From 66aaeafa0b245ef6021c93690989727c959e41ed Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 16 Oct 2026 18:58:48 -0400 Subject: [PATCH 1/5] Let an enriched docket placeholder be enriched again _plan_enrichment required uuid_owner[row_uuid] == the row's own key. uuid_owner records FIRST ownership and never moves, so a placeholder that was itself the successor of an earlier enrichment (geography-free -> geography-only) never owns its UUID, and its observation fell to the mint path and was refused as UUID reuse. This blocked usaspending.dod.prime_award_obligations (a2649fd9) from taking its FY2026 observation (Thesis run 38962977031). Drop that conjunct. The live-binding conditions that remain (enrichment-shaped, the row's key live, and bound to row_uuid now) are the contract and still refuse stale, superseded, retired, handed-over and reclaimed rows. The mint-path refusal now says "first bound to", since uuid_owner is not the current binding. Co-Authored-By: Claude Opus 5.5 --- scripts/build_series_catalog.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/scripts/build_series_catalog.py b/scripts/build_series_catalog.py index 8b6f47c9..c514b0f4 100644 --- a/scripts/build_series_catalog.py +++ b/scripts/build_series_catalog.py @@ -2133,7 +2133,6 @@ def _plan_enrichment( placeholder whose live binding owns ``row_uuid``; True if planned. """ prior_own_key = UuidRegistry.entry_key(prior) - owner_key = registry.uuid_owner.get(uuid_module.UUID(row_uuid).int) prior_geo = prior.get("geography") or None geo = geography or {} enrichment_shaped = ( @@ -2149,14 +2148,17 @@ def _plan_enrichment( ) ) ) - # uuid_owner records FIRST ownership and never moves, so alone it - # would accept a stale catalog row still carrying a UUID its key - # has since superseded away from. The contract is about the LIVE - # binding: demand it be row_uuid itself. Staging replay would - # reject such a plan anyway; this fails at plan time instead. + # The contract is the LIVE binding: the row's own key must be live + # and bound to row_uuid right now. That alone refuses a stale row + # still carrying a UUID its key has since superseded away from, or + # a key that was retired, handed over or reclaimed. Staging replay + # would reject such a plan anyway; this fails at plan time instead. + # Historical ownership (uuid_owner) is not part of the contract: it + # records FIRST ownership and never moves, so a placeholder that + # was itself the successor of an earlier enrichment never owns its + # UUID and could never be enriched again. if not ( enrichment_shaped - and owner_key == prior_own_key and registry.is_live(prior_own_key) and registry.binding(prior_own_key) == row_uuid ): @@ -2164,8 +2166,11 @@ def _plan_enrichment( # Docket-placeholder enrichment: the binding MOVES to the observed # identity via an explicit retire + succeeds pair. UUID continuity # is preserved, so no ceremony flag is needed — this holds for the - # first enrichment and for a re-enrichment after a withdraw/reclaim - # cycle (the grammar accepts succeeds on a retired successor key). + # first enrichment, for a chain of them (a geography-free + # placeholder enriched to a geography-only one, then to the + # observed identity), and for a re-enrichment after a + # withdraw/reclaim cycle (the grammar accepts succeeds on a retired + # successor key). plan["enrich_retires"].append( dict( _registry_event( @@ -2285,7 +2290,8 @@ def resolve_uuid( return row_uuid raise SystemExit( f"identity {canon_key} would mint uuid {row_uuid}, which " - f"the registry already binds to {owner_key} — UUID reuse " + f"the registry already knows (first bound to {owner_key}) " + "— UUID reuse " "requires an explicit ceremony (merge via curated alias + " "--allow-remint, or placeholder enrichment)" ) From adb190152a4f31e23af581f0f62d50a1c15e32f6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 16 Oct 2026 19:14:57 -0400 Subject: [PATCH 2/5] Pin chained placeholder enrichment and the stale-row refusals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regressions: - the production state (registry lines 196/249/250 for a2649fd9): the geography-only placeholder takes its first print as exactly the retire + succeeds pair Thesis's append guard admits; - the chain through main (docket entry learns a country, then the first print lands): no ceremony, one UUID, --check after each hop; - six stale rows after a chain (superseded, retired, handed over, reclaimed, chain root, another series' UUID) still refused at plan time by the live-binding conditions alone, artifacts byte-untouched. Properties (Hypothesis): - every builder-reachable chain, any country, vintage and entity, keeps one UUID, writes one retire + succeeds pair per hop, passes --check and is byte-idempotent; - in any grammar-admitted history, a live binding its key did not first own was established by succeeds (why the owner conjunct only ever refused chains); - differential: the builder plans an enrichment exactly when UuidRegistry accepts the pair, everything it plans stages and agrees with its catalog, and refusals are plan-time diagnostics. Counterfactually verified: with the owner conjunct restored, both regressions, the chain property and the differential property fail (the differential on 5/5 seeds, shrinking to the dod shape); the stale-row tests pass either way. The mint-path diagnostic now says "already knows (first bound to …)", so test_mint_never_reuses_a_bound_uuid matches that. Co-Authored-By: Claude Opus 5.5 --- docs/catalog-curation-backlog.md | 11 + tests/test_build_series_catalog.py | 658 ++++++++++++++++++++++++++++- 2 files changed, 668 insertions(+), 1 deletion(-) diff --git a/docs/catalog-curation-backlog.md b/docs/catalog-curation-backlog.md index d729b6f1..5c854429 100644 --- a/docs/catalog-curation-backlog.md +++ b/docs/catalog-curation-backlog.md @@ -171,6 +171,17 @@ were closed on 2026-08-09 by chronicle#153: replay rejects the hand-forged plan before any write reaches disk, and the builder end-to-end refuses the stale catalog, leaving both artifacts untouched. +- 2026-10-16: #153 added the live-binding check but kept the first-owner + conjunct (`uuid_owner[row_uuid] == prior_own_key`) alongside it. Because + the first owner never moves, that conjunct refused every second + enrichment hop. A placeholder that had already been enriched once (a + geography-free placeholder enriched to geography-only when its docket + entry learned a country) could not take its first print. The conjunct is + gone, and the live binding alone is the contract. + `usaspending.dod.prime_award_obligations` (a2649fd9) hit this in Thesis + run 38962977031. Tests now pin the chain and the stale-row refusals. A + property test also checks that the builder plans an enrichment exactly + when the registry grammar admits the retire + succeeds pair. ## Country geography on Colorado-scoped concepts, and one raw series id (2026-09-04) diff --git a/tests/test_build_series_catalog.py b/tests/test_build_series_catalog.py index 8bfd0fb7..860dde7a 100644 --- a/tests/test_build_series_catalog.py +++ b/tests/test_build_series_catalog.py @@ -16,6 +16,7 @@ import pathlib import sys import tempfile +import uuid import pytest from hypothesis import HealthCheck, given, settings @@ -2014,7 +2015,7 @@ def test_mint_never_reuses_a_bound_uuid(tmp_path: pathlib.Path) -> None: }, ] } - with pytest.raises(SystemExit, match="already binds"): + with pytest.raises(SystemExit, match=r"already knows \(first bound to"): _build( tmp_path, [_row("new.one"), _row("new.two")], @@ -4424,3 +4425,658 @@ def observed_total(catalog: dict) -> int: undone = json.loads((tmp_path / "catalog.json").read_text()) assert _without_digests(undone) == _without_digests(before) assert bsc.main(argv + ["--check"]) == 0 + + +# --- chained placeholder enrichment (2026-10-16) ---------------------------- +# +# A docket placeholder can be enriched more than once. The geography-free +# placeholder becomes a geography-only one when its docket entry learns a +# country, and that one becomes the observed identity when the first print +# lands. Each hop is a retire + succeeds pair carrying one UUID. The guard is +# the LIVE binding of the row's own key. uuid_owner records first ownership +# and never moves, so it names the first hop's predecessor, never the +# second's: requiring it (the guard before this fix) refused every second +# hop. Thesis run 38962977031 hit exactly that on +# usaspending.dod.prime_award_obligations. + +DOD = "usaspending.dod.prime_award_obligations" +CHAIN_NOTE = "docket placeholder enriched by first observed identity" +US_UNDECLARED = {"level": "country", "id": "0100000US", "vintage": None} +U3 = "cccccccc-3333-4333-8333-333333333333" +U4 = "eeeeeeee-5555-4555-8555-555555555555" + + +def _key(fields: dict) -> tuple[str, str, str]: + return bsc.UuidRegistry.entry_key(fields) + + +def _owner(registry: bsc.UuidRegistry, row_uuid: str): + return registry.uuid_owner.get(uuid.UUID(row_uuid).int) + + +def _plan_events(plan: dict) -> list[dict]: + """Every registry event a plan stages, in main's order.""" + return ( + plan["enrich_retires"] + plan["mints"] + plan["revives"] + + plan["supersedes"] + plan["retire_pending"] + ) + + +def _event_kind(event: dict) -> str: + for marker in ("succeeds", "supersedes", "retired", "revived", "reclaimed"): + if marker in event: + return marker + return "mint" + + +def _registry_lines(tmp_path: pathlib.Path) -> list[dict]: + return [ + json.loads(line) + for line in (tmp_path / "registry.jsonl").read_text().splitlines() + ] + + +def _dod_observation() -> dict: + """The FY2026 first print as Thesis records it (registered snapshot).""" + return _row( + f"{DOD}.fy2026", + rid=f"{DOD}.fy2026.registered_query_snapshot", + unit="billions USD", + period={"type": "year", "value": "2026"}, + ) + + +def _dod_chain() -> list[dict]: + """Registry lines 196, 249 and 250 for a2649fd9, with U1 for the UUID.""" + return [ + _mint(DOD, U1), + dict(_mint(DOD, U1), retired=True, note=CHAIN_NOTE), + dict( + _mint(DOD, U1, geography=US_UNDECLARED), + succeeds={"concept": DOD, "geography": None, "entity": None}, + ), + ] + + +def _docket_row(concept: str, row_uuid: str, geography: dict | None) -> dict: + return { + "uuid": row_uuid, + "concept": concept, + "geography": geography, + "entity": None, + "aliases": [], + "status": "docket-only", + } + + +def test_a_placeholder_enriched_once_is_enriched_again( + tmp_path: pathlib.Path, +) -> None: + # Production repro (Thesis run 38962977031, 2026-10-16). The catalog + # still holds the geography-only placeholder docket-only; its first + # print must take the second hop instead of falling through to the + # UUID-reuse refusal. + registry = _registry(tmp_path, _dod_chain()) + geo_key = _key({"concept": DOD, "geography": US_UNDECLARED, "entity": None}) + # The trap: the live placeholder holds U1 without having first owned it. + assert registry.is_live(geo_key) and registry.binding(geo_key) == U1 + assert _owner(registry, U1) != geo_key + existing = { + "series": [ + _docket_row(DOD, U1, dict(bsc.COUNTRY_GEOGRAPHY["US"])) + ] + } + catalog, plan = _build( + tmp_path, [_dod_observation()], existing=existing, + registry_entries=_dod_chain(), + ) + (row,) = catalog["series"] + assert (row["concept"], row["uuid"], row["status"]) == (DOD, U1, "observed") + assert row["entity"] == ECONOMY + # Exactly the pair the diagnosis's patched generator wrote (and that + # thesis#362's append guard admits): retire the geography-only + # placeholder, then succeed it at the observed identity, one UUID. + assert [ + bsc.UuidRegistry.render_entry(e) for e in _plan_events(plan) + ] == [ + bsc.UuidRegistry.render_entry(e) + for e in ( + dict(_mint(DOD, U1, geography=US_UNDECLARED), retired=True, + note=CHAIN_NOTE), + dict( + _mint(DOD, U1, geography=US_IDENTITY, entity=ECONOMY), + succeeds={"concept": DOD, "geography": US_UNDECLARED, + "entity": None}, + ), + ) + ] + assert not plan["dropped"] + staged = registry.stage(_plan_events(plan)) + observed_key = _key(row) + assert staged.is_live(observed_key) and staged.binding(observed_key) == U1 + assert not staged.is_live(geo_key) and geo_key in staged.consumed + assert [k for k, e in staged.live_bindings() if e["uuid"] == U1] == [ + observed_key + ] + assert bsc.registry_agreement_problems(catalog, staged) == [] + + +def test_placeholder_chain_keeps_one_uuid_through_main( + tmp_path: pathlib.Path, +) -> None: + # The same chain end to end through main: the docket entry starts with + # no country, learns "US" (first hop, docket to docket), then its first + # print lands (second hop). Neither hop needs a ceremony flag, the UUID + # never changes, and --check passes after each. + entry = {"series": DOD, "cadence": "annual", + "extras": {"targetUnit": "billions USD"}} + bystander = _row("bls.cps.unemployment_rate") + argv = _repo(tmp_path, [bystander], seed={"series": [entry]}) + + def rows_by_concept() -> dict[str, dict]: + catalog = json.loads((tmp_path / "catalog.json").read_text()) + return {r["concept"]: r for r in catalog["series"]} + + assert bsc.main(argv) == 0 + placeholder = rows_by_concept()[DOD] + assert placeholder["status"] == "docket-only" + assert placeholder["geography"] is None + lineage = placeholder["uuid"] + bystander_uuid = rows_by_concept()["bls.cps.unemployment_rate"]["uuid"] + + entry["extras"]["country"] = "US" + (tmp_path / "seed.json").write_text( + json.dumps({"series": [entry]}), encoding="utf-8" + ) + assert bsc.main(argv) == 0 + country = rows_by_concept()[DOD] + assert (country["uuid"], country["status"]) == (lineage, "docket-only") + assert country["geography"] == bsc.COUNTRY_GEOGRAPHY["US"] + assert bsc.main(argv + ["--check"]) == 0 + + (tmp_path / "obs.jsonl").write_text( + "".join(json.dumps(r) + "\n" for r in [bystander, _dod_observation()]), + encoding="utf-8", + ) + assert bsc.main(argv) == 0 + observed = rows_by_concept()[DOD] + assert (observed["uuid"], observed["status"]) == (lineage, "observed") + assert observed["entity"] == ECONOMY + assert rows_by_concept()["bls.cps.unemployment_rate"]["uuid"] == ( + bystander_uuid + ) + assert bsc.main(argv + ["--check"]) == 0 + + events = _registry_lines(tmp_path) + assert [ + (e["geography"], e["entity"], _event_kind(e)) + for e in events + if e["concept"] == DOD + ] == [ + (None, None, "mint"), + (None, None, "retired"), + (US_UNDECLARED, None, "succeeds"), + (US_UNDECLARED, None, "retired"), + (US_IDENTITY, ECONOMY, "succeeds"), + ] + assert {e["uuid"] for e in events if e["concept"] == DOD} == {lineage} + assert not any( + _event_kind(e) in ("supersedes", "reclaimed") for e in events + ) + catalog_bytes = (tmp_path / "catalog.json").read_bytes() + registry_bytes = (tmp_path / "registry.jsonl").read_bytes() + assert bsc.main(argv) == 0 + assert (tmp_path / "catalog.json").read_bytes() == catalog_bytes + assert (tmp_path / "registry.jsonl").read_bytes() == registry_bytes + + +# Each stage is (docket entry declares a country, observation present). +_CHAINS = ( + ((False, False), (True, False)), + ((False, False), (False, True)), + ((True, False), (True, True)), + ((False, False), (True, False), (True, True)), +) + + +@settings( + max_examples=60, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given( + chain=st.sampled_from(_CHAINS), + country=st.sampled_from(sorted(bsc.COUNTRY_GEOGRAPHY)), + vintage=st.sampled_from(["current", "2020", None]), + entity=st.sampled_from([ECONOMY, HAND_ENTITY]), +) +def test_any_placeholder_chain_keeps_one_uuid_and_passes_check( + tmp_path_factory: pytest.TempPathFactory, + chain: tuple, + country: str, + vintage: str | None, + entity: dict, +) -> None: + """For every chain of placeholder enrichments the builder can reach + (geography-free -> country, geography-free -> observed, country -> + observed, and all three hops), in any country, at any observed + vintage and entity: every hop writes exactly one retire + succeeds + pair handing over the binding it retired, needs no ceremony flag, + keeps the lineage's one UUID, leaves exactly one live binding for it + (at the row's identity), leaves unrelated series untouched, passes + --check, and is byte-idempotent.""" + tmp_path = tmp_path_factory.mktemp("chain") + concept = "agency.chain.series" + neighbour = {"series": "agency.chain.neighbour", "cadence": "monthly", + "extras": {"country": "US", "targetUnit": "percent"}} + bystander = _row("bls.cps.unemployment_rate") + observation = _row( + concept, + geography={**bsc.COUNTRY_GEOGRAPHY[country], "vintage": vintage}, + entity=entity, + ) + + def write_stage(stage: tuple[bool, bool]) -> None: + declares_country, observed = stage + extras = {"targetUnit": "percent"} + if declares_country: + extras["country"] = country + seed = {"series": [ + {"series": concept, "cadence": "monthly", "extras": extras}, + neighbour, + ]} + (tmp_path / "seed.json").write_text(json.dumps(seed), encoding="utf-8") + rows = [bystander] + ([observation] if observed else []) + (tmp_path / "obs.jsonl").write_text( + "".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8" + ) + + def catalog_rows() -> list[dict]: + return json.loads((tmp_path / "catalog.json").read_text())["series"] + + argv = _repo(tmp_path, []) + write_stage(chain[0]) + assert bsc.main(argv) == 0 + (first,) = [r for r in catalog_rows() if r["concept"] == concept] + lineage = first["uuid"] + others = { + _key(r): r["uuid"] for r in catalog_rows() if r["concept"] != concept + } + for stage in chain[1:]: + before = len(_registry_lines(tmp_path)) + write_stage(stage) + assert bsc.main(argv) == 0 # no --allow-remint + (row,) = [r for r in catalog_rows() if r["concept"] == concept] + assert row["uuid"] == lineage + assert row["status"] == ("observed" if stage[1] else "docket-only") + hop = _registry_lines(tmp_path)[before:] + assert [_event_kind(e) for e in hop] == ["retired", "succeeds"] + assert {e["uuid"] for e in hop} == {lineage} + assert _key(hop[1]["succeeds"]) == _key(hop[0]) + assert _key(hop[1]) == _key(row) + assert { + _key(r): r["uuid"] for r in catalog_rows() if r["concept"] != concept + } == others + registry = bsc.UuidRegistry.load(tmp_path / "registry.jsonl") + assert [ + k for k, e in registry.live_bindings() if e["uuid"] == lineage + ] == [_key(row)] + assert bsc.main(argv + ["--check"]) == 0 + snapshot = ( + (tmp_path / "catalog.json").read_bytes(), + (tmp_path / "registry.jsonl").read_bytes(), + ) + assert bsc.main(argv) == 0 + assert ( + (tmp_path / "catalog.json").read_bytes(), + (tmp_path / "registry.jsonl").read_bytes(), + ) == snapshot + assert { + e["uuid"] for e in _registry_lines(tmp_path) if e["concept"] == concept + } == {lineage} + + +def _stale_after_chain(scenario: str): + """(registry events, stale docket-only row, observed identity): the + DOD chain (geography-free -> geography-only, U1) and then whatever made + the committed catalog's row stale.""" + chain = _dod_chain() + observed_economy = {"concept": DOD, "geography": US_IDENTITY, + "entity": ECONOMY} + observed_hand = dict(observed_economy, entity=HAND_ENTITY) + geo_row = dict(bsc.COUNTRY_GEOGRAPHY["US"]) + handed_on = chain + [ + dict(_mint(DOD, U1, geography=US_UNDECLARED), retired=True, + note=CHAIN_NOTE), + dict(observed_economy, uuid=U1, + succeeds={"concept": DOD, "geography": US_UNDECLARED, + "entity": None}), + ] + if scenario == "superseded": + # A curated remint moved the placeholder to U2; the row kept U1. + events = chain + [ + dict(_mint(DOD, U2, geography=US_UNDECLARED), supersedes=U1, + note="curated remint of the placeholder"), + ] + return events, _docket_row(DOD, U1, geo_row), observed_economy + if scenario == "retired": + # The docket entry was withdrawn (gated retire); the row lingers. + events = chain + [ + dict(_mint(DOD, U1, geography=US_UNDECLARED), retired=True, + note="docket entry withdrawn"), + ] + return events, _docket_row(DOD, U1, geo_row), observed_economy + if scenario == "handed over": + # The placeholder already took its second hop; the row predates it. + return handed_on, _docket_row(DOD, U1, geo_row), observed_hand + if scenario == "reclaimed": + # After the hand-over the observation was withdrawn and the + # placeholder re-established on a fresh UUID; the row kept U1. + events = handed_on + [ + dict(observed_economy, uuid=U1, retired=True, + note="observation withdrawn upstream"), + dict(_mint(DOD, U2, geography=US_UNDECLARED), reclaimed=True, + note="placeholder re-established after handover"), + ] + return events, _docket_row(DOD, U1, geo_row), observed_hand + if scenario == "chain root": + # The row is the geography-free placeholder the first hop retired. + return chain, _docket_row(DOD, U1, None), observed_economy + if scenario == "another series' uuid": + # The row carries a UUID that another live series holds. + events = chain + [_mint("agency.other", U3)] + return events, _docket_row(DOD, U3, geo_row), observed_economy + raise AssertionError(scenario) + + +@pytest.mark.parametrize( + "scenario", + ["superseded", "retired", "handed over", "reclaimed", "chain root", + "another series' uuid"], +) +def test_stale_rows_after_a_chain_are_still_refused( + tmp_path: pathlib.Path, scenario: str +) -> None: + # The case the dropped owner check was written for: a catalog row that + # no longer carries its key's LIVE binding. With the owner check gone, + # the live-binding conditions alone must refuse each one at plan time + # with the ceremony diagnostic, and main must leave both artifacts + # byte-untouched. + events, stale_row, observed = _stale_after_chain(scenario) + registry = _registry(tmp_path, events) + stale_key = _key(stale_row) + assert not ( + registry.is_live(stale_key) + and registry.binding(stale_key) == stale_row["uuid"] + ) + observation = _row( + DOD, geography=dict(observed["geography"], name="United States"), + entity=observed["entity"], + ) + with pytest.raises(SystemExit, match="explicit ceremony"): + _build( + tmp_path, [observation], existing={"series": [stale_row]}, + registry_entries=events, + ) + argv = _repo(tmp_path, [observation], seed={"series": []}) + _registry(tmp_path, events) + (tmp_path / "catalog.json").write_text( + json.dumps({"series": [stale_row]}) + "\n", encoding="utf-8" + ) + catalog_before = (tmp_path / "catalog.json").read_bytes() + registry_before = (tmp_path / "registry.jsonl").read_bytes() + with pytest.raises(SystemExit, match="explicit ceremony"): + bsc.main(argv) + assert (tmp_path / "catalog.json").read_bytes() == catalog_before + assert (tmp_path / "registry.jsonl").read_bytes() == registry_before + + +# A small identity universe for generated registry histories: two concepts +# (so a row can carry another series' UUID), geographies at every +# refinement level, three entities, and six UUIDs (so stale rows collide +# with known values). +_GEN_CONCEPT, _GEN_OTHER = "agency.lineage", "agency.elsewhere" +_PLACEHOLDER_GEOS = ( + None, + {"level": "country", "id": "0100000US", "vintage": None}, + {"level": "country", "id": "0100000US", "vintage": "current"}, + {"level": "country", "id": "0100000US", "vintage": "2020"}, + {"level": "country", "id": "GB", "vintage": None}, +) +_OBSERVED_GEOS = ( + {"level": "country", "id": "0100000US", "vintage": "current"}, + {"level": "country", "id": "0100000US", "vintage": "2020"}, + {"level": "country", "id": "GB", "vintage": "current"}, +) +_GEN_GEOS = _PLACEHOLDER_GEOS + _OBSERVED_GEOS[2:] +_GEN_ENTITIES = (None, ECONOMY, HAND_ENTITY) +_GEN_UUIDS = ( + U1, U2, U3, U4, + "ffffffff-6666-4666-8666-666666666666", + "abababab-7777-4777-8777-777777777777", +) +_GEN_IDENTS = tuple( + {"concept": concept, "geography": geography, "entity": entity} + for concept in (_GEN_CONCEPT, _GEN_OTHER) + for geography in _GEN_GEOS + for entity in _GEN_ENTITIES +) + + +def _ident(entry: dict) -> dict: + return {key: entry[key] for key in ("concept", "geography", "entity")} + + +def _refines(placeholder: dict | None, geography: dict | None) -> bool: + """Whether an enrichment hop from ``placeholder`` to ``geography`` is + plausible. Used only to steer generation toward chains; the registry + grammar alone decides what a history may contain.""" + if placeholder is None: + return True + geography = geography or {} + return (placeholder["level"], placeholder["id"]) == ( + geography.get("level"), geography.get("id") + ) and placeholder["vintage"] in (None, geography.get("vintage")) + + +def _generated_registry(data: st.DataObject) -> bsc.UuidRegistry: + """A registry grown from random events that the grammar accepts. + + Each step draws a move the current state allows: mint an unbound + identity on an unused UUID, retire a live binding, revive a retired + one, supersede or reclaim onto an unused UUID, or take an enrichment + hop (retire an entity-less binding and succeed it at a refinement). + The move is kept only if UuidRegistry accepts it, so every history is + one the grammar itself admits. Hops chain, so placeholders enriched + once, twice and three times occur alongside stale, superseded, + retired, handed-over and reclaimed keys.""" + registry = bsc.UuidRegistry(pathlib.Path("generated.jsonl"), b"") + for _ in range(data.draw(st.integers(1, 12), label="steps")): + held = list(registry.latest.values()) + live = [e for e in held if not e.get("retired")] + retired = [e for e in held if e.get("retired")] + consumed = [e for e in retired if _key(e) in registry.consumed] + unbound = [i for i in _GEN_IDENTS if _key(i) not in registry.latest] + unused = [u for u in _GEN_UUIDS if _owner(registry, u) is None] + moves = [] + if unbound and unused: + moves += ["mint", "mint"] + if live: + moves += ["retire"] + if live and unused: + moves += ["supersede"] + if retired: + moves += ["revive"] + if consumed and unused: + moves += ["reclaim"] + if any(e["entity"] is None for e in live): + moves += ["hop", "hop", "hop"] + if not moves: + break + move = data.draw(st.sampled_from(moves)) + if move == "mint": + # Placeholders (entity-less) are minted twice as often. + pool = [i for i in unbound if i["entity"] is None] * 2 + unbound + events = [dict(data.draw(st.sampled_from(pool)), + uuid=data.draw(st.sampled_from(unused)))] + elif move == "retire": + entry = data.draw(st.sampled_from(live)) + events = [dict(_ident(entry), uuid=entry["uuid"], retired=True, + note="generated retirement")] + elif move == "supersede": + entry = data.draw(st.sampled_from(live)) + events = [dict(_ident(entry), + uuid=data.draw(st.sampled_from(unused)), + supersedes=entry["uuid"], + note="generated curated remint")] + elif move == "revive": + entry = data.draw(st.sampled_from(retired)) + events = [dict(_ident(entry), uuid=entry["uuid"], revived=True)] + elif move == "reclaim": + entry = data.draw(st.sampled_from(consumed)) + events = [dict(_ident(entry), + uuid=data.draw(st.sampled_from(unused)), + reclaimed=True, + note="generated reclaim after handover")] + else: + entry = data.draw(st.sampled_from( + [e for e in live if e["entity"] is None] + )) + targets = [ + ident for ident in _GEN_IDENTS + if ident["concept"] == entry["concept"] + and _key(ident) != _key(entry) + and _refines(entry["geography"], ident["geography"]) + ] + # Entity-less targets are weighted up so hops chain. + target = data.draw(st.sampled_from( + [t for t in targets if t["entity"] is None] * 3 + targets + )) + events = [ + dict(_ident(entry), uuid=entry["uuid"], retired=True, + note=CHAIN_NOTE), + dict(target, uuid=entry["uuid"], succeeds=_ident(entry)), + ] + try: + registry = registry.stage(events) + except SystemExit: + continue + return registry + + +@settings(max_examples=400, deadline=None) +@given(data=st.data()) +def test_a_live_binding_its_key_never_first_owned_came_by_succeeds( + data: st.DataObject, +) -> None: + """Why the owner check was redundant where it was right. In every + history the grammar admits, a live binding (K, U) either has K as U's + first owner, or K's binding was established by a succeeds event. So + the conjunct uuid_owner[U] == K, on top of the live-binding + conditions, refused nothing but enrichment chains.""" + registry = _generated_registry(data) + for key, entry in registry.live_bindings(): + if _owner(registry, entry["uuid"]) == key: + continue + establishing = [ + e for e in registry.entries + if _key(e) == key and _event_kind(e) not in ("retired", "revived") + ] + assert _event_kind(establishing[-1]) == "succeeds", (key, entry) + + +@settings( + max_examples=400, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given(data=st.data()) +def test_enrichment_is_planned_exactly_when_the_grammar_admits_it( + tmp_path_factory: pytest.TempPathFactory, data: st.DataObject +) -> None: + """Differential: plan-time guard vs. the registry grammar. + + For any history the grammar admits, any docket-only row (live, stale, + superseded, retired, handed over, reclaimed, the root of a chain, or + carrying another series' UUID) and any observation of its concept: + + * completeness: the builder plans the retire + succeeds pair exactly + when UuidRegistry would accept that very pair (so no enrichment + chain is refused); + * soundness: everything the builder plans stages cleanly and agrees + with the catalog it built (so no stale row slips through); + * a refusal is a plan-time diagnostic, never a crash; + * when no enrichment is planned, the row takes its own key's binding + or a UUID new to the registry, never a UUID another key holds. + """ + tmp_path = tmp_path_factory.mktemp("differential") + registry = _generated_registry(data) + (tmp_path / "registry.jsonl").write_bytes(registry.raw) + placeholders = [ + e for e in registry.latest.values() + if e["concept"] == _GEN_CONCEPT and e["entity"] is None + ] + # Steer toward rows at keys the history holds, carrying their last + # binding: live ones (the chained case lives here) and stale ones + # (retired, handed over). Any geography with any UUID stays possible. + candidates = ( + [(e["geography"], e["uuid"]) for e in placeholders + if not e.get("retired")] * 8 + + [(e["geography"], e["uuid"]) for e in placeholders] * 2 + + [(g, u) for g in _PLACEHOLDER_GEOS for u in _GEN_UUIDS] + ) + geography, row_uuid = data.draw(st.sampled_from(candidates)) + placeholder = {"concept": _GEN_CONCEPT, "geography": geography, + "entity": None} + # Observations that refine the row's geography are weighted up; any + # observed geography stays possible (cross-country, vintage clash). + observed = { + "concept": _GEN_CONCEPT, + "geography": data.draw(st.sampled_from( + [g for g in _OBSERVED_GEOS if _refines(geography, g)] * 3 + + list(_OBSERVED_GEOS) + )), + "entity": data.draw(st.sampled_from((ECONOMY, HAND_ENTITY))), + } + retire = dict(placeholder, uuid=row_uuid, retired=True, note=CHAIN_NOTE) + succeed = dict(observed, uuid=row_uuid, succeeds=placeholder) + try: + registry.stage([retire, succeed]) + admitted = True + except SystemExit: + admitted = False + observation = _row( + _GEN_CONCEPT, + geography=dict(observed["geography"], name=None), + entity=observed["entity"], + ) + try: + catalog, plan = _build( + tmp_path, [observation], + existing={"series": [_docket_row(_GEN_CONCEPT, row_uuid, + geography)]}, + ) + except SystemExit as exc: + assert not admitted, exc + assert "explicit ceremony" in str(exc) or ( + "superseding UUIDs must be new" in str(exc) + ), exc + return + hop = [e for e in plan["mints"] if "succeeds" in e] + assert bool(hop) == admitted + (row,) = catalog["series"] + if admitted: + assert row["uuid"] == row_uuid + assert [ + bsc.UuidRegistry.render_entry(e) + for e in plan["enrich_retires"] + hop + ] == [bsc.UuidRegistry.render_entry(e) for e in (retire, succeed)] + else: + assert row["uuid"] == registry.binding(_key(observed)) or ( + _owner(registry, row["uuid"]) is None + ) + # main attaches the ceremony note to gated events before staging. + for event in plan["supersedes"] + plan["retire_pending"]: + event["note"] = "generated ceremony" + staged = registry.stage(_plan_events(plan)) + assert bsc.registry_agreement_problems(catalog, staged) == [] From ac8652da07a0713145cf034682b8c2202c30b791 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 18 Oct 2026 12:10:21 -0400 Subject: [PATCH 3/5] Refuse a consumed or cross-concept predecessor at plan time Independent review of #360 found a state the live binding misses. A placeholder B hands its lineage to C, then (docket loses and regains its country) takes a fresh UUID back from a coarser placeholder A. B is live and bound to that UUID but still consumed, and the grammar never lets a predecessor hand over twice. Without the dropped owner conjunct the builder planned retire(B) + C succeeds B, which staging then rejected. _plan_enrichment now also requires prior_own_key not in registry.consumed, so it refuses at plan time again (as base did). The review's LOW finding, pre-existing: a docket entry matched to a placeholder through a curated alias keyed the docket row under the alias, so the planned succeeds crossed concepts and died in staging. Enrichment now also requires the prior's concept to equal the canonical one. Observed buckets always take the prior's concept, so only that docket path changes, from a staging failure to a plan-time refusal. Tests: - the review's lifecycle through main (seven events, B live and consumed), refused with and without --allow-remint, artifacts byte-untouched; - the cross-concept docket alias, refused at plan time; - the registry generator gains a "return" move (mint a coarser placeholder and hop it onto a retired key that already handed over), and retired hop targets and consumed rows are weighted up, so the differential property reaches live-but-consumed keys. Counterfactuals: without the consumed check, the lifecycle regression and the differential property fail on 5/5 seeds; without the concept check, the alias regression fails. Co-Authored-By: Claude Opus 5.5 --- scripts/build_series_catalog.py | 19 +++- tests/test_build_series_catalog.py | 151 ++++++++++++++++++++++++++--- 2 files changed, 153 insertions(+), 17 deletions(-) diff --git a/scripts/build_series_catalog.py b/scripts/build_series_catalog.py index c514b0f4..0af4c366 100644 --- a/scripts/build_series_catalog.py +++ b/scripts/build_series_catalog.py @@ -2135,9 +2135,13 @@ def _plan_enrichment( prior_own_key = UuidRegistry.entry_key(prior) prior_geo = prior.get("geography") or None geo = geography or {} + # Lineage never crosses concepts. Observed buckets always take the + # prior's concept; a docket entry matched through a curated alias + # can name another one, and the grammar would refuse that succeeds. enrichment_shaped = ( prior.get("status") == "docket-only" and prior.get("entity") is None + and prior.get("concept") == canon_key[0] and ( prior_geo is None or ( @@ -2148,11 +2152,15 @@ def _plan_enrichment( ) ) ) - # The contract is the LIVE binding: the row's own key must be live - # and bound to row_uuid right now. That alone refuses a stale row - # still carrying a UUID its key has since superseded away from, or - # a key that was retired, handed over or reclaimed. Staging replay - # would reject such a plan anyway; this fails at plan time instead. + # The contract is the LIVE binding: the row's own key must be live, + # bound to row_uuid right now, and still able to hand its lineage + # over. Liveness and binding refuse a stale row still carrying a + # UUID its key has since superseded away from, and a key that was + # retired, handed over or reclaimed. A key hands over at most once: + # one that handed over and later returned as the successor of + # another hop is live again but stays consumed, and the grammar + # refuses it as a predecessor. Staging replay would reject each of + # these plans anyway; this fails at plan time instead. # Historical ownership (uuid_owner) is not part of the contract: it # records FIRST ownership and never moves, so a placeholder that # was itself the successor of an earlier enrichment never owns its @@ -2161,6 +2169,7 @@ def _plan_enrichment( enrichment_shaped and registry.is_live(prior_own_key) and registry.binding(prior_own_key) == row_uuid + and prior_own_key not in registry.consumed ): return False # Docket-placeholder enrichment: the binding MOVES to the observed diff --git a/tests/test_build_series_catalog.py b/tests/test_build_series_catalog.py index 860dde7a..7aa0c75a 100644 --- a/tests/test_build_series_catalog.py +++ b/tests/test_build_series_catalog.py @@ -4831,6 +4831,97 @@ def test_stale_rows_after_a_chain_are_still_refused( assert (tmp_path / "registry.jsonl").read_bytes() == registry_before +def test_a_returned_placeholder_never_hands_over_twice( + tmp_path: pathlib.Path, +) -> None: + # Independent-review counterexample (chronicle#360, 2026-10-18). A + # placeholder B hands U1 to its observed identity C. The observation is + # withdrawn and the docket loses its country (fresh placeholder A/U2), + # then the country returns and B takes U2 from A. B is live and bound + # to U2, but it is still consumed: it already handed over once, and the + # grammar never lets a predecessor hand over twice. When C's + # observation returns, the builder must refuse at plan time. Without the + # consumed check it planned retire(B/U2) + C succeeds B, which staging + # rejected. Base refused too (B never first-owned U2), so this pins the + # status quo for this lineage, not a regression. + concept = "agency.lineage" + bystander = _row("bls.cps.unemployment_rate") + observed = _row(concept) + + def stage(country: bool, rows: list[dict]) -> None: + extras = {"targetUnit": "percent"} + if country: + extras["country"] = "US" + seed = {"series": [ + {"series": concept, "cadence": "monthly", "extras": extras}, + ]} + (tmp_path / "seed.json").write_text(json.dumps(seed), encoding="utf-8") + (tmp_path / "obs.jsonl").write_text( + "".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8" + ) + + argv = _repo(tmp_path, []) + remint = ["--allow-remint", "--remint-note", "observation withdrawn"] + stage(True, [bystander]) + assert bsc.main(argv) == 0 # mint B/U1 + stage(True, [bystander, observed]) + assert bsc.main(argv) == 0 # retire B, C succeeds B with U1 + stage(False, [bystander]) + assert bsc.main(argv + remint) == 0 # mint A/U2, retire C + stage(True, [bystander]) + assert bsc.main(argv) == 0 # retire A, B succeeds A with U2 + assert bsc.main(argv + ["--check"]) == 0 + a_key = (concept, bsc._geo_key(None), bsc._entity_key(None)) + b_key = (concept, bsc._geo_key(US_UNDECLARED), bsc._entity_key(None)) + c_key = (concept, bsc._geo_key(US_IDENTITY), bsc._entity_key(ECONOMY)) + assert [ + (_key(e), _event_kind(e)) + for e in _registry_lines(tmp_path) + if e["concept"] == concept + ] == [ + (b_key, "mint"), + (b_key, "retired"), + (c_key, "succeeds"), + (a_key, "mint"), + (c_key, "retired"), + (a_key, "retired"), + (b_key, "succeeds"), + ] + registry = bsc.UuidRegistry.load(tmp_path / "registry.jsonl") + u2 = registry.binding(a_key) + assert registry.is_live(b_key) and registry.binding(b_key) == u2 + assert b_key in registry.consumed # the state the live binding misses + + stage(True, [bystander, observed]) + catalog_before = (tmp_path / "catalog.json").read_bytes() + registry_before = (tmp_path / "registry.jsonl").read_bytes() + for flags in ([], remint): + with pytest.raises(SystemExit, match="superseding UUIDs must be new"): + bsc.main(argv + flags) + assert (tmp_path / "catalog.json").read_bytes() == catalog_before + assert (tmp_path / "registry.jsonl").read_bytes() == registry_before + + +def test_a_docket_alias_never_plans_a_cross_concept_hop( + tmp_path: pathlib.Path, +) -> None: + # Independent-review finding (chronicle#360, LOW, pre-existing): a + # docket entry naming a placeholder by its curated alias keys the + # docket row under the alias, so the planned succeeds crossed concepts + # and died only in staging replay. The builder now refuses at plan time. + existing = {"series": [ + dict(_docket_row("agency.a", U1, None), aliases=["agency.b"]), + ]} + docket = {"series": [{"series": "agency.b", "cadence": "monthly", + "extras": {"country": "US", + "targetUnit": "percent"}}]} + with pytest.raises(SystemExit, match="explicit ceremony"): + _build( + tmp_path, [], existing=existing, + registry_entries=[_mint("agency.a", U1)], docket=docket, + ) + + # A small identity universe for generated registry histories: two concepts # (so a row can carry another series' UUID), geographies at every # refinement level, three entities, and six UUIDs (so stale rows collide @@ -4884,12 +4975,15 @@ def _generated_registry(data: st.DataObject) -> bsc.UuidRegistry: Each step draws a move the current state allows: mint an unbound identity on an unused UUID, retire a live binding, revive a retired - one, supersede or reclaim onto an unused UUID, or take an enrichment - hop (retire an entity-less binding and succeed it at a refinement). + one, supersede or reclaim onto an unused UUID, take an enrichment hop + (retire an entity-less binding and succeed it at a refinement), or + return (mint a coarser placeholder and hop it onto a retired key that + already handed over). The move is kept only if UuidRegistry accepts it, so every history is one the grammar itself admits. Hops chain, so placeholders enriched once, twice and three times occur alongside stale, superseded, - retired, handed-over and reclaimed keys.""" + retired, handed-over and reclaimed keys, and keys that handed over and + then returned as a later hop's successor (live, but consumed).""" registry = bsc.UuidRegistry(pathlib.Path("generated.jsonl"), b"") for _ in range(data.draw(st.integers(1, 12), label="steps")): held = list(registry.latest.values()) @@ -4911,6 +5005,22 @@ def _generated_registry(data: st.DataObject) -> bsc.UuidRegistry: moves += ["reclaim"] if any(e["entity"] is None for e in live): moves += ["hop", "hop", "hop"] + # A consumed placeholder can take a lineage again as a later hop's + # successor (main writes this when a docket entry loses its + # country, which mints a fresh coarser placeholder, and regains + # it): mint a coarser placeholder and hop it straight onto the key. + returns = [ + (coarser, entry) + for entry in consumed + if entry["entity"] is None + for coarser in unbound + if coarser["concept"] == entry["concept"] + and coarser["entity"] is None + and _key(coarser) != _key(entry) + and _refines(coarser["geography"], entry["geography"]) + ] + if returns and unused: + moves += ["return", "return"] if not moves: break move = data.draw(st.sampled_from(moves)) @@ -4938,6 +5048,14 @@ def _generated_registry(data: st.DataObject) -> bsc.UuidRegistry: uuid=data.draw(st.sampled_from(unused)), reclaimed=True, note="generated reclaim after handover")] + elif move == "return": + coarser, entry = data.draw(st.sampled_from(returns)) + fresh = data.draw(st.sampled_from(unused)) + events = [ + dict(coarser, uuid=fresh), + dict(coarser, uuid=fresh, retired=True, note=CHAIN_NOTE), + dict(_ident(entry), uuid=fresh, succeeds=coarser), + ] else: entry = data.draw(st.sampled_from( [e for e in live if e["entity"] is None] @@ -4948,9 +5066,14 @@ def _generated_registry(data: st.DataObject) -> bsc.UuidRegistry: and _key(ident) != _key(entry) and _refines(entry["geography"], ident["geography"]) ] - # Entity-less targets are weighted up so hops chain. + # Entity-less targets are weighted up so hops chain, and retired + # ones further still so lineages return onto keys that already + # handed over (live again, but consumed). + retired_keys = {_key(e) for e in retired} target = data.draw(st.sampled_from( - [t for t in targets if t["entity"] is None] * 3 + targets + [t for t in targets if _key(t) in retired_keys] * 6 + + [t for t in targets if t["entity"] is None] * 3 + + targets )) events = [ dict(_ident(entry), uuid=entry["uuid"], retired=True, @@ -4996,9 +5119,10 @@ def test_enrichment_is_planned_exactly_when_the_grammar_admits_it( ) -> None: """Differential: plan-time guard vs. the registry grammar. - For any history the grammar admits, any docket-only row (live, stale, - superseded, retired, handed over, reclaimed, the root of a chain, or - carrying another series' UUID) and any observation of its concept: + For any history the grammar admits, any docket-only row (live, live + but already handed over, stale, superseded, retired, handed over, + reclaimed, the root of a chain, or carrying another series' UUID) and + any observation of its concept: * completeness: the builder plans the retire + succeeds pair exactly when UuidRegistry would accept that very pair (so no enrichment @@ -5017,11 +5141,14 @@ def test_enrichment_is_planned_exactly_when_the_grammar_admits_it( if e["concept"] == _GEN_CONCEPT and e["entity"] is None ] # Steer toward rows at keys the history holds, carrying their last - # binding: live ones (the chained case lives here) and stale ones - # (retired, handed over). Any geography with any UUID stays possible. + # binding: live ones (the chained case lives here), live ones that + # already handed over (consumed), and stale ones (retired, handed + # over). Any geography with any UUID stays possible. + live = [e for e in placeholders if not e.get("retired")] candidates = ( - [(e["geography"], e["uuid"]) for e in placeholders - if not e.get("retired")] * 8 + [(e["geography"], e["uuid"]) for e in live] * 8 + + [(e["geography"], e["uuid"]) for e in live + if _key(e) in registry.consumed] * 16 + [(e["geography"], e["uuid"]) for e in placeholders] * 2 + [(g, u) for g in _PLACEHOLDER_GEOS for u in _GEN_UUIDS] ) From 5be9ff8ac41f1fb4a198f14f7b35504bab7fac3a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 18 Oct 2026 14:09:29 -0400 Subject: [PATCH 4/5] Name the cause when a placeholder may not hand over Round-2 review of #360 (LOW L1, L3; NIT N1): - L1 (new in this PR): a docket entry matching a placeholder through a curated alias was refused with the UUID-reuse message, whose remedy (curated alias + --allow-remint) cannot help. - L3 (pre-existing): a placeholder that already handed over once was refused with "superseding UUIDs must be new", blaming a remint nobody made. When the placeholder holds the row's UUID live but either hand-over rule fails, _plan_enrichment now raises a diagnostic naming the cause: lineage never crosses concepts (name the docket entry by the placeholder's own concept), or a placeholder hands over at most once (delete its catalog row and rerun with --allow-remint). Both callers already refused every such row, since the UUID is registered, so only the message changes. Tests exercise each suggested curation: renaming the docket entry enriches cleanly, and deleting the consumed placeholder's row then --allow-remint revives the observed identity's own UUID and retires the placeholder, with --check passing. _event_kind now reads planned events too (absent markers there are None, not missing). N1: the backlog bullet and docstring said the live binding alone is the contract; they now state the hand-over rules too. Co-Authored-By: Claude Opus 5.5 --- docs/catalog-curation-backlog.md | 17 +++++-- scripts/build_series_catalog.py | 71 ++++++++++++++++++-------- tests/test_build_series_catalog.py | 80 ++++++++++++++++++++++++------ 3 files changed, 127 insertions(+), 41 deletions(-) diff --git a/docs/catalog-curation-backlog.md b/docs/catalog-curation-backlog.md index 5c854429..13f0a009 100644 --- a/docs/catalog-curation-backlog.md +++ b/docs/catalog-curation-backlog.md @@ -176,12 +176,19 @@ were closed on 2026-08-09 by chronicle#153: the first owner never moves, that conjunct refused every second enrichment hop. A placeholder that had already been enriched once (a geography-free placeholder enriched to geography-only when its docket - entry learned a country) could not take its first print. The conjunct is - gone, and the live binding alone is the contract. + entry learned a country) could not take its first print. `usaspending.dod.prime_award_obligations` (a2649fd9) hit this in Thesis - run 38962977031. Tests now pin the chain and the stale-row refusals. A - property test also checks that the builder plans an enrichment exactly - when the registry grammar admits the retire + succeeds pair. + run 38962977031. The conjunct is gone (#360). The contract is now the + live binding plus the grammar's hand-over rules: the placeholder's key + is live and bound to the row's UUID, it is the same concept as the + identity it hands over to, and it has not handed over before. A key that + handed over and later returned as the successor of another hop is live + again but stays consumed. When the binding holds but either hand-over + rule fails, the builder refuses at plan time and names the cause. + Tests pin the chain, the stale-row refusals, both hand-over refusals + and the curation each refusal suggests. A property test checks that the + builder plans an enrichment exactly when the registry grammar admits + the retire + succeeds pair. ## Country geography on Colorado-scoped concepts, and one raw series id (2026-09-04) diff --git a/scripts/build_series_catalog.py b/scripts/build_series_catalog.py index 0af4c366..685c5f59 100644 --- a/scripts/build_series_catalog.py +++ b/scripts/build_series_catalog.py @@ -2129,19 +2129,23 @@ def _plan_enrichment( entity: dict | None, row_uuid: str, ) -> bool: - """Plan a retire + succeeds pair when ``prior`` is a genuine - placeholder whose live binding owns ``row_uuid``; True if planned. + """Plan a retire + succeeds pair when ``prior`` is a docket + placeholder that holds ``row_uuid`` live and may hand it over to + ``canon_key``; True if planned. + + A placeholder that holds ``row_uuid`` live but may not hand it over + (``canon_key`` is another concept, or the placeholder already + handed over once) raises a diagnostic naming that cause. Every + caller would refuse such a row anyway, since the UUID is + registered, but with a UUID-reuse or remint message that misstates + why. """ prior_own_key = UuidRegistry.entry_key(prior) prior_geo = prior.get("geography") or None geo = geography or {} - # Lineage never crosses concepts. Observed buckets always take the - # prior's concept; a docket entry matched through a curated alias - # can name another one, and the grammar would refuse that succeeds. - enrichment_shaped = ( + placeholder_shaped = ( prior.get("status") == "docket-only" and prior.get("entity") is None - and prior.get("concept") == canon_key[0] and ( prior_geo is None or ( @@ -2152,26 +2156,49 @@ def _plan_enrichment( ) ) ) - # The contract is the LIVE binding: the row's own key must be live, - # bound to row_uuid right now, and still able to hand its lineage - # over. Liveness and binding refuse a stale row still carrying a - # UUID its key has since superseded away from, and a key that was - # retired, handed over or reclaimed. A key hands over at most once: - # one that handed over and later returned as the successor of - # another hop is live again but stays consumed, and the grammar - # refuses it as a predecessor. Staging replay would reject each of - # these plans anyway; this fails at plan time instead. - # Historical ownership (uuid_owner) is not part of the contract: it - # records FIRST ownership and never moves, so a placeholder that - # was itself the successor of an earlier enrichment never owns its - # UUID and could never be enriched again. + # The contract is the LIVE binding plus the grammar's hand-over + # rules. The row's own key must be live and bound to row_uuid right + # now: that refuses a stale row still carrying a UUID its key has + # since superseded away from, and a key that was retired, handed + # over or reclaimed. Staging replay would reject each of these + # plans anyway; this fails at plan time instead. Historical + # ownership (uuid_owner) is not part of the contract: it records + # FIRST ownership and never moves, so a placeholder that was itself + # the successor of an earlier enrichment never owns its UUID and + # could never be enriched again. if not ( - enrichment_shaped + placeholder_shaped and registry.is_live(prior_own_key) and registry.binding(prior_own_key) == row_uuid - and prior_own_key not in registry.consumed ): return False + # Lineage never crosses concepts. Observed buckets always take the + # prior's concept; a docket entry matched through a curated alias + # can name another one, and the grammar refuses that succeeds. + if prior.get("concept") != canon_key[0]: + raise SystemExit( + f"identity {canon_key} matches docket placeholder " + f"{prior_own_key} (uuid {row_uuid}) only through the " + f"curated alias {canon_key[0]!r}; lineage never crosses " + "concepts, so the placeholder cannot be enriched into it — " + f"name the docket entry {prior['concept']!r}, the " + "placeholder's own concept" + ) + # A key hands over at most once. One that handed over and later + # returned as the successor of another hop is live again, but it + # stays consumed and the grammar refuses it as a predecessor. + if prior_own_key in registry.consumed: + raise SystemExit( + f"identity {canon_key} would take uuid {row_uuid} from docket " + f"placeholder {prior_own_key}, which already handed its " + "lineage over once and holds this uuid as the successor of a " + "later enrichment; a placeholder hands over at most once. " + "Curate it: for example, delete the placeholder's catalog row " + "and rerun with --allow-remint, so the observed identity " + "takes its own lineage (reviving its retired binding, or " + "minting if it has none) and the placeholder's binding " + "retires" + ) # Docket-placeholder enrichment: the binding MOVES to the observed # identity via an explicit retire + succeeds pair. UUID continuity # is preserved, so no ceremony flag is needed — this holds for the diff --git a/tests/test_build_series_catalog.py b/tests/test_build_series_catalog.py index 7aa0c75a..ec326087 100644 --- a/tests/test_build_series_catalog.py +++ b/tests/test_build_series_catalog.py @@ -4463,8 +4463,9 @@ def _plan_events(plan: dict) -> list[dict]: def _event_kind(event: dict) -> str: + # Planned events carry absent markers as None; rendered lines omit them. for marker in ("succeeds", "supersedes", "retired", "revived", "reclaimed"): - if marker in event: + if event.get(marker) is not None: return marker return "mint" @@ -4840,10 +4841,11 @@ def test_a_returned_placeholder_never_hands_over_twice( # then the country returns and B takes U2 from A. B is live and bound # to U2, but it is still consumed: it already handed over once, and the # grammar never lets a predecessor hand over twice. When C's - # observation returns, the builder must refuse at plan time. Without the - # consumed check it planned retire(B/U2) + C succeeds B, which staging - # rejected. Base refused too (B never first-owned U2), so this pins the - # status quo for this lineage, not a regression. + # observation returns, the builder must refuse at plan time, naming the + # cause. Without the consumed check it planned retire(B/U2) + C + # succeeds B, which staging rejected. Base refused too (B never + # first-owned U2), but with a remint message that blamed a supersede + # nobody made. The curation the diagnostic suggests must then work. concept = "agency.lineage" bystander = _row("bls.cps.unemployment_rate") observed = _row(concept) @@ -4896,11 +4898,37 @@ def stage(country: bool, rows: list[dict]) -> None: catalog_before = (tmp_path / "catalog.json").read_bytes() registry_before = (tmp_path / "registry.jsonl").read_bytes() for flags in ([], remint): - with pytest.raises(SystemExit, match="superseding UUIDs must be new"): + with pytest.raises(SystemExit, match="hands over at most once"): bsc.main(argv + flags) assert (tmp_path / "catalog.json").read_bytes() == catalog_before assert (tmp_path / "registry.jsonl").read_bytes() == registry_before + # The suggested curation: delete the placeholder's catalog row and + # rerun with --allow-remint. C revives its own U1; B's U2 retires. + catalog = json.loads(catalog_before) + catalog["series"] = [ + r for r in catalog["series"] if r["concept"] != concept + ] + (tmp_path / "catalog.json").write_text( + json.dumps(catalog, indent=2) + "\n", encoding="utf-8" + ) + assert bsc.main(argv) == 1 # retiring B's live binding is gated + assert bsc.main(argv + remint) == 0 + assert bsc.main(argv + ["--check"]) == 0 + registry = bsc.UuidRegistry.load(tmp_path / "registry.jsonl") + u1 = registry.binding(c_key) + assert registry.is_live(c_key) and u1 != u2 + assert not registry.is_live(b_key) and registry.binding(b_key) == u2 + assert [ + (_key(e), _event_kind(e)) + for e in _registry_lines(tmp_path)[len(registry_before.splitlines()):] + ] == [(c_key, "revived"), (b_key, "retired")] + (row,) = [ + r for r in json.loads((tmp_path / "catalog.json").read_text())["series"] + if r["concept"] == concept + ] + assert (row["uuid"], row["status"], _key(row)) == (u1, "observed", c_key) + def test_a_docket_alias_never_plans_a_cross_concept_hop( tmp_path: pathlib.Path, @@ -4908,18 +4936,37 @@ def test_a_docket_alias_never_plans_a_cross_concept_hop( # Independent-review finding (chronicle#360, LOW, pre-existing): a # docket entry naming a placeholder by its curated alias keys the # docket row under the alias, so the planned succeeds crossed concepts - # and died only in staging replay. The builder now refuses at plan time. + # and died only in staging replay. The builder now refuses at plan time + # naming the cause, and the remedy it names (call the docket entry by + # the placeholder's own concept) enriches cleanly. existing = {"series": [ dict(_docket_row("agency.a", U1, None), aliases=["agency.b"]), ]} - docket = {"series": [{"series": "agency.b", "cadence": "monthly", - "extras": {"country": "US", - "targetUnit": "percent"}}]} - with pytest.raises(SystemExit, match="explicit ceremony"): + entry = {"series": "agency.b", "cadence": "monthly", + "extras": {"country": "US", "targetUnit": "percent"}} + with pytest.raises(SystemExit, match="never crosses concepts") as refused: _build( tmp_path, [], existing=existing, - registry_entries=[_mint("agency.a", U1)], docket=docket, + registry_entries=[_mint("agency.a", U1)], + docket={"series": [entry]}, ) + assert "name the docket entry 'agency.a'" in str(refused.value) + catalog, plan = _build( + tmp_path, [], existing=existing, + registry_entries=[_mint("agency.a", U1)], + docket={"series": [dict(entry, series="agency.a")]}, + ) + (row,) = catalog["series"] + assert (row["concept"], row["uuid"], row["status"]) == ( + "agency.a", U1, "docket-only" + ) + assert [_event_kind(e) for e in _plan_events(plan)] == [ + "retired", "succeeds" + ] + staged = _registry(tmp_path, [_mint("agency.a", U1)]).stage( + _plan_events(plan) + ) + assert bsc.registry_agreement_problems(catalog, staged) == [] # A small identity universe for generated registry histories: two concepts @@ -5185,8 +5232,13 @@ def test_enrichment_is_planned_exactly_when_the_grammar_admits_it( ) except SystemExit as exc: assert not admitted, exc - assert "explicit ceremony" in str(exc) or ( - "superseding UUIDs must be new" in str(exc) + assert any( + diagnostic in str(exc) + for diagnostic in ( + "explicit ceremony", + "superseding UUIDs must be new", + "hands over at most once", + ) ), exc return hop = [e for e in plan["mints"] if "succeeds" in e] From 10bd12e202e94363151c3def620a2704be5e5dfc Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 18 Oct 2026 14:31:04 -0400 Subject: [PATCH 5/5] Say only what is always true in the consumed-placeholder refusal Delta review of 5be9ff8a (CHANGES; F1 blocking, F2 and F3 non-blocking): the refusal for a placeholder that already handed over prescribed a recipe (delete its catalog row, rerun with --allow-remint) and promised outcomes that do not always hold: - F1: if the returning observation is spelled by a curated alias held only on the placeholder row, deleting the row loses that mapping; the rerun mints a separate alias-named identity and leaves the placeholder live. - F2: an observed identity that is itself consumed reclaims a fresh UUID rather than reviving its retired one. - F3: a returned placeholder can later supersede its UUID while staying consumed, so "holds this uuid as the successor of a later enrichment" can be false. The message now states only the cause: the placeholder already handed a lineage over once and remains consumed, so it cannot hand this one over and the lineage needs explicit curation. Refusal behaviour is unchanged (the review found zero raise-versus-plan differences over 478,632 old/new input pairs). The lifecycle test keeps demonstrating one curation that works for its simple lineage, labelled as such, and the backlog bullet records the caveats instead of a recipe. Co-Authored-By: Claude Opus 5.5 --- docs/catalog-curation-backlog.md | 16 +++++++++++----- scripts/build_series_catalog.py | 17 ++++++++--------- tests/test_build_series_catalog.py | 8 ++++++-- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/docs/catalog-curation-backlog.md b/docs/catalog-curation-backlog.md index 13f0a009..15a90e41 100644 --- a/docs/catalog-curation-backlog.md +++ b/docs/catalog-curation-backlog.md @@ -184,11 +184,17 @@ were closed on 2026-08-09 by chronicle#153: identity it hands over to, and it has not handed over before. A key that handed over and later returned as the successor of another hop is live again but stays consumed. When the binding holds but either hand-over - rule fails, the builder refuses at plan time and names the cause. - Tests pin the chain, the stale-row refusals, both hand-over refusals - and the curation each refusal suggests. A property test checks that the - builder plans an enrichment exactly when the registry grammar admits - the retire + succeeds pair. + rule fails, the builder refuses at plan time and names the cause. The + cross-concept refusal names its remedy (call the docket entry by the + placeholder's concept). The consumed refusal prescribes none, because + the right curation depends on the lineage. Deleting the placeholder's + catalog row and rerunning with --allow-remint resolves the simple case, + but it loses curated aliases held only on that row, and an observed + identity that is itself consumed reclaims a fresh UUID instead of + reviving. Tests pin the chain, the stale-row refusals, both hand-over + refusals, the cross-concept remedy and the simple consumed curation. A + property test checks that the builder plans an enrichment exactly when + the registry grammar admits the retire + succeeds pair. ## Country geography on Colorado-scoped concepts, and one raw series id (2026-09-04) diff --git a/scripts/build_series_catalog.py b/scripts/build_series_catalog.py index 685c5f59..2f8fb414 100644 --- a/scripts/build_series_catalog.py +++ b/scripts/build_series_catalog.py @@ -2186,18 +2186,17 @@ def _plan_enrichment( ) # A key hands over at most once. One that handed over and later # returned as the successor of another hop is live again, but it - # stays consumed and the grammar refuses it as a predecessor. + # stays consumed and the grammar refuses it as a predecessor. How + # to resolve that depends on the lineage (curated aliases, whether + # the observed identity has a binding of its own), so the + # diagnostic names the cause and prescribes no recipe. if prior_own_key in registry.consumed: raise SystemExit( f"identity {canon_key} would take uuid {row_uuid} from docket " - f"placeholder {prior_own_key}, which already handed its " - "lineage over once and holds this uuid as the successor of a " - "later enrichment; a placeholder hands over at most once. " - "Curate it: for example, delete the placeholder's catalog row " - "and rerun with --allow-remint, so the observed identity " - "takes its own lineage (reviving its retired binding, or " - "minting if it has none) and the placeholder's binding " - "retires" + f"placeholder {prior_own_key}, which already handed a lineage " + "over once and remains consumed; a placeholder hands over at " + "most once, so it cannot hand this one over — this lineage " + "needs explicit curation" ) # Docket-placeholder enrichment: the binding MOVES to the observed # identity via an explicit retire + succeeds pair. UUID continuity diff --git a/tests/test_build_series_catalog.py b/tests/test_build_series_catalog.py index 3636633c..16898a28 100644 --- a/tests/test_build_series_catalog.py +++ b/tests/test_build_series_catalog.py @@ -4857,7 +4857,10 @@ def test_a_returned_placeholder_never_hands_over_twice( # cause. Without the consumed check it planned retire(B/U2) + C # succeeds B, which staging rejected. Base refused too (B never # first-owned U2), but with a remint message that blamed a supersede - # nobody made. The curation the diagnostic suggests must then work. + # nobody made. The diagnostic names the cause and prescribes nothing: + # the right curation depends on the lineage (a delta review showed that + # deleting the placeholder's row loses curated aliases held only there, + # and that a consumed observed identity reclaims rather than revives). concept = "agency.lineage" bystander = _row("bls.cps.unemployment_rate") observed = _row(concept) @@ -4915,7 +4918,8 @@ def stage(country: bool, rows: list[dict]) -> None: assert (tmp_path / "catalog.json").read_bytes() == catalog_before assert (tmp_path / "registry.jsonl").read_bytes() == registry_before - # The suggested curation: delete the placeholder's catalog row and + # One curation that resolves this lineage (no curated aliases on B, and + # C has an unconsumed retired binding): delete B's catalog row and # rerun with --allow-remint. C revives its own U1; B's U2 retires. catalog = json.loads(catalog_before) catalog["series"] = [