From 4010c4b84bf564d9faf2be4f76f3cd4116dddf51 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 15 Oct 2026 07:43:12 -0400 Subject: [PATCH 01/10] Prepare d1386 Route A rerun for October 18 Pin CT ladder and current crosswalk, separate base and release feeds, and prepare fail-closed lineage evidence plus launchd operator sequence. Co-Authored-By: Claude Opus 5.5 --- ...s-modal-stage-route-a-rerun-base-plan.json | 61 +++ docs/us-modal-stage-runbook.md | 26 +- docs/us-route-a-rerun-20261018.env.example | 67 +++ docs/us-route-a-rerun-20261018.md | 486 ++++++++++++++++++ docs/us-route-a-rerun-prep-report.md | 95 ++++ .../engine_free/us/test_route_a_driver.py | 106 ++++ .../test_us_modal_stage_puf_support_base.py | 78 ++- .../us/test_us_route_a_local_reference.py | 266 ++++++++++ tools/modal_us_stage_plan.py | 16 +- tools/route_a/README.md | 10 + tools/route_a/make_local_reference.py | 293 +++++++++++ tools/route_a/route_a.env.example | 3 + tools/route_a/route_a.sh | 12 +- 13 files changed, 1505 insertions(+), 14 deletions(-) create mode 100644 docs/us-modal-stage-route-a-rerun-base-plan.json create mode 100644 docs/us-route-a-rerun-20261018.env.example create mode 100644 docs/us-route-a-rerun-20261018.md create mode 100644 docs/us-route-a-rerun-prep-report.md create mode 100644 packages/microcosm-build/tests/engine_free/us/test_us_route_a_local_reference.py create mode 100644 tools/route_a/make_local_reference.py diff --git a/docs/us-modal-stage-route-a-rerun-base-plan.json b/docs/us-modal-stage-route-a-rerun-base-plan.json new file mode 100644 index 0000000000..3f756dd2c6 --- /dev/null +++ b/docs/us-modal-stage-route-a-rerun-base-plan.json @@ -0,0 +1,61 @@ +{ + "schema": "microcosm-modal-us-stage-plan/1", + "tool": "us-puf-support-base-rerun", + "stage": "all", + "run_id": "route-a-rerun-base-20261018-pending", + "source": { + "commit": "PICK_MAIN_COMMIT_ON_2026_10_18", + "branch": "main" + }, + "inputs": { + "asec_2024_h5": { + "uri": "volume://cas/sha256/ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d/census_cps_2024.h5", + "sha256": "ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d" + }, + "asec_2023_h5": { + "uri": "volume://cas/sha256/cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88/census_cps_2023.h5", + "sha256": "cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88" + }, + "asec_2022_h5": { + "uri": "volume://cas/sha256/7ccca976284bb47815d84460cc4f75a0a65d26d7754ab0a0f417de351b3d474e/census_cps_2022.h5", + "sha256": "7ccca976284bb47815d84460cc4f75a0a65d26d7754ab0a0f417de351b3d474e" + }, + "puf_2024_h5": { + "uri": "volume://cas/sha256/7669f5b5281f20080e77204f9bd4aabfad0aa101fa283e22caf9ba8d61d4d6df/puf_2024.h5", + "sha256": "7669f5b5281f20080e77204f9bd4aabfad0aa101fa283e22caf9ba8d61d4d6df" + }, + "puf_2015_csv": { + "uri": "volume://cas/sha256/0a7fd643edb1acc55c507db795914b41d232922be78c149b58d111f4672499df/puf_2015.csv", + "sha256": "0a7fd643edb1acc55c507db795914b41d232922be78c149b58d111f4672499df" + }, + "acs_2022_h5": { + "uri": "volume://cas/sha256/0b319b496f19a6913066f9c5ea572edfda3d78a187be6f375846617d0b441bd4/acs_2022.h5", + "sha256": "0b319b496f19a6913066f9c5ea572edfda3d78a187be6f375846617d0b441bd4" + }, + "asec_education_2022_zip": { + "uri": "volume://cas/sha256/d2e000250782adfbdd7f29c82b66d866591a30f0d330496698ec19f9c784ce11/asecpub23csv.zip", + "sha256": "d2e000250782adfbdd7f29c82b66d866591a30f0d330496698ec19f9c784ce11" + }, + "asec_education_2023_zip": { + "uri": "volume://cas/sha256/cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09/asecpub24csv.zip", + "sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09" + }, + "asec_education_2024_zip": { + "uri": "volume://cas/sha256/318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b/asecpub25csv.zip", + "sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b" + }, + "base_ledger_facts": { + "uri": "volume://cas/sha256/b85437390021777e746f507c5890305496baf5fc7f2c78ba08ddb090f4839801/consumer_facts_us_c5e5bf8.jsonl", + "sha256": "b85437390021777e746f507c5890305496baf5fc7f2c78ba08ddb090f4839801" + }, + "block_ladder_npz": { + "uri": "volume://cas/sha256/6840b990acdfa2003d7723da5595e3cfff7205a4fa3d8d6206ed835c85c3f233/us_block_ladder_2020.npz", + "sha256": "6840b990acdfa2003d7723da5595e3cfff7205a4fa3d8d6206ed835c85c3f233" + } + }, + "env": { + "PYTHONUNBUFFERED": "1" + }, + "max_wall_seconds": 14400, + "nonpreemptible": true +} diff --git a/docs/us-modal-stage-runbook.md b/docs/us-modal-stage-runbook.md index 378656d602..2a1016e82f 100644 --- a/docs/us-modal-stage-runbook.md +++ b/docs/us-modal-stage-runbook.md @@ -3,12 +3,15 @@ Epic #956 acceleration item E: heavy US stages should not have to queue on the one 128 GiB build machine. This runbook covers the smallest working path: a registered tool run on Modal from a pinned commit, with inputs fetched by -digest and outputs listed with sha256 receipts. Four tools are registered +digest and outputs listed with sha256 receipts. Five tools are registered (`TOOLS` in the plan module): `tools/build_us_acs_local_release.py` (tool `us-acs-local-release`, stages `materialize`, `calibrate`, `qa`, `finalize`, `package`, or `all`), Route A's PUF-support base, `tools/build_us_puf_support_base.py` (tool `us-puf-support-base`, stage -`all`; see "Route A's base stage"), the ACS multispine staging builder, +`all`; see "Route A's base stage"), its current-main rerun registration +`us-puf-support-base-rerun` (same script and stage, with the reviewed current +district crosswalk; see [the 18 October rerun](us-route-a-rerun-20261018.md)), +the ACS multispine staging builder, `tools/build_us_acs_multispine_base.py` (tool `us-acs-multispine-staging`, stage `all`; see "ACS multispine staging"), and `runner-smoke` (stage `smoke`), an inline script that proves the run path on any pushed commit (see @@ -443,6 +446,14 @@ across containers would let preemption lose one chunk instead of the run. ## Route A's base stage +For Max's d1386 rerun on **18 October 2026**, use +[the rerun preparation and launch sequence](us-route-a-rerun-20261018.md) and +`us-modal-stage-route-a-rerun-base-plan.json`. Its commit is intentionally +unresolved. That procedure separates the historical base feed from main's +new release feed, pins the current in-tree crosswalk with a scoped registration, +and requires fresh local 000/001 evidence for the selected commit and CT ladder. +The historical plan/reference below cannot qualify that rerun. + On 29 September 2026 Max ruled to run only Route A's base stage on Modal: non-preemptible, capped at 4 hours of wall, about $15. The release, its preflight and certification stay on the build machine. @@ -527,7 +538,7 @@ Two kinds of test hold the command: command with its paths tokenized (`packages/microcosm-build/tests/fixtures/modal_us_stage/route_a_base_command_4b57d15a287c.json`), and the registration's owned flags equal the flags of this tree's tool. -- Only in a clone that has commit `4b57d15a2`, four tests in +- Only in a clone that has commit `4b57d15a2`, five tests in `test_us_modal_stage_puf_support_base.py` that read that commit with `git show` and skip without it: - `test_the_tool_s_own_parser_reads_the_same_build_from_both_commands`: @@ -543,10 +554,13 @@ Two kinds of test hold the command: its `_stage_run_config` locks, from the Modal argv, the run config the local run locked (paths by file name), including `PYTHONHASHSEED` `0` when the runner's tool environment removed a container's value, and a - forced `12345` is what compare-lineage refuses. + forced `12345` is what compare-lineage refuses; + - `test_rerun_crosswalk_refuses_the_historical_tree`: the rerun registration + refuses that commit's historical crosswalk bytes while retaining its + separately reviewed current-main pin. CI checks out only the commit under test (`actions/checkout` fetches one -commit by default), so there those four tests report a skip rather than +commit by default), so there those five tests report a skip rather than pass. Run them in a full clone before a paid run (step 2 below). The plan is `docs/us-modal-stage-route-a-base-plan.json` (run @@ -825,7 +839,7 @@ python3 tools/modal_us_stage_plan.py upload-commands --shell \ # 2. Validate locally: the argv, class base (cpu_limit 4.0), the $14.63 # estimate and the $14.90 ceiling at the request # (estimated_usd_at_timeout). Then run the base's tests, and confirm the -# four pinned-commit tests passed rather than skipped (-rs lists skips). +# five pinned-commit tests passed rather than skipped (-rs lists skips). python3 tools/modal_us_stage_plan.py validate docs/us-modal-stage-route-a-base-plan.json uv run pytest -rs packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py diff --git a/docs/us-route-a-rerun-20261018.env.example b/docs/us-route-a-rerun-20261018.env.example new file mode 100644 index 0000000000..ec623b98ac --- /dev/null +++ b/docs/us-route-a-rerun-20261018.env.example @@ -0,0 +1,67 @@ +# shellcheck shell=bash +# d1386 preparation, 2026-10-15. Copy to the machine-local 20261018 directory. +# COMMIT is deliberately empty. Recheck the marked settings on launch day. +# Copy to route_a.env, or point ROUTE_A_ENV at a machine-local settings file. +# No credentials belong here. Environment overrides win over these defaults. +# All machine locations are configured here; the driver uses colocated helpers. +: "${PE:=/Users/maxghenis/PolicyEngine}" +: "${MAIN:=$PE/microcosm}" +: "${WT_ROOT:=$PE/_worktrees}" +# Use a fresh root for a new attempt. Do not reuse a pre-CT base's ACCEPTED marker. +: "${RUN_ROOT:=$PE/_recovered/scratch-backup/893/route-a-rerun-20261018/runs}" +: "${CHAIN_ROOT:=$PE/_recovered/scratch-backup/893/overnight-20260923}" +: "${CHAIN_LOG:=$CHAIN_ROOT/chain.log}" +: "${DISK_PATH:=/System/Volumes/Data}" +: "${UV:=$HOME/.local/bin/uv}" +: "${PYTHON:=python3}" +: "${AGENT_SECRET:=$HOME/bin/agent-secret}" +: "${STORAGE:=$PE/policyengine-us-data/policyengine_us_data/storage}" +: "${EDU:=$PE/_buildm-runtime/inputs/asec_education}" +: "${FEED:=$PE/_buildh-runtime/inputs/consumer_facts_us_51aa40f.jsonl}" +# The driver preserves the published feed/crosswalk pins by default. If the +# chosen commit changed them, set FEED_SHA, FEED_BYTES and CROSSWALK_SHA here +# alongside FEED, after checking that commit's packaged pins. +# d713: CT-fixed ladder; route_a.sh verifies its digest and size. +: "${LADDER:=$PE/_build_artifacts/us-ct-cbsa/us_block_ladder_2020.npz}" +: "${SSI:=$PE/_buildo-runtime/inputs/attempt6_basis_schema3_seed.json}" +: "${SCF:=$PE/_buildm-runtime/inputs/scf_cache/rscfp2022.dta}" + +# Choose a pushed build commit. Empty COMMIT refuses a real run. +: "${COMMIT:=}" +: "${REQUIRE_ON_MAIN:=1}" +: "${REQUIRE_SPM_ROLE:=1}" +: "${PREFLIGHT_COMMIT:=}" +: "${PREFLIGHT_NEW_LINEAGE_ARGS:=--new-lineage}" +: "${EARLY_START_AFTER_HEAVY:=0}" +: "${PRUNE_BASE_CHECKPOINTS:=1}" +: "${RELEASE_DESPITE_PREFLIGHT_FAIL:=0}" +: "${RESOURCE_WAIT_HOURS:=12}" +: "${RELEASE_RAM_GIB:=50}" +: "${RELEASE_WALL_SECONDS:=345600}" +# 0 is the sole opt-out and restores --no-staging without a credential lookup. +: "${ROUTE_A_STAGING=1}" + +# Retain d122 whole-base dense national/state settings and the published run's +# export-mass reference. Review the historical tail waiver against the new export. +: "${DENSE_RELEASE_D122:=1}" +: "${EXPORT_MASS_REF_H5:=$PE/_buildg-runtime/forensics/populace_us_2024.h5}" +: "${RELEASE_EXTRA_ARGS:=--maximum-microsim-batch-size 2000 --target-surface national_state --dense-default-dataset --export-input-mass-reference-h5 $EXPORT_MASS_REF_H5}" +: "${QRF_TAIL_EXCLUSIONS:=$CHAIN_ROOT/route-a/qrf_tail_exclusions_routea_d490_20260930.json}" +: "${QRF_TAIL_EXCLUSIONS_SHA:=fa5760993ebcd1386f02c0b1c4c4e867a0c4b1e689a33774a41f623f95132d69}" +# Optional base feed override: BASE_LEDGER_FACTS and BASE_LEDGER_FACTS_SHA. + +# REVIEW: selected main must still pin this release feed. It changed since 4b57d15a2. +: "${FEED_SHA:=965a29ac9458edb58a05184917ee4a19772fd3dc7b4588db6eb6da5803641aac}" +: "${FEED_BYTES:=169038757}" +# Keep the historical base feed explicitly; its CD facts are unchanged by #940. +: "${BASE_LEDGER_FACTS:=$PE/_buildh-runtime/inputs/consumer_facts_us_c5e5bf8.jsonl}" +: "${BASE_LEDGER_FACTS_SHA:=b85437390021777e746f507c5890305496baf5fc7f2c78ba08ddb090f4839801}" +# REVIEW: selected main must still carry the NC-fixed in-tree crosswalk. +: "${CROSSWALK_SHA:=a347303fff3fea145f758488c43cb94355df5a8cb5e512552b2ee3791d2aa233}" +# REVIEW: historical export-mass reference is inherited, not re-certified here. +# REVIEW: d490 must be re-measured on the new lineage. Update file and SHA together. +# Allowed without another ruling: threshold 0.75, same or fewer columns only. +# REVIEW: d122 dense/national_state, batch 2000, RAM/wall/wait limits are inherited. +# REVIEW: PREFLIGHT_COMMIT defaults to COMMIT; keep both on the chosen main tree. +# Explicit staging destination; no credential value is stored here. +export POPULACE_STAGING_REPO_ID=policyengine/populace-us-staging diff --git a/docs/us-route-a-rerun-20261018.md b/docs/us-route-a-rerun-20261018.md new file mode 100644 index 0000000000..7361aa2683 --- /dev/null +++ b/docs/us-route-a-rerun-20261018.md @@ -0,0 +1,486 @@ +# Route A rerun on 18 October 2026 (d1386) + +Prepared on 15 October against cached `origin/main` +`412f916536b66577e4e0b42c707285085df7c302`. Fetching was denied by this +workspace's sandbox; these findings must be checked against the commit selected +on 18 October. Nothing in this preparation launched a build or called Modal. + +Max's d1386 selects whatever has merged to `main` on 18 October. It must contain +`9802825200929c445856d969f4231287c185d784` (#1186, the #1178 Social Security +fix). The base runs on Modal; release, preflights and certification run locally +through `tools/route_a/`. Staging is enabled. Publication remains separate. + +The assigned checkout is an older detached commit and its shared Git metadata +is read-only. Preparation therefore used an ignored, extracted cached-main tree +inside the assigned workspace, with independent local Git metadata. The portable +patch/bundle is against that main snapshot; apply it to a current-main prep +worktree before running this sequence. No caller metadata was written. +The requested external settings directory was also outside the writable +sandbox. The prepared, uncommitted settings fallback is +`build/route-a-rerun-20261018/route_a.env`; the reproducible template is +`docs/us-route-a-rerun-20261018.env.example`. Install it at the requested +machine-local path in step 1. Never commit `route_a.env`. + +## Inputs and registration + +The new plan is `docs/us-modal-stage-route-a-rerun-base-plan.json`. +`PICK_MAIN_COMMIT_ON_2026_10_18` is deliberately not a SHA: existing `validate` +returns 2 while it remains. Step 1 writes a concrete machine-local plan without +editing the template. All eleven inputs use the runbook's +`volume://cas/sha256//` layout on +`microcosm-us-stage-inputs`; the volume name is not part of the URI. + +| Input or setting | Change since `4b57d15a2` | +| --- | --- | +| Block ladder | d713 CT-fixed `6840b990acdfa2003d7723da5595e3cfff7205a4fa3d8d6206ed835c85c3f233`, 18,991,218 bytes; replaces `7ba39b95…` | +| ASEC H5s, income years 2024/2023/2022 | Same pins as the historical plan, in that order | +| ASEC education, income years 2022/2023/2024 | Same three archive pins; the first also seeds the unemployment-weeks cache | +| ASEC 2025 | Main added it to general defaults; Route A explicitly supplies 2024/2023/2022, so no fourth source is added | +| Processed PUF, TY2015 raw PUF, ACS 2022 | Same pins as the historical plan | +| Base feed | Retains `consumer_facts_us_c5e5bf8.jsonl`, SHA `b85437390021777e746f507c5890305496baf5fc7f2c78ba08ddb090f4839801` | +| Release feed | Main repinned under #940 to `consumer_facts_us_51aa40f.jsonl`, SHA `965a29ac9458edb58a05184917ee4a19772fd3dc7b4588db6eb6da5803641aac`, 169,038,757 bytes, 40,178 rows; the machine file was verified | +| In-tree district crosswalk | NC correction changed `c7cb040b…` to `a347303fff3fea145f758488c43cb94355df5a8cb5e512552b2ee3791d2aa233`, 78,202 bytes | + +The release feed retains all 39,158 old facts and adds 1,020 TY2023 state AGI +facts; see `us-chronicle-feed-repin.md`. Base district counts are unchanged, +so retaining its old feed is intentional. `BASE_LEDGER_FACTS` and its SHA are +explicitly independent from release `FEED`, `FEED_SHA` and `FEED_BYTES`. +The base builder itself is unchanged since `4b57d15a2`; runtime code has changed, +including the raw-PUF partnership self-employment correction. Identical input +pins do not imply identical outputs. + +`us-puf-support-base-rerun` preserves the historical base registration's argv, +inputs, cache seed, hash-seed handling, mirroring and resources, and pins the +current crosswalk. The historical `us-puf-support-base` registration and its +plan/reference remain unchanged. Both refuse an unrecognized crosswalk. The +driver's feed and crosswalk expectations are configurable, with historical +defaults; this rerun's settings explicitly pin the new values. If main changes +any pin again, review and update the affected settings/registration before +launching; do not suppress a check or substitute unreviewed bytes. + +The base retains non-preemptible placement, four requested/limited CPUs, +112 GiB memory, 14,400 seconds of tool budget and a 16,500-second class timeout. +The planner estimates $14.63 including reserve and $14.90 at the timeout and +request, below the $15 base cap. As the existing runbook explains, this is not +an absolute billed ceiling: memory above the request, checks, startup, image +builds and storage add cost. A second paid attempt needs Max's go. No resource +limit or cap interpretation is changed here. + +## Lineage decision + +Recommend a fresh local reference from **only named stages +`source_construction` and `pre_clone_enrichment`**, on the selected clean commit +and CT ladder, followed by strict `compare-lineage`. These stages add no Modal +cost. The retained local interrupted attempt's +`overnight-20260923/route-a/run-4b57d15a287c/base-sup.failed-1790651711/RESULT.json` +reports 2,054.6823 seconds elapsed and 12,516,982,784 bytes peak RSS (11.66 GiB), +after 000/001 completed. These are whole-attempt measurements, not isolated +000/001 timings; no retained local stage profile permits finer attribution. +Its two compared checkpoint files total 1,485,203,161 bytes. The current source +stage also writes an auxiliary raw-ASEC checkpoint; that total is not the full +local disk requirement. Allow RAM/disk headroom and +use supervision; shared free-disk decline cannot be attributed to those stages. + +The old Modal receipt reports 6,234.9 seconds tool wall, 6,281.6 seconds container +wall, 70.69 GiB peak RSS and a $5.67 list-price estimate for the completed old +base. These observations do not bound this rerun's time or cost. + +A new hand-off requires a COMPLETED, non-budget-stopped receipt, strict +`base-out` verification, the actual Modal context matching its receipt digest, +and a reference with the chosen commit, all eleven input digests, normalized +all-stage argv/environment, pipeline, run config, source-code digest, +dependency versions, hash seed 0 and actual 000/001 checkpoint digests. Every +comparison must pass. Interpreter and other thread-variable differences are +reported; only hash seed is compared among thread variables. Equality covers +two stages, not the later 22, district/ladder assignment or final export. +Release gates and certification still run. + +Do not relabel the old reference with a new commit/ladder or derive a local +reference from Modal outputs. `make_local_reference.py` reads the actual local +context/checkpoints and validates their inputs and source identity. Keep its +JSON and the supporting local evidence with this run. + +The old `base-sup/PROVENANCE.md` records a strict mismatch at stage 001: +313/432,523 `person.pre_subsidy_rent` values differed, maximum absolute +`8.7e-11`, relative `6.08e-15`. A prior session used a narrow exception. That +does not authorize this run. Any mismatch, skipping the new local reference, +or relaxing the hand-off needs **Max's explicit decision** with the new +evidence. There is no automatic waiver path in this preparation. Stop before +writing `ACCEPTED` while that decision is outstanding. + +## Settings to review on 18 October + +- `COMMIT`: empty until selected from freshly fetched main; must include #1186. +- `FEED`, `FEED_SHA`, `FEED_BYTES`, `BASE_LEDGER_FACTS`, its SHA and + `CROSSWALK_SHA`: recheck against the selected tree. The two feeds differ on + purpose. ASEC/PUF/ACS/education pins must also remain compatible. +- `PREFLIGHT_COMMIT` and `PREFLIGHT_NEW_LINEAGE_ARGS`: empty commit defaults to + the build commit; verify its parser and gates still accept `--new-lineage`. +- `EXPORT_MASS_REF_H5`: inherited + `$PE/_buildg-runtime/forensics/populace_us_2024.h5` (354,374,956 bytes). + Review its suitability and digest as the comparison reference for this run. + The driver does not include this file in `inputs.resolved.json`; record its + digest separately. The release must retain its export-mass gate. +- `QRF_TAIL_EXCLUSIONS` and `_SHA`: the d490 historical register is provisional + for this lineage. Re-measure and update the file and SHA together. Its four + columns are `bond_assets`, `estate_income`, + `long_term_capital_gains_on_collectibles`, and + `w2_wages_from_qualified_business`. Under d490 a new measurement may retain + threshold **0.75** and the same or a subset of these columns without another + ruling. Drop stale entries. Adding a column or lowering the threshold needs + Max's ruling. Its old reasons and expiry do not attest a new export. The + pre-solve dry run uses base weights; the final weighted-export gate still + determines whether the register is current. +- `DENSE_RELEASE_D122` and `RELEASE_EXTRA_ARGS`: retain the historical dense + whole-base, national/state target surface and batch size 2000. The original + d122 text described the first release; confirm this remains the intended + configuration for the rerun. +- `RELEASE_RAM_GIB=50`, `RELEASE_WALL_SECONDS=345600`, + `RESOURCE_WAIT_HOURS=12` and batch size 2000: inherited admission/budget + settings, unmeasured on the new commit and current load. The 110 GiB RSS + cap remains enforced. +- Staging: `ROUTE_A_STAGING=1`; destination explicitly + `policyengine/populace-us-staging`. Check runtime helper access and dashboard + delivery on launch day. Do not fetch/store credentials during preparation. +- Small-H5 guard fixture: choose a real small written export with enough + households for at least two batches. Do not substitute a full export for it. + +## Exact launch-day sequence + +Run the blocks in **Bash**, with `set -euo pipefail`, from a prep checkout +containing these changes. `PREP` is that checkout; `W` below is a separate, +clean checkout of the chosen build commit. Nothing writes to the old run or +`microcosm-route-a-4b57d15a287c` worktree. There is no automatic date scheduler. + +### 1. Select main, install settings and fill the plan + +```bash +set -euo pipefail +PE=/Users/maxghenis/PolicyEngine +MAIN=$PE/microcosm +PREP=$PE/_worktrees/route-a-rerun-prep +LOCAL=$PE/_recovered/scratch-backup/893/route-a-rerun-20261018 +SETTINGS=$LOCAL/route_a.env +PLAN=$LOCAL/base-plan.json +LSUBMIT=$PE/_recovered/scratch-backup/893/bin/lsubmit.sh +cd "$PREP" +git -C "$MAIN" fetch origin +C=$(git -C "$MAIN" rev-parse 'origin/main^{commit}') +git -C "$MAIN" merge-base --is-ancestor 9802825200929c445856d969f4231287c185d784 "$C" +mkdir -p "$LOCAL" +# Once only: preserve any existing operator edits. +test -e "$SETTINGS" || cp docs/us-route-a-rerun-20261018.env.example "$SETTINGS" +python3 - "$MAIN" "$C" "$PLAN" "$SETTINGS" <<'PY' +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +main, commit, output, settings = sys.argv[1:] +def shown(path): + return subprocess.check_output(["git", "-C", main, "show", f"{commit}:{path}"]) + +feed = json.loads(shown("packages/microcosm-build/src/microcosm/build/us/chronicle_feed.json")) +if feed["facts_sha256"] != "965a29ac9458edb58a05184917ee4a19772fd3dc7b4588db6eb6da5803641aac": + raise SystemExit("REFUSED: main changed the release feed; review its pins/settings") +crosswalk = "packages/microcosm-build/src/microcosm/build/us_runtime/data/congressional_district_vintage_crosswalk.csv" +if hashlib.sha256(shown(crosswalk)).hexdigest() != "a347303fff3fea145f758488c43cb94355df5a8cb5e512552b2ee3791d2aa233": + raise SystemExit("REFUSED: main changed the crosswalk; review the rerun registration") +plan = json.loads(Path("docs/us-modal-stage-route-a-rerun-base-plan.json").read_text()) +plan["source"]["commit"] = commit +plan["run_id"] = f"route-a-rerun-base-20261018-{commit[:12]}" +text = Path(settings).read_text() +if ': "${COMMIT:=}"' not in text: + raise SystemExit("REFUSED: settings already pinned; review before replacing COMMIT") +if Path(output).exists(): + raise SystemExit("REFUSED: concrete plan already exists; preserve the attempt identity") +Path(output).write_text(json.dumps(plan, indent=2) + "\n") +Path(settings).write_text(text.replace(': "${COMMIT:=}"', ': "${COMMIT:=' + commit + '}"')) +print(commit) +PY +python3 tools/modal_us_stage_plan.py validate "$PLAN" > "$LOCAL/plan-validation.json" +# Review SETTINGS and the uncertainties above before proceeding. +. "$SETTINGS" +test "$COMMIT" = "$C" +test "$ROUTE_A_STAGING" = 1 +shasum -a 256 "$EXPORT_MASS_REF_H5" > "$LOCAL/export-mass-reference.sha256" +``` + +Use a fresh shell or clear stale Route A environment overrides: defaults in +the settings file intentionally let nonempty environment values win. `lsubmit` +does not inherit them, so persist every intended change in the settings file. + +### 2. Engine-free checks, resolve and missing input uploads + +```bash +uv run pytest -n 4 -rs \ + packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py \ + packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py \ + packages/microcosm-build/tests/engine_free/us/test_route_a_driver.py \ + packages/microcosm-build/tests/engine_free/us/test_us_route_a_local_reference.py +uv run ruff check . +uv run ruff format --check tools/modal_us_stage_plan.py tools/route_a/make_local_reference.py \ + packages/microcosm-build/tests/engine_free/us/test_route_a_driver.py \ + packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py \ + packages/microcosm-build/tests/engine_free/us/test_us_route_a_local_reference.py +python3 tools/ci_test_plan.py verify +ROUTE_A_ENV="$SETTINGS" bash tools/route_a/route_a.sh --resolve | tee "$LOCAL/resolve-initial.log" +# Resolve exit 0 alone is insufficient: it can report commit blockers. +if rg 'BLOCKER' "$LOCAL/resolve-initial.log"; then exit 1; fi +# Inspect inputs.resolved.json; every registered file must be verified. +python3 tools/modal_us_stage_plan.py upload-commands "$PLAN" --shell \ + "asec_2024_h5=$STORAGE/census_cps_2024.h5" \ + "asec_2023_h5=$STORAGE/census_cps_2023.h5" \ + "asec_2022_h5=$STORAGE/census_cps_2022.h5" \ + "puf_2024_h5=$STORAGE/puf_2024.h5" "puf_2015_csv=$STORAGE/puf_2015.csv" \ + "acs_2022_h5=$STORAGE/acs_2022.h5" \ + "asec_education_2022_zip=$EDU/asecpub23csv.zip" \ + "asec_education_2023_zip=$EDU/asecpub24csv.zip" \ + "asec_education_2024_zip=$EDU/asecpub25csv.zip" \ + "base_ledger_facts=$BASE_LEDGER_FACTS" "block_ladder_npz=$LADDER" \ + > "$LOCAL/upload-base-inputs.sh" +# Hashes are checked before this script is emitted; it skips existing CAS files. +# The CT upload may already have completed; do not overwrite unrelated paths. +bash "$LOCAL/upload-base-inputs.sh" +MODAL=$(command -v modal) +MICROCOSM_MODAL_PLAN="$PLAN" "$MODAL" run tools/modal_us_stage.py \ + > "$LOCAL/modal-check.log" 2>&1 +``` + +Preparation also ran repository-wide `ruff format --check .` with locked Ruff +0.15.16: 45 unchanged cached-main files fail that check. All five changed Python +files pass; the scoped launch-day command above checks them without rewriting +unrelated sources or their identities. Reassess this baseline after fetching. + +Read the check report: no problems, branch/source/crosswalk/eleven inputs +verified, disk admission, write probe and hash seed accepted. A failed check +blocks the paid launch. Check all five pinned-commit tests passed +instead of skipped in the full-history checkout. No Modal command is run as +part of preparing this document. + +### 3. Launch the Modal base under launchd + +```bash +"$LSUBMIT" route-a-rerun-20261018-base "$LOCAL/base.launchd.log" "$PREP" \ + /usr/bin/env MICROCOSM_MODAL_PLAN="$PLAN" \ + "$MODAL" run --detach tools/modal_us_stage.py --run +``` + +Keep launchd's default `ProcessType=Standard`; do not set `KEEPALIVE=1`. +Use the same reviewed plan for this one paid attempt. A failure or budget stop +does not authorize an automatic relaunch. + +### 4. Produce the local reference (scheduled, supervised local work) + +This step runs engines and must wait for machine admission. It was not run +during preparation. Create the clean chosen-commit checkout and locked US env. +The script below supervises each named stage with the existing local base +limits: 100 GiB RSS cap, 80 GiB RAM/70 GiB disk admission, six-hour wall limit +and persistent 20 GiB disk floor. Admission refusal needs a later operator +retry; it does not silently run on the overloaded machine. It never invokes +`--stage all` locally. + +```bash +W=$WT_ROOT/microcosm-route-a-${C:0:12} +if [ ! -d "$W" ]; then git -C "$MAIN" worktree add --detach "$W" "$C"; fi +test "$(git -C "$W" rev-parse HEAD)" = "$C" +test -z "$(git -C "$W" status --porcelain)" +(cd "$W" && "$UV" sync --all-packages --locked --extra us) +REFROOT=$LOCAL/local-reference +REF=$LOCAL/base-local-reference.json +mkdir -p "$REFROOT/base-checkpoints" +BASE_ARGS=(--checkpoint-dir "$REFROOT/base-checkpoints" + --asec-h5 "2024=$STORAGE/census_cps_2024.h5" + --asec-h5 "2023=$STORAGE/census_cps_2023.h5" + --asec-h5 "2022=$STORAGE/census_cps_2022.h5" + --asec-h5-sha256 2024=ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d + --asec-h5-sha256 2023=cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88 + --asec-h5-sha256 2022=7ccca976284bb47815d84460cc4f75a0a65d26d7754ab0a0f417de351b3d474e + --puf-h5 "$STORAGE/puf_2024.h5" --puf-source-year-csv "$STORAGE/puf_2015.csv" + --acs-h5 "$STORAGE/acs_2022.h5" + --asec-education-source "2022=$EDU/asecpub23csv.zip" + --asec-education-source "2023=$EDU/asecpub24csv.zip" + --asec-education-source "2024=$EDU/asecpub25csv.zip" + --target-year 2024 --seed 0 --n-estimators 32 --ledger-facts "$BASE_LEDGER_FACTS" + --assign-congressional-districts + --congressional-district-vintage-crosswalk "$W/packages/microcosm-build/src/microcosm/build/us_runtime/data/congressional_district_vintage_crosswalk.csv" + --congressional-district-seed 0 --block-ladder-artifact "$LADDER" + --out "$REFROOT/base-out") +{ + echo '#!/bin/bash' + echo 'set -euo pipefail' + declare -p W PREP PLAN REFROOT REF BASE_ARGS + cat <<'BASH' +for stage in source_construction pre_clone_enrichment; do + python3 - "$REFROOT/$stage-config.json" "$W" "$stage" "${BASE_ARGS[@]}" <<'PY' +import json +import sys +from pathlib import Path +output, worktree, stage, *args = sys.argv[1:] +gib = 1024 ** 3 +config = { + "cwd": worktree, + "argv": [worktree + "/.venv/bin/python", "-B", "tools/build_us_puf_support_base.py", *args, "--stage", stage], + "env": {"PYTHONHASHSEED": "0", "PYTHONUNBUFFERED": "1"}, + "authorization": "d1386: local 000/001 comparison evidence only", + "limits": {"wall_seconds": 21600, "cpu_seconds": 64800, + "rss_bytes": 100*gib, "available_ram_admission_bytes": 80*gib, + "disk_admission_bytes": 70*gib, "disk_floor_bytes": 20*gib, + "disk_floor_persist_seconds": 180, + "output_bytes": 2*gib, "log_bytes": 512*1024**2}, +} +with Path(output).open("x") as stream: + json.dump(config, stream, indent=2) +PY + "$W/.venv/bin/python" "$PREP/tools/route_a/supervise.py" \ + "$REFROOT/$stage-sup" "$REFROOT/$stage-config.json" +done +python3 "$PREP/tools/route_a/make_local_reference.py" --plan "$PLAN" \ + --checkpoint-root "$REFROOT/base-checkpoints" --repo-root "$W" --out "$REF" +BASH +} > "$LOCAL/local-reference.sh" +"$LSUBMIT" route-a-rerun-20261018-reference "$LOCAL/reference.launchd.log" "$PREP" \ + /bin/bash "$LOCAL/local-reference.sh" +``` + +Wait for this job to finish successfully and `$REF` to exist before step 5. +Retain the per-stage supervisor records. On refusal/failure, inspect them and +move only this attempt's failed supervisor/config files aside before retrying; +the two stages' completed-prefix evidence remains in the fresh reference root. + +### 5. Fetch, compare and hand off + +After the Modal base completes, fetch only the receipt, output and context. +Check the CLI's downloaded directory layout before selecting `RECEIPT`. + +```bash +R=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["run_id"])' "$PLAN") +FETCH=$LOCAL/modal-runs/$R +mkdir -p "$FETCH/state" +"$MODAL" volume get microcosm-us-stage-runs "runs/$R/receipts" "$FETCH/" +"$MODAL" volume get microcosm-us-stage-runs "runs/$R/state/base-out" "$FETCH/state/" +"$MODAL" volume get microcosm-us-stage-runs \ + "runs/$R/state/base-checkpoints/stage_run_context.json" "$FETCH/" +RECEIPT=$(python3 - "$FETCH" <<'PY' +import sys +from pathlib import Path +paths = list((Path(sys.argv[1]) / "receipts").glob("all-*.json")) +if len(paths) != 1: + raise SystemExit("REFUSED: expected one attempt receipt; inspect all receipts manually") +print(paths[0]) +PY +) +CONTEXT=$FETCH/stage_run_context.json +LINEAGE=$FETCH/compare-lineage.json +RUN=$RUN_ROOT/run-${C:0:12} +( + set -euo pipefail + test ! -e "$RUN/base-out" && test ! -e "$RUN/base-sup" + python3 tools/modal_us_stage_plan.py verify-receipt "$RECEIPT" \ + --state-root "$FETCH/state" --prefix base-out --strict + python3 tools/modal_us_stage_plan.py compare-lineage "$RECEIPT" \ + --reference "$REF" --run-context "$CONTEXT" > "$LINEAGE" + mkdir -p "$RUN/base-out" + cp -p "$FETCH/state/base-out/"* "$RUN/base-out/" + python3 tools/modal_us_stage_plan.py verify-receipt "$RECEIPT" \ + --state-root "$RUN" --prefix base-out --strict + H5=$RUN/base-out/base_populace_us_2024_puf_support.h5 + shasum -a 256 "$H5" > "$RUN/base.sha256" + if [ -e "$RUN/base-checkpoints" ]; then + mv "$RUN/base-checkpoints" "$RUN/base-checkpoints.local-$(date +%s)" + fi + mkdir -p "$RUN/base-sup" + cp "$RECEIPT" "$RUN/base-sup/modal-receipt.json" + cp "$CONTEXT" "$RUN/base-sup/modal-stage_run_context.json" + cp "$LINEAGE" "$RUN/base-sup/compare-lineage.json" + cp "$REF" "$RUN/base-sup/local-reference.json" + RECEIPT_SHA=$(shasum -a 256 "$RECEIPT" | cut -c1-64) + printf '{\n "status": "MODAL_RECEIPT",\n "returncode": 0,\n "refusal": null,\n "receipt_sha256": "%s",\n "compare_lineage": "compare-lineage.json"\n}\n' \ + "$RECEIPT_SHA" > "$RUN/base-sup/RESULT.json" + echo "returncode=0 $(date '+%F %T') modal receipt sha256 $RECEIPT_SHA" \ + > "$RUN/base-sup/ACCEPTED" +) +``` + +`ACCEPTED` is last. If any comparison fails, stop and retain the evidence for +Max; do not write the marker. A partly executed copy block needs inspection +before any retry. Do not remove old run evidence. + +### 6. Release guards, register review and release under launchd + +Regenerate the maintained eight-household, five-state fixture into this run's +scratch, then use it as `SMALL_H5`. The old guard fixture no longer exists. Its +retained PR4 round-2 report measured the sweep at 6,351.77 seconds (106 minutes), +4.75 GiB peak RSS and 28 serial workers, so schedule that check too. From `W`, run +the complete guard sweep and the release's pre-solve register dry run before +starting the driver. These engine checks are deliberately deferred until launch +day. The dry run's inherited d490 register is a candidate to grade, not proof +that it applies to the new weighted export. Resolve any certain failure and +review AT-RISK results before release. + +```bash +{ + echo '#!/bin/bash' + echo 'set -euo pipefail' + declare -p W LOCAL RUN SETTINGS + cat <<'BASH' +. "$SETTINGS" +(cd "$W" && .venv/bin/python - "$LOCAL" <<'PY' +import sys +from pathlib import Path +from tools.sweep_us_post_export_scoring import _load_builder +from test_support.microcosm_build.us_post_export_scoring import ( + _GUARD_HOUSEHOLDS, _write_engine_h5, +) +_write_engine_h5(_load_builder(), Path(sys.argv[1]) / "guard-fixture", _GUARD_HOUSEHOLDS) +PY +) +SMALL_H5=$LOCAL/guard-fixture/populace_us_2024.h5 +(cd "$W" && .venv/bin/python tools/sweep_us_post_export_scoring.py \ + --dataset-path "$SMALL_H5" --batch-size 3 --reforms-per-worker 4 \ + --output "$LOCAL/guard-sweep.json") +EXTRA=($RELEASE_EXTRA_ARGS) +gate_rc=0 +(cd "$W" && .venv/bin/python tools/dry_run_us_release_gates.py \ + --json-out "$LOCAL/dry-run-gates.json" -- \ + --base-h5 "$RUN/base-out/base_populace_us_2024_puf_support.h5" \ + --ledger-facts "$FEED" --ledger-facts-sha256 "$FEED_SHA" \ + --qrf-tail-concentration-exclusions "$QRF_TAIL_EXCLUSIONS" \ + --ssi-take-up-prior-weight-basis "$SSI" \ + --ssi-take-up-prior-weight-basis-sha256 25fe8af50a99d717f3408b2de7f0849d2307d4f05b1a7d55d2703999002fff0a \ + --scf-summary-extract "$SCF" --out "$LOCAL/gate-dry-run" \ + --release-id route-a-rerun-20261018-gate-check \ + --checkpoint-root "$LOCAL/gate-checkpoints" --seed 0 --no-staging "${EXTRA[@]}") || gate_rc=$? +case "$gate_rc" in + 0) echo 'Release guards completed cleanly' ;; + 2) echo 'AT-RISK: review dry-run-gates.json before starting release'; exit 2 ;; + *) echo 'Release guard dry run failed'; exit "$gate_rc" ;; +esac +BASH +} > "$LOCAL/release-checks.sh" +"$LSUBMIT" route-a-rerun-20261018-checks "$LOCAL/checks.launchd.log" "$PREP" \ + /bin/bash "$LOCAL/release-checks.sh" +``` + +Wait for the checks job to finish. Inspect its exit status and both reports; +an exit 2 requires operator review of AT-RISK findings before continuing. +A failed sweep or a certain gate failure blocks release. Then start the driver: + +```bash +# If settings/register changed during review, source again and resolve again. +. "$SETTINGS" +ROUTE_A_ENV="$SETTINGS" bash tools/route_a/route_a.sh --resolve | tee "$LOCAL/resolve-release.log" +if rg 'BLOCKER' "$LOCAL/resolve-release.log"; then exit 1; fi +test -f "$RUN/base-sup/ACCEPTED" +"$LSUBMIT" route-a-rerun-20261018-release "$LOCAL/release.launchd.log" "$PREP" \ + /usr/bin/env ROUTE_A_ENV="$SETTINGS" /bin/bash "$PREP/tools/route_a/route_a.sh" +``` + +`lsubmit.sh` passes HOME/PATH and explicit job variables only. The `/usr/bin/env` +arguments above are necessary for plan/settings delivery. The driver skips the +accepted Modal base and runs prefetch, base preflight, release with staging, +release preflight and offline publisher preflight. There is no release-only +flag. Confirm staging telemetry arrives and retain the local hand-off and final +gate evidence. A `ROUTE_A_DONE.json` is a candidate hand-off with +`published: false`; publication requires its separate decision. diff --git a/docs/us-route-a-rerun-prep-report.md b/docs/us-route-a-rerun-prep-report.md new file mode 100644 index 0000000000..a275ed52f5 --- /dev/null +++ b/docs/us-route-a-rerun-prep-report.md @@ -0,0 +1,95 @@ +# Route A preparation report — 15 October 2026 + +Prepared d1386 against cached main +`412f916536b66577e4e0b42c707285085df7c302`; no data build, Modal command, +credential lookup, upload, publication or push ran. The selected 18 October +commit is deliberately unset. See [the launch sequence](us-route-a-rerun-20261018.md) +for input findings, all uncertain settings, lineage requirements and commands. + +## Files + +- `docs/us-modal-stage-route-a-rerun-base-plan.json`: refuses validation until + the commit placeholder is filled; CT ladder and ten historical base inputs. +- `docs/us-route-a-rerun-20261018.env.example`: reproducible settings, with new + release feed, independent old base feed, new crosswalk, staging and d490. +- `docs/us-route-a-rerun-20261018.md`: exact launch-day commands and decisions. +- `docs/us-modal-stage-runbook.md`, `tools/route_a/README.md`: rerun links and + registration/settings documentation. +- `tools/modal_us_stage_plan.py`: scoped rerun ToolSpec; preserves old replay. +- `tools/route_a/route_a.sh`, `route_a.env.example`: configurable explicit feed + digest/size and crosswalk pin with historical defaults. +- `tools/route_a/make_local_reference.py`: derives evidence from real local + 000/001 checkpoints, context and verified inputs/source; launches nothing. +- `packages/microcosm-build/tests/engine_free/us/test_route_a_driver.py`, + `test_us_modal_stage_puf_support_base.py`, `test_us_route_a_local_reference.py`: + input-pin, registration, placeholder and provenance/refusal coverage. + +## Validation + +- 377 isolated engine-free tool tests passed, at most four pytest workers and + one native numerical thread per worker. All added tests passed. +- `ruff check .` passed using locked Ruff 0.15.16. +- `ruff format --check` passed on all five changed Python files. The full + repository check reports 45 formatting failures; every failing file's Git + object hash was checked against cached main and is unchanged. They were + preserved. +- `python3 tools/ci_test_plan.py verify` passed: 705 registered test files. +- Driver/settings and all seven documented Bash blocks pass `bash -n`. +- The documented pin-filling command was exercised using temporary plan and + settings files and cached main. It produces a valid plan, refuses repinning, + and the untouched template returns validation exit 2. The concrete plan's + request-price timeout estimate is below $15 ($14.90). + +The isolated pytest run used `--confcutdir=packages/microcosm-build/tests/engine_free/us` +to avoid the unrelated root telemetry fixture's installed-distribution needs. +The ordinary fixture-based invocation was attempted and failed because +`microcosm-fit` distribution metadata is absent. No project wheels/editable +packages were built. Four existing tests were excluded from the passing run: + +- `test_the_tool_s_own_parser_reads_the_same_build_from_both_commands` +- `test_the_pinned_tool_locks_the_reference_s_run_config_from_the_modal_argv` +- `test_the_seed_stops_the_tool_s_census_download` +- `test_wrapper_secret_reaches_only_child_environment` + +The first three require installed workspace distribution metadata; the last +requires `ps`, denied by the sandbox. Those checks must run in the launch-day +locked environment; the runbook's full test command includes them. This report +does not claim certification or a fully green ordinary environment. + +## Sandbox and hand-off + +Only the assigned workspace was written. Both `git fetch origin` and staging +files in the assigned checkout failed because Git metadata under the caller's +repository is read-only. The intended `_worktrees/` and machine-local settings +destinations are outside the writable sandbox too. + +An independent cached-main checkout with its own Git metadata was therefore +prepared at `build/route-a-rerun-prep-main/` inside this workspace. Its branch is +`route-a-rerun-prep`; its commit is exported in the workspace-root +`route-a-rerun-prep.bundle` and `route-a-rerun-prep.patch`. These are portable +changes against cached main, avoiding the assigned checkout's older parent. +The assigned outer checkout remains **uncommitted**. It also holds copies of +the changed files for inspection. Use the bundle/patch to integrate with main; +do not treat the old outer checkout as a runnable current-main tree. + +The actual uncommitted settings fallback is +`build/route-a-rerun-20261018/route_a.env`. The requested +`/Users/maxghenis/PolicyEngine/_recovered/scratch-backup/893/route-a-rerun-20261018/route_a.env` +was not written. Install it with step 1 of the launch sequence; no actual +`route_a.env` is committed in the bundle. + +Outside this sandbox, recover the committed prep worktree without pushing: + +```bash +PE=/Users/maxghenis/PolicyEngine +MAIN=$PE/microcosm +ASSIGNED=/Users/maxghenis/.subfleet/worktrees/20261015-052227-route-a-rerun-prep +git -C "$MAIN" fetch "$ASSIGNED/route-a-rerun-prep.bundle" refs/heads/route-a-rerun-prep +git -C "$MAIN" worktree add -b route-a-rerun-prep "$PE/_worktrees/route-a-rerun-prep" FETCH_HEAD +``` + +That prep tree contains reviewed runner/driver changes. On 18 October, fetch +main again and select the build commit independently as the launch sequence +requires. If applying to a newer prep branch, use the patch or cherry-pick the +bundle's commit and review any conflicts. No historical run or worktree was +modified. diff --git a/packages/microcosm-build/tests/engine_free/us/test_route_a_driver.py b/packages/microcosm-build/tests/engine_free/us/test_route_a_driver.py index 1f4a45ada7..4cb59ccc83 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_route_a_driver.py +++ b/packages/microcosm-build/tests/engine_free/us/test_route_a_driver.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import importlib.util import json import os @@ -103,6 +104,111 @@ def _example_extra_args() -> str: return result.stdout.decode() +def _input_settings_result( + tmp_path: Path, overrides: dict[str, str], commands: str +) -> subprocess.CompletedProcess[bytes]: + source = (ROUTE_A_TOOLS / "route_a.sh").read_text(encoding="utf-8") + prefix, delimiter, _ = source.partition("FETCH_OK=1\n") + assert delimiter + # Only initialize settings and define functions. The remainder fetches + # refs and may launch stages, so this harness never executes it. + script = tmp_path / "input-settings-check.sh" + script.write_text(prefix + commands, encoding="utf-8") + required = ( + "PE MAIN WT_ROOT RUN_ROOT CHAIN_ROOT CHAIN_LOG DISK_PATH UV PYTHON " + "STORAGE EDU FEED LADDER SSI SCF AGENT_SECRET" + ).split() + settings = {key: str(tmp_path / key.lower()) for key in required} + settings.update(PYTHON=sys.executable, **overrides) + settings_path = tmp_path / "route_a.env" + settings_path.write_text( + "\n".join(f"{key}={shlex.quote(value)}" for key, value in settings.items()) + + "\n", + encoding="utf-8", + ) + return subprocess.run( + ["bash", str(script)], + env={"PATH": os.environ["PATH"], "ROUTE_A_ENV": str(settings_path)}, + capture_output=True, + check=False, + timeout=20, + ) + + +@pytest.mark.parametrize("override", [False, True]) +def test_feed_and_crosswalk_pins_keep_defaults_or_settings_overrides( + tmp_path: Path, override: bool +) -> None: + historical_feed = "b85437390021777e746f507c5890305496baf5fc7f2c78ba08ddb090f4839801" + historical_crosswalk = ( + "c7cb040b1f57ca2ea2adcbfe60cc2b250ca23acbc4b640cd421e766fa54c1aec" + ) + overrides = ( + { + "FEED_SHA": "a" * 64, + "FEED_BYTES": "123", + "CROSSWALK_SHA": "b" * 64, + "BASE_LEDGER_FACTS": str(tmp_path / "historical-base-feed.jsonl"), + "BASE_LEDGER_FACTS_SHA": historical_feed, + } + if override + else {} + ) + result = _input_settings_result( + tmp_path, + overrides, + '\ninput_rows\nprintf "crosswalk|%s\\n" "$CROSSWALK_SHA"\n', + ) + assert result.returncode == 0, result.stderr.decode() + rows = { + row[0]: row[1:] + for row in map(lambda s: s.split("|"), result.stdout.decode().splitlines()) + } + release = rows["release_ledger_facts"] + base = rows["base_ledger_facts"] + assert release[1:3] == ( + ["a" * 64, "123"] if override else [historical_feed, "164603204"] + ) + assert rows["crosswalk"] == ["b" * 64 if override else historical_crosswalk] + assert base[1] == historical_feed + assert base[0] == (overrides["BASE_LEDGER_FACTS"] if override else release[0]) + + +@pytest.mark.parametrize("mismatch", [None, "sha", "size"]) +def test_overridden_feed_pin_still_verifies_digest_and_size( + tmp_path: Path, mismatch: str | None +) -> None: + feed = tmp_path / "new-feed.jsonl" + payload = b'{"fixture": "new release feed"}\n' + feed.write_bytes(payload) + result = _input_settings_result( + tmp_path, + { + "FEED": str(feed), + "FEED_SHA": "0" * 64 + if mismatch == "sha" + else hashlib.sha256(payload).hexdigest(), + "FEED_BYTES": str(len(payload) + (mismatch == "size")), + }, + """ +# Exercise the driver's registered release row with real tiny bytes; no +# production input or git repository is read. Adapt its BSD stat call for CI. +registered_rows=$(input_rows) +input_rows() { printf '%s\\n' "$registered_rows" | awk -F '|' '$1 == "release_ledger_facts"'; } +stat() { "$PYTHON" -c 'import os,sys; print(os.stat(sys.argv[1]).st_size)' "$3"; } +verify_inputs +printf '%s\\n' "${#INPUT_PROBLEMS[@]}" +""", + ) + assert result.returncode == 0, result.stderr.decode() + assert result.stdout.decode().strip() == ("0" if mismatch is None else "1") + report = json.loads((tmp_path / "run_root" / "inputs.resolved.json").read_text()) + assert len(report["inputs"]) == 1 + assert report["inputs"][0]["status"] == ( + "ok" if mismatch is None else f"{mismatch}_mismatch" + ) + + def test_shell_scripts_are_valid_bash() -> None: scripts = _shell_sources() assert scripts diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py b/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py index acb05a9120..23690d4d4e 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py @@ -671,6 +671,22 @@ def test_loose_base_plans_are_refused(mutate, message: str) -> None: plan_lib.parse_plan(data) +def test_rerun_template_refuses_validation_until_the_commit_is_selected(capsys) -> None: + path = ROOT / "docs" / "us-modal-stage-route-a-rerun-base-plan.json" + data = json.loads(path.read_text()) + assert data["source"]["commit"] == "PICK_MAIN_COMMIT_ON_2026_10_18" + assert plan_lib.main(["validate", str(path)]) == 2 + assert ( + "source.commit must be a full 40-hex lowercase sha" in capsys.readouterr().err + ) + data["source"]["commit"] = _committed_plan().commit + plan = plan_lib.parse_plan(data) + assert plan.tool is plan_lib.US_PUF_SUPPORT_BASE_RERUN + assert plan.inputs["block_ladder_npz"].sha256 == ( + "6840b990acdfa2003d7723da5595e3cfff7205a4fa3d8d6206ed835c85c3f233" + ) + + def test_the_budget_ceiling_is_the_timeout_less_the_base_s_overhead() -> None: ceiling = plan_lib.BASE.timeout_s - plan_lib.BASE_RUNNER_OVERHEAD_SECONDS assert ceiling == 14_700 @@ -1189,6 +1205,61 @@ def no_network(*_args, **_kwargs): # --------------------------------------------------------------------------- # +def test_rerun_registration_preserves_the_historical_base_contract() -> None: + rerun = plan_lib.US_PUF_SUPPORT_BASE_RERUN + assert plan_lib.TOOLS[rerun.name] is rerun + assert ( + dataclasses.replace(rerun, name=BASE.name, tree_files=BASE.tree_files) == BASE + ) + assert BASE.tree_files["cd_vintage_crosswalk"] is plan_lib.CD_VINTAGE_CROSSWALK + assert plan_lib.CD_VINTAGE_CROSSWALK.sha256 == ( + "c7cb040b1f57ca2ea2adcbfe60cc2b250ca23acbc4b640cd421e766fa54c1aec" + ) + historical = _committed_plan() + data = base_plan_data(tool=rerun.name) + plan = plan_lib.parse_plan(data) + assert plan_lib.planned_argv(plan) == plan_lib.planned_argv(historical) + assert plan.tool.stages["all"].resources is plan_lib.BASE + assert plan.inputs == historical.inputs + assert plan.max_wall_seconds == historical.max_wall_seconds == 14_400 + assert plan_lib.estimated_usd_at_timeout(plan) == 14.90 + assert plan_lib.estimated_usd_at_timeout(plan) <= plan_lib.BASE_COST_CAP_USD + + +def test_rerun_crosswalk_pin_verifies_the_current_tree() -> None: + (row,) = plan_lib.tree_file_rows(plan_lib.US_PUF_SUPPORT_BASE_RERUN, ROOT) + assert "problem" not in row + assert row["sha256"] == ( + "a347303fff3fea145f758488c43cb94355df5a8cb5e512552b2ee3791d2aa233" + ) + assert row["bytes"] == 78_202 + + +def test_rerun_crosswalk_refuses_the_historical_tree( + pinned_source: str, tmp_path: Path +) -> None: + crosswalk = plan_lib.CD_VINTAGE_CROSSWALK_RERUN + historical = subprocess.run( + [ + "git", + "-C", + str(ROOT), + "show", + f"{_committed_plan().commit}:{crosswalk.path}", + ], + capture_output=True, + check=True, + ).stdout + assert hashlib.sha256(historical).hexdigest() == ( + plan_lib.CD_VINTAGE_CROSSWALK.sha256 + ) + path = tmp_path / crosswalk.path + path.parent.mkdir(parents=True) + path.write_bytes(historical) + (row,) = plan_lib.tree_file_rows(plan_lib.US_PUF_SUPPORT_BASE_RERUN, tmp_path) + assert "another version" in row["problem"] + + def test_tree_file_rows_verify_the_clone(tmp_path: Path) -> None: rel = "packages/x/data/crosswalk.csv" good = b"source,target,weight\n" @@ -2428,8 +2499,8 @@ def test_the_hand_off_never_overwrites_a_base(tmp_path: Path) -> None: def test_the_runbook_names_every_registered_tool() -> None: text = RUNBOOK.read_text() intro = text[: text.index("Two files do the work")] - assert len(plan_lib.TOOLS) == 4 - assert "Four tools are registered" in intro + assert len(plan_lib.TOOLS) == 5 + assert "Five tools are registered" in intro for name, tool in plan_lib.TOOLS.items(): assert f"`{name}`" in intro, name if tool.script is not None: @@ -2443,6 +2514,7 @@ def test_the_runbook_names_every_registered_tool() -> None: "test_the_owned_flags_are_the_pinned_tool_s_flags", "test_the_registered_crosswalk_digest_is_4b57d15a2_s", "test_the_pinned_tool_locks_the_reference_s_run_config_from_the_modal_argv", + "test_rerun_crosswalk_refuses_the_historical_tree", ) @@ -2463,4 +2535,4 @@ def test_the_runbook_names_every_pinned_commit_test() -> None: text = RUNBOOK.read_text() for name in PINNED_COMMIT_TESTS: assert f"`{name}`" in text, name - assert "four tests in\n `test_us_modal_stage_puf_support_base.py`" in text + assert "five tests in\n `test_us_modal_stage_puf_support_base.py`" in text diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_route_a_local_reference.py b/packages/microcosm-build/tests/engine_free/us/test_us_route_a_local_reference.py new file mode 100644 index 0000000000..f5a99981d3 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_us_route_a_local_reference.py @@ -0,0 +1,266 @@ +"""Bind Route A's local comparison reference to actual files and chosen code.""" + +from __future__ import annotations + +import dataclasses +import hashlib +import importlib.util +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +from test_support.git_repositories import NO_BACKGROUND_MAINTENANCE +from test_support.microcosm_build.us_modal_stage_plan_tool import ( + ROOT, + base_plan_data, + plan_lib, +) + + +@pytest.fixture +def helper(monkeypatch): + monkeypatch.setattr("sys.path", list(__import__("sys").path)) + spec = importlib.util.spec_from_file_location( + "route_a_make_local_reference", ROOT / "tools/route_a/make_local_reference.py" + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +@pytest.fixture(params=["us-puf-support-base", "us-puf-support-base-rerun"]) +def evidence(tmp_path, monkeypatch, helper, request): + # Real temporary input/checkpoint bytes; only the archive home-cache + # registry pin is disabled so tiny synthetic source files can be used. + monkeypatch.setitem( + helper.stage_plan.TOOLS, + request.param, + dataclasses.replace(helper.stage_plan.TOOLS[request.param], home_seeds=()), + ) + monkeypatch.setattr(helper, "_committed_source_sha256", lambda *_args: "a" * 64) + data = base_plan_data() + data["tool"] = request.param + config = json.loads( + (ROOT / "docs/us-modal-stage-route-a-base-local-reference.json").read_text() + )["run_config"] + config["builder_code_identity"]["source_sha256"] = "a" * 64 + for role, ref in data["inputs"].items(): + filename = ref["uri"].rsplit("/", 1)[1] + source = tmp_path / "inputs" / role / filename + source.parent.mkdir(parents=True) + source.write_bytes(f"synthetic {role}".encode()) + digest = hashlib.sha256(source.read_bytes()).hexdigest() + ref.update(uri=f"volume://cas/sha256/{digest}/{filename}", sha256=digest) + if role.startswith("asec_education_"): + config["asec_education_source"][role.split("_")[2]] = str(source) + elif role.startswith("asec_"): + year = role.split("_")[1] + config["asec_h5_sha256"][year] = digest + config["asec_h5"] = [ + f"{year}={source}" if item.startswith(year + "=") else item + for item in config["asec_h5"] + ] + else: + key = { + "acs_2022_h5": "acs_h5", + "puf_2024_h5": "puf_h5", + "puf_2015_csv": "puf_source_year_csv", + "base_ledger_facts": "ledger_facts", + "block_ladder_npz": "block_ladder_artifact", + }[role] + config[key] = str(source) + if role in {"puf_2024_h5", "puf_2015_csv"}: + config[key + "_sha256"] = digest + crosswalk = tmp_path / "congressional_district_vintage_crosswalk.csv" + crosswalk.write_text("synthetic crosswalk\n") + config["congressional_district_vintage_crosswalk"] = str(crosswalk) + selected_tool = helper.stage_plan.TOOLS[request.param] + monkeypatch.setitem( + helper.stage_plan.TOOLS, + request.param, + dataclasses.replace( + selected_tool, + tree_files={ + "cd_vintage_crosswalk": dataclasses.replace( + selected_tool.tree_files["cd_vintage_crosswalk"], + sha256=hashlib.sha256(crosswalk.read_bytes()).hexdigest(), + ) + }, + ), + ) + config["out"] = str(tmp_path / "base-out") + checkpoints = tmp_path / "base-checkpoints" + checkpoints.mkdir() + names = ["source_construction", "pre_clone_enrichment"] + records = [] + for index, name in enumerate(names): + filename = f"{index:03d}_{name}.frame.h5" + path = checkpoints / filename + path.write_bytes(f"synthetic {name} checkpoint".encode()) + records.append( + { + "stage": name, + "checkpoint_filename": filename, + "checkpoint_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + ) + records[0]["metadata"] = { + "base_source": { + "sources": [ + {"year": year, "sha256": data["inputs"][f"asec_{year}_h5"]["sha256"]} + for year in helper.stage_plan.PUF_SUPPORT_BASE_ASEC_YEARS + ] + } + } + records[1]["metadata"] = {"acs_sha256": data["inputs"]["acs_2022_h5"]["sha256"]} + context = { + "artifact_kind": "populace_outer_stage_run_context", + "completed": names, + "pipeline": { + "stages": [ + {"name": name, "description": name + " → frame"} for name in names + ] + }, + "run_config": config, + "stage_records": records, + } + context["pipeline_sha256"] = hashlib.sha256( + json.dumps( + context["pipeline"], + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode() + ).hexdigest() + context_path = checkpoints / "stage_run_context.json" + context_path.write_text(json.dumps(context)) + plan_path = tmp_path / "plan.json" + plan_path.write_text(json.dumps(data)) + return plan_path, checkpoints, context, context_path + + +def test_reference_uses_actual_context_and_checkpoints(helper, evidence, tmp_path): + plan, checkpoints, context, _context_path = evidence + reference = helper.make_reference(plan, checkpoints, tmp_path) + assert reference["commit"] == json.loads(plan.read_text())["source"]["commit"] + assert reference["run_config"] == plan_lib.mask_config_paths(context["run_config"]) + assert reference["pipeline"] == context["pipeline"] + assert reference["outputs"] == { + f"base-checkpoints/{record['checkpoint_filename']}": record["checkpoint_sha256"] + for record in context["stage_records"] + } + assert reference["argv"][0] == "" + assert reference["argv"][reference["argv"].index("--stage") + 1] == "all" + assert reference["stages_compared"] == context["completed"] + + +@pytest.mark.parametrize( + ("change", "message"), + [ + ("checkpoint", "differs from its context record"), + ("ladder", "block_ladder_npz sha256 differs"), + ("puf_pin", "puf_h5_sha256 recorded pin differs"), + ("source", "source identity differs"), + ("hash_seed", "PYTHONHASHSEED must be 0"), + ("extra_stage", "exactly completed stages 000 and 001"), + ("seed", "run_config.seed differs"), + ("pipeline", "pipeline digest differs"), + ("environment", "plan non-thread environment"), + ("acs_record", "ACS enrichment-stage digest differs"), + ], +) +def test_refuses_evidence_mismatches(helper, evidence, tmp_path, change, message): + plan, checkpoints, context, context_path = evidence + if change == "checkpoint": + (checkpoints / "001_pre_clone_enrichment.frame.h5").write_bytes(b"changed") + elif change == "ladder": + Path(context["run_config"]["block_ladder_artifact"]).write_bytes(b"old ladder") + elif change == "puf_pin": + context["run_config"]["puf_h5_sha256"] = "f" * 64 + elif change == "source": + context["run_config"]["builder_code_identity"]["source_sha256"] = "f" * 64 + elif change == "hash_seed": + context["run_config"]["thread_environment"]["PYTHONHASHSEED"] = "12345" + elif change == "extra_stage": + context["completed"].append("clone_feature_extraction") + elif change == "pipeline": + context["pipeline_sha256"] = "f" * 64 + elif change == "environment": + plan_data = json.loads(plan.read_text()) + plan_data["env"]["MICROCOSM_UNRECORDED_SETTING"] = "1" + plan.write_text(json.dumps(plan_data)) + elif change == "acs_record": + context["stage_records"][1]["metadata"]["acs_sha256"] = "f" * 64 + else: + context["run_config"]["seed"] = 1 + context_path.write_text(json.dumps(context)) + with pytest.raises(ValueError, match=message): + helper.make_reference(plan, checkpoints, tmp_path) + + +def test_cli_refuses_overwrite_and_writes_no_acceptance(helper, evidence, tmp_path): + plan, checkpoints, _context, _context_path = evidence + output = tmp_path / "reference.json" + args = [ + "--plan", + str(plan), + "--checkpoint-root", + str(checkpoints), + "--out", + str(output), + ] + assert helper.main(args) == 0 + original = output.read_bytes() + assert helper.main(args) == 2 + assert output.read_bytes() == original + assert not (tmp_path / "ACCEPTED").exists() + + +def test_committed_source_digest_ignores_dirty_checkout(helper, tmp_path): + repo = tmp_path / "repo" + repo.mkdir() + identity = Path("packages/microcosm-build/src/microcosm/build/code_identity.py") + (repo / identity).parent.mkdir(parents=True) + shutil.copyfile(ROOT / identity, repo / identity) + (repo / "tools").mkdir() + (repo / "tools/build_us_puf_support_base.py").write_text("# synthetic builder\n") + (repo / "pyproject.toml").write_text("# synthetic project\n") + (repo / "uv.lock").write_text("# synthetic lock\n") + + def git(*args): + return subprocess.check_output( + ["git", *NO_BACKGROUND_MAINTENANCE, "-C", str(repo), *args], text=True + ).strip() + + git("init", "--quiet") + git("add", ".") + git( + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.com", + "commit", + "--quiet", + "-m", + "fixture", + ) + commit = git("rev-parse", "HEAD") + expected = helper._committed_source_sha256(repo, commit) + (repo / "tools/build_us_puf_support_base.py").write_text("# changed working copy\n") + assert helper._committed_source_sha256(repo, commit) == expected + git("add", ".") + git( + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.com", + "commit", + "--quiet", + "-m", + "changed", + ) + assert helper._committed_source_sha256(repo, git("rev-parse", "HEAD")) != expected diff --git a/tools/modal_us_stage_plan.py b/tools/modal_us_stage_plan.py index 06217d613b..0cbace7fff 100644 --- a/tools/modal_us_stage_plan.py +++ b/tools/modal_us_stage_plan.py @@ -42,7 +42,7 @@ import subprocess import sys from collections.abc import Callable, Iterable, Mapping, Sequence -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from itertools import zip_longest from pathlib import Path, PurePosixPath @@ -888,6 +888,19 @@ def _puf_support_base_argv( stage_hash_seed_default=HASH_SEED, ) +# Keep the historical registration and its local comparison reference pinned to +# 4b57d15a2. The 2026-10-18 rerun uses main's reviewed NC crosswalk correction +# (d8d96d926), with the same base command, inputs and resource budget. +CD_VINTAGE_CROSSWALK_RERUN = TreeFile( + path=CD_VINTAGE_CROSSWALK.path, + sha256="a347303fff3fea145f758488c43cb94355df5a8cb5e512552b2ee3791d2aa233", +) +US_PUF_SUPPORT_BASE_RERUN = replace( + US_PUF_SUPPORT_BASE, + name="us-puf-support-base-rerun", + tree_files={"cd_vintage_crosswalk": CD_VINTAGE_CROSSWALK_RERUN}, +) + # --- The retained ACS local-release staging builder ----------------------- # The shim delegates to tools/_legacy/build_us_acs_multispine_base.py. It # accepts no --stage: "all" is the runner's single stage, not a builder flag. @@ -1005,6 +1018,7 @@ def _acs_multispine_staging_argv( US_ACS_LOCAL_RELEASE, RUNNER_SMOKE, US_PUF_SUPPORT_BASE, + US_PUF_SUPPORT_BASE_RERUN, US_ACS_MULTISPINE_STAGING, ) } diff --git a/tools/route_a/README.md b/tools/route_a/README.md index 4d66455414..86eb1733d7 100644 --- a/tools/route_a/README.md +++ b/tools/route_a/README.md @@ -224,3 +224,13 @@ Telemetry is written locally under and supervisor resource series do not emit these staging records. If uploads fail, dashboard visibility is not guaranteed. A staging upload does not publish the release or update production `latest.json`. + +## 18 October rerun (d1386) + +Use [the prepared rerun sequence](../../docs/us-route-a-rerun-20261018.md) +and [its settings template](../../docs/us-route-a-rerun-20261018.env.example). +For a newer committed feed/crosswalk, set `FEED_SHA`, `FEED_BYTES` and +`CROSSWALK_SHA` together with the corresponding paths after checking the chosen +commit. Historical defaults remain unchanged. `BASE_LEDGER_FACTS` and its SHA +can intentionally retain the old base-only feed while the release uses main's +new pin. An actual machine-local `route_a.env` must stay outside Git. diff --git a/tools/route_a/make_local_reference.py b/tools/route_a/make_local_reference.py new file mode 100644 index 0000000000..12411df23a --- /dev/null +++ b/tools/route_a/make_local_reference.py @@ -0,0 +1,293 @@ +"""Record existing local stage 000/001 evidence; never launch a build. + +The checkpoint context supplies the actual config, pipeline and recorded frame +digests. Every configured input is hashed again against the filled Modal plan. +The recorded code identity must equal the chosen commit's committed sources. +This creates comparison evidence, never an exception or an ACCEPTED marker. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import shutil +import subprocess +import sys +import tarfile +import tempfile +from pathlib import Path, PurePosixPath + +TOOLS = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(TOOLS)) +import modal_us_stage_plan as stage_plan # noqa: E402 + + +def _committed_source_sha256(repo_root: Path, commit: str) -> str: + """Use the shared identity function on sources extracted from git objects.""" + + with tempfile.TemporaryDirectory(prefix="route-a-reference-") as temporary: + root = Path(temporary) + with subprocess.Popen( + [ + "git", + "-C", + str(repo_root), + "archive", + commit, + "tools/build_us_puf_support_base.py", + "pyproject.toml", + "uv.lock", + "packages", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) as process: + assert process.stdout is not None + with tarfile.open(fileobj=process.stdout, mode="r|") as archive: + for member in archive: + path = PurePosixPath(member.name) + packaged = ( + len(path.parts) >= 4 + and path.parts[0] == "packages" + and path.parts[2] == "src" + and path.suffix in {".json", ".py", ".toml", ".yaml", ".yml"} + ) + if not packaged and member.name not in { + "tools/build_us_puf_support_base.py", + "pyproject.toml", + "uv.lock", + }: + continue + if not member.isfile() or path.is_absolute() or ".." in path.parts: + raise ValueError(f"unsupported committed source: {member.name}") + stream = archive.extractfile(member) + assert stream is not None + target = root / member.name + target.parent.mkdir(parents=True, exist_ok=True) + with target.open("wb") as output: + shutil.copyfileobj(stream, output) + _stdout, stderr = process.communicate() + if process.returncode: + raise ValueError( + f"cannot read chosen commit: {stderr.decode().strip()}" + ) + identity_path = ( + root / "packages/microcosm-build/src/microcosm/build/code_identity.py" + ) + spec = importlib.util.spec_from_file_location( + "route_a_code_identity", identity_path + ) + if spec is None or spec.loader is None: + raise ValueError("chosen commit lacks the shared code-identity helper") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.builder_code_identity( + root, + tool_path=root / "tools/build_us_puf_support_base.py", + distributions=(), + )["source_sha256"] + + +def _input_paths(config: dict) -> dict[str, Path]: + paths = { + f"asec_{year}_h5": Path(filename) + for item in config["asec_h5"] + for year, filename in [item.split("=", 1)] + } + paths.update( + { + f"asec_education_{year}_zip": Path(filename) + for year, filename in config["asec_education_source"].items() + } + ) + for role, key in { + "puf_2024_h5": "puf_h5", + "puf_2015_csv": "puf_source_year_csv", + "acs_2022_h5": "acs_h5", + "base_ledger_facts": "ledger_facts", + "block_ladder_npz": "block_ladder_artifact", + }.items(): + paths[role] = Path(config[key]) + return paths + + +def make_reference(plan_path: Path, checkpoint_root: Path, repo_root: Path) -> dict: + """Refuse mismatched evidence and return a reference for the actual files.""" + + raw_plan, plan = stage_plan.load_plan(plan_path) + if ( + plan.tool.name not in {"us-puf-support-base", "us-puf-support-base-rerun"} + or plan.stage != "all" + ): + raise ValueError("a Route A PUF-support base all-stage plan is required") + context_path = checkpoint_root / "stage_run_context.json" + context = json.loads(context_path.read_text()) + names = ["source_construction", "pre_clone_enrichment"] + if context.get("completed") != names: + raise ValueError("local context must have exactly completed stages 000 and 001") + if context.get("artifact_kind") != "populace_outer_stage_run_context": + raise ValueError("not an outer-stage run context") + config = context["run_config"] + if config["builder_code_identity"]["source_sha256"] != _committed_source_sha256( + repo_root, plan.commit + ): + raise ValueError("local source identity differs from the chosen commit") + if config["thread_environment"].get("PYTHONHASHSEED") != "0": + raise ValueError("local PYTHONHASHSEED must be 0") + nonthread_env = { + name: value + for name, value in raw_plan.get("env", {}).items() + if name not in config["thread_environment"] + } + if nonthread_env != {"PYTHONUNBUFFERED": "1"}: + raise ValueError("plan non-thread environment must be only PYTHONUNBUFFERED=1") + + # These are the registry's fixed command settings and the builder defaults + # for flags it withholds. A later config key is retained for compare-lineage. + settings = { + "target_year": stage_plan.PUF_SUPPORT_BASE_TARGET_YEAR, + "seed": stage_plan.PUF_SUPPORT_BASE_SEED, + "n_estimators": stage_plan.PUF_SUPPORT_BASE_N_ESTIMATORS, + "congressional_district_seed": stage_plan.PUF_SUPPORT_BASE_CD_SEED, + "assign_congressional_districts": True, + "geography_ladder_seed": 0, + "allow_geography_ladder_gate_failures": False, + "without_block_ladder": False, + "equivalence_deterministic_h5_metadata": False, + "base_h5": None, + "asec_max_households": None, + "support_spine_spec": None, + "asec_2023_weeks_unemployed_source": None, + } + for key, value in settings.items(): + if key not in config or config[key] != value: + raise ValueError( + f"local run_config.{key} differs from the registered command" + ) + if Path(config["out"]).name != stage_plan.PUF_SUPPORT_BASE_OUT: + raise ValueError("local --out must have the planned base-out basename") + crosswalk = Path(config["congressional_district_vintage_crosswalk"]) + tree_pin = plan.tool.tree_files["cd_vintage_crosswalk"] + if crosswalk.name != Path(tree_pin.path).name: + raise ValueError("local crosswalk filename differs from the planned command") + if stage_plan.sha256_file(crosswalk)[0] != tree_pin.sha256: + raise ValueError("local crosswalk differs from the registered tree pin") + + paths = _input_paths(config) + if set(paths) != set(plan.inputs): + raise ValueError("local input roles differ from the plan") + inputs = {} + for role, ref in plan.inputs.items(): + filename = Path(stage_plan.input_local_path(stage_plan.WORK_ROOT, ref)).name + if paths[role].name != filename: + raise ValueError(f"local {role} filename differs from the planned command") + digest, _size = stage_plan.sha256_file(paths[role]) + if digest != ref.sha256: + raise ValueError(f"local {role} sha256 differs from the plan") + inputs[role] = digest + for year in stage_plan.PUF_SUPPORT_BASE_ASEC_YEARS: + if config["asec_h5_sha256"].get(str(year)) != inputs[f"asec_{year}_h5"]: + raise ValueError(f"local ASEC {year} recorded pin differs from its bytes") + for key, role in { + "puf_h5_sha256": "puf_2024_h5", + "puf_source_year_csv_sha256": "puf_2015_csv", + }.items(): + if config.get(key) != inputs[role]: + raise ValueError(f"local {key} recorded pin differs from its bytes") + if [item.split("=", 1)[0] for item in config["asec_h5"]] != [ + str(year) for year in stage_plan.PUF_SUPPORT_BASE_ASEC_YEARS + ]: + raise ValueError("local ASEC order differs from the registered command") + + outputs = {} + records = context["stage_records"] + if [record["stage"] for record in records] != names: + raise ValueError("local stage records do not match completed stages 000/001") + source_digests = { + f"asec_{source['year']}_h5": source["sha256"] + for source in records[0]["metadata"]["base_source"]["sources"] + } + for role, digest in source_digests.items(): + if inputs.get(role) != digest: + raise ValueError(f"local {role} source-stage digest differs from its bytes") + if set(source_digests) != { + f"asec_{year}_h5" for year in stage_plan.PUF_SUPPORT_BASE_ASEC_YEARS + }: + raise ValueError("local source-stage ASEC roles differ from the plan") + if records[1]["metadata"]["acs_sha256"] != inputs["acs_2022_h5"]: + raise ValueError("local ACS enrichment-stage digest differs from its bytes") + for index, (name, record) in enumerate(zip(names, records, strict=True)): + filename = f"{index:03d}_{name}.frame.h5" + digest, _size = stage_plan.sha256_file(checkpoint_root / filename) + if ( + record["checkpoint_filename"] != filename + or record["checkpoint_sha256"] != digest + ): + raise ValueError(f"local {filename} differs from its context record") + outputs[f"base-checkpoints/{filename}"] = digest + pipeline = context["pipeline"] + if [item["name"] for item in pipeline["stages"]][:2] != names: + raise ValueError("local pipeline does not start with stages 000/001") + # Replay StagePipeline.sha256's public JSON wire representation without + # importing its numpy/pandas runtime (outer_stage_runtime.py). + pipeline_sha256 = hashlib.sha256( + json.dumps( + pipeline, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + ).hexdigest() + if context["pipeline_sha256"] != pipeline_sha256: + raise ValueError("local pipeline digest differs from its context record") + return { + "schema": stage_plan.LOCAL_REFERENCE_SCHEMA, + "description": ( + "Generated from existing local completed stages 000/001, their recorded " + "config and actual frame/input bytes. argv is the registered all-stage " + "command whose settings these named stages used. No build was launched; " + "stages 002 onward have no local comparison here." + ), + "tool": plan.tool.name, + "stage": plan.stage, + "commit": plan.commit, + "argv": stage_plan.tokenize_stage_argv(stage_plan.planned_argv(plan)), + "env": raw_plan.get("env", {}), + "inputs": inputs, + "outputs": outputs, + "stages_compared": names, + "run_context_path": "base-checkpoints/stage_run_context.json", + "pipeline": pipeline, + "pipeline_sha256": pipeline_sha256, + "run_config": stage_plan.mask_config_paths(config), + "local_evidence": { + "context_sha256": stage_plan.sha256_file(context_path)[0], + "note": "Input bytes were rehashed at reference creation; retain the local context and checkpoints.", + }, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--plan", type=Path, required=True) + parser.add_argument("--checkpoint-root", type=Path, required=True) + parser.add_argument("--repo-root", type=Path, default=TOOLS.parent) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args(argv) + try: + if args.out.exists(): + raise FileExistsError(f"output already exists: {args.out}") + reference = make_reference(args.plan, args.checkpoint_root, args.repo_root) + with args.out.open("x") as output: + output.write(json.dumps(reference, indent=2) + "\n") + except (ValueError, OSError, KeyError, TypeError, tarfile.TarError) as error: + print(f"REFUSED: {error}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/route_a/route_a.env.example b/tools/route_a/route_a.env.example index e9b3250457..89788a6078 100644 --- a/tools/route_a/route_a.env.example +++ b/tools/route_a/route_a.env.example @@ -16,6 +16,9 @@ : "${STORAGE:=$PE/policyengine-us-data/policyengine_us_data/storage}" : "${EDU:=$PE/_buildm-runtime/inputs/asec_education}" : "${FEED:=$PE/_buildh-runtime/inputs/consumer_facts_us_c5e5bf8.jsonl}" +# The driver preserves the published feed/crosswalk pins by default. If the +# chosen commit changed them, set FEED_SHA, FEED_BYTES and CROSSWALK_SHA here +# alongside FEED, after checking that commit's packaged pins. # d713: CT-fixed ladder; route_a.sh verifies its digest and size. : "${LADDER:=$PE/_build_artifacts/us-ct-cbsa/us_block_ladder_2020.npz}" : "${SSI:=$PE/_buildo-runtime/inputs/attempt6_basis_schema3_seed.json}" diff --git a/tools/route_a/route_a.sh b/tools/route_a/route_a.sh index 79dc61c704..b0eeec7fb7 100755 --- a/tools/route_a/route_a.sh +++ b/tools/route_a/route_a.sh @@ -50,13 +50,17 @@ LOG=$RUN_ROOT/route_a.log # Inputs, resolved and hashed on this machine on 2026-09-23 (see status.md). # Format: role|path|sha256|bytes|where main pins it # --------------------------------------------------------------------------- -FEED_SHA=b85437390021777e746f507c5890305496baf5fc7f2c78ba08ddb090f4839801 +# Preserve the published defaults; a rerun's settings must pin the feed and +# crosswalk at its chosen commit when those inputs have changed on main. +: "${FEED_SHA:=b85437390021777e746f507c5890305496baf5fc7f2c78ba08ddb090f4839801}" +: "${FEED_BYTES:=164603204}" # The base stage reads the feed only for the SOI congressional-district return # counts. On 2026-09-23 the pinned feed and the 2026-09-16 base feed # (consumer_facts_builde_aging_v5.jsonl, a5d34d4a...) gave the same 436-row # distribution through main's congressional_district_distribution_from_ledger_facts; # after the vintage crosswalk, 73 weights differ by at most 2.2e-16 relative. -# The pinned feed is used for both stages so the lineage has one feed identity. +# Both stages default to this feed. Explicit BASE_LEDGER_FACTS and its SHA can +# retain a different base feed while the release follows its committed pin. BASE_LEDGER_FACTS=${BASE_LEDGER_FACTS:-$FEED} BASE_LEDGER_FACTS_SHA=${BASE_LEDGER_FACTS_SHA:-$FEED_SHA} # d713 (Max 2026-10-03): the next certified build uses the Connecticut-fixed ladder from microcosm#1072 @@ -71,7 +75,7 @@ ASEC_2024_SHA=ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d ASEC_2023_SHA=cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88 ASEC_2022_SHA=7ccca976284bb47815d84460cc4f75a0a65d26d7754ab0a0f417de351b3d474e CROSSWALK_REL=packages/microcosm-build/src/microcosm/build/us_runtime/data/congressional_district_vintage_crosswalk.csv -CROSSWALK_SHA=c7cb040b1f57ca2ea2adcbfe60cc2b250ca23acbc4b640cd421e766fa54c1aec +: "${CROSSWALK_SHA:=c7cb040b1f57ca2ea2adcbfe60cc2b250ca23acbc4b640cd421e766fa54c1aec}" FEED_PIN_REL=packages/microcosm-build/src/microcosm/build/us/chronicle_feed.json ASEC_PIN_REL=packages/microcosm-build/src/microcosm/build/us_runtime/asec_sources.py SPM_ROLE_REL=packages/microcosm-build/src/microcosm/build/us_runtime/spm_independence_role.py @@ -96,7 +100,7 @@ asec_education_2023_zip|$EDU/asecpub24csv.zip|cdb39cdac34bef99dd0940ab28e306f692 asec_education_2024_zip|$EDU/asecpub25csv.zip|318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b|147271429|us_runtime/education_assistance_source.py base_ledger_facts|$BASE_LEDGER_FACTS|$BASE_LEDGER_FACTS_SHA|-|us/chronicle_feed.json when it is the pinned feed block_ladder_npz|$LADDER|6840b990acdfa2003d7723da5595e3cfff7205a4fa3d8d6206ed835c85c3f233|18991218|d713: Connecticut-fixed ladder rebuilt at microcosm#1072 (af98853e); _build_artifacts/us-ct-cbsa/verify_vs_route_a.json -release_ledger_facts|$FEED|$FEED_SHA|164603204|us/chronicle_feed.json facts_sha256 (Chronicle c5e5bf8, bare feed, 39,158 rows) +release_ledger_facts|$FEED|$FEED_SHA|$FEED_BYTES|us/chronicle_feed.json facts_sha256 ${TAIL:+qrf_tail_exclusions|$TAIL|${QRF_TAIL_EXCLUSIONS_SHA:-9bd497dc0d03dd793b53a228979e99408e62fdd2430bdd0d04df5fc7576e6229}|-|operator-supplied register; the release records its sha in diagnostics} ssi_take_up_prior_basis|$SSI|$SSI_SHA|4782|--ssi-take-up-prior-weight-basis-sha256 scf_summary_extract|$SCF|6b8dd2d935a76ed225ddebc80fb2db22a467f0c80d9a1acaa67b4584aa4bafd1|24904185|us/source_stages.json, us_runtime/scf_wealth.py From f053b3eaadc0e083ece14b460705455c3e8f9ad6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 19 Oct 2026 12:39:19 -0400 Subject: [PATCH 02/10] Record Max's d1711/d1712 rulings and supervise the launch-day checks - The QRF tail register: d490 covers this build (d1711), since the d177/d450 waivers expire at the next certified build. - Stage-001 lineage: a repeat of the 9/30 float-rounding class may be accepted (d1712); anything else still stops for Max. - Step 6 runs the guard fixture, guard sweep and gate dry run under supervise.py, with RSS caps, admission and the disk floor. - Step 4's limits are sized to stages 000/001 rather than the whole base. - PREP points at the real checkout; the stale sandbox hand-off text is gone. - The reference's argv and environment are registry-derived; say so. Co-Authored-By: Claude Opus 5.5 --- docs/us-route-a-rerun-20261018.env.example | 5 +- docs/us-route-a-rerun-20261018.md | 140 ++++++++++++++------- docs/us-route-a-rerun-prep-report.md | 43 ++----- 3 files changed, 104 insertions(+), 84 deletions(-) diff --git a/docs/us-route-a-rerun-20261018.env.example b/docs/us-route-a-rerun-20261018.env.example index ec623b98ac..bcabd8451b 100644 --- a/docs/us-route-a-rerun-20261018.env.example +++ b/docs/us-route-a-rerun-20261018.env.example @@ -59,8 +59,9 @@ # REVIEW: selected main must still carry the NC-fixed in-tree crosswalk. : "${CROSSWALK_SHA:=a347303fff3fea145f758488c43cb94355df5a8cb5e512552b2ee3791d2aa233}" # REVIEW: historical export-mass reference is inherited, not re-certified here. -# REVIEW: d490 must be re-measured on the new lineage. Update file and SHA together. -# Allowed without another ruling: threshold 0.75, same or fewer columns only. +# REVIEW: re-measure the register on the new lineage. Update file and SHA together. +# d1711 (Max 2026-10-19): d490 covers this build. Threshold 0.75, same or fewer +# columns, renewed to 2026-12-31 or the next certified build. A new column stops. # REVIEW: d122 dense/national_state, batch 2000, RAM/wall/wait limits are inherited. # REVIEW: PREFLIGHT_COMMIT defaults to COMMIT; keep both on the chosen main tree. # Explicit staging destination; no credential value is stored here. diff --git a/docs/us-route-a-rerun-20261018.md b/docs/us-route-a-rerun-20261018.md index 7361aa2683..c9477e79c7 100644 --- a/docs/us-route-a-rerun-20261018.md +++ b/docs/us-route-a-rerun-20261018.md @@ -1,25 +1,19 @@ # Route A rerun on 18 October 2026 (d1386) -Prepared on 15 October against cached `origin/main` -`412f916536b66577e4e0b42c707285085df7c302`. Fetching was denied by this -workspace's sandbox; these findings must be checked against the commit selected -on 18 October. Nothing in this preparation launched a build or called Modal. +Prepared on 15 October against `origin/main` +`412f916536b66577e4e0b42c707285085df7c302`. Nothing in this preparation +launched a build or called Modal. The 18 October start did not happen; Max +confirmed on 19 October that the build launches that day on current `main` +without #1275 (d1386, d1556). -Max's d1386 selects whatever has merged to `main` on 18 October. It must contain +Max's d1386 selects whatever has merged to `main` at launch. It must contain `9802825200929c445856d969f4231287c185d784` (#1186, the #1178 Social Security fix). The base runs on Modal; release, preflights and certification run locally through `tools/route_a/`. Staging is enabled. Publication remains separate. -The assigned checkout is an older detached commit and its shared Git metadata -is read-only. Preparation therefore used an ignored, extracted cached-main tree -inside the assigned workspace, with independent local Git metadata. The portable -patch/bundle is against that main snapshot; apply it to a current-main prep -worktree before running this sequence. No caller metadata was written. -The requested external settings directory was also outside the writable -sandbox. The prepared, uncommitted settings fallback is -`build/route-a-rerun-20261018/route_a.env`; the reproducible template is -`docs/us-route-a-rerun-20261018.env.example`. Install it at the requested -machine-local path in step 1. Never commit `route_a.env`. +The machine-local settings file is +`$PE/_recovered/scratch-backup/893/route-a-rerun-20261018/route_a.env`, from +`docs/us-route-a-rerun-20261018.env.example`. Never commit `route_a.env`. ## Inputs and registration @@ -87,10 +81,13 @@ base. These observations do not bound this rerun's time or cost. A new hand-off requires a COMPLETED, non-budget-stopped receipt, strict `base-out` verification, the actual Modal context matching its receipt digest, -and a reference with the chosen commit, all eleven input digests, normalized -all-stage argv/environment, pipeline, run config, source-code digest, -dependency versions, hash seed 0 and actual 000/001 checkpoint digests. Every -comparison must pass. Interpreter and other thread-variable differences are +and a reference with the chosen commit, all eleven input digests, pipeline, run +config, source-code digest, dependency versions, hash seed 0 and actual 000/001 +checkpoint digests. Every comparison must pass. The reference's argv and +environment are written from the plan's registry (`planned_argv(plan)`), not +read from the local run, so their comparison only confirms the registry; the +strict masked `run_config`, source and dependency identity, hash seed, +pipeline and checkpoint digests carry the evidence. Interpreter and other thread-variable differences are reported; only hash seed is compared among thread variables. Equality covers two stages, not the later 22, district/ladder assignment or final export. Release gates and certification still run. @@ -102,11 +99,14 @@ JSON and the supporting local evidence with this run. The old `base-sup/PROVENANCE.md` records a strict mismatch at stage 001: 313/432,523 `person.pre_subsidy_rent` values differed, maximum absolute -`8.7e-11`, relative `6.08e-15`. A prior session used a narrow exception. That -does not authorize this run. Any mismatch, skipping the new local reference, -or relaxing the hand-off needs **Max's explicit decision** with the new -evidence. There is no automatic waiver path in this preparation. Stop before -writing `ACCEPTED` while that decision is outstanding. +`8.7e-11`, relative `6.08e-15`. A prior session accepted it with a scripted check. +Max ruled on 19 October (d1712) that a repeat of exactly that class may be +accepted for this run: every entity table has the same shape and index, totals +are equal, and every differing value is within 1e-12 relative. The check that +establishes this is recorded with the run's provenance. Any other mismatch, +skipping the new local reference, or relaxing the hand-off still needs **Max's +explicit decision** with the new evidence. Stop before writing `ACCEPTED` while +that decision is outstanding. ## Settings to review on 18 October @@ -125,10 +125,13 @@ writing `ACCEPTED` while that decision is outstanding. for this lineage. Re-measure and update the file and SHA together. Its four columns are `bond_assets`, `estate_income`, `long_term_capital_gains_on_collectibles`, and - `w2_wages_from_qualified_business`. Under d490 a new measurement may retain - threshold **0.75** and the same or a subset of these columns without another - ruling. Drop stale entries. Adding a column or lowering the threshold needs - Max's ruling. Its old reasons and expiry do not attest a new export. The + `w2_wages_from_qualified_business`. The d177/d450 waivers behind them expire + at the next certified build, which is this one. Max ruled on 19 October + (d1711) that d490 covers this build: start with this register, and if the + measured columns over 0.75 are the same or fewer, apply them on the same + terms, renewed to expire 2026-12-31 or at the next certified build. Drop + stale entries. Adding a column or lowering the threshold still needs Max's + ruling. The old reasons and expiry do not attest a new export. The pre-solve dry run uses base weights; the final weighted-export gate still determines whether the register is current. - `DENSE_RELEASE_D122` and `RELEASE_EXTRA_ARGS`: retain the historical dense @@ -158,7 +161,7 @@ clean checkout of the chosen build commit. Nothing writes to the old run or set -euo pipefail PE=/Users/maxghenis/PolicyEngine MAIN=$PE/microcosm -PREP=$PE/_worktrees/route-a-rerun-prep +PREP=$PE/_worktrees/microcosm-route-a-rerun-prep LOCAL=$PE/_recovered/scratch-backup/893/route-a-rerun-20261018 SETTINGS=$LOCAL/route_a.env PLAN=$LOCAL/base-plan.json @@ -275,9 +278,10 @@ does not authorize an automatic relaunch. This step runs engines and must wait for machine admission. It was not run during preparation. Create the clean chosen-commit checkout and locked US env. -The script below supervises each named stage with the existing local base -limits: 100 GiB RSS cap, 80 GiB RAM/70 GiB disk admission, six-hour wall limit -and persistent 20 GiB disk floor. Admission refusal needs a later operator +The script below supervises each named stage with limits sized to these two +stages, not the whole base: a 32 GiB RSS cap (the interrupted whole attempt +peaked at 11.66 GiB), 24 GiB RAM and 30 GiB disk admission, a six-hour wall +limit and the persistent 20 GiB disk floor. Admission refusal needs a later operator retry; it does not silently run on the overloaded machine. It never invokes `--stage all` locally. @@ -325,8 +329,8 @@ config = { "env": {"PYTHONHASHSEED": "0", "PYTHONUNBUFFERED": "1"}, "authorization": "d1386: local 000/001 comparison evidence only", "limits": {"wall_seconds": 21600, "cpu_seconds": 64800, - "rss_bytes": 100*gib, "available_ram_admission_bytes": 80*gib, - "disk_admission_bytes": 70*gib, "disk_floor_bytes": 20*gib, + "rss_bytes": 32*gib, "available_ram_admission_bytes": 24*gib, + "disk_admission_bytes": 30*gib, "disk_floor_bytes": 20*gib, "disk_floor_persist_seconds": 180, "output_bytes": 2*gib, "log_bytes": 512*1024**2}, } @@ -419,30 +423,74 @@ day. The dry run's inherited d490 register is a candidate to grade, not proof that it applies to the new weighted export. Resolve any certain failure and review AT-RISK results before release. +Each check runs under `tools/route_a/supervise.py`, like step 4, so the RSS +cap, RAM and disk admission and the disk floor apply. The dry run's limits +follow the driver's preflight stage scaled up to a full base load; the sweep's +follow its measured 4.75 GiB peak. A supervisor refusal prints its reason and +returns 99. + ```bash { echo '#!/bin/bash' echo 'set -euo pipefail' - declare -p W LOCAL RUN SETTINGS + declare -p W PREP LOCAL RUN SETTINGS cat <<'BASH' . "$SETTINGS" -(cd "$W" && .venv/bin/python - "$LOCAL" <<'PY' +CHECKS=$LOCAL/release-checks +mkdir -p "$CHECKS" +# supervised NAME RSS_GIB RAM_GIB DISK_GIB WALL_S CPU_S -- ARGV... +# Prints the child's return code, or 99 when the supervisor refused or killed it. +supervised() { + local name=$1 rss=$2 ram=$3 disk=$4 wall=$5 cpu=$6 + shift 7 + python3 -c ' +import json, sys +output, worktree, rss, ram, disk, wall, cpu, *argv = sys.argv[1:] +gib = 1024 ** 3 +config = { + "cwd": worktree, + "argv": argv, + "env": {"PYTHONHASHSEED": "0", "PYTHONUNBUFFERED": "1"}, + "authorization": "d1386: release guard checks before the release", + "limits": {"wall_seconds": int(wall), "cpu_seconds": int(cpu), + "rss_bytes": int(rss) * gib, "available_ram_admission_bytes": int(ram) * gib, + "disk_admission_bytes": int(disk) * gib, "disk_floor_bytes": 20 * gib, + "disk_floor_persist_seconds": 180, + "output_bytes": 2 * gib, "log_bytes": 512 * 1024 ** 2}, +} +with open(output, "x") as stream: + json.dump(config, stream, indent=2) +' "$CHECKS/$name-config.json" "$W" "$rss" "$ram" "$disk" "$wall" "$cpu" "$@" + "$W/.venv/bin/python" "$PREP/tools/route_a/supervise.py" \ + "$CHECKS/$name-sup" "$CHECKS/$name-config.json" >&2 || true + python3 -c ' +import json, sys +result = json.load(open(sys.argv[1])) +if result.get("status") == "REFUSED_ADMISSION" or result.get("refusal"): + print("supervisor:", result.get("status"), result.get("refusal"), file=sys.stderr) + print(99) +else: + print(result["returncode"]) +' "$CHECKS/$name-sup/RESULT.json" +} +fixture_rc=$(supervised guard-fixture 8 6 30 3600 3600 -- \ + "$W/.venv/bin/python" -c ' import sys from pathlib import Path from tools.sweep_us_post_export_scoring import _load_builder -from test_support.microcosm_build.us_post_export_scoring import ( - _GUARD_HOUSEHOLDS, _write_engine_h5, -) +from test_support.microcosm_build.us_post_export_scoring import _GUARD_HOUSEHOLDS, _write_engine_h5 _write_engine_h5(_load_builder(), Path(sys.argv[1]) / "guard-fixture", _GUARD_HOUSEHOLDS) -PY -) +' "$LOCAL") +[ "$fixture_rc" = 0 ] || { echo "Guard fixture failed ($fixture_rc)"; exit 1; } SMALL_H5=$LOCAL/guard-fixture/populace_us_2024.h5 -(cd "$W" && .venv/bin/python tools/sweep_us_post_export_scoring.py \ +sweep_rc=$(supervised guard-sweep 16 12 30 21600 43200 -- \ + "$W/.venv/bin/python" tools/sweep_us_post_export_scoring.py \ --dataset-path "$SMALL_H5" --batch-size 3 --reforms-per-worker 4 \ --output "$LOCAL/guard-sweep.json") +[ "$sweep_rc" = 0 ] || { echo "Guard sweep failed ($sweep_rc)"; exit 1; } EXTRA=($RELEASE_EXTRA_ARGS) -gate_rc=0 -(cd "$W" && .venv/bin/python tools/dry_run_us_release_gates.py \ +gate_rc=$(supervised gate-dry-run 64 40 30 21600 43200 -- \ + "$W/.venv/bin/python" tools/dry_run_us_release_gates.py \ --json-out "$LOCAL/dry-run-gates.json" -- \ --base-h5 "$RUN/base-out/base_populace_us_2024_puf_support.h5" \ --ledger-facts "$FEED" --ledger-facts-sha256 "$FEED_SHA" \ @@ -451,11 +499,11 @@ gate_rc=0 --ssi-take-up-prior-weight-basis-sha256 25fe8af50a99d717f3408b2de7f0849d2307d4f05b1a7d55d2703999002fff0a \ --scf-summary-extract "$SCF" --out "$LOCAL/gate-dry-run" \ --release-id route-a-rerun-20261018-gate-check \ - --checkpoint-root "$LOCAL/gate-checkpoints" --seed 0 --no-staging "${EXTRA[@]}") || gate_rc=$? + --checkpoint-root "$LOCAL/gate-checkpoints" --seed 0 --no-staging "${EXTRA[@]}") case "$gate_rc" in 0) echo 'Release guards completed cleanly' ;; 2) echo 'AT-RISK: review dry-run-gates.json before starting release'; exit 2 ;; - *) echo 'Release guard dry run failed'; exit "$gate_rc" ;; + *) echo "Release guard dry run failed ($gate_rc)"; exit 1 ;; esac BASH } > "$LOCAL/release-checks.sh" diff --git a/docs/us-route-a-rerun-prep-report.md b/docs/us-route-a-rerun-prep-report.md index a275ed52f5..206d82a4b9 100644 --- a/docs/us-route-a-rerun-prep-report.md +++ b/docs/us-route-a-rerun-prep-report.md @@ -56,40 +56,11 @@ requires `ps`, denied by the sandbox. Those checks must run in the launch-day locked environment; the runbook's full test command includes them. This report does not claim certification or a fully green ordinary environment. -## Sandbox and hand-off +## Hand-off -Only the assigned workspace was written. Both `git fetch origin` and staging -files in the assigned checkout failed because Git metadata under the caller's -repository is read-only. The intended `_worktrees/` and machine-local settings -destinations are outside the writable sandbox too. - -An independent cached-main checkout with its own Git metadata was therefore -prepared at `build/route-a-rerun-prep-main/` inside this workspace. Its branch is -`route-a-rerun-prep`; its commit is exported in the workspace-root -`route-a-rerun-prep.bundle` and `route-a-rerun-prep.patch`. These are portable -changes against cached main, avoiding the assigned checkout's older parent. -The assigned outer checkout remains **uncommitted**. It also holds copies of -the changed files for inspection. Use the bundle/patch to integrate with main; -do not treat the old outer checkout as a runnable current-main tree. - -The actual uncommitted settings fallback is -`build/route-a-rerun-20261018/route_a.env`. The requested -`/Users/maxghenis/PolicyEngine/_recovered/scratch-backup/893/route-a-rerun-20261018/route_a.env` -was not written. Install it with step 1 of the launch sequence; no actual -`route_a.env` is committed in the bundle. - -Outside this sandbox, recover the committed prep worktree without pushing: - -```bash -PE=/Users/maxghenis/PolicyEngine -MAIN=$PE/microcosm -ASSIGNED=/Users/maxghenis/.subfleet/worktrees/20261015-052227-route-a-rerun-prep -git -C "$MAIN" fetch "$ASSIGNED/route-a-rerun-prep.bundle" refs/heads/route-a-rerun-prep -git -C "$MAIN" worktree add -b route-a-rerun-prep "$PE/_worktrees/route-a-rerun-prep" FETCH_HEAD -``` - -That prep tree contains reviewed runner/driver changes. On 18 October, fetch -main again and select the build commit independently as the launch sequence -requires. If applying to a newer prep branch, use the patch or cherry-pick the -bundle's commit and review any conflicts. No historical run or worktree was -modified. +The preparation lane's sandbox could not write Git metadata, so it handed the +change over as a bundle. The main session committed it as branch +`route-a-rerun-prep` (PR #1289), checked out at +`$PE/_worktrees/microcosm-route-a-rerun-prep`. The machine-local settings file +is `$PE/_recovered/scratch-backup/893/route-a-rerun-20261018/route_a.env` and +is never committed. No historical run or worktree was modified. From 54f6dbc34bb7f5e3be5708fee5ed71399a326914 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 19 Oct 2026 12:41:46 -0400 Subject: [PATCH 03/10] Let the launch-day guard checks run before the base and resume safely The guard fixture and sweep need no base, so GUARDS_ONLY=1 runs them while the Modal base builds, as its own job and script. The full run then reuses their recorded results and requires ACCEPTED before the gate dry run. A check with an existing supervisor directory is never rerun; a supervisor refusal reports 99. Co-Authored-By: Claude Opus 5.5 --- docs/us-route-a-rerun-20261018.md | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/docs/us-route-a-rerun-20261018.md b/docs/us-route-a-rerun-20261018.md index c9477e79c7..66a1da1f7e 100644 --- a/docs/us-route-a-rerun-20261018.md +++ b/docs/us-route-a-rerun-20261018.md @@ -427,22 +427,36 @@ Each check runs under `tools/route_a/supervise.py`, like step 4, so the RSS cap, RAM and disk admission and the disk floor apply. The dry run's limits follow the driver's preflight stage scaled up to a full base load; the sweep's follow its measured 4.75 GiB peak. A supervisor refusal prints its reason and -returns 99. +returns 99. The fixture and sweep don't need the base, so they can run while it +builds: generate and submit the script with `GUARDS_ONLY=1` (job +`route-a-rerun-20261018-guards`), then again with `GUARDS_ONLY=0` (job +`route-a-rerun-20261018-checks`) once `ACCEPTED` exists. The second run reuses the recorded guard results. ```bash +GUARDS_ONLY=${GUARDS_ONLY:-0} +# Separate files: bash reads a running script as it goes, so never rewrite one in use. +if [ "$GUARDS_ONLY" = 1 ]; then CHECK_NAME=guards; else CHECK_NAME=checks; fi { echo '#!/bin/bash' echo 'set -euo pipefail' - declare -p W PREP LOCAL RUN SETTINGS + declare -p W PREP LOCAL RUN SETTINGS GUARDS_ONLY cat <<'BASH' . "$SETTINGS" CHECKS=$LOCAL/release-checks mkdir -p "$CHECKS" # supervised NAME RSS_GIB RAM_GIB DISK_GIB WALL_S CPU_S -- ARGV... # Prints the child's return code, or 99 when the supervisor refused or killed it. +# A check that already has a supervisor directory is not rerun: its recorded +# result is reused (move a failed attempt's -config.json and -sup aside to retry). supervised() { local name=$1 rss=$2 ram=$3 disk=$4 wall=$5 cpu=$6 shift 7 + if [ -e "$CHECKS/$name-sup" ]; then + if [ ! -f "$CHECKS/$name-sup/RESULT.json" ]; then + echo "$name: supervisor directory without RESULT.json; still running or crashed" >&2 + echo 99; return + fi + else python3 -c ' import json, sys output, worktree, rss, ram, disk, wall, cpu, *argv = sys.argv[1:] @@ -463,6 +477,7 @@ with open(output, "x") as stream: ' "$CHECKS/$name-config.json" "$W" "$rss" "$ram" "$disk" "$wall" "$cpu" "$@" "$W/.venv/bin/python" "$PREP/tools/route_a/supervise.py" \ "$CHECKS/$name-sup" "$CHECKS/$name-config.json" >&2 || true + fi python3 -c ' import json, sys result = json.load(open(sys.argv[1])) @@ -488,6 +503,8 @@ sweep_rc=$(supervised guard-sweep 16 12 30 21600 43200 -- \ --dataset-path "$SMALL_H5" --batch-size 3 --reforms-per-worker 4 \ --output "$LOCAL/guard-sweep.json") [ "$sweep_rc" = 0 ] || { echo "Guard sweep failed ($sweep_rc)"; exit 1; } +if [ "$GUARDS_ONLY" = 1 ]; then echo 'Guard checks passed; gate dry run deferred'; exit 0; fi +test -f "$RUN/base-sup/ACCEPTED" EXTRA=($RELEASE_EXTRA_ARGS) gate_rc=$(supervised gate-dry-run 64 40 30 21600 43200 -- \ "$W/.venv/bin/python" tools/dry_run_us_release_gates.py \ @@ -506,9 +523,9 @@ case "$gate_rc" in *) echo "Release guard dry run failed ($gate_rc)"; exit 1 ;; esac BASH -} > "$LOCAL/release-checks.sh" -"$LSUBMIT" route-a-rerun-20261018-checks "$LOCAL/checks.launchd.log" "$PREP" \ - /bin/bash "$LOCAL/release-checks.sh" +} > "$LOCAL/release-$CHECK_NAME.sh" +"$LSUBMIT" "route-a-rerun-20261018-$CHECK_NAME" "$LOCAL/$CHECK_NAME.launchd.log" "$PREP" \ + /bin/bash "$LOCAL/release-$CHECK_NAME.sh" ``` Wait for the checks job to finish. Inspect its exit status and both reports; From 0779d42804e6689cbcdf0a12f421da86e4daa23d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 19 Oct 2026 13:01:10 -0400 Subject: [PATCH 04/10] Spawn Modal stages so a sleeping launcher can't cancel them The d1386 base's first attempt was cancelled after 822 s, 99 s after the launching Mac went to sleep on battery: `modal run --detach ... --run` waits on the call, and a client that goes away cancels it. `--run --spawn` spawns the stage and returns its function call id. A probe on 2026-10-19 showed a call spawned under --detach running to completion after its client exited. The receipt and attempt record land on the runs volume either way, and the launch doc now waits for the receipt there. Co-Authored-By: Claude Opus 5.5 --- docs/us-route-a-rerun-20261018.md | 20 ++++++++++-- .../us/test_us_modal_stage_plan_tool.py | 32 +++++++++++++++++++ tools/modal_us_stage.py | 28 ++++++++++++++-- 3 files changed, 75 insertions(+), 5 deletions(-) diff --git a/docs/us-route-a-rerun-20261018.md b/docs/us-route-a-rerun-20261018.md index 66a1da1f7e..7dca2a6958 100644 --- a/docs/us-route-a-rerun-20261018.md +++ b/docs/us-route-a-rerun-20261018.md @@ -267,12 +267,23 @@ part of preparing this document. ```bash "$LSUBMIT" route-a-rerun-20261018-base "$LOCAL/base.launchd.log" "$PREP" \ /usr/bin/env MICROCOSM_MODAL_PLAN="$PLAN" \ - "$MODAL" run --detach tools/modal_us_stage.py --run + "$MODAL" run --detach tools/modal_us_stage.py --run --spawn ``` +`--spawn` returns once the stage is spawned and prints its +`function_call_id`; under `--detach` the call then runs on Modal whether or not +this Mac stays awake. Without it, the client waits on the call, and a client +that goes away can cancel it: on 19 October the first attempt +(`all-20261019T163840131111Z`) was cancelled after 822 s, 99 s after the Mac +went to sleep on battery. The stage writes its receipt to +`runs/$R/receipts/` on `microcosm-us-stage-runs` either way; step 5 waits for +it there. To block on the call instead: +`modal.FunctionCall.from_id("").get()`. + Keep launchd's default `ProcessType=Standard`; do not set `KEEPALIVE=1`. -Use the same reviewed plan for this one paid attempt. A failure or budget stop -does not authorize an automatic relaunch. +Use the same reviewed plan for each paid attempt. A failure, cancellation or +budget stop does not authorize an automatic relaunch: each relaunch is another +paid run and needs Max's go (runbook, "The budget stop"). ### 4. Produce the local reference (scheduled, supervised local work) @@ -356,6 +367,9 @@ the two stages' completed-prefix evidence remains in the fresh reference root. ### 5. Fetch, compare and hand off After the Modal base completes, fetch only the receipt, output and context. +Completion means a receipt under `runs/$R/receipts/` +(`"$MODAL" volume ls microcosm-us-stage-runs "runs/$R/receipts"`); the +attempt record under `runs/$R/attempts/` says `finished` and its `outcome`. Check the CLI's downloaded directory layout before selecting `RECEIPT`. ```bash diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py b/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py index 4ac5c47379..fc41c2b191 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py @@ -1457,6 +1457,38 @@ def test_app_main_exits_nonzero_unless_the_stage_completed( assert json.loads(capsys.readouterr().out)["stopped_at_budget"] is stopped +def test_app_main_spawn_returns_the_call_id_without_waiting( + app, monkeypatch, capsys +) -> None: + data = _plan_data(max_wall_seconds=600) + plan = plan_lib.parse_plan(data) + monkeypatch.setattr(app, "_LOADED", (data, plan)) + runner = MagicMock() + runner.spawn.return_value = MagicMock(object_id="fc-123") + monkeypatch.setitem(app.RUNNERS, ("heavy", False), runner) + app.main(run=True, spawn=True) + runner.spawn.assert_called_once_with(data) + runner.remote.assert_not_called() + assert json.loads(capsys.readouterr().out) == { + "run_id": plan.run_id, + "stage": plan.stage, + "function_call_id": "fc-123", + } + + +def test_app_main_refuses_spawn_without_run(app, monkeypatch) -> None: + data = _plan_data(max_wall_seconds=600) + monkeypatch.setattr(app, "_LOADED", (data, plan_lib.parse_plan(data))) + runner = MagicMock() + monkeypatch.setitem(app.RUNNERS, ("heavy", False), runner) + check = MagicMock() + monkeypatch.setattr(app, "check_stage", check) + with pytest.raises(SystemExit, match="--spawn needs --run"): + app.main(spawn=True) + runner.spawn.assert_not_called() + check.remote.assert_not_called() + + @pytest.mark.parametrize("key", plan_lib.THREAD_ENV_KEYS) def test_every_documented_thread_count_is_allowlisted(key) -> None: assert plan_lib._ENV_KEY.fullmatch(key) diff --git a/tools/modal_us_stage.py b/tools/modal_us_stage.py index da86a43a6b..3810e70bff 100644 --- a/tools/modal_us_stage.py +++ b/tools/modal_us_stage.py @@ -1150,11 +1150,22 @@ def run_stage_acs_staging_nonpreemptible(plan_data: dict) -> dict: @app.local_entrypoint() -def main(run: bool = False) -> None: - """Check the plan (default) or, with --run, execute the stage.""" +def main(run: bool = False, spawn: bool = False) -> None: + """Check the plan (default) or, with --run, execute the stage. + + ``--run`` waits for the stage and prints its receipt. ``--run --spawn`` + spawns the stage, prints its function call id and returns. Under + ``modal run --detach`` the spawned call keeps running after this client + exits, so a client that sleeps or is killed cannot cancel it. A waited-on + call can be cancelled that way: on 2026-10-19 the Route A rerun base was + cancelled 99 s after the launching Mac went to sleep. Either way the + stage writes its receipt and attempt record to the runs volume. + """ assert _LOADED is not None plan_data, plan = _LOADED + if spawn and not run: + raise SystemExit("--spawn needs --run") if not run: report = check_stage.remote(plan_data) print(json.dumps(report, indent=2)) @@ -1163,6 +1174,19 @@ def main(run: bool = False) -> None: print("CHECK OK") return runner = RUNNERS[(plan.resources.name, plan.nonpreemptible)] + if spawn: + call = runner.spawn(plan_data) + print( + json.dumps( + { + "run_id": plan.run_id, + "stage": plan.stage, + "function_call_id": call.object_id, + }, + indent=2, + ) + ) + return receipt = runner.remote(plan_data) brief = { key: receipt[key] From c0fb05094b8b853d934c0ee86887d9572a562361 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 19 Oct 2026 14:12:09 -0400 Subject: [PATCH 05/10] Clone the fetched base into the run instead of copying it Step 5 copied the 2.4 GB base-out from the fetch directory into the run, which doubled its disk use on a host held near the release's 45 GiB admission. `cp -pc` makes an APFS clone that shares the fetched blocks. A 64 MB test moved free space by 0.7 MB instead of 64. Co-Authored-By: Claude Opus 5.5 --- docs/us-route-a-rerun-20261018.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/us-route-a-rerun-20261018.md b/docs/us-route-a-rerun-20261018.md index 7dca2a6958..76ac06a19a 100644 --- a/docs/us-route-a-rerun-20261018.md +++ b/docs/us-route-a-rerun-20261018.md @@ -400,7 +400,8 @@ RUN=$RUN_ROOT/run-${C:0:12} python3 tools/modal_us_stage_plan.py compare-lineage "$RECEIPT" \ --reference "$REF" --run-context "$CONTEXT" > "$LINEAGE" mkdir -p "$RUN/base-out" - cp -p "$FETCH/state/base-out/"* "$RUN/base-out/" + # APFS clone: the run copy shares the fetched blocks, so it costs no extra disk. + cp -pc "$FETCH/state/base-out/"* "$RUN/base-out/" python3 tools/modal_us_stage_plan.py verify-receipt "$RECEIPT" \ --state-root "$RUN" --prefix base-out --strict H5=$RUN/base-out/base_populace_us_2024_puf_support.h5 From fd557a50d3da604390287392070a9b0dd2358b7a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 19 Oct 2026 14:56:35 -0400 Subject: [PATCH 06/10] Give the gate dry run a release id the US contract accepts The release refuses any id that doesn't start with populace-us-, and the dry run stops where the release would. So the launch doc's route-a-rerun-20261018-gate-check failed after 119 s on 2026-10-19 without grading a gate. Use populace-us-2024-gate-check-. Co-Authored-By: Claude Opus 5.5 --- docs/us-route-a-rerun-20261018.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/us-route-a-rerun-20261018.md b/docs/us-route-a-rerun-20261018.md index 76ac06a19a..c25ade085f 100644 --- a/docs/us-route-a-rerun-20261018.md +++ b/docs/us-route-a-rerun-20261018.md @@ -530,7 +530,7 @@ gate_rc=$(supervised gate-dry-run 64 40 30 21600 43200 -- \ --ssi-take-up-prior-weight-basis "$SSI" \ --ssi-take-up-prior-weight-basis-sha256 25fe8af50a99d717f3408b2de7f0849d2307d4f05b1a7d55d2703999002fff0a \ --scf-summary-extract "$SCF" --out "$LOCAL/gate-dry-run" \ - --release-id route-a-rerun-20261018-gate-check \ + --release-id "populace-us-2024-gate-check-${COMMIT:0:12}" \ --checkpoint-root "$LOCAL/gate-checkpoints" --seed 0 --no-staging "${EXTRA[@]}") case "$gate_rc" in 0) echo 'Release guards completed cleanly' ;; From f3341e5079d5386604d9b4f8f436457e33333e0b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 19 Oct 2026 17:31:47 -0400 Subject: [PATCH 07/10] Answer the Opus review of the launch-day changes - Spawn: the docstring, launch doc and runbook no longer claim a receipt "either way". A refusal or error leaves only an attempt record, a rejected plan leaves nothing, and a killed container's record goes stale. Step 5 names all four end states; step 3 records the call id in base.fc and refuses a second paid launch while it exists. Every runbook launch now uses --run --spawn. - Step 6: a check's config records W's HEAD and every file argument's sha256. A recorded result is reused only for an identical config (logged); a changed register refuses. Scripts are written aside and moved in, never over a running job's. - d1712: tools/route_a/lineage_class_check.py, the check run on 2026-10-19 (same syntax tree; formatted), with example and Hypothesis tests; step 5b runs it. - d1711: step 6b renews the QRF register before the release. - Step 7 starts the release from a read-only export of the driver and checks AC power. RUN is defined in step 4, and BLOCKER checks use grep. Co-Authored-By: Claude Opus 5.5 --- docs/us-modal-stage-runbook.md | 17 +- docs/us-route-a-rerun-20261018.md | 202 ++++++++++++++---- .../us/test_route_a_lineage_class_check.py | 164 ++++++++++++++ tools/modal_us_stage.py | 34 ++- tools/route_a/lineage_class_check.py | 142 ++++++++++++ 5 files changed, 504 insertions(+), 55 deletions(-) create mode 100644 packages/microcosm-build/tests/engine_free/us/test_route_a_lineage_class_check.py create mode 100644 tools/route_a/lineage_class_check.py diff --git a/docs/us-modal-stage-runbook.md b/docs/us-modal-stage-runbook.md index 2a1016e82f..702b051c16 100644 --- a/docs/us-modal-stage-runbook.md +++ b/docs/us-modal-stage-runbook.md @@ -223,8 +223,13 @@ python3 tools/modal_us_stage_plan.py validate plan.json # attempt of the run that may still be running. MICROCOSM_MODAL_PLAN=plan.json modal run tools/modal_us_stage.py -# 4. Run the stage. --detach keeps it running if this terminal goes away; -# the receipt lands on the runs volume either way. Set "max_wall_seconds" +# 4. Run the stage. --spawn returns once the stage is spawned, and --detach +# keeps the app up after this client exits, so a terminal or Mac that goes +# away can't cancel it. Without --spawn the client waits on the call, and +# on 2026-10-19 a waiting call was cancelled 99 s after its Mac went to +# sleep. Read the outcome from the runs volume: a receipt under +# runs//receipts/, or an attempt record whose outcome is refused or +# error (see modal_us_stage.main). Set "max_wall_seconds" # in the plan to cap the cost below the class's hard timeout: the runner # stops the tool then (SIGTERM to the tool's process group, which the # tool runs in alone, so a stage child it spawned stops too; SIGKILL to @@ -240,7 +245,7 @@ MICROCOSM_MODAL_PLAN=plan.json modal run tools/modal_us_stage.py # comes off it (see "Preemption and restarts" below). A stop at the # budget writes a receipt, so launching the same plan again gets the # whole budget again. -MICROCOSM_MODAL_PLAN=plan.json modal run --detach tools/modal_us_stage.py --run +MICROCOSM_MODAL_PLAN=plan.json modal run --detach tools/modal_us_stage.py --run --spawn # 5. Fetch the state and verify it against the receipt. A receipt whose # status is not COMPLETED, or that says stopped_at_budget, does not @@ -342,7 +347,7 @@ python3 tools/modal_us_stage_plan.py digest /acs_multispine_staging.h5 # expect f335f573…; then run the printed `modal volume put …` line MICROCOSM_MODAL_PLAN=docs/us-modal-stage-example-plan.json modal run tools/modal_us_stage.py MICROCOSM_MODAL_PLAN=docs/us-modal-stage-example-plan.json \ - modal run --detach tools/modal_us_stage.py --run + modal run --detach tools/modal_us_stage.py --run --spawn ``` ## Acceptance attempt: 23 September materialize on the state surface @@ -863,7 +868,7 @@ MICROCOSM_MODAL_PLAN=docs/us-modal-stage-route-a-base-plan.json \ # throttles CPU use above the limit (the request), and memory above # 112 GiB would bill above that. MICROCOSM_MODAL_PLAN=docs/us-modal-stage-route-a-base-plan.json \ - modal run --detach tools/modal_us_stage.py --run + modal run --detach tools/modal_us_stage.py --run --spawn # 5. Fetch the receipts and base-out only, not the 44 GB of checkpoints, # and verify base-out against the receipt. verify-receipt refuses a @@ -1240,7 +1245,7 @@ bash staging-inputs.sh # disk admission, write probe, PYTHONHASHSEED, ledger and receipt state. MICROCOSM_MODAL_PLAN="$PLAN" modal run tools/modal_us_stage.py # Run: the plan requests non-preemptible placement and a 16-hour budget. -MICROCOSM_MODAL_PLAN="$PLAN" modal run --detach tools/modal_us_stage.py --run +MICROCOSM_MODAL_PLAN="$PLAN" modal run --detach tools/modal_us_stage.py --run --spawn # After completion, fetch the entire state (including logs) and receipts. R=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["run_id"])' "$PLAN") diff --git a/docs/us-route-a-rerun-20261018.md b/docs/us-route-a-rerun-20261018.md index c25ade085f..d401a09ca1 100644 --- a/docs/us-route-a-rerun-20261018.md +++ b/docs/us-route-a-rerun-20261018.md @@ -102,8 +102,11 @@ The old `base-sup/PROVENANCE.md` records a strict mismatch at stage 001: `8.7e-11`, relative `6.08e-15`. A prior session accepted it with a scripted check. Max ruled on 19 October (d1712) that a repeat of exactly that class may be accepted for this run: every entity table has the same shape and index, totals -are equal, and every differing value is within 1e-12 relative. The check that -establishes this is recorded with the run's provenance. Any other mismatch, +are equal, and every differing value is within 1e-12 relative. Step 5b runs +`tools/route_a/lineage_class_check.py`, which is narrower: any difference +outside `person.pre_subsidy_rent` stops for Max, and so does a total that +differs by more than 1e-12 relative. Its JSON verdict goes into the run's +provenance. Any other mismatch, skipping the new local reference, or relaxing the hand-off still needs **Max's explicit decision** with the new evidence. Stop before writing `ACCEPTED` while that decision is outstanding. @@ -230,7 +233,7 @@ uv run ruff format --check tools/modal_us_stage_plan.py tools/route_a/make_local python3 tools/ci_test_plan.py verify ROUTE_A_ENV="$SETTINGS" bash tools/route_a/route_a.sh --resolve | tee "$LOCAL/resolve-initial.log" # Resolve exit 0 alone is insufficient: it can report commit blockers. -if rg 'BLOCKER' "$LOCAL/resolve-initial.log"; then exit 1; fi +if grep -q 'BLOCKER' "$LOCAL/resolve-initial.log"; then exit 1; fi # Inspect inputs.resolved.json; every registered file must be verified. python3 tools/modal_us_stage_plan.py upload-commands "$PLAN" --shell \ "asec_2024_h5=$STORAGE/census_cps_2024.h5" \ @@ -265,20 +268,34 @@ part of preparing this document. ### 3. Launch the Modal base under launchd ```bash +# One paid attempt at a time: base.fc records the spawned call. +if [ -e "$LOCAL/base.fc" ]; then + echo "REFUSED: $LOCAL/base.fc records spawned call $(cat "$LOCAL/base.fc"); check it before another paid launch" + exit 1 +fi "$LSUBMIT" route-a-rerun-20261018-base "$LOCAL/base.launchd.log" "$PREP" \ /usr/bin/env MICROCOSM_MODAL_PLAN="$PLAN" \ "$MODAL" run --detach tools/modal_us_stage.py --run --spawn +until grep -q '"function_call_id"' "$LOCAL/base.launchd.log" 2>/dev/null; do sleep 5; done +grep -o 'fc-[A-Za-z0-9]*' "$LOCAL/base.launchd.log" | head -1 > "$LOCAL/base.fc" +test -s "$LOCAL/base.fc" ``` -`--spawn` returns once the stage is spawned and prints its -`function_call_id`; under `--detach` the call then runs on Modal whether or not -this Mac stays awake. Without it, the client waits on the call, and a client -that goes away can cancel it: on 19 October the first attempt -(`all-20261019T163840131111Z`) was cancelled after 822 s, 99 s after the Mac -went to sleep on battery. The stage writes its receipt to -`runs/$R/receipts/` on `microcosm-us-stage-runs` either way; step 5 waits for -it there. To block on the call instead: -`modal.FunctionCall.from_id("").get()`. +`--spawn` returns once the stage is spawned and prints its `function_call_id`. +Run it under `--detach`. Without `--spawn` the client waits on the call, and on +19 October the first attempt (`all-20261019T163840131111Z`) was cancelled +after 822 s, 99 s after the Mac went to sleep on battery. The cancellation is +attributed to the sleep from its timing. A probe that day showed a spawned call +finishing after its client exited; client sleep itself was not tested. + +The launcher's exit code no longer says how the stage ended; the runs volume +does (see step 5). To block on the call instead, use the modal tool's own +interpreter (system `python3` has no `modal`): +`~/.local/share/uv/tools/modal/bin/python -c 'import modal; print(modal.FunctionCall.from_id("").get())'`. +It returns the receipt, or re-raises the error of a rejected, refused or failed +attempt. Remove `base.fc` only when its call has ended and a relaunch has +Max's go. A launchd job's own exit status is in +`launchctl print gui/$(id -u)/com.maxghenis.claude893.