diff --git a/docs/us-modal-stage-route-a-rerun-base-plan.json b/docs/us-modal-stage-route-a-rerun-base-plan.json new file mode 100644 index 0000000000..3f756dd2c6 --- /dev/null +++ b/docs/us-modal-stage-route-a-rerun-base-plan.json @@ -0,0 +1,61 @@ +{ + "schema": "microcosm-modal-us-stage-plan/1", + "tool": "us-puf-support-base-rerun", + "stage": "all", + "run_id": "route-a-rerun-base-20261018-pending", + "source": { + "commit": "PICK_MAIN_COMMIT_ON_2026_10_18", + "branch": "main" + }, + "inputs": { + "asec_2024_h5": { + "uri": "volume://cas/sha256/ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d/census_cps_2024.h5", + "sha256": "ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d" + }, + "asec_2023_h5": { + "uri": "volume://cas/sha256/cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88/census_cps_2023.h5", + "sha256": "cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88" + }, + "asec_2022_h5": { + "uri": "volume://cas/sha256/7ccca976284bb47815d84460cc4f75a0a65d26d7754ab0a0f417de351b3d474e/census_cps_2022.h5", + "sha256": "7ccca976284bb47815d84460cc4f75a0a65d26d7754ab0a0f417de351b3d474e" + }, + "puf_2024_h5": { + "uri": "volume://cas/sha256/7669f5b5281f20080e77204f9bd4aabfad0aa101fa283e22caf9ba8d61d4d6df/puf_2024.h5", + "sha256": "7669f5b5281f20080e77204f9bd4aabfad0aa101fa283e22caf9ba8d61d4d6df" + }, + "puf_2015_csv": { + "uri": "volume://cas/sha256/0a7fd643edb1acc55c507db795914b41d232922be78c149b58d111f4672499df/puf_2015.csv", + "sha256": "0a7fd643edb1acc55c507db795914b41d232922be78c149b58d111f4672499df" + }, + "acs_2022_h5": { + "uri": "volume://cas/sha256/0b319b496f19a6913066f9c5ea572edfda3d78a187be6f375846617d0b441bd4/acs_2022.h5", + "sha256": "0b319b496f19a6913066f9c5ea572edfda3d78a187be6f375846617d0b441bd4" + }, + "asec_education_2022_zip": { + "uri": "volume://cas/sha256/d2e000250782adfbdd7f29c82b66d866591a30f0d330496698ec19f9c784ce11/asecpub23csv.zip", + "sha256": "d2e000250782adfbdd7f29c82b66d866591a30f0d330496698ec19f9c784ce11" + }, + "asec_education_2023_zip": { + "uri": "volume://cas/sha256/cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09/asecpub24csv.zip", + "sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09" + }, + "asec_education_2024_zip": { + "uri": "volume://cas/sha256/318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b/asecpub25csv.zip", + "sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b" + }, + "base_ledger_facts": { + "uri": "volume://cas/sha256/b85437390021777e746f507c5890305496baf5fc7f2c78ba08ddb090f4839801/consumer_facts_us_c5e5bf8.jsonl", + "sha256": "b85437390021777e746f507c5890305496baf5fc7f2c78ba08ddb090f4839801" + }, + "block_ladder_npz": { + "uri": "volume://cas/sha256/6840b990acdfa2003d7723da5595e3cfff7205a4fa3d8d6206ed835c85c3f233/us_block_ladder_2020.npz", + "sha256": "6840b990acdfa2003d7723da5595e3cfff7205a4fa3d8d6206ed835c85c3f233" + } + }, + "env": { + "PYTHONUNBUFFERED": "1" + }, + "max_wall_seconds": 14400, + "nonpreemptible": true +} diff --git a/docs/us-modal-stage-runbook.md b/docs/us-modal-stage-runbook.md index 378656d602..1d33ed5d57 100644 --- a/docs/us-modal-stage-runbook.md +++ b/docs/us-modal-stage-runbook.md @@ -3,12 +3,15 @@ Epic #956 acceleration item E: heavy US stages should not have to queue on the one 128 GiB build machine. This runbook covers the smallest working path: a registered tool run on Modal from a pinned commit, with inputs fetched by -digest and outputs listed with sha256 receipts. Four tools are registered +digest and outputs listed with sha256 receipts. Five tools are registered (`TOOLS` in the plan module): `tools/build_us_acs_local_release.py` (tool `us-acs-local-release`, stages `materialize`, `calibrate`, `qa`, `finalize`, `package`, or `all`), Route A's PUF-support base, `tools/build_us_puf_support_base.py` (tool `us-puf-support-base`, stage -`all`; see "Route A's base stage"), the ACS multispine staging builder, +`all`; see "Route A's base stage"), its current-main rerun registration +`us-puf-support-base-rerun` (same script and stage, with the reviewed current +district crosswalk; see [the 18 October rerun](us-route-a-rerun-20261018.md)), +the ACS multispine staging builder, `tools/build_us_acs_multispine_base.py` (tool `us-acs-multispine-staging`, stage `all`; see "ACS multispine staging"), and `runner-smoke` (stage `smoke`), an inline script that proves the run path on any pushed commit (see @@ -220,8 +223,13 @@ python3 tools/modal_us_stage_plan.py validate plan.json # attempt of the run that may still be running. MICROCOSM_MODAL_PLAN=plan.json modal run tools/modal_us_stage.py -# 4. Run the stage. --detach keeps it running if this terminal goes away; -# the receipt lands on the runs volume either way. Set "max_wall_seconds" +# 4. Run the stage. --spawn returns once the stage is spawned, and under +# --detach a probe on 2026-10-19 saw the spawned call finish after its +# client exited (client sleep was not tested). Without --spawn the client waits on the call, and +# on 2026-10-19 a waiting call was cancelled 99 s after its Mac went to +# sleep. Read the outcome from the runs volume: a receipt under +# runs//receipts/, or an attempt record whose outcome is refused or +# error (see modal_us_stage.main). Set "max_wall_seconds" # in the plan to cap the cost below the class's hard timeout: the runner # stops the tool then (SIGTERM to the tool's process group, which the # tool runs in alone, so a stage child it spawned stops too; SIGKILL to @@ -237,7 +245,7 @@ MICROCOSM_MODAL_PLAN=plan.json modal run tools/modal_us_stage.py # comes off it (see "Preemption and restarts" below). A stop at the # budget writes a receipt, so launching the same plan again gets the # whole budget again. -MICROCOSM_MODAL_PLAN=plan.json modal run --detach tools/modal_us_stage.py --run +MICROCOSM_MODAL_PLAN=plan.json modal run --detach tools/modal_us_stage.py --run --spawn # 5. Fetch the state and verify it against the receipt. A receipt whose # status is not COMPLETED, or that says stopped_at_budget, does not @@ -339,7 +347,7 @@ python3 tools/modal_us_stage_plan.py digest /acs_multispine_staging.h5 # expect f335f573…; then run the printed `modal volume put …` line MICROCOSM_MODAL_PLAN=docs/us-modal-stage-example-plan.json modal run tools/modal_us_stage.py MICROCOSM_MODAL_PLAN=docs/us-modal-stage-example-plan.json \ - modal run --detach tools/modal_us_stage.py --run + modal run --detach tools/modal_us_stage.py --run --spawn ``` ## Acceptance attempt: 23 September materialize on the state surface @@ -443,6 +451,14 @@ across containers would let preemption lose one chunk instead of the run. ## Route A's base stage +For Max's d1386 rerun on **18 October 2026**, use +[the rerun preparation and launch sequence](us-route-a-rerun-20261018.md) and +`us-modal-stage-route-a-rerun-base-plan.json`. Its commit is intentionally +unresolved. That procedure separates the historical base feed from main's +new release feed, pins the current in-tree crosswalk with a scoped registration, +and requires fresh local 000/001 evidence for the selected commit and CT ladder. +The historical plan/reference below cannot qualify that rerun. + On 29 September 2026 Max ruled to run only Route A's base stage on Modal: non-preemptible, capped at 4 hours of wall, about $15. The release, its preflight and certification stay on the build machine. @@ -527,7 +543,7 @@ Two kinds of test hold the command: command with its paths tokenized (`packages/microcosm-build/tests/fixtures/modal_us_stage/route_a_base_command_4b57d15a287c.json`), and the registration's owned flags equal the flags of this tree's tool. -- Only in a clone that has commit `4b57d15a2`, four tests in +- Only in a clone that has commit `4b57d15a2`, five tests in `test_us_modal_stage_puf_support_base.py` that read that commit with `git show` and skip without it: - `test_the_tool_s_own_parser_reads_the_same_build_from_both_commands`: @@ -543,10 +559,13 @@ Two kinds of test hold the command: its `_stage_run_config` locks, from the Modal argv, the run config the local run locked (paths by file name), including `PYTHONHASHSEED` `0` when the runner's tool environment removed a container's value, and a - forced `12345` is what compare-lineage refuses. + forced `12345` is what compare-lineage refuses; + - `test_rerun_crosswalk_refuses_the_historical_tree`: the rerun registration + refuses that commit's historical crosswalk bytes while retaining its + separately reviewed current-main pin. CI checks out only the commit under test (`actions/checkout` fetches one -commit by default), so there those four tests report a skip rather than +commit by default), so there those five tests report a skip rather than pass. Run them in a full clone before a paid run (step 2 below). The plan is `docs/us-modal-stage-route-a-base-plan.json` (run @@ -825,7 +844,7 @@ python3 tools/modal_us_stage_plan.py upload-commands --shell \ # 2. Validate locally: the argv, class base (cpu_limit 4.0), the $14.63 # estimate and the $14.90 ceiling at the request # (estimated_usd_at_timeout). Then run the base's tests, and confirm the -# four pinned-commit tests passed rather than skipped (-rs lists skips). +# five pinned-commit tests passed rather than skipped (-rs lists skips). python3 tools/modal_us_stage_plan.py validate docs/us-modal-stage-route-a-base-plan.json uv run pytest -rs packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py @@ -849,7 +868,7 @@ MICROCOSM_MODAL_PLAN=docs/us-modal-stage-route-a-base-plan.json \ # throttles CPU use above the limit (the request), and memory above # 112 GiB would bill above that. MICROCOSM_MODAL_PLAN=docs/us-modal-stage-route-a-base-plan.json \ - modal run --detach tools/modal_us_stage.py --run + modal run --detach tools/modal_us_stage.py --run --spawn # 5. Fetch the receipts and base-out only, not the 44 GB of checkpoints, # and verify base-out against the receipt. verify-receipt refuses a @@ -1226,7 +1245,7 @@ bash staging-inputs.sh # disk admission, write probe, PYTHONHASHSEED, ledger and receipt state. MICROCOSM_MODAL_PLAN="$PLAN" modal run tools/modal_us_stage.py # Run: the plan requests non-preemptible placement and a 16-hour budget. -MICROCOSM_MODAL_PLAN="$PLAN" modal run --detach tools/modal_us_stage.py --run +MICROCOSM_MODAL_PLAN="$PLAN" modal run --detach tools/modal_us_stage.py --run --spawn # After completion, fetch the entire state (including logs) and receipts. R=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["run_id"])' "$PLAN") diff --git a/docs/us-route-a-rerun-20261018.env.example b/docs/us-route-a-rerun-20261018.env.example new file mode 100644 index 0000000000..890e71261c --- /dev/null +++ b/docs/us-route-a-rerun-20261018.env.example @@ -0,0 +1,69 @@ +# shellcheck shell=bash +# d1386 preparation, 2026-10-15. Copy to the machine-local 20261018 directory. +# COMMIT is deliberately empty. Recheck the marked settings on launch day. +# Copy to route_a.env, or point ROUTE_A_ENV at a machine-local settings file. +# No credentials belong here. Environment overrides win over these defaults. +# All machine locations are configured here; the driver uses colocated helpers. +: "${PE:=/Users/maxghenis/PolicyEngine}" +: "${MAIN:=$PE/microcosm}" +: "${WT_ROOT:=$PE/_worktrees}" +# Use a fresh root for a new attempt. Do not reuse a pre-CT base's ACCEPTED marker. +: "${RUN_ROOT:=$PE/_recovered/scratch-backup/893/route-a-rerun-20261018/runs}" +: "${CHAIN_ROOT:=$PE/_recovered/scratch-backup/893/overnight-20260923}" +: "${CHAIN_LOG:=$CHAIN_ROOT/chain.log}" +: "${DISK_PATH:=/System/Volumes/Data}" +: "${UV:=$HOME/.local/bin/uv}" +: "${PYTHON:=python3}" +: "${AGENT_SECRET:=$HOME/bin/agent-secret}" +: "${STORAGE:=$PE/policyengine-us-data/policyengine_us_data/storage}" +: "${EDU:=$PE/_buildm-runtime/inputs/asec_education}" +: "${FEED:=$PE/_buildh-runtime/inputs/consumer_facts_us_51aa40f.jsonl}" +# The driver preserves the published feed/crosswalk pins by default. If the +# chosen commit changed them, set FEED_SHA, FEED_BYTES and CROSSWALK_SHA here +# alongside FEED, after checking that commit's packaged pins. +# d713: CT-fixed ladder; route_a.sh verifies its digest and size. +: "${LADDER:=$PE/_build_artifacts/us-ct-cbsa/us_block_ladder_2020.npz}" +: "${SSI:=$PE/_buildo-runtime/inputs/attempt6_basis_schema3_seed.json}" +: "${SCF:=$PE/_buildm-runtime/inputs/scf_cache/rscfp2022.dta}" + +# Choose a pushed build commit. Empty COMMIT refuses a real run. +: "${COMMIT:=}" +: "${REQUIRE_ON_MAIN:=1}" +: "${REQUIRE_SPM_ROLE:=1}" +: "${PREFLIGHT_COMMIT:=}" +: "${PREFLIGHT_NEW_LINEAGE_ARGS:=--new-lineage}" +: "${EARLY_START_AFTER_HEAVY:=0}" +: "${PRUNE_BASE_CHECKPOINTS:=1}" +: "${RELEASE_DESPITE_PREFLIGHT_FAIL:=0}" +: "${RESOURCE_WAIT_HOURS:=12}" +: "${RELEASE_RAM_GIB:=50}" +: "${RELEASE_WALL_SECONDS:=345600}" +# 0 is the sole opt-out and restores --no-staging without a credential lookup. +: "${ROUTE_A_STAGING=1}" + +# Retain d122 whole-base dense national/state settings and the published run's +# export-mass reference. Review the historical tail waiver against the new export. +: "${DENSE_RELEASE_D122:=1}" +: "${EXPORT_MASS_REF_H5:=$PE/_buildg-runtime/forensics/populace_us_2024.h5}" +: "${RELEASE_EXTRA_ARGS:=--maximum-microsim-batch-size 2000 --target-surface national_state --dense-default-dataset --export-input-mass-reference-h5 $EXPORT_MASS_REF_H5}" +: "${QRF_TAIL_EXCLUSIONS:=$CHAIN_ROOT/route-a/qrf_tail_exclusions_routea_d490_20260930.json}" +: "${QRF_TAIL_EXCLUSIONS_SHA:=fa5760993ebcd1386f02c0b1c4c4e867a0c4b1e689a33774a41f623f95132d69}" +# Optional base feed override: BASE_LEDGER_FACTS and BASE_LEDGER_FACTS_SHA. + +# REVIEW: selected main must still pin this release feed. It changed since 4b57d15a2. +: "${FEED_SHA:=965a29ac9458edb58a05184917ee4a19772fd3dc7b4588db6eb6da5803641aac}" +: "${FEED_BYTES:=169038757}" +# Keep the historical base feed explicitly; its CD facts are unchanged by #940. +: "${BASE_LEDGER_FACTS:=$PE/_buildh-runtime/inputs/consumer_facts_us_c5e5bf8.jsonl}" +: "${BASE_LEDGER_FACTS_SHA:=b85437390021777e746f507c5890305496baf5fc7f2c78ba08ddb090f4839801}" +# REVIEW: selected main must still carry the NC-fixed in-tree crosswalk. +: "${CROSSWALK_SHA:=a347303fff3fea145f758488c43cb94355df5a8cb5e512552b2ee3791d2aa233}" +# REVIEW: historical export-mass reference is inherited, not re-certified here. +# REVIEW: re-measure the register on the new lineage. Update file and SHA together. +# d1711 (Max 2026-10-19): d490 covers this build. Threshold 0.75, same or fewer +# columns, on the same terms, renewed to 2026-12-31 or the next certified build. +# A new column stops for Max. +# REVIEW: d122 dense/national_state, batch 2000, RAM/wall/wait limits are inherited. +# REVIEW: PREFLIGHT_COMMIT defaults to COMMIT; keep both on the chosen main tree. +# Explicit staging destination; no credential value is stored here. +export POPULACE_STAGING_REPO_ID=policyengine/populace-us-staging diff --git a/docs/us-route-a-rerun-20261018.md b/docs/us-route-a-rerun-20261018.md new file mode 100644 index 0000000000..569ba5a950 --- /dev/null +++ b/docs/us-route-a-rerun-20261018.md @@ -0,0 +1,749 @@ +# Route A rerun on 18 October 2026 (d1386) + +Prepared on 15 October against `origin/main` +`412f916536b66577e4e0b42c707285085df7c302`. Nothing in this preparation +launched a build or called Modal. The 18 October start did not happen; Max +confirmed on 19 October that the build launches that day on current `main` +without #1275 (d1386, d1556). + +Max's d1386 selects whatever has merged to `main` at launch. It must contain +`9802825200929c445856d969f4231287c185d784` (#1186, the #1178 Social Security +fix). The base runs on Modal; release, preflights and certification run locally +through `tools/route_a/`. Staging is enabled. Publication remains separate. + +The machine-local settings file is +`$PE/_recovered/scratch-backup/893/route-a-rerun-20261018/route_a.env`, from +`docs/us-route-a-rerun-20261018.env.example`. Never commit `route_a.env`. + +## Inputs and registration + +The new plan is `docs/us-modal-stage-route-a-rerun-base-plan.json`. +`PICK_MAIN_COMMIT_ON_2026_10_18` is deliberately not a SHA: existing `validate` +returns 2 while it remains. Step 1 writes a concrete machine-local plan without +editing the template. All eleven inputs use the runbook's +`volume://cas/sha256//` layout on +`microcosm-us-stage-inputs`; the volume name is not part of the URI. + +| Input or setting | Change since `4b57d15a2` | +| --- | --- | +| Block ladder | d713 CT-fixed `6840b990acdfa2003d7723da5595e3cfff7205a4fa3d8d6206ed835c85c3f233`, 18,991,218 bytes; replaces `7ba39b95…` | +| ASEC H5s, income years 2024/2023/2022 | Same pins as the historical plan, in that order | +| ASEC education, income years 2022/2023/2024 | Same three archive pins; the first also seeds the unemployment-weeks cache | +| ASEC 2025 | Main added it to general defaults; Route A explicitly supplies 2024/2023/2022, so no fourth source is added | +| Processed PUF, TY2015 raw PUF, ACS 2022 | Same pins as the historical plan | +| Base feed | Retains `consumer_facts_us_c5e5bf8.jsonl`, SHA `b85437390021777e746f507c5890305496baf5fc7f2c78ba08ddb090f4839801` | +| Release feed | Main repinned under #940 to `consumer_facts_us_51aa40f.jsonl`, SHA `965a29ac9458edb58a05184917ee4a19772fd3dc7b4588db6eb6da5803641aac`, 169,038,757 bytes, 40,178 rows; the machine file was verified | +| In-tree district crosswalk | NC correction changed `c7cb040b…` to `a347303fff3fea145f758488c43cb94355df5a8cb5e512552b2ee3791d2aa233`, 78,202 bytes | + +The release feed retains all 39,158 old facts and adds 1,020 TY2023 state AGI +facts; see `us-chronicle-feed-repin.md`. Base district counts are unchanged, +so retaining its old feed is intentional. `BASE_LEDGER_FACTS` and its SHA are +explicitly independent from release `FEED`, `FEED_SHA` and `FEED_BYTES`. +The base builder itself is unchanged since `4b57d15a2`; runtime code has changed, +including the raw-PUF partnership self-employment correction. Identical input +pins do not imply identical outputs. + +`us-puf-support-base-rerun` preserves the historical base registration's argv, +inputs, cache seed, hash-seed handling, mirroring and resources, and pins the +current crosswalk. The historical `us-puf-support-base` registration and its +plan/reference remain unchanged. Both refuse an unrecognized crosswalk. The +driver's feed and crosswalk expectations are configurable, with historical +defaults; this rerun's settings explicitly pin the new values. If main changes +any pin again, review and update the affected settings/registration before +launching; do not suppress a check or substitute unreviewed bytes. + +The base retains non-preemptible placement, four requested/limited CPUs, +112 GiB memory, 14,400 seconds of tool budget and a 16,500-second class timeout. +The planner estimates $14.63 including reserve and $14.90 at the timeout and +request, below the $15 base cap. As the existing runbook explains, this is not +an absolute billed ceiling: memory above the request, checks, startup, image +builds and storage add cost. A second paid attempt needs Max's go. No resource +limit or cap interpretation is changed here. + +## Lineage decision + +Recommend a fresh local reference from **only named stages +`source_construction` and `pre_clone_enrichment`**, on the selected clean commit +and CT ladder, followed by strict `compare-lineage`. These stages add no Modal +cost. The retained local interrupted attempt's +`overnight-20260923/route-a/run-4b57d15a287c/base-sup.failed-1790651711/RESULT.json` +reports 2,054.6823 seconds elapsed and 12,516,982,784 bytes peak RSS (11.66 GiB), +after 000/001 completed. These are whole-attempt measurements, not isolated +000/001 timings; no retained local stage profile permits finer attribution. +Its two compared checkpoint files total 1,485,203,161 bytes. The current source +stage also writes an auxiliary raw-ASEC checkpoint; that total is not the full +local disk requirement. Allow RAM/disk headroom and +use supervision; shared free-disk decline cannot be attributed to those stages. + +The old Modal receipt reports 6,234.9 seconds tool wall, 6,281.6 seconds container +wall, 70.69 GiB peak RSS and a $5.67 list-price estimate for the completed old +base. These observations do not bound this rerun's time or cost. + +A new hand-off requires a COMPLETED, non-budget-stopped receipt, strict +`base-out` verification, the actual Modal context matching its receipt digest, +and a reference with the chosen commit, all eleven input digests, pipeline, run +config, source-code digest, dependency versions, hash seed 0 and actual 000/001 +checkpoint digests. Every comparison must pass. The reference's argv and +environment are written from the plan's registry (`planned_argv(plan)`), not +read from the local run, so their comparison only confirms the registry; the +strict masked `run_config`, source and dependency identity, hash seed, +pipeline and checkpoint digests carry the evidence. Interpreter and other thread-variable differences are +reported; only hash seed is compared among thread variables. Equality covers +two stages, not the later 22, district/ladder assignment or final export. +Release gates and certification still run. + +Do not relabel the old reference with a new commit/ladder or derive a local +reference from Modal outputs. `make_local_reference.py` reads the actual local +context/checkpoints and validates their inputs and source identity. Keep its +JSON and the supporting local evidence with this run. + +The old `base-sup/PROVENANCE.md` records a strict mismatch at stage 001: +313/432,523 `person.pre_subsidy_rent` values differed, maximum absolute +`8.7e-11`, relative `6.08e-15`. A prior session accepted it with a scripted check. +Max ruled on 19 October (d1712) that a repeat of exactly that class may be +accepted for this run: every entity table has the same shape and index, totals +are equal, and every differing value is within 1e-12 relative. Step 5b runs +`tools/route_a/lineage_class_check.py`, which is narrower: any difference +outside `person.pre_subsidy_rent` stops for Max, and so does a total that +differs by more than 1e-12 relative. Its JSON verdict goes into the run's +provenance. Any other mismatch, +skipping the new local reference, or relaxing the hand-off still needs **Max's +explicit decision** with the new evidence. Stop before writing `ACCEPTED` while +that decision is outstanding. + +## Settings to review on 18 October + +- `COMMIT`: empty until selected from freshly fetched main; must include #1186. +- `FEED`, `FEED_SHA`, `FEED_BYTES`, `BASE_LEDGER_FACTS`, its SHA and + `CROSSWALK_SHA`: recheck against the selected tree. The two feeds differ on + purpose. ASEC/PUF/ACS/education pins must also remain compatible. +- `PREFLIGHT_COMMIT` and `PREFLIGHT_NEW_LINEAGE_ARGS`: empty commit defaults to + the build commit; verify its parser and gates still accept `--new-lineage`. +- `EXPORT_MASS_REF_H5`: inherited + `$PE/_buildg-runtime/forensics/populace_us_2024.h5` (354,374,956 bytes). + Review its suitability and digest as the comparison reference for this run. + The driver does not include this file in `inputs.resolved.json`; record its + digest separately. The release must retain its export-mass gate. +- `QRF_TAIL_EXCLUSIONS` and `_SHA`: the d490 historical register is provisional + for this lineage. Re-measure and update the file and SHA together. Its four + columns are `bond_assets`, `estate_income`, + `long_term_capital_gains_on_collectibles`, and + `w2_wages_from_qualified_business`. The d177/d450 waivers behind them expire + at the next certified build, which is this one. Max ruled on 19 October + (d1711) that d490 covers this build: start with this register, and if the + measured columns over 0.75 are the same or fewer, apply them on the same + terms, renewed to expire 2026-12-31 or at the next certified build. Drop + stale entries. Adding a column or lowering the threshold still needs Max's + ruling. The old reasons and expiry do not attest a new export. The + pre-solve dry run uses base weights; the final weighted-export gate still + determines whether the register is current. +- `DENSE_RELEASE_D122` and `RELEASE_EXTRA_ARGS`: retain the historical dense + whole-base, national/state target surface and batch size 2000. The original + d122 text described the first release; confirm this remains the intended + configuration for the rerun. +- `RELEASE_RAM_GIB=50`, `RELEASE_WALL_SECONDS=345600`, + `RESOURCE_WAIT_HOURS=12` and batch size 2000: inherited admission/budget + settings, unmeasured on the new commit and current load. The 110 GiB RSS + cap remains enforced. +- Staging: `ROUTE_A_STAGING=1`; destination explicitly + `policyengine/populace-us-staging`. Check runtime helper access and dashboard + delivery on launch day. Do not fetch/store credentials during preparation. +- Small-H5 guard fixture: choose a real small written export with enough + households for at least two batches. Do not substitute a full export for it. + +## Exact launch-day sequence + +Run the blocks in **Bash**, with `set -euo pipefail`, from a prep checkout +containing these changes. `PREP` is that checkout; `W` below is a separate, +clean checkout of the chosen build commit. Nothing writes to the old run or +`microcosm-route-a-4b57d15a287c` worktree. There is no automatic date scheduler. + +### 1. Select main, install settings and fill the plan + +```bash +set -euo pipefail +PE=/Users/maxghenis/PolicyEngine +MAIN=$PE/microcosm +PREP=$PE/_worktrees/microcosm-route-a-rerun-prep +LOCAL=$PE/_recovered/scratch-backup/893/route-a-rerun-20261018 +SETTINGS=$LOCAL/route_a.env +PLAN=$LOCAL/base-plan.json +LSUBMIT=$PE/_recovered/scratch-backup/893/bin/lsubmit.sh +cd "$PREP" +git -C "$MAIN" fetch origin +C=$(git -C "$MAIN" rev-parse 'origin/main^{commit}') +git -C "$MAIN" merge-base --is-ancestor 9802825200929c445856d969f4231287c185d784 "$C" +mkdir -p "$LOCAL" +# Once only: preserve any existing operator edits. +test -e "$SETTINGS" || cp docs/us-route-a-rerun-20261018.env.example "$SETTINGS" +python3 - "$MAIN" "$C" "$PLAN" "$SETTINGS" <<'PY' +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +main, commit, output, settings = sys.argv[1:] +def shown(path): + return subprocess.check_output(["git", "-C", main, "show", f"{commit}:{path}"]) + +feed = json.loads(shown("packages/microcosm-build/src/microcosm/build/us/chronicle_feed.json")) +if feed["facts_sha256"] != "965a29ac9458edb58a05184917ee4a19772fd3dc7b4588db6eb6da5803641aac": + raise SystemExit("REFUSED: main changed the release feed; review its pins/settings") +crosswalk = "packages/microcosm-build/src/microcosm/build/us_runtime/data/congressional_district_vintage_crosswalk.csv" +if hashlib.sha256(shown(crosswalk)).hexdigest() != "a347303fff3fea145f758488c43cb94355df5a8cb5e512552b2ee3791d2aa233": + raise SystemExit("REFUSED: main changed the crosswalk; review the rerun registration") +plan = json.loads(Path("docs/us-modal-stage-route-a-rerun-base-plan.json").read_text()) +plan["source"]["commit"] = commit +plan["run_id"] = f"route-a-rerun-base-20261018-{commit[:12]}" +text = Path(settings).read_text() +if ': "${COMMIT:=}"' not in text: + raise SystemExit("REFUSED: settings already pinned; review before replacing COMMIT") +if Path(output).exists(): + raise SystemExit("REFUSED: concrete plan already exists; preserve the attempt identity") +Path(output).write_text(json.dumps(plan, indent=2) + "\n") +Path(settings).write_text(text.replace(': "${COMMIT:=}"', ': "${COMMIT:=' + commit + '}"')) +print(commit) +PY +python3 tools/modal_us_stage_plan.py validate "$PLAN" > "$LOCAL/plan-validation.json" +# Review SETTINGS and the uncertainties above before proceeding. +. "$SETTINGS" +test "$COMMIT" = "$C" +test "$ROUTE_A_STAGING" = 1 +shasum -a 256 "$EXPORT_MASS_REF_H5" > "$LOCAL/export-mass-reference.sha256" +``` + +Use a fresh shell or clear stale Route A environment overrides: defaults in +the settings file intentionally let nonempty environment values win. `lsubmit` +does not inherit them, so persist every intended change in the settings file. + +### 2. Engine-free checks, resolve and missing input uploads + +```bash +uv run pytest -n 4 -rs \ + packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py \ + packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py \ + packages/microcosm-build/tests/engine_free/us/test_route_a_driver.py \ + packages/microcosm-build/tests/engine_free/us/test_us_route_a_local_reference.py +uv run ruff check . +uv run ruff format --check tools/modal_us_stage_plan.py tools/route_a/make_local_reference.py \ + packages/microcosm-build/tests/engine_free/us/test_route_a_driver.py \ + packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py \ + packages/microcosm-build/tests/engine_free/us/test_us_route_a_local_reference.py +python3 tools/ci_test_plan.py verify +ROUTE_A_ENV="$SETTINGS" bash tools/route_a/route_a.sh --resolve | tee "$LOCAL/resolve-initial.log" +# Resolve exit 0 alone is insufficient: it can report commit blockers. +if grep -q 'BLOCKER' "$LOCAL/resolve-initial.log"; then exit 1; fi +# Inspect inputs.resolved.json; every registered file must be verified. +python3 tools/modal_us_stage_plan.py upload-commands "$PLAN" --shell \ + "asec_2024_h5=$STORAGE/census_cps_2024.h5" \ + "asec_2023_h5=$STORAGE/census_cps_2023.h5" \ + "asec_2022_h5=$STORAGE/census_cps_2022.h5" \ + "puf_2024_h5=$STORAGE/puf_2024.h5" "puf_2015_csv=$STORAGE/puf_2015.csv" \ + "acs_2022_h5=$STORAGE/acs_2022.h5" \ + "asec_education_2022_zip=$EDU/asecpub23csv.zip" \ + "asec_education_2023_zip=$EDU/asecpub24csv.zip" \ + "asec_education_2024_zip=$EDU/asecpub25csv.zip" \ + "base_ledger_facts=$BASE_LEDGER_FACTS" "block_ladder_npz=$LADDER" \ + > "$LOCAL/upload-base-inputs.sh" +# Hashes are checked before this script is emitted; it skips existing CAS files. +# The CT upload may already have completed; do not overwrite unrelated paths. +bash "$LOCAL/upload-base-inputs.sh" +MODAL=$(command -v modal) +MICROCOSM_MODAL_PLAN="$PLAN" "$MODAL" run tools/modal_us_stage.py \ + > "$LOCAL/modal-check.log" 2>&1 +``` + +Preparation also ran repository-wide `ruff format --check .` with locked Ruff +0.15.16: 45 unchanged cached-main files fail that check. All five changed Python +files pass; the scoped launch-day command above checks them without rewriting +unrelated sources or their identities. Reassess this baseline after fetching. + +Read the check report: no problems, branch/source/crosswalk/eleven inputs +verified, disk admission, write probe and hash seed accepted. A failed check +blocks the paid launch. Check all five pinned-commit tests passed +instead of skipped in the full-history checkout. No Modal command is run as +part of preparing this document. + +### 3. Launch the Modal base under launchd + +```bash +# One paid attempt at a time: base.fc records the spawned call. +if [ -e "$LOCAL/base.fc" ]; then + echo "REFUSED: $LOCAL/base.fc records spawned call $(cat "$LOCAL/base.fc"); check it before another paid launch" + exit 1 +fi +# launchd appends to its log, so each attempt gets its own. +ATTEMPT_LOG=$LOCAL/base.launchd.$(date -u +%Y%m%dT%H%M%SZ).log +"$LSUBMIT" route-a-rerun-20261018-base "$ATTEMPT_LOG" "$PREP" \ + /usr/bin/env MICROCOSM_MODAL_PLAN="$PLAN" \ + "$MODAL" run --detach tools/modal_us_stage.py --run --spawn +# Wait at most 30 minutes for this attempt's call id. Stop early if the launcher +# exits without one (a refused check or an auth error). +for _ in $(seq 360); do + grep -q '"function_call_id"' "$ATTEMPT_LOG" 2>/dev/null && break + state=$(launchctl print "gui/$(id -u)/com.maxghenis.claude893.route-a-rerun-20261018-base" 2>/dev/null || true) + case "$state" in *"state = running"*|*"state = spawn scheduled"*) ;; *) break ;; esac + sleep 5 +done +FC=$(sed -n 's/.*"function_call_id": "\(fc-[A-Za-z0-9]*\)".*/\1/p' "$ATTEMPT_LOG" | sed -n 1p) +test -n "$FC" || { echo "No call id in $ATTEMPT_LOG; read it before anything else"; exit 1; } +echo "$FC" > "$LOCAL/base.fc" +``` + +`--spawn` returns once the stage is spawned and prints its `function_call_id`. +Run it under `--detach`. Without `--spawn` the client waits on the call, and on +19 October the first attempt (`all-20261019T163840131111Z`) was cancelled +after 822 s, 99 s after the Mac went to sleep on battery. The cancellation is +attributed to the sleep from its timing. A probe that day showed a spawned call +finishing after its client exited; client sleep itself was not tested. + +The launcher's exit code no longer says how the stage ended; the runs volume +does (see step 5). To block on the call instead, use the modal tool's own +interpreter (system `python3` has no `modal`): +`~/.local/share/uv/tools/modal/bin/python -c 'import modal; print(modal.FunctionCall.from_id("").get())'`. +It returns the receipt (COMPLETED or FAILED), or re-raises the error of an +attempt that raised: a rejected plan, a refusal or an exception. +Remove `base.fc` only when its call has ended and a relaunch has Max's go. A launchd job's own exit status is in +`launchctl print gui/$(id -u)/com.maxghenis.claude893.