Prepare the d1386 Route A rerun: CT-ladder base plan, fresh lineage evidence, launch-day sequence

docs/us-modal-stage-route-a-rerun-base-plan.json

@@ -0,0 +1,61 @@
+{
+  "schema": "microcosm-modal-us-stage-plan/1",
+  "tool": "us-puf-support-base-rerun",
+  "stage": "all",
+  "run_id": "route-a-rerun-base-20261018-pending",
+  "source": {
+    "commit": "PICK_MAIN_COMMIT_ON_2026_10_18",
+    "branch": "main"
+  },
+  "inputs": {
+    "asec_2024_h5": {
+      "uri": "volume://cas/sha256/ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d/census_cps_2024.h5",
+      "sha256": "ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d"
+    },
+    "asec_2023_h5": {
+      "uri": "volume://cas/sha256/cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88/census_cps_2023.h5",
+      "sha256": "cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88"
+    },
+    "asec_2022_h5": {
+      "uri": "volume://cas/sha256/7ccca976284bb47815d84460cc4f75a0a65d26d7754ab0a0f417de351b3d474e/census_cps_2022.h5",
+      "sha256": "7ccca976284bb47815d84460cc4f75a0a65d26d7754ab0a0f417de351b3d474e"
+    },
+    "puf_2024_h5": {
+      "uri": "volume://cas/sha256/7669f5b5281f20080e77204f9bd4aabfad0aa101fa283e22caf9ba8d61d4d6df/puf_2024.h5",
+      "sha256": "7669f5b5281f20080e77204f9bd4aabfad0aa101fa283e22caf9ba8d61d4d6df"
+    },
+    "puf_2015_csv": {
+      "uri": "volume://cas/sha256/0a7fd643edb1acc55c507db795914b41d232922be78c149b58d111f4672499df/puf_2015.csv",
+      "sha256": "0a7fd643edb1acc55c507db795914b41d232922be78c149b58d111f4672499df"
+    },
+    "acs_2022_h5": {
+      "uri": "volume://cas/sha256/0b319b496f19a6913066f9c5ea572edfda3d78a187be6f375846617d0b441bd4/acs_2022.h5",
+      "sha256": "0b319b496f19a6913066f9c5ea572edfda3d78a187be6f375846617d0b441bd4"
+    },
+    "asec_education_2022_zip": {
+      "uri": "volume://cas/sha256/d2e000250782adfbdd7f29c82b66d866591a30f0d330496698ec19f9c784ce11/asecpub23csv.zip",
+      "sha256": "d2e000250782adfbdd7f29c82b66d866591a30f0d330496698ec19f9c784ce11"
+    },
+    "asec_education_2023_zip": {
+      "uri": "volume://cas/sha256/cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09/asecpub24csv.zip",
+      "sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09"
+    },
+    "asec_education_2024_zip": {
+      "uri": "volume://cas/sha256/318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b/asecpub25csv.zip",
+      "sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b"
+    },
+    "base_ledger_facts": {
+      "uri": "volume://cas/sha256/b85437390021777e746f507c5890305496baf5fc7f2c78ba08ddb090f4839801/consumer_facts_us_c5e5bf8.jsonl",
+      "sha256": "b85437390021777e746f507c5890305496baf5fc7f2c78ba08ddb090f4839801"
+    },
+    "block_ladder_npz": {
+      "uri": "volume://cas/sha256/6840b990acdfa2003d7723da5595e3cfff7205a4fa3d8d6206ed835c85c3f233/us_block_ladder_2020.npz",
+      "sha256": "6840b990acdfa2003d7723da5595e3cfff7205a4fa3d8d6206ed835c85c3f233"
+    }
+  },
+  "env": {
+    "PYTHONUNBUFFERED": "1"
+  },
+  "max_wall_seconds": 14400,
+  "nonpreemptible": true
+}

docs/us-modal-stage-runbook.md

@@ -3,12 +3,15 @@
 Epic #956 acceleration item E: heavy US stages should not have to queue on
 the one 128 GiB build machine. This runbook covers the smallest working path:
 a registered tool run on Modal from a pinned commit, with inputs fetched by
-digest and outputs listed with sha256 receipts. Four tools are registered
+digest and outputs listed with sha256 receipts. Five tools are registered
 (`TOOLS` in the plan module): `tools/build_us_acs_local_release.py` (tool
 `us-acs-local-release`, stages `materialize`, `calibrate`, `qa`,
 `finalize`, `package`, or `all`), Route A's PUF-support base,
 `tools/build_us_puf_support_base.py` (tool `us-puf-support-base`, stage
-`all`; see "Route A's base stage"), the ACS multispine staging builder,
+`all`; see "Route A's base stage"), its current-main rerun registration
+`us-puf-support-base-rerun` (same script and stage, with the reviewed current
+district crosswalk; see [the 18 October rerun](us-route-a-rerun-20261018.md)),
+the ACS multispine staging builder,
 `tools/build_us_acs_multispine_base.py` (tool `us-acs-multispine-staging`,
 stage `all`; see "ACS multispine staging"), and `runner-smoke` (stage
 `smoke`), an inline script that proves the run path on any pushed commit (see
@@ -220,8 +223,13 @@ python3 tools/modal_us_stage_plan.py validate plan.json
 #    attempt of the run that may still be running.
 MICROCOSM_MODAL_PLAN=plan.json modal run tools/modal_us_stage.py
 
-# 4. Run the stage. --detach keeps it running if this terminal goes away;
-#    the receipt lands on the runs volume either way. Set "max_wall_seconds"
+# 4. Run the stage. --spawn returns once the stage is spawned, and under
+#    --detach a probe on 2026-10-19 saw the spawned call finish after its
+#    client exited (client sleep was not tested). Without --spawn the client waits on the call, and
+#    on 2026-10-19 a waiting call was cancelled 99 s after its Mac went to
+#    sleep. Read the outcome from the runs volume: a receipt under
+#    runs/<run_id>/receipts/, or an attempt record whose outcome is refused or
+#    error (see modal_us_stage.main). Set "max_wall_seconds"
 #    in the plan to cap the cost below the class's hard timeout: the runner
 #    stops the tool then (SIGTERM to the tool's process group, which the
 #    tool runs in alone, so a stage child it spawned stops too; SIGKILL to
@@ -237,7 +245,7 @@ MICROCOSM_MODAL_PLAN=plan.json modal run tools/modal_us_stage.py
 #    comes off it (see "Preemption and restarts" below). A stop at the
 #    budget writes a receipt, so launching the same plan again gets the
 #    whole budget again.
-MICROCOSM_MODAL_PLAN=plan.json modal run --detach tools/modal_us_stage.py --run
+MICROCOSM_MODAL_PLAN=plan.json modal run --detach tools/modal_us_stage.py --run --spawn
 
 # 5. Fetch the state and verify it against the receipt. A receipt whose
 #    status is not COMPLETED, or that says stopped_at_budget, does not
@@ -339,7 +347,7 @@ python3 tools/modal_us_stage_plan.py digest <that dir>/acs_multispine_staging.h5
 # expect f335f573…; then run the printed `modal volume put …` line
 MICROCOSM_MODAL_PLAN=docs/us-modal-stage-example-plan.json modal run tools/modal_us_stage.py
 MICROCOSM_MODAL_PLAN=docs/us-modal-stage-example-plan.json \
-  modal run --detach tools/modal_us_stage.py --run
+  modal run --detach tools/modal_us_stage.py --run --spawn
 ```
 
 ## Acceptance attempt: 23 September materialize on the state surface
@@ -443,6 +451,14 @@ across containers would let preemption lose one chunk instead of the run.
 
 ## Route A's base stage
 
+For Max's d1386 rerun on **18 October 2026**, use
+[the rerun preparation and launch sequence](us-route-a-rerun-20261018.md) and
+`us-modal-stage-route-a-rerun-base-plan.json`. Its commit is intentionally
+unresolved. That procedure separates the historical base feed from main's
+new release feed, pins the current in-tree crosswalk with a scoped registration,
+and requires fresh local 000/001 evidence for the selected commit and CT ladder.
+The historical plan/reference below cannot qualify that rerun.
+
 On 29 September 2026 Max ruled to run only Route A's base stage on Modal:
 non-preemptible, capped at 4 hours of wall, about $15. The release, its
 preflight and certification stay on the build machine.
@@ -527,7 +543,7 @@ Two kinds of test hold the command:
   command with its paths tokenized
   (`packages/microcosm-build/tests/fixtures/modal_us_stage/route_a_base_command_4b57d15a287c.json`),
   and the registration's owned flags equal the flags of this tree's tool.
-- Only in a clone that has commit `4b57d15a2`, four tests in
+- Only in a clone that has commit `4b57d15a2`, five tests in
   `test_us_modal_stage_puf_support_base.py` that read that commit with
   `git show` and skip without it:
   - `test_the_tool_s_own_parser_reads_the_same_build_from_both_commands`:
@@ -543,10 +559,13 @@ Two kinds of test hold the command:
     its `_stage_run_config` locks, from the Modal argv, the run config the
     local run locked (paths by file name), including `PYTHONHASHSEED` `0`
     when the runner's tool environment removed a container's value, and a
-    forced `12345` is what compare-lineage refuses.
+    forced `12345` is what compare-lineage refuses;
+  - `test_rerun_crosswalk_refuses_the_historical_tree`: the rerun registration
+    refuses that commit's historical crosswalk bytes while retaining its
+    separately reviewed current-main pin.
 
 CI checks out only the commit under test (`actions/checkout` fetches one
-commit by default), so there those four tests report a skip rather than
+commit by default), so there those five tests report a skip rather than
 pass. Run them in a full clone before a paid run (step 2 below).
 
 The plan is `docs/us-modal-stage-route-a-base-plan.json` (run
@@ -825,7 +844,7 @@ python3 tools/modal_us_stage_plan.py upload-commands --shell \
 # 2. Validate locally: the argv, class base (cpu_limit 4.0), the $14.63
 #    estimate and the $14.90 ceiling at the request
 #    (estimated_usd_at_timeout). Then run the base's tests, and confirm the
-#    four pinned-commit tests passed rather than skipped (-rs lists skips).
+#    five pinned-commit tests passed rather than skipped (-rs lists skips).
 python3 tools/modal_us_stage_plan.py validate docs/us-modal-stage-route-a-base-plan.json
 uv run pytest -rs packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py
 
@@ -849,7 +868,7 @@ MICROCOSM_MODAL_PLAN=docs/us-modal-stage-route-a-base-plan.json \
 #    throttles CPU use above the limit (the request), and memory above
 #    112 GiB would bill above that.
 MICROCOSM_MODAL_PLAN=docs/us-modal-stage-route-a-base-plan.json \
-  modal run --detach tools/modal_us_stage.py --run
+  modal run --detach tools/modal_us_stage.py --run --spawn
 
 # 5. Fetch the receipts and base-out only, not the 44 GB of checkpoints,
 #    and verify base-out against the receipt. verify-receipt refuses a
@@ -1226,7 +1245,7 @@ bash staging-inputs.sh
 # disk admission, write probe, PYTHONHASHSEED, ledger and receipt state.
 MICROCOSM_MODAL_PLAN="$PLAN" modal run tools/modal_us_stage.py
 # Run: the plan requests non-preemptible placement and a 16-hour budget.
-MICROCOSM_MODAL_PLAN="$PLAN" modal run --detach tools/modal_us_stage.py --run
+MICROCOSM_MODAL_PLAN="$PLAN" modal run --detach tools/modal_us_stage.py --run --spawn
 
 # After completion, fetch the entire state (including logs) and receipts.
 R=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["run_id"])' "$PLAN")

docs/us-route-a-rerun-20261018.env.example

@@ -0,0 +1,69 @@
+# shellcheck shell=bash
+# d1386 preparation, 2026-10-15. Copy to the machine-local 20261018 directory.
+# COMMIT is deliberately empty. Recheck the marked settings on launch day.
+# Copy to route_a.env, or point ROUTE_A_ENV at a machine-local settings file.
+# No credentials belong here. Environment overrides win over these defaults.
+# All machine locations are configured here; the driver uses colocated helpers.
+: "${PE:=/Users/maxghenis/PolicyEngine}"
+: "${MAIN:=$PE/microcosm}"
+: "${WT_ROOT:=$PE/_worktrees}"
+# Use a fresh root for a new attempt. Do not reuse a pre-CT base's ACCEPTED marker.
+: "${RUN_ROOT:=$PE/_recovered/scratch-backup/893/route-a-rerun-20261018/runs}"
+: "${CHAIN_ROOT:=$PE/_recovered/scratch-backup/893/overnight-20260923}"
+: "${CHAIN_LOG:=$CHAIN_ROOT/chain.log}"
+: "${DISK_PATH:=/System/Volumes/Data}"
+: "${UV:=$HOME/.local/bin/uv}"
+: "${PYTHON:=python3}"
+: "${AGENT_SECRET:=$HOME/bin/agent-secret}"
+: "${STORAGE:=$PE/policyengine-us-data/policyengine_us_data/storage}"
+: "${EDU:=$PE/_buildm-runtime/inputs/asec_education}"
+: "${FEED:=$PE/_buildh-runtime/inputs/consumer_facts_us_51aa40f.jsonl}"
+# The driver preserves the published feed/crosswalk pins by default. If the
+# chosen commit changed them, set FEED_SHA, FEED_BYTES and CROSSWALK_SHA here
+# alongside FEED, after checking that commit's packaged pins.
+# d713: CT-fixed ladder; route_a.sh verifies its digest and size.
+: "${LADDER:=$PE/_build_artifacts/us-ct-cbsa/us_block_ladder_2020.npz}"
+: "${SSI:=$PE/_buildo-runtime/inputs/attempt6_basis_schema3_seed.json}"
+: "${SCF:=$PE/_buildm-runtime/inputs/scf_cache/rscfp2022.dta}"
+
+# Choose a pushed build commit. Empty COMMIT refuses a real run.
+: "${COMMIT:=}"
+: "${REQUIRE_ON_MAIN:=1}"
+: "${REQUIRE_SPM_ROLE:=1}"
+: "${PREFLIGHT_COMMIT:=}"
+: "${PREFLIGHT_NEW_LINEAGE_ARGS:=--new-lineage}"
+: "${EARLY_START_AFTER_HEAVY:=0}"
+: "${PRUNE_BASE_CHECKPOINTS:=1}"
+: "${RELEASE_DESPITE_PREFLIGHT_FAIL:=0}"
+: "${RESOURCE_WAIT_HOURS:=12}"
+: "${RELEASE_RAM_GIB:=50}"
+: "${RELEASE_WALL_SECONDS:=345600}"
+# 0 is the sole opt-out and restores --no-staging without a credential lookup.
+: "${ROUTE_A_STAGING=1}"
+
+# Retain d122 whole-base dense national/state settings and the published run's
+# export-mass reference. Review the historical tail waiver against the new export.
+: "${DENSE_RELEASE_D122:=1}"
+: "${EXPORT_MASS_REF_H5:=$PE/_buildg-runtime/forensics/populace_us_2024.h5}"
+: "${RELEASE_EXTRA_ARGS:=--maximum-microsim-batch-size 2000 --target-surface national_state --dense-default-dataset --export-input-mass-reference-h5 $EXPORT_MASS_REF_H5}"
+: "${QRF_TAIL_EXCLUSIONS:=$CHAIN_ROOT/route-a/qrf_tail_exclusions_routea_d490_20260930.json}"
+: "${QRF_TAIL_EXCLUSIONS_SHA:=fa5760993ebcd1386f02c0b1c4c4e867a0c4b1e689a33774a41f623f95132d69}"
+# Optional base feed override: BASE_LEDGER_FACTS and BASE_LEDGER_FACTS_SHA.
+
+# REVIEW: selected main must still pin this release feed. It changed since 4b57d15a2.
+: "${FEED_SHA:=965a29ac9458edb58a05184917ee4a19772fd3dc7b4588db6eb6da5803641aac}"
+: "${FEED_BYTES:=169038757}"
+# Keep the historical base feed explicitly; its CD facts are unchanged by #940.
+: "${BASE_LEDGER_FACTS:=$PE/_buildh-runtime/inputs/consumer_facts_us_c5e5bf8.jsonl}"
+: "${BASE_LEDGER_FACTS_SHA:=b85437390021777e746f507c5890305496baf5fc7f2c78ba08ddb090f4839801}"
+# REVIEW: selected main must still carry the NC-fixed in-tree crosswalk.
+: "${CROSSWALK_SHA:=a347303fff3fea145f758488c43cb94355df5a8cb5e512552b2ee3791d2aa233}"
+# REVIEW: historical export-mass reference is inherited, not re-certified here.
+# REVIEW: re-measure the register on the new lineage. Update file and SHA together.
+# d1711 (Max 2026-10-19): d490 covers this build. Threshold 0.75, same or fewer
+# columns, on the same terms, renewed to 2026-12-31 or the next certified build.
+# A new column stops for Max.
+# REVIEW: d122 dense/national_state, batch 2000, RAM/wall/wait limits are inherited.
+# REVIEW: PREFLIGHT_COMMIT defaults to COMMIT; keep both on the chosen main tree.
+# Explicit staging destination; no credential value is stored here.
+export POPULACE_STAGING_REPO_ID=policyengine/populace-us-staging