diff --git a/.github/workflows/validate-deployment.yml b/.github/workflows/validate-deployment.yml index 4a75676..9fff2e8 100644 --- a/.github/workflows/validate-deployment.yml +++ b/.github/workflows/validate-deployment.yml @@ -43,9 +43,9 @@ jobs: # income, liveness) completes in a few minutes. The investment-closure # regression is a real closure solve (minutes on the server) and is the # authoritative HARD GATE in the OBR repo CI (local, pre-deploy); here it - # runs on the weekly Sunday cron and on a manual `deep` dispatch. force_build guarantees the deployed - # model is always freshly cloned, so the fast suite is enough to confirm a - # functional deploy on the automatic path. + # runs on the weekly Sunday cron and on a manual `deep` dispatch. Deployment + # fetches the reviewed commits recorded in modal_app.py, so the fast suite + # is enough to confirm a functional deploy on the automatic path. # 35 min: the slow leg now includes the OBR bridge end-to-end test (up to # ~9 min against a cold server) on top of the closure probe (~7 min); the # 20-min cap already timed out once on 2026-07-17. diff --git a/integration/modal_app.py b/integration/modal_app.py index 0971281..b7cbf2f 100644 --- a/integration/modal_app.py +++ b/integration/modal_app.py @@ -118,13 +118,27 @@ FRB_REPO = f"{HOME}/us-frb-model" INTEGRATION = str(Path(__file__).parent) -# POLICYENGINE_MACRO_IMAGE_SOURCE=github (used by CI) clones the model repos from -# GitHub main at image-build time instead of copying the local checkouts — -# same absolute paths, so all data-file resolution is unchanged. +# POLICYENGINE_MACRO_IMAGE_SOURCE=github (used by CI) fetches reviewed model +# commits instead of copying local checkouts. The absolute paths stay the same, +# so all data-file resolution is unchanged. GITHUB_SOURCE = os.environ.get("POLICYENGINE_MACRO_IMAGE_SOURCE") == "github" OBR_URL = "https://github.com/PolicyEngine/obr-macroeconomic-model" BOE_URL = "https://github.com/PolicyEngine/boe-var-model" FRB_URL = "https://github.com/PolicyEngine/us-frb-model" +HANK_URL = "https://github.com/PolicyEngine/us-hank-model" +OBR_REVISION = "85ada9be070a34a4a4a0a65d92f8f148a476ccb1" +BOE_REVISION = "3ab56259224c20a0bf65caa523e6e7440e56b37e" +FRB_REVISION = "ca80394c99e9aeae410723ef3f51b9fdf9b84a77" +HANK_REVISION = "b9b3130d47d02edd07668531dfb4fce54f241b9b" +SOURCE_REVISION_ENV = { + "POLICYENGINE_MACRO_SOURCE_REVISION_HANK": HANK_REVISION, +} +if GITHUB_SOURCE: + SOURCE_REVISION_ENV.update({ + "POLICYENGINE_MACRO_SOURCE_REVISION_OBR": OBR_REVISION, + "POLICYENGINE_MACRO_SOURCE_REVISION_BOE": BOE_REVISION, + "POLICYENGINE_MACRO_SOURCE_REVISION_FRB": FRB_REVISION, + }) # Keep the image lean: skip the 412MB dashboard, VCS, caches, docs. # "**/.venv" matters more than it looks: a local checkout that has been set up @@ -214,21 +228,29 @@ # Tarball URL, not git+: this pip layer runs before (and without) # apt_install("git"), so a VCS requirement fails with "Cannot find # command 'git'" during the Modal image build. - "us-hank-model @ https://github.com/PolicyEngine/us-hank-model/archive/refs/heads/main.tar.gz", + f"us-hank-model @ {HANK_URL}/archive/{HANK_REVISION}.tar.gz", ) ) if GITHUB_SOURCE: - # force_build=True: the clone command string never changes, so without it - # Modal caches this layer and every redeploy reuses a STALE checkout — - # model fixes merged to OBR/BoE main would never reach production. Forcing a - # rebuild re-clones the current main on each github-source deploy (~10s). + # Fetch reviewed commits rather than mutable branches. Updating a model is + # an explicit source change, and the revision is also returned in every + # adapter's provenance. image = image.apt_install("git").run_commands( - f"git clone --depth 1 {OBR_URL} {OBR_REPO}", - f"git clone --depth 1 {BOE_URL} {BOE_REPO}", - f"git clone --depth 1 {FRB_URL} {FRB_REPO}", + f"mkdir -p {HOME}", + f"git init {OBR_REPO}", + f"git -C {OBR_REPO} remote add origin {OBR_URL}", + f"git -C {OBR_REPO} fetch --depth 1 origin {OBR_REVISION}", + f"git -C {OBR_REPO} checkout --detach FETCH_HEAD", + f"git init {BOE_REPO}", + f"git -C {BOE_REPO} remote add origin {BOE_URL}", + f"git -C {BOE_REPO} fetch --depth 1 origin {BOE_REVISION}", + f"git -C {BOE_REPO} checkout --detach FETCH_HEAD", + f"git init {FRB_REPO}", + f"git -C {FRB_REPO} remote add origin {FRB_URL}", + f"git -C {FRB_REPO} fetch --depth 1 origin {FRB_REVISION}", + f"git -C {FRB_REPO} checkout --detach FETCH_HEAD", _FRB_PRUNE, - force_build=True, ) else: image = ( @@ -247,6 +269,7 @@ image = ( image + .env(SOURCE_REVISION_ENV) .add_local_dir(INTEGRATION, remote_path=f"{HOME}/macro/integration", copy=True, ignore=_IGNORE + ["modal_app.py"]) # Editable installs keep each package's __file__ inside its repo, so the diff --git a/integration/pyproject.toml b/integration/pyproject.toml index 55f3933..b960097 100644 --- a/integration/pyproject.toml +++ b/integration/pyproject.toml @@ -20,10 +20,10 @@ dependencies = [ # particular, frbus includes model.xml and LONGBASE.TXT inside its wheel, so a # clean non-editable installation is a supported access path. models = [ - "obr-macro-model @ git+https://github.com/PolicyEngine/obr-macroeconomic-model", - "boe_var @ git+https://github.com/PolicyEngine/boe-var-model", - "frbus @ git+https://github.com/PolicyEngine/us-frb-model", - "us-hank-model @ git+https://github.com/PolicyEngine/us-hank-model", + "obr-macro-model @ git+https://github.com/PolicyEngine/obr-macroeconomic-model@85ada9be070a34a4a4a0a65d92f8f148a476ccb1", + "boe_var @ git+https://github.com/PolicyEngine/boe-var-model@3ab56259224c20a0bf65caa523e6e7440e56b37e", + "frbus @ git+https://github.com/PolicyEngine/us-frb-model@ca80394c99e9aeae410723ef3f51b9fdf9b84a77", + "us-hank-model @ git+https://github.com/PolicyEngine/us-hank-model@b9b3130d47d02edd07668531dfb4fce54f241b9b", # [models] pulls the pinned policyengine-uk / policyengine-us country # packages that pe.uk / pe.us need at runtime. "policyengine[models]>=4,<5", diff --git a/integration/src/policyengine_macro/core.py b/integration/src/policyengine_macro/core.py index 728c28d..78a9f34 100644 --- a/integration/src/policyengine_macro/core.py +++ b/integration/src/policyengine_macro/core.py @@ -47,9 +47,29 @@ def _provenance( "og-uk": "https://github.com/PolicyEngine/og-uk", "pe-microsim": "https://github.com/PolicyEngine/policyengine", "obr-macro": "https://github.com/PolicyEngine/obr-macroeconomic-model", + "boe-svar": "https://github.com/PolicyEngine/boe-var-model", + "frb-us": "https://github.com/PolicyEngine/us-frb-model", "og+microsim": "https://github.com/PolicyEngine/macro", "us-hank": "https://github.com/PolicyEngine/us-hank-model", } + revision_env = { + "obr-macro": "POLICYENGINE_MACRO_SOURCE_REVISION_OBR", + "boe-svar": "POLICYENGINE_MACRO_SOURCE_REVISION_BOE", + "frb-us": "POLICYENGINE_MACRO_SOURCE_REVISION_FRB", + "us-hank": "POLICYENGINE_MACRO_SOURCE_REVISION_HANK", + } + source_revision = os.environ.get(revision_env.get(model_id, "")) + if not source_revision: + source_revision = f"installed {distribution} {package_version}" + reproducibility = ( + "Check out the recorded source URL at the recorded source revision, " + "install the recorded adapter version, and rerun the serialized " + "request against the recorded baseline and data vintage." + if re.fullmatch(r"[0-9a-f]{40}", source_revision) + else + "Install the recorded package versions and rerun the serialized " + "request against the recorded baseline and data vintage." + ) return { "model_id": model_id, "package": distribution, @@ -57,16 +77,13 @@ def _provenance( "model_version": package_version, "adapter_version": _package_version("policyengine-macro"), "source_url": source_urls.get(model_id, "https://github.com/PolicyEngine/macro"), - "source_revision": f"installed {distribution} {package_version}", + "source_revision": source_revision, "data_vintage": data_vintage, "baseline_vintage": baseline, "baseline": baseline, "estimation_sample": estimation_sample, "run_at": datetime.now(timezone.utc).isoformat(), - "reproducibility": ( - "Install the recorded package versions and rerun the serialized " - "request against the recorded baseline and data vintage." - ), + "reproducibility": reproducibility, } diff --git a/integration/tests/test_core.py b/integration/tests/test_core.py index 1afd045..ce10b4f 100644 --- a/integration/tests/test_core.py +++ b/integration/tests/test_core.py @@ -42,6 +42,22 @@ def test_latest_shocks_rejects_unsafe_draws_before_import(draws): core.svar_latest_shocks(draws=draws) +def test_provenance_uses_exact_deployed_source_revision(monkeypatch): + revision = "a" * 40 + monkeypatch.setenv("POLICYENGINE_MACRO_SOURCE_REVISION_HANK", revision) + provenance = core._provenance( + model_id="us-hank", + distribution="us-hank-model", + data_vintage="paper calibration", + baseline="steady state", + ) + assert provenance["source_revision"] == revision + assert provenance["source_url"] == ( + "https://github.com/PolicyEngine/us-hank-model" + ) + assert "recorded source revision" in provenance["reproducibility"] + + def test_summary_parses(): s = core.svar_summary() if "error" in s: diff --git a/integration/tests/test_source_pins.py b/integration/tests/test_source_pins.py new file mode 100644 index 0000000..0b59da1 --- /dev/null +++ b/integration/tests/test_source_pins.py @@ -0,0 +1,43 @@ +"""Deployment source revisions must be exact and synchronized.""" + +import ast +from pathlib import Path +import re + + +INTEGRATION = Path(__file__).resolve().parents[1] +PIN_NAMES = ( + "OBR_REVISION", + "BOE_REVISION", + "FRB_REVISION", + "HANK_REVISION", +) + + +def _modal_pins() -> dict[str, str]: + tree = ast.parse((INTEGRATION / "modal_app.py").read_text()) + pins: dict[str, str] = {} + for node in tree.body: + if not isinstance(node, ast.Assign) or len(node.targets) != 1: + continue + target = node.targets[0] + if ( + isinstance(target, ast.Name) + and target.id in PIN_NAMES + and isinstance(node.value, ast.Constant) + ): + pins[target.id] = node.value.value + return pins + + +def test_modal_model_sources_are_full_git_revisions(): + pins = _modal_pins() + assert set(pins) == set(PIN_NAMES) + for revision in pins.values(): + assert re.fullmatch(r"[0-9a-f]{40}", revision) + + +def test_optional_model_dependencies_use_deployment_revisions(): + pyproject = (INTEGRATION / "pyproject.toml").read_text() + for revision in _modal_pins().values(): + assert f"@{revision}" in pyproject