From eda7fdadd1937472d3b1e9e0f0cdb8bfaccd1f02 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 11 Oct 2026 04:20:05 +0800 Subject: [PATCH 01/16] ci: fail the suite on compile-environment skips by named reason --- .github/workflows/gate.yml | 12 +- .github/workflows/test.yml | 69 +++++++- tests/ci_env_skip_allowance.json | 18 ++ tests/conftest.py | 22 +++ tests/skip_gate.py | 294 +++++++++++++++++++++++++++++++ 5 files changed, 413 insertions(+), 2 deletions(-) create mode 100644 tests/ci_env_skip_allowance.json create mode 100644 tests/conftest.py create mode 100644 tests/skip_gate.py diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index 698c132..861cd99 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -125,8 +125,18 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install with dev extras run: python -m pip install -e ".[dev]" + - name: Assert the compile-environment skip gate is enabled + run: | + python - <<'PY' + import os + import sys + value = os.environ.get("PINEFORGE_SKIP_GATE", "").strip().lower() + if value == "off": + sys.stderr.write("PINEFORGE_SKIP_GATE=off is not allowed in CI: the skip gate must run\n") + sys.exit(1) + PY - name: Run test suite - run: python -m pytest -q + run: python -m pytest -q -rs gate: if: always() diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c430086..0f844be 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -85,5 +85,72 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install with dev extras run: python -m pip install -e ".[dev]" + - name: Assert the compile-environment skip gate is enabled + run: | + python - <<'PY' + import os + import sys + value = os.environ.get("PINEFORGE_SKIP_GATE", "").strip().lower() + if value == "off": + sys.stderr.write("PINEFORGE_SKIP_GATE=off is not allowed in CI: the skip gate must run\n") + sys.exit(1) + PY - name: Run test suite - run: python -m pytest -q + run: python -m pytest -q -rs + + engine-built: + needs: preflight-docs + if: needs.preflight-docs.outputs.docs_only != 'true' + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + PYTHONHASHSEED: "0" + ENGINE_COMMIT: 4e01e96b5165b5d5e46d72bfbbf88ae6c035a6e9 + PINEFORGE_SKIP_PROFILE: engine_built + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: "3.14" + - name: Install with dev extras + run: python -m pip install -e ".[dev]" + - name: Install the compile toolchain and Eigen + run: | + sudo apt-get update + sudo apt-get install -y libeigen3-dev clang cmake g++ ninja-build + - name: Clone and build the paired engine + run: | + set -euo pipefail + git clone https://github.com/pineforge-4pass/pineforge-engine.git engine + git -C engine checkout "$ENGINE_COMMIT" + cmake -B build -S engine -DCMAKE_BUILD_TYPE=Release + cmake --build build -j"$(nproc)" + lib="$(find "$PWD/build" -name libpineforge.a -print -quit)" + if [ -z "$lib" ]; then + echo "the engine build produced no libpineforge.a" >&2 + exit 1 + fi + echo "libpineforge.a at: $lib" + echo "PINEFORGE_ENGINE_LIB=$lib" >> "$GITHUB_ENV" + echo "PINEFORGE_ENGINE_INCLUDE=$PWD/engine/include" >> "$GITHUB_ENV" + - name: Assert the built library exists + run: | + set -euo pipefail + test -f "$PINEFORGE_ENGINE_LIB" || { echo "no library at $PINEFORGE_ENGINE_LIB" >&2; exit 1; } + test -f "$PINEFORGE_ENGINE_INCLUDE/pineforge/engine.hpp" || { echo "no engine headers under $PINEFORGE_ENGINE_INCLUDE" >&2; exit 1; } + - name: Assert the compile-environment skip gate is enabled + run: | + python - <<'PY' + import os + import sys + value = os.environ.get("PINEFORGE_SKIP_GATE", "").strip().lower() + if value == "off": + sys.stderr.write("PINEFORGE_SKIP_GATE=off is not allowed in CI: the skip gate must run\n") + sys.exit(1) + PY + - name: Run the test suite with the engine present + run: python -m pytest -q -rs diff --git a/tests/ci_env_skip_allowance.json b/tests/ci_env_skip_allowance.json new file mode 100644 index 0000000..9220bcf --- /dev/null +++ b/tests/ci_env_skip_allowance.json @@ -0,0 +1,18 @@ +{ + "_note": "Allowance for the compile-environment skip gate (tests/skip_gate.py). Keys are the EXACT skip reason text (not a normalised class) so a renamed reason is drift too. 'default' is the profile the engine-less hosted-CI pytest job runs; 'engine_built' is the profile the engine-built job runs and lists every reason at 0, so any compile-environment skip fails it. UNMEASURED: every count below is a placeholder written without a spot run; the supervisor replaces them from the measured run before the PR opens, and the gate fails loudly on any mismatch. Reasons were read from the pytest.skip(...) / pytestmark skipif calls in tests/.", + "_unmeasured": true, + "default": { + "No built libpineforge.a. Set PINEFORGE_ENGINE_LIB to the archive (or configure a build*/lib/ beside the engine checkout) to run the emitted translation unit.": 1131, + "built libpineforge not found; set PINEFORGE_ENGINE_LIB": 20, + "deriving the host members needs clang's AST dump; no clang++ found": 1, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 0 + }, + "engine_built": { + "No built libpineforge.a. Set PINEFORGE_ENGINE_LIB to the archive (or configure a build*/lib/ beside the engine checkout) to run the emitted translation unit.": 0, + "built libpineforge not found; set PINEFORGE_ENGINE_LIB": 0, + "deriving the host members needs clang's AST dump; no clang++ found": 0, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 0 + } +} diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..83bc6dc --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,22 @@ +"""pytest configuration for the pineforge-codegen test suite. + +Loads the compile-environment skip gate (:mod:`tests.skip_gate`), which fails +the session at the end when a skipped test's exact reason is not listed in +``tests/ci_env_skip_allowance.json`` (or its count differs). Without this a +machine with no engine library, Eigen or clang skips every test that compiles +an emitted translation unit, pytest exits 0, and the suite looks green while a +third of it never ran. + +The gate is ON by default; ``PINEFORGE_SKIP_GATE=off`` disables it, and the CI +workflow asserts that variable is not set the gate cannot be turned off +silently. +""" + +from __future__ import annotations + +from tests import skip_gate + +# Re-exported so pytest registers the session-end hook from this conftest. +from tests.skip_gate import pytest_sessionfinish # noqa: F401 + +__all__ = ["pytest_sessionfinish", "skip_gate"] diff --git a/tests/skip_gate.py b/tests/skip_gate.py new file mode 100644 index 0000000..20b6f24 --- /dev/null +++ b/tests/skip_gate.py @@ -0,0 +1,294 @@ +"""Session-end pytest gate that fails the run on compile-environment skips. + +Hosted CI runs ``python -m pytest -q`` on a machine with no engine library, +no Eigen and no clang, so every test that compiles an emitted translation +unit *skips*. Because they skip, pytest exits 0 and the badge is green while +roughly a third of the suite never runs. A gate that can pass on missing +input is not a gate. + +This module turns the skips into a verdict. At session end it collects every +skipped test's reason from the terminal reporter, normalises each reason to a +class (a reason is in the COMPILE-ENVIRONMENT class when it mentions a +compile-environment knob, see ``COMPILE_ENV_PATTERNS``), and compares the +*exact* reason text against the allowance file +``tests/ci_env_skip_allowance.json``: + + {"default": {"": , ...}, + "engine_built": {"": 0, ...}} + +The exact reason TEXT -- not the class -- is the key, so a reason that was +reworded is drift too. The profile is chosen by ``PINEFORGE_SKIP_PROFILE`` +(default ``default``); the CI ``engine-built`` job sets ``engine_built``. + +The session FAILS (non-zero exit) when: + + (a) a COMPILE-ENVIRONMENT skip's exact reason is not a key of the profile; + (b) a listed reason's count differs from what was actually observed; + (c) a listed reason with count > 0 was NOT observed (a stale allowance); + (d) the session reported no skip summary at all, while the profile lists + any reason with count > 0 (a plugin that never ran is not a gate). + +The gate is ON by default and may be turned off only with +``PINEFORGE_SKIP_GATE=off`` -- the CI workflow asserts that variable is not +set, because a gate that can be disabled silently would defeat the point. +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +# The allowance file lives beside this module. +ALLOWANCE_PATH = Path(__file__).resolve().parent / "ci_env_skip_allowance.json" + +# A skip reason is in the COMPILE-ENVIRONMENT class when it mentions any of +# these knobs (case-insensitive substring match). The list is deliberately +# exhaustive about *wording families*: the engine library archive, the Eigen +# and generated headers, the compiler/AST tooling, the parity corpus and the +# population-sources directory a compile or byte-comparison test needs. +COMPILE_ENV_PATTERNS = ( + "No built libpineforge.a", + "set PINEFORGE_ENGINE_LIB", + "no clang++", + "PINEFORGE_ENGINE_CORPUS", + "PINEFORGE_STRING_SETTINGS_SOURCES", + "population sources require", +) + +CLASS_COMPILE_ENV = "COMPILE-ENVIRONMENT" + +GATE_ENV = "PINEFORGE_SKIP_GATE" +PROFILE_ENV = "PINEFORGE_SKIP_PROFILE" +DEFAULT_PROFILE = "default" + + +def gate_disabled(env=None): + """True when the gate is turned off via ``PINEFORGE_SKIP_GATE=off``. + + This is the single source of truth for the disable switch: the workflow + assertion imports and calls it, so the CI check and the gate can never + disagree about the normalization (a shell reimplementation could, e.g. + on whitespace or a non-ASCII case). + """ + env = os.environ if env is None else env + return str(env.get(GATE_ENV, "")).strip().lower() == "off" + + +def classify(reason): + """Return the class of a skip reason, or ``None`` when it is not compile-env. + + Matching is a case-insensitive substring test against + ``COMPILE_ENV_PATTERNS`` (the allowance keys on the *exact* reason text; + this only decides which reasons the gate must see in the file). + """ + if not reason: + return None + text = str(reason).lower() + for pattern in COMPILE_ENV_PATTERNS: + if pattern.lower() in text: + return CLASS_COMPILE_ENV + return None + + +def load_allowance(path=ALLOWANCE_PATH): + """Read the allowance JSON, or return ``None`` when it is missing/unreadable.""" + path = Path(path) + if not path.is_file(): + return None + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + + +def profile_entries(allowance, profile_name): + """Return ``{reason: count}`` for a profile, or ``None`` when it is absent. + + Keys beginning with ``_`` are file metadata (a ``_note`` comment field) + and are ignored, so the allowance can carry human notes without polluting + the reason set. + """ + if not isinstance(allowance, dict): + return None + profile = allowance.get(profile_name) + if not isinstance(profile, dict): + return None + return {k: v for k, v in profile.items() if not str(k).startswith("_")} + + +def skip_reason(report): + """Extract the reason text from a skipped ``TestReport``. + + pytest records a skip's ``longrepr`` as the 3-tuple + ``(location, lineno, reason)``; some plugins hand back a plain string. + Both are normalised to the reason text, with any ``"Skipped: "`` display + prefix stripped, so the same skip compares equal however it is reported. + """ + longrepr = getattr(report, "longrepr", None) + if isinstance(longrepr, tuple) and len(longrepr) >= 3: + reason = longrepr[2] + elif longrepr is not None: + reason = longrepr + else: + reason = "" + text = str(reason) + if text.startswith("Skipped: "): + text = text[len("Skipped: "):] + return text + + +def observed_counts(skipped_reports): + """Map each skipped test's exact reason text to how many tests reported it.""" + counts: dict[str, int] = {} + for report in skipped_reports: + reason = skip_reason(report) + counts[reason] = counts.get(reason, 0) + 1 + return counts + + +def check(profile, observed, *, summary_present=True): + """Compare observed skip counts against a profile; return a problem list. + + ``profile`` maps an exact reason text to its expected count; ``observed`` + maps an exact reason text to its count. An empty list means PASS. + """ + if not isinstance(profile, dict): + return ["the selected allowance profile is missing or empty"] + + problems: list[str] = [] + + # (d) no skip summary at all while the profile expects occurrences. + expects_any = any(isinstance(v, int) and v > 0 for v in profile.values()) + if not summary_present and expects_any: + listed = sum(1 for v in profile.values() if isinstance(v, int) and v > 0) + problems.append( + "no skip summary was reported by the terminal reporter, but the " + f"profile lists {listed} reason(s) with count > 0" + ) + + # (a) a compile-environment skip whose exact reason is unlisted. + for reason, count in sorted(observed.items()): + if classify(reason) == CLASS_COMPILE_ENV and reason not in profile: + problems.append( + f"unlisted {CLASS_COMPILE_ENV} skip reason ({count}x): {reason!r}" + ) + + # (b) / (c) listed reasons that did not match the observation. + for reason, expected in sorted(profile.items()): + if not isinstance(expected, int): + problems.append(f"allowance count for {reason!r} is not an integer: {expected!r}") + continue + got = observed.get(reason, 0) + if expected == got: + continue + if expected > 0 and got == 0: + problems.append( + f"stale allowance: listed {reason!r} expects {expected} but it was " + "NOT observed" + ) + else: + problems.append( + f"count mismatch: {reason!r} expected {expected}, observed {got}" + ) + + return problems + + +def _format_counts(counts): + """Render a ``{reason: count}`` mapping as one ``n x reason`` line.""" + if not counts: + return "(no compile-environment skips)" + return "; ".join(f"{n}x {reason!r}" for reason, n in sorted(counts.items())) + + +def run_gate(terminalreporter, allowance_path=ALLOWANCE_PATH, env=None): + """Evaluate the gate and return ``(problems, observed)``. + + Returns ``(None, observed)`` when the gate is disabled via + ``PINEFORGE_SKIP_GATE=off`` (the caller prints nothing). + """ + env = os.environ if env is None else env + if gate_disabled(env): + return None, {} + + stats = getattr(terminalreporter, "stats", None) + summary_present = isinstance(stats, dict) + skipped_reports = stats.get("skipped", []) if summary_present else [] + observed = observed_counts(skipped_reports) + + allowance = load_allowance(allowance_path) + if allowance is None: + return ( + [f"the allowance file is missing or unreadable: {allowance_path}"], + observed, + ) + + profile_name = str(env.get(PROFILE_ENV, "") or DEFAULT_PROFILE) + profile = profile_entries(allowance, profile_name) + if profile is None: + return ( + [f"the allowance file has no {profile_name!r} profile"], + observed, + ) + + problems = check(profile, observed, summary_present=summary_present) + return problems, observed + + +def _emit(terminalreporter, lines): + """Print ``lines`` through the terminal reporter, falling back to stderr.""" + write_line = getattr(terminalreporter, "write_line", None) + for line in lines: + if callable(write_line): + write_line(line) + else: + print(line, file=sys.stderr) + + +def pytest_sessionfinish(session, exitstatus): + """Fire the gate at session end and fail the session on any drift. + + ``pytest_sessionfinish`` runs even when the session has failures; after + the hook returns pytest reads ``session.exitstatus`` back, so setting it + to 1 here makes the whole process exit non-zero. + """ + terminalreporter = session.config.pluginmanager.getplugin("terminalreporter") + + # A collection-only run executes no test, so it has no skip summary to + # judge; failing it on the allowance's expected counts would be noise. + if getattr(session.config.option, "collectonly", False): + return + + problems, observed = run_gate(terminalreporter) + + if problems is None: # gate disabled + return + + compile_env = { + reason: count + for reason, count in observed.items() + if classify(reason) == CLASS_COMPILE_ENV + } + + if problems: + _emit(terminalreporter, [""]) + _emit(terminalreporter, ["compile-environment skip gate FAILED:"]) + for problem in problems: + _emit(terminalreporter, [f" - {problem}"]) + _emit( + terminalreporter, + [f" observed compile-environment skips: {_format_counts(compile_env)}"], + ) + session.exitstatus = 1 + return + + _emit( + terminalreporter, + [ + "compile-environment skip gate OK: " + f"observed={len(observed)} reason(s) total, " + f"{_format_counts(compile_env)}" + ], + ) From 07c744d39e69099a469eceda6aa0a44d5b3e5805 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 11 Oct 2026 04:20:05 +0800 Subject: [PATCH 02/16] ci: widen the skip gate's environment patterns and measure the allowance The first spot run proved the pattern list was incomplete: on a machine with no engine variables set at all the suite skips at the INCLUDE check, not at the library check, and 191 such skips were invisible to the gate. Added PINEFORGE_ENGINE_INCLUDE and "engine include / compiler unavailable" to the compile-environment patterns, and replaced the placeholder counts in the default profile with the numbers that run produced. --- tests/ci_env_skip_allowance.json | 31 +++++++++++++++++++++---------- tests/skip_gate.py | 7 +++++++ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/tests/ci_env_skip_allowance.json b/tests/ci_env_skip_allowance.json index 9220bcf..cb62d16 100644 --- a/tests/ci_env_skip_allowance.json +++ b/tests/ci_env_skip_allowance.json @@ -1,18 +1,29 @@ { - "_note": "Allowance for the compile-environment skip gate (tests/skip_gate.py). Keys are the EXACT skip reason text (not a normalised class) so a renamed reason is drift too. 'default' is the profile the engine-less hosted-CI pytest job runs; 'engine_built' is the profile the engine-built job runs and lists every reason at 0, so any compile-environment skip fails it. UNMEASURED: every count below is a placeholder written without a spot run; the supervisor replaces them from the measured run before the PR opens, and the gate fails loudly on any mismatch. Reasons were read from the pytest.skip(...) / pytestmark skipif calls in tests/.", - "_unmeasured": true, + "_note": "Allowance for the compile-environment skip gate (tests/skip_gate.py). Keys are the EXACT skip reason text (a renamed reason is drift too). 'default' is the profile the engine-less hosted-CI pytest job runs; its counts were MEASURED on a spot box on 2026-10-11 04:19 Taipei (c7i.4xlarge, Ubuntu 24.04, python 3.12, no engine variables set at all) - which is the same shape as the engine-less CI job, except that CI runs a python 3.11-3.14 matrix, so a count that moves with the interpreter is EXPECTED to differ there and the gate will say so by name. 'engine_built' lists every reason at 0 for the CI job that clones and builds the engine, so any compile-environment skip fails that job.", + "_unmeasured": false, + "_measured_on": "2026-10-11T04:19+08:00 spot c7i.4xlarge ap-southeast-1a, engine-less, no PINEFORGE_* set", "default": { - "No built libpineforge.a. Set PINEFORGE_ENGINE_LIB to the archive (or configure a build*/lib/ beside the engine checkout) to run the emitted translation unit.": 1131, - "built libpineforge not found; set PINEFORGE_ENGINE_LIB": 20, - "deriving the host members needs clang's AST dump; no clang++ found": 1, - "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, - "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 0 + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 148, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 43, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 1, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 1, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 1, + "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, + "engine include / compiler unavailable": 0, + "No built libpineforge.a. Set PINEFORGE_ENGINE_LIB to the archive (or configure a build*/lib/ beside the engine checkout) to run the emitted translation unit.": 0, + "built libpineforge not found; set PINEFORGE_ENGINE_LIB": 0, + "deriving the host members needs clang's AST dump; no clang++ found": 0 }, "engine_built": { + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, + "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 0, + "engine include / compiler unavailable": 0, "No built libpineforge.a. Set PINEFORGE_ENGINE_LIB to the archive (or configure a build*/lib/ beside the engine checkout) to run the emitted translation unit.": 0, "built libpineforge not found; set PINEFORGE_ENGINE_LIB": 0, - "deriving the host members needs clang's AST dump; no clang++ found": 0, - "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, - "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 0 + "deriving the host members needs clang's AST dump; no clang++ found": 0 } } diff --git a/tests/skip_gate.py b/tests/skip_gate.py index 20b6f24..100d774 100644 --- a/tests/skip_gate.py +++ b/tests/skip_gate.py @@ -49,6 +49,13 @@ class (a reason is in the COMPILE-ENVIRONMENT class when it mentions a # and generated headers, the compiler/AST tooling, the parity corpus and the # population-sources directory a compile or byte-comparison test needs. COMPILE_ENV_PATTERNS = ( + # Added 10-11 04:20 after a spot run proved the list was incomplete: on a machine with NO + # engine variables set at all, the tests skip at the INCLUDE check, not at the library check, + # and those skips were invisible to this gate (191 of them in one run) - a hole of exactly the + # class the gate exists to close. The include check comes first, so its wording, not the + # library's, is what hosted CI actually produces. + "PINEFORGE_ENGINE_INCLUDE", + "engine include / compiler unavailable", "No built libpineforge.a", "set PINEFORGE_ENGINE_LIB", "no clang++", From c6ae613841505a041dcb8acc1d5a2269a6bc0131 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 11 Oct 2026 04:28:33 +0800 Subject: [PATCH 03/16] ci: make the skip gate default-deny over every skip reason --- .github/workflows/gate.yml | 9 ++-- .github/workflows/test.yml | 22 +++++--- tests/ci_env_skip_allowance.json | 37 +++++++------ tests/skip_gate.py | 91 +++++++++++++++++++++----------- 4 files changed, 100 insertions(+), 59 deletions(-) diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index 861cd99..50e17c9 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -101,6 +101,9 @@ jobs: path: release.json if-no-files-found: ignore + # Engineless job. It runs a SINGLE interpreter because the skip gate's + # `default` allowance counts were measured on python 3.12; a count that moves + # with the interpreter would make the gate's drift signal meaningless. pytest: needs: preflight-docs if: needs.preflight-docs.outputs.docs_only != 'true' && (github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name == github.repository) @@ -108,10 +111,6 @@ jobs: timeout-minutes: 30 permissions: contents: read - strategy: - fail-fast: false - matrix: - python-version: ["3.11", "3.12", "3.13", "3.14"] env: PYTHONHASHSEED: "0" steps: @@ -122,7 +121,7 @@ jobs: persist-credentials: false - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 with: - python-version: ${{ matrix.python-version }} + python-version: "3.12" - name: Install with dev extras run: python -m pip install -e ".[dev]" - name: Assert the compile-environment skip gate is enabled diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0f844be..61261cd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,15 +63,15 @@ jobs: echo 'docs_only=false' >> "$GITHUB_OUTPUT" fi + # Engine-less job. Unlike `engine-built` this one runs a SINGLE interpreter: + # the skip gate's `default` allowance counts were measured on python 3.12, + # and a count that moves with the interpreter would make the gate's drift + # signal meaningless. That is the only reason the two jobs differ. pytest: needs: preflight-docs if: needs.preflight-docs.outputs.docs_only != 'true' runs-on: ubuntu-latest timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - python-version: ["3.11", "3.12", "3.13", "3.14"] env: PYTHONHASHSEED: "0" steps: @@ -82,7 +82,7 @@ jobs: persist-credentials: false - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 with: - python-version: ${{ matrix.python-version }} + python-version: "3.12" - name: Install with dev extras run: python -m pip install -e ".[dev]" - name: Assert the compile-environment skip gate is enabled @@ -95,14 +95,22 @@ jobs: sys.stderr.write("PINEFORGE_SKIP_GATE=off is not allowed in CI: the skip gate must run\n") sys.exit(1) PY - - name: Run test suite + - name: Run test suite (single interpreter: the default allowance was measured on 3.12) run: python -m pytest -q -rs + # Engine-building job. Unlike the engine-less `pytest` job this one KEEPS the + # 3.11-3.14 matrix: with the engine present the compile-environment allowance + # is zero, so no counted skip can move with the interpreter, and the matrix + # costs nothing but proves the build on every supported version. engine-built: needs: preflight-docs if: needs.preflight-docs.outputs.docs_only != 'true' runs-on: ubuntu-latest timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] env: PYTHONHASHSEED: "0" ENGINE_COMMIT: 4e01e96b5165b5d5e46d72bfbbf88ae6c035a6e9 @@ -115,7 +123,7 @@ jobs: persist-credentials: false - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 with: - python-version: "3.14" + python-version: ${{ matrix.python-version }} - name: Install with dev extras run: python -m pip install -e ".[dev]" - name: Install the compile toolchain and Eigen diff --git a/tests/ci_env_skip_allowance.json b/tests/ci_env_skip_allowance.json index cb62d16..ff3d925 100644 --- a/tests/ci_env_skip_allowance.json +++ b/tests/ci_env_skip_allowance.json @@ -1,29 +1,36 @@ { - "_note": "Allowance for the compile-environment skip gate (tests/skip_gate.py). Keys are the EXACT skip reason text (a renamed reason is drift too). 'default' is the profile the engine-less hosted-CI pytest job runs; its counts were MEASURED on a spot box on 2026-10-11 04:19 Taipei (c7i.4xlarge, Ubuntu 24.04, python 3.12, no engine variables set at all) - which is the same shape as the engine-less CI job, except that CI runs a python 3.11-3.14 matrix, so a count that moves with the interpreter is EXPECTED to differ there and the gate will say so by name. 'engine_built' lists every reason at 0 for the CI job that clones and builds the engine, so any compile-environment skip fails that job.", + "_note": "DEFAULT-DENY allowance for the skip gate (tests/skip_gate.py). Every skip reason that occurs, whatever its wording, must appear here with its exact count; there is no allow-list of 'known' wordings. classify()/COMPILE_ENV_PATTERNS only label the log lines and name which reasons the 'engine_built' profile pins at 0. Keys are the EXACT skip reason text (a renamed reason is drift too). 'default' is the profile the engine-less hosted-CI pytest job runs; its counts are the WHOLE skip population of the measured run (2,358 skips across 13 reasons), not only the compile-environment ones. 'engine_built' is the profile the engine-building CI job runs: the compile-environment reasons are pinned at 0 so any such skip fails that job; see _engine_built_note.", "_unmeasured": false, - "_measured_on": "2026-10-11T04:19+08:00 spot c7i.4xlarge ap-southeast-1a, engine-less, no PINEFORGE_* set", + "_measured_on": "2026-10-11 spot box (Ubuntu 24.04, python 3.12), engine-less with NO PINEFORGE_* variables set; the counts are the WHOLE skip population of that run (2,358 skips across 13 reasons), not only the compile-environment skips", + "_engine_built_note": "The engine-building CI job runs this profile. Its compile-environment reasons are pinned at 0 because building the engine must remove every one. The other counts are copies of the engine-less measurement and are EXPECTED to be corrected from this job's own first run: when a count is wrong the gate prints the exact row to write here, which is how this profile is meant to be filled.", "default": { - "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 148, - "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 43, - "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 1, + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 2261, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 44, + "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, + "engine has no coded run-stop interface": 8, + "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, + "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, + "got empty parameter set for (case_name)": 3, "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 1, + "authoritative engine-corpus source not available": 1, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 1, "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 1, - "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, - "engine include / compiler unavailable": 0, - "No built libpineforge.a. Set PINEFORGE_ENGINE_LIB to the archive (or configure a build*/lib/ beside the engine checkout) to run the emitted translation unit.": 0, - "built libpineforge not found; set PINEFORGE_ENGINE_LIB": 0, - "deriving the host members needs clang's AST dump; no clang++ found": 0 + "engine include / compiler unavailable": 1, + "got empty parameter set for (pine_file)": 1 }, "engine_built": { "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, - "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, + "engine has no coded run-stop interface": 8, + "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, + "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 0, + "got empty parameter set for (case_name)": 3, "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "authoritative engine-corpus source not available": 1, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, - "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 0, "engine include / compiler unavailable": 0, - "No built libpineforge.a. Set PINEFORGE_ENGINE_LIB to the archive (or configure a build*/lib/ beside the engine checkout) to run the emitted translation unit.": 0, - "built libpineforge not found; set PINEFORGE_ENGINE_LIB": 0, - "deriving the host members needs clang's AST dump; no clang++ found": 0 + "got empty parameter set for (pine_file)": 1 } } diff --git a/tests/skip_gate.py b/tests/skip_gate.py index 100d774..712b3f2 100644 --- a/tests/skip_gate.py +++ b/tests/skip_gate.py @@ -1,4 +1,4 @@ -"""Session-end pytest gate that fails the run on compile-environment skips. +"""Session-end pytest gate that fails the run on ANY unallowed skip. Hosted CI runs ``python -m pytest -q`` on a machine with no engine library, no Eigen and no clang, so every test that compiles an emitted translation @@ -6,27 +6,37 @@ roughly a third of the suite never runs. A gate that can pass on missing input is not a gate. -This module turns the skips into a verdict. At session end it collects every -skipped test's reason from the terminal reporter, normalises each reason to a -class (a reason is in the COMPILE-ENVIRONMENT class when it mentions a -compile-environment knob, see ``COMPILE_ENV_PATTERNS``), and compares the -*exact* reason text against the allowance file +This module turns the skips into a verdict, and it is **DEFAULT-DENY**: every +skip reason that occurs, whatever its wording, is a row the allowance must +name with its exact count. There is no allow-list of "known" wordings -- an +earlier version keyed only on compile-environment patterns, so a skip reason +the list did not know passed silently (a spot run produced 2,261 skips under +a wording the list missed). At session end the gate collects every skipped +test's reason from the terminal reporter and compares the *exact* reason text +and count against the selected profile of the allowance file ``tests/ci_env_skip_allowance.json``: {"default": {"": , ...}, - "engine_built": {"": 0, ...}} + "engine_built": {"": , ...}} -The exact reason TEXT -- not the class -- is the key, so a reason that was +The exact reason TEXT -- not a class -- is the key, so a reason that was reworded is drift too. The profile is chosen by ``PINEFORGE_SKIP_PROFILE`` (default ``default``); the CI ``engine-built`` job sets ``engine_built``. +``classify`` and ``COMPILE_ENV_PATTERNS`` remain, but only to LABEL the log +lines (and to name, for the ``engine_built`` profile, the reasons that must be +zero there). They never decide pass or fail: the decision is the table below +alone. + The session FAILS (non-zero exit) when: - (a) a COMPILE-ENVIRONMENT skip's exact reason is not a key of the profile; - (b) a listed reason's count differs from what was actually observed; + (a) an observed skip reason is not a key of the selected profile; + (b) a listed reason's count differs from what was observed; (c) a listed reason with count > 0 was NOT observed (a stale allowance); (d) the session reported no skip summary at all, while the profile lists - any reason with count > 0 (a plugin that never ran is not a gate). + any reason with count > 0 (a plugin that never ran is not a gate); + (e) the selected profile is missing or empty, or the allowance file is + missing or unreadable. The gate is ON by default and may be turned off only with ``PINEFORGE_SKIP_GATE=off`` -- the CI workflow asserts that variable is not @@ -44,9 +54,12 @@ class (a reason is in the COMPILE-ENVIRONMENT class when it mentions a ALLOWANCE_PATH = Path(__file__).resolve().parent / "ci_env_skip_allowance.json" # A skip reason is in the COMPILE-ENVIRONMENT class when it mentions any of -# these knobs (case-insensitive substring match). The list is deliberately -# exhaustive about *wording families*: the engine library archive, the Eigen -# and generated headers, the compiler/AST tooling, the parity corpus and the +# these knobs (case-insensitive substring match). This list is NO LONGER a +# decision: the gate is default-deny, so an unknown reason fails wherever it +# comes from. The class only LABELS the log lines and names the reasons the +# ``engine_built`` profile pins at 0. It is deliberately exhaustive about +# *wording families*: the engine library archive, the Eigen and generated +# headers, the compiler/AST tooling, the parity corpus and the # population-sources directory a compile or byte-comparison test needs. COMPILE_ENV_PATTERNS = ( # Added 10-11 04:20 after a spot run proved the list was incomplete: on a machine with NO @@ -87,8 +100,10 @@ def classify(reason): """Return the class of a skip reason, or ``None`` when it is not compile-env. Matching is a case-insensitive substring test against - ``COMPILE_ENV_PATTERNS`` (the allowance keys on the *exact* reason text; - this only decides which reasons the gate must see in the file). + ``COMPILE_ENV_PATTERNS``. This is used ONLY to label the log lines and to + name the reasons the ``engine_built`` profile pins at 0 -- it never + decides whether the gate passes. The decision keys on the *exact* reason + text against the selected profile. """ if not reason: return None @@ -113,16 +128,19 @@ def load_allowance(path=ALLOWANCE_PATH): def profile_entries(allowance, profile_name): """Return ``{reason: count}`` for a profile, or ``None`` when it is absent. - Keys beginning with ``_`` are file metadata (a ``_note`` comment field) - and are ignored, so the allowance can carry human notes without polluting - the reason set. + A missing profile, a profile that is not a mapping, and an EMPTY profile + all return ``None``: an empty table allows nothing and explains nothing, + so the gate treats it as a fail (condition (e)). Keys beginning with + ``_`` are file metadata (a ``_note`` comment field) and are ignored, so the + allowance can carry human notes without polluting the reason set. """ if not isinstance(allowance, dict): return None profile = allowance.get(profile_name) if not isinstance(profile, dict): return None - return {k: v for k, v in profile.items() if not str(k).startswith("_")} + entries = {k: v for k, v in profile.items() if not str(k).startswith("_")} + return entries or None def skip_reason(report): @@ -158,6 +176,8 @@ def observed_counts(skipped_reports): def check(profile, observed, *, summary_present=True): """Compare observed skip counts against a profile; return a problem list. + DEFAULT-DENY: this does not consult ``classify`` or any pattern list -- + EVERY observed reason must be a key of the profile, with the exact count. ``profile`` maps an exact reason text to its expected count; ``observed`` maps an exact reason text to its count. An empty list means PASS. """ @@ -175,12 +195,10 @@ def check(profile, observed, *, summary_present=True): f"profile lists {listed} reason(s) with count > 0" ) - # (a) a compile-environment skip whose exact reason is unlisted. + # (a) ANY observed reason the profile does not list (default-deny). for reason, count in sorted(observed.items()): - if classify(reason) == CLASS_COMPILE_ENV and reason not in profile: - problems.append( - f"unlisted {CLASS_COMPILE_ENV} skip reason ({count}x): {reason!r}" - ) + if reason not in profile: + problems.append(f"unlisted skip reason ({count}x): {reason!r}") # (b) / (c) listed reasons that did not match the observation. for reason, expected in sorted(profile.items()): @@ -206,7 +224,7 @@ def check(profile, observed, *, summary_present=True): def _format_counts(counts): """Render a ``{reason: count}`` mapping as one ``n x reason`` line.""" if not counts: - return "(no compile-environment skips)" + return "(none)" return "; ".join(f"{n}x {reason!r}" for reason, n in sorted(counts.items())) @@ -236,7 +254,7 @@ def run_gate(terminalreporter, allowance_path=ALLOWANCE_PATH, env=None): profile = profile_entries(allowance, profile_name) if profile is None: return ( - [f"the allowance file has no {profile_name!r} profile"], + [f"the allowance file has no usable {profile_name!r} profile (missing or empty)"], observed, ) @@ -281,12 +299,20 @@ def pytest_sessionfinish(session, exitstatus): if problems: _emit(terminalreporter, [""]) - _emit(terminalreporter, ["compile-environment skip gate FAILED:"]) + _emit( + terminalreporter, + ["skip gate FAILED (default-deny: every observed skip reason must " + "be an allowance key with its exact count):"], + ) for problem in problems: _emit(terminalreporter, [f" - {problem}"]) _emit( terminalreporter, - [f" observed compile-environment skips: {_format_counts(compile_env)}"], + [f" compile-environment skips: {_format_counts(compile_env)}"], + ) + _emit( + terminalreporter, + [f" observed skips: {_format_counts(observed)}"], ) session.exitstatus = 1 return @@ -294,8 +320,9 @@ def pytest_sessionfinish(session, exitstatus): _emit( terminalreporter, [ - "compile-environment skip gate OK: " - f"observed={len(observed)} reason(s) total, " - f"{_format_counts(compile_env)}" + "skip gate OK (default-deny): " + f"observed={len(observed)} reason(s), " + f"compile-environment={len(compile_env)}, " + f"observed skips: {_format_counts(observed)}" ], ) From 7e533e44476e5b0bc8762b37543284a0d2d1c6fb Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 11 Oct 2026 05:24:24 +0800 Subject: [PATCH 04/16] ci: close the six bypasses the review found in the skip gate --- .github/workflows/gate.yml | 5 ++++- .github/workflows/test.yml | 12 +++++++++--- tests/ci_env_skip_allowance.json | 2 +- tests/skip_gate.py | 20 ++++++++++++++------ 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index 50e17c9..85622bb 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -135,7 +135,10 @@ jobs: sys.exit(1) PY - name: Run test suite - run: python -m pytest -q -rs + run: | + set -eo pipefail + python -m pytest -q -rs 2>&1 | tee pytest.log + grep -q '^skip gate OK (default-deny)' pytest.log gate: if: always() diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 61261cd..40968e9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -95,8 +95,11 @@ jobs: sys.stderr.write("PINEFORGE_SKIP_GATE=off is not allowed in CI: the skip gate must run\n") sys.exit(1) PY - - name: Run test suite (single interpreter: the default allowance was measured on 3.12) - run: python -m pytest -q -rs + - name: "Run test suite (single interpreter: the default allowance was measured on 3.12)" + run: | + set -eo pipefail + python -m pytest -q -rs 2>&1 | tee pytest.log + grep -q '^skip gate OK (default-deny)' pytest.log # Engine-building job. Unlike the engine-less `pytest` job this one KEEPS the # 3.11-3.14 matrix: with the engine present the compile-environment allowance @@ -161,4 +164,7 @@ jobs: sys.exit(1) PY - name: Run the test suite with the engine present - run: python -m pytest -q -rs + run: | + set -eo pipefail + python -m pytest -q -rs 2>&1 | tee pytest.log + grep -q '^skip gate OK (default-deny)' pytest.log diff --git a/tests/ci_env_skip_allowance.json b/tests/ci_env_skip_allowance.json index ff3d925..b3a8403 100644 --- a/tests/ci_env_skip_allowance.json +++ b/tests/ci_env_skip_allowance.json @@ -2,7 +2,7 @@ "_note": "DEFAULT-DENY allowance for the skip gate (tests/skip_gate.py). Every skip reason that occurs, whatever its wording, must appear here with its exact count; there is no allow-list of 'known' wordings. classify()/COMPILE_ENV_PATTERNS only label the log lines and name which reasons the 'engine_built' profile pins at 0. Keys are the EXACT skip reason text (a renamed reason is drift too). 'default' is the profile the engine-less hosted-CI pytest job runs; its counts are the WHOLE skip population of the measured run (2,358 skips across 13 reasons), not only the compile-environment ones. 'engine_built' is the profile the engine-building CI job runs: the compile-environment reasons are pinned at 0 so any such skip fails that job; see _engine_built_note.", "_unmeasured": false, "_measured_on": "2026-10-11 spot box (Ubuntu 24.04, python 3.12), engine-less with NO PINEFORGE_* variables set; the counts are the WHOLE skip population of that run (2,358 skips across 13 reasons), not only the compile-environment skips", - "_engine_built_note": "The engine-building CI job runs this profile. Its compile-environment reasons are pinned at 0 because building the engine must remove every one. The other counts are copies of the engine-less measurement and are EXPECTED to be corrected from this job's own first run: when a count is wrong the gate prints the exact row to write here, which is how this profile is meant to be filled.", + "_engine_built_note": "UNMEASURED. The engine-building CI job runs this profile, but the counts below are NOT a measurement of that job: they are copies of the engine-less `default` run and are EXPECTED to be WRONG. That job neither sets PINEFORGE_ENGINE_CORPUS nor PINEFORGE_STRING_SETTINGS_SOURCES (the parity corpus and the population sources are private submodules absent from the public pineforge-engine clone the job checks out), so its corpus and population tests do not run and its counts WILL differ from the engine-less job's. Expect the FIRST engine-built run to FAIL and the gate to print the exact rows to write here; paste those rows in to measure this profile. The compile-environment rows (the 'PINEFORGE_ENGINE_INCLUDE not set or invalid...' and 'set PINEFORGE_ENGINE_INCLUDE...' rows, 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') must stay 0: building the engine must remove every one.", "default": { "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 2261, "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 44, diff --git a/tests/skip_gate.py b/tests/skip_gate.py index 712b3f2..df838fd 100644 --- a/tests/skip_gate.py +++ b/tests/skip_gate.py @@ -186,13 +186,11 @@ def check(profile, observed, *, summary_present=True): problems: list[str] = [] - # (d) no skip summary at all while the profile expects occurrences. - expects_any = any(isinstance(v, int) and v > 0 for v in profile.values()) - if not summary_present and expects_any: - listed = sum(1 for v in profile.values() if isinstance(v, int) and v > 0) + # (d) no skip summary at all: a session with no summary ALWAYS fails, + # whatever the profile expects -- a plugin that never ran is not a gate. + if not summary_present: problems.append( - "no skip summary was reported by the terminal reporter, but the " - f"profile lists {listed} reason(s) with count > 0" + "no skip summary was reported by the terminal reporter" ) # (a) ANY observed reason the profile does not list (default-deny). @@ -236,6 +234,12 @@ def run_gate(terminalreporter, allowance_path=ALLOWANCE_PATH, env=None): """ env = os.environ if env is None else env if gate_disabled(env): + # A workflow must not be able to turn the gate off: keyed on the GitHub + # Actions variable, so the refusal stands even if the workflow deletes + # its own assertion step. Only reached when the gate is disabled, so a + # normal Actions run (gate on) is unaffected. + if env.get("GITHUB_ACTIONS") == "true": + return ["PINEFORGE_SKIP_GATE=off is not allowed under GitHub Actions"], {} return None, {} stats = getattr(terminalreporter, "stats", None) @@ -289,6 +293,10 @@ def pytest_sessionfinish(session, exitstatus): problems, observed = run_gate(terminalreporter) if problems is None: # gate disabled + _emit( + terminalreporter, + ["skip gate DISABLED (PINEFORGE_SKIP_GATE=off): no skip verdict was produced"], + ) return compile_env = { From 047811df44c8f438fdd20870b88f29eeca72e056 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 11 Oct 2026 05:41:09 +0800 Subject: [PATCH 05/16] ci: make the skip gate's verdict start on its own line --- .github/workflows/gate.yml | 2 ++ .github/workflows/test.yml | 4 ++++ tests/skip_gate.py | 8 ++++++++ 3 files changed, 14 insertions(+) diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index 85622bb..7a4ebdf 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -138,6 +138,8 @@ jobs: run: | set -eo pipefail python -m pytest -q -rs 2>&1 | tee pytest.log + # Anchored to line start: the gate emits a leading empty line, since + # pytest's -q progress line (".....[100%]") can lack a trailing newline. grep -q '^skip gate OK (default-deny)' pytest.log gate: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 40968e9..4e67a3e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -99,6 +99,8 @@ jobs: run: | set -eo pipefail python -m pytest -q -rs 2>&1 | tee pytest.log + # Anchored to line start: the gate emits a leading empty line, since + # pytest's -q progress line (".....[100%]") can lack a trailing newline. grep -q '^skip gate OK (default-deny)' pytest.log # Engine-building job. Unlike the engine-less `pytest` job this one KEEPS the @@ -167,4 +169,6 @@ jobs: run: | set -eo pipefail python -m pytest -q -rs 2>&1 | tee pytest.log + # Anchored to line start: the gate emits a leading empty line, since + # pytest's -q progress line (".....[100%]") can lack a trailing newline. grep -q '^skip gate OK (default-deny)' pytest.log diff --git a/tests/skip_gate.py b/tests/skip_gate.py index df838fd..890df6d 100644 --- a/tests/skip_gate.py +++ b/tests/skip_gate.py @@ -293,6 +293,9 @@ def pytest_sessionfinish(session, exitstatus): problems, observed = run_gate(terminalreporter) if problems is None: # gate disabled + # Leading empty line: pytest's -q progress output (".....[100%]") can + # lack a trailing newline, which would glue this verdict to it. + _emit(terminalreporter, [""]) _emit( terminalreporter, ["skip gate DISABLED (PINEFORGE_SKIP_GATE=off): no skip verdict was produced"], @@ -306,6 +309,8 @@ def pytest_sessionfinish(session, exitstatus): } if problems: + # Leading empty line: pytest's -q progress output (".....[100%]") can + # lack a trailing newline, which would glue this verdict to it. _emit(terminalreporter, [""]) _emit( terminalreporter, @@ -325,6 +330,9 @@ def pytest_sessionfinish(session, exitstatus): session.exitstatus = 1 return + # Leading empty line: pytest's -q progress output (".....[100%]") can lack + # a trailing newline, which would glue this verdict to it. + _emit(terminalreporter, [""]) _emit( terminalreporter, [ From 630fb831cffffa4385f6a5140a04661d1ce22c04 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 11 Oct 2026 06:49:04 +0800 Subject: [PATCH 06/16] ci: build the engine inside the pull-request gate --- .github/workflows/gate.yml | 98 ++++++++++++++++++++++++++++++-- .github/workflows/test.yml | 80 ++------------------------ tests/ci_env_skip_allowance.json | 4 +- 3 files changed, 102 insertions(+), 80 deletions(-) diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index 7a4ebdf..e6f9bc6 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -142,9 +142,98 @@ jobs: # pytest's -q progress line (".....[100%]") can lack a trailing newline. grep -q '^skip gate OK (default-deny)' pytest.log + # Engine-building job, MOVED here from test.yml so this workflow can depend + # on it: `needs:` cannot name a job in another workflow, so with the job only + # in test.yml the blocking aggregation below could not wait for it. This + # workflow runs on `pull_request_target`, where a job that builds + # pull-request code is the classic hole, so the job is held to the SAME safety + # shape as the engine-less `pytest` job above: the fork-guard clause in `if:`, + # a read-only `permissions: contents: read` (the workflow default is + # `permissions: {}`), a checkout of the PR head with `persist-credentials: + # false`, and no secret anywhere. Unlike the engine-less `pytest` job it KEEPS + # the 3.11-3.14 matrix: with the engine present the compile-environment + # allowance is zero, so no counted skip can move with the interpreter, and the + # matrix costs nothing but proves the build on every supported version. + engine-built: + needs: preflight-docs + if: needs.preflight-docs.outputs.docs_only != 'true' && (github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name == github.repository) + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] + env: + PYTHONHASHSEED: "0" + ENGINE_COMMIT: 4e01e96b5165b5d5e46d72bfbbf88ae6c035a6e9 + CORPUS_COMMIT: b40aa8ecbe3caa9ad05244cdfd9ee8138da845e4 + PINEFORGE_SKIP_PROFILE: engine_built + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: ${{ matrix.python-version }} + - name: Install with dev extras + run: python -m pip install -e ".[dev]" + - name: Install the compile toolchain and Eigen + run: | + sudo apt-get update + sudo apt-get install -y libeigen3-dev clang cmake g++ ninja-build + - name: Clone and build the paired engine + run: | + set -euo pipefail + git clone https://github.com/pineforge-4pass/pineforge-engine.git engine + git -C engine checkout "$ENGINE_COMMIT" + cmake -B build -S engine -DCMAKE_BUILD_TYPE=Release + cmake --build build -j"$(nproc)" + lib="$(find "$PWD/build" -name libpineforge.a -print -quit)" + if [ -z "$lib" ]; then + echo "the engine build produced no libpineforge.a" >&2 + exit 1 + fi + echo "libpineforge.a at: $lib" + echo "PINEFORGE_ENGINE_LIB=$lib" >> "$GITHUB_ENV" + echo "PINEFORGE_ENGINE_INCLUDE=$PWD/engine/include" >> "$GITHUB_ENV" + - name: Clone the public parity corpus at the engine's pinned gitlink + run: | + set -euo pipefail + git clone https://github.com/pineforge-4pass/pineforge-corpus.git corpus + git -C corpus checkout "$CORPUS_COMMIT" + # The corpus resolver requires a `data/` directory under the root. + test -d corpus/data || { echo "no data/ under the pinned corpus checkout" >&2; exit 1; } + echo "PINEFORGE_ENGINE_CORPUS=$PWD/corpus" >> "$GITHUB_ENV" + - name: Assert the built library exists + run: | + set -euo pipefail + test -f "$PINEFORGE_ENGINE_LIB" || { echo "no library at $PINEFORGE_ENGINE_LIB" >&2; exit 1; } + test -f "$PINEFORGE_ENGINE_INCLUDE/pineforge/engine.hpp" || { echo "no engine headers under $PINEFORGE_ENGINE_INCLUDE" >&2; exit 1; } + - name: Assert the compile-environment skip gate is enabled + run: | + python - <<'PY' + import os + import sys + value = os.environ.get("PINEFORGE_SKIP_GATE", "").strip().lower() + if value == "off": + sys.stderr.write("PINEFORGE_SKIP_GATE=off is not allowed in CI: the skip gate must run\n") + sys.exit(1) + PY + - name: Run the test suite with the engine present + run: | + set -eo pipefail + python -m pytest -q -rs 2>&1 | tee pytest.log + # Anchored to line start: the gate emits a leading empty line, since + # pytest's -q progress line (".....[100%]") can lack a trailing newline. + grep -q '^skip gate OK (default-deny)' pytest.log + gate: if: always() - needs: [preflight-docs, differential, pytest] + needs: [preflight-docs, differential, pytest, engine-built] runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -159,18 +248,19 @@ jobs: DOCS_ONLY: ${{ needs.preflight-docs.outputs.docs_only }} DIFFERENTIAL_RESULT: ${{ needs.differential.result }} PYTEST_RESULT: ${{ needs.pytest.result }} + ENGINE_BUILT_RESULT: ${{ needs.engine-built.result }} SAME_REPO: ${{ github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name == github.repository }} run: | set -euo pipefail state=failure description='Codegen gate did not pass' if [ "$PREFLIGHT_RESULT" = success ]; then - if [ "$DOCS_ONLY" = true ] && [ "$DIFFERENTIAL_RESULT" = skipped ] && [ "$PYTEST_RESULT" = skipped ]; then + if [ "$DOCS_ONLY" = true ] && [ "$DIFFERENTIAL_RESULT" = skipped ] && [ "$PYTEST_RESULT" = skipped ] && [ "$ENGINE_BUILT_RESULT" = skipped ]; then state=success description='docs-only preflight+docs; no sweep' - elif [ "$DOCS_ONLY" != true ] && [ "$SAME_REPO" = true ] && [ "$DIFFERENTIAL_RESULT" = success ] && [ "$PYTEST_RESULT" = success ]; then + elif [ "$DOCS_ONLY" != true ] && [ "$SAME_REPO" = true ] && [ "$DIFFERENTIAL_RESULT" = success ] && [ "$PYTEST_RESULT" = success ] && [ "$ENGINE_BUILT_RESULT" = success ]; then state=success - description='Full codegen differential gate and pytest passed' + description='Full codegen differential gate, pytest and engine-built suite passed' fi fi if [ "$state" != success ] && [ "$SAME_REPO" != true ]; then diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4e67a3e..aebad6f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,10 +63,12 @@ jobs: echo 'docs_only=false' >> "$GITHUB_OUTPUT" fi - # Engine-less job. Unlike `engine-built` this one runs a SINGLE interpreter: - # the skip gate's `default` allowance counts were measured on python 3.12, - # and a count that moves with the interpreter would make the gate's drift - # signal meaningless. That is the only reason the two jobs differ. + # Engine-less job. It runs a SINGLE interpreter: the skip gate's `default` + # allowance counts were measured on python 3.12, and a count that moves with + # the interpreter would make the gate's drift signal meaningless. + # The engine-building twin of this job lives in gate.yml, where the blocking + # aggregation can `needs:` it (a `needs:` cannot name a job in another + # workflow); it is not run here. pytest: needs: preflight-docs if: needs.preflight-docs.outputs.docs_only != 'true' @@ -102,73 +104,3 @@ jobs: # Anchored to line start: the gate emits a leading empty line, since # pytest's -q progress line (".....[100%]") can lack a trailing newline. grep -q '^skip gate OK (default-deny)' pytest.log - - # Engine-building job. Unlike the engine-less `pytest` job this one KEEPS the - # 3.11-3.14 matrix: with the engine present the compile-environment allowance - # is zero, so no counted skip can move with the interpreter, and the matrix - # costs nothing but proves the build on every supported version. - engine-built: - needs: preflight-docs - if: needs.preflight-docs.outputs.docs_only != 'true' - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - python-version: ["3.11", "3.12", "3.13", "3.14"] - env: - PYTHONHASHSEED: "0" - ENGINE_COMMIT: 4e01e96b5165b5d5e46d72bfbbf88ae6c035a6e9 - PINEFORGE_SKIP_PROFILE: engine_built - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - with: - ref: ${{ github.sha }} - fetch-depth: 0 - persist-credentials: false - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 - with: - python-version: ${{ matrix.python-version }} - - name: Install with dev extras - run: python -m pip install -e ".[dev]" - - name: Install the compile toolchain and Eigen - run: | - sudo apt-get update - sudo apt-get install -y libeigen3-dev clang cmake g++ ninja-build - - name: Clone and build the paired engine - run: | - set -euo pipefail - git clone https://github.com/pineforge-4pass/pineforge-engine.git engine - git -C engine checkout "$ENGINE_COMMIT" - cmake -B build -S engine -DCMAKE_BUILD_TYPE=Release - cmake --build build -j"$(nproc)" - lib="$(find "$PWD/build" -name libpineforge.a -print -quit)" - if [ -z "$lib" ]; then - echo "the engine build produced no libpineforge.a" >&2 - exit 1 - fi - echo "libpineforge.a at: $lib" - echo "PINEFORGE_ENGINE_LIB=$lib" >> "$GITHUB_ENV" - echo "PINEFORGE_ENGINE_INCLUDE=$PWD/engine/include" >> "$GITHUB_ENV" - - name: Assert the built library exists - run: | - set -euo pipefail - test -f "$PINEFORGE_ENGINE_LIB" || { echo "no library at $PINEFORGE_ENGINE_LIB" >&2; exit 1; } - test -f "$PINEFORGE_ENGINE_INCLUDE/pineforge/engine.hpp" || { echo "no engine headers under $PINEFORGE_ENGINE_INCLUDE" >&2; exit 1; } - - name: Assert the compile-environment skip gate is enabled - run: | - python - <<'PY' - import os - import sys - value = os.environ.get("PINEFORGE_SKIP_GATE", "").strip().lower() - if value == "off": - sys.stderr.write("PINEFORGE_SKIP_GATE=off is not allowed in CI: the skip gate must run\n") - sys.exit(1) - PY - - name: Run the test suite with the engine present - run: | - set -eo pipefail - python -m pytest -q -rs 2>&1 | tee pytest.log - # Anchored to line start: the gate emits a leading empty line, since - # pytest's -q progress line (".....[100%]") can lack a trailing newline. - grep -q '^skip gate OK (default-deny)' pytest.log diff --git a/tests/ci_env_skip_allowance.json b/tests/ci_env_skip_allowance.json index b3a8403..74b155b 100644 --- a/tests/ci_env_skip_allowance.json +++ b/tests/ci_env_skip_allowance.json @@ -2,7 +2,7 @@ "_note": "DEFAULT-DENY allowance for the skip gate (tests/skip_gate.py). Every skip reason that occurs, whatever its wording, must appear here with its exact count; there is no allow-list of 'known' wordings. classify()/COMPILE_ENV_PATTERNS only label the log lines and name which reasons the 'engine_built' profile pins at 0. Keys are the EXACT skip reason text (a renamed reason is drift too). 'default' is the profile the engine-less hosted-CI pytest job runs; its counts are the WHOLE skip population of the measured run (2,358 skips across 13 reasons), not only the compile-environment ones. 'engine_built' is the profile the engine-building CI job runs: the compile-environment reasons are pinned at 0 so any such skip fails that job; see _engine_built_note.", "_unmeasured": false, "_measured_on": "2026-10-11 spot box (Ubuntu 24.04, python 3.12), engine-less with NO PINEFORGE_* variables set; the counts are the WHOLE skip population of that run (2,358 skips across 13 reasons), not only the compile-environment skips", - "_engine_built_note": "UNMEASURED. The engine-building CI job runs this profile, but the counts below are NOT a measurement of that job: they are copies of the engine-less `default` run and are EXPECTED to be WRONG. That job neither sets PINEFORGE_ENGINE_CORPUS nor PINEFORGE_STRING_SETTINGS_SOURCES (the parity corpus and the population sources are private submodules absent from the public pineforge-engine clone the job checks out), so its corpus and population tests do not run and its counts WILL differ from the engine-less job's. Expect the FIRST engine-built run to FAIL and the gate to print the exact rows to write here; paste those rows in to measure this profile. The compile-environment rows (the 'PINEFORGE_ENGINE_INCLUDE not set or invalid...' and 'set PINEFORGE_ENGINE_INCLUDE...' rows, 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') must stay 0: building the engine must remove every one.", + "_engine_built_note": "UNMEASURED. The engine-building CI job runs this profile, but the counts below are NOT a measurement of that job: they are copies of the engine-less `default` run and are EXPECTED to be WRONG. That job now DOES set PINEFORGE_ENGINE_CORPUS, cloning the public pineforge-4pass/pineforge-corpus at b40aa8ecbe3caa9ad05244cdfd9ee8138da845e4 (the gitlink the pinned engine commit 4e01e96b5165b5d5e46d72bfbbf88ae6c035a6e9 records), so its corpus tests DO run. It does NOT set PINEFORGE_STRING_SETTINGS_SOURCES, because the population sources that variable would point at are NOT public: this repository carries no fixture for them and the public corpus checkout above has no file matching the sha12 names the test globs, so its six population-source cases skip -- hence that row is 6, not 0. Expect the FIRST engine-built run to FAIL on the other rows and the gate to print the exact rows to write here; paste those rows in to measure this profile. The compile-environment rows (the 'PINEFORGE_ENGINE_INCLUDE not set or invalid...' and 'set PINEFORGE_ENGINE_INCLUDE...' rows, 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') must stay 0: building the engine and pinning the public corpus must remove every one.", "default": { "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 2261, "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 44, @@ -24,7 +24,7 @@ "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, "engine has no coded run-stop interface": 8, "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, - "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 0, + "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, "got empty parameter set for (case_name)": 3, "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, "authoritative engine-corpus source not available": 1, From cba1446cfaef5b0d1058e60141535d6cda7153ee Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 11 Oct 2026 06:49:04 +0800 Subject: [PATCH 07/16] ci: fetch the corpus where the engine reads it, and give the job time --- .github/workflows/gate.yml | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index e6f9bc6..adc1fb6 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -158,7 +158,11 @@ jobs: needs: preflight-docs if: needs.preflight-docs.outputs.docs_only != 'true' && (github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name == github.repository) runs-on: ubuntu-latest - timeout-minutes: 60 + # The suite is minutes-to-hours: the engine-less run alone took 47 minutes + # with 1,181 skips, and building the engine turns 1,131 of those skips into + # real work (plus the corpus's compile tests), on a 4-vCPU hosted runner. + # 360 is GitHub's maximum for a hosted job. + timeout-minutes: 360 permissions: contents: read strategy: @@ -190,6 +194,15 @@ jobs: set -euo pipefail git clone https://github.com/pineforge-4pass/pineforge-engine.git engine git -C engine checkout "$ENGINE_COMMIT" + # The corpus must live INSIDE the engine checkout, at /corpus: the e2e harness (tests/_e2e.py) and the engine's own + # derive_corpus_feeds.py both resolve it there, so a corpus fetched + # beside the engine leaves engine/corpus an empty gitlink and every + # e2e test skips for want of its 1m feed. The engine's .gitmodules + # already points the `corpus` submodule at the public HTTPS URL, so + # no URL rewrite is needed; the update checks out the pinned gitlink. + git -C engine submodule update --init corpus + git -C engine/corpus checkout "$CORPUS_COMMIT" cmake -B build -S engine -DCMAKE_BUILD_TYPE=Release cmake --build build -j"$(nproc)" lib="$(find "$PWD/build" -name libpineforge.a -print -quit)" @@ -200,14 +213,14 @@ jobs: echo "libpineforge.a at: $lib" echo "PINEFORGE_ENGINE_LIB=$lib" >> "$GITHUB_ENV" echo "PINEFORGE_ENGINE_INCLUDE=$PWD/engine/include" >> "$GITHUB_ENV" - - name: Clone the public parity corpus at the engine's pinned gitlink + echo "PINEFORGE_ENGINE_CORPUS=$PWD/engine/corpus" >> "$GITHUB_ENV" + - name: Assert the public parity corpus is present under the engine run: | set -euo pipefail - git clone https://github.com/pineforge-4pass/pineforge-corpus.git corpus - git -C corpus checkout "$CORPUS_COMMIT" - # The corpus resolver requires a `data/` directory under the root. - test -d corpus/data || { echo "no data/ under the pinned corpus checkout" >&2; exit 1; } - echo "PINEFORGE_ENGINE_CORPUS=$PWD/corpus" >> "$GITHUB_ENV" + # The corpus resolver and the e2e harness require a `data/` tree + # under the root the harness resolves: /corpus/data. + test -d "$PINEFORGE_ENGINE_CORPUS/data" || { echo "no data/ under $PINEFORGE_ENGINE_CORPUS" >&2; exit 1; } + test -f "$PINEFORGE_ENGINE_CORPUS/data/ohlcv_ETH-USDT-USDT_1m.csv" || { echo "the corpus is missing its 1m feed" >&2; exit 1; } - name: Assert the built library exists run: | set -euo pipefail From 6538412967f0b74d95c1771e660b8c91ea5fa82e Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 11 Oct 2026 06:49:04 +0800 Subject: [PATCH 08/16] ci: fetch the corpus's LFS feed and assert its real size --- .github/workflows/gate.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index adc1fb6..6e005e7 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -185,10 +185,11 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install with dev extras run: python -m pip install -e ".[dev]" - - name: Install the compile toolchain and Eigen + - name: Install the compile toolchain, Eigen and git-lfs run: | sudo apt-get update - sudo apt-get install -y libeigen3-dev clang cmake g++ ninja-build + sudo apt-get install -y libeigen3-dev clang cmake g++ ninja-build git-lfs + git lfs install --skip-repo - name: Clone and build the paired engine run: | set -euo pipefail @@ -203,6 +204,12 @@ jobs: # no URL rewrite is needed; the update checks out the pinned gitlink. git -C engine submodule update --init corpus git -C engine/corpus checkout "$CORPUS_COMMIT" + # The corpus's 1m feed is stored in Git LFS: a plain clone/submodule + # update leaves a ~130-byte POINTER at data/ohlcv_ETH-USDT-USDT_1m.csv + # that passes a bare `test -f`, while the ~176 MB of data the e2e + # harness needs is absent. Pull the pinned commit's LFS objects so the + # real feed is on disk; the assertion below then re-checks its size. + git -C engine/corpus lfs pull cmake -B build -S engine -DCMAKE_BUILD_TYPE=Release cmake --build build -j"$(nproc)" lib="$(find "$PWD/build" -name libpineforge.a -print -quit)" @@ -220,7 +227,15 @@ jobs: # The corpus resolver and the e2e harness require a `data/` tree # under the root the harness resolves: /corpus/data. test -d "$PINEFORGE_ENGINE_CORPUS/data" || { echo "no data/ under $PINEFORGE_ENGINE_CORPUS" >&2; exit 1; } - test -f "$PINEFORGE_ENGINE_CORPUS/data/ohlcv_ETH-USDT-USDT_1m.csv" || { echo "the corpus is missing its 1m feed" >&2; exit 1; } + # A bare `test -f` is not enough: the feed is stored in Git LFS, so an + # unsmudged ~130-byte pointer satisfies it while the data is absent. + # Mirror the harness (tests/_e2e.py: `feed.stat().st_size < 1_000_000`) + # and require a real file of at least 1,000,000 bytes. + feed="$PINEFORGE_ENGINE_CORPUS/data/ohlcv_ETH-USDT-USDT_1m.csv" + if [ ! -f "$feed" ] || [ "$(stat -c%s "$feed")" -lt 1000000 ]; then + echo "the corpus 1m feed is missing or an unsmudged LFS pointer (need >= 1000000 bytes): $feed" >&2 + exit 1 + fi - name: Assert the built library exists run: | set -euo pipefail From 91b016c7299b3184c4afd85fbd930d95e74648dd Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 11 Oct 2026 07:01:43 +0800 Subject: [PATCH 09/16] ci: build once and run the engine suite in shards --- .github/workflows/gate.yml | 216 +++++++++++++++++++++++++++---- tests/ci_env_skip_allowance.json | 170 ++++++++++++++++++++++++ 2 files changed, 364 insertions(+), 22 deletions(-) diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index 6e005e7..bc306e7 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -150,30 +150,35 @@ jobs: # shape as the engine-less `pytest` job above: the fork-guard clause in `if:`, # a read-only `permissions: contents: read` (the workflow default is # `permissions: {}`), a checkout of the PR head with `persist-credentials: - # false`, and no secret anywhere. Unlike the engine-less `pytest` job it KEEPS - # the 3.11-3.14 matrix: with the engine present the compile-environment - # allowance is zero, so no counted skip can move with the interpreter, and the - # matrix costs nothing but proves the build on every supported version. - engine-built: + # false`, and no secret anywhere. + # + # It BUILDS ONCE and SPLITS; the running happens in `engine-shard` below. One + # job cannot both build and run the whole suite inside a hosted runner's + # hours: the engine-less run measured 2843.49s (0:47:23) with 1,181 skips, + # building the engine turns 1,131 of those into real work, the corpus adds + # ~314 compile tests, and a hosted runner has at most 4 vCPUs. So this job + # builds the engine, pins its corpus, computes the deterministic shard split + # and publishes all of it as the `engine-env` artifact the shard matrix + # consumes -- no shard rebuilds anything. + engine-build: needs: preflight-docs if: needs.preflight-docs.outputs.docs_only != 'true' && (github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name == github.repository) runs-on: ubuntu-latest - # The suite is minutes-to-hours: the engine-less run alone took 47 minutes - # with 1,181 skips, and building the engine turns 1,131 of those skips into - # real work (plus the corpus's compile tests), on a 4-vCPU hosted runner. - # 360 is GitHub's maximum for a hosted job. + # The clone + LFS pull + engine build, the collect-only sweep and the + # artifact upload. 360 is GitHub's maximum for a hosted job. timeout-minutes: 360 permissions: contents: read - strategy: - fail-fast: false - matrix: - python-version: ["3.11", "3.12", "3.13", "3.14"] env: PYTHONHASHSEED: "0" ENGINE_COMMIT: 4e01e96b5165b5d5e46d72bfbbf88ae6c035a6e9 CORPUS_COMMIT: b40aa8ecbe3caa9ad05244cdfd9ee8138da845e4 - PINEFORGE_SKIP_PROFILE: engine_built + # The shard count the split below writes and the matrix of `engine-shard` + # mirrors. 10: the engine-built full run is 2-4 hours of work on a 4-vCPU + # runner (the ruling's own assumption), so a tenth is ~18 minutes of + # nominal work -- wide headroom under the 120-minute shard timeout even + # for the shard that draws the heaviest modules. + SHARD_COUNT: "10" steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 with: @@ -182,7 +187,7 @@ jobs: persist-credentials: false - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 with: - python-version: ${{ matrix.python-version }} + python-version: "3.12" - name: Install with dev extras run: python -m pip install -e ".[dev]" - name: Install the compile toolchain, Eigen and git-lfs @@ -251,17 +256,178 @@ jobs: sys.stderr.write("PINEFORGE_SKIP_GATE=off is not allowed in CI: the skip gate must run\n") sys.exit(1) PY - - name: Run the test suite with the engine present + - name: Split the collected test files into deterministic shards + run: | + set -euo pipefail + # Collect with the engine env set, so the corpus tests are collected + # as real work (an unresolved engine collects none of them). + python -m pytest --collect-only -q > collect.txt + # The collected TEST FILES. `--collect-only -q` emits one line per + # collected node id ("tests/x.py::test_y") on pytest 8 and one + # "tests/x.py: " per file on pytest 9, so accept BOTH shapes + # and nothing else: a "::" line contributes its path prefix, a + # "path: " line contributes its path. The warnings summary + # and any other prose line matches neither and drops out. `sort -u` + # makes the split a pure function of the SORTED list: never `ls`, + # never filesystem order, never a hash of anything. + mkdir -p shards + awk '/::/ { t=$1; sub(/::.*/, "", t); if (t ~ /\.py$/) print t; next } + /^[^ ]*\.py: [0-9]+$/ { t=$1; sub(/:$/, "", t); print t }' \ + collect.txt | sort -u > shards/all.txt + test -s shards/all.txt || { echo "collection produced no test files" >&2; exit 1; } + n="$SHARD_COUNT" + for i in $(seq 0 $((n - 1))); do : > "shards/shard-$i.txt"; done + # Round-robin on the sorted list: file k goes to shard k % n. + i=0 + while IFS= read -r f; do + printf '%s\n' "$f" >> "shards/shard-$((i % n)).txt" + i=$((i + 1)) + done < shards/all.txt + # UNION CHECK. `cat shards/shard-*.txt | sort | uniq | wc -l` must + # equal `wc -l < shards/all.txt`, and the total line count across the + # shards must equal it too, so every collected file sits in exactly + # one shard: no file lost, none duplicated. The diff proves the union + # IS all.txt, not merely the same size. + all_count="$(wc -l < shards/all.txt)" + union_count="$(cat shards/shard-*.txt | sort -u | wc -l)" + placed_count="$(cat shards/shard-*.txt | wc -l)" + echo "collected test files: $all_count" + echo "union across shards: $union_count" + echo "lines across shards: $placed_count" + [ "$union_count" -eq "$all_count" ] || { + echo "the shard union does not cover every collected test file" >&2; exit 1; } + [ "$placed_count" -eq "$all_count" ] || { + echo "a collected test file sits in more than one shard" >&2; exit 1; } + diff -u <(sort shards/all.txt) <(cat shards/shard-*.txt | sort -u) || { + echo "the shard union differs from the collected file list" >&2; exit 1; } + # INDEPENDENT CROSS-CHECK. The union equality above alone cannot fail + # on a parse that silently DROPPED a collected file: the shards are + # built from all.txt, so a short all.txt yields a short union that + # still matches. Compare against the test files on disk instead -- + # `find tests -name 'test_*.py'` is the whole tree pytest collects + # from here -- so a dropped file fails the build loudly. + disk_count="$(find tests -name 'test_*.py' | wc -l)" + echo "test files on disk: $disk_count" + [ "$disk_count" -eq "$all_count" ] || { + echo "the collected file list does not cover every tests/**/test_*.py on disk" >&2; exit 1; } + diff -u <(find tests -name 'test_*.py' | sort) shards/all.txt || { + echo "the collected file list differs from the test files on disk" >&2; exit 1; } + # A shard that runs NOTHING must not pass. + for i in $(seq 0 $((n - 1))); do + test -s "shards/shard-$i.txt" || { echo "shard $i is empty" >&2; exit 1; } + done + # Record the split in the job log (the lists are small). + for i in $(seq 0 $((n - 1))); do + echo "--- shard $i: $(wc -l < "shards/shard-$i.txt") file(s)" + sed 's/^/ /' "shards/shard-$i.txt" + done + - name: Stage the engine environment for the shards + run: | + set -euo pipefail + # Sized deliberately so a shard runs WITHOUT rebuilding: the engine + # headers and the CMake-generated version header a compile test needs, + # the built runtime the e2e tests link, the engine `scripts/` the e2e + # harness imports (run_strategy.py, derive_corpus_feeds.py), the + # corpus's `.pine` strategy tree (the compile and emit sweeps glob it) + # and `corpus/data` (the e2e 1m feed and its derived 15m feed). The + # corpus's ~437 MB of per-strategy TradingView tapes under + # validation/ are NOT shipped: no test reads them. + mkdir -p engine-env/build/include engine-env/build/lib engine-env/corpus + cp -a engine/include engine-env/include + cp -a build/include/pineforge engine-env/build/include/pineforge + cp -a build/lib/libpineforge.a engine-env/build/lib/libpineforge.a + cp -a engine/scripts engine-env/scripts + rm -rf engine-env/scripts/__pycache__ + rsync -a --exclude='/data' --include='*/' --include='*.pine' --exclude='*' \ + engine/corpus/ engine-env/corpus/ + rsync -a engine/corpus/data/ engine-env/corpus/data/ + cp -a shards engine-env/shards + du -sh engine-env + test -f engine-env/include/pineforge/engine.hpp + test -f engine-env/build/include/pineforge/version.h + test -f engine-env/build/lib/libpineforge.a + test -f engine-env/scripts/run_strategy.py + test -d engine-env/corpus/data + - name: Upload the engine environment + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: engine-env + path: engine-env/ + if-no-files-found: error + + # The shard matrix: N jobs that each run ONE slice of the suite the build job + # pinned, against the `engine-env` artifact -- no shard rebuilds the engine. + # `fail-fast: false` so every shard reports; a failing shard must be visible + # as a failure, never absorbed. Same safety shape as `engine-build`: fork + # guard, read-only contents, head-sha checkout, no secret. + engine-shard: + needs: [preflight-docs, engine-build] + if: needs.preflight-docs.outputs.docs_only != 'true' && (github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name == github.repository) + runs-on: ubuntu-latest + timeout-minutes: 120 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + # Mirrors SHARD_COUNT in `engine-build`; the allowance file carries one + # `engine_built_shard_` profile per entry, for the same N. + shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + env: + PYTHONHASHSEED: "0" + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16e87065 + with: + python-version: "3.12" + - name: Install with dev extras + run: python -m pip install -e ".[dev]" + - name: Install Eigen, clang and g++ + run: | + sudo apt-get update + sudo apt-get install -y libeigen3-dev clang g++ + - name: Download the engine environment + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: engine-env + path: engine-env + - name: Export the engine environment for this shard + run: | + set -euo pipefail + test -f engine-env/include/pineforge/engine.hpp || { echo "artifact has no engine headers" >&2; exit 1; } + test -f engine-env/build/lib/libpineforge.a || { echo "artifact has no libpineforge.a" >&2; exit 1; } + test -d engine-env/corpus/data || { echo "artifact has no corpus data" >&2; exit 1; } + test -f engine-env/scripts/run_strategy.py || { echo "artifact has no engine scripts" >&2; exit 1; } + test -s "engine-env/shards/shard-${{ matrix.shard }}.txt" || { echo "shard ${{ matrix.shard }} is empty" >&2; exit 1; } + echo "PINEFORGE_ENGINE_LIB=$PWD/engine-env/build/lib/libpineforge.a" >> "$GITHUB_ENV" + echo "PINEFORGE_ENGINE_INCLUDE=$PWD/engine-env/include" >> "$GITHUB_ENV" + echo "PINEFORGE_ENGINE_CORPUS=$PWD/engine-env/corpus" >> "$GITHUB_ENV" + echo "PINEFORGE_SKIP_PROFILE=engine_built_shard_${{ matrix.shard }}" >> "$GITHUB_ENV" + - name: Assert the compile-environment skip gate is enabled + run: | + python - <<'PY' + import os + import sys + value = os.environ.get("PINEFORGE_SKIP_GATE", "").strip().lower() + if value == "off": + sys.stderr.write("PINEFORGE_SKIP_GATE=off is not allowed in CI: the skip gate must run\n") + sys.exit(1) + PY + - name: Run this shard with the engine present run: | set -eo pipefail - python -m pytest -q -rs 2>&1 | tee pytest.log + python -m pytest -q -rs $(cat "engine-env/shards/shard-${{ matrix.shard }}.txt") 2>&1 | tee pytest.log # Anchored to line start: the gate emits a leading empty line, since # pytest's -q progress line (".....[100%]") can lack a trailing newline. grep -q '^skip gate OK (default-deny)' pytest.log gate: if: always() - needs: [preflight-docs, differential, pytest, engine-built] + needs: [preflight-docs, differential, pytest, engine-build, engine-shard] runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -276,19 +442,25 @@ jobs: DOCS_ONLY: ${{ needs.preflight-docs.outputs.docs_only }} DIFFERENTIAL_RESULT: ${{ needs.differential.result }} PYTEST_RESULT: ${{ needs.pytest.result }} - ENGINE_BUILT_RESULT: ${{ needs.engine-built.result }} + # `needs..result` for a MATRIX job is the WORST result across the + # matrix, so `engine-shard` reading `success` means EVERY shard + # succeeded (with `fail-fast: false` one failed shard makes it + # `failure`). A build that fails leaves the shards `skipped`, which is + # not `success` either, so a failed build fails the gate too. + ENGINE_BUILD_RESULT: ${{ needs.engine-build.result }} + ENGINE_SHARD_RESULT: ${{ needs.engine-shard.result }} SAME_REPO: ${{ github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name == github.repository }} run: | set -euo pipefail state=failure description='Codegen gate did not pass' if [ "$PREFLIGHT_RESULT" = success ]; then - if [ "$DOCS_ONLY" = true ] && [ "$DIFFERENTIAL_RESULT" = skipped ] && [ "$PYTEST_RESULT" = skipped ] && [ "$ENGINE_BUILT_RESULT" = skipped ]; then + if [ "$DOCS_ONLY" = true ] && [ "$DIFFERENTIAL_RESULT" = skipped ] && [ "$PYTEST_RESULT" = skipped ] && [ "$ENGINE_BUILD_RESULT" = skipped ] && [ "$ENGINE_SHARD_RESULT" = skipped ]; then state=success description='docs-only preflight+docs; no sweep' - elif [ "$DOCS_ONLY" != true ] && [ "$SAME_REPO" = true ] && [ "$DIFFERENTIAL_RESULT" = success ] && [ "$PYTEST_RESULT" = success ] && [ "$ENGINE_BUILT_RESULT" = success ]; then + elif [ "$DOCS_ONLY" != true ] && [ "$SAME_REPO" = true ] && [ "$DIFFERENTIAL_RESULT" = success ] && [ "$PYTEST_RESULT" = success ] && [ "$ENGINE_BUILD_RESULT" = success ] && [ "$ENGINE_SHARD_RESULT" = success ]; then state=success - description='Full codegen differential gate, pytest and engine-built suite passed' + description='Full codegen differential gate, pytest and every engine-built shard passed' fi fi if [ "$state" != success ] && [ "$SAME_REPO" != true ]; then diff --git a/tests/ci_env_skip_allowance.json b/tests/ci_env_skip_allowance.json index 74b155b..72b25f9 100644 --- a/tests/ci_env_skip_allowance.json +++ b/tests/ci_env_skip_allowance.json @@ -32,5 +32,175 @@ "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, "engine include / compiler unavailable": 0, "got empty parameter set for (pine_file)": 1 + }, + "engine_built_shard_0": { + "_note": "UNMEASURED: engine_built_shard_0 is the profile the `engine-shard` matrix job runs for shard 0 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own round-robin slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", + "_unmeasured": true, + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, + "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, + "engine has no coded run-stop interface": 8, + "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, + "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, + "got empty parameter set for (case_name)": 3, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "authoritative engine-corpus source not available": 1, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, + "engine include / compiler unavailable": 0, + "got empty parameter set for (pine_file)": 1 + }, + "engine_built_shard_1": { + "_note": "UNMEASURED: engine_built_shard_1 is the profile the `engine-shard` matrix job runs for shard 1 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own round-robin slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", + "_unmeasured": true, + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, + "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, + "engine has no coded run-stop interface": 8, + "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, + "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, + "got empty parameter set for (case_name)": 3, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "authoritative engine-corpus source not available": 1, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, + "engine include / compiler unavailable": 0, + "got empty parameter set for (pine_file)": 1 + }, + "engine_built_shard_2": { + "_note": "UNMEASURED: engine_built_shard_2 is the profile the `engine-shard` matrix job runs for shard 2 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own round-robin slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", + "_unmeasured": true, + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, + "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, + "engine has no coded run-stop interface": 8, + "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, + "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, + "got empty parameter set for (case_name)": 3, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "authoritative engine-corpus source not available": 1, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, + "engine include / compiler unavailable": 0, + "got empty parameter set for (pine_file)": 1 + }, + "engine_built_shard_3": { + "_note": "UNMEASURED: engine_built_shard_3 is the profile the `engine-shard` matrix job runs for shard 3 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own round-robin slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", + "_unmeasured": true, + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, + "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, + "engine has no coded run-stop interface": 8, + "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, + "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, + "got empty parameter set for (case_name)": 3, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "authoritative engine-corpus source not available": 1, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, + "engine include / compiler unavailable": 0, + "got empty parameter set for (pine_file)": 1 + }, + "engine_built_shard_4": { + "_note": "UNMEASURED: engine_built_shard_4 is the profile the `engine-shard` matrix job runs for shard 4 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own round-robin slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", + "_unmeasured": true, + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, + "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, + "engine has no coded run-stop interface": 8, + "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, + "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, + "got empty parameter set for (case_name)": 3, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "authoritative engine-corpus source not available": 1, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, + "engine include / compiler unavailable": 0, + "got empty parameter set for (pine_file)": 1 + }, + "engine_built_shard_5": { + "_note": "UNMEASURED: engine_built_shard_5 is the profile the `engine-shard` matrix job runs for shard 5 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own round-robin slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", + "_unmeasured": true, + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, + "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, + "engine has no coded run-stop interface": 8, + "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, + "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, + "got empty parameter set for (case_name)": 3, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "authoritative engine-corpus source not available": 1, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, + "engine include / compiler unavailable": 0, + "got empty parameter set for (pine_file)": 1 + }, + "engine_built_shard_6": { + "_note": "UNMEASURED: engine_built_shard_6 is the profile the `engine-shard` matrix job runs for shard 6 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own round-robin slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", + "_unmeasured": true, + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, + "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, + "engine has no coded run-stop interface": 8, + "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, + "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, + "got empty parameter set for (case_name)": 3, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "authoritative engine-corpus source not available": 1, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, + "engine include / compiler unavailable": 0, + "got empty parameter set for (pine_file)": 1 + }, + "engine_built_shard_7": { + "_note": "UNMEASURED: engine_built_shard_7 is the profile the `engine-shard` matrix job runs for shard 7 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own round-robin slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", + "_unmeasured": true, + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, + "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, + "engine has no coded run-stop interface": 8, + "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, + "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, + "got empty parameter set for (case_name)": 3, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "authoritative engine-corpus source not available": 1, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, + "engine include / compiler unavailable": 0, + "got empty parameter set for (pine_file)": 1 + }, + "engine_built_shard_8": { + "_note": "UNMEASURED: engine_built_shard_8 is the profile the `engine-shard` matrix job runs for shard 8 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own round-robin slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", + "_unmeasured": true, + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, + "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, + "engine has no coded run-stop interface": 8, + "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, + "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, + "got empty parameter set for (case_name)": 3, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "authoritative engine-corpus source not available": 1, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, + "engine include / compiler unavailable": 0, + "got empty parameter set for (pine_file)": 1 + }, + "engine_built_shard_9": { + "_note": "UNMEASURED: engine_built_shard_9 is the profile the `engine-shard` matrix job runs for shard 9 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own round-robin slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", + "_unmeasured": true, + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, + "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, + "engine has no coded run-stop interface": 8, + "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, + "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, + "got empty parameter set for (case_name)": 3, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "authoritative engine-corpus source not available": 1, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, + "engine include / compiler unavailable": 0, + "got empty parameter set for (pine_file)": 1 } } From cd61bd40006b098cf553da51b552575c03b9780c Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 11 Oct 2026 07:35:45 +0800 Subject: [PATCH 10/16] ci: fix the shard action pin, ship docker, and hash the shard split --- .github/workflows/gate.yml | 30 +++++++++++------ tests/ci_env_skip_allowance.json | 21 ++++++------ tests/conftest.py | 2 +- tests/skip_gate.py | 57 +++++++++++++++++++++++--------- 4 files changed, 73 insertions(+), 37 deletions(-) diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index bc306e7..738cc6d 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -216,7 +216,7 @@ jobs: # real feed is on disk; the assertion below then re-checks its size. git -C engine/corpus lfs pull cmake -B build -S engine -DCMAKE_BUILD_TYPE=Release - cmake --build build -j"$(nproc)" + cmake --build build -j"$(nproc)" --target pineforge lib="$(find "$PWD/build" -name libpineforge.a -print -quit)" if [ -z "$lib" ]; then echo "the engine build produced no libpineforge.a" >&2 @@ -268,8 +268,8 @@ jobs: # and nothing else: a "::" line contributes its path prefix, a # "path: " line contributes its path. The warnings summary # and any other prose line matches neither and drops out. `sort -u` - # makes the split a pure function of the SORTED list: never `ls`, - # never filesystem order, never a hash of anything. + # gives a stable basis that never depends on `ls` or on filesystem + # order; a file's shard is a pure function of each sorted path. mkdir -p shards awk '/::/ { t=$1; sub(/::.*/, "", t); if (t ~ /\.py$/) print t; next } /^[^ ]*\.py: [0-9]+$/ { t=$1; sub(/:$/, "", t); print t }' \ @@ -277,11 +277,14 @@ jobs: test -s shards/all.txt || { echo "collection produced no test files" >&2; exit 1; } n="$SHARD_COUNT" for i in $(seq 0 $((n - 1))); do : > "shards/shard-$i.txt"; done - # Round-robin on the sorted list: file k goes to shard k % n. - i=0 + # Stable assignment: a file's shard is a pure function of ITS OWN PATH + # (POSIX cksum of the path, modulo n), not of its position in the list. + # A positional split moves every later file one shard over whenever a + # test file is added or removed, which changes the per-shard skip + # counts the allowance pins on roughly every other merge. while IFS= read -r f; do - printf '%s\n' "$f" >> "shards/shard-$((i % n)).txt" - i=$((i + 1)) + h="$(printf '%s' "$f" | cksum | cut -d' ' -f1)" + printf '%s\n' "$f" >> "shards/shard-$((h % n)).txt" done < shards/all.txt # UNION CHECK. `cat shards/shard-*.txt | sort | uniq | wc -l` must # equal `wc -l < shards/all.txt`, and the total line count across the @@ -327,9 +330,10 @@ jobs: # Sized deliberately so a shard runs WITHOUT rebuilding: the engine # headers and the CMake-generated version header a compile test needs, # the built runtime the e2e tests link, the engine `scripts/` the e2e - # harness imports (run_strategy.py, derive_corpus_feeds.py), the - # corpus's `.pine` strategy tree (the compile and emit sweeps glob it) - # and `corpus/data` (the e2e 1m feed and its derived 15m feed). The + # harness imports (run_strategy.py, derive_corpus_feeds.py) and the + # engine `docker/` (run_json.py, which the coded run-stop test runs), + # the corpus's `.pine` strategy tree (the compile and emit sweeps glob + # it) and `corpus/data` (the e2e 1m feed and its derived 15m feed). The # corpus's ~437 MB of per-strategy TradingView tapes under # validation/ are NOT shipped: no test reads them. mkdir -p engine-env/build/include engine-env/build/lib engine-env/corpus @@ -337,6 +341,7 @@ jobs: cp -a build/include/pineforge engine-env/build/include/pineforge cp -a build/lib/libpineforge.a engine-env/build/lib/libpineforge.a cp -a engine/scripts engine-env/scripts + cp -a engine/docker engine-env/docker rm -rf engine-env/scripts/__pycache__ rsync -a --exclude='/data' --include='*/' --include='*.pine' --exclude='*' \ engine/corpus/ engine-env/corpus/ @@ -347,6 +352,7 @@ jobs: test -f engine-env/build/include/pineforge/version.h test -f engine-env/build/lib/libpineforge.a test -f engine-env/scripts/run_strategy.py + test -f engine-env/docker/run_json.py test -d engine-env/corpus/data - name: Upload the engine environment uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 @@ -354,6 +360,7 @@ jobs: name: engine-env path: engine-env/ if-no-files-found: error + retention-days: 1 # The shard matrix: N jobs that each run ONE slice of the suite the build job # pinned, against the `engine-env` artifact -- no shard rebuilds the engine. @@ -381,7 +388,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha || github.sha }} fetch-depth: 0 persist-credentials: false - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16e87065 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 with: python-version: "3.12" - name: Install with dev extras @@ -402,6 +409,7 @@ jobs: test -f engine-env/build/lib/libpineforge.a || { echo "artifact has no libpineforge.a" >&2; exit 1; } test -d engine-env/corpus/data || { echo "artifact has no corpus data" >&2; exit 1; } test -f engine-env/scripts/run_strategy.py || { echo "artifact has no engine scripts" >&2; exit 1; } + test -f engine-env/docker/run_json.py || { echo "artifact has no engine docker/" >&2; exit 1; } test -s "engine-env/shards/shard-${{ matrix.shard }}.txt" || { echo "shard ${{ matrix.shard }} is empty" >&2; exit 1; } echo "PINEFORGE_ENGINE_LIB=$PWD/engine-env/build/lib/libpineforge.a" >> "$GITHUB_ENV" echo "PINEFORGE_ENGINE_INCLUDE=$PWD/engine-env/include" >> "$GITHUB_ENV" diff --git a/tests/ci_env_skip_allowance.json b/tests/ci_env_skip_allowance.json index 72b25f9..55d5281 100644 --- a/tests/ci_env_skip_allowance.json +++ b/tests/ci_env_skip_allowance.json @@ -19,6 +19,7 @@ "got empty parameter set for (pine_file)": 1 }, "engine_built": { + "_note": "engine_built is kept for a MANUAL single-job run; the CI jobs now run the engine_built_shard_ profiles, one per leg of the `engine-shard` matrix, so this profile is not the one they run.", "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, @@ -34,7 +35,7 @@ "got empty parameter set for (pine_file)": 1 }, "engine_built_shard_0": { - "_note": "UNMEASURED: engine_built_shard_0 is the profile the `engine-shard` matrix job runs for shard 0 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own round-robin slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", + "_note": "UNMEASURED: engine_built_shard_0 is the profile the `engine-shard` matrix job runs for shard 0 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own hash slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", "_unmeasured": true, "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, @@ -51,7 +52,7 @@ "got empty parameter set for (pine_file)": 1 }, "engine_built_shard_1": { - "_note": "UNMEASURED: engine_built_shard_1 is the profile the `engine-shard` matrix job runs for shard 1 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own round-robin slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", + "_note": "UNMEASURED: engine_built_shard_1 is the profile the `engine-shard` matrix job runs for shard 1 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own hash slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", "_unmeasured": true, "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, @@ -68,7 +69,7 @@ "got empty parameter set for (pine_file)": 1 }, "engine_built_shard_2": { - "_note": "UNMEASURED: engine_built_shard_2 is the profile the `engine-shard` matrix job runs for shard 2 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own round-robin slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", + "_note": "UNMEASURED: engine_built_shard_2 is the profile the `engine-shard` matrix job runs for shard 2 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own hash slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", "_unmeasured": true, "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, @@ -85,7 +86,7 @@ "got empty parameter set for (pine_file)": 1 }, "engine_built_shard_3": { - "_note": "UNMEASURED: engine_built_shard_3 is the profile the `engine-shard` matrix job runs for shard 3 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own round-robin slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", + "_note": "UNMEASURED: engine_built_shard_3 is the profile the `engine-shard` matrix job runs for shard 3 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own hash slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", "_unmeasured": true, "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, @@ -102,7 +103,7 @@ "got empty parameter set for (pine_file)": 1 }, "engine_built_shard_4": { - "_note": "UNMEASURED: engine_built_shard_4 is the profile the `engine-shard` matrix job runs for shard 4 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own round-robin slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", + "_note": "UNMEASURED: engine_built_shard_4 is the profile the `engine-shard` matrix job runs for shard 4 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own hash slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", "_unmeasured": true, "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, @@ -119,7 +120,7 @@ "got empty parameter set for (pine_file)": 1 }, "engine_built_shard_5": { - "_note": "UNMEASURED: engine_built_shard_5 is the profile the `engine-shard` matrix job runs for shard 5 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own round-robin slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", + "_note": "UNMEASURED: engine_built_shard_5 is the profile the `engine-shard` matrix job runs for shard 5 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own hash slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", "_unmeasured": true, "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, @@ -136,7 +137,7 @@ "got empty parameter set for (pine_file)": 1 }, "engine_built_shard_6": { - "_note": "UNMEASURED: engine_built_shard_6 is the profile the `engine-shard` matrix job runs for shard 6 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own round-robin slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", + "_note": "UNMEASURED: engine_built_shard_6 is the profile the `engine-shard` matrix job runs for shard 6 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own hash slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", "_unmeasured": true, "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, @@ -153,7 +154,7 @@ "got empty parameter set for (pine_file)": 1 }, "engine_built_shard_7": { - "_note": "UNMEASURED: engine_built_shard_7 is the profile the `engine-shard` matrix job runs for shard 7 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own round-robin slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", + "_note": "UNMEASURED: engine_built_shard_7 is the profile the `engine-shard` matrix job runs for shard 7 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own hash slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", "_unmeasured": true, "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, @@ -170,7 +171,7 @@ "got empty parameter set for (pine_file)": 1 }, "engine_built_shard_8": { - "_note": "UNMEASURED: engine_built_shard_8 is the profile the `engine-shard` matrix job runs for shard 8 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own round-robin slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", + "_note": "UNMEASURED: engine_built_shard_8 is the profile the `engine-shard` matrix job runs for shard 8 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own hash slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", "_unmeasured": true, "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, @@ -187,7 +188,7 @@ "got empty parameter set for (pine_file)": 1 }, "engine_built_shard_9": { - "_note": "UNMEASURED: engine_built_shard_9 is the profile the `engine-shard` matrix job runs for shard 9 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own round-robin slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", + "_note": "UNMEASURED: engine_built_shard_9 is the profile the `engine-shard` matrix job runs for shard 9 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own hash slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", "_unmeasured": true, "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, diff --git a/tests/conftest.py b/tests/conftest.py index 83bc6dc..6de6da0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,7 @@ third of it never ran. The gate is ON by default; ``PINEFORGE_SKIP_GATE=off`` disables it, and the CI -workflow asserts that variable is not set the gate cannot be turned off +workflow asserts that the variable is not set, so the gate cannot be turned off silently. """ diff --git a/tests/skip_gate.py b/tests/skip_gate.py index 890df6d..c7ffb19 100644 --- a/tests/skip_gate.py +++ b/tests/skip_gate.py @@ -21,12 +21,14 @@ The exact reason TEXT -- not a class -- is the key, so a reason that was reworded is drift too. The profile is chosen by ``PINEFORGE_SKIP_PROFILE`` -(default ``default``); the CI ``engine-built`` job sets ``engine_built``. +(default ``default``); the CI ``engine-shard`` matrix sets +``engine_built_shard_`` for each of its legs (the single-job ``engine_built`` +profile is kept only for a manual run, not for CI). ``classify`` and ``COMPILE_ENV_PATTERNS`` remain, but only to LABEL the log -lines (and to name, for the ``engine_built`` profile, the reasons that must be -zero there). They never decide pass or fail: the decision is the table below -alone. +lines and to enforce, for an ``engine_built*`` profile, that every +compile-environment reason is pinned at 0. They never decide pass or fail +otherwise: the decision is the table below alone. The session FAILS (non-zero exit) when: @@ -36,7 +38,10 @@ (d) the session reported no skip summary at all, while the profile lists any reason with count > 0 (a plugin that never ran is not a gate); (e) the selected profile is missing or empty, or the allowance file is - missing or unreadable. + missing or unreadable; + (f) the selected profile's name starts with ``engine_built`` and it allows + a nonzero count for any compile-environment reason (building the engine + must remove every one, so a nonzero allowance fails the run). The gate is ON by default and may be turned off only with ``PINEFORGE_SKIP_GATE=off`` -- the CI workflow asserts that variable is not @@ -87,10 +92,10 @@ def gate_disabled(env=None): """True when the gate is turned off via ``PINEFORGE_SKIP_GATE=off``. - This is the single source of truth for the disable switch: the workflow - assertion imports and calls it, so the CI check and the gate can never - disagree about the normalization (a shell reimplementation could, e.g. - on whitespace or a non-ASCII case). + This is the single source of truth for the disable switch: the gate + normalizes the variable here. CI does not call this; it asserts the + variable itself with a short inline check that the value is not ``off``, + so a workflow cannot turn the gate off silently. """ env = os.environ if env is None else env return str(env.get(GATE_ENV, "")).strip().lower() == "off" @@ -100,10 +105,10 @@ def classify(reason): """Return the class of a skip reason, or ``None`` when it is not compile-env. Matching is a case-insensitive substring test against - ``COMPILE_ENV_PATTERNS``. This is used ONLY to label the log lines and to - name the reasons the ``engine_built`` profile pins at 0 -- it never - decides whether the gate passes. The decision keys on the *exact* reason - text against the selected profile. + ``COMPILE_ENV_PATTERNS``. It labels the log lines and, for an + ``engine_built*`` profile, enforces that every compile-environment reason + is pinned at 0 (see ``check``); otherwise the decision keys on the *exact* + reason text against the selected profile. """ if not reason: return None @@ -173,13 +178,17 @@ def observed_counts(skipped_reports): return counts -def check(profile, observed, *, summary_present=True): +def check(profile, observed, *, summary_present=True, profile_name=None): """Compare observed skip counts against a profile; return a problem list. DEFAULT-DENY: this does not consult ``classify`` or any pattern list -- EVERY observed reason must be a key of the profile, with the exact count. ``profile`` maps an exact reason text to its expected count; ``observed`` maps an exact reason text to its count. An empty list means PASS. + + When ``profile_name`` starts with ``engine_built``, any compile-environment + reason (per ``classify``) allowed a NONZERO count is a problem: building the + engine and pinning the corpus must remove every such skip. """ if not isinstance(profile, dict): return ["the selected allowance profile is missing or empty"] @@ -216,6 +225,22 @@ def check(profile, observed, *, summary_present=True): f"count mismatch: {reason!r} expected {expected}, observed {got}" ) + # (f) an engine_built profile must pin every compile-environment reason at 0: + # building the engine is exactly what removes them, so a nonzero allowance + # is a broken gate, not a tolerance. + if profile_name is not None and str(profile_name).startswith("engine_built"): + for reason, expected in sorted(profile.items()): + if ( + isinstance(expected, int) + and expected != 0 + and classify(reason) == CLASS_COMPILE_ENV + ): + problems.append( + f"engine_built profile {profile_name!r} allows {expected} " + f"compile-environment skip(s) for {reason!r}; the engine build " + "must remove every one, so the allowance must pin it at 0" + ) + return problems @@ -262,7 +287,9 @@ def run_gate(terminalreporter, allowance_path=ALLOWANCE_PATH, env=None): observed, ) - problems = check(profile, observed, summary_present=summary_present) + problems = check( + profile, observed, summary_present=summary_present, profile_name=profile_name + ) return problems, observed From 248744573e6adbb336f03164f91e11c6a512e2f1 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 11 Oct 2026 07:53:09 +0800 Subject: [PATCH 11/16] ci: exempt the sources a build cannot fetch from the zero-skip rule --- tests/ci_env_skip_allowance.json | 4 +- tests/skip_gate.py | 50 +++++++++++++++++------- tests/test_skip_gate_profiles.py | 67 ++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 16 deletions(-) create mode 100644 tests/test_skip_gate_profiles.py diff --git a/tests/ci_env_skip_allowance.json b/tests/ci_env_skip_allowance.json index 55d5281..809d2ad 100644 --- a/tests/ci_env_skip_allowance.json +++ b/tests/ci_env_skip_allowance.json @@ -1,8 +1,8 @@ { - "_note": "DEFAULT-DENY allowance for the skip gate (tests/skip_gate.py). Every skip reason that occurs, whatever its wording, must appear here with its exact count; there is no allow-list of 'known' wordings. classify()/COMPILE_ENV_PATTERNS only label the log lines and name which reasons the 'engine_built' profile pins at 0. Keys are the EXACT skip reason text (a renamed reason is drift too). 'default' is the profile the engine-less hosted-CI pytest job runs; its counts are the WHOLE skip population of the measured run (2,358 skips across 13 reasons), not only the compile-environment ones. 'engine_built' is the profile the engine-building CI job runs: the compile-environment reasons are pinned at 0 so any such skip fails that job; see _engine_built_note.", + "_note": "DEFAULT-DENY allowance for the skip gate (tests/skip_gate.py). Every skip reason that occurs, whatever its wording, must appear here with its exact count; there is no allow-list of 'known' wordings. classify()/COMPILE_ENV_PATTERNS only label the log lines and name which reasons the 'engine_built' profile pins at 0 -- every compile-environment reason EXCEPT the ones building the engine cannot remove (ENGINE_BUILD_KEEPS in tests/skip_gate.py: the population-string-settings sources, which are not public), which keep their measured count. Keys are the EXACT skip reason text (a renamed reason is drift too). 'default' is the profile the engine-less hosted-CI pytest job runs; its counts are the WHOLE skip population of the measured run (2,358 skips across 13 reasons), not only the compile-environment ones. 'engine_built' is the profile the engine-building CI job runs: the compile-environment reasons the build removes are pinned at 0 so any such skip fails that job, while the reasons it cannot remove keep their measured count; see _engine_built_note.", "_unmeasured": false, "_measured_on": "2026-10-11 spot box (Ubuntu 24.04, python 3.12), engine-less with NO PINEFORGE_* variables set; the counts are the WHOLE skip population of that run (2,358 skips across 13 reasons), not only the compile-environment skips", - "_engine_built_note": "UNMEASURED. The engine-building CI job runs this profile, but the counts below are NOT a measurement of that job: they are copies of the engine-less `default` run and are EXPECTED to be WRONG. That job now DOES set PINEFORGE_ENGINE_CORPUS, cloning the public pineforge-4pass/pineforge-corpus at b40aa8ecbe3caa9ad05244cdfd9ee8138da845e4 (the gitlink the pinned engine commit 4e01e96b5165b5d5e46d72bfbbf88ae6c035a6e9 records), so its corpus tests DO run. It does NOT set PINEFORGE_STRING_SETTINGS_SOURCES, because the population sources that variable would point at are NOT public: this repository carries no fixture for them and the public corpus checkout above has no file matching the sha12 names the test globs, so its six population-source cases skip -- hence that row is 6, not 0. Expect the FIRST engine-built run to FAIL on the other rows and the gate to print the exact rows to write here; paste those rows in to measure this profile. The compile-environment rows (the 'PINEFORGE_ENGINE_INCLUDE not set or invalid...' and 'set PINEFORGE_ENGINE_INCLUDE...' rows, 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') must stay 0: building the engine and pinning the public corpus must remove every one.", + "_engine_built_note": "UNMEASURED. The engine-building CI job runs this profile, but the counts below are NOT a measurement of that job: they are copies of the engine-less `default` run and are EXPECTED to be WRONG. That job now DOES set PINEFORGE_ENGINE_CORPUS, cloning the public pineforge-4pass/pineforge-corpus at b40aa8ecbe3caa9ad05244cdfd9ee8138da845e4 (the gitlink the pinned engine commit 4e01e96b5165b5d5e46d72bfbbf88ae6c035a6e9 records), so its corpus tests DO run. It does NOT set PINEFORGE_STRING_SETTINGS_SOURCES, because the population sources that variable would point at are NOT public: this repository carries no fixture for them and the public corpus checkout above has no file matching the sha12 names the test globs, so its six population-source cases skip -- hence that row is 6, not 0. Expect the FIRST engine-built run to FAIL on the other rows and the gate to print the exact rows to write here; paste those rows in to measure this profile. The compile-environment rows (the 'PINEFORGE_ENGINE_INCLUDE not set or invalid...' and 'set PINEFORGE_ENGINE_INCLUDE...' rows, 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') must stay 0: building the engine and pinning the public corpus must remove every one -- every one EXCEPT the 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row, which stays 6 because the sources that knob points at are not public, so the build cannot remove it (ENGINE_BUILD_KEEPS in tests/skip_gate.py).", "default": { "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 2261, "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 44, diff --git a/tests/skip_gate.py b/tests/skip_gate.py index c7ffb19..e9f4569 100644 --- a/tests/skip_gate.py +++ b/tests/skip_gate.py @@ -27,8 +27,10 @@ ``classify`` and ``COMPILE_ENV_PATTERNS`` remain, but only to LABEL the log lines and to enforce, for an ``engine_built*`` profile, that every -compile-environment reason is pinned at 0. They never decide pass or fail -otherwise: the decision is the table below alone. +compile-environment reason building the engine removes is pinned at 0 -- the +ones it cannot remove (``ENGINE_BUILD_KEEPS``: the population-string-settings +sources, which are not public) keep their measured count. They never decide +pass or fail otherwise: the decision is the table below alone. The session FAILS (non-zero exit) when: @@ -40,8 +42,11 @@ (e) the selected profile is missing or empty, or the allowance file is missing or unreadable; (f) the selected profile's name starts with ``engine_built`` and it allows - a nonzero count for any compile-environment reason (building the engine - must remove every one, so a nonzero allowance fails the run). + a nonzero count for any compile-environment reason building the engine + must remove -- that is, every such reason EXCEPT the ones it cannot + remove (``ENGINE_BUILD_KEEPS``: the population-string-settings sources, + which are not public), each of which keeps its measured count. A + nonzero allowance for a reason the build removes fails the run. The gate is ON by default and may be turned off only with ``PINEFORGE_SKIP_GATE=off`` -- the CI workflow asserts that variable is not @@ -62,7 +67,9 @@ # these knobs (case-insensitive substring match). This list is NO LONGER a # decision: the gate is default-deny, so an unknown reason fails wherever it # comes from. The class only LABELS the log lines and names the reasons the -# ``engine_built`` profile pins at 0. It is deliberately exhaustive about +# ``engine_built`` profile pins at 0 -- apart from ``ENGINE_BUILD_KEEPS``, the +# compile-environment reasons building the engine cannot remove, which keep +# their measured count. It is deliberately exhaustive about # *wording families*: the engine library archive, the Eigen and generated # headers, the compiler/AST tooling, the parity corpus and the # population-sources directory a compile or byte-comparison test needs. @@ -84,6 +91,11 @@ CLASS_COMPILE_ENV = "COMPILE-ENVIRONMENT" +# Compile-environment reasons that building the engine does NOT remove: the population +# sources the string-settings tests read are not public, so those six skips stay real in an +# engine-built run and an engine_built* profile pins them at their measured count, not at 0. +ENGINE_BUILD_KEEPS = ("PINEFORGE_STRING_SETTINGS_SOURCES", "population sources require") + GATE_ENV = "PINEFORGE_SKIP_GATE" PROFILE_ENV = "PINEFORGE_SKIP_PROFILE" DEFAULT_PROFILE = "default" @@ -107,8 +119,10 @@ def classify(reason): Matching is a case-insensitive substring test against ``COMPILE_ENV_PATTERNS``. It labels the log lines and, for an ``engine_built*`` profile, enforces that every compile-environment reason - is pinned at 0 (see ``check``); otherwise the decision keys on the *exact* - reason text against the selected profile. + building the engine removes is pinned at 0 -- the ones it cannot remove + (``ENGINE_BUILD_KEEPS``) keep their measured count (see ``check``); + otherwise the decision keys on the *exact* reason text against the + selected profile. """ if not reason: return None @@ -181,14 +195,19 @@ def observed_counts(skipped_reports): def check(profile, observed, *, summary_present=True, profile_name=None): """Compare observed skip counts against a profile; return a problem list. - DEFAULT-DENY: this does not consult ``classify`` or any pattern list -- - EVERY observed reason must be a key of the profile, with the exact count. + DEFAULT-DENY: the table alone decides -- EVERY observed reason must be a + key of the profile, with the exact count. (``classify`` and + ``ENGINE_BUILD_KEEPS`` are consulted only by the extra ``engine_built`` rule + below, never to decide pass or fail on their own.) ``profile`` maps an exact reason text to its expected count; ``observed`` maps an exact reason text to its count. An empty list means PASS. When ``profile_name`` starts with ``engine_built``, any compile-environment - reason (per ``classify``) allowed a NONZERO count is a problem: building the - engine and pinning the corpus must remove every such skip. + reason (per ``classify``) allowed a NONZERO count is a problem -- EXCEPT the + reasons building the engine cannot remove (``ENGINE_BUILD_KEEPS``: the + population-string-settings sources, which are not public), which keep their + measured count: building the engine and pinning the corpus must remove every + other such skip. """ if not isinstance(profile, dict): return ["the selected allowance profile is missing or empty"] @@ -225,15 +244,18 @@ def check(profile, observed, *, summary_present=True, profile_name=None): f"count mismatch: {reason!r} expected {expected}, observed {got}" ) - # (f) an engine_built profile must pin every compile-environment reason at 0: - # building the engine is exactly what removes them, so a nonzero allowance - # is a broken gate, not a tolerance. + # (f) an engine_built profile must pin every compile-environment reason at 0 + # EXCEPT the reasons building the engine cannot remove (``ENGINE_BUILD_KEEPS``: + # the population-string-settings sources, which are not public): building the + # engine removes every other one, so a nonzero allowance for one is a broken + # gate, not a tolerance. if profile_name is not None and str(profile_name).startswith("engine_built"): for reason, expected in sorted(profile.items()): if ( isinstance(expected, int) and expected != 0 and classify(reason) == CLASS_COMPILE_ENV + and not any(p.lower() in str(reason).lower() for p in ENGINE_BUILD_KEEPS) ): problems.append( f"engine_built profile {profile_name!r} allows {expected} " diff --git a/tests/test_skip_gate_profiles.py b/tests/test_skip_gate_profiles.py new file mode 100644 index 0000000..1e921c2 --- /dev/null +++ b/tests/test_skip_gate_profiles.py @@ -0,0 +1,67 @@ +"""The shipped ``engine_built*`` profiles must satisfy the gate's own rules. + +Engine-free, environment-free and self-contained: this test loads the allowance +JSON beside :mod:`tests.skip_gate` and, for every profile whose name starts +with ``engine_built``, runs ``skip_gate.check`` with the profile's OWN declared +counts as the observation (its metadata keys stripped). The profile then has +to come back clean -- so a rule that contradicts the shipped data, for example +an ``engine_built`` rule demanding a 0 the build cannot produce, fails HERE, in +the engine-less run, instead of only on the CI leg that actually executes that +profile. + +No engine, no ``PINEFORGE_*`` variable, no fixture and no skip: it reads the +shipped file and calls the pure ``check`` function. If a profile is +inconsistent, the assertion names it (the fix is either the profile's counts or +the rule, never this test). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from tests import skip_gate + +_METADATA_PREFIX = "_" + + +def _load_allowance(): + """Return the shipped allowance JSON, failing loudly if it is unreadable.""" + path = Path(skip_gate.ALLOWANCE_PATH) + return json.loads(path.read_text(encoding="utf-8")) + + +def _rows(profile): + """A profile's skip rows: everything but its ``_``-prefixed metadata. + + ``_note``, ``_unmeasured`` and any other ``_`` key are human metadata, not + skip reasons, so they are stripped from both the profile and the + observation passed to ``check`` -- ``check`` iterates a profile's keys and + would flag a ``_note``'s string value as a non-integer count. + """ + return { + key: value + for key, value in profile.items() + if not str(key).startswith(_METADATA_PREFIX) + } + + +def test_engine_built_profiles_pass_the_gate_on_their_own_counts(): + allowance = _load_allowance() + names = [name for name in allowance if str(name).startswith("engine_built")] + assert names, "no engine_built* profile in the shipped allowance" + + failures = [] + for name in names: + rows = _rows(allowance[name]) + problems = skip_gate.check( + rows, rows, summary_present=True, profile_name=name + ) + if problems: + failures.append((name, problems)) + + assert not failures, "\n".join( + f"{name} is inconsistent with the skip gate's own rules:\n " + + "\n ".join(problems) + for name, problems in failures + ) From effb6aba24749ea193ae03214295658ed79ca07f Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 11 Oct 2026 05:30:43 +0800 Subject: [PATCH 12/16] tests: update the gate-corpus fixture comment wording --- .../ok/validation__order-process-on-close-false-01.pine | 2 +- .../ok/validation__order-process-on-close-true-01.pine | 2 +- .../ok/validation__pyramid-cash-fractional-commission-01.pine | 2 +- .../ok/validation__pyramid-deferred-flip-close-all-01.pine | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/gate-corpus/ok/validation__order-process-on-close-false-01.pine b/tests/gate-corpus/ok/validation__order-process-on-close-false-01.pine index fdc3540..a6c3df7 100644 --- a/tests/gate-corpus/ok/validation__order-process-on-close-false-01.pine +++ b/tests/gate-corpus/ok/validation__order-process-on-close-false-01.pine @@ -14,7 +14,7 @@ // exit on EMA cross-under. ≥10 cycles over 36k bars. // // TV setup: 15m chart, ETH-USDT-USDT, same window as -// data/ohlcv_ETH-USDT-USDT_15m.csv. Manual TV capture later. +// data/ohlcv_ETH-USDT-USDT_15m.csv. TradingView strategy report for this window. //@version=6 strategy("PF process-orders 01 - on_close=false", shorttitle="POC_p01_FALSE", overlay=true, initial_capital=1000000, currency=currency.USD, diff --git a/tests/gate-corpus/ok/validation__order-process-on-close-true-01.pine b/tests/gate-corpus/ok/validation__order-process-on-close-true-01.pine index 5c80294..ff74540 100644 --- a/tests/gate-corpus/ok/validation__order-process-on-close-true-01.pine +++ b/tests/gate-corpus/ok/validation__order-process-on-close-true-01.pine @@ -15,7 +15,7 @@ // bars. // // TV setup: 15m chart, ETH-USDT-USDT, same window as -// data/ohlcv_ETH-USDT-USDT_15m.csv. Manual TV capture later. +// data/ohlcv_ETH-USDT-USDT_15m.csv. TradingView strategy report for this window. //@version=6 strategy("PF process-orders 02 - on_close=true", shorttitle="POC_p02_TRUE", overlay=true, initial_capital=1000000, currency=currency.USD, diff --git a/tests/gate-corpus/ok/validation__pyramid-cash-fractional-commission-01.pine b/tests/gate-corpus/ok/validation__pyramid-cash-fractional-commission-01.pine index ade9c61..39b6441 100644 --- a/tests/gate-corpus/ok/validation__pyramid-cash-fractional-commission-01.pine +++ b/tests/gate-corpus/ok/validation__pyramid-cash-fractional-commission-01.pine @@ -16,7 +16,7 @@ // (hour==23, minute==45) closes all. ≥10 cycles over 36k bars. // // TV setup: 15m chart, ETH-USDT-USDT, same window as -// data/ohlcv_ETH-USDT-USDT_15m.csv. Manual TV capture later. +// data/ohlcv_ETH-USDT-USDT_15m.csv. TradingView strategy report for this window. //@version=6 strategy("PF Pyramid carry 02 - cash fractional", shorttitle="PYR_p02_CASH", overlay=true, initial_capital=1000000, currency=currency.USD, diff --git a/tests/gate-corpus/ok/validation__pyramid-deferred-flip-close-all-01.pine b/tests/gate-corpus/ok/validation__pyramid-deferred-flip-close-all-01.pine index 2dc5d2a..97d81d1 100644 --- a/tests/gate-corpus/ok/validation__pyramid-deferred-flip-close-all-01.pine +++ b/tests/gate-corpus/ok/validation__pyramid-deferred-flip-close-all-01.pine @@ -18,7 +18,7 @@ // next bar after the flat reset. // // TV setup: 15m chart, ETH-USDT-USDT, same window as -// data/ohlcv_ETH-USDT-USDT_15m.csv. Manual TV capture later. +// data/ohlcv_ETH-USDT-USDT_15m.csv. TradingView strategy report for this window. //@version=6 strategy("PF Pyramid carry 01 - deferred flip", shorttitle="PYR_p01_FLIP", overlay=true, initial_capital=1000000, currency=currency.USD, From eef9d9561d1431649abe90130d4f6d5a45e7d2f1 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 11 Oct 2026 08:11:54 +0800 Subject: [PATCH 13/16] ci: cache the corpus LFS objects and stop re-running on a PR edit --- .github/workflows/gate.yml | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index 738cc6d..d9ccb85 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -5,7 +5,9 @@ on: schedule: - cron: "27 5 * * *" pull_request_target: - types: [opened, synchronize, reopened, edited, ready_for_review] + # `edited` is deliberately absent: an edit to a PR's title or body used to + # re-run this whole workflow, LFS pull and engine build included. + types: [opened, synchronize, reopened, ready_for_review] permissions: {} @@ -195,7 +197,7 @@ jobs: sudo apt-get update sudo apt-get install -y libeigen3-dev clang cmake g++ ninja-build git-lfs git lfs install --skip-repo - - name: Clone and build the paired engine + - name: Clone the paired engine and check out its pinned corpus run: | set -euo pipefail git clone https://github.com/pineforge-4pass/pineforge-engine.git engine @@ -209,12 +211,31 @@ jobs: # no URL rewrite is needed; the update checks out the pinned gitlink. git -C engine submodule update --init corpus git -C engine/corpus checkout "$CORPUS_COMMIT" - # The corpus's 1m feed is stored in Git LFS: a plain clone/submodule - # update leaves a ~130-byte POINTER at data/ohlcv_ETH-USDT-USDT_1m.csv - # that passes a bare `test -f`, while the ~176 MB of data the e2e - # harness needs is absent. Pull the pinned commit's LFS objects so the - # real feed is on disk; the assertion below then re-checks its size. + # The corpus's 1m feed is stored in Git LFS: a plain clone/submodule + # update leaves a ~130-byte POINTER at data/ohlcv_ETH-USDT-USDT_1m.csv + # that passes a bare `test -f`, while the ~176 MB of data the e2e + # harness needs is absent. `corpus` is a SUBMODULE, so its LFS store is + # the submodule's own git dir, engine/.git/modules/corpus/lfs/objects + # (not engine/corpus/.git/...): cache exactly that store, keyed by the + # pinned gitlink sha the job already exports as CORPUS_COMMIT. The key is + # content-addressed, so the objects are pulled only when CORPUS_COMMIT + # moves; a restored cache turns the `lfs pull` below into a no-op, and + # the >= 1000000-byte feed assert that follows re-checks the real bytes. + # The cache step MUST sit before the pull. + - name: Restore the corpus LFS object cache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 + with: + path: engine/.git/modules/corpus/lfs/objects + key: corpus-lfs-${{ env.CORPUS_COMMIT }} + # The pull still runs: it is what smudges the pointer files into the real + # feed on disk, and with a cache hit it downloads nothing. + - name: Pull the pinned corpus LFS objects + run: | + set -euo pipefail git -C engine/corpus lfs pull + - name: Build the paired engine + run: | + set -euo pipefail cmake -B build -S engine -DCMAKE_BUILD_TYPE=Release cmake --build build -j"$(nproc)" --target pineforge lib="$(find "$PWD/build" -name libpineforge.a -print -quit)" From c34b472aa513e1e1d51da84d214af1808fc8e459 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 11 Oct 2026 08:33:02 +0800 Subject: [PATCH 14/16] ci: leave the corpus to the cache and the pull, not the checkout --- .github/workflows/gate.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index d9ccb85..eecb18d 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -198,6 +198,8 @@ jobs: sudo apt-get install -y libeigen3-dev clang cmake g++ ninja-build git-lfs git lfs install --skip-repo - name: Clone the paired engine and check out its pinned corpus + env: + GIT_LFS_SKIP_SMUDGE: "1" run: | set -euo pipefail git clone https://github.com/pineforge-4pass/pineforge-engine.git engine @@ -220,10 +222,10 @@ jobs: # pinned gitlink sha the job already exports as CORPUS_COMMIT. The key is # content-addressed, so the objects are pulled only when CORPUS_COMMIT # moves; a restored cache turns the `lfs pull` below into a no-op, and - # the >= 1000000-byte feed assert that follows re-checks the real bytes. + # the >= 1000000-byte feed assert that follows re-checks the real size. # The cache step MUST sit before the pull. - name: Restore the corpus LFS object cache - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: engine/.git/modules/corpus/lfs/objects key: corpus-lfs-${{ env.CORPUS_COMMIT }} From 1ba52e1a3c7d3b2b02618be1d37f519b81cfcad7 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 11 Oct 2026 10:04:07 +0800 Subject: [PATCH 15/16] ci: measure the per-shard skip allowance from the first gate run --- tests/ci_env_skip_allowance.json | 185 +++++++------------------------ 1 file changed, 37 insertions(+), 148 deletions(-) diff --git a/tests/ci_env_skip_allowance.json b/tests/ci_env_skip_allowance.json index 809d2ad..4beaa5c 100644 --- a/tests/ci_env_skip_allowance.json +++ b/tests/ci_env_skip_allowance.json @@ -35,173 +35,62 @@ "got empty parameter set for (pine_file)": 1 }, "engine_built_shard_0": { - "_note": "UNMEASURED: engine_built_shard_0 is the profile the `engine-shard` matrix job runs for shard 0 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own hash slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", - "_unmeasured": true, - "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, - "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, - "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, - "engine has no coded run-stop interface": 8, - "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, - "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, - "got empty parameter set for (case_name)": 3, - "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, - "authoritative engine-corpus source not available": 1, - "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, - "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, - "engine include / compiler unavailable": 0, - "got empty parameter set for (pine_file)": 1 + "_note": "MEASURED: engine_built_shard_0 is the profile the `engine-shard` matrix job runs for shard 0 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. This shard's skip population is EMPTY, which is correct: with a summary present and no skips, default-deny passes.", + "_unmeasured": false, + "_measured_on": "MEASURED on run 38099268385, shard 0 (engine-shard matrix leg), 2026-10-11" }, "engine_built_shard_1": { - "_note": "UNMEASURED: engine_built_shard_1 is the profile the `engine-shard` matrix job runs for shard 1 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own hash slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", - "_unmeasured": true, - "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, - "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, - "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, - "engine has no coded run-stop interface": 8, - "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, - "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, - "got empty parameter set for (case_name)": 3, - "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, - "authoritative engine-corpus source not available": 1, - "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, - "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, - "engine include / compiler unavailable": 0, - "got empty parameter set for (pine_file)": 1 + "_note": "MEASURED: engine_built_shard_1 is the profile the `engine-shard` matrix job runs for shard 1 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. This shard's skip population is EMPTY, which is correct: with a summary present and no skips, default-deny passes.", + "_unmeasured": false, + "_measured_on": "MEASURED on run 38099268385, shard 1 (engine-shard matrix leg), 2026-10-11" }, "engine_built_shard_2": { - "_note": "UNMEASURED: engine_built_shard_2 is the profile the `engine-shard` matrix job runs for shard 2 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own hash slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", - "_unmeasured": true, - "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, - "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, - "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, - "engine has no coded run-stop interface": 8, - "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, - "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, - "got empty parameter set for (case_name)": 3, - "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, - "authoritative engine-corpus source not available": 1, - "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, - "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, - "engine include / compiler unavailable": 0, - "got empty parameter set for (pine_file)": 1 + "_note": "MEASURED: engine_built_shard_2 is the profile the `engine-shard` matrix job runs for shard 2 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. This shard's skip population is EMPTY, which is correct: with a summary present and no skips, default-deny passes.", + "_unmeasured": false, + "_measured_on": "MEASURED on run 38099268385, shard 2 (engine-shard matrix leg), 2026-10-11" }, "engine_built_shard_3": { - "_note": "UNMEASURED: engine_built_shard_3 is the profile the `engine-shard` matrix job runs for shard 3 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own hash slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", - "_unmeasured": true, - "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, - "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, - "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, - "engine has no coded run-stop interface": 8, - "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, - "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, - "got empty parameter set for (case_name)": 3, - "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, - "authoritative engine-corpus source not available": 1, - "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, - "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, - "engine include / compiler unavailable": 0, - "got empty parameter set for (pine_file)": 1 + "_note": "MEASURED: engine_built_shard_3 is the profile the `engine-shard` matrix job runs for shard 3 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one.", + "_unmeasured": false, + "_measured_on": "MEASURED on run 38099268385, shard 3 (engine-shard matrix leg), 2026-10-11", + "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 1 }, "engine_built_shard_4": { - "_note": "UNMEASURED: engine_built_shard_4 is the profile the `engine-shard` matrix job runs for shard 4 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own hash slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", - "_unmeasured": true, - "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, - "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, - "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, - "engine has no coded run-stop interface": 8, - "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, - "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, - "got empty parameter set for (case_name)": 3, - "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, - "authoritative engine-corpus source not available": 1, - "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, - "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, - "engine include / compiler unavailable": 0, - "got empty parameter set for (pine_file)": 1 + "_note": "MEASURED: engine_built_shard_4 is the profile the `engine-shard` matrix job runs for shard 4 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one.", + "_unmeasured": false, + "_measured_on": "MEASURED on run 38099268385, shard 4 (engine-shard matrix leg), 2026-10-11", + "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 5 }, "engine_built_shard_5": { - "_note": "UNMEASURED: engine_built_shard_5 is the profile the `engine-shard` matrix job runs for shard 5 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own hash slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", - "_unmeasured": true, - "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, - "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, - "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, - "engine has no coded run-stop interface": 8, - "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, - "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, - "got empty parameter set for (case_name)": 3, - "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, - "authoritative engine-corpus source not available": 1, - "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, - "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, - "engine include / compiler unavailable": 0, - "got empty parameter set for (pine_file)": 1 + "_note": "MEASURED: engine_built_shard_5 is the profile the `engine-shard` matrix job runs for shard 5 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one.", + "_unmeasured": false, + "_measured_on": "MEASURED on run 38099268385, shard 5 (engine-shard matrix leg), 2026-10-11", + "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 25 }, "engine_built_shard_6": { - "_note": "UNMEASURED: engine_built_shard_6 is the profile the `engine-shard` matrix job runs for shard 6 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own hash slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", - "_unmeasured": true, - "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, - "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, - "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, - "engine has no coded run-stop interface": 8, - "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, - "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, - "got empty parameter set for (case_name)": 3, - "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, - "authoritative engine-corpus source not available": 1, - "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, - "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, - "engine include / compiler unavailable": 0, - "got empty parameter set for (pine_file)": 1 + "_note": "MEASURED: engine_built_shard_6 is the profile the `engine-shard` matrix job runs for shard 6 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. This shard's skip population is EMPTY, which is correct: with a summary present and no skips, default-deny passes.", + "_unmeasured": false, + "_measured_on": "MEASURED on run 38099268385, shard 6 (engine-shard matrix leg), 2026-10-11" }, "engine_built_shard_7": { - "_note": "UNMEASURED: engine_built_shard_7 is the profile the `engine-shard` matrix job runs for shard 7 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own hash slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", - "_unmeasured": true, - "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, - "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, - "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, - "engine has no coded run-stop interface": 8, - "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, + "_note": "MEASURED: engine_built_shard_7 is the profile the `engine-shard` matrix job runs for shard 7 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row is 6 because that knob's sources are not public, so the build cannot close it (ENGINE_BUILD_KEEPS in tests/skip_gate.py).", + "_unmeasured": false, + "_measured_on": "MEASURED on run 38099268385, shard 7 (engine-shard matrix leg), 2026-10-11", "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, - "got empty parameter set for (case_name)": 3, - "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, - "authoritative engine-corpus source not available": 1, - "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, - "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, - "engine include / compiler unavailable": 0, "got empty parameter set for (pine_file)": 1 }, "engine_built_shard_8": { - "_note": "UNMEASURED: engine_built_shard_8 is the profile the `engine-shard` matrix job runs for shard 8 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own hash slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", - "_unmeasured": true, - "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, - "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, - "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, - "engine has no coded run-stop interface": 8, - "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, - "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, - "got empty parameter set for (case_name)": 3, - "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, - "authoritative engine-corpus source not available": 1, - "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, - "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, - "engine include / compiler unavailable": 0, - "got empty parameter set for (pine_file)": 1 + "_note": "MEASURED: engine_built_shard_8 is the profile the `engine-shard` matrix job runs for shard 8 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one.", + "_unmeasured": false, + "_measured_on": "MEASURED on run 38099268385, shard 8 (engine-shard matrix leg), 2026-10-11", + "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 1, + "got empty parameter set for (case_name)": 2 }, "engine_built_shard_9": { - "_note": "UNMEASURED: engine_built_shard_9 is the profile the `engine-shard` matrix job runs for shard 9 of 10. It is a COPY of the `engine_built` profile (the compile-environment reasons at 0, every other reason at its engine_built count) and is EXPECTED to be wrong: a shard runs only its own hash slice of the collected test files, so a reason whose tests all land in another shard is counted in neither direction here. The compile-environment rows ('PINEFORGE_ENGINE_INCLUDE not set or invalid...', 'set PINEFORGE_ENGINE_INCLUDE...', 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') are 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard's FIRST run prints the exact rows to paste in; replace the copied counts with those rows and set _unmeasured per profile to false when done. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row stays at 6, its `engine_built` value: that knob's sources are not public, so the skip is real and the artifact cannot close it (see `_engine_built_note`).", - "_unmeasured": true, - "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, - "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, - "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, - "engine has no coded run-stop interface": 8, - "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, - "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, - "got empty parameter set for (case_name)": 3, - "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, - "authoritative engine-corpus source not available": 1, - "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, - "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, - "engine include / compiler unavailable": 0, - "got empty parameter set for (pine_file)": 1 + "_note": "MEASURED: engine_built_shard_9 is the profile the `engine-shard` matrix job runs for shard 9 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one.", + "_unmeasured": false, + "_measured_on": "MEASURED on run 38099268385, shard 9 (engine-shard matrix leg), 2026-10-11", + "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 1, + "got empty parameter set for (case_name)": 1 } } From 3490c9156ff98388a43ab97bab64db285801d38f Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 11 Oct 2026 10:15:02 +0800 Subject: [PATCH 16/16] ci: correct the shard 8 key and give the empty shards their zero rows --- tests/ci_env_skip_allowance.json | 42 +++++++++++++++++++++++++------- tests/test_skip_gate_profiles.py | 7 ++++++ 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/tests/ci_env_skip_allowance.json b/tests/ci_env_skip_allowance.json index 4beaa5c..b782cd7 100644 --- a/tests/ci_env_skip_allowance.json +++ b/tests/ci_env_skip_allowance.json @@ -35,19 +35,37 @@ "got empty parameter set for (pine_file)": 1 }, "engine_built_shard_0": { - "_note": "MEASURED: engine_built_shard_0 is the profile the `engine-shard` matrix job runs for shard 0 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. This shard's skip population is EMPTY, which is correct: with a summary present and no skips, default-deny passes.", + "_note": "MEASURED: engine_built_shard_0 is the profile the `engine-shard` matrix job runs for shard 0 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard observed no skips; the profile carries the six compile-environment rows at 0 because the gate rejects a profile with no reason rows at all, so the zero rows are what makes an observing-nothing shard legal.", "_unmeasured": false, - "_measured_on": "MEASURED on run 38099268385, shard 0 (engine-shard matrix leg), 2026-10-11" + "_measured_on": "MEASURED on run 38099268385, shard 0 (engine-shard matrix leg), 2026-10-11", + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, + "engine include / compiler unavailable": 0 }, "engine_built_shard_1": { - "_note": "MEASURED: engine_built_shard_1 is the profile the `engine-shard` matrix job runs for shard 1 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. This shard's skip population is EMPTY, which is correct: with a summary present and no skips, default-deny passes.", + "_note": "MEASURED: engine_built_shard_1 is the profile the `engine-shard` matrix job runs for shard 1 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard observed no skips; the profile carries the six compile-environment rows at 0 because the gate rejects a profile with no reason rows at all, so the zero rows are what makes an observing-nothing shard legal.", "_unmeasured": false, - "_measured_on": "MEASURED on run 38099268385, shard 1 (engine-shard matrix leg), 2026-10-11" + "_measured_on": "MEASURED on run 38099268385, shard 1 (engine-shard matrix leg), 2026-10-11", + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, + "engine include / compiler unavailable": 0 }, "engine_built_shard_2": { - "_note": "MEASURED: engine_built_shard_2 is the profile the `engine-shard` matrix job runs for shard 2 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. This shard's skip population is EMPTY, which is correct: with a summary present and no skips, default-deny passes.", + "_note": "MEASURED: engine_built_shard_2 is the profile the `engine-shard` matrix job runs for shard 2 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard observed no skips; the profile carries the six compile-environment rows at 0 because the gate rejects a profile with no reason rows at all, so the zero rows are what makes an observing-nothing shard legal.", "_unmeasured": false, - "_measured_on": "MEASURED on run 38099268385, shard 2 (engine-shard matrix leg), 2026-10-11" + "_measured_on": "MEASURED on run 38099268385, shard 2 (engine-shard matrix leg), 2026-10-11", + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, + "engine include / compiler unavailable": 0 }, "engine_built_shard_3": { "_note": "MEASURED: engine_built_shard_3 is the profile the `engine-shard` matrix job runs for shard 3 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one.", @@ -68,9 +86,15 @@ "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 25 }, "engine_built_shard_6": { - "_note": "MEASURED: engine_built_shard_6 is the profile the `engine-shard` matrix job runs for shard 6 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. This shard's skip population is EMPTY, which is correct: with a summary present and no skips, default-deny passes.", + "_note": "MEASURED: engine_built_shard_6 is the profile the `engine-shard` matrix job runs for shard 6 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard observed no skips; the profile carries the six compile-environment rows at 0 because the gate rejects a profile with no reason rows at all, so the zero rows are what makes an observing-nothing shard legal.", "_unmeasured": false, - "_measured_on": "MEASURED on run 38099268385, shard 6 (engine-shard matrix leg), 2026-10-11" + "_measured_on": "MEASURED on run 38099268385, shard 6 (engine-shard matrix leg), 2026-10-11", + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, + "engine include / compiler unavailable": 0 }, "engine_built_shard_7": { "_note": "MEASURED: engine_built_shard_7 is the profile the `engine-shard` matrix job runs for shard 7 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row is 6 because that knob's sources are not public, so the build cannot close it (ENGINE_BUILD_KEEPS in tests/skip_gate.py).", @@ -83,7 +107,7 @@ "_note": "MEASURED: engine_built_shard_8 is the profile the `engine-shard` matrix job runs for shard 8 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one.", "_unmeasured": false, "_measured_on": "MEASURED on run 38099268385, shard 8 (engine-shard matrix leg), 2026-10-11", - "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 1, + "the pre-lane codegen (7a39cb3) is not in this checkout's history": 1, "got empty parameter set for (case_name)": 2 }, "engine_built_shard_9": { diff --git a/tests/test_skip_gate_profiles.py b/tests/test_skip_gate_profiles.py index 1e921c2..90e992e 100644 --- a/tests/test_skip_gate_profiles.py +++ b/tests/test_skip_gate_profiles.py @@ -53,6 +53,13 @@ def test_engine_built_profiles_pass_the_gate_on_their_own_counts(): failures = [] for name in names: + # A profile with no usable reason rows (missing or empty) is rejected by + # the gate's real path: ``profile_entries`` returns None and the profile + # would silently allow nothing. Fail here instead of passing silently. + assert skip_gate.profile_entries(allowance, name) is not None, ( + f"{name} has no usable reason rows " + "(profile_entries returned None)" + ) rows = _rows(allowance[name]) problems = skip_gate.check( rows, rows, summary_present=True, profile_name=name