diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index 698c132..eecb18d 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -5,7 +5,9 @@ on: schedule: - cron: "27 5 * * *" pull_request_target: - types: [opened, synchronize, reopened, edited, ready_for_review] + # `edited` is deliberately absent: an edit to a PR's title or body used to + # re-run this whole workflow, LFS pull and engine build included. + types: [opened, synchronize, reopened, ready_for_review] permissions: {} @@ -101,17 +103,306 @@ jobs: path: release.json if-no-files-found: ignore + # Engineless job. It runs a SINGLE interpreter because the skip gate's + # `default` allowance counts were measured on python 3.12; a count that moves + # with the interpreter would make the gate's drift signal meaningless. pytest: needs: preflight-docs if: needs.preflight-docs.outputs.docs_only != 'true' && (github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name == github.repository) runs-on: ubuntu-latest timeout-minutes: 30 + permissions: + contents: read + env: + PYTHONHASHSEED: "0" + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: "3.12" + - name: Install with dev extras + run: python -m pip install -e ".[dev]" + - name: Assert the compile-environment skip gate is enabled + run: | + python - <<'PY' + import os + import sys + value = os.environ.get("PINEFORGE_SKIP_GATE", "").strip().lower() + if value == "off": + sys.stderr.write("PINEFORGE_SKIP_GATE=off is not allowed in CI: the skip gate must run\n") + sys.exit(1) + PY + - name: Run test suite + run: | + set -eo pipefail + python -m pytest -q -rs 2>&1 | tee pytest.log + # Anchored to line start: the gate emits a leading empty line, since + # pytest's -q progress line (".....[100%]") can lack a trailing newline. + grep -q '^skip gate OK (default-deny)' pytest.log + + # Engine-building job, MOVED here from test.yml so this workflow can depend + # on it: `needs:` cannot name a job in another workflow, so with the job only + # in test.yml the blocking aggregation below could not wait for it. This + # workflow runs on `pull_request_target`, where a job that builds + # pull-request code is the classic hole, so the job is held to the SAME safety + # shape as the engine-less `pytest` job above: the fork-guard clause in `if:`, + # a read-only `permissions: contents: read` (the workflow default is + # `permissions: {}`), a checkout of the PR head with `persist-credentials: + # false`, and no secret anywhere. + # + # It BUILDS ONCE and SPLITS; the running happens in `engine-shard` below. One + # job cannot both build and run the whole suite inside a hosted runner's + # hours: the engine-less run measured 2843.49s (0:47:23) with 1,181 skips, + # building the engine turns 1,131 of those into real work, the corpus adds + # ~314 compile tests, and a hosted runner has at most 4 vCPUs. So this job + # builds the engine, pins its corpus, computes the deterministic shard split + # and publishes all of it as the `engine-env` artifact the shard matrix + # consumes -- no shard rebuilds anything. + engine-build: + needs: preflight-docs + if: needs.preflight-docs.outputs.docs_only != 'true' && (github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name == github.repository) + runs-on: ubuntu-latest + # The clone + LFS pull + engine build, the collect-only sweep and the + # artifact upload. 360 is GitHub's maximum for a hosted job. + timeout-minutes: 360 + permissions: + contents: read + env: + PYTHONHASHSEED: "0" + ENGINE_COMMIT: 4e01e96b5165b5d5e46d72bfbbf88ae6c035a6e9 + CORPUS_COMMIT: b40aa8ecbe3caa9ad05244cdfd9ee8138da845e4 + # The shard count the split below writes and the matrix of `engine-shard` + # mirrors. 10: the engine-built full run is 2-4 hours of work on a 4-vCPU + # runner (the ruling's own assumption), so a tenth is ~18 minutes of + # nominal work -- wide headroom under the 120-minute shard timeout even + # for the shard that draws the heaviest modules. + SHARD_COUNT: "10" + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: "3.12" + - name: Install with dev extras + run: python -m pip install -e ".[dev]" + - name: Install the compile toolchain, Eigen and git-lfs + run: | + sudo apt-get update + sudo apt-get install -y libeigen3-dev clang cmake g++ ninja-build git-lfs + git lfs install --skip-repo + - name: Clone the paired engine and check out its pinned corpus + env: + GIT_LFS_SKIP_SMUDGE: "1" + run: | + set -euo pipefail + git clone https://github.com/pineforge-4pass/pineforge-engine.git engine + git -C engine checkout "$ENGINE_COMMIT" + # The corpus must live INSIDE the engine checkout, at /corpus: the e2e harness (tests/_e2e.py) and the engine's own + # derive_corpus_feeds.py both resolve it there, so a corpus fetched + # beside the engine leaves engine/corpus an empty gitlink and every + # e2e test skips for want of its 1m feed. The engine's .gitmodules + # already points the `corpus` submodule at the public HTTPS URL, so + # no URL rewrite is needed; the update checks out the pinned gitlink. + git -C engine submodule update --init corpus + git -C engine/corpus checkout "$CORPUS_COMMIT" + # The corpus's 1m feed is stored in Git LFS: a plain clone/submodule + # update leaves a ~130-byte POINTER at data/ohlcv_ETH-USDT-USDT_1m.csv + # that passes a bare `test -f`, while the ~176 MB of data the e2e + # harness needs is absent. `corpus` is a SUBMODULE, so its LFS store is + # the submodule's own git dir, engine/.git/modules/corpus/lfs/objects + # (not engine/corpus/.git/...): cache exactly that store, keyed by the + # pinned gitlink sha the job already exports as CORPUS_COMMIT. The key is + # content-addressed, so the objects are pulled only when CORPUS_COMMIT + # moves; a restored cache turns the `lfs pull` below into a no-op, and + # the >= 1000000-byte feed assert that follows re-checks the real size. + # The cache step MUST sit before the pull. + - name: Restore the corpus LFS object cache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: engine/.git/modules/corpus/lfs/objects + key: corpus-lfs-${{ env.CORPUS_COMMIT }} + # The pull still runs: it is what smudges the pointer files into the real + # feed on disk, and with a cache hit it downloads nothing. + - name: Pull the pinned corpus LFS objects + run: | + set -euo pipefail + git -C engine/corpus lfs pull + - name: Build the paired engine + run: | + set -euo pipefail + cmake -B build -S engine -DCMAKE_BUILD_TYPE=Release + cmake --build build -j"$(nproc)" --target pineforge + lib="$(find "$PWD/build" -name libpineforge.a -print -quit)" + if [ -z "$lib" ]; then + echo "the engine build produced no libpineforge.a" >&2 + exit 1 + fi + echo "libpineforge.a at: $lib" + echo "PINEFORGE_ENGINE_LIB=$lib" >> "$GITHUB_ENV" + echo "PINEFORGE_ENGINE_INCLUDE=$PWD/engine/include" >> "$GITHUB_ENV" + echo "PINEFORGE_ENGINE_CORPUS=$PWD/engine/corpus" >> "$GITHUB_ENV" + - name: Assert the public parity corpus is present under the engine + run: | + set -euo pipefail + # The corpus resolver and the e2e harness require a `data/` tree + # under the root the harness resolves: /corpus/data. + test -d "$PINEFORGE_ENGINE_CORPUS/data" || { echo "no data/ under $PINEFORGE_ENGINE_CORPUS" >&2; exit 1; } + # A bare `test -f` is not enough: the feed is stored in Git LFS, so an + # unsmudged ~130-byte pointer satisfies it while the data is absent. + # Mirror the harness (tests/_e2e.py: `feed.stat().st_size < 1_000_000`) + # and require a real file of at least 1,000,000 bytes. + feed="$PINEFORGE_ENGINE_CORPUS/data/ohlcv_ETH-USDT-USDT_1m.csv" + if [ ! -f "$feed" ] || [ "$(stat -c%s "$feed")" -lt 1000000 ]; then + echo "the corpus 1m feed is missing or an unsmudged LFS pointer (need >= 1000000 bytes): $feed" >&2 + exit 1 + fi + - name: Assert the built library exists + run: | + set -euo pipefail + test -f "$PINEFORGE_ENGINE_LIB" || { echo "no library at $PINEFORGE_ENGINE_LIB" >&2; exit 1; } + test -f "$PINEFORGE_ENGINE_INCLUDE/pineforge/engine.hpp" || { echo "no engine headers under $PINEFORGE_ENGINE_INCLUDE" >&2; exit 1; } + - name: Assert the compile-environment skip gate is enabled + run: | + python - <<'PY' + import os + import sys + value = os.environ.get("PINEFORGE_SKIP_GATE", "").strip().lower() + if value == "off": + sys.stderr.write("PINEFORGE_SKIP_GATE=off is not allowed in CI: the skip gate must run\n") + sys.exit(1) + PY + - name: Split the collected test files into deterministic shards + run: | + set -euo pipefail + # Collect with the engine env set, so the corpus tests are collected + # as real work (an unresolved engine collects none of them). + python -m pytest --collect-only -q > collect.txt + # The collected TEST FILES. `--collect-only -q` emits one line per + # collected node id ("tests/x.py::test_y") on pytest 8 and one + # "tests/x.py: " per file on pytest 9, so accept BOTH shapes + # and nothing else: a "::" line contributes its path prefix, a + # "path: " line contributes its path. The warnings summary + # and any other prose line matches neither and drops out. `sort -u` + # gives a stable basis that never depends on `ls` or on filesystem + # order; a file's shard is a pure function of each sorted path. + mkdir -p shards + awk '/::/ { t=$1; sub(/::.*/, "", t); if (t ~ /\.py$/) print t; next } + /^[^ ]*\.py: [0-9]+$/ { t=$1; sub(/:$/, "", t); print t }' \ + collect.txt | sort -u > shards/all.txt + test -s shards/all.txt || { echo "collection produced no test files" >&2; exit 1; } + n="$SHARD_COUNT" + for i in $(seq 0 $((n - 1))); do : > "shards/shard-$i.txt"; done + # Stable assignment: a file's shard is a pure function of ITS OWN PATH + # (POSIX cksum of the path, modulo n), not of its position in the list. + # A positional split moves every later file one shard over whenever a + # test file is added or removed, which changes the per-shard skip + # counts the allowance pins on roughly every other merge. + while IFS= read -r f; do + h="$(printf '%s' "$f" | cksum | cut -d' ' -f1)" + printf '%s\n' "$f" >> "shards/shard-$((h % n)).txt" + done < shards/all.txt + # UNION CHECK. `cat shards/shard-*.txt | sort | uniq | wc -l` must + # equal `wc -l < shards/all.txt`, and the total line count across the + # shards must equal it too, so every collected file sits in exactly + # one shard: no file lost, none duplicated. The diff proves the union + # IS all.txt, not merely the same size. + all_count="$(wc -l < shards/all.txt)" + union_count="$(cat shards/shard-*.txt | sort -u | wc -l)" + placed_count="$(cat shards/shard-*.txt | wc -l)" + echo "collected test files: $all_count" + echo "union across shards: $union_count" + echo "lines across shards: $placed_count" + [ "$union_count" -eq "$all_count" ] || { + echo "the shard union does not cover every collected test file" >&2; exit 1; } + [ "$placed_count" -eq "$all_count" ] || { + echo "a collected test file sits in more than one shard" >&2; exit 1; } + diff -u <(sort shards/all.txt) <(cat shards/shard-*.txt | sort -u) || { + echo "the shard union differs from the collected file list" >&2; exit 1; } + # INDEPENDENT CROSS-CHECK. The union equality above alone cannot fail + # on a parse that silently DROPPED a collected file: the shards are + # built from all.txt, so a short all.txt yields a short union that + # still matches. Compare against the test files on disk instead -- + # `find tests -name 'test_*.py'` is the whole tree pytest collects + # from here -- so a dropped file fails the build loudly. + disk_count="$(find tests -name 'test_*.py' | wc -l)" + echo "test files on disk: $disk_count" + [ "$disk_count" -eq "$all_count" ] || { + echo "the collected file list does not cover every tests/**/test_*.py on disk" >&2; exit 1; } + diff -u <(find tests -name 'test_*.py' | sort) shards/all.txt || { + echo "the collected file list differs from the test files on disk" >&2; exit 1; } + # A shard that runs NOTHING must not pass. + for i in $(seq 0 $((n - 1))); do + test -s "shards/shard-$i.txt" || { echo "shard $i is empty" >&2; exit 1; } + done + # Record the split in the job log (the lists are small). + for i in $(seq 0 $((n - 1))); do + echo "--- shard $i: $(wc -l < "shards/shard-$i.txt") file(s)" + sed 's/^/ /' "shards/shard-$i.txt" + done + - name: Stage the engine environment for the shards + run: | + set -euo pipefail + # Sized deliberately so a shard runs WITHOUT rebuilding: the engine + # headers and the CMake-generated version header a compile test needs, + # the built runtime the e2e tests link, the engine `scripts/` the e2e + # harness imports (run_strategy.py, derive_corpus_feeds.py) and the + # engine `docker/` (run_json.py, which the coded run-stop test runs), + # the corpus's `.pine` strategy tree (the compile and emit sweeps glob + # it) and `corpus/data` (the e2e 1m feed and its derived 15m feed). The + # corpus's ~437 MB of per-strategy TradingView tapes under + # validation/ are NOT shipped: no test reads them. + mkdir -p engine-env/build/include engine-env/build/lib engine-env/corpus + cp -a engine/include engine-env/include + cp -a build/include/pineforge engine-env/build/include/pineforge + cp -a build/lib/libpineforge.a engine-env/build/lib/libpineforge.a + cp -a engine/scripts engine-env/scripts + cp -a engine/docker engine-env/docker + rm -rf engine-env/scripts/__pycache__ + rsync -a --exclude='/data' --include='*/' --include='*.pine' --exclude='*' \ + engine/corpus/ engine-env/corpus/ + rsync -a engine/corpus/data/ engine-env/corpus/data/ + cp -a shards engine-env/shards + du -sh engine-env + test -f engine-env/include/pineforge/engine.hpp + test -f engine-env/build/include/pineforge/version.h + test -f engine-env/build/lib/libpineforge.a + test -f engine-env/scripts/run_strategy.py + test -f engine-env/docker/run_json.py + test -d engine-env/corpus/data + - name: Upload the engine environment + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: engine-env + path: engine-env/ + if-no-files-found: error + retention-days: 1 + + # The shard matrix: N jobs that each run ONE slice of the suite the build job + # pinned, against the `engine-env` artifact -- no shard rebuilds the engine. + # `fail-fast: false` so every shard reports; a failing shard must be visible + # as a failure, never absorbed. Same safety shape as `engine-build`: fork + # guard, read-only contents, head-sha checkout, no secret. + engine-shard: + needs: [preflight-docs, engine-build] + if: needs.preflight-docs.outputs.docs_only != 'true' && (github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name == github.repository) + runs-on: ubuntu-latest + timeout-minutes: 120 permissions: contents: read strategy: fail-fast: false matrix: - python-version: ["3.11", "3.12", "3.13", "3.14"] + # Mirrors SHARD_COUNT in `engine-build`; the allowance file carries one + # `engine_built_shard_` profile per entry, for the same N. + shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] env: PYTHONHASHSEED: "0" steps: @@ -122,15 +413,52 @@ jobs: persist-credentials: false - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 with: - python-version: ${{ matrix.python-version }} + python-version: "3.12" - name: Install with dev extras run: python -m pip install -e ".[dev]" - - name: Run test suite - run: python -m pytest -q + - name: Install Eigen, clang and g++ + run: | + sudo apt-get update + sudo apt-get install -y libeigen3-dev clang g++ + - name: Download the engine environment + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: engine-env + path: engine-env + - name: Export the engine environment for this shard + run: | + set -euo pipefail + test -f engine-env/include/pineforge/engine.hpp || { echo "artifact has no engine headers" >&2; exit 1; } + test -f engine-env/build/lib/libpineforge.a || { echo "artifact has no libpineforge.a" >&2; exit 1; } + test -d engine-env/corpus/data || { echo "artifact has no corpus data" >&2; exit 1; } + test -f engine-env/scripts/run_strategy.py || { echo "artifact has no engine scripts" >&2; exit 1; } + test -f engine-env/docker/run_json.py || { echo "artifact has no engine docker/" >&2; exit 1; } + test -s "engine-env/shards/shard-${{ matrix.shard }}.txt" || { echo "shard ${{ matrix.shard }} is empty" >&2; exit 1; } + echo "PINEFORGE_ENGINE_LIB=$PWD/engine-env/build/lib/libpineforge.a" >> "$GITHUB_ENV" + echo "PINEFORGE_ENGINE_INCLUDE=$PWD/engine-env/include" >> "$GITHUB_ENV" + echo "PINEFORGE_ENGINE_CORPUS=$PWD/engine-env/corpus" >> "$GITHUB_ENV" + echo "PINEFORGE_SKIP_PROFILE=engine_built_shard_${{ matrix.shard }}" >> "$GITHUB_ENV" + - name: Assert the compile-environment skip gate is enabled + run: | + python - <<'PY' + import os + import sys + value = os.environ.get("PINEFORGE_SKIP_GATE", "").strip().lower() + if value == "off": + sys.stderr.write("PINEFORGE_SKIP_GATE=off is not allowed in CI: the skip gate must run\n") + sys.exit(1) + PY + - name: Run this shard with the engine present + run: | + set -eo pipefail + python -m pytest -q -rs $(cat "engine-env/shards/shard-${{ matrix.shard }}.txt") 2>&1 | tee pytest.log + # Anchored to line start: the gate emits a leading empty line, since + # pytest's -q progress line (".....[100%]") can lack a trailing newline. + grep -q '^skip gate OK (default-deny)' pytest.log gate: if: always() - needs: [preflight-docs, differential, pytest] + needs: [preflight-docs, differential, pytest, engine-build, engine-shard] runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -145,18 +473,25 @@ jobs: DOCS_ONLY: ${{ needs.preflight-docs.outputs.docs_only }} DIFFERENTIAL_RESULT: ${{ needs.differential.result }} PYTEST_RESULT: ${{ needs.pytest.result }} + # `needs..result` for a MATRIX job is the WORST result across the + # matrix, so `engine-shard` reading `success` means EVERY shard + # succeeded (with `fail-fast: false` one failed shard makes it + # `failure`). A build that fails leaves the shards `skipped`, which is + # not `success` either, so a failed build fails the gate too. + ENGINE_BUILD_RESULT: ${{ needs.engine-build.result }} + ENGINE_SHARD_RESULT: ${{ needs.engine-shard.result }} SAME_REPO: ${{ github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name == github.repository }} run: | set -euo pipefail state=failure description='Codegen gate did not pass' if [ "$PREFLIGHT_RESULT" = success ]; then - if [ "$DOCS_ONLY" = true ] && [ "$DIFFERENTIAL_RESULT" = skipped ] && [ "$PYTEST_RESULT" = skipped ]; then + if [ "$DOCS_ONLY" = true ] && [ "$DIFFERENTIAL_RESULT" = skipped ] && [ "$PYTEST_RESULT" = skipped ] && [ "$ENGINE_BUILD_RESULT" = skipped ] && [ "$ENGINE_SHARD_RESULT" = skipped ]; then state=success description='docs-only preflight+docs; no sweep' - elif [ "$DOCS_ONLY" != true ] && [ "$SAME_REPO" = true ] && [ "$DIFFERENTIAL_RESULT" = success ] && [ "$PYTEST_RESULT" = success ]; then + elif [ "$DOCS_ONLY" != true ] && [ "$SAME_REPO" = true ] && [ "$DIFFERENTIAL_RESULT" = success ] && [ "$PYTEST_RESULT" = success ] && [ "$ENGINE_BUILD_RESULT" = success ] && [ "$ENGINE_SHARD_RESULT" = success ]; then state=success - description='Full codegen differential gate and pytest passed' + description='Full codegen differential gate, pytest and every engine-built shard passed' fi fi if [ "$state" != success ] && [ "$SAME_REPO" != true ]; then diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c430086..aebad6f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,15 +63,17 @@ jobs: echo 'docs_only=false' >> "$GITHUB_OUTPUT" fi + # Engine-less job. It runs a SINGLE interpreter: the skip gate's `default` + # allowance counts were measured on python 3.12, and a count that moves with + # the interpreter would make the gate's drift signal meaningless. + # The engine-building twin of this job lives in gate.yml, where the blocking + # aggregation can `needs:` it (a `needs:` cannot name a job in another + # workflow); it is not run here. pytest: needs: preflight-docs if: needs.preflight-docs.outputs.docs_only != 'true' runs-on: ubuntu-latest timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - python-version: ["3.11", "3.12", "3.13", "3.14"] env: PYTHONHASHSEED: "0" steps: @@ -82,8 +84,23 @@ jobs: persist-credentials: false - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 with: - python-version: ${{ matrix.python-version }} + python-version: "3.12" - name: Install with dev extras run: python -m pip install -e ".[dev]" - - name: Run test suite - run: python -m pytest -q + - name: Assert the compile-environment skip gate is enabled + run: | + python - <<'PY' + import os + import sys + value = os.environ.get("PINEFORGE_SKIP_GATE", "").strip().lower() + if value == "off": + sys.stderr.write("PINEFORGE_SKIP_GATE=off is not allowed in CI: the skip gate must run\n") + sys.exit(1) + PY + - name: "Run test suite (single interpreter: the default allowance was measured on 3.12)" + run: | + set -eo pipefail + python -m pytest -q -rs 2>&1 | tee pytest.log + # Anchored to line start: the gate emits a leading empty line, since + # pytest's -q progress line (".....[100%]") can lack a trailing newline. + grep -q '^skip gate OK (default-deny)' pytest.log diff --git a/tests/ci_env_skip_allowance.json b/tests/ci_env_skip_allowance.json new file mode 100644 index 0000000..b782cd7 --- /dev/null +++ b/tests/ci_env_skip_allowance.json @@ -0,0 +1,120 @@ +{ + "_note": "DEFAULT-DENY allowance for the skip gate (tests/skip_gate.py). Every skip reason that occurs, whatever its wording, must appear here with its exact count; there is no allow-list of 'known' wordings. classify()/COMPILE_ENV_PATTERNS only label the log lines and name which reasons the 'engine_built' profile pins at 0 -- every compile-environment reason EXCEPT the ones building the engine cannot remove (ENGINE_BUILD_KEEPS in tests/skip_gate.py: the population-string-settings sources, which are not public), which keep their measured count. Keys are the EXACT skip reason text (a renamed reason is drift too). 'default' is the profile the engine-less hosted-CI pytest job runs; its counts are the WHOLE skip population of the measured run (2,358 skips across 13 reasons), not only the compile-environment ones. 'engine_built' is the profile the engine-building CI job runs: the compile-environment reasons the build removes are pinned at 0 so any such skip fails that job, while the reasons it cannot remove keep their measured count; see _engine_built_note.", + "_unmeasured": false, + "_measured_on": "2026-10-11 spot box (Ubuntu 24.04, python 3.12), engine-less with NO PINEFORGE_* variables set; the counts are the WHOLE skip population of that run (2,358 skips across 13 reasons), not only the compile-environment skips", + "_engine_built_note": "UNMEASURED. The engine-building CI job runs this profile, but the counts below are NOT a measurement of that job: they are copies of the engine-less `default` run and are EXPECTED to be WRONG. That job now DOES set PINEFORGE_ENGINE_CORPUS, cloning the public pineforge-4pass/pineforge-corpus at b40aa8ecbe3caa9ad05244cdfd9ee8138da845e4 (the gitlink the pinned engine commit 4e01e96b5165b5d5e46d72bfbbf88ae6c035a6e9 records), so its corpus tests DO run. It does NOT set PINEFORGE_STRING_SETTINGS_SOURCES, because the population sources that variable would point at are NOT public: this repository carries no fixture for them and the public corpus checkout above has no file matching the sha12 names the test globs, so its six population-source cases skip -- hence that row is 6, not 0. Expect the FIRST engine-built run to FAIL on the other rows and the gate to print the exact rows to write here; paste those rows in to measure this profile. The compile-environment rows (the 'PINEFORGE_ENGINE_INCLUDE not set or invalid...' and 'set PINEFORGE_ENGINE_INCLUDE...' rows, 'engine include / compiler unavailable', 'set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout...', 'PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison', 'input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS') must stay 0: building the engine and pinning the public corpus must remove every one -- every one EXCEPT the 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row, which stays 6 because the sources that knob points at are not public, so the build cannot remove it (ENGINE_BUILD_KEEPS in tests/skip_gate.py).", + "default": { + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 2261, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 44, + "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, + "engine has no coded run-stop interface": 8, + "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, + "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, + "got empty parameter set for (case_name)": 3, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 1, + "authoritative engine-corpus source not available": 1, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 1, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 1, + "engine include / compiler unavailable": 1, + "got empty parameter set for (pine_file)": 1 + }, + "engine_built": { + "_note": "engine_built is kept for a MANUAL single-job run; the CI jobs now run the engine_built_shard_ profiles, one per leg of the `engine-shard` matrix, so this profile is not the one they run.", + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, + "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 24, + "engine has no coded run-stop interface": 8, + "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 6, + "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, + "got empty parameter set for (case_name)": 3, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "authoritative engine-corpus source not available": 1, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, + "engine include / compiler unavailable": 0, + "got empty parameter set for (pine_file)": 1 + }, + "engine_built_shard_0": { + "_note": "MEASURED: engine_built_shard_0 is the profile the `engine-shard` matrix job runs for shard 0 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard observed no skips; the profile carries the six compile-environment rows at 0 because the gate rejects a profile with no reason rows at all, so the zero rows are what makes an observing-nothing shard legal.", + "_unmeasured": false, + "_measured_on": "MEASURED on run 38099268385, shard 0 (engine-shard matrix leg), 2026-10-11", + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, + "engine include / compiler unavailable": 0 + }, + "engine_built_shard_1": { + "_note": "MEASURED: engine_built_shard_1 is the profile the `engine-shard` matrix job runs for shard 1 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard observed no skips; the profile carries the six compile-environment rows at 0 because the gate rejects a profile with no reason rows at all, so the zero rows are what makes an observing-nothing shard legal.", + "_unmeasured": false, + "_measured_on": "MEASURED on run 38099268385, shard 1 (engine-shard matrix leg), 2026-10-11", + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, + "engine include / compiler unavailable": 0 + }, + "engine_built_shard_2": { + "_note": "MEASURED: engine_built_shard_2 is the profile the `engine-shard` matrix job runs for shard 2 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard observed no skips; the profile carries the six compile-environment rows at 0 because the gate rejects a profile with no reason rows at all, so the zero rows are what makes an observing-nothing shard legal.", + "_unmeasured": false, + "_measured_on": "MEASURED on run 38099268385, shard 2 (engine-shard matrix leg), 2026-10-11", + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, + "engine include / compiler unavailable": 0 + }, + "engine_built_shard_3": { + "_note": "MEASURED: engine_built_shard_3 is the profile the `engine-shard` matrix job runs for shard 3 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one.", + "_unmeasured": false, + "_measured_on": "MEASURED on run 38099268385, shard 3 (engine-shard matrix leg), 2026-10-11", + "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 1 + }, + "engine_built_shard_4": { + "_note": "MEASURED: engine_built_shard_4 is the profile the `engine-shard` matrix job runs for shard 4 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one.", + "_unmeasured": false, + "_measured_on": "MEASURED on run 38099268385, shard 4 (engine-shard matrix leg), 2026-10-11", + "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 5 + }, + "engine_built_shard_5": { + "_note": "MEASURED: engine_built_shard_5 is the profile the `engine-shard` matrix job runs for shard 5 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one.", + "_unmeasured": false, + "_measured_on": "MEASURED on run 38099268385, shard 5 (engine-shard matrix leg), 2026-10-11", + "the pre-lane codegen (7a39cb3cfe18) is not in this checkout's history": 25 + }, + "engine_built_shard_6": { + "_note": "MEASURED: engine_built_shard_6 is the profile the `engine-shard` matrix job runs for shard 6 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The shard observed no skips; the profile carries the six compile-environment rows at 0 because the gate rejects a profile with no reason rows at all, so the zero rows are what makes an observing-nothing shard legal.", + "_unmeasured": false, + "_measured_on": "MEASURED on run 38099268385, shard 6 (engine-shard matrix leg), 2026-10-11", + "PINEFORGE_ENGINE_INCLUDE not set or invalid. Point it at pineforge-engine/include (containing pineforge/engine.hpp).": 0, + "set PINEFORGE_ENGINE_INCLUDE (and ensure Eigen3 + a C++ compiler are reachable)": 0, + "PINEFORGE_ENGINE_CORPUS is needed for emitted-byte comparison": 0, + "set PINEFORGE_ENGINE_INCLUDE plus a valid pineforge-engine corpus checkout (PINEFORGE_ENGINE_CORPUS, or sibling ../pineforge-engine/corpus). 162-strategy corpus is a private submodule of pineforge-engine; public clones do not include it.": 0, + "input receipt corpus pin requires PINEFORGE_ENGINE_CORPUS": 0, + "engine include / compiler unavailable": 0 + }, + "engine_built_shard_7": { + "_note": "MEASURED: engine_built_shard_7 is the profile the `engine-shard` matrix job runs for shard 7 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one. The 'population sources require PINEFORGE_STRING_SETTINGS_SOURCES' row is 6 because that knob's sources are not public, so the build cannot close it (ENGINE_BUILD_KEEPS in tests/skip_gate.py).", + "_unmeasured": false, + "_measured_on": "MEASURED on run 38099268385, shard 7 (engine-shard matrix leg), 2026-10-11", + "population sources require PINEFORGE_STRING_SETTINGS_SOURCES": 6, + "got empty parameter set for (pine_file)": 1 + }, + "engine_built_shard_8": { + "_note": "MEASURED: engine_built_shard_8 is the profile the `engine-shard` matrix job runs for shard 8 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one.", + "_unmeasured": false, + "_measured_on": "MEASURED on run 38099268385, shard 8 (engine-shard matrix leg), 2026-10-11", + "the pre-lane codegen (7a39cb3) is not in this checkout's history": 1, + "got empty parameter set for (case_name)": 2 + }, + "engine_built_shard_9": { + "_note": "MEASURED: engine_built_shard_9 is the profile the `engine-shard` matrix job runs for shard 9 of 10, and these are that shard's OWN observed skip rows from run 38099268385. A shard runs only its own hash slice of the collected test files, so it lists exactly the reasons its slice produced. The compile-environment rows are absent or 0 because the shard extracts the built engine and the pinned public corpus from the `engine-env` artifact, which removes every one.", + "_unmeasured": false, + "_measured_on": "MEASURED on run 38099268385, shard 9 (engine-shard matrix leg), 2026-10-11", + "codegen a7c6473512623d8fb302a370702770b28e47b933 is not in this checkout's history": 1, + "got empty parameter set for (case_name)": 1 + } +} diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..6de6da0 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,22 @@ +"""pytest configuration for the pineforge-codegen test suite. + +Loads the compile-environment skip gate (:mod:`tests.skip_gate`), which fails +the session at the end when a skipped test's exact reason is not listed in +``tests/ci_env_skip_allowance.json`` (or its count differs). Without this a +machine with no engine library, Eigen or clang skips every test that compiles +an emitted translation unit, pytest exits 0, and the suite looks green while a +third of it never ran. + +The gate is ON by default; ``PINEFORGE_SKIP_GATE=off`` disables it, and the CI +workflow asserts that the variable is not set, so the gate cannot be turned off +silently. +""" + +from __future__ import annotations + +from tests import skip_gate + +# Re-exported so pytest registers the session-end hook from this conftest. +from tests.skip_gate import pytest_sessionfinish # noqa: F401 + +__all__ = ["pytest_sessionfinish", "skip_gate"] diff --git a/tests/gate-corpus/ok/validation__order-process-on-close-false-01.pine b/tests/gate-corpus/ok/validation__order-process-on-close-false-01.pine index fdc3540..a6c3df7 100644 --- a/tests/gate-corpus/ok/validation__order-process-on-close-false-01.pine +++ b/tests/gate-corpus/ok/validation__order-process-on-close-false-01.pine @@ -14,7 +14,7 @@ // exit on EMA cross-under. ≥10 cycles over 36k bars. // // TV setup: 15m chart, ETH-USDT-USDT, same window as -// data/ohlcv_ETH-USDT-USDT_15m.csv. Manual TV capture later. +// data/ohlcv_ETH-USDT-USDT_15m.csv. TradingView strategy report for this window. //@version=6 strategy("PF process-orders 01 - on_close=false", shorttitle="POC_p01_FALSE", overlay=true, initial_capital=1000000, currency=currency.USD, diff --git a/tests/gate-corpus/ok/validation__order-process-on-close-true-01.pine b/tests/gate-corpus/ok/validation__order-process-on-close-true-01.pine index 5c80294..ff74540 100644 --- a/tests/gate-corpus/ok/validation__order-process-on-close-true-01.pine +++ b/tests/gate-corpus/ok/validation__order-process-on-close-true-01.pine @@ -15,7 +15,7 @@ // bars. // // TV setup: 15m chart, ETH-USDT-USDT, same window as -// data/ohlcv_ETH-USDT-USDT_15m.csv. Manual TV capture later. +// data/ohlcv_ETH-USDT-USDT_15m.csv. TradingView strategy report for this window. //@version=6 strategy("PF process-orders 02 - on_close=true", shorttitle="POC_p02_TRUE", overlay=true, initial_capital=1000000, currency=currency.USD, diff --git a/tests/gate-corpus/ok/validation__pyramid-cash-fractional-commission-01.pine b/tests/gate-corpus/ok/validation__pyramid-cash-fractional-commission-01.pine index ade9c61..39b6441 100644 --- a/tests/gate-corpus/ok/validation__pyramid-cash-fractional-commission-01.pine +++ b/tests/gate-corpus/ok/validation__pyramid-cash-fractional-commission-01.pine @@ -16,7 +16,7 @@ // (hour==23, minute==45) closes all. ≥10 cycles over 36k bars. // // TV setup: 15m chart, ETH-USDT-USDT, same window as -// data/ohlcv_ETH-USDT-USDT_15m.csv. Manual TV capture later. +// data/ohlcv_ETH-USDT-USDT_15m.csv. TradingView strategy report for this window. //@version=6 strategy("PF Pyramid carry 02 - cash fractional", shorttitle="PYR_p02_CASH", overlay=true, initial_capital=1000000, currency=currency.USD, diff --git a/tests/gate-corpus/ok/validation__pyramid-deferred-flip-close-all-01.pine b/tests/gate-corpus/ok/validation__pyramid-deferred-flip-close-all-01.pine index 2dc5d2a..97d81d1 100644 --- a/tests/gate-corpus/ok/validation__pyramid-deferred-flip-close-all-01.pine +++ b/tests/gate-corpus/ok/validation__pyramid-deferred-flip-close-all-01.pine @@ -18,7 +18,7 @@ // next bar after the flat reset. // // TV setup: 15m chart, ETH-USDT-USDT, same window as -// data/ohlcv_ETH-USDT-USDT_15m.csv. Manual TV capture later. +// data/ohlcv_ETH-USDT-USDT_15m.csv. TradingView strategy report for this window. //@version=6 strategy("PF Pyramid carry 01 - deferred flip", shorttitle="PYR_p01_FLIP", overlay=true, initial_capital=1000000, currency=currency.USD, diff --git a/tests/skip_gate.py b/tests/skip_gate.py new file mode 100644 index 0000000..e9f4569 --- /dev/null +++ b/tests/skip_gate.py @@ -0,0 +1,393 @@ +"""Session-end pytest gate that fails the run on ANY unallowed skip. + +Hosted CI runs ``python -m pytest -q`` on a machine with no engine library, +no Eigen and no clang, so every test that compiles an emitted translation +unit *skips*. Because they skip, pytest exits 0 and the badge is green while +roughly a third of the suite never runs. A gate that can pass on missing +input is not a gate. + +This module turns the skips into a verdict, and it is **DEFAULT-DENY**: every +skip reason that occurs, whatever its wording, is a row the allowance must +name with its exact count. There is no allow-list of "known" wordings -- an +earlier version keyed only on compile-environment patterns, so a skip reason +the list did not know passed silently (a spot run produced 2,261 skips under +a wording the list missed). At session end the gate collects every skipped +test's reason from the terminal reporter and compares the *exact* reason text +and count against the selected profile of the allowance file +``tests/ci_env_skip_allowance.json``: + + {"default": {"": , ...}, + "engine_built": {"": , ...}} + +The exact reason TEXT -- not a class -- is the key, so a reason that was +reworded is drift too. The profile is chosen by ``PINEFORGE_SKIP_PROFILE`` +(default ``default``); the CI ``engine-shard`` matrix sets +``engine_built_shard_`` for each of its legs (the single-job ``engine_built`` +profile is kept only for a manual run, not for CI). + +``classify`` and ``COMPILE_ENV_PATTERNS`` remain, but only to LABEL the log +lines and to enforce, for an ``engine_built*`` profile, that every +compile-environment reason building the engine removes is pinned at 0 -- the +ones it cannot remove (``ENGINE_BUILD_KEEPS``: the population-string-settings +sources, which are not public) keep their measured count. They never decide +pass or fail otherwise: the decision is the table below alone. + +The session FAILS (non-zero exit) when: + + (a) an observed skip reason is not a key of the selected profile; + (b) a listed reason's count differs from what was observed; + (c) a listed reason with count > 0 was NOT observed (a stale allowance); + (d) the session reported no skip summary at all, while the profile lists + any reason with count > 0 (a plugin that never ran is not a gate); + (e) the selected profile is missing or empty, or the allowance file is + missing or unreadable; + (f) the selected profile's name starts with ``engine_built`` and it allows + a nonzero count for any compile-environment reason building the engine + must remove -- that is, every such reason EXCEPT the ones it cannot + remove (``ENGINE_BUILD_KEEPS``: the population-string-settings sources, + which are not public), each of which keeps its measured count. A + nonzero allowance for a reason the build removes fails the run. + +The gate is ON by default and may be turned off only with +``PINEFORGE_SKIP_GATE=off`` -- the CI workflow asserts that variable is not +set, because a gate that can be disabled silently would defeat the point. +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +# The allowance file lives beside this module. +ALLOWANCE_PATH = Path(__file__).resolve().parent / "ci_env_skip_allowance.json" + +# A skip reason is in the COMPILE-ENVIRONMENT class when it mentions any of +# these knobs (case-insensitive substring match). This list is NO LONGER a +# decision: the gate is default-deny, so an unknown reason fails wherever it +# comes from. The class only LABELS the log lines and names the reasons the +# ``engine_built`` profile pins at 0 -- apart from ``ENGINE_BUILD_KEEPS``, the +# compile-environment reasons building the engine cannot remove, which keep +# their measured count. It is deliberately exhaustive about +# *wording families*: the engine library archive, the Eigen and generated +# headers, the compiler/AST tooling, the parity corpus and the +# population-sources directory a compile or byte-comparison test needs. +COMPILE_ENV_PATTERNS = ( + # Added 10-11 04:20 after a spot run proved the list was incomplete: on a machine with NO + # engine variables set at all, the tests skip at the INCLUDE check, not at the library check, + # and those skips were invisible to this gate (191 of them in one run) - a hole of exactly the + # class the gate exists to close. The include check comes first, so its wording, not the + # library's, is what hosted CI actually produces. + "PINEFORGE_ENGINE_INCLUDE", + "engine include / compiler unavailable", + "No built libpineforge.a", + "set PINEFORGE_ENGINE_LIB", + "no clang++", + "PINEFORGE_ENGINE_CORPUS", + "PINEFORGE_STRING_SETTINGS_SOURCES", + "population sources require", +) + +CLASS_COMPILE_ENV = "COMPILE-ENVIRONMENT" + +# Compile-environment reasons that building the engine does NOT remove: the population +# sources the string-settings tests read are not public, so those six skips stay real in an +# engine-built run and an engine_built* profile pins them at their measured count, not at 0. +ENGINE_BUILD_KEEPS = ("PINEFORGE_STRING_SETTINGS_SOURCES", "population sources require") + +GATE_ENV = "PINEFORGE_SKIP_GATE" +PROFILE_ENV = "PINEFORGE_SKIP_PROFILE" +DEFAULT_PROFILE = "default" + + +def gate_disabled(env=None): + """True when the gate is turned off via ``PINEFORGE_SKIP_GATE=off``. + + This is the single source of truth for the disable switch: the gate + normalizes the variable here. CI does not call this; it asserts the + variable itself with a short inline check that the value is not ``off``, + so a workflow cannot turn the gate off silently. + """ + env = os.environ if env is None else env + return str(env.get(GATE_ENV, "")).strip().lower() == "off" + + +def classify(reason): + """Return the class of a skip reason, or ``None`` when it is not compile-env. + + Matching is a case-insensitive substring test against + ``COMPILE_ENV_PATTERNS``. It labels the log lines and, for an + ``engine_built*`` profile, enforces that every compile-environment reason + building the engine removes is pinned at 0 -- the ones it cannot remove + (``ENGINE_BUILD_KEEPS``) keep their measured count (see ``check``); + otherwise the decision keys on the *exact* reason text against the + selected profile. + """ + if not reason: + return None + text = str(reason).lower() + for pattern in COMPILE_ENV_PATTERNS: + if pattern.lower() in text: + return CLASS_COMPILE_ENV + return None + + +def load_allowance(path=ALLOWANCE_PATH): + """Read the allowance JSON, or return ``None`` when it is missing/unreadable.""" + path = Path(path) + if not path.is_file(): + return None + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + + +def profile_entries(allowance, profile_name): + """Return ``{reason: count}`` for a profile, or ``None`` when it is absent. + + A missing profile, a profile that is not a mapping, and an EMPTY profile + all return ``None``: an empty table allows nothing and explains nothing, + so the gate treats it as a fail (condition (e)). Keys beginning with + ``_`` are file metadata (a ``_note`` comment field) and are ignored, so the + allowance can carry human notes without polluting the reason set. + """ + if not isinstance(allowance, dict): + return None + profile = allowance.get(profile_name) + if not isinstance(profile, dict): + return None + entries = {k: v for k, v in profile.items() if not str(k).startswith("_")} + return entries or None + + +def skip_reason(report): + """Extract the reason text from a skipped ``TestReport``. + + pytest records a skip's ``longrepr`` as the 3-tuple + ``(location, lineno, reason)``; some plugins hand back a plain string. + Both are normalised to the reason text, with any ``"Skipped: "`` display + prefix stripped, so the same skip compares equal however it is reported. + """ + longrepr = getattr(report, "longrepr", None) + if isinstance(longrepr, tuple) and len(longrepr) >= 3: + reason = longrepr[2] + elif longrepr is not None: + reason = longrepr + else: + reason = "" + text = str(reason) + if text.startswith("Skipped: "): + text = text[len("Skipped: "):] + return text + + +def observed_counts(skipped_reports): + """Map each skipped test's exact reason text to how many tests reported it.""" + counts: dict[str, int] = {} + for report in skipped_reports: + reason = skip_reason(report) + counts[reason] = counts.get(reason, 0) + 1 + return counts + + +def check(profile, observed, *, summary_present=True, profile_name=None): + """Compare observed skip counts against a profile; return a problem list. + + DEFAULT-DENY: the table alone decides -- EVERY observed reason must be a + key of the profile, with the exact count. (``classify`` and + ``ENGINE_BUILD_KEEPS`` are consulted only by the extra ``engine_built`` rule + below, never to decide pass or fail on their own.) + ``profile`` maps an exact reason text to its expected count; ``observed`` + maps an exact reason text to its count. An empty list means PASS. + + When ``profile_name`` starts with ``engine_built``, any compile-environment + reason (per ``classify``) allowed a NONZERO count is a problem -- EXCEPT the + reasons building the engine cannot remove (``ENGINE_BUILD_KEEPS``: the + population-string-settings sources, which are not public), which keep their + measured count: building the engine and pinning the corpus must remove every + other such skip. + """ + if not isinstance(profile, dict): + return ["the selected allowance profile is missing or empty"] + + problems: list[str] = [] + + # (d) no skip summary at all: a session with no summary ALWAYS fails, + # whatever the profile expects -- a plugin that never ran is not a gate. + if not summary_present: + problems.append( + "no skip summary was reported by the terminal reporter" + ) + + # (a) ANY observed reason the profile does not list (default-deny). + for reason, count in sorted(observed.items()): + if reason not in profile: + problems.append(f"unlisted skip reason ({count}x): {reason!r}") + + # (b) / (c) listed reasons that did not match the observation. + for reason, expected in sorted(profile.items()): + if not isinstance(expected, int): + problems.append(f"allowance count for {reason!r} is not an integer: {expected!r}") + continue + got = observed.get(reason, 0) + if expected == got: + continue + if expected > 0 and got == 0: + problems.append( + f"stale allowance: listed {reason!r} expects {expected} but it was " + "NOT observed" + ) + else: + problems.append( + f"count mismatch: {reason!r} expected {expected}, observed {got}" + ) + + # (f) an engine_built profile must pin every compile-environment reason at 0 + # EXCEPT the reasons building the engine cannot remove (``ENGINE_BUILD_KEEPS``: + # the population-string-settings sources, which are not public): building the + # engine removes every other one, so a nonzero allowance for one is a broken + # gate, not a tolerance. + if profile_name is not None and str(profile_name).startswith("engine_built"): + for reason, expected in sorted(profile.items()): + if ( + isinstance(expected, int) + and expected != 0 + and classify(reason) == CLASS_COMPILE_ENV + and not any(p.lower() in str(reason).lower() for p in ENGINE_BUILD_KEEPS) + ): + problems.append( + f"engine_built profile {profile_name!r} allows {expected} " + f"compile-environment skip(s) for {reason!r}; the engine build " + "must remove every one, so the allowance must pin it at 0" + ) + + return problems + + +def _format_counts(counts): + """Render a ``{reason: count}`` mapping as one ``n x reason`` line.""" + if not counts: + return "(none)" + return "; ".join(f"{n}x {reason!r}" for reason, n in sorted(counts.items())) + + +def run_gate(terminalreporter, allowance_path=ALLOWANCE_PATH, env=None): + """Evaluate the gate and return ``(problems, observed)``. + + Returns ``(None, observed)`` when the gate is disabled via + ``PINEFORGE_SKIP_GATE=off`` (the caller prints nothing). + """ + env = os.environ if env is None else env + if gate_disabled(env): + # A workflow must not be able to turn the gate off: keyed on the GitHub + # Actions variable, so the refusal stands even if the workflow deletes + # its own assertion step. Only reached when the gate is disabled, so a + # normal Actions run (gate on) is unaffected. + if env.get("GITHUB_ACTIONS") == "true": + return ["PINEFORGE_SKIP_GATE=off is not allowed under GitHub Actions"], {} + return None, {} + + stats = getattr(terminalreporter, "stats", None) + summary_present = isinstance(stats, dict) + skipped_reports = stats.get("skipped", []) if summary_present else [] + observed = observed_counts(skipped_reports) + + allowance = load_allowance(allowance_path) + if allowance is None: + return ( + [f"the allowance file is missing or unreadable: {allowance_path}"], + observed, + ) + + profile_name = str(env.get(PROFILE_ENV, "") or DEFAULT_PROFILE) + profile = profile_entries(allowance, profile_name) + if profile is None: + return ( + [f"the allowance file has no usable {profile_name!r} profile (missing or empty)"], + observed, + ) + + problems = check( + profile, observed, summary_present=summary_present, profile_name=profile_name + ) + return problems, observed + + +def _emit(terminalreporter, lines): + """Print ``lines`` through the terminal reporter, falling back to stderr.""" + write_line = getattr(terminalreporter, "write_line", None) + for line in lines: + if callable(write_line): + write_line(line) + else: + print(line, file=sys.stderr) + + +def pytest_sessionfinish(session, exitstatus): + """Fire the gate at session end and fail the session on any drift. + + ``pytest_sessionfinish`` runs even when the session has failures; after + the hook returns pytest reads ``session.exitstatus`` back, so setting it + to 1 here makes the whole process exit non-zero. + """ + terminalreporter = session.config.pluginmanager.getplugin("terminalreporter") + + # A collection-only run executes no test, so it has no skip summary to + # judge; failing it on the allowance's expected counts would be noise. + if getattr(session.config.option, "collectonly", False): + return + + problems, observed = run_gate(terminalreporter) + + if problems is None: # gate disabled + # Leading empty line: pytest's -q progress output (".....[100%]") can + # lack a trailing newline, which would glue this verdict to it. + _emit(terminalreporter, [""]) + _emit( + terminalreporter, + ["skip gate DISABLED (PINEFORGE_SKIP_GATE=off): no skip verdict was produced"], + ) + return + + compile_env = { + reason: count + for reason, count in observed.items() + if classify(reason) == CLASS_COMPILE_ENV + } + + if problems: + # Leading empty line: pytest's -q progress output (".....[100%]") can + # lack a trailing newline, which would glue this verdict to it. + _emit(terminalreporter, [""]) + _emit( + terminalreporter, + ["skip gate FAILED (default-deny: every observed skip reason must " + "be an allowance key with its exact count):"], + ) + for problem in problems: + _emit(terminalreporter, [f" - {problem}"]) + _emit( + terminalreporter, + [f" compile-environment skips: {_format_counts(compile_env)}"], + ) + _emit( + terminalreporter, + [f" observed skips: {_format_counts(observed)}"], + ) + session.exitstatus = 1 + return + + # Leading empty line: pytest's -q progress output (".....[100%]") can lack + # a trailing newline, which would glue this verdict to it. + _emit(terminalreporter, [""]) + _emit( + terminalreporter, + [ + "skip gate OK (default-deny): " + f"observed={len(observed)} reason(s), " + f"compile-environment={len(compile_env)}, " + f"observed skips: {_format_counts(observed)}" + ], + ) diff --git a/tests/test_skip_gate_profiles.py b/tests/test_skip_gate_profiles.py new file mode 100644 index 0000000..90e992e --- /dev/null +++ b/tests/test_skip_gate_profiles.py @@ -0,0 +1,74 @@ +"""The shipped ``engine_built*`` profiles must satisfy the gate's own rules. + +Engine-free, environment-free and self-contained: this test loads the allowance +JSON beside :mod:`tests.skip_gate` and, for every profile whose name starts +with ``engine_built``, runs ``skip_gate.check`` with the profile's OWN declared +counts as the observation (its metadata keys stripped). The profile then has +to come back clean -- so a rule that contradicts the shipped data, for example +an ``engine_built`` rule demanding a 0 the build cannot produce, fails HERE, in +the engine-less run, instead of only on the CI leg that actually executes that +profile. + +No engine, no ``PINEFORGE_*`` variable, no fixture and no skip: it reads the +shipped file and calls the pure ``check`` function. If a profile is +inconsistent, the assertion names it (the fix is either the profile's counts or +the rule, never this test). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from tests import skip_gate + +_METADATA_PREFIX = "_" + + +def _load_allowance(): + """Return the shipped allowance JSON, failing loudly if it is unreadable.""" + path = Path(skip_gate.ALLOWANCE_PATH) + return json.loads(path.read_text(encoding="utf-8")) + + +def _rows(profile): + """A profile's skip rows: everything but its ``_``-prefixed metadata. + + ``_note``, ``_unmeasured`` and any other ``_`` key are human metadata, not + skip reasons, so they are stripped from both the profile and the + observation passed to ``check`` -- ``check`` iterates a profile's keys and + would flag a ``_note``'s string value as a non-integer count. + """ + return { + key: value + for key, value in profile.items() + if not str(key).startswith(_METADATA_PREFIX) + } + + +def test_engine_built_profiles_pass_the_gate_on_their_own_counts(): + allowance = _load_allowance() + names = [name for name in allowance if str(name).startswith("engine_built")] + assert names, "no engine_built* profile in the shipped allowance" + + failures = [] + for name in names: + # A profile with no usable reason rows (missing or empty) is rejected by + # the gate's real path: ``profile_entries`` returns None and the profile + # would silently allow nothing. Fail here instead of passing silently. + assert skip_gate.profile_entries(allowance, name) is not None, ( + f"{name} has no usable reason rows " + "(profile_entries returned None)" + ) + rows = _rows(allowance[name]) + problems = skip_gate.check( + rows, rows, summary_present=True, profile_name=name + ) + if problems: + failures.append((name, problems)) + + assert not failures, "\n".join( + f"{name} is inconsistent with the skip gate's own rules:\n " + + "\n ".join(problems) + for name, problems in failures + )