diff --git a/examples/python-pytest/README.md b/examples/python-pytest/README.md new file mode 100644 index 00000000..caad1628 --- /dev/null +++ b/examples/python-pytest/README.md @@ -0,0 +1,75 @@ +# Python/pytest: unchanged PASS and deliberate-write REVIEW + +Historical example, archived at ExecSurface **v0.1.0-alpha.5**; it has not been +re-run against v1.0.0. This recipe learns and checks the **same real pytest +command** against a tiny public fixture using that pinned Alpha.5 release. The current public support +scope is Linux x86_64 with native `ptrace`; `doctor` must succeed on the host. +This is a reproducible integration example, not independent external validation. +Independent Alpha.5 findings belong in [issue #118](https://github.com/AETHERXGLOBAL/execsurface/issues/118). + +From the repository root, first obtain the checksum-verified Alpha.5 binary using +the installation notes of the [immutable Alpha.5 release](https://github.com/AETHERXGLOBAL/execsurface/releases/tag/v0.1.0-alpha.5). Keep its reviewed +path in `EXEC_SURFACE_BIN`; this recipe records its digest and checks its version. +Then create a Python 3.12 or 3.13 environment and install the hash-pinned test +selection. Installation may download packages; the observed test uses no network +or external services. + +```sh +python3.13 -m venv /tmp/execsurface-pytest-env +/tmp/execsurface-pytest-env/bin/python -m pip install \ + --require-hashes --only-binary=:all: \ + -r examples/python-pytest/requirements.lock + +EXEC_SURFACE_BIN=/absolute/path/to/checksum-verified/execsurface +/tmp/execsurface-pytest-env/bin/python examples/python-pytest/run.py \ + --execsurface "$EXEC_SURFACE_BIN" \ + --output /tmp/execsurface-pytest-first-run +``` + +Use a new output directory each time. The runner refuses to overwrite a retained +run, copies its two public fixture files, and retains the native command receipts, +stdout/stderr, baseline and verdict JSON. A failed expectation writes `failure.json` +and exits nonzero, retaining the failure. `--python` may explicitly select another +interpreter with the exact dependencies installed; otherwise the runner uses its +own interpreter. + +The observed command is: + +```sh +python -m pytest -q -s -p no:cacheprovider test_fixture.py +``` + +The runner also disables plugin autoload and bytecode writes. `-s` disables +pytest's random temporary-file output capture; the **runner** still retains both +streams. Default pytest output capture was tested and made unchanged runs report +REVIEW for varying scratch-file names/deleted inode paths. These explicit fixture +choices keep this small workload deterministic without changing ExecSurface +normalization, policy or acceptance thresholds. They are not a recipe for hiding +arbitrary application drift. + +The runner performs the ordinary `learn --output ... --workspace ... --label ...` +and `check --baseline ... --workspace ... --json-output ...` flows: + +| Step | Pytest result | ExecSurface result | Asserted exit | +| --- | --- | --- | --- | +| Learn `write_extra: false` | One passing test | Baseline written | 0 | +| Check unchanged fixture | One passing test | PASS; no drift | 0 | +| Set public fixture `write_extra: true`, check again | One passing test | REVIEW; added `$WORKSPACE/drift.txt` write | 10 | + +The same assertion, command, Python interpreter and baseline remain selected. +Only the public fixture's boolean changes. The test then writes the harmless +`drift.txt` inside its fresh workspace. The runner verifies its native file-write +finding and verifies the baseline bytes did not change. + +The **baseline** records the selected observed execution surface. The **policy** +interprets differences; this recipe uses the unchanged built-in REVIEW policy for +unmatched drift. It does not relearn after drift or weaken policy to get PASS. +ERROR, BLOCK, unexpected exit, lost report or an unavailable observer fails the +recipe rather than counting as either expected outcome. + +The fixture contains no secrets. It adds no environment enumeration or +secret/file-content collection. ExecSurface's existing metadata observer may +record local executable and filesystem paths; review retained receipts before +sharing them. A PASS here establishes this selected observed fixture comparison; +it does not establish containment, malware safety, complete observation of every +possible effect or independent custody. diff --git a/examples/python-pytest/fixtures/fixture.json b/examples/python-pytest/fixtures/fixture.json new file mode 100644 index 00000000..d3ce7175 --- /dev/null +++ b/examples/python-pytest/fixtures/fixture.json @@ -0,0 +1 @@ +{"left":2,"right":3,"write_extra":false} diff --git a/examples/python-pytest/fixtures/test_fixture.py b/examples/python-pytest/fixtures/test_fixture.py new file mode 100644 index 00000000..2da2decb --- /dev/null +++ b/examples/python-pytest/fixtures/test_fixture.py @@ -0,0 +1,10 @@ +"""One deterministic pytest test; optional extra write is deliberate runtime drift.""" +import json +from pathlib import Path + + +def test_public_fixture(): + selection = json.loads(Path('fixture.json').read_text()) + assert selection['left'] + selection['right'] == 5 + if selection['write_extra']: + Path('drift.txt').write_text('controlled fixture drift\n') diff --git a/examples/python-pytest/requirements.in b/examples/python-pytest/requirements.in new file mode 100644 index 00000000..9471b3d9 --- /dev/null +++ b/examples/python-pytest/requirements.in @@ -0,0 +1 @@ +pytest==8.4.2 diff --git a/examples/python-pytest/requirements.lock b/examples/python-pytest/requirements.lock new file mode 100644 index 00000000..289e0e7f --- /dev/null +++ b/examples/python-pytest/requirements.lock @@ -0,0 +1,22 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --python python3.13 --generate-hashes --output-file requirements.lock requirements.in +iniconfig==2.3.0 \ + --hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \ + --hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12 + # via pytest +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via pytest +pluggy==1.6.0 \ + --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ + --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + # via pytest +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via pytest +pytest==8.4.2 \ + --hash=sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01 \ + --hash=sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79 + # via -r execsurface-pytest/examples/python-pytest/requirements.in diff --git a/examples/python-pytest/run.py b/examples/python-pytest/run.py new file mode 100644 index 00000000..ac480121 --- /dev/null +++ b/examples/python-pytest/run.py @@ -0,0 +1,94 @@ +"""Learn and check real pytest, retaining unchanged PASS and deliberate-write REVIEW.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + + +class RecipeError(ValueError): + """A native command or declared recipe assertion did not establish its outcome.""" + + +def invoke(command: list[str], root: Path, name: str, expected: int) -> None: + """Retain command, both streams and actual exit before checking expectation.""" + env = os.environ.copy() + env.update(PYTHONDONTWRITEBYTECODE='1', PYTEST_DISABLE_PLUGIN_AUTOLOAD='1') + try: + result = subprocess.run(command, cwd=root, env=env, capture_output=True, timeout=60, check=False) + except subprocess.TimeoutExpired as error: + (root / (name + '.stdout')).write_bytes(error.stdout or b'') + (root / (name + '.stderr')).write_bytes(error.stderr or b'') + (root / (name + '-command.json')).write_text(json.dumps({'command': command, 'returncode': None, 'expectedReturncode': expected, 'outcome': 'timeout'}, indent=2) + '\n') + raise + (root / (name + '.stdout')).write_bytes(result.stdout) + (root / (name + '.stderr')).write_bytes(result.stderr) + receipt = {'command': command, 'returncode': result.returncode, 'expectedReturncode': expected} + (root / (name + '-command.json')).write_text(json.dumps(receipt, indent=2) + '\n') + if result.returncode != expected: + raise RecipeError(f'{name}: expected exit {expected}, got {result.returncode}; retained receipts: {root}') + + +def check(binary: str, python: str, root: Path, name: str, expected: int, verdict: str) -> None: + """Assert the native structured verdict and unchanged reviewed baseline bytes.""" + command = [binary, 'check', '--baseline', str(root / 'execsurface.lock.json'), '--workspace', str(root), '--json-output', str(root / (name + '-verdict.json')), '--', python, '-m', 'pytest', '-q', '-s', '-p', 'no:cacheprovider', 'test_fixture.py'] + invoke(command, root, name, expected) + report = json.loads((root / (name + '-verdict.json')).read_bytes()) + if report['target'] != {'exit_code': 0, 'signal': None}: + raise RecipeError(f'{name}: pytest did not exit successfully; retained {root}') + if report['policy']['source'] != 'builtin:review-unmatched-drift': + raise RecipeError(f'{name}: selected built-in policy differs; retained {root}') + if verdict == 'REVIEW' and not any(finding['change'] == 'added' and finding['effect_kind'] == 'file_write' and finding['evidence']['effect']['target']['value'] == '$WORKSPACE/drift.txt' for finding in report['findings']): + raise RecipeError(f'{name}: deliberate file_write finding absent; retained {root}') + if report['verdict'] != verdict.lower(): + raise RecipeError(f'{name}: expected {verdict} verdict; retained {root}') + + +def run(binary: Path, python: Path, output: Path) -> dict: + """Create a fresh synthetic workspace and preserve both positive and drift evidence.""" + binary = binary.resolve(strict=True) + # Do not resolve Python's venv symlink: that loses the selected test environment. + python = python.absolute() + output = output.absolute() + output.mkdir(parents=True, exist_ok=False) + fixture = Path(__file__).with_name('fixtures') + for name in ('test_fixture.py', 'fixture.json'): + shutil.copyfile(fixture / name, output / name) + selected = {'binarySha256': hashlib.sha256(binary.read_bytes()).hexdigest(), 'pythonExecutable': str(python), 'fixtureSha256': hashlib.sha256((fixture / 'test_fixture.py').read_bytes()).hexdigest(), 'policy': 'unchanged built-in REVIEW for unmatched drift', 'privacy': 'public synthetic fixture; no environment or secret/file-content collection added'} + (output / 'selection.json').write_text(json.dumps(selected, indent=2) + '\n') + try: + invoke([str(binary), '--version'], output, 'version', 0) + if (output / 'version.stdout').read_text().strip() != 'execsurface 0.1.0-alpha.5': + raise RecipeError('recipe requires explicitly selected ExecSurface Alpha.5') + invoke([str(binary), 'doctor'], output, 'doctor', 0) + command = [str(binary), 'learn', '--output', str(output / 'execsurface.lock.json'), '--label', 'python-pytest-fixture', '--workspace', str(output), '--', str(python), '-m', 'pytest', '-q', '-s', '-p', 'no:cacheprovider', 'test_fixture.py'] + invoke(command, output, 'learn', 0) + before = (output / 'execsurface.lock.json').read_bytes() + check(str(binary), str(python), output, 'unchanged', 0, 'PASS') + # Only this public fixture value changes; command and pytest assertion stay the same. + (output / 'fixture.json').write_text('{"left":2,"right":3,"write_extra":true}\n') + check(str(binary), str(python), output, 'drift', 10, 'REVIEW') + if (output / 'execsurface.lock.json').read_bytes() != before: + raise RecipeError('baseline changed while checking drift') + if (output / 'drift.txt').read_text() != 'controlled fixture drift\n': + raise RecipeError('deliberate native write is missing') + report = {'status': 'verified-recipe', 'unchanged': {'verdict': 'PASS', 'exit': 0}, 'drift': {'verdict': 'REVIEW', 'exit': 10}, 'baselineSha256': hashlib.sha256(before).hexdigest(), 'scope': 'same-operator Alpha.5 Python/pytest fixture; runtime drift example, not independent security validation', 'receipts': str(output)} + except (OSError, ValueError, subprocess.TimeoutExpired) as error: + (output / 'failure.json').write_text(json.dumps({'status': 'failed', 'reason': str(error)}, indent=2) + '\n') + raise + (output / 'recipe-report.json').write_text(json.dumps(report, indent=2) + '\n') + return report + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--execsurface', type=Path, required=True) + parser.add_argument('--python', type=Path, default=Path(sys.executable)) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + print(json.dumps(run(args.execsurface, args.python, args.output), indent=2))