From 24e51c7f0b20f92abb63f356a385b3cc43bb5fb0 Mon Sep 17 00:00:00 2001 From: maxime-fleury Date: Fri, 9 Oct 2026 21:19:19 +0200 Subject: [PATCH] Audit hashes of regenerated gzip containers and propose a rule Two observations establish the failure mode: #202's canonical check requires the sha256 of a container it regenerates (exact_complex_flow_lift.py:409 and :419, required by source-assisted-v4/verify.py:162) and fails on unchanged inputs, while #194 passes in the same environment; and PR212's pinned archive rebuilds byte for byte as a tar payload but not as a container. The repository already has the fixes in use -- #194 decompresses before comparing, #217 pins payload hashes and #207 checks semantic fields -- so this adds a scanner that locates the pattern, the audit with file and line evidence, and one CONTRIBUTING.md paragraph stating the rule once. Generated with Codebuff Co-Authored-By: Codebuff --- .../CONTRIBUTING-note.md | 38 +++++ .../gzip-container-reproducibility/README.md | 153 ++++++++++++++++++ .../gzip_hash_audit.py | 118 ++++++++++++++ 3 files changed, 309 insertions(+) create mode 100644 research/gzip-container-reproducibility/CONTRIBUTING-note.md create mode 100644 research/gzip-container-reproducibility/README.md create mode 100644 research/gzip-container-reproducibility/gzip_hash_audit.py diff --git a/research/gzip-container-reproducibility/CONTRIBUTING-note.md b/research/gzip-container-reproducibility/CONTRIBUTING-note.md new file mode 100644 index 0000000000..d000c266fd --- /dev/null +++ b/research/gzip-container-reproducibility/CONTRIBUTING-note.md @@ -0,0 +1,38 @@ +# Proposed `CONTRIBUTING.md` paragraph + +Insertion point: as one new paragraph immediately after the paragraph that ends + +> Include regenerated certificates and patches in the same change. Run the +> selected incremental target, `make entrance-bank-verify`, as well as the checks +> affected by your changes. The current proof and bank scheduling supplement are +> supplied as Markdown; historical PDF targets belong to their respective +> checkpoints. Review changes to claims in the README and note together. + +and before "New contributions are under the repository's Apache-2.0 license." +The quote is verbatim from `CONTRIBUTING.md` at this branch's base (`main`, tip +`3b6b6689`); the paragraph around it stays unchanged. + +## Text to insert + +> **Regenerated artifacts and hashes.** When a check regenerates a file, compare +> payloads, not containers: the bytes of a gzip or tar container depend on the writing +> interpreter, zlib and tar build, so a pinned container digest can fail on unchanged +> inputs and can hide a real change behind an environment difference. Decompress before +> comparing (`#194`'s regeneration check does this: +> `old, data = gzip.decompress(old), gzip.decompress(data)`), or pin the payload's digest +> or the mathematical fields instead of the container's. Pinning the sha256 of a +> container that is committed stays fine, because those bytes are fixed in the repository +> and no check regenerates them. If a submission records a container digest, it should +> label it as an environment receipt, record the interpreter and zlib versions beside it, +> and not require it to match across environments. + +## Why this paragraph, and not a patch to a specific package + +The rule applies to any package that regenerates a container, and at least five merged or +open packages touch it. The one observed failure is #202's canonical check, whose integer +κ and every other leaf reproduce; the fix `#207` already prefers is to check the lift +through `exact_scalar_program_sha256` and the complex profile's fields. A general sentence +in `CONTRIBUTING.md` states the rule once, where a reviewer will meet it before writing +the next such check, instead of being re-litigated per package. The audit, its evidence +and the scanner that reproduces it are in +`research/gzip-container-reproducibility/`. diff --git a/research/gzip-container-reproducibility/README.md b/research/gzip-container-reproducibility/README.md new file mode 100644 index 0000000000..a94819c39e --- /dev/null +++ b/research/gzip-container-reproducibility/README.md @@ -0,0 +1,153 @@ +# Hashes of regenerated gzip containers: audit and convention + +**No κ, no certificate change, no new mathematics.** This is a reproducibility audit of +one pattern in this repository, plus a convention and a scanner for it, prompted by an +observed failure and an independent replay. + +## The pattern and why it breaks + +A package writes a container with `gzip.compress(...)`, hashes the **compressed** bytes, +and requires that hash to equal a value committed or pinned when the container was +produced elsewhere. The deflate stream of a gzip container depends on the writing +interpreter's zlib build (and on block splitting), so the check can fail on unchanged +inputs and can hide a real change behind an environment difference. + +Two observations in this repository establish the failure mode, from opposite directions: + +1. **A check that fails on unchanged inputs.** #202's headline verifier, + `research/source-assisted-v4/verify.py`, stops at line 162 with + `AssertionError: Canonical certificate does not reproduce`. Exactly seven certificate + leaves differ and all of them derive from one value: the sha256 of the compressed + lift certificate, written by + `research/source-assisted/decision/exact_complex_flow_lift.py:409` + (`gzip.compress(encoded, compresslevel=9, mtime=0)`) and recorded at line 419 as + `certificate_sha256=digest(cert_path)`, where `digest()` is a sha256 of the **file** + (line 40-41). Every other leaf, including κ, the complex saving, the bit effective + saving and the assembly records, is equal. The independent replay in PR #214 + reproduces this on CPython 3.13.16 with the pinned numpy 2.3.5 / scipy 1.17.0, and + reports the same failure under CPython 3.12.3 for #194's own verifier, whose lift + inputs are unchanged between the two commits. #194's CI passed the same check under + the same interpreter and package versions. +2. **A rebuild instruction that does not reproduce.** PR #212 documents its pinned + archive as byte-for-byte rebuildable with + `git archive --format=tar a1175449 | gzip -n -9`. The **tar payload** is byte + identical to the commit (verified), but the shipped container is not: it hashes + `316aa694…` at 1,405,363 bytes, while a rebuild here gives `cb71d72d…` at 1,404,558 + bytes, and no zlib level 1-9 reproduces the shipped length. The archive's own file + hashes are still checked by `ceiling.py`, so no number changes. + +## What the repository already does correctly + +The right shape is not new here, and it is used in more than one place: + +* **Compare the payload.** The regeneration check in `paired_cube_bit_word.py` (present at + #194's head and carried into the packages that inherit it) carries the reason in a + comment: + ```python + if name.endswith('.gz'): # gzip bytes depend on the zlib build; compare the compressed content + old, data = gzip.decompress(old), gzip.decompress(data) + require(old == data, 'byte-identical regeneration of ' + name) + ``` + and #194's `bit/prove.py` does the same (`assert record == json.loads(gzip.decompress(target.read_bytes())), 'Exact prime witnesses changed'`). +* **Pin payload hashes or semantic fields.** #217's `expected-bit.json` pins the payload + hashes of `graph_p12.json`, `word_p12.json` and `frames_p12.json` (its `emitted_sha256`), + and `run_bit.py` records `prime_canonical_sha256 = sha256(witness_bytes)` next to the + container digest. #207's `verify_inner.py:23` and `29-30` check + `exact_scalar_program_sha256` and the complex profile's mathematical fields instead of + a container. +* **Pin committed containers.** Pinning the sha256 of a `.gz` file that is committed is + safe: those bytes are fixed in the repository and no check regenerates them. That is + what `research/aligned-final-pricing-result/SOURCE.json:12-21` and the vendored + upstream `SOURCE.json` files do. +* **Record both, and require only the payload.** `research/pair-assembly/frame/frame_compile.py:41-49`, + inherited by #194, #202, #207, #212 and #216, writes `frame-word-{h}.json.gz` and records + `gzip_sha256=sha256(path.read_bytes())` next to `word_sha256=sha256(raw)`; #217's + `run_bit.py:151` records `prime_witness_sha256` next to `prime_canonical_sha256`. Both + make the payload the canonical value; PR #207/#210/#216's + `chart_witness_sha256=hashlib.sha256((HERE/'pr200-gauge-charts.json.gz').read_bytes())` + and #194/#202's `certificate_sha256=digest(cert_path)` record a container digest alone. + Recorded-only digests do not fail in the runs we read, but a future comparison could + mistake one for a canonical check. + +## Scanner + +`gzip_hash_audit.py` is a heuristic reader that locates the pattern and prints the +evidence. It is not a proof of absence: it flags a file when it hashes a container path, +separates files that also decompress, and reports comparisons it can see in the same +file. Cross-file comparisons (a digest recorded in one file and required in another, as +in observation 1) must be read by hand, which is why the two observations above are +stated as chains of file and line rather than as scanner output. + +```sh +python3 research/gzip-container-reproducibility/gzip_hash_audit.py +``` + +Findings from a run on the exact heads, each checked out as a full tree +(`git worktree add --detach `), on 2026-10-09. "Risky" is the scanner's +`RISK` class: a file that writes a container and records a digest of it. + +| Head | `.py` files | `RISK` | flagged | +|---|---:|---:|---| +| this repository's `main` `3b6b668` | 639 | 3 | `research/pair-assembly/frame/frame_compile.py:48`, `research/machine-transfer-verification/stream-transfer/experiments/balanced_split.py:64`, `scripts/experiments/probe_pair_ranked.py:41` | +| #194 `a8c8778` | 578 | 3 | those three, with `frame_compile.py` inherited, plus `research/source-assisted/decision/exact_complex_flow_lift.py:419` | +| #202 `8d8d67b` | 605 | 3 | the same three; the `exact_complex_flow_lift.py` record is the one observation 1 follows | +| #207 `cd14825` | 650 | 4 | the merged tree's three plus `research/coordinated-crossover-pr200/geometry/audit-gauge-charts.py:50` | +| #212 `9b6ad03` | 640 | 3 | the merged tree's three | +| #216 `34abdc5` | 660 | 5 | the merged tree's three plus `audit-gauge-charts.py:50` in `coordinated-crossover-pr200/` and in its vendored `coordinated-crossover-pr200-v2/baseline/` | +| #217 `b739fc2` | 654 | 3 | the merged tree's three; its `run_bit.py:151` records the container digest *and* the payload digest, and the scanner does not flag it | + +Three files are flagged at every head, and they read differently once opened: + +* `frame_compile.py:41-49` writes `frame-word-{h}.json.gz` and records + `gzip_sha256=sha256(path.read_bytes())` next to `word_sha256=sha256(raw)`. The payload + digest is the canonical one; the container digest is a record. It has observation 1's + shape only if a later check regenerates that container and requires the recorded value, + which is what the `#202` chain does for a different file, and what nothing does for this + one in the trees scanned here. +* `balanced_split.py:64` records `gzip_sha256` of the container it writes, and + `stream-transfer/experiments/verify_balanced_split.py:67` — run with + `--candidate ` — asserts `digest(word) == meta['gzip_sha256']` against the receipt + in that directory. Both files travel together out of the same run, so this is a pinned + container in the sense of the convention below; it becomes observation 1 if a harness + regenerates the container from the candidate's inputs while keeping the receipt. It is + the one place in the merged tree where a container digest is *required* against a stored + value, so it is the first thing to check if the pattern bites again. +* `probe_pair_ranked.py:41` records `profile_sha256=sha256(path.read_bytes())` for a file + it writes, into its own `screen.json`. The merged tree only pins the script's own digest + (`research/coordinated-frames-and-entrance-banks/SOURCE.json`) and points at it from a + review note; no reader of the recorded value appears in the trees scanned here. + +`audit-gauge-charts.py:50` (`#207`, `#216`) records `chart_witness_sha256` of a +regenerated `.gz` in a committed audit JSON. We found no run that requires it; it is a +record of the environment in which the chart was produced, which is what the convention +below asks such a record to say. + +## Convention + +1. **A check must compare payloads, not containers.** Decompress before comparing, or + compare a deterministic uncompressed serialization. `#194`'s + `gzip.decompress`-then-`require` is the precedent to copy. +2. **A pin should name a payload or a semantic field.** Pin the payload's sha256, or the + mathematical fields, or the interpreter-independent values (as #207's + `exact_scalar_program_sha256` and #217's `emitted_sha256` do). Pinning the sha256 of a + **committed** container stays fine. +3. **A container digest may be recorded, but only as an environment receipt.** If a + submission keeps one, it should say so, record the interpreter and zlib versions + beside it, and never require it to match across environments - otherwise an + environment difference is indistinguishable from a changed input. +4. **Prefer the payload check when a canonical check fails this way.** Re-point the check + at semantic fields (the fix #207 already implements for #202's supplier), rather than + regenerating the container in the reference environment, which leaves the same fault + for the next reader. + +`CONTRIBUTING-note.md` carries the same rule as a paragraph written for +`CONTRIBUTING.md`, after the paragraph that asks contributors to include regenerated +certificates. Placing it there is the maintainers' call. + +## Scope and credits + +This note audits a pattern, not any author's mathematics; nothing here contradicts a +submitted number or a verifier that passes. Observation 1 is PR #214's diagnosis, +reproduced here by reading the cited lines; observation 2 is from the audit of PR #212. +The convention follows the practice already present in #194, #207 and #217. Scanner and +note prepared with Anthropic Claude assistance; Apache-2.0. diff --git a/research/gzip-container-reproducibility/gzip_hash_audit.py b/research/gzip-container-reproducibility/gzip_hash_audit.py new file mode 100644 index 0000000000..e03c18f0e9 --- /dev/null +++ b/research/gzip-container-reproducibility/gzip_hash_audit.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Find gzip-container hashing in a checkout, and classify it. + +The failure this audits: a package writes a gzip container with `gzip.compress(...)` +(or `GzipFile`), hashes the *compressed* bytes, and requires that hash to equal a +value committed or pinned when the container was produced in someone else's +environment. The deflate stream of a gzip container depends on the writing +interpreter and zlib build, so that check can fail on unchanged inputs. Observed: +#202's headline verifier stops with `Canonical certificate does not reproduce` on +CPython 3.13.16 with the pinned numpy/scipy (independent replay, PR #214), while +#194 -- #202 one commit earlier with unchanged lift inputs -- passes in the same +environment. PR #212's pinned `baseline-pr200.tar.gz` shows the same effect in the +other direction: its tar payload reproduces byte for byte from the commit, its +container does not. + +Two shapes are fine, and both are already used in this repository: + + * compare the payload: `gzip.decompress` before comparing, as #194's regeneration + check does -- + if name.endswith('.gz'): # gzip bytes depend on the zlib build + old, data = gzip.decompress(old), gzip.decompress(data) + require(old == data, ...) + and as #194's `bit/prove.py` does (`assert record == json.loads(gzip.decompress(...))`). + * pin a payload hash or the semantic fields: #217's `expected-bit.json` pins the + payload hashes of `graph_p12.json`, `word_p12.json`, `frames_p12.json`; #207's + `verify_inner.py` compares `exact_scalar_program_sha256` and the complex profile's + fields instead of the lift container. + +A third fine shape is pinning the sha256 of a *committed* `.gz` file: those bytes are +fixed in the repository and are never regenerated by a check. + +This scanner is a heuristic reader, not a proof. It reports three things per file: +whether a container is written, whether a container path is hashed, and whether that +digest is compared in the same file. Cross-file comparisons (a digest recorded in +one file and required in another) have to be read by hand; the note lists them. + +Usage: + python3 gzip_hash_audit.py ROOT [ROOT ...] + python3 gzip_hash_audit.py --strict ROOT # exit 1 if a container digest is hashed +""" +import argparse +import re +import sys +from pathlib import Path + +WRITE = re.compile(r'gzip\.compress|GzipFile\(|gzip\.open\([^\n]*[\'"][wax]') +GZ_PATH = re.compile(r'(\w+)\s*=\s*[^\n]*[\'"][^\'"\n]*\.gz|with_suffix\(\s*[\'"][^\'"\n]*\.gz') +HASH_CALL = re.compile(r'\b(digest|sha256|sha1)\s*\(') +COMPARE = re.compile(r'(==|!=)[^\n]*\b\w*(_sha256|_sha1|digest)\b|\b\w*(_sha256|_sha1)\b[^\n]*(==|!=)') +SKIP_DIRS = {'.git', '.work', '__pycache__', '.venv', 'node_modules', 'site-packages'} + + +def classify(path): + try: + text = path.read_text(errors='replace') + except OSError: + return None + if 'gzip' not in text and 'GzipFile' not in text: + return None + lines = text.splitlines() + writes = [n for n, line in enumerate(lines, 1) if WRITE.search(line)] + gz_vars = set() + for line in lines: + for name in re.findall(r'(\w+)\s*=\s*[^\n]*[\'"][^\'"\n]*\.gz', line): + gz_vars.add(name) + for name in re.findall(r'(\w+)\s*=\s*[^\n]*with_suffix\(\s*[\'"][^\'"\n]*\.gz', line): + gz_vars.add(name) + hashes, evidence = [], [] + for n, line in enumerate(lines, 1): + if not HASH_CALL.search(line): + continue + if '.gz' in line or any(re.search(r'\b%s\b' % re.escape(v), line) for v in gz_vars): + hashes.append(n) + evidence.append('%s:%d: %s' % (path.name, n, line.strip()[:160])) + compares = [n for n, line in enumerate(lines, 1) if COMPARE.search(line)] + if not writes and not hashes: + return None + return dict(path=str(path), writes=writes, hashes=hashes, compares=compares, + decompress=bool(re.search(r'gzip\.decompress', text)), evidence=evidence[:4]) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('roots', nargs='+', type=Path) + ap.add_argument('--strict', action='store_true') + args = ap.parse_args() + risky = 0 + for root in args.roots: + root = root.resolve() + print('=== %s' % root) + for path in sorted(root.rglob('*.py')): + if any(part in SKIP_DIRS for part in path.parts): + continue + info = classify(path) + if info is None: + continue + if info['hashes'] and not info['decompress']: + flag, risky = 'RISK', risky + 1 + elif info['hashes']: + flag = 'MIX ' + else: + flag = ' ' + print(' %s writes=%d container_digest=%d compared_here=%d decompress=%s %s' + % (flag, len(info['writes']), len(info['hashes']), len(info['compares']), + info['decompress'], info['path'])) + for line in info['evidence']: + print(' %s' % line) + print(' scanned .py files: %d' % sum(1 for _ in root.rglob('*.py'))) + if args.strict and risky: + print('FAIL %d file(s) hash a container path with no decompress-compare' % risky) + return 1 + print('OK scanned; %d risky file(s)' % risky) + return 0 + + +if __name__ == '__main__': + sys.exit(main())