From 8d7e01925177cb3268c8157bfddd2bae8a706ac5 Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Tue, 27 Oct 2026 15:47:09 +0000 Subject: [PATCH 1/6] ci: retire custom-code budgets and document generated-file ownership --- .castiron-ratchet.json | 5 ----- AGENTS.md | 19 +++++++++++-------- CONTRIBUTING.md | 27 +++++++++++---------------- 3 files changed, 22 insertions(+), 29 deletions(-) delete mode 100644 .castiron-ratchet.json diff --git a/.castiron-ratchet.json b/.castiron-ratchet.json deleted file mode 100644 index 7c2f8ca0f..000000000 --- a/.castiron-ratchet.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "schema_version": 2, - "generated_files": "strict", - "max_custom_patch_lines": 0 -} diff --git a/AGENTS.md b/AGENTS.md index 86eecb297..77765534b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,15 +7,18 @@ generator owns the code you are changing, preserve existing generated behavior, authoritative source when it is available. The `openai-java-example/` directory is not modified by the generator. -## Custom-code budget +## Generated-file ownership -Follow [the custom-code guidance](scripts/castiron/CUSTOM_CODE.md). Budget changes -belong in a separate PR containing only `.castiron-ratchet.json`, with an explicit justification -in the PR description. Increases require a **human approving review** before merging. -Agents may investigate and draft proposals, but must not approve budget increases -(including through a human's credentials) or bypass the gate. Do not weaken -counting, broaden exclusions, or alter generation metadata to make a change pass. -The checker and effective budget come from main, not the PR. Keep default CODEOWNERS. +Never hand-edit generated SDK files, including generated tests, documentation, +workflows, or the generated-file inventory. Change the authoritative API schema, +configuration, or compiler, then regenerate. Put SDK-specific behavior in +separately owned handwritten helpers and extension points. + +Regeneration replaces generator-owned files and deletes retired generated paths. +Handwritten edits to generated files are overwritten; they are never replayed or +merged into new output. Preserve separately owned handwritten files and resolve +ownership collisions explicitly. Enforce this rule through code review; there is +no customization budget or additional generated-file CI gate. ## Security requirements diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6febab8f4..311bd658a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,22 +24,17 @@ management. The framework-neutral SDK requires Java 8, while development require Kotlin toolchain. See the [Java version support policy](docs/version-support-policy.md) for artifact-level runtime, framework, lifecycle, and release rules. -## Custom-code budget +## Generated-file ownership -The custom-code budget counts additions plus deletions in the remaining patch -against verified generated output. `.castiron-ratchet.json` defines this repository's -ceiling. CI uses the checker and budget on main, not the PR's proposed versions. +Do not edit generated SDK files by hand. Fix the API schema, generation +configuration, or compiler and regenerate instead. Put SDK-specific behavior in +separately owned handwritten helpers and extension points. -Budget changes must be in a separate PR modifying **only `.castiron-ratchet.json`**. -Justify the current usage, proposed ceiling, and why fixing generation is not -appropriate in the PR description. Increases require a **human approving review** -and must merge before an SDK change relies on them. Agents may draft proposals, -but must not approve increases or bypass the gate. Keep default CODEOWNERS. -Lower the ceiling after cleanup while retaining headroom; decreases must still -fit the measured usage. - -See [custom-code technical details](scripts/castiron/CUSTOM_CODE.md) for accounting, -local checks, trusted CI, and activation instructions. +Regeneration replaces generated files and removes retired generated paths. +Handwritten edits to generated output are overwritten, not carried forward. +Separately owned handwritten files are preserved. Resolve ownership conflicts +explicitly before regeneration. Code review enforces this rule; customization +budgets and their CI checks are retired. ## Security expectations @@ -90,8 +85,8 @@ artifact's eligibility on a given JVM come from `gradle/version-support.properti ## Modifying or adding code -Most of the SDK is generated code. Modifications to code will be persisted between generations, but may -result in merge conflicts between manual patches and changes from the generator. The generator will never +Most of the SDK is generated code. Never edit generated files by hand; change +the authoritative generation inputs or compiler and regenerate instead. The generator will never modify the contents of the `openai-java-example/` directory. ## Adding and running examples From 8a046bfe8dee6ade3601b7e416e4d2d75b2f9934 Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Tue, 27 Oct 2026 16:09:59 +0000 Subject: [PATCH 2/6] ci: remove retired budget publishers from security models --- docs/architecture/security-model.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/architecture/security-model.md b/docs/architecture/security-model.md index d3b27b2a8..7917dcb05 100644 --- a/docs/architecture/security-model.md +++ b/docs/architecture/security-model.md @@ -1,5 +1,10 @@ # OpenAI Java SDK security model +Generated files are updated through authoritative inputs or the compiler and +regeneration; handwritten edits to them are forbidden. Code review enforces this +ownership rule. The customization-budget reporter and publishers are retired +and are not part of the active CI or security controls. + This is the repository's single canonical detailed threat model. Use it for security review and Codex Security scans of the revision being reviewed. SECURITY.md remains the authority for private disclosure and reportability @@ -106,13 +111,10 @@ not exposed to the attestation action. boundary where independently mutable data enters a parser/evaluator, where runtime/network data reaches a sensitive sink, or where PR-controlled code can reach protected CI or release credentials. -- Candidate Git objects can still be lower-trust data when a separately - privileged workflow evaluates them. The workflow_run custom-code workflow - checks out trusted main, computes from current PR Git objects, and freshness - checks the exact head before publishing status; breaking that evaluator boundary - remains a security finding. (.github/workflows/castiron-custom-code-comment.yml:34-57; - .github/workflows/castiron-custom-code-comment.yml:74-98; - .github/workflows/castiron-custom-code-comment.yml:128-181) +- Candidate Git objects remain lower-trust data when a separately privileged + evaluator handles them. Such an evaluator must independently constrain that + data before accessing protected CI or release authority. + - Merged, reviewed repository source intentionally executes with trusted build/release authority. The security boundary is externally enforced branch governance, CODEOWNERS/rulesets, verified release selection, and protected @@ -171,7 +173,6 @@ These are review hypotheses, not confirmed findings. | High | Untrusted webhook bytes are parsed or acted on without valid signature/freshness checks. | Embedding application exposes webhook ingress and relies on SDK verification. | Forged event acceptance. | Required headers, bounded timestamp tolerance, HMAC-SHA256, timing-safe compare, then parse; no replay deduplication. | Preserve verify-before-parse ordering and focused regression tests; embedding applications must deduplicate accepted webhook IDs when replay matters. | openai-java-core/src/main/kotlin/com/openai/services/WebhookVerification.kt:19-99; openai-java-core/src/main/kotlin/com/openai/services/blocking/WebhookServiceImpl.kt:33-41 | | Medium | Independently mutable API/SSE JSON crosses a parser boundary and causes unsafe interpretation, resource exhaustion, or sensitive exception/log output. | Attacker controls an API/network response or configured proxy; sink is reachable in caller deployment. | Client failure, data exposure, or downstream misuse. | Strict mapper defaults, typed parsing, stream iteration, and explicit DEBUG diagnostics. | Keep parsing robust and large-payload-compatible; avoid treating model output as executable authority without caller controls. | openai-java-core/src/main/kotlin/com/openai/core/ObjectMappers.kt:34-116; openai-java-core/src/main/kotlin/com/openai/core/handlers/SseHandler.kt:19-60; AGENTS.md:49-66 | | Medium | Credentials or customer payloads reach an unintended destination or uncontrolled log. | Lower-trust actor independently controls base URL/proxy/TLS/log sink, or diagnostics are enabled in an unsafe deployment. | Secret or customer-data disclosure. | Production URL default, caller-owned configuration, auth wrappers, and default sensitive-header redaction; DEBUG is explicit. | Preserve redaction and destination/TLS controls; document caller obligations. | openai-java-core/src/main/kotlin/com/openai/core/ClientOptions.kt:151-172; openai-java-core/src/main/kotlin/com/openai/core/ClientOptions.kt:601-665; openai-java-core/src/main/kotlin/com/openai/core/http/LoggingHttpClient.kt:24-48; openai-java-core/src/main/kotlin/com/openai/core/http/LoggingHttpClient.kt:91-142 | -| Medium | A separately privileged evaluator consumes mutable PR Git objects, artifacts, or cache entries as trusted code/data. | Privileged workflow reads candidate state without binding/ref validation. | Privilege crossing beyond ordinary PR CI. | Custom-code workflow checks out main, uses current PR objects, and freshness-checks exact head; PR cache scope remains isolated. | Preserve trusted checkout and exact-head/digest binding. | .github/workflows/castiron-custom-code-comment.yml:34-57; .github/workflows/castiron-custom-code-comment.yml:128-181; .github/workflows/ci.yml:120-129 | | Low / not a boundary by itself | A PR changes a checked-in example, test, fixture, build script, or generated source and ordinary PR CI executes it. | The same PR author already controls the tracked executable checkout. | No new capability beyond the unprivileged PR job. | PR CI has read-only contents and no release credentials; cache scope is enforced outside PR code. | Investigate only if execution crosses an independent input or protected-authority boundary. | .github/workflows/ci.yml:8-20; .github/workflows/ci.yml:33-48; .github/workflows/ci.yml:92-107 | ## Severity calibration (critical, high, medium, low) From 3b5f84974fa38e04ae92a6eb21e43391c499fc98 Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Tue, 27 Oct 2026 16:33:15 +0000 Subject: [PATCH 3/6] docs: retire stale customization-budget operating guidance --- tasks/plan.md | 4 ++++ tasks/todo.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/tasks/plan.md b/tasks/plan.md index c2292cbba..b45a5a1e0 100644 --- a/tasks/plan.md +++ b/tasks/plan.md @@ -1,5 +1,9 @@ # Java generated/custom code transition +> Historical transition record: budgets and reporter commands below are retired. +> Follow [AGENTS.md](../AGENTS.md#generated-file-ownership) for current ownership +> guidance. Generated output is replaced; handwritten edits are not replayed. + ## Consolidated rollout checkpoint — October 20, 2026 (Pacific) This checkpoint supersedes the older allocation and pending-review notes below; diff --git a/tasks/todo.md b/tasks/todo.md index 0bf075a70..4439646d7 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -1,5 +1,9 @@ # Java generated/custom transition tasks +> Historical transition checklist: budgets and reporter commands below are retired. +> Follow [AGENTS.md](../AGENTS.md#generated-file-ownership) for current ownership +> guidance. Generated output is replaced; handwritten edits are not replayed. + ## Consolidated rollout checkpoint — October 20, 2026 (Pacific) This checkpoint supersedes the older allocation and pending-review notes below; From 9214e3c6610aa1a5f26a9b32f6e0670fab3b5c19 Mon Sep 17 00:00:00 2001 From: jbeckwith-oai <269036200+jbeckwith-oai@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:45:04 +0000 Subject: [PATCH 4/6] chore(api): Retire obsolete customization reporting Castiron-Internal-PR: https://github.com/openai/openai-java-internal/pull/377 Castiron-Source-SHA: e3cbc36f43a0f45a8d1dec31e8f4035cd02c4fe2 Castiron-Public-Base-SHA: 4311dba26c34f5d813105f94f2a76ba9073561dc --- .castiron.stats.yml | 7 +- .../castiron-custom-code-comment.yml | 904 ---------- .github/workflows/castiron-custom-code.yml | 116 -- scripts/castiron/CUSTOM_CODE.md | 298 --- scripts/castiron/README.md | 29 - scripts/castiron/custom_code_budget.py | 978 ---------- scripts/castiron/custom_code_policy.py | 731 -------- scripts/castiron/custom_code_report.py | 1534 ---------------- scripts/castiron/custom_code_test_support.py | 116 -- .../castiron/fixtures/github_publisher.cjs | 183 -- scripts/castiron/generated-files.json | 76 - scripts/castiron/ruff.toml | 3 - scripts/castiron/test_custom_code_budget.py | 430 ----- scripts/castiron/test_custom_code_github.py | 1048 ----------- scripts/castiron/test_custom_code_policy.py | 988 ---------- .../test_custom_code_policy_source.py | 198 -- .../castiron/test_custom_code_publication.py | 600 ------- scripts/castiron/test_custom_code_report.py | 1591 ----------------- .../test_custom_code_review_revocation.py | 986 ---------- .../test_custom_code_status_publication.py | 782 -------- scripts/castiron/test_custom_code_strict.py | 774 -------- 21 files changed, 3 insertions(+), 12369 deletions(-) delete mode 100644 .github/workflows/castiron-custom-code-comment.yml delete mode 100644 .github/workflows/castiron-custom-code.yml delete mode 100644 scripts/castiron/CUSTOM_CODE.md delete mode 100644 scripts/castiron/README.md delete mode 100755 scripts/castiron/custom_code_budget.py delete mode 100755 scripts/castiron/custom_code_policy.py delete mode 100755 scripts/castiron/custom_code_report.py delete mode 100644 scripts/castiron/custom_code_test_support.py delete mode 100644 scripts/castiron/fixtures/github_publisher.cjs delete mode 100644 scripts/castiron/ruff.toml delete mode 100644 scripts/castiron/test_custom_code_budget.py delete mode 100644 scripts/castiron/test_custom_code_github.py delete mode 100644 scripts/castiron/test_custom_code_policy.py delete mode 100644 scripts/castiron/test_custom_code_policy_source.py delete mode 100644 scripts/castiron/test_custom_code_publication.py delete mode 100644 scripts/castiron/test_custom_code_report.py delete mode 100644 scripts/castiron/test_custom_code_review_revocation.py delete mode 100644 scripts/castiron/test_custom_code_status_publication.py delete mode 100644 scripts/castiron/test_custom_code_strict.py diff --git a/.castiron.stats.yml b/.castiron.stats.yml index 93c7e9f78..417e87d26 100644 --- a/.castiron.stats.yml +++ b/.castiron.stats.yml @@ -1,8 +1,7 @@ schema_version: 1 -generation_id: 68bd942a-6d9f-44d3-8c8b-46ae55d85e50 +generation_id: 9e3623b2-d4e0-44b9-907b-93835ac5946a openapi_spec_hash: 3273c8e0a3c60aa72222e3b5970513fd openapi_transformed_spec_hash: da0a1022fdcb8ba16fe3417d9b670a6d config_hash: 8a39abf288ab81f39720996b7bfd990a -codegen_sha: 3e9a4b126b6d25d5776122bb165289701c36e63d -codegen_hash: cc631164dc74d37b508a404ddfb80481610c613607a383bfc941836918e2d22c -public_codegen_sha: eeb7c1db53408f178c5b8c63a027b112d47fcd59 +codegen_sha: 72c32974dfdec5fd9be954a92a4aaca7fd9ba0fc +codegen_hash: 5bbc3afa0a60dc7f537bb2553a476ae5438ff0f0e1e61b6feb54ee69f0a0268c diff --git a/.github/workflows/castiron-custom-code-comment.yml b/.github/workflows/castiron-custom-code-comment.yml deleted file mode 100644 index 74f04360a..000000000 --- a/.github/workflows/castiron-custom-code-comment.yml +++ /dev/null @@ -1,904 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -name: Castiron custom code comment - -on: - workflow_run: - workflows: [Castiron custom code] - types: [completed] - -permissions: {} - -concurrency: - group: castiron-custom-code-comment-${{ github.event.workflow_run.head_repository.id }}-${{ github.event.workflow_run.head_branch }} - cancel-in-progress: false - queue: max # Preserve newer pending evaluations if older runs arrive out of order. - -jobs: - review-change: - name: Invalidate reviewed policy statuses - if: github.event.workflow_run.event == 'pull_request_review' - runs-on: ubuntu-latest - timeout-minutes: 5 - concurrency: - group: castiron-budget-writes-${{ github.repository_id }} - cancel-in-progress: false - queue: max - permissions: - actions: write - contents: read - pull-requests: read - statuses: write - steps: - - name: Re-evaluate current PR and queue after review changes - env: - GH_TOKEN: ${{ github.token }} - GIT_CONFIG_COUNT: '2' - GIT_CONFIG_KEY_0: credential.helper - GIT_CONFIG_VALUE_0: '' - GIT_CONFIG_KEY_1: credential.https://github.com.helper - GIT_CONFIG_VALUE_1: '!gh auth git-credential' - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - // Trusted main orchestration only. Signal fields never choose status targets. - const repo = `${context.repo.owner}/${context.repo.repo}`; - const signal = context.payload.workflow_run; - const {data: run} = await github.rest.actions.getWorkflowRun({...context.repo, run_id: signal.id}); - const {data: workflow} = await github.rest.actions.getWorkflow({ - ...context.repo, workflow_id: 'castiron-custom-code.yml', - }); - if (!Number.isSafeInteger(run.workflow_id) || run.workflow_id !== workflow.id || - workflow.path !== '.github/workflows/castiron-custom-code.yml') return; - if (run.repository.full_name !== repo || run.event !== 'pull_request_review' || - run.status !== 'completed' || run.id !== signal.id || run.run_attempt !== signal.run_attempt || - run.head_sha !== signal.head_sha || run.path.split('@', 1)[0] !== '.github/workflows/castiron-custom-code.yml') { - throw new Error('Unexpected or superseded review signal'); - } - async function mergeBase(base, head) { - const {data: comparison} = await github.rest.repos.compareCommitsWithBasehead({ - ...context.repo, basehead: `${base}...${head}`, per_page: 1, - }); - const sha = comparison.merge_base_commit?.sha; - if (!/^[0-9a-f]{40}$/.test(sha || '')) throw new Error('Invalid policy merge base'); - return sha; - } - async function requiresHuman(pr, targetSha, mainSha) { - if (!/^[0-9a-f]{40}$/.test(targetSha || '')) throw new Error('Invalid authenticated target'); - if (!/^[0-9a-f]{40}$/.test(mainSha || '')) throw new Error('Invalid authenticated main'); - const path = require('node:path'); - const fs = require('node:fs'); - const root = fs.mkdtempSync(path.join(process.env.RUNNER_TEMP, 'castiron-review-')); - try { - // Bind executable policy and its dependency to the authenticated - // main snapshot for this decision, including after a retry. - for (const name of ['custom_code_policy.py', 'custom_code_report.py']) { - const {data: file} = await github.rest.repos.getContent({ - ...context.repo, path: `scripts/castiron/${name}`, ref: mainSha, - }); - if (file.type !== 'file' || file.encoding !== 'base64' || - !Number.isSafeInteger(file.size) || file.size <= 0 || file.size > 1048576 || - typeof file.content !== 'string') throw new Error('Invalid trusted policy file'); - const content = Buffer.from(file.content, 'base64'); - if (content.length !== file.size) throw new Error('Invalid trusted policy content'); - fs.writeFileSync(path.join(root, name), content, {flag: 'wx'}); - } - const result = await exec.getExecOutput('python3', ['-I', - path.join(root, 'custom_code_policy.py'), 'review-required', - '--repository', repo, '--repo', path.join(root, 'objects.git'), - '--base', targetSha, '--head', pr.head.sha, '--main', mainSha], {silent: true}); - const decision = JSON.parse(result.stdout); - if (typeof decision.requires_human !== 'boolean') throw new Error('Invalid review decision'); - return decision.requires_human; - } finally { - fs.rmSync(root, {recursive: true, force: true}); - } - } - const dirtyTargets = new Set(); - const matchesSource = pr => Boolean(run.head_repository?.id && run.head_repository?.full_name && run.head_branch && - pr.head?.ref === run.head_branch && pr.head?.repo?.id === run.head_repository.id && - pr.head?.repo?.full_name === run.head_repository.full_name); - // Retry discovery within this job timeout while targets are changing. - while (true) { - const explicitAssociation = Boolean(run.pull_requests?.length); - let pulls = run.pull_requests?.length ? run.pull_requests : await github.paginate( - github.rest.repos.listPullRequestsAssociatedWithCommit, {...context.repo, commit_sha: run.head_sha}); - let hasAssociation = true; - let matching = []; - for (let attempt = 0; attempt < 2; attempt++) { - const candidates = []; - for (const item of pulls) { - const {data: pr} = await github.rest.pulls.get({...context.repo, pull_number: item.number}); - if (pr.state !== 'open' || pr.base.ref !== 'next' || pr.base.repo.full_name !== repo || - (item.head?.sha && pr.head.sha !== item.head.sha) || !/^[0-9a-f]{40}$/.test(pr.head.sha)) continue; - if (!matchesSource(pr)) continue; - if (run.head_sha !== pr.head.sha) { - const {data: commit} = await github.rest.repos.getCommit({...context.repo, ref: run.head_sha}); - if (commit.parents.length !== 2 || commit.parents[1].sha !== pr.head.sha || - !/^[0-9a-f]{40}$/.test(commit.parents[0].sha) || - await mergeBase(commit.parents[0].sha, pr.base.sha) !== commit.parents[0].sha) continue; - // Associated reviews may name an older merge after next advances. - // Branch-only discovery still requires the live merge and base. - if (!hasAssociation) { - const {data: merge} = await github.rest.git.getRef({...context.repo, ref: `pull/${pr.number}/merge`}); - if (merge.object.sha !== run.head_sha || commit.parents[0].sha !== pr.base.sha) continue; - } - } - candidates.push(pr); - } - matching = candidates.length === 1 ? candidates : candidates.filter(pr => - Number.isSafeInteger(run.head_repository?.id) && run.head_repository.id > 0 && - pr.head.repo?.id === run.head_repository.id && typeof run.head_branch === 'string' && - run.head_branch.length > 0 && pr.head.ref === run.head_branch); - if (matching.length || explicitAssociation || attempt) break; - if (!run.head_repository?.id || !run.head_repository.full_name || !run.head_branch) return; - pulls = await github.paginate(github.rest.pulls.list, { - ...context.repo, state: 'open', head: `${run.head_repository.owner.login}:${run.head_branch}`, - }); - hasAssociation = false; - } - if (!matching.length) return; - if (matching.length !== 1) throw new Error('Review signal identifies ambiguous next PRs'); - const pr = matching[0]; - const files = await github.paginate(github.rest.pulls.listFiles, {...context.repo, pull_number: pr.number}); - if (!files.some(file => file.filename === '.castiron-ratchet.json')) return; - const {data: target} = await github.rest.git.getRef({...context.repo, ref: 'heads/next'}); - const targetSha = target.object.sha; - const {data: main} = await github.rest.git.getRef({...context.repo, ref: 'heads/main'}); - const mainSha = main.object.sha; - let requiresReview; - try { - requiresReview = await requiresHuman(pr, targetSha, mainSha); - } catch (error) { - const {data: currentMain} = await github.rest.git.getRef({...context.repo, ref: 'heads/main'}); - if (currentMain.object.sha !== mainSha) continue; - throw error; - } - if (!requiresReview) { - const [{data: current}, {data: target}, main] = await Promise.all([ - github.rest.pulls.get({...context.repo, pull_number: pr.number}), - github.rest.git.getRef({...context.repo, ref: 'heads/next'}), - mainSha && github.rest.git.getRef({...context.repo, ref: 'heads/main'}), - ]); - if (current.state !== 'open' || current.head.sha !== pr.head.sha || current.base.ref !== 'next' || - current.base.repo.full_name !== repo || target.object.sha !== targetSha || - (main && main.data.object.sha !== mainSha)) continue; - return; - } - // Read authoritative reviews, including edits: submittedAt alone misses - // removal/restoration of an attestation. Never trust candidate event artifacts. - let reviewCursor = null; - const latest = new Map(); - do { - const result = await github.graphql(`query($owner:String!,$name:String!,$number:Int!,$cursor:String) { - repository(owner:$owner,name:$name) { pullRequest(number:$number) { - reviews(first:100,after:$cursor) { nodes { fullDatabaseId state body updatedAt - author { login __typename } authorAssociation commit { oid } } - pageInfo { hasNextPage endCursor } } - } } - }`, {...context.repo, name: context.repo.repo, number: pr.number, cursor: reviewCursor}); - const page = result.repository.pullRequest.reviews; - for (const review of page.nodes) { - const value = review.fullDatabaseId; - if (!(typeof value === 'string' || Number.isSafeInteger(value)) || - !/^[1-9][0-9]*$/.test(String(value))) throw new Error('Invalid review identity'); - const identity = String(value); - if (['PENDING', 'COMMENTED'].includes(review.state) || !review.author) continue; - const previous = latest.get(review.author.login); - if (!previous || BigInt(identity) > BigInt(previous.identity)) latest.set(review.author.login, {...review, identity}); - } - if (!page.pageInfo.hasNextPage) break; - if (!page.pageInfo.endCursor || page.pageInfo.endCursor === reviewCursor) throw new Error('Invalid review cursor'); - reviewCursor = page.pageInfo.endCursor; - } while (true); - // Same v2 whole-body contract as the selector/publisher. This can only - // request reevaluation; the normal publisher still decides authorization. - const approvals = [...latest.values()].filter(review => - review.state === 'APPROVED' && review.commit?.oid === pr.head.sha && - review.author.__typename === 'User' && !review.author.login.endsWith('[bot]') && - review.author.login !== pr.user.login && - ['OWNER', 'MEMBER', 'COLLABORATOR'].includes(review.authorAssociation) && - typeof review.body === 'string' && - review.body.trim() === 'I personally reviewed and approve this policy change as a human: ' + pr.head.sha); - const authorized = approvals.length > 0; - if (authorized && (run.actor?.type !== 'User' || - !approvals.some(review => review.author.login === run.actor.login))) return; - for (const review of approvals) { - if (!Number.isFinite(Date.parse(review.updatedAt))) throw new Error('Missing review update time'); - } - const fingerprint = require('node:crypto').createHash('sha256').update(JSON.stringify( - [pr.head.sha, approvals.map(review => [review.identity, review.updatedAt]).sort((a, b) => a[0].localeCompare(b[0]))] - )).digest('hex'); - const marker = `[review ${authorized ? 'approve' : 'revoke'} ${fingerprint}]`; - async function queue() { - let cursor = null; - const entries = []; - do { - const result = await github.graphql(`query($owner:String!,$name:String!,$cursor:String) { - repository(owner:$owner,name:$name) { mergeQueue(branch:"next") { - entries(first:100,after:$cursor) { nodes { headCommit { oid } pullRequest { number headRefOid } } - pageInfo { hasNextPage endCursor } } - } } - }`, {...context.repo, name: context.repo.repo, cursor}); - const page = result.repository.mergeQueue?.entries; - if (!page) return []; - entries.push(...page.nodes); - if (!page.pageInfo.hasNextPage) return entries; - if (!page.pageInfo.endCursor || page.pageInfo.endCursor === cursor) throw new Error('Invalid queue cursor'); - cursor = page.pageInfo.endCursor; - } while (true); - } - const entries = await queue(); - const index = entries.findIndex(entry => entry.pullRequest?.number === pr.number && entry.pullRequest.headRefOid === pr.head.sha); - // Later queue entries can include the affected constituent. Re-evaluate all of them. - const targets = new Map([[pr.head.sha, 'pull_request']]); - for (const entry of index < 0 ? [] : entries.slice(index)) { - if (entry.headCommit === null) continue; // Queue preparation has not produced statuses yet. - if (!/^[0-9a-f]{40}$/.test(entry.headCommit.oid)) throw new Error('Invalid queue SHA'); - targets.set(entry.headCommit.oid, 'merge_group'); - } - // Revoke the authenticated snapshot before slower reads. Pending on an obsolete - // affected SHA cannot authorize a merge, and a later race must not retain success. - const url = `${context.serverUrl}/${repo}/actions/runs/${context.runId}`; - const activeTargets = new Map(); - const priorWriters = new Map(); - async function priorWriter(status) { - if (!/\[review (?:approve|revoke) [0-9a-f]{64}\]$/.test(status.description || '')) return null; - const identity = status.target_url?.slice(`${context.serverUrl}/${repo}/actions/runs/`.length); - if (!/^[1-9][0-9]*$/.test(identity || '') || !Number.isSafeInteger(Number(identity))) return null; - if (!priorWriters.has(identity)) { - const {data: definition} = await github.rest.actions.getWorkflow({ - ...context.repo, workflow_id: 'castiron-custom-code-comment.yml', - }); - let writer; - try { - ({data: writer} = await github.rest.actions.getWorkflowRun({...context.repo, run_id: Number(identity)})); - } catch (error) { - if (error.status !== 404) throw error; - priorWriters.set(identity, null); // Deleted runs cannot authenticate a marker. - return null; - } - const authenticated = definition.path === '.github/workflows/castiron-custom-code-comment.yml' && - writer.workflow_id === definition.id && writer.repository.full_name === repo && - writer.event === 'workflow_run' && writer.id === Number(identity); - priorWriters.set(identity, authenticated ? { - completed: writer.status === 'completed' && writer.conclusion === 'success', - } : null); - } - return priorWriters.get(identity); - } - const names = ['Castiron next / budget-only change', 'Castiron next / custom-code budget']; - for (const [sha, event] of targets) { - const statuses = await github.paginate(github.rest.repos.listCommitStatusesForRef, {...context.repo, ref: sha}); - const trusted = statuses.filter(status => status.creator?.login === 'github-actions[bot]' && - status.target_url?.startsWith(`${context.serverUrl}/${repo}/actions/runs/`)); - const current = new Map(); - for (const status of trusted) if (!current.has(status.context)) current.set(status.context, status); - // Only current pending markers coalesce. A later publisher failure - // supersedes recovery even if a restored review has the same timestamp. - const changed = []; - for (const name of names) { - const status = current.get(name); - // A failed/cancelled writer may have stopped after pending but - // before dispatch. Re-stamp and retry; only successful writers coalesce. - const writer = status?.state === 'pending' ? await priorWriter(status) : null; - const handled = status?.state === 'success' || - (status?.state === 'pending' && status.description?.endsWith(marker) && writer?.completed); - if (authorized ? !handled : - status?.state === 'success' || (writer && !writer.completed)) changed.push(name); - } - if (authorized && names.every(name => current.get(name)?.state === 'success')) continue; - for (const name of changed) { - await github.rest.repos.createCommitStatus({...context.repo, sha, context: name, state: 'pending', - description: `Review changed; reevaluation pending. ${marker}`, target_url: url}); - dirtyTargets.add(sha); - } - if (dirtyTargets.has(sha)) activeTargets.set(sha, event); - } - if (!activeTargets.size) return; - const reruns = []; - for (const [sha, event] of activeTargets) { - const runs = await github.paginate(github.rest.actions.listWorkflowRuns, { - ...context.repo, workflow_id: 'castiron-custom-code.yml', head_sha: sha, event, per_page: 100, - }); - const matches = []; - for (const item of runs) { - if (item.repository.full_name !== repo || item.event !== event || item.head_sha !== sha || - item.path.split('@', 1)[0] !== '.github/workflows/castiron-custom-code.yml') continue; - if (event === 'pull_request') { - const source = item.head_repository; - if (!source?.id || !source.full_name || !item.head_branch) continue; - const explicitAssociation = Boolean(item.pull_requests?.length); - let associated = item.pull_requests?.length ? item.pull_requests : await github.paginate( - github.rest.repos.listPullRequestsAssociatedWithCommit, {...context.repo, commit_sha: sha}); - const live = []; - for (let attempt = 0; attempt < 2; attempt++) { - for (const reference of associated) { - const {data: pull} = await github.rest.pulls.get({...context.repo, pull_number: reference.number}); - if (pull.state === 'open' && pull.head.sha === sha && pull.base.repo.full_name === repo && - pull.head.repo?.id === source.id && pull.head.repo?.full_name === source.full_name && - pull.head.ref === item.head_branch && ['main', 'next'].includes(pull.base.ref)) live.push(pull); - } - if (live.length || explicitAssociation || attempt) break; - associated = await github.paginate(github.rest.pulls.list, { - ...context.repo, state: 'open', head: `${source.owner.login}:${item.head_branch}`, - }); - } - // Match the canonical reporter's main preference for shared-head discovery. - const main = live.filter(pull => pull.base.ref === 'main'); - const selected = main.length ? main : live; - if (selected.length !== 1 || selected[0].number !== pr.number) continue; - } - matches.push(item); - } - matches.sort((a, b) => b.id - a.id); - if (matches[0]?.status === 'completed') reruns.push(matches[0].id); - } - // Retry against the new snapshot; already-observed affected statuses stay pending. - const {data: current} = await github.rest.pulls.get({...context.repo, pull_number: pr.number}); - const {data: base} = await github.rest.git.getRef({...context.repo, ref: 'heads/next'}); - if (current.state !== 'open' || current.head.sha !== pr.head.sha || current.base.ref !== 'next' || - current.base.repo.full_name !== repo || base.object.sha !== targetSha || - JSON.stringify(await queue()) !== JSON.stringify(entries)) continue; - // Only normal authenticated reporter runs may restore success. Never infer approval here. - for (const run_id of reruns) await github.rest.actions.reRunWorkflow({...context.repo, run_id}); - return; - } - - main-change: - name: Invalidate next queue after main advances - if: github.event.workflow_run.event == 'push' && github.event.workflow_run.head_branch == 'main' - runs-on: ubuntu-latest - timeout-minutes: 5 - concurrency: - group: castiron-budget-writes-${{ github.repository_id }} - cancel-in-progress: false - queue: max - permissions: - actions: write - contents: read - pull-requests: read - statuses: write - steps: - - name: Re-evaluate live next queue after main advances - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - // Main pushes only invalidate. The normal authenticated evaluator - // and serialized publisher are the only paths that restore success. - const repo = `${context.repo.owner}/${context.repo.repo}`; - const signal = context.payload.workflow_run; - const {data: run} = await github.rest.actions.getWorkflowRun({...context.repo, run_id: signal.id}); - const {data: workflow} = await github.rest.actions.getWorkflow({ - ...context.repo, workflow_id: 'castiron-custom-code.yml', - }); - if (!Number.isSafeInteger(run.workflow_id) || run.workflow_id !== workflow.id || - workflow.path !== '.github/workflows/castiron-custom-code.yml') return; - if (run.repository.full_name !== repo || run.head_repository?.full_name !== repo || - run.event !== 'push' || run.head_branch !== 'main' || run.status !== 'completed' || - run.id !== signal.id || run.run_attempt !== signal.run_attempt || run.head_sha !== signal.head_sha || - !/^[0-9a-f]{40}$/.test(run.head_sha) || - run.path.split('@', 1)[0] !== '.github/workflows/castiron-custom-code.yml') { - throw new Error('Unexpected or superseded main signal'); - } - const {data: main} = await github.rest.git.getRef({...context.repo, ref: 'heads/main'}); - // A newer main push has its own signal. An old retry must not churn - // candidates already qualified against that newer snapshot. - if (run.head_sha !== main.object.sha) return; - const targets = new Set(); - let cursor = null; - do { - const result = await github.graphql(`query($owner:String!,$name:String!,$branch:String!,$cursor:String) { - repository(owner:$owner,name:$name) { mergeQueue(branch:$branch) { - entries(first:100,after:$cursor) { - nodes { headCommit { oid } } - pageInfo { hasNextPage endCursor } - } - } } - }`, {owner: context.repo.owner, name: context.repo.repo, branch: 'next', cursor}); - const page = result.repository.mergeQueue?.entries; - if (!page) break; - for (const entry of page.nodes) { - if (entry.headCommit === null) continue; // Unprepared entries have no statuses to invalidate. - const sha = entry.headCommit?.oid; - if (!/^[0-9a-f]{40}$/.test(sha || '')) throw new Error('Invalid live queue SHA'); - targets.add(sha); - } - if (!page.pageInfo.hasNextPage) break; - const next = page.pageInfo.endCursor; - if (!next || next === cursor) throw new Error('Invalid queue pagination'); - cursor = next; - } while (true); - const url = `${context.serverUrl}/${repo}/actions/runs/${context.runId}`; - // Invalidate the entire live next queue before slower rerun discovery. - // This also covers bootstraps whose policy pin is now stale. Ordinary - // pinned-policy candidates can regain success through normal checks. - for (const sha of targets) { - for (const name of ['Castiron next / budget-only change', 'Castiron next / custom-code budget']) { - await github.rest.repos.createCommitStatus({...context.repo, sha, context: name, state: 'pending', - description: `Main advanced; reevaluation pending. [main ${main.object.sha}]`, target_url: url}); - } - } - for (const sha of targets) { - const runs = await github.paginate(github.rest.actions.listWorkflowRuns, { - ...context.repo, workflow_id: 'castiron-custom-code.yml', head_sha: sha, event: 'merge_group', per_page: 100, - }); - const matches = runs.filter(item => item.repository.full_name === repo && - item.workflow_id === workflow.id && item.event === 'merge_group' && item.head_sha === sha && - typeof item.head_branch === 'string' && item.head_branch.startsWith('gh-readonly-queue/next/') && - item.path.split('@', 1)[0] === '.github/workflows/castiron-custom-code.yml' && - Number.isSafeInteger(item.id) && item.id > 0); - matches.sort((a, b) => b.id - a.id); - // A running evaluation will check main at publication. Do not - // restart an older completed run when a newer one is in flight. - if (matches[0]?.status === 'completed') { - await github.rest.actions.reRunWorkflow({...context.repo, run_id: matches[0].id}); - } - } - - compute: - name: Compute trusted custom-code report - if: contains(fromJSON('["pull_request", "merge_group"]'), github.event.workflow_run.event) && (github.event.workflow_run.path == '.github/workflows/castiron-custom-code.yml' || startsWith(github.event.workflow_run.path, '.github/workflows/castiron-custom-code.yml@')) - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - actions: read - pull-requests: read - outputs: - selected-policy-sha: ${{ steps.select-policy.outputs.policy_sha }} - authenticated-main-sha: ${{ steps.select-policy.outputs.authenticated_main_sha }} - main-snapshot: ${{ steps.select-policy.outputs.main_snapshot }} - selected-base-sha: ${{ steps.select-policy.outputs.base_sha }} - selected-head-sha: ${{ steps.select-policy.outputs.head_sha }} - selected-branch: ${{ steps.select-policy.outputs.target_branch }} - selected-bootstrap: ${{ steps.select-policy.outputs.bootstrap }} - policy-reviews: ${{ steps.select-policy.outputs.policy_reviews }} - human-review-rules: ${{ steps.select-policy.outputs.human_review_rules }} - number: ${{ steps.report.outputs.number }} - artifact-id: ${{ steps.artifact.outputs.artifact-id }} - artifact-run-attempt: ${{ github.run_attempt }} - target-branch: ${{ steps.budget.outputs.target_branch }} - policy-sha: ${{ steps.budget.outputs.policy_sha }} - base-sha: ${{ steps.budget.outputs.base_sha }} - head-sha: ${{ steps.budget.outputs.head_sha }} - isolation: ${{ steps.budget.outputs.isolation }} - budget: ${{ steps.budget.outputs.budget }} - steps: - # Selecting main here pins executable checker code and budget policy. - # The reporter separately captures the live main/next target baseline. - - name: Check out the trusted reporter - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: main - persist-credentials: false - - # The selector and its module come from current main. The later evaluator - # loads its own policy helpers from the immutable pinned checkout. - - name: Select reviewed policy from the trusted target base - id: select-policy - env: - GH_TOKEN: ${{ github.token }} - REPOSITORY: ${{ github.repository }} - GIT_CONFIG_COUNT: '2' - GIT_CONFIG_KEY_0: credential.helper - GIT_CONFIG_VALUE_0: '' - GIT_CONFIG_KEY_1: credential.https://github.com.helper - GIT_CONFIG_VALUE_1: '!gh auth git-credential' - run: | - trusted_main="$(git rev-parse HEAD)" - printf 'authenticated_main_sha=%s\n' "$trusted_main" >> "$GITHUB_OUTPUT" - python3 -I scripts/castiron/custom_code_budget.py select-policy \ - --repository "$REPOSITORY" --event-path "$GITHUB_EVENT_PATH" \ - --trusted-sha "$trusted_main" \ - --repo "$RUNNER_TEMP/castiron-policy-objects.git" \ - --out "$RUNNER_TEMP/castiron-policy-selection.json" - - - name: Check out the authenticated main policy version - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ steps.select-policy.outputs.policy_sha }} - path: .castiron-policy-checker - persist-credentials: false - - - name: Compute from trusted main and target branch Git objects - id: report - if: github.event.workflow_run.event == 'pull_request' - env: - GH_TOKEN: ${{ github.token }} - GIT_CONFIG_COUNT: '2' - GIT_CONFIG_KEY_0: credential.helper - GIT_CONFIG_VALUE_0: '' - GIT_CONFIG_KEY_1: credential.https://github.com.helper - GIT_CONFIG_VALUE_1: '!gh auth git-credential' - REPOSITORY: ${{ github.repository }} - RUN_ID: ${{ github.event.workflow_run.id }} - RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} - run: | - python3 -I .castiron-policy-checker/scripts/castiron/custom_code_report.py trusted-report \ - --repo "$RUNNER_TEMP/castiron-objects.git" \ - --base "$(git -C .castiron-policy-checker rev-parse HEAD)" \ - --repository "$REPOSITORY" --run-id "$RUN_ID" --run-attempt "$RUN_ATTEMPT" \ - --out "$RUNNER_TEMP/castiron-custom-code" - if test -f "$RUNNER_TEMP/castiron-custom-code/context.json"; then - number=$(jq -er '.pr' "$RUNNER_TEMP/castiron-custom-code/context.json") - printf 'number=%s\n' "$number" >> "$GITHUB_OUTPUT" - cat "$RUNNER_TEMP/castiron-custom-code/summary.md" >> "$GITHUB_STEP_SUMMARY" - fi - - - name: Upload the trusted report and patch - id: artifact - if: steps.report.outputs.number != '' - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: castiron-custom-code-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/castiron-custom-code/ - if-no-files-found: error - retention-days: 7 - - # The prior step's report was computed here from - # Git objects by main's reporter, not downloaded from the candidate run. - - name: Evaluate main's custom-code budget - id: budget - if: always() - continue-on-error: true # A budget failure must not suppress the existing report comment. - env: - GH_TOKEN: ${{ github.token }} - REPOSITORY: ${{ github.repository }} - SOURCE_EVENT: ${{ github.event.workflow_run.event }} - GIT_CONFIG_COUNT: '2' - GIT_CONFIG_KEY_0: credential.helper - GIT_CONFIG_VALUE_0: '' - GIT_CONFIG_KEY_1: credential.https://github.com.helper - GIT_CONFIG_VALUE_1: '!gh auth git-credential' - run: | - trusted_sha=$(git -C .castiron-policy-checker rev-parse HEAD) - reuse=() - if [[ "$SOURCE_EVENT" == pull_request ]]; then - reuse=(--trusted-report-dir "$RUNNER_TEMP/castiron-custom-code") - fi - python3 -I .castiron-policy-checker/scripts/castiron/custom_code_budget.py github \ - --repository "$REPOSITORY" --event-path "$GITHUB_EVENT_PATH" \ - --trusted-sha "$trusted_sha" --repo "$RUNNER_TEMP/castiron-objects.git" \ - --selected-policy-json "$RUNNER_TEMP/castiron-policy-selection.json" \ - "${reuse[@]}" --out "$RUNNER_TEMP/custom-code-budget" - - - name: Add the budget to the run summary - if: always() - run: | - if test -f "$RUNNER_TEMP/custom-code-budget/summary.md"; then - cat "$RUNNER_TEMP/custom-code-budget/summary.md" >> "$GITHUB_STEP_SUMMARY" - fi - - - name: Upload trusted budget measurements - if: always() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: custom-code-budget-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/custom-code-budget/ - if-no-files-found: error - retention-days: 7 - - budget-status: - concurrency: - group: castiron-budget-writes-${{ github.repository_id }} - cancel-in-progress: false - queue: max - name: Publish custom-code budget checks - needs: compute - if: always() && !cancelled() && needs.compute.result != 'skipped' - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - actions: read - pull-requests: read - statuses: write - steps: - # This workflow definition is from main. No candidate checkout/artifacts. - - name: Publish exact-head statuses after checking freshness - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - SELECTED_POLICY_SHA: ${{ needs.compute.outputs.selected-policy-sha }} - AUTHENTICATED_MAIN_SHA: ${{ needs.compute.outputs.authenticated-main-sha }} - MAIN_SNAPSHOT: ${{ needs.compute.outputs.main-snapshot }} - SELECTED_BASE_SHA: ${{ needs.compute.outputs.selected-base-sha }} - SELECTED_HEAD_SHA: ${{ needs.compute.outputs.selected-head-sha }} - SELECTED_BRANCH: ${{ needs.compute.outputs.selected-branch }} - SELECTED_BOOTSTRAP: ${{ needs.compute.outputs.selected-bootstrap }} - POLICY_REVIEWS: ${{ needs.compute.outputs.policy-reviews }} - HUMAN_REVIEW_RULES: ${{ needs.compute.outputs.human-review-rules }} - TARGET_BRANCH: ${{ needs.compute.outputs.target-branch }} - POLICY_SHA: ${{ needs.compute.outputs.policy-sha }} - BASE_SHA: ${{ needs.compute.outputs.base-sha }} - HEAD_SHA: ${{ needs.compute.outputs.head-sha }} - ISOLATION_RESULT: ${{ needs.compute.outputs.isolation }} - BUDGET_RESULT: ${{ needs.compute.outputs.budget }} - PUBLISH_ATTEMPT: ${{ github.run_attempt }} - with: - script: | - const event = context.payload.workflow_run; - const {data: run} = await github.rest.actions.getWorkflowRun({...context.repo, run_id: event.id}); - const {data: workflow} = await github.rest.actions.getWorkflow({ - ...context.repo, workflow_id: 'castiron-custom-code.yml', - }); - if (!Number.isSafeInteger(run.workflow_id) || run.workflow_id !== workflow.id || - workflow.path !== '.github/workflows/castiron-custom-code.yml') return; - if (run.event !== event.event || run.head_sha !== event.head_sha || run.run_attempt !== event.run_attempt || - run.status !== 'completed' || run.path.split('@', 1)[0] !== '.github/workflows/castiron-custom-code.yml' || - run.repository.full_name !== `${context.repo.owner}/${context.repo.repo}`) return; - const matchesSource = pr => Boolean(run.head_repository?.id && run.head_repository?.full_name && run.head_branch && - pr.head?.sha === run.head_sha && pr.head?.ref === run.head_branch && - pr.head?.repo?.id === run.head_repository.id && pr.head?.repo?.full_name === run.head_repository.full_name); - const head = run.head_sha; - if (!/^[0-9a-f]{40}$/.test(head)) throw new Error('Invalid candidate SHA'); - const base = process.env.BASE_SHA; - const policy = process.env.POLICY_SHA; - const evaluatedBranch = process.env.TARGET_BRANCH; - let branch, number; - let fresh = /^[0-9a-f]{40}$/.test(base) && /^[0-9a-f]{40}$/.test(policy) && head === process.env.HEAD_SHA; - if (run.event === 'pull_request') { - if (!run.head_repository?.id || !run.head_repository.full_name || !run.head_branch) return; - const explicitAssociation = Boolean(run.pull_requests?.length); - let pulls = run.pull_requests.length ? run.pull_requests : await github.paginate( - github.rest.repos.listPullRequestsAssociatedWithCommit, {...context.repo, commit_sha: head}); - const current = []; - for (let attempt = 0; attempt < 2; attempt++) { - for (const pull of pulls) { - const {data: pr} = await github.rest.pulls.get({...context.repo, pull_number: pull.number}); - if (pr.state === 'open' && matchesSource(pr) && - ['main', 'next'].includes(pr.base.ref) && pr.base.repo.full_name === `${context.repo.owner}/${context.repo.repo}`) current.push(pr); - } - if (current.length || explicitAssociation || attempt) break; - pulls = await github.paginate(github.rest.pulls.list, { - ...context.repo, state: 'open', head: `${run.head_repository.owner.login}:${run.head_branch}`, - }); - } - // Match compute's preference for required main checks when - // fallback discovery finds an advisory next PR for the same head. - const main = current.filter(pr => pr.base.ref === 'main'); - const selected = main.length ? main : current; - if (selected.length !== 1) return; - branch = selected[0].base.ref; - number = selected[0].number; - } else { - const match = run.head_branch.match(/^gh-readonly-queue\/(main|next)\//); - if (run.event !== 'merge_group' || !match) return; - branch = match[1]; - } - // Bind every target to the trusted selector, including next-to-main retargeting. - fresh = fresh && process.env.SELECTED_BRANCH === branch && - process.env.SELECTED_BASE_SHA === base && process.env.SELECTED_HEAD_SHA === head && - process.env.SELECTED_POLICY_SHA === policy; - fresh = fresh && evaluatedBranch === branch; - const bootstrap = process.env.SELECTED_BOOTSTRAP; - fresh = fresh && (bootstrap === 'false' || (branch === 'next' && bootstrap === 'true')); - const url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - // Concurrency serializes writes, but does not order runs or retries. - // Keep an older evaluation from overwriting a newer result for this head. - const order = [run.id, run.run_attempt, Number(context.runId), Number(process.env.PUBLISH_ATTEMPT)]; - const marker = `[evaluation ${order.join(':')}]`; - const statuses = await github.paginate(github.rest.repos.listCommitStatusesForRef, {...context.repo, ref: head}); - const prefix = branch === 'next' ? 'Castiron next' : 'Castiron'; - const names = [`${prefix} / budget-only change`, `${prefix} / custom-code budget`]; - for (const status of statuses) { - if (!names.includes(status.context) || status.creator?.login !== 'github-actions[bot]') continue; - const prior = status.description?.match(/\[evaluation (\d+):(\d+):(\d+):(\d+)\]$/); - if (!prior || status.target_url !== `${url.slice(0, url.lastIndexOf('/') + 1)}${prior[3]}`) continue; - const previous = prior.slice(1).map(Number); - const different = previous.findIndex((value, index) => value !== order[index]); - if (different !== -1 && previous[different] > order[different]) return; - } - // Selection is not a lasting approval. Recheck live reviews after - // compute and immediately before attaching required statuses. - if (branch === 'next') { - try { - const proofs = JSON.parse(process.env.POLICY_REVIEWS); - const rules = JSON.parse(process.env.HUMAN_REVIEW_RULES); - if (!Array.isArray(proofs) || rules.version !== 2 || !Array.isArray(rules.associations) || - typeof rules.negative_pattern !== 'string' || typeof rules.attestation_prefix !== 'string' || - !rules.attestation_prefix) throw new Error('Missing trusted review contract'); - const negative = new RegExp(rules.negative_pattern, 'i'); - const seen = new Set(); - for (const proof of proofs) { - if (!Number.isSafeInteger(proof.number) || proof.number <= 0 || !/^[0-9a-f]{40}$/.test(proof.head_sha) || - seen.has(proof.number) || (run.event === 'pull_request' && (proof.number !== number || proof.head_sha !== head))) { - throw new Error('Invalid policy review receipt'); - } - seen.add(proof.number); - const {data: pr} = await github.rest.pulls.get({...context.repo, pull_number: proof.number}); - if (pr.state !== 'open' || pr.head.sha !== proof.head_sha || pr.base.ref !== 'next' || - pr.base.repo.full_name !== `${context.repo.owner}/${context.repo.repo}`) throw new Error('Policy PR changed'); - let reviewCursor = null; - const latest = new Map(); - do { - const result = await github.graphql(`query($owner:String!,$name:String!,$number:Int!,$cursor:String) { - repository(owner:$owner,name:$name) { pullRequest(number:$number) { - reviews(first:100,after:$cursor) { nodes { fullDatabaseId state body - author { login __typename } authorAssociation commit { oid } } - pageInfo { hasNextPage endCursor } } - } } - }`, {...context.repo, name: context.repo.repo, number: proof.number, cursor: reviewCursor}); - const page = result.repository.pullRequest.reviews; - for (const review of page.nodes) { - const value = review.fullDatabaseId; - if (!(typeof value === 'string' || Number.isSafeInteger(value)) || - !/^[1-9][0-9]*$/.test(String(value))) throw new Error('Invalid review identity'); - const identity = String(value); - if (['PENDING', 'COMMENTED'].includes(review.state) || !review.author) continue; - const previous = latest.get(review.author.login); - if (!previous || BigInt(identity) > BigInt(previous.identity)) latest.set(review.author.login, {...review, identity}); - } - if (!page.pageInfo.hasNextPage) break; - if (!page.pageInfo.endCursor || page.pageInfo.endCursor === reviewCursor) throw new Error('Invalid review cursor'); - reviewCursor = page.pageInfo.endCursor; - } while (true); - const approved = [...latest.values()].some(review => review.state === 'APPROVED' && - review.commit?.oid === proof.head_sha && review.author.__typename === 'User' && - rules.associations.includes(review.authorAssociation) && !review.author.login.endsWith('[bot]') && - review.author.login !== pr.user.login && typeof review.body === 'string' && - review.body.trim() === rules.attestation_prefix + proof.head_sha && - !negative.test(review.body)); - if (!approved) throw new Error('Human policy approval is no longer valid'); - } - } catch { fresh = false; } - } - // Recheck after pagination, immediately before attaching statuses. - if (run.event === 'pull_request') { - const {data: pr} = await github.rest.pulls.get({...context.repo, pull_number: number}); - if (pr.state !== 'open' || !matchesSource(pr) || pr.base.ref !== branch || - pr.base.repo.full_name !== `${context.repo.owner}/${context.repo.repo}`) return; - } else { - // A removed/rebuilt queue entry must not receive a passing result. - let cursor = null, member = false; - do { - const response = await github.graphql(`query($owner:String!,$name:String!,$branch:String!,$cursor:String) { - repository(owner:$owner,name:$name) { mergeQueue(branch:$branch) { - entries(first:100,after:$cursor) { - nodes { headCommit { oid } } - pageInfo { hasNextPage endCursor } - } - } } - }`, {owner: context.repo.owner, name: context.repo.repo, branch, cursor}); - const entries = response.repository.mergeQueue?.entries; - if (!entries) break; - member = entries.nodes.some(entry => entry.headCommit?.oid === head); - if (member || !entries.pageInfo.hasNextPage) break; - const next = entries.pageInfo.endCursor; - if (!next || next === cursor) throw new Error('Invalid queue pagination'); - cursor = next; - } while (true); - const {data: target} = await github.rest.git.getRef({...context.repo, ref: `heads/${branch}`}); - fresh = fresh && base === target.object.sha && (branch === 'next' || policy === target.object.sha); - fresh = fresh && member; - } - if (branch === 'next' && run.event === 'pull_request') { - const {data: target} = await github.rest.git.getRef({...context.repo, ref: 'heads/next'}); - fresh = fresh && base === target.object.sha; - } - // Main pushes invalidate next evaluations, even when their reviewed - // checker pin stays unchanged. Do not restore success from a run - // computed before that invalidation. This is the reporter checkout, - // not the independently reviewed (and possibly older) policy pin. - if (branch === 'next') { - const authenticatedMain = process.env.AUTHENTICATED_MAIN_SHA || ''; - const {data: currentMain} = await github.rest.git.getRef({...context.repo, ref: 'heads/main'}); - fresh = fresh && /^[0-9a-f]{40}$/.test(authenticatedMain) && - currentMain.object.sha === authenticatedMain; - } - // Bootstrap may adopt only the still-current authenticated main tree. - // This value comes from our selector, never from candidate artifacts. - if (process.env.MAIN_SNAPSHOT) { - const {data: currentMain} = await github.rest.git.getRef({...context.repo, ref: 'heads/main'}); - fresh = fresh && branch === 'next' && process.env.MAIN_SNAPSHOT === policy && - currentMain.object.sha === policy; - } - // Bootstrap authorization was evaluated against authenticated main, - // carried in SELECTED_POLICY_SHA. A newer main requires reevaluation. - if (branch === 'next' && bootstrap === 'true') { - const {data: main} = await github.rest.git.getRef({...context.repo, ref: 'heads/main'}); - fresh = fresh && policy === main.object.sha; - } - for (const [name, result] of [ - [names[0], process.env.ISOLATION_RESULT], - [names[1], process.env.BUDGET_RESULT], - ]) { - const state = fresh && result === 'success' ? 'success' : 'failure'; - const description = !fresh ? 'Evaluation unavailable or queue base changed; inspect the trusted run and rerun.' - : `${state === 'success' ? 'Passed' : 'Failed'}: ${branch} ${base.slice(0, 12)}, policy ${policy.slice(0, 12)}.`; - await github.rest.repos.createCommitStatus({...context.repo, sha: head, context: name, - state, description: `${description} ${marker}`, target_url: url}); - } - - comment: - name: Update custom-code comment - needs: compute - if: always() && !cancelled() && github.event.workflow_run.event == 'pull_request' && (needs.compute.result == 'failure' || needs.compute.outputs.number != '') - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - actions: read - pull-requests: write - steps: - - name: Check out the trusted publisher - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.workflow_sha }} - persist-credentials: false - - - name: Download this workflow's trusted report - if: needs.compute.result == 'success' - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - artifact-ids: ${{ needs.compute.outputs.artifact-id }} - merge-multiple: true - path: ${{ runner.temp }}/castiron-custom-code - - - name: Create or update the single report comment - id: publish - if: needs.compute.result == 'success' - env: - GH_TOKEN: ${{ github.token }} - REPOSITORY: ${{ github.repository }} - PR_NUMBER: ${{ needs.compute.outputs.number }} - RUN_ID: ${{ github.event.workflow_run.id }} - RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} - ARTIFACT_RUN_ID: ${{ github.run_id }} - ARTIFACT_RUN_ATTEMPT: ${{ needs.compute.outputs.artifact-run-attempt }} - run: | - python3 -I scripts/castiron/custom_code_report.py comment \ - --report "$RUNNER_TEMP/castiron-custom-code/report.json" \ - --repository "$REPOSITORY" --pr "$PR_NUMBER" --run-id "$RUN_ID" \ - --run-attempt "$RUN_ATTEMPT" \ - --artifact-run-id "$ARTIFACT_RUN_ID" --artifact-run-attempt "$ARTIFACT_RUN_ATTEMPT" - - - name: Publish a trusted failure status - if: always() && !cancelled() && steps.publish.outcome != 'success' - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - const marker = ''; - const event = context.payload.workflow_run; - const {data: run} = await github.rest.actions.getWorkflowRun({...context.repo, run_id: event.id}); - const {data: workflow} = await github.rest.actions.getWorkflow({ - ...context.repo, workflow_id: 'castiron-custom-code.yml', - }); - if (!Number.isSafeInteger(run.workflow_id) || run.workflow_id !== workflow.id || - workflow.path !== '.github/workflows/castiron-custom-code.yml') return; - if (run.id !== event.id || run.event !== 'pull_request' || run.event !== event.event || - run.head_sha !== event.head_sha || run.run_attempt !== event.run_attempt || run.status !== 'completed' || - run.path.split('@', 1)[0] !== '.github/workflows/castiron-custom-code.yml' || - run.repository.full_name !== `${context.repo.owner}/${context.repo.repo}`) return; - const matchesSource = pr => Boolean(run.head_repository?.id && run.head_repository?.full_name && run.head_branch && - pr.head?.sha === run.head_sha && pr.head?.ref === run.head_branch && - pr.head?.repo?.id === run.head_repository.id && pr.head?.repo?.full_name === run.head_repository.full_name); - if (!run.head_repository?.id || !run.head_repository.full_name || !run.head_branch) return; - const explicitAssociation = Boolean(run.pull_requests?.length); - let pulls = run.pull_requests?.length ? run.pull_requests : await github.paginate(github.rest.repos.listPullRequestsAssociatedWithCommit, {...context.repo, commit_sha: run.head_sha}); - const current = []; - for (let attempt = 0; attempt < 2; attempt++) { - for (const number of new Set(pulls.map(pull => pull.number))) { - const {data: pr} = await github.rest.pulls.get({...context.repo, pull_number: number}); - if (pr.state === 'open' && matchesSource(pr) && ['main', 'next'].includes(pr.base.ref) && - pr.base.repo.full_name === `${context.repo.owner}/${context.repo.repo}`) current.push({number, branch: pr.base.ref}); - } - if (current.length || explicitAssociation || attempt) break; - pulls = await github.paginate(github.rest.pulls.list, { - ...context.repo, state: 'open', head: `${run.head_repository.owner.login}:${run.head_branch}`, - }); - } - const main = current.filter(pr => pr.branch === 'main'); - const selected = main.length ? main : current; - if (selected.length !== 1) return; - const {number, branch} = selected[0]; - const comments = await github.paginate(github.rest.issues.listComments, {...context.repo, issue_number: number}); - const previous = comments.find(c => c.user?.type === 'Bot' && c.user?.login === 'github-actions[bot]' && c.body?.startsWith(marker)); - const prior = previous?.body?.match(//); - if (prior && (Number(prior[1]) > run.id || (Number(prior[1]) === run.id && Number(prior[2]) > run.run_attempt))) return; - const {data: pr} = await github.rest.pulls.get({...context.repo, pull_number: number}); - if (pr.state !== 'open' || !matchesSource(pr) || pr.base.ref !== branch || - pr.base.repo.full_name !== `${context.repo.owner}/${context.repo.repo}`) return; - const url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${run.id}`; - const body = `${marker}\n\n## Castiron custom code\n\n⚠️ Report unavailable for \`${run.head_sha.slice(0, 12)}\`.\n\nThe report setup or validation failed. [Inspect the workflow run](${url}).\n\n`; - if (previous) await github.rest.issues.updateComment({...context.repo, comment_id: previous.id, body}); - else await github.rest.issues.createComment({...context.repo, issue_number: number, body}); diff --git a/.github/workflows/castiron-custom-code.yml b/.github/workflows/castiron-custom-code.yml deleted file mode 100644 index afe153609..000000000 --- a/.github/workflows/castiron-custom-code.yml +++ /dev/null @@ -1,116 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -name: Castiron custom code - -on: - push: - branches: [main] - pull_request: - types: [opened, synchronize, reopened, ready_for_review, edited] - pull_request_review: - types: [submitted, edited, dismissed] - # Notify the existing trusted handler for queue candidates. - merge_group: - types: [checks_requested] - -permissions: - contents: read - -concurrency: - group: castiron-custom-code-${{ github.event_name }}-${{ github.event.pull_request.number || github.event.merge_group.head_ref }} - cancel-in-progress: false - -env: - REPORTER_SHA256: ddb15c9a162d26a6c9abf1b4fefcd80f2e685d9d255af729863258b7ecd08e2d - -jobs: - queue-signal: - if: github.event_name == 'merge_group' || github.event_name == 'pull_request_review' || github.event_name == 'push' - runs-on: ubuntu-latest - timeout-minutes: 1 - permissions: {} - steps: - - run: echo 'The workflow_run handler on main evaluates this candidate independently.' - - # Candidate tests run on a separate runner and never precede a token-bearing step. - candidate-tests: - if: github.event_name == 'pull_request' - name: Castiron / reporter tests - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - steps: - - name: Check out the pull request - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.event.pull_request.head.sha }} - persist-credentials: false - - - name: Test snapshot isolation and the custom-code budget - run: python3 -m unittest discover -s scripts/castiron -p 'test_custom_code*.py' - - report: - if: github.event_name == 'pull_request' - name: Castiron / baseline consistency - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Record the pull request - env: - PR_NUMBER: ${{ github.event.pull_request.number }} - REPOSITORY: ${{ github.repository }} - RUN_ID: ${{ github.run_id }} - RUN_ATTEMPT: ${{ github.run_attempt }} - run: | - mkdir -p "$RUNNER_TEMP/castiron-custom-code" - jq -n --argjson pr "$PR_NUMBER" --arg repository "$REPOSITORY" \ - --argjson run "$RUN_ID" --argjson attempt "$RUN_ATTEMPT" \ - '{pr: $pr, repository: $repository, run: $run, attempt: $attempt}' \ - > "$RUNNER_TEMP/castiron-custom-code/context.json" - - - name: Check out the pull request - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - persist-credentials: false - - - name: Verify the reporter matches its generated workflow - run: printf '%s %s\n' "$REPORTER_SHA256" scripts/castiron/custom_code_report.py | sha256sum --check --strict - - - name: Validate the codegen hash and report custom code - env: - GH_TOKEN: ${{ github.token }} - GIT_CONFIG_COUNT: '2' - GIT_CONFIG_KEY_0: credential.helper - GIT_CONFIG_VALUE_0: '' - GIT_CONFIG_KEY_1: credential.https://github.com.helper - GIT_CONFIG_VALUE_1: '!gh auth git-credential' - BASE_SHA: ${{ github.event.pull_request.base.sha }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - TARGET_BRANCH: ${{ github.event.pull_request.base.ref }} - PUBLIC_REPOSITORY: ${{ !github.event.repository.private }} - run: | - git fetch --quiet --no-tags origin "$BASE_SHA" "$HEAD_SHA" - mode=() - if [[ "$PUBLIC_REPOSITORY" == true ]]; then mode=(--public); fi - python3 -I scripts/castiron/custom_code_report.py report \ - --base "$BASE_SHA" --head "$HEAD_SHA" --target-branch "$TARGET_BRANCH" \ - --fetch --require-head-hash "${mode[@]}" \ - --out "$RUNNER_TEMP/castiron-custom-code" - - - name: Add the report to the run summary - if: always() - run: | - if test -f "$RUNNER_TEMP/castiron-custom-code/summary.md"; then - cat "$RUNNER_TEMP/castiron-custom-code/summary.md" >> "$GITHUB_STEP_SUMMARY" - fi - - - name: Upload report and current custom-code patch - if: always() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: castiron-custom-code-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/castiron-custom-code/ - if-no-files-found: error - retention-days: 7 diff --git a/scripts/castiron/CUSTOM_CODE.md b/scripts/castiron/CUSTOM_CODE.md deleted file mode 100644 index 87cee14ef..000000000 --- a/scripts/castiron/CUSTOM_CODE.md +++ /dev/null @@ -1,298 +0,0 @@ - - -# Custom code - -The custom-code reporter measures the SDK's remaining customization of generated -files. The budget gate checks that measurement against the repository's policy. -See [CONTRIBUTING.md](../../CONTRIBUTING.md#custom-code-budget) for budget changes -and human review, and [AGENTS.md](../../AGENTS.md#custom-code-budget) for agent rules. - -## What is counted - -Repositories may separately activate `"ratchet": "go"` in their policy after -this checker reaches trusted main. Activation must be a human-reviewed, -policy-only PR; SDK promotion does not carry policy edits. Keep the existing -`max_custom_patch_lines` ceiling. - -The Go ratchet rejects net growth in generated-owned custom patch additions -plus deletions **and physical lines of handwritten Go** compared with the -captured live target. It includes helpers, tests, comments, blank lines, nested -modules, and SDK-owned CI Go retained in private checkpoints. Non-text and -nonregular handwritten Go fail validation. Both revisions use their own verified -generated snapshots, read as Git objects without executing candidate code. - -Stale PRs compare with the live target, not their historical merge base. Merge -queues check the complete combined candidate. Every landed reduction becomes -the next baseline automatically. An activated ratchet cannot be disabled; -raising the independent patch-only ceiling does not restore retired headroom. -Existing policy isolation, human review, and snapshot ownership rules still apply. - -The budget is **added lines plus deleted lines in the entire current custom patch** -against the candidate's verified pure-generated snapshot. It is not the PR's diff -against main, and additions and deletions never cancel. Replacing 500 generated -lines with 500 handwritten lines costs 1,000 lines. Deleting a generated file -counts all its removed lines. Restoring generated content reduces the budget. - -The checker reuses `scripts/castiron/custom_code_report.py` without changing its -snapshot/hash validation or generated-file ownership. It counts generated-owned -files, including runtime code, tests, and generated documentation. Wholly -handwritten-only files are outside this existing report's scope. Changes in -generation ownership remain visible in the report; do not change exclusions or -checkpoints to hide customization. Non-text custom patches fail as uncountable. - -The ceiling is defined in [`.castiron-ratchet.json`](../../.castiron-ratchet.json). -Migration policy contains an integer `schema_version` of 1 and a -nonnegative integer `max_custom_patch_lines`. Strict schema v2 is described below. -Missing, deleted, renamed, executable, symlinked, -malformed, or unsupported policy files fail closed. There is no disable flag or -automatic budget update command. - -Budget-file isolation is checked against the complete PR diff, not just its latest -commit. Increases never apply to their own PR or merge group. Decreases must also -fit the measured usage. Human approval is contributor and agent policy plus normal -review, not an automated claim that GitHub can identify who operated an account. - -## Trusted CI - -### Strict generated-file ownership - -Schema v1 remains migration mode with its existing patch budget. An SDK with -zero customizations can opt into schema v2 through a policy-only PR: - -```json -{ "schema_version": 2, "generated_files": "strict", "max_custom_patch_lines": 0 } -``` - -First land shared compiler/verifier support and regenerate the SDK, including -`scripts/castiron/generated-files.json`. Then activate strict policy. Both the -main policy and a proposed activation are enforced; strict policy cannot be -downgraded. A policy change cannot be combined with SDK or generated-source edits. -SDK promotion rejects policy-file changes; submit the policy-only PR directly -in the SDK repository. - -Strict mode compares generated paths, final file bytes, and Git modes against -the manifest in the verified generated snapshot. The candidate's manifest is -also checked exactly. Header removal, deletion, renaming, symlink substitution, -or manifest edits cannot exempt a generated file. Generated workflow/action -hashes are covered by the manifest even though the legacy patch counter and -public snapshot publication continue to exclude those subtrees. Handwritten -files and repository-owned governance remain outside generated ownership. -Private checkpoints retain SDK-owned CI workflows and actions from their parent; -the generated ownership inventory does not claim those inherited files. - -The existing required `Castiron / custom-code budget` status fails on any strict -violation. Its trusted-main execution and merge-queue checks remain mandatory. -Local generation validates before integration and publication using compiler-owned -code. New generated snapshots still require the existing generation/promotion -provenance and review controls; a self-computed hash is not approval to publish. - -Agents must fix inputs/compiler code or handwritten extensions, regenerate, and -verify. Never patch generated files or the manifest, disable strict mode, change -ownership exclusions, or rewrite provenance to get a passing result. Verification -failure must preserve edits for inspection rather than overwrite them. - -The custom-code workflow pair separates candidate execution from trusted checks: - -- `castiron-custom-code.yml` runs proposed offline tests and the advisory report - on `pull_request` with read-only permissions. -- `castiron-custom-code-comment.yml` handles `workflow_run` from **main**. Its - read-only selector authenticates the target base and selects a main commit. - Main uses its captured current commit. Next uses the reviewed `next_policy` - version stored in its protected target base. A second checkout executes only - that authenticated main version of the reporter/checker and reads its policy. - It independently resolves the live PR target (`main` or `next`) and captures - the target baseline and candidate SHA. For main, the target baseline is the - pinned main checkout; for next, it is the live next ref, not stale PR metadata. - It reads only Git objects in a fresh bare repository, never candidate code. -- An unprivileged `merge_group` job in the first workflow signals that a candidate - needs checking. The trusted handler resolves the target from the queue ref, - requires an effective merge-queue rule on that target, and verifies current - queue membership and target ancestry. It checks isolation against the target - per constituent PR and budgets the complete candidate using trusted main policy. - A policy increase in next or the same group cannot grant a higher limit. - -A separate publisher with no checkout attaches these statuses only to PRs and -merge groups targeting `main`, after rechecking the PR head/target or queue base freshness: - -- `Castiron / budget-only change` -- `Castiron / custom-code budget` - -Next uses separate `Castiron next / budget-only change` and -`Castiron next / custom-code budget` contexts, so identical commit SHAs on the -two tracks cannot overwrite each other's authorization. The policy version is -fixed by next's protected base; advancing main does not silently change it. -Changing the reviewed pin changes next's own base and rebuilds its merge queue. -Review invalidation reads the effective checker's inert `PIN_REVIEW_POLICY` -capability marker at that exact revision. Legacy or unknown versions retain -human review for every pin edit; only the recognized effective-policy contract -allows non-increasing forward pin changes to use ordinary review. No checker -code is executed by the invalidation handler. - -The effective policy is read from the selected authenticated main commit. The -next base selects that version; a candidate never selects its own executing -checker or grants itself a larger budget. Target baseline, trusted policy, and candidate identities are recorded -separately in the report and budget result. Reusing a report requires matching -branch, policy, and head identities; retargeted PRs fail closed. - -PR results describe a captured snapshot even if a branch moves during computation. -They do not authorize merging without queue validation. Queue checks require -current target baseline and its approved policy version (current main for main, -reviewed pin for next). Immediately before publishing, the handler rechecks the -PR head/target or queue membership and the bound target baseline/policy identities. A stale queue evaluation cannot publish success. Missing -snapshots, invalid hashes, unavailable membership, and policy errors fail closed. - -The queue signal cannot supply a passing result, and removing it leaves required -statuses missing. No write-capable job executes from the candidate's workflow -definition. See GitHub's [workflow_run trust model](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#workflow_run) -and [merge-queue behavior](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue). - -The trusted run summary reports additions, deletions, total, mixed-file count, -headroom, largest patches, and exact policy/candidate/generated revisions. The -existing custom-code comment remains unchanged, including when the budget fails. -The trusted compute job reuses its own report, never the candidate's artifacts. -The checker, workflows, and offline tests are generated from shared Castiron -templates. The budget policy remains repository-owned and is never generated. -Repository-specific customizations are preserved through the normal three-way -merge during generation. - -## Local verification - -Run from the SDK repository with Python 3.10+; no SDK dependencies are needed: - -```sh -python3 -m unittest discover -s scripts/castiron -p 'test_custom_code*.py' -``` - -Use a **trusted checkout's** script to inspect committed revisions: - -```sh -python3 -I scripts/castiron/custom_code_budget.py check \ - --repo /path/to/sdk --base FULL_MAIN_SHA --head FULL_CANDIDATE_SHA \ - --public --fetch --out /tmp/custom-code-budget -``` - -The inspected repo's `origin` must match its snapshot source. Omit `--public` -for the private SDK repository's checkpoints. Local checks are diagnostic and -use the supplied base as policy; CI independently binds the target, main policy, -and candidate to live GitHub metadata. - -## Activation order - -1. Merge the initial budget policy in its budget-only PR after human review. -2. Merge the tooling and guidance separately. Do not enable required - statuses before the policy and trusted checker exist on main. There is no - permanent bootstrap exemption in the checker. -3. Exercise a passing PR, over-budget PR, mixed budget/code PR, a fork PR, and a - merge-queue candidate. Policy increases must still use the old limit. -4. Have a human administrator retain/enable **require merge queue** and add both - exact status names above to the main ruleset, bound to GitHub Actions. Retain - normal review and CODEOWNERS settings. Exercise a main-policy decrease after - an unchanged PR head passed, and confirm its queued candidate uses the new limit. - The tests job is separate from these authoritative statuses. - -This is an accidental-change guardrail, not an adversarial approval boundary. -Workflow definitions and ruleset changes still have the normal repository review -and administrator controls; GitHub Actions status names alone are not a unique -workflow identity. Stronger organization-required workflow enforcement can be -added later. - -## Activating next - -Land shared tooling in Castiron first, then generate and review its SDK update -onto main. Add a reviewed policy pin in a separate `.castiron-ratchet.json`-only -PR targeting next; preserve the existing budget/schema fields: - -```json -"next_policy": { - "repository": "openai/your-sdk", - "main_sha": "" -} -``` - -The SHA must belong to that same repository's authenticated main first-parent -history and support this pin protocol. Selection reads the protected target -base, never the candidate. Pin-only updates and initial pin adoption use ordinary -CODEOWNERS and required reviews when the pin does not roll back to an older main -checker, the effective pinned budget stays the same or decreases, and its schema, -strictness and ratchet mode stay unchanged. Initial adoption compares checker age -and effective policy with authenticated bootstrap main, not the unpinned target's -local policy fields; later updates compare with the current protected -target pin and effective policy, even for stale PRs. A -checker rollback, an effective pinned-budget increase or mode transition, a local numeric budget increase, or Go -ratchet activation still requires an exact-head human approving review from a -repository owner, member, or collaborator. Bot reviews, stale reviews and dismissed -approvals do not authorize those changes. - -For those human-required changes, the approving reviewer must explicitly attest that -they personally performed the human review. The entire review body must be the -following sentence, substituting the exact full current head SHA: - -```text -I personally reviewed and approve this policy change as a human: <40-character head SHA> -``` - -Blank reviews, ordinary code-review text, quoted attestations, and attestations for -an earlier head do not qualify. Agents must never submit this attestation, even -through a human account. This is an explicit reviewer attestation, not a claim that -GitHub account type or team membership proves who operated the account. Ordinary -SDK code reviews continue to accept automation under their normal repository gates. - -For initial adoption only, a base with no pin may accept an exact policy-only -pin-initialization PR under authenticated current-main checker/policy, with the -same effective-budget and mode review requirements. The proposed pin is validated but never executed -for its own initialization. A malformed base pin cannot enter this path; other -changes without a base pin fail closed. Ordinary pin updates execute under the -previous approved version and cannot include SDK/tooling changes. - -An existing unpinned `next` may instead adopt the complete authenticated current-main -snapshot when next is an ancestor of main and main already uses strict ownership -with zero custom lines. The canonical `sdk prepare-next-baseline` command preserves -all main bytes, modes and existing public checkpoints, adding only the exact -same-main policy pin. This exception cannot include additional edits, a different -pin, a relaxed policy or multiple queue constituents. Ordinary checkpoint, manifest, -zero-budget and repository review checks remain mandatory. Main must still match -at final publication; if it moves, prepare and qualify a fresh candidate. This is -adoption of already-public output, not regeneration or a budget increase. - -Keep required status activation separate until hosted tests have exercised main -and next PRs, forks, retargeting, policy changes, and both merge queues. Require -the target-specific next contexts after observing their genuine producers; -never reuse main's context names on next. Establish missing ordinary CI triggers -through a reviewed trusted workflow path before the policy-only initialization. -Preserve reviews, CODEOWNERS, merge queues, and all existing main protections. -This policy contract does not enable package releases from next. - -For `next` policy activations and budget increases, the trusted computation also -records the exact constituent PR numbers and head SHAs requiring human approval. -The isolated status publisher rechecks those live PRs and their latest reviews -immediately before publication. Dismissed, replaced, stale, automated or missing -approvals fail closed even if ordinary automated approvals keep the queue live. -The publisher consumes the selector's versioned review rules, with cross-runtime -predicate fixtures; candidate artifacts never supply authorization or outcomes. - -### Review changes after publication - -The unprivileged reporter workflow signals submitted, edited and dismissed reviews. -For a current next PR changing the policy file, the trusted main handler marks its -required budget statuses and affected live queue statuses pending, then requests -normal reporter reruns. Only the ordinary authenticated evaluator may restore -success; the review signal never selects executable code, approval rules or an -arbitrary status target. Fork runs with empty PR lists use authenticated commit -associations and open-PR lookup, retaining live head/merge binding. The handler -marks each authenticated snapshot pending before slow run pagination, then retries -discovery if the PR or queue changes. Pending statuses on superseded affected -SHAs cannot authorize a merge; observed successes stay revoked even if a later -lookup fails. Reporter reruns must resolve to this same PR under the canonical -main-preference rule, not merely share its head SHA. Missing or ambiguous reporter -runs remain pending. Status writers share a serialization group. - -Retain native required approving reviews and CODEOWNERS at merge time. Review-event -delivery and reevaluation are asynchronous; this mechanism is not an atomic human -approval check at merge. In particular, automated reviews using human accounts do -not become human policy authorization. Do not weaken native gates or treat this -handler as their replacement. A failed invalidation or rerun must be investigated; -missing reporter runs remain pending until the normal trusted run is requested. -Install and validate the review signal on the protected target before activating -next policy checks, including actual PR and queue revocation after dismissal. - -The trusted selector also checks this exact active-checker contract before granting a pin-only review exemption. Legacy or unrecognized contracts retain the required exact-head review in the selection receipt, so the final publisher revalidates it even without a review-change event. diff --git a/scripts/castiron/README.md b/scripts/castiron/README.md deleted file mode 100644 index c445a641b..000000000 --- a/scripts/castiron/README.md +++ /dev/null @@ -1,29 +0,0 @@ - - -# Castiron custom-code reporting - -Castiron maintains shared templates for these files. Prefer changing those templates -for cross-SDK improvements; repository-specific customizations use the normal -three-way merge and are allowed. -The reporter uses Python 3.10+, Git, and `gh`; it does not import SDK code. - -Run `python3 -m unittest discover -s scripts/castiron -p 'test_custom_code*.py'` -for the offline suite. Install Node.js to exercise the workflow publishers too. -See [CUSTOM_CODE.md](CUSTOM_CODE.md) for budget policy and activation. -The report comment includes commands to inspect the exact custom-code patch. -Public reporting uses only public snapshots and needs no private repository access. - -The workflow validates the recorded `codegen_hash`. -Its hash format is documented in the reporter. Only `.github/actions/` and -`.github/workflows/` are excluded from the content hash. - -The read-only pull-request workflow runs on every branch, including drafts and -forks. A separate read-only `workflow_run` job computes the authoritative report from -the captured main checkout and GitHub-associated PR head. Merge groups are -checked independently against current main. It fetches those objects into a new bare repository and never checks -out or executes PR code. The comment-writing job consumes only the artifact -from that trusted job, rechecks freshness, and links to its report and patch. -PR-produced reports are advisory run output, not the published assessment. The -publisher becomes active once its workflow is on the default branch. No branch -allowlist or repository variable is needed. Never execute PR-controlled code with -write credentials. Changing either workflow may require one-time AM permission. diff --git a/scripts/castiron/custom_code_budget.py b/scripts/castiron/custom_code_budget.py deleted file mode 100755 index 88b808532..000000000 --- a/scripts/castiron/custom_code_budget.py +++ /dev/null @@ -1,978 +0,0 @@ -#!/usr/bin/env python3 -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -"""SDK custom-code budget gate. Run only from a trusted checkout, never PR code. - -Reuses Castiron's vendored snapshot verifier and generated-file accounting. This -file and its workflows are generated from shared Castiron templates. -""" - -from __future__ import annotations - -import argparse -import ast -import hashlib -import html -import importlib.util -import json -import os -import subprocess -import sys -from pathlib import Path -from typing import Any, cast - -# Load only the policy helpers beside this immutable checker, including under -I. -_spec = importlib.util.spec_from_file_location( - "castiron_budget_policy", Path(__file__).with_name("custom_code_policy.py") -) -assert _spec is not None and _spec.loader is not None -policy = importlib.util.module_from_spec(_spec) -sys.modules[_spec.name] = policy -_spec.loader.exec_module(policy) -report = policy.report -unique_keys = policy.unique_keys -read_policy = policy.read_policy -changed_paths = policy.changed_paths -queued_entries = policy.queued_entries -is_main_snapshot_bootstrap = policy.is_main_snapshot_bootstrap - -POLICY = ".castiron-ratchet.json" -NEXT_POLICY_VERSION = 1 -ISOLATION_MESSAGE = ( - "Budget changes require a separate, budget-only PR. " - "Put the justification in the PR description." -) - - -def read_budget(repo: Path, revision: str) -> int: - return cast(int, read_policy(repo, revision)["max_custom_patch_lines"]) - - -def strict_policy(repo: Path, revision: str) -> bool: - return cast(int, read_policy(repo, revision)["schema_version"]) == 2 - - -def go_ratchet(repo: Path, revision: str) -> bool: - return cast(bool, read_policy(repo, revision).get("ratchet") == "go") - - -MANIFEST = b"scripts/castiron/generated-files.json" -# Accept the relocated tooling only from the verified checkpoint. Deploy this -# reader before changing the producer so trusted-main checks can validate moves. -MANIFEST_PATHS = (MANIFEST, b"generated/castiron/generated-files.json") -# Derived from the renderer's repository-file inventory, also used by checkpoint -# publication to distinguish generated tooling from protected SDK-owned CI. -GENERATED_GITHUB_PATHS = frozenset( - { - b".github/workflows/castiron-custom-code.yml", - b".github/workflows/castiron-custom-code-comment.yml", - } -) - - -# Audited inventory of the authenticated pre-inventory checker format. These -# historical snapshots bind the checker bytes and manifest through their codegen -# hash. Never infer their inventory from the running compiler or candidate tree. -LEGACY_CHECKPOINT_INVENTORIES = { - "e504abba2701f4733eb4b8ce26856cca596e13feab1e7e323adc8e7d2ccb64bc": frozenset( - { - b".github/workflows/castiron-custom-code-comment.yml", - b".github/workflows/castiron-custom-code.yml", - } - ), -} - - -def checkpoint_github_inventory(source: bytes) -> frozenset[bytes]: - """Read literal compiler inventory from verified checkpoint data, without execution.""" - values = [ - node.value - for node in ast.parse(source).body - if isinstance(node, ast.Assign) - and any( - isinstance(target, ast.Name) and target.id == "GENERATED_GITHUB_PATHS" - for target in node.targets - ) - ] - if not values: - historical = LEGACY_CHECKPOINT_INVENTORIES.get(hashlib.sha256(source).hexdigest()) - if historical is not None: - return historical - if len(values) != 1: - raise ValueError("checkpoint must contain one generated GitHub inventory") - value = values[0] - if ( - not isinstance(value, ast.Call) - or not isinstance(value.func, ast.Name) - or value.func.id != "frozenset" - or len(value.args) != 1 - or value.keywords - ): - raise ValueError("checkpoint generated GitHub inventory must be literal") - literal_paths: object = ast.literal_eval(value.args[0]) - # An empty renderer inventory is written with an empty pair of braces. - if isinstance(literal_paths, dict) and not literal_paths: - return frozenset() - if not isinstance(literal_paths, set) or any( - not isinstance(path, bytes) or not path.startswith(b".github/") - for path in cast(set[object], literal_paths) - ): - raise ValueError("invalid checkpoint generated GitHub inventory") - return frozenset(cast(set[bytes], literal_paths)) - - -def generated_manifest(repo: Path, generated: str) -> dict[bytes, dict[str, str]]: - """Read ownership from the verified snapshot, never the candidate's manifest.""" - entries = report.tree_entries(repo, generated, include_workflows=True) - manifests = [path for path in MANIFEST_PATHS if path in entries] - if len(manifests) > 1: - raise ValueError("checkpoint contains ambiguous generated ownership manifests") - manifest_path = manifests[0] if manifests else MANIFEST - entry = entries.get(manifest_path) - if entry is None or entry.mode != b"100644": - raise ValueError( - "strict mode requires a generated ownership manifest; regenerate with Castiron" - ) - blobs = report.read_blobs(repo, entries) - inventory_path = manifest_path.rsplit(b"/", 1)[0] + b"/custom_code_budget.py" - inventory_entry = entries.get(inventory_path) - if inventory_entry is None or inventory_entry.mode not in (b"100644", b"100755"): - raise ValueError("checkpoint is missing its generated GitHub inventory") - github_paths = checkpoint_github_inventory(blobs[inventory_entry.oid]) - value = json.loads(blobs[entry.oid], object_pairs_hook=unique_keys) - if not isinstance(value, dict) or set(cast(dict[str, Any], value)) != { - "schema_version", - "files", - }: - raise ValueError("invalid generated ownership manifest") - value = cast(dict[str, Any], value) - if ( - type(value["schema_version"]) is not int - or value["schema_version"] != 1 - or not isinstance(value["files"], dict) - ): - raise ValueError("invalid generated ownership manifest") - files: dict[bytes, dict[str, str]] = {} - for name, identity in cast(dict[Any, Any], value["files"]).items(): - if ( - not isinstance(name, str) - or not name - or "\\" in name - or "\0" in name - or any(part in ("", ".", "..", ".git") for part in name.split("/")) - ): - raise ValueError("invalid generated manifest path") - path = name.encode("utf-8") - if path in MANIFEST_PATHS: - raise ValueError("generated manifest cannot include itself") - # A matching identity cannot turn inherited, SDK-owned CI into generated - # output. Use this verified checkpoint's literal compiler inventory, not - # the executing verifier's version or candidate Python/manifest claims. - # Public checkpoints may omit the private workflow/action blobs. - if not report.included(path) and path not in github_paths: - raise ValueError(f"generated manifest claims SDK-owned GitHub path: {name}") - if not isinstance(identity, dict) or set(cast(dict[str, Any], identity)) != { - "mode", - "sha256", - }: - raise ValueError(f"invalid generated identity: {name}") - identity = cast(dict[str, Any], identity) - if ( - identity["mode"] not in ("100644", "100755") - or not isinstance(identity["sha256"], str) - or not report.HASH.fullmatch(identity["sha256"]) - ): - raise ValueError(f"invalid generated identity: {name}") - files[path] = identity - # Sanitized public checkpoints omit GitHub blobs, not their ownership entries. - # Otherwise an omitted identity would also escape strict candidate checks. - if missing := github_paths - files.keys(): - raise ValueError( - f"generated manifest omits generated GitHub inventory path: {min(missing).decode()}" - ) - # Checkpoints retain SDK-owned workflow/action subtrees from their parent. - # Those are outside generated ownership. Every other snapshot path, including - # renderer-owned GitHub files omitted from the legacy hash, must be listed. - # Public snapshots retain generated identities without the private blobs. - snapshot_paths = set(entries) - {manifest_path} - if any( - report.included(path) or path in github_paths for path in snapshot_paths - files.keys() - ) or any(report.included(path) for path in files.keys() - snapshot_paths): - raise ValueError("generated manifest does not cover the verified snapshot exactly") - for path, identity in files.items(): - if path in entries: - actual = entries[path] - if ( - actual.mode.decode() != identity["mode"] - or hashlib.sha256(blobs[actual.oid]).hexdigest() != identity["sha256"] - ): - raise ValueError(f"manifest disagrees with generated snapshot: {path.decode()}") - # The manifest is itself immutable, bound by the existing snapshot hash. - files[manifest_path] = { - "mode": "100644", - "sha256": hashlib.sha256(blobs[entry.oid]).hexdigest(), - } - return files - - -def verify_generated( - repo: Path, - generated: str, - head: str, - *, - previous: str | None = None, - worktree: bool = False, -) -> list[str]: - files = generated_manifest(repo, generated) - old = generated_manifest(repo, previous) if previous and previous != generated else files - entries = report.tree_entries(repo, head, include_workflows=True) - blobs = report.read_blobs(repo, {p: e for p, e in entries.items() if p in files}) - failures: list[str] = [] - for path, identity in files.items(): - entry = entries.get(path) - content = blobs.get(entry.oid) if entry else None - mode = entry.mode.decode() if entry else None - if worktree: - full = repo / path.decode() - # Never follow a symlink introduced by an agent or formatter. - unsafe = any( - (repo / Path(*Path(path.decode()).parts[:i])).is_symlink() - for i in range(1, len(Path(path.decode()).parts) + 1) - ) - if unsafe or not full.is_file(): - content, mode = None, None - else: - content = full.read_bytes() - mode = "100755" if full.stat().st_mode & 0o111 else "100644" - if ( - content is None - or mode != identity["mode"] - or hashlib.sha256(content).hexdigest() != identity["sha256"] - ): - failures.append( - f"{path.decode()}: differs from Castiron output; change inputs/compiler or handwritten code" - ) - for path in old.keys() - files.keys(): - retired = repo / path.decode() - present = ( - retired.is_symlink() or (retired.exists() and not retired.is_dir()) - if worktree - else path in entries - ) - if present: - failures.append(f"{path.decode()}: retired generated file remains in candidate") - return failures - - -def check_isolation(repo: Path, base: str, head: str) -> int | None: - # Validate even if the policy was removed by an earlier commit in this PR. - proposed = read_budget(repo, head) - paths = changed_paths(repo, base, head) - if POLICY.encode() not in paths: - return None - if strict_policy(repo, base) and not strict_policy(repo, head): - raise ValueError("strict generated-file ownership cannot be downgraded") - if go_ratchet(repo, base) and not go_ratchet(repo, head): - raise ValueError("Go customization ratchet cannot be disabled") - if paths != {POLICY.encode()}: - raise ValueError(ISOLATION_MESSAGE) - return proposed - - -def count_custom_lines(result: dict[str, Any]) -> tuple[int, int]: - if result.get("status") != "ok": - raise ValueError("custom-code report could not verify the generated baseline") - added = removed = 0 - for file in result["files"]: - if not file["custom_after"]: - continue - counts = [file["added"], file["removed"]] - if any(not isinstance(n, str) or not n.isascii() or not n.isdecimal() for n in counts): - raise ValueError(f"cannot count non-text customization: {file['path']}") - added += int(counts[0]) - removed += int(counts[1]) - return added, removed - - -def outcome(state: str, description: str) -> dict[str, str]: - return {"state": state, "description": description} - - -def go_customization(repo: Path, revision: str, measured: dict[str, Any]) -> dict[str, Any]: - """Count patches plus physical handwritten Go, using only verified Git objects.""" - added, removed = count_custom_lines(measured) - # Use the reporter's ownership view. Private snapshots also retain SDK-owned - # CI files excluded from that view; those Go files are still handwritten. - generated = report.tree_entries(repo, measured["after"]["commit"]) - handwritten = { - path: entry - for path, entry in report.tree_entries(repo, revision, include_workflows=True).items() - if path.endswith(b".go") and path not in generated - } - for path, entry in handwritten.items(): - if entry.mode not in {b"100644", b"100755"}: - raise ValueError(f"nonregular handwritten Go: {path!r}") - blobs = report.read_blobs(repo, handwritten) - files: list[dict[str, Any]] = [] - for path, entry in sorted(handwritten.items()): - content = blobs[entry.oid] - content.decode("utf-8") - if b"\0" in content: - raise ValueError(f"non-text handwritten Go: {path!r}") - lines = content.count(b"\n") + int(bool(content) and not content.endswith(b"\n")) - files.append({"path": path.decode("utf-8"), "lines": lines}) - handwritten_lines = sum(file["lines"] for file in files) - return { - "revision": revision, - "generated": measured["after"], - "patch_lines": added + removed, - "handwritten_lines": handwritten_lines, - "handwritten_files": files, - "total": added + removed + handwritten_lines, - } - - -def evaluate( - repo: Path, - base: str, - head: str, - *, - public: bool, - fetch: bool = False, - pull_heads: list[str] | None = None, - measurement: tuple[dict[str, Any], bytes] | None = None, - policy_sha: str | None = None, - target_branch: str = "main", - main_snapshot: str | None = None, - repository: str | None = None, -) -> tuple[dict[str, Any], bytes]: - """In a queue, isolate each PR but budget the complete merged candidate.""" - base, head = report.require_sha(base), report.require_sha(head) - policy = cast(str, report.require_sha(policy_sha or base)) - result: dict[str, Any] = { - "base_sha": base, - "target_branch": target_branch, - "policy_sha": policy, - "head_sha": head, - "checks": {}, - } - checks = result["checks"] - proposed_limits: list[int] = [] - try: - read_budget(repo, policy) - read_budget(repo, base) - read_budget(repo, head) - if pull_heads is not None and not pull_heads: - raise ValueError("merge group has no verified constituent PRs") - if main_snapshot is not None and ( - target_branch != "next" - or main_snapshot != policy - or repository is None - or (pull_heads is not None and len(pull_heads) != 1) - or not is_main_snapshot_bootstrap( - repo, - repository, - base, - head, - policy, - constituent=pull_heads[0] - if pull_heads is not None and len(pull_heads) == 1 - else None, - ) - or any( - not is_main_snapshot_bootstrap(repo, repository, base, candidate, policy) - for candidate in (pull_heads if pull_heads is not None else [head]) - ) - ): - raise ValueError("invalid authenticated main snapshot bootstrap") - for pr_head in pull_heads if pull_heads is not None else [head]: - proposed = 0 if main_snapshot is not None else check_isolation(repo, base, pr_head) - if proposed is not None: - proposed_limits.append(proposed) - checks["isolation"] = outcome( - "success", - "Exact authenticated strict-zero main snapshot adoption." - if main_snapshot is not None - else "Budget changes are isolated; human review is required for increases.", - ) - except (report.ReportError, ValueError, UnicodeError) as exc: - checks["isolation"] = outcome("failure", str(exc)) - - patch = b"" - try: - limit = read_budget(repo, policy) - result["limit"] = limit - measured, patch = ( - measurement - if measurement is not None - else report.build_report( - repo, - base, - head, - public=public, - fetch=fetch, - require_head_hash=True, - target_branch=target_branch, - ) - ) - added, removed = count_custom_lines(measured) - total = added + removed - result.update( - additions=added, - deletions=removed, - total=total, - mixed_files=measured["counts"]["after"], - generated=measured["after"], - files=measured["files"], - ) - # Increases never apply to their own PR/group. Decreases must be viable too. - checked_limit = min([limit, *proposed_limits]) - result["checked_limit"] = checked_limit - state = "success" if total <= checked_limit else "failure" - description = f"+{added} / -{removed} = {total} custom lines; limit {checked_limit}." - if any(go_ratchet(repo, revision) for revision in (policy, base, head)): - # The report's historical merge base is for presentation only. Measure - # the captured live target against its own hash-verified snapshot. - baseline, _ = report.build_report( - repo, - base, - base, - public=public, - fetch=fetch, - require_head_hash=True, - target_branch=target_branch, - ) - before = go_customization(repo, base, baseline) - after = go_customization(repo, head, measured) - delta = after["total"] - before["total"] - result["ratchet"] = {"kind": "go", "before": before, "after": after, "delta": delta} - description += ( - f" Go customization {before['total']} -> {after['total']} (delta {delta:+d})." - ) - if delta > 0: - state = "failure" - if go_ratchet(repo, base) and not go_ratchet(repo, head): - state = "failure" - description += " Go customization ratchet cannot be disabled." - # Main's strict policy applies even if the candidate tries to remove it. - # A proposed activation must itself satisfy the stricter contract. - if strict_policy(repo, policy) or strict_policy(repo, base) or strict_policy(repo, head): - violations = verify_generated( - repo, - measured["after"]["commit"], - head, - previous=(measured["before"] or measured["adoption_baseline"])["commit"], - ) - result["strict_violations"] = violations - if violations: - state = "failure" - description = "Strict generated-file verification failed: " + "; ".join(violations) - else: - description += " All generated files exactly match the verified manifest." - checks["budget"] = outcome(state, description) - except (report.ReportError, ValueError, UnicodeError, KeyError) as exc: - checks["budget"] = outcome("failure", str(exc)) - return result, patch - - -def write_result(out: Path, result: dict[str, Any], patch: bytes = b"") -> None: - out.mkdir(parents=True, exist_ok=True) - (out / "budget.json").write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") - (out / "custom-code.patch").write_bytes(patch) - lines = ["## Custom-code budget", ""] - for name in ("isolation", "budget"): - check = result["checks"][name] - lines.append(f"- **{name}: {check['state']}** — {html.escape(check['description'])}") - if "total" in result: - lines += [ - "", - f"Mixed files: **{result['mixed_files']}**. Base-budget headroom: **{result['limit'] - result['total']} lines**.", - ] - lines += [ - "", - f"Policy: `{result['policy_sha']}` · candidate: `{result['head_sha']}`.", - f"Target {result.get('target_branch', 'baseline')}: `{result['base_sha']}`.", - f"Verified generated snapshot: `{result['generated']['commit']}`.", - "", - "| Largest custom patches | Added | Deleted |", - "|---|---:|---:|", - ] - files = [f for f in result["files"] if f["custom_after"]] - for file in sorted(files, key=lambda f: int(f["added"]) + int(f["removed"]), reverse=True)[ - :10 - ]: - path = html.escape(file["path"]).replace("|", "|").replace("\n", " ") - lines.append(f"| {path} | {file['added']} | {file['removed']} |") - ownership = [f for f in result["files"] if f["category"] == "no_longer_generated"] - if ownership: - lines += ["", "**No longer generated (outside the current budget):**"] - lines += [f"- {html.escape(f['path'])}" for f in ownership] - lines += [ - "", - "Budget increases require a separate budget-only PR, explicit justification, and a human approving review. Agents may not approve or bypass an increase.", - ] - summary = "\n".join(lines) + "\n" - (out / "summary.md").write_text(summary) - print(summary) - - -def github_evaluate( - repo: Path, - repository: str, - event: dict[str, Any], - trusted_sha: str, - trusted_report_dir: Path | None = None, - selection: dict[str, Any] | None = None, -) -> tuple[dict[str, Any], bytes]: - """Fetch data into a fresh bare repo; the executable checkout remains trusted.""" - if not report.REPOSITORY.fullmatch(repository): - raise ValueError("invalid repository") - root = f"repos/{repository}" - metadata = report.api("GET", root) - if event["repository"]["full_name"] != repository: - raise ValueError("event repository mismatch") - branch = metadata["default_branch"] - if branch != "main": - raise ValueError("budget gate requires main as the default branch") - main = report.require_sha(trusted_sha) - signal = event["workflow_run"] - run_id = signal["id"] - if type(run_id) is not int or run_id <= 0: - raise ValueError("invalid source run ID") - run = report.api("GET", f"{root}/actions/runs/{run_id}") - if ( - run["head_sha"] != signal["head_sha"] - or run["repository"]["full_name"] != repository - or run["path"].split("@", 1)[0] != ".github/workflows/castiron-custom-code.yml" - or run["status"] != "completed" - or run["run_attempt"] != signal["run_attempt"] - or run["event"] != signal["event"] - ): - raise ValueError("unexpected or superseded source workflow run") - head = report.require_sha(run["head_sha"]) - measured = None - if run["event"] == "pull_request": - current = report.current_pull(repository, run) - if current is None: - raise ValueError( - "source run must identify exactly one current PR targeting main or next" - ) - _, branch = current - if trusted_report_dir is not None: - measured = json.loads((trusted_report_dir / "report.json").read_text()) - base = report.require_sha(measured["target_base_sha"]) - else: - base = ( - main - if branch == "main" - else report.require_sha( - report.api("GET", f"{root}/git/ref/heads/{branch}")["object"]["sha"] - ) - ) - elif run["event"] == "merge_group": - # Both the target baseline and trusted policy must still be current. - branch = next( - ( - name - for name in report.TARGET_BRANCHES - if run["head_branch"].startswith(f"gh-readonly-queue/{name}/") - ), - "", - ) - if not branch: - raise ValueError("queue signal does not target main or next") - current_main = report.require_sha( - report.api("GET", f"{root}/git/ref/heads/main")["object"]["sha"] - ) - if branch == "main" and main != current_main: - raise ValueError("trusted checkout is stale; rerun against current main") - base = ( - main - if branch == "main" - else report.require_sha( - report.api("GET", f"{root}/git/ref/heads/{branch}")["object"]["sha"] - ) - ) - else: - raise ValueError("unsupported budget event") - # PR statuses can outlive their base revision. Require the queue's fresh, - # combined candidate check before merging rather than relying on those alone. - page = 1 - while True: - rules = report.api("GET", f"{root}/rules/branches/{branch}?per_page=100&page={page}") - if any(rule["type"] == "merge_queue" for rule in rules): - break - if len(rules) < 100: - raise ValueError( - f"{branch} must require a merge queue before activating the budget gate" - ) - page += 1 - # This receipt is created locally by the trusted-main selector before the - # pinned checkout. It must never come from candidate workflow artifacts. - if ( - selection is None - or type(selection.get("schema_version")) is not int - or any( - selection.get(key) != value - for key, value in { - "schema_version": 1, - "target_branch": branch, - "base_sha": base, - "head_sha": head, - "policy_sha": main, - }.items() - ) - ): - raise ValueError("trusted policy selection does not match evaluation") - measurement = None - if trusted_report_dir is not None: - # Only the preceding main-branch step may supply this directory. Never - # download a candidate artifact here. Reuse its verified report and bare - # object store, avoiding a second snapshot fetch/hash/diff computation. - if run["event"] != "pull_request": - raise ValueError("only PR runs can reuse the trusted report") - assert measured is not None - if ( - measured["target_branch"] != branch - or measured["policy_sha"] != main - or measured["head_sha"] != head - or (branch == "main" and base != main) - ): - raise ValueError("trusted report does not match the captured base/head") - if report.git(repo, "rev-parse", "--is-bare-repository").strip() != b"true": - raise ValueError("trusted report must use a bare object store") - measurement = (measured, (trusted_report_dir / "custom-code.patch").read_bytes()) - else: - if repo.exists(): - raise ValueError("Git object directory must be fresh") - subprocess.run(["git", "init", "--bare", str(repo)], check=True, capture_output=True) - report.git(repo, "remote", "add", "origin", f"https://github.com/{repository}.git") - report.git( - repo, "fetch", "--quiet", "--no-tags", "origin", *dict.fromkeys((main, base, head)) - ) - pull_heads: list[str] | None = None - if run["event"] == "merge_group": - if report.git(repo, "merge-base", base, head).decode().strip() != base: - raise ValueError("merge group does not include current target; retry the current group") - pull_heads = [] - entries = queued_entries(repository, branch) - if not any(queue_head == head for _, queue_head in entries): - raise ValueError("merge-group head is no longer in the current queue") - for candidate, queue_head in entries: - report.git(repo, "fetch", "--quiet", "--no-tags", "origin", queue_head) - ancestor = report.git(repo, "merge-base", queue_head, head).decode().strip() - if ancestor == queue_head: - report.git(repo, "fetch", "--quiet", "--no-tags", "origin", candidate) - pull_heads.append(candidate) - if not pull_heads: - raise ValueError("cannot verify merge-group PR membership; retry the current group") - main_snapshot = selection.get("main_snapshot") or None - if main_snapshot is not None: - if ( - branch != "next" - or main_snapshot != main - or report.api("GET", f"{root}/git/ref/heads/main")["object"]["sha"] != main - ): - raise ValueError("authenticated main snapshot changed; retry bootstrap") - # This evaluator has its own bare object store, including when reusing - # the PR report. SHA fetches do not create the selector's named main ref. - report.git( - repo, - "fetch", - "--quiet", - "--no-tags", - "origin", - "+refs/heads/main:refs/remotes/origin/main", - ) - if report.git(repo, "rev-parse", "refs/remotes/origin/main").decode().strip() != main: - raise ValueError("authenticated main changed during snapshot fetch; retry") - if not is_main_snapshot_bootstrap( - repo, - repository, - base, - head, - main, - constituent=pull_heads[0] if pull_heads is not None and len(pull_heads) == 1 else None, - ): - raise ValueError("authenticated main snapshot changed; retry bootstrap") - if branch == "next": - assert selection is not None - base_pin = read_policy(repo, base).get("next_policy") - if base_pin is None: - # evaluate() checks each verified queue constituent for policy isolation. - if ( - selection.get("bootstrap") is not True - or "next_policy" not in read_policy(repo, head) - or ( - main_snapshot is None - and pull_heads is None - and changed_paths(repo, base, head) != {POLICY.encode()} - ) - ): - raise ValueError("unreviewed next policy initialization") - elif selection.get("bootstrap") is not False or base_pin != { - "repository": repository, - "main_sha": main, - }: - raise ValueError("trusted next base pin changed; rerun policy selection") - result, patch = evaluate( - repo, - base, - head, - public=not metadata["private"], - fetch=True, - pull_heads=pull_heads, - measurement=measurement, - policy_sha=main, - target_branch=branch, - main_snapshot=main_snapshot, - repository=repository, - ) - result["target_branch"] = branch - return result, patch - - -def prepare_main_snapshot( - repo: Path, repository: str, base: str, authenticated_main: str, *, fetch: bool = False -) -> str: - """Create and verify an adoption commit without altering a ref, index or metadata.""" - base, authenticated_main = map(report.require_sha, (base, authenticated_main)) - strict = {"schema_version": 2, "max_custom_patch_lines": 0, "generated_files": "strict"} - main_policy = read_policy(repo, authenticated_main) - if { - key: value for key, value in main_policy.items() if key != "ratchet" - } != strict or "next_policy" in read_policy(repo, base): - raise ValueError("main snapshot preparation requires strict-zero main and unpinned next") - if policy.merge_base(repo, base, authenticated_main) != base: - raise ValueError("protected next must be an ancestor of authenticated main") - value = { - **main_policy, - "next_policy": {"repository": repository, "main_sha": authenticated_main}, - } - blob = report.git( - repo, - "hash-object", - "-w", - "--stdin", - input_bytes=(json.dumps(value, indent=2) + "\n").encode(), - ).strip() - entries = report.git(repo, "ls-tree", "-z", authenticated_main).split(b"\0") - # Replace only the root policy blob, preserving every other entry and mode. - replacement = b"100644 blob " + blob + b"\t" + POLICY.encode() - tree = ( - report.git( - repo, - "mktree", - "-z", - input_bytes=b"\0".join( - replacement if entry.partition(b"\t")[2] == POLICY.encode() else entry - for entry in entries - ), - ) - .decode() - .strip() - ) - head = ( - report.git( - repo, - "commit-tree", - tree, - "-p", - base, - "-p", - authenticated_main, - "-m", - "Adopt verified strict-zero main baseline on next", - ) - .decode() - .strip() - ) - if not is_main_snapshot_bootstrap(repo, repository, base, head, authenticated_main): - raise ValueError("prepared snapshot does not match authenticated main") - result, _ = evaluate( - repo, - base, - head, - public=True, - fetch=fetch, - target_branch="next", - policy_sha=authenticated_main, - main_snapshot=authenticated_main, - repository=repository, - ) - if not all(check["state"] == "success" for check in result["checks"].values()): - raise ValueError("main snapshot verification failed: " + json.dumps(result["checks"])) - return cast(str, head) - - -def read_public_api(path: str) -> Any: - # Reuse the local gh login without reading or printing its credential. - output = subprocess.run( - ["gh", "api", "--hostname", "github.com", "--method", "GET", path], - capture_output=True, - check=True, - timeout=60, - ) - return json.loads(output.stdout) - - -def prepare_next_baseline(repo: Path, repository: str, branch: str) -> dict[str, str]: - """Authenticate public refs and create one new local review branch; never push.""" - if not report.REPOSITORY.fullmatch(repository) or not branch.startswith("codex/"): - raise ValueError("expected a repository and a new codex/ review branch") - report.git(repo, "check-ref-format", "--branch", branch) - origin = report.git(repo, "remote", "get-url", "origin").decode().strip() - if origin not in report.github_origin_urls(repository): - raise ValueError("preparation requires the public repository as origin") - root = f"repos/{repository}" - metadata = read_public_api(root) - if ( - metadata["private"] - or metadata["default_branch"] != "main" - or metadata["full_name"] != repository - ): - raise ValueError("preparation requires a public repository with default main") - - def ref(name: str) -> str: - return cast( - str, - report.require_sha(read_public_api(f"{root}/git/ref/heads/{name}")["object"]["sha"]), - ) - - main, base = ref("main"), ref("next") - report.git( - repo, - "fetch", - "--quiet", - "--no-tags", - "origin", - "+refs/heads/main:refs/remotes/origin/main", - "+refs/heads/next:refs/remotes/origin/next", - ) - for name, expected in (("main", main), ("next", base)): - if ( - report.git(repo, "rev-parse", f"refs/remotes/origin/{name}").decode().strip() - != expected - ): - raise ValueError("public refs changed during preparation; retry") - head = prepare_main_snapshot(repo, repository, base, main, fetch=True) - if ref("main") != main or ref("next") != base: - raise ValueError("public refs changed during verification; retry") - # Compare-and-create only. Existing review branches and working trees are preserved. - report.git(repo, "update-ref", f"refs/heads/{branch}", head, "0" * 40) - return {"head_sha": head, "base_sha": base, "main_sha": main, "branch": branch} - - -def main() -> int: - if sys.argv[1:2] == ["select-policy"]: - return cast(int, policy.main(sys.argv[1:])) - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="command", required=True) - commands.add_parser("review-rules") - prepare = commands.add_parser("prepare-next-baseline") - prepare.add_argument("--repo", type=Path, required=True) - prepare.add_argument("--repository", required=True) - prepare.add_argument("--branch", required=True) - generated = commands.add_parser("verify-generated") - generated.add_argument("--repo", type=Path, required=True) - generated.add_argument("--policy-ref", required=True) - generated.add_argument("--generated", required=True) - generated.add_argument("--previous-generated") - generated.add_argument("--head", required=True) - generated.add_argument("--worktree", action="store_true") - local = commands.add_parser("check") - local.add_argument("--base", required=True) - local.add_argument("--head", required=True) - local.add_argument("--public", action="store_true") - local.add_argument("--fetch", action="store_true") - local.add_argument("--target-branch", choices=sorted(report.TARGET_BRANCHES), default="main") - github = commands.add_parser("github") - github.add_argument("--repository", required=True) - github.add_argument("--event-path", type=Path, required=True) - github.add_argument("--trusted-sha", required=True) - github.add_argument("--trusted-report-dir", type=Path) - github.add_argument("--selected-policy-json", type=Path) - for command in (local, github): - command.add_argument("--repo", type=Path, required=True) - command.add_argument("--out", type=Path, required=True) - args = parser.parse_args() - if args.command == "review-rules": - print(json.dumps(policy.HUMAN_REVIEW_RULES, separators=(",", ":"))) - return 0 - if args.command == "prepare-next-baseline": - try: - preparation = prepare_next_baseline(args.repo, args.repository, args.branch) - except (ValueError, OSError, report.ReportError, subprocess.SubprocessError) as exc: - print(str(exc), file=sys.stderr) - return 1 - print(json.dumps(preparation, sort_keys=True)) - return 0 - if args.command == "verify-generated": - # Local pipeline integration pins policy and checkpoint before merging. - # Legacy SDKs without a policy retain their existing behavior. - try: - policy_ref = report.require_sha(args.policy_ref) - if not report.git(args.repo, "ls-tree", policy_ref, "--", POLICY): - return 0 - if not strict_policy(args.repo, policy_ref): - return 0 - failures = verify_generated( - args.repo, - args.generated, - args.head, - previous=args.previous_generated, - worktree=args.worktree, - ) - if failures: - raise ValueError("\n".join(failures)) - except (ValueError, OSError, report.ReportError) as exc: - print(str(exc), file=sys.stderr) - return 1 - return 0 - try: - if args.command == "check": - result, patch = evaluate( - args.repo, - args.base, - args.head, - public=args.public, - fetch=args.fetch, - target_branch=args.target_branch, - ) - else: - result, patch = github_evaluate( - args.repo, - args.repository, - json.loads(args.event_path.read_text()), - args.trusted_sha, - args.trusted_report_dir, - json.loads(args.selected_policy_json.read_text()) - if args.selected_policy_json - else None, - ) - except ( - report.ReportError, - ValueError, - OSError, - KeyError, - TypeError, - subprocess.SubprocessError, - ) as exc: - result = { - "checks": {name: outcome("failure", str(exc)) for name in ("isolation", "budget")} - } - patch = b"" - write_result(args.out, result, patch) - if args.command == "github" and "GITHUB_OUTPUT" in os.environ: - with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: - output.writelines( - f"{name}={result['checks'][name]['state']}\n" for name in ("isolation", "budget") - ) - output.write(f"target_branch={result.get('target_branch', '')}\n") - output.write(f"policy_sha={result.get('policy_sha', '')}\n") - output.write(f"base_sha={result.get('base_sha', '')}\n") - output.write(f"head_sha={result.get('head_sha', '')}\n") - return 0 if all(c["state"] == "success" for c in result["checks"].values()) else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/castiron/custom_code_policy.py b/scripts/castiron/custom_code_policy.py deleted file mode 100755 index 8d06a3057..000000000 --- a/scripts/castiron/custom_code_policy.py +++ /dev/null @@ -1,731 +0,0 @@ -#!/usr/bin/env python3 -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -"""Select reviewed policy using current main; emit a local, versioned receipt. - -The pinned budget evaluator remains a separate executable authority. This module -never executes candidate code, and receipts must not come from candidate artifacts. -""" - -from __future__ import annotations - -import argparse -import ast -import importlib.util -import json -import os -import re -import sys -from pathlib import Path -from typing import Any, TypedDict, cast - -# Explicitly load the reporter beside this trusted script, including under python -I. -# The inspected repository is only a Git object store, not a Python import path. -_spec = importlib.util.spec_from_file_location( - "castiron_budget_reporter", Path(__file__).with_name("custom_code_report.py") -) -assert _spec is not None and _spec.loader is not None -report = importlib.util.module_from_spec(_spec) -sys.modules[_spec.name] = report -_spec.loader.exec_module(report) - -POLICY = ".castiron-ratchet.json" -NEXT_POLICY_VERSION = 1 -# Inert capability marker read by trusted review invalidation at this exact revision. -PIN_REVIEW_POLICY = "effective-policy-v1" -SELECTION_RECEIPT_VERSION = 1 - - -def unique_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: - value: dict[str, Any] = {} - for key, item in pairs: - if key in value: - raise ValueError(f"duplicate policy key: {key}") - value[key] = item - return value - - -def read_policy(repo: Path, revision: str) -> dict[str, Any]: - revision = report.require_sha(revision) - entry = report.git(repo, "ls-tree", revision, "--", POLICY).split() - if len(entry) != 4 or entry[:2] != [b"100644", b"blob"]: - raise ValueError(f"{POLICY} must exist as a regular, non-executable file at {revision}") - if int(report.git(repo, "cat-file", "-s", entry[2].decode())) > 4096: - raise ValueError("budget file exceeds 4096 bytes") - value = json.loads( - report.git(repo, "show", f"{revision}:{POLICY}"), object_pairs_hook=unique_keys - ) - if not isinstance(value, dict): - raise ValueError("budget file must be an object") # noqa: TRY004 - malformed policy is a value error - value = cast(dict[str, Any], value) - version = value.get("schema_version") - if type(version) is not int or version not in (1, 2): - raise ValueError("unsupported budget schema_version") - keys = {"schema_version", "max_custom_patch_lines"} - if "ratchet" in value: - keys.add("ratchet") - if value["ratchet"] != "go": - raise ValueError("ratchet must be go") - if version == 2: - keys.add("generated_files") - if "next_policy" in value: - keys.add("next_policy") - pin: Any = value["next_policy"] - if not isinstance(pin, dict) or set(cast(dict[str, Any], pin)) != { - "repository", - "main_sha", - }: - raise ValueError("next_policy must contain exactly repository and main_sha") - if not isinstance(pin["repository"], str) or not report.REPOSITORY.fullmatch( - pin["repository"] - ): - raise ValueError("invalid next_policy repository") - if not isinstance(pin["main_sha"], str): - raise ValueError("invalid next_policy main_sha") - report.require_sha(pin["main_sha"]) - if set(value) != keys: - raise ValueError("budget file contains missing or unsupported fields") - limit = value["max_custom_patch_lines"] - if type(limit) is not int or limit < 0: - raise ValueError("max_custom_patch_lines must be a nonnegative integer") - if version == 2 and (value["generated_files"] != "strict" or limit != 0): - raise ValueError("schema v2 requires generated_files=strict and a zero budget") - return value - - -def next_policy_sha(repo: Path, repository: str, base: str, authenticated_main: str) -> str: - """Select executable policy only from an authenticated target base, never its candidate.""" - base = report.require_sha(base) - authenticated_main = report.require_sha(authenticated_main) - if not report.REPOSITORY.fullmatch(repository): - raise ValueError("invalid policy repository") - origin = report.git(repo, "remote", "get-url", "origin").decode().strip() - if origin not in report.github_origin_urls(repository): - raise ValueError("policy repository origin mismatch") - fetched_main = report.git(repo, "rev-parse", "refs/remotes/origin/main").decode().strip() - if fetched_main != authenticated_main: - raise ValueError("authenticated main changed during policy selection; retry") - pin = read_policy(repo, base).get("next_policy") - # read_policy validates the shape of every present next_policy pin. - if pin is None: - raise ValueError("trusted next base has no reviewed policy pin") - if pin["repository"] != repository: - raise ValueError("next policy pin belongs to another repository") - policy = cast(str, report.require_sha(pin["main_sha"])) - # A reachable side-branch commit is not an adopted main policy revision. - adopted = report.git(repo, "rev-list", "--first-parent", authenticated_main).splitlines() - if policy.encode() not in adopted: - raise ValueError("next policy pin is not in authenticated main first-parent history") - read_policy(repo, policy) - # Reject pre-protocol checkers rather than executing code that cannot bind a next pin. - source = report.git(repo, "show", f"{policy}:scripts/castiron/custom_code_budget.py") - tree = ast.parse(source) - capable = any( - isinstance(node, ast.Assign) - and any( - isinstance(target, ast.Name) and target.id == "NEXT_POLICY_VERSION" - for target in node.targets - ) - and isinstance(node.value, ast.Constant) - and type(node.value.value) is int - and node.value.value == NEXT_POLICY_VERSION - for node in tree.body - ) - if not capable: - raise ValueError("pinned checker does not support reviewed next policies") - return policy - - -def supports_effective_pin_review(repo: Path, revision: str) -> bool: - """Read the effective checker's inert contract; never execute its source.""" - revision = report.require_sha(revision) - path = "scripts/castiron/custom_code_policy.py" - entry = report.git(repo, "ls-tree", revision, "--", path).split() - if len(entry) != 4 or entry[0] not in (b"100644", b"100755") or entry[1] != b"blob": - return False - source = report.git(repo, "show", f"{revision}:{path}") - declarations = re.findall(rb"^PIN_REVIEW_POLICY = .*$", source, re.MULTILINE) - return declarations == [b'PIN_REVIEW_POLICY = "effective-policy-v1"'] - - -class HumanReviewRules(TypedDict): - version: int - associations: list[str] - negative_pattern: str - attestation_prefix: str - - -HUMAN_REVIEW_RULES: HumanReviewRules = { - "version": 2, - "attestation_prefix": "I personally reviewed and approve this policy change as a human: ", - "associations": ["OWNER", "MEMBER", "COLLABORATOR"], - "negative_pattern": ( - r"automated[\s_:-]*(?:source[\s_:-]*)?review|source[- ]only review|" - r"(?:not|isn't|does not constitute|does not substitute for)\s+(?:an?\s+)?human\s+approv|" - r"non[- ]human[- ]approval" - ), -} - - -def has_human_approval(reviews: list[dict[str, Any]], author: str, head: str) -> bool: - """Versioned review predicate; publisher consumes these same trusted rules.""" - latest: dict[str, dict[str, Any]] = {} - for review in reviews: - # Discussion does not replace an approval; blocking or dismissed - # reviews still supersede it, and edited attestations are checked below. - if review["state"] not in ("PENDING", "COMMENTED") and review.get("user") is not None: - login = review["user"]["login"] - if login not in latest or review["id"] > latest[login]["id"]: - latest[login] = review - return any( - review["state"] == "APPROVED" - and review["commit_id"] == head - and review["user"]["type"] == "User" - and review.get("author_association") in HUMAN_REVIEW_RULES["associations"] - and not review["user"]["login"].endswith("[bot]") - and review["user"]["login"] != author - and (review.get("body") or "").strip() == HUMAN_REVIEW_RULES["attestation_prefix"] + head - and not re.search( - HUMAN_REVIEW_RULES["negative_pattern"], review.get("body") or "", re.IGNORECASE - ) - for review in latest.values() - ) - - -def require_human_policy_review(repository: str, head: str, number: int | None = None) -> int: - """Only live, exact-head human reviews authorize a next policy activation/update.""" - root = f"repos/{repository}" - pulls = ( - [{"number": number}] - if number is not None - else report.api("GET", f"{root}/commits/{head}/pulls?per_page=100") - ) - if not pulls: - # Fork runs can have no commit association. Discover candidates from - # the authenticated base repository, then apply the same live checks - # below. Never accept an approval based only on search metadata. - page = 1 - while True: - candidates = report.api( - "GET", f"{root}/pulls?state=open&base=next&per_page=100&page={page}" - ) - pulls.extend(item for item in candidates if item["head"]["sha"] == head) - if len(candidates) < 100: - break - page += 1 - current: list[dict[str, Any]] = [] - for item in pulls: - pull = report.api("GET", f"{root}/pulls/{item['number']}") - if ( - pull["state"] == "open" - and pull["head"]["sha"] == head - and pull["base"]["ref"] == "next" - and pull["base"]["repo"]["full_name"] == repository - ): - current.append(pull) - if len(current) != 1: - raise ValueError("policy change must identify one open next PR") - pull = current[0] - all_reviews: list[dict[str, Any]] = [] - page = 1 - while True: - reviews = report.api( - "GET", f"{root}/pulls/{pull['number']}/reviews?per_page=100&page={page}" - ) - all_reviews.extend(reviews) - if len(reviews) < 100: - break - page += 1 - if not has_human_approval(all_reviews, pull["user"]["login"], head): - raise ValueError("next policy changes require an exact-head human approving review") - return int(pull["number"]) - - -def requires_human_policy_review( - repo: Path, repository: str, base: str, head: str, authenticated_main: str -) -> bool: - """Decide review requirements from Git data using the selector's policy rules.""" - base_policy = read_policy(repo, base) - candidate_policy = read_policy(repo, head) - candidate_base_policy = read_policy(repo, merge_base(repo, base, head)) - budget_increased = ( - candidate_policy["max_custom_patch_lines"] > candidate_base_policy["max_custom_patch_lines"] - ) - ratchet_activated = ( - candidate_policy.get("ratchet") == "go" and candidate_base_policy.get("ratchet") != "go" - ) - pin_changed = candidate_policy.get("next_policy") != candidate_base_policy.get("next_policy") - if not pin_changed: - return budget_increased or ratchet_activated - policy = ( - next_policy_sha(repo, repository, base, authenticated_main) - if "next_policy" in base_policy - else authenticated_main - ) - candidate = head - proposed = read_policy(repo, next_policy_sha(repo, repository, candidate, authenticated_main)) - # Authored changes come from the merge base, but their safety must be - # measured against the active policy: the protected target's pin, or - # authenticated main during bootstrap. - previous = read_policy(repo, policy) - # Bootstrap executes authenticated main; initial adoption must not silently - # replace it with older checker code, even when policy fields match. - previous_checker = ( - base_policy["next_policy"]["main_sha"] - if "next_policy" in base_policy - else authenticated_main - ) - rolled_back = bool( - report.git( - repo, - "rev-list", - "--first-parent", - f"{candidate_policy['next_policy']['main_sha']}..{previous_checker}", - ).strip() - ) - # Compare effective policies, not the candidate's unchanged local ceiling. - # Schema/strictness or ratchet transitions also retain human authorization. - pin_requires_human = ( - not supports_effective_pin_review(repo, previous_checker) - or rolled_back - or proposed["max_custom_patch_lines"] > previous["max_custom_patch_lines"] - or proposed["schema_version"] != previous["schema_version"] - or proposed.get("generated_files") != previous.get("generated_files") - or proposed.get("ratchet") != previous.get("ratchet") - ) - return budget_increased or ratchet_activated or pin_requires_human - - -def is_main_snapshot_bootstrap( - repo: Path, - repository: str, - base: str, - head: str, - authenticated_main: str, - *, - constituent: str | None = None, -) -> bool: - """Recognize only an exact, strict-zero authenticated main snapshot adoption. - - This does not verify generated ownership: the ordinary budget evaluator must - still authenticate the unchanged main checkpoints and their complete manifest. - No candidate file is executed and no generation metadata is rewritten. - """ - base, head, authenticated_main = map(report.require_sha, (base, head, authenticated_main)) - if "next_policy" in read_policy(repo, base): - return False - strict = {"schema_version": 2, "max_custom_patch_lines": 0, "generated_files": "strict"} - main_policy = read_policy(repo, authenticated_main) - if {key: value for key, value in main_policy.items() if key != "ratchet"} != strict: - return False - expected_policy = { - **main_policy, - "next_policy": {"repository": repository, "main_sha": authenticated_main}, - } - if read_policy(repo, head) != expected_policy: - return False - # Also validates the configured repository, fetched main and protocol marker. - if next_policy_sha(repo, repository, head, authenticated_main) != authenticated_main: - return False - if merge_base(repo, base, authenticated_main) != base or merge_base(repo, base, head) != base: - return False - paths = set( - report.git( - repo, - "diff", - "--name-only", - "-z", - "--no-renames", - "--no-ext-diff", - "--no-textconv", - authenticated_main, - head, - "--", - ).split(b"\0") - ) - {b""} - if paths != {POLICY.encode()}: - return False - parents = report.git(repo, "rev-list", "--parents", "-n", "1", head).decode().split()[1:] - expected = list(dict.fromkeys((base, authenticated_main))) - if constituent is not None and head != constituent: - if not is_main_snapshot_bootstrap(repo, repository, base, constituent, authenticated_main): - return False - expected = [base, constituent] - if parents != expected: - return False - # The complete tree comparison binds stats/inventory bytes and file modes. - # Require an existing public snapshot; this is not legacy metadata recovery. - return bool(report.read_stats(repo, authenticated_main).get("public_codegen_sha")) - - -def select_policy( - repo: Path, - repository: str, - branch: str, - base: str, - head: str, - authenticated_main: str, - *, - pull_heads: list[str] | None = None, - pull_numbers: dict[str, int] | None = None, -) -> dict[str, Any]: - """The trusted selector runs before checking out any pinned executable code.""" - base, head, authenticated_main = map(report.require_sha, (base, head, authenticated_main)) - if branch == "main": - return { - "schema_version": SELECTION_RECEIPT_VERSION, - "target_branch": branch, - "base_sha": base, - "head_sha": head, - "policy_sha": authenticated_main, - "bootstrap": False, - } - if branch != "next": - raise ValueError("unsupported policy target") - # Malformed policies are errors, never a reason to enter bootstrap mode. - base_policy = read_policy(repo, base) - head_policy = read_policy(repo, head) - bootstrap = "next_policy" not in base_policy - snapshot_bootstrap = bootstrap and is_main_snapshot_bootstrap( - repo, - repository, - base, - head, - authenticated_main, - constituent=pull_heads[0] if pull_heads is not None and len(pull_heads) == 1 else None, - ) - if ( - snapshot_bootstrap - and pull_heads is not None - and ( - len(pull_heads) != 1 - or not is_main_snapshot_bootstrap( - repo, repository, base, pull_heads[0], authenticated_main - ) - ) - ): - raise ValueError("main snapshot bootstrap requires one exact verified constituent") - if bootstrap: - if "next_policy" not in head_policy or ( - not snapshot_bootstrap - and pull_heads is None - and changed_paths(repo, base, head) != {POLICY.encode()} - ): - raise ValueError("missing next pin allows only reviewed policy-only initialization") - policy = authenticated_main - else: - policy = next_policy_sha(repo, repository, base, authenticated_main) - # No candidate-selected version executes, including in a policy-only PR. - head_base_policy = read_policy(repo, merge_base(repo, base, head)) - if head_policy.get("next_policy") != head_base_policy.get("next_policy"): - if "next_policy" not in head_policy or ( - not snapshot_bootstrap - and pull_heads is None - and changed_paths(repo, base, head) != {POLICY.encode()} - ): - raise ValueError("next pin changes must be policy-only and cannot remove the pin") - next_policy_sha(repo, repository, head, authenticated_main) - policy_reviews: list[dict[str, Any]] = [] - bootstrap_validated = False - candidates = pull_heads if pull_heads is not None else [head] - if not candidates: - raise ValueError("next evaluation has no verified constituent PR") - for candidate in candidates: - candidate_policy = read_policy(repo, candidate) - candidate_base = merge_base(repo, base, candidate) - candidate_base_policy = read_policy(repo, candidate_base) - pin_changed = candidate_policy.get("next_policy") != candidate_base_policy.get( - "next_policy" - ) - budget_increased = ( - candidate_policy["max_custom_patch_lines"] - > candidate_base_policy["max_custom_patch_lines"] - ) - ratchet_activated = ( - candidate_policy.get("ratchet") == "go" and candidate_base_policy.get("ratchet") != "go" - ) - if ( - not snapshot_bootstrap - and (pin_changed or budget_increased or ratchet_activated) - and changed_paths(repo, base, candidate) != {POLICY.encode()} - ): - raise ValueError("each next policy activation or increase must be policy-only") - if requires_human_policy_review(repo, repository, base, candidate, authenticated_main): - number = require_human_policy_review( - repository, candidate, pull_numbers[candidate] if pull_numbers is not None else None - ) - policy_reviews.append({"number": number, "head_sha": candidate}) - # Keep activation tied to its verified policy-only constituent, even - # when current main's effective-policy rules require no new approval. - if pin_changed and candidate_policy.get("next_policy") == head_policy.get("next_policy"): - bootstrap_validated = True - if bootstrap and not bootstrap_validated: - raise ValueError("bootstrap requires a verified policy-only constituent") - return { - "schema_version": SELECTION_RECEIPT_VERSION, - "target_branch": branch, - "base_sha": base, - "head_sha": head, - "policy_sha": policy, - "bootstrap": bootstrap, - "main_snapshot": authenticated_main if snapshot_bootstrap else "", - "policy_reviews": policy_reviews, - "human_review_rules": HUMAN_REVIEW_RULES, - } - - -def select_github_policy( - repo: Path, - repository: str, - event: dict[str, Any], - trusted_sha: str, - *, - source_workflow: str = ".github/workflows/castiron-custom-code.yml", -) -> dict[str, Any]: - """Authenticate event, refs and graph in a fresh store before choosing checker code.""" - if source_workflow not in ( - ".github/workflows/castiron-custom-code.yml", - ".github/workflows/ci.yml", - ): - raise ValueError("unsupported policy source workflow") - if ( - not report.REPOSITORY.fullmatch(repository) - or event["repository"]["full_name"] != repository - ): - raise ValueError("policy event repository mismatch") - root = f"repos/{repository}" - metadata = report.api("GET", root) - if metadata["default_branch"] != "main": - raise ValueError("policy selector requires the main default branch") - signal = event["workflow_run"] - run = report.api("GET", f"{root}/actions/runs/{signal['id']}") - if ( - run["repository"]["full_name"] != repository - or run["head_sha"] != signal["head_sha"] - or run["status"] != "completed" - or run["event"] != signal["event"] - or run["run_attempt"] != signal["run_attempt"] - or run["path"].split("@", 1)[0] != source_workflow - ): - raise ValueError("unexpected or superseded policy source run") - workflow = report.api("GET", f"{root}/actions/workflows/{source_workflow.rsplit('/', 1)[1]}") - if ( - type(run.get("workflow_id")) is not int - or run["workflow_id"] != workflow["id"] - or workflow["path"] != source_workflow - ): - raise ValueError("policy source run is not the canonical workflow") - head = report.require_sha(run["head_sha"]) - pull_numbers: dict[str, int] = {} - if run["event"] == "pull_request": - current = report.current_pull(repository, run) - if current is None: - raise ValueError("policy run must identify one current target PR") - number, branch = current - pull_numbers[head] = number - elif run["event"] == "merge_group": - branch = next( - ( - name - for name in report.TARGET_BRANCHES - if run["head_branch"].startswith(f"gh-readonly-queue/{name}/") - ), - "", - ) - if not branch: - raise ValueError("policy queue target is unsupported") - else: - raise ValueError("unsupported policy source event") - trusted_sha = report.require_sha(trusted_sha) - if branch == "main": - if source_workflow != ".github/workflows/castiron-custom-code.yml": - raise ValueError("CI policy adapter is restricted to next") - return select_policy(repo, repository, branch, trusted_sha, head, trusted_sha) - main = report.require_sha(report.api("GET", f"{root}/git/ref/heads/main")["object"]["sha"]) - base = report.require_sha(report.api("GET", f"{root}/git/ref/heads/next")["object"]["sha"]) - if repo.exists(): - raise ValueError("policy object directory must be fresh") - repo.mkdir() - report.git(repo, "init", "--quiet", "--bare") - report.git(repo, "remote", "add", "origin", f"https://github.com/{repository}.git") - report.git( - repo, - "fetch", - "--quiet", - "--no-tags", - "origin", - "+refs/heads/main:refs/remotes/origin/main", - "+refs/heads/next:refs/remotes/origin/next", - head, - ) - if report.git(repo, "rev-parse", "refs/remotes/origin/next").decode().strip() != base: - raise ValueError("next base changed during selection; retry") - heads: list[str] | None = None - if run["event"] == "merge_group": - if report.git(repo, "merge-base", base, head).decode().strip() != base: - raise ValueError("policy queue does not contain current next base") - entries = queued_pull_entries(repository, branch) - if not any(queue == head for _, queue, _ in entries): - raise ValueError("policy queue entry is no longer live") - heads = [] - for candidate, queue, number in entries: - report.git(repo, "fetch", "--quiet", "--no-tags", "origin", queue) - if report.git(repo, "merge-base", queue, head).decode().strip() == queue: - report.git(repo, "fetch", "--quiet", "--no-tags", "origin", candidate) - if candidate in pull_numbers: - raise ValueError("policy queue has multiple PRs for one head") - pull_numbers[candidate] = number - heads.append(candidate) - if not heads: - raise ValueError("policy queue has no verified constituent PRs") - return select_policy( - repo, repository, branch, base, head, main, pull_heads=heads, pull_numbers=pull_numbers - ) - - -def merge_base(repo: Path, base: str, head: str) -> str: - """Distinguish actual PR policy edits from later changes on the target.""" - return cast( - str, - report.git(repo, "merge-base", report.require_sha(base), report.require_sha(head)) - .decode() - .strip(), - ) - - -def changed_paths(repo: Path, base: str, head: str) -> set[bytes]: - """Use the entire PR diff, not its last commit or a truncated API file list.""" - start = merge_base(repo, base, head) - return set( - report.git( - repo, - "diff", - "--name-only", - "-z", - "--no-renames", - "--no-ext-diff", - "--no-textconv", - start, - head, - "--", - ).split(b"\0") - ) - {b""} - - -def queued_entries(repository: str, branch: str) -> list[tuple[str, str]]: - return [(head, queue) for head, queue, _ in queued_pull_entries(repository, branch)] - - -def queued_pull_entries(repository: str, branch: str) -> list[tuple[str, str, int]]: - """Query the actual queue; never infer PR membership from commit messages.""" - owner, name = repository.split("/") - query = """query($owner:String!,$name:String!,$branch:String!,$cursor:String) { - repository(owner:$owner,name:$name) { mergeQueue(branch:$branch) { - entries(first:100,after:$cursor) { - nodes { pullRequest { number headRefOid } headCommit { oid } } - pageInfo { hasNextPage endCursor } - } - } } - }""" - cursor = None - heads: list[tuple[str, str, int]] = [] - while True: - response = report.api( - "POST", - "graphql", - { - "query": query, - "variables": {"owner": owner, "name": name, "branch": branch, "cursor": cursor}, - }, - ) - entries = response["data"]["repository"]["mergeQueue"]["entries"] - for node in entries["nodes"]: - # Entries still awaiting preparation have no synthetic queue commit. - if node["headCommit"] is not None: - heads.append( - ( - report.require_sha(node["pullRequest"]["headRefOid"]), - report.require_sha(node["headCommit"]["oid"]), - int(node["pullRequest"]["number"]), - ) - ) - page = entries["pageInfo"] - if not page["hasNextPage"]: - return heads - if not page["endCursor"] or page["endCursor"] == cursor: - raise ValueError("invalid merge-queue pagination") - cursor = page["endCursor"] - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="command", required=True) - review = commands.add_parser("review-required") - review.add_argument("--repo", type=Path, required=True) - review.add_argument("--repository", required=True) - review.add_argument("--base", required=True) - review.add_argument("--head", required=True) - review.add_argument("--main", required=True) - selection = commands.add_parser("select-policy") - selection.add_argument("--repo", type=Path, required=True) - selection.add_argument("--repository", required=True) - selection.add_argument("--event-path", type=Path, required=True) - selection.add_argument("--trusted-sha", required=True) - selection.add_argument("--out", type=Path, required=True) - selection.add_argument( - "--source-workflow", - choices=(".github/workflows/castiron-custom-code.yml", ".github/workflows/ci.yml"), - default=".github/workflows/castiron-custom-code.yml", - ) - args = parser.parse_args(argv) - if args.command == "review-required": - if not report.REPOSITORY.fullmatch(args.repository): - raise ValueError("invalid policy repository") - base, head, main = map(report.require_sha, (args.base, args.head, args.main)) - if args.repo.exists(): - raise ValueError("policy object directory must be fresh") - args.repo.mkdir() - report.git(args.repo, "init", "--quiet", "--bare") - report.git( - args.repo, "remote", "add", "origin", f"https://github.com/{args.repository}.git" - ) - report.git( - args.repo, - "fetch", - "--quiet", - "--no-tags", - "origin", - "+refs/heads/main:refs/remotes/origin/main", - base, - head, - ) - if report.git(args.repo, "rev-parse", "refs/remotes/origin/main").decode().strip() != main: - raise ValueError("authenticated main changed during review evaluation; retry") - print( - json.dumps( - { - "requires_human": requires_human_policy_review( - args.repo, args.repository, base, head, main - ) - } - ) - ) - return 0 - selected = select_github_policy( - args.repo, - args.repository, - json.loads(args.event_path.read_text()), - args.trusted_sha, - source_workflow=args.source_workflow, - ) - args.out.write_text(json.dumps(selected, sort_keys=True) + "\n") - if "GITHUB_OUTPUT" in os.environ: - with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: - output.writelines( - f"{key}={json.dumps(value) if isinstance(value, (bool, list, dict)) else value}\n" - for key, value in selected.items() - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/castiron/custom_code_report.py b/scripts/castiron/custom_code_report.py deleted file mode 100755 index 28b827905..000000000 --- a/scripts/castiron/custom_code_report.py +++ /dev/null @@ -1,1534 +0,0 @@ -#!/usr/bin/env python3 -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -# Regenerate with Castiron; do not edit this file by hand. -"""Standalone, vendorable Castiron generated-content hashing and reporting. - -Hash v1: SHA256(domain || u64(entry count) || entries), where each entry is -u64(path length) || path bytes || six ASCII Git mode bytes || u64(blob length) -|| blob bytes. Integers are unsigned big-endian. Entries sort by raw Git path. -Only .github/actions/ and .github/workflows/ are excluded. No SDK imports. -""" - -from __future__ import annotations - -import argparse -import base64 -import collections -import hashlib -import html -import json -import os -import re -import struct -import subprocess -import sys -import tempfile -from contextlib import ExitStack -from dataclasses import dataclass -from pathlib import Path -from typing import Any, cast -from urllib.parse import urlencode - -DOMAIN = b"castiron-codegen-v1\0" -MARKER = "" -RUN_MARKER = re.compile(r"") -SHA = re.compile(r"[0-9a-f]{40}\Z") -HASH = re.compile(r"[0-9a-f]{64}\Z") -REPOSITORY = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\Z") -TARGET_BRANCHES = ("main", "next") -EXCLUDED = (b".github/actions/", b".github/workflows/") -PUBLIC_BLOCKED_PATHS = { - b".castiron", - b".github", - b".gitmodules", - b"CODEOWNERS", - b"SECURITY.md", - b"docs/CODEOWNERS", - b"docs/SECURITY.md", -} -LABELS = { - "newly_customized": "Newly customized", - "removed": "Customization removed", - "existing_changed": "Existing customization changed", - "first_adoption": "Current customization; legacy baseline unavailable", - "baseline_changed": "Generated baseline changed", - "unchanged": "Existing customization unchanged", - "no_longer_generated": "No longer generated", - "newly_generation_owned": "Now generated; already existed", -} - - -class ReportError(ValueError): - """The report cannot establish a trustworthy comparison.""" - - -def github_origin_urls(repository: str) -> set[str]: - """Accepted URL spellings for one exact GitHub repository.""" - if not REPOSITORY.fullmatch(repository): - raise ReportError("invalid GitHub repository") - return { - f"https://github.com/{repository}.git", - f"https://github.com/{repository}", - f"git@github.com:{repository}.git", - } - - -@dataclass(frozen=True) -class Entry: - mode: bytes - oid: str - - -def git(repo: Path, *args: str, input_bytes: bytes | None = None) -> bytes: - env = os.environ.copy() - # Preserve ordinary Git authentication/configuration, but do not inherit - # repository routing from the caller (for example, a Git hook). - for name in ( - "GIT_DIR", - "GIT_COMMON_DIR", - "GIT_WORK_TREE", - "GIT_INDEX_FILE", - "GIT_PREFIX", - "GIT_OBJECT_DIRECTORY", - "GIT_ALTERNATE_OBJECT_DIRECTORIES", - ): - env.pop(name, None) - env["GIT_NO_REPLACE_OBJECTS"] = "1" - with ExitStack() as resources: - stdin = None - if input_bytes is not None: - # Batched object reads can fill both pipes on macOS. Feed requests - # through a private, automatically removed file while draining output. - stdin = resources.enter_context(tempfile.TemporaryFile()) - stdin.write(input_bytes) - stdin.seek(0) - result = subprocess.run( - ["git", "--no-replace-objects", "-C", str(repo), *args], - stdin=stdin, - capture_output=True, - env=env, - check=False, - ) - if result.returncode: - raise ReportError(f"git {args[0]} failed: {result.stderr.decode(errors='replace').strip()}") - return result.stdout - - -def require_sha(value: str) -> str: - if not SHA.fullmatch(value): - raise ReportError("expected a full Git commit SHA") - return value - - -def included(path: bytes) -> bool: - return not path.startswith(EXCLUDED) - - -def tree_entries(repo: Path, commit: str, *, include_workflows: bool = False) -> dict[bytes, Entry]: - result: dict[bytes, Entry] = {} - for record in git(repo, "ls-tree", "-r", "-z", require_sha(commit)).split(b"\0"): - if not record: - continue - metadata, path = record.split(b"\t", 1) - mode, kind, oid = metadata.split(b" ") - if not include_workflows and not included(path): - continue - if kind != b"blob" or mode not in {b"100644", b"100755", b"120000"}: - raise ReportError(f"unsupported Git entry: {path!r}") - result[path] = Entry(mode, oid.decode("ascii")) - return result - - -def read_blobs(repo: Path, entries: dict[bytes, Entry]) -> dict[str, bytes]: - oids = sorted({entry.oid for entry in entries.values()}) - output = git( - repo, "cat-file", "--batch", input_bytes="".join(f"{oid}\n" for oid in oids).encode() - ) - result: dict[str, bytes] = {} - offset = 0 - for oid in oids: - end = output.index(b"\n", offset) - actual, kind, size = output[offset:end].split(b" ") - if actual.decode() != oid or kind != b"blob": - raise ReportError("unexpected Git object response") - offset = end + 1 - length = int(size) - result[oid] = output[offset : offset + length] - offset += length - if output[offset : offset + 1] != b"\n": - raise ReportError("truncated Git object response") - offset += 1 - if offset != len(output): - raise ReportError("unexpected trailing Git object data") - return result - - -def hash_codegen_commit(repo: Path, commit: str) -> str: - entries = tree_entries(repo, commit) - blobs = read_blobs(repo, entries) - digest = hashlib.sha256(DOMAIN + struct.pack(">Q", len(entries))) - for path, entry in sorted(entries.items()): - content = blobs[entry.oid] - digest.update(struct.pack(">Q", len(path))) - digest.update(path) - digest.update(entry.mode) - digest.update(struct.pack(">Q", len(content))) - digest.update(content) - return digest.hexdigest() - - -def read_stats(repo: Path, commit: str) -> dict[str, str]: - entry = git(repo, "ls-tree", require_sha(commit), "--", ".castiron.stats.yml") - if not entry.startswith((b"100644 blob ", b"100755 blob ")): - raise ReportError("Castiron stats must be a regular Git file") - text = git(repo, "show", f"{require_sha(commit)}:.castiron.stats.yml").decode("utf-8") - values: dict[str, str] = {} - for line in text.splitlines(): - if not line.strip() or line.lstrip().startswith("#"): - continue - key, sep, value = line.partition(":") - if not sep or key.strip() in values: - raise ReportError("invalid or duplicate stats field") - values[key.strip()] = value.strip().strip("\"'") - if values.get("schema_version") != "1": - raise ReportError("unsupported Castiron stats schema") - require_sha(values.get("codegen_sha", "")) - if not re.fullmatch( - r"[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}", - values.get("generation_id", ""), - ): - raise ReportError("invalid generation ID") - if "codegen_hash" in values and not HASH.fullmatch(values["codegen_hash"]): - raise ReportError("invalid codegen_hash") - if "public_codegen_sha" in values: - require_sha(values["public_codegen_sha"]) - return values - - -def public_metadata(repo: Path, commit: str) -> dict[str, str]: - message = git(repo, "show", "-s", "--format=%B", require_sha(commit)).decode("utf-8") - fields: dict[str, str] = {} - for key in ("Version", "Generation-ID", "Hash", "Branch"): - matches = re.findall(rf"^Castiron-Public-Codegen-{key}: (.+)$", message, re.MULTILINE) - if len(matches) != 1: - raise ReportError("invalid public codegen metadata") - fields[key] = matches[0] - if fields["Version"] != "1" or not HASH.fullmatch(fields["Hash"]): - raise ReportError("unsupported public codegen metadata") - targets = re.findall(r"^Castiron-Public-Codegen-Target: (.*)$", message, re.MULTILINE) - if targets: - if len(targets) != 1 or targets[0] not in TARGET_BRANCHES: - raise ReportError("invalid public codegen target") - fields["Target"] = targets[0] - if not fields["Branch"].startswith("codegen/"): - raise ReportError("invalid public codegen branch") - git(repo, "check-ref-format", f"refs/heads/{fields['Branch']}") - for key in ("Adoption-Base", "Adoption-Source"): - values = re.findall(rf"^Castiron-Public-Codegen-{key}: (.+)$", message, re.MULTILINE) - if values: - if len(values) != 1: - raise ReportError("duplicate adoption metadata") - fields[key] = require_sha(values[0]) - if ("Adoption-Base" in fields) != ("Adoption-Source" in fields): - raise ReportError("incomplete adoption metadata") - return fields - - -def legacy_snapshot_on_main(repo: Path, commit: str, *, fetch: bool) -> bool: - """Legacy snapshots have no target; trust only adoption in public main history.""" - if fetch: - git( - repo, - "fetch", - "--quiet", - "--no-tags", - "origin", - "+refs/heads/main:refs/remotes/origin/main", - ) - revisions = ( - git( - repo, - "log", - "--first-parent", - "--format=%H", - f"-G{require_sha(commit)}", - "refs/remotes/origin/main", - "--", - ".castiron.stats.yml", - ) - .decode() - .splitlines() - ) - for revision in revisions: - try: - if read_stats(repo, revision).get("public_codegen_sha") == commit: - return True - except ReportError: - continue - return False - - -def validate_public_target( - repo: Path, - commit: str, - metadata: dict[str, str], - *, - target_branch: str, - fetch: bool, - bootstrap_lineage: str | None = None, -) -> None: - if target_branch not in TARGET_BRANCHES: - raise ReportError("unsupported report target") - if target_branch != "next" and ( - metadata["Branch"] == "codegen/castiron/next" - or metadata["Branch"].startswith("codegen/castiron/next/") - ): - raise ReportError("next checkpoint cannot authorize a main report") - if target_branch == "main": - if metadata.get("Target") == "next": - raise ReportError("next checkpoint cannot authorize a main report") - if "Target" not in metadata and not legacy_snapshot_on_main(repo, commit, fetch=fetch): - if bootstrap_lineage is None: - raise ReportError("legacy snapshot has no verified main adoption") - # Pre-bootstrap SDK stats have no public snapshot pointer. Their - # baseline may be an ancestor of an adopted snapshot, never adopted - # directly. Validate the lineage independently before trusting it. - validate_public_target( - repo, - bootstrap_lineage, - public_metadata(repo, bootstrap_lineage), - target_branch=target_branch, - fetch=fetch, - ) - git(repo, "merge-base", "--is-ancestor", commit, bootstrap_lineage) - - -def resolve_public_baseline( - repo: Path, - revision: str, - *, - fetch: bool, - require_hash: bool, - lineage: str | None = None, - target_branch: str = "main", -) -> dict[str, Any]: - stats = read_stats(repo, revision) - commit = stats.get("public_codegen_sha") - if commit is None: - # Bootstrap old public main without changing its schema-v1 stats. Search - # only the already-verified public snapshot lineage supplied by PR head. - if lineage is None: - raise ReportError("stats do not contain public_codegen_sha") - for candidate in git(repo, "rev-list", require_sha(lineage)).decode().splitlines(): - if public_metadata(repo, candidate)["Generation-ID"] == stats["generation_id"]: - commit = candidate - break - if commit is None: - raise ReportError("public snapshot lineage does not contain the base generation") - elif fetch: - git(repo, "fetch", "--quiet", "--no-tags", "origin", commit) - metadata = public_metadata(repo, commit) - validate_public_target( - repo, - commit, - metadata, - target_branch=target_branch, - fetch=fetch, - bootstrap_lineage=lineage if stats.get("public_codegen_sha") is None else None, - ) - if metadata["Generation-ID"] != stats["generation_id"]: - raise ReportError("public codegen generation ID does not match stats") - ref = f"refs/heads/{metadata['Branch']}" - tracking = f"refs/castiron/report/{hashlib.sha256(ref.encode()).hexdigest()}" - if fetch: - git(repo, "fetch", "--quiet", "--no-tags", "origin", f"+{ref}:{tracking}") - else: - tracking = f"refs/remotes/origin/{metadata['Branch']}" - git(repo, "merge-base", "--is-ancestor", commit, tracking) - actual = hash_codegen_commit(repo, commit) - if actual != metadata["Hash"]: - raise ReportError("public codegen content does not match snapshot metadata") - expected = stats.get("codegen_hash") - if expected is None and require_hash: - raise ReportError("head stats do not contain codegen_hash") - if expected is not None and expected != actual: - raise ReportError(f"codegen_hash mismatch: expected {expected}, got {actual}") - return { - "commit": commit, - "generation_id": stats["generation_id"], - "hash": actual, - "hash_recorded": expected is not None, - } - - -def public_snapshot_entries( - source: Path, - commit: str, - destination: Path, - public_base: str, - approved_tree: str | None = None, -) -> dict[bytes, Entry]: - entries = tree_entries(source, commit) - approved = tree_entries(destination, public_base) - if approved_tree is not None: - approved.update(tree_entries(destination, approved_tree)) - for path in entries: - if path in PUBLIC_BLOCKED_PATHS or path.startswith((b".castiron/", b".github/")): - raise ReportError(f"refusing to publish private or governance snapshot path: {path!r}") - if path not in approved: - raise ReportError( - f"snapshot path is absent from the approved public SDK tree: {path!r}" - ) - return entries - - -def copy_generated_tree( - source: Path, - commit: str, - destination: Path, - public_base: str, - approved_tree: str | None = None, -) -> str: - """Transfer approved blobs/trees only, never source commits or ancestors.""" - entries = public_snapshot_entries(source, commit, destination, public_base, approved_tree) - for oid, blob in read_blobs(source, entries).items(): - actual = git(destination, "hash-object", "-w", "--stdin", input_bytes=blob).decode().strip() - if actual != oid: - raise ReportError("source and destination Git object formats differ") - directories: dict[bytes, list[bytes]] = {b"": []} - for path, entry in entries.items(): - parent, _, name = path.rpartition(b"/") - directories.setdefault(parent, []).append( - entry.mode + b" blob " + entry.oid.encode() + b"\t" + name + b"\0" - ) - while parent: - parent = parent.rpartition(b"/")[0] - directories.setdefault(parent, []) - for directory in sorted(directories, key=lambda p: (p.count(b"/"), len(p)), reverse=True): - tree = git( - destination, "mktree", "-z", input_bytes=b"".join(directories[directory]) - ).strip() - if not directory: - return tree.decode() - parent, _, name = directory.rpartition(b"/") - directories[parent].append(b"040000 tree " + tree + b"\t" + name + b"\0") - raise ReportError("could not construct public generated tree") - - -def create_public_snapshot( - destination: Path, - tree: str, - generation: str, - content_hash: str, - branch: str, - parent: str | None, - adoption: tuple[str, str] | None = None, -) -> str: - target = ( - "next" - if branch == "codegen/castiron/next" or branch.startswith("codegen/castiron/next/") - else "main" - ) - message = ( - "Castiron generated snapshot\n\n" - "Castiron-Public-Codegen-Version: 1\n" - f"Castiron-Public-Codegen-Generation-ID: {generation}\n" - f"Castiron-Public-Codegen-Hash: {content_hash}\n" - f"Castiron-Public-Codegen-Branch: {branch}\n" - f"Castiron-Public-Codegen-Target: {target}\n" - ) - if adoption is not None: - if target != "next": - raise ReportError("adoption snapshot must target next") - message += f"Castiron-Public-Codegen-Adoption-Base: {require_sha(adoption[0])}\n" - message += f"Castiron-Public-Codegen-Adoption-Source: {require_sha(adoption[1])}\n" - parents = ["-p", require_sha(parent)] if parent is not None else [] - snapshot = ( - git( - destination, - "-c", - "user.name=Castiron", - "-c", - "user.email=noreply@openai.com", - "commit-tree", - tree, - *parents, - "-m", - message, - ) - .decode() - .strip() - ) - require_sha(snapshot) - if hash_codegen_commit(destination, snapshot) != content_hash: - raise ReportError("public snapshot content hash differs from private checkpoint") - return snapshot - - -def first_adoption_source(repo: Path, base: str, *, target_branch: str, fetch: bool) -> str: - """Authority is the protected target base policy, never a candidate receipt.""" - base = require_sha(base) - if target_branch != "next": - raise ReportError("first adoption is next-only") - if git(repo, "ls-tree", base, "--", ".castiron.stats.yml"): - raise ReportError("existing/corrupt Castiron stats require ordinary recovery") - if git(repo, "log", "--full-history", "--format=%H", base, "--", ".castiron.stats.yml").strip(): - raise ReportError("deleted Castiron stats cannot authorize first adoption") - if not git(repo, "ls-tree", base, "--", ".stats.yml").startswith(b"100644 blob "): - raise ReportError("first adoption requires a legacy SDK target") - if not git(repo, "ls-tree", base, "--", ".castiron-ratchet.json").startswith(b"100644 blob "): - raise ReportError("first adoption requires an approved target policy") - - def unique(pairs: list[tuple[str, Any]]) -> dict[str, Any]: - result: dict[str, Any] = {} - for key, value in pairs: - if key in result: - raise ReportError("duplicate policy field") - result[key] = value - return result - - policy = json.loads( - git(repo, "show", f"{base}:.castiron-ratchet.json"), object_pairs_hook=unique - ) - pin = policy.get("next_policy") - if not isinstance(pin, dict) or set(cast(dict[str, Any], pin)) != {"repository", "main_sha"}: - raise ReportError("first adoption requires the protected next-policy pin") - pin = cast(dict[str, Any], pin) - repository = pin["repository"] - if repository not in {"openai/openai-java", "openai/openai-cli"}: - raise ReportError("first adoption is only supported for legacy Java/CLI") - remote = git(repo, "remote", "get-url", "origin").decode().strip() - allowed = github_origin_urls(repository) | github_origin_urls(f"{repository}-internal") - if remote not in allowed: - raise ReportError("adoption policy repository does not match SDK origin") - source = require_sha(pin["main_sha"]) - main_ref = "refs/castiron/adoption-main" if fetch else "refs/remotes/origin/main" - if fetch: - git( - repo, - "fetch", - "--quiet", - "--no-tags", - f"https://github.com/{repository}.git", - f"+refs/heads/main:{main_ref}", - ) - if source not in git(repo, "rev-list", "--first-parent", main_ref).decode().splitlines(): - raise ReportError("adoption source is not on authenticated main history") - source_policy = json.loads( - git(repo, "show", f"{source}:.castiron-ratchet.json"), object_pairs_hook=unique - ) - if {key: value for key, value in policy.items() if key != "next_policy"} != source_policy: - raise ReportError("first adoption must preserve approved policy limits") - if not read_stats(repo, source).get("public_codegen_sha"): - raise ReportError( - "approved main must have a genuine public checkpoint before first adoption" - ) - return source - - -def adoption_head( - repo: Path, - base: str, - source: str, - head: str, - after: dict[str, Any], - *, - public: bool, -) -> None: - git(repo, "merge-base", "--is-ancestor", base, head) - if not public: - # Private initialization retains the authenticated source parent. Normal - # promotion deliberately publishes a single-parent target-branch commit. - git(repo, "merge-base", "--is-ancestor", source, head) - if git(repo, "show", f"{base}:.castiron-ratchet.json") != git( - repo, "show", f"{head}:.castiron-ratchet.json" - ): - raise ReportError("first adoption must preserve protected target policy") - original, current = read_stats(repo, source), read_stats(repo, head) - if original["generation_id"] == current["generation_id"]: - raise ReportError("first adoption requires a new generation") - for key in ("config_hash", "openapi_spec_hash"): - if not original.get(key) or current.get(key) != original[key]: - raise ReportError("first adoption must preserve authenticated main inputs") - if public: - # Public authorization comes from the protected pin and verified public - # snapshot ancestry, never from a candidate-supplied adoption receipt. - git(repo, "merge-base", "--is-ancestor", original["public_codegen_sha"], after["commit"]) - metadata = public_metadata(repo, after["commit"]) - if ( - metadata.get("Target") != "next" - or metadata.get("Adoption-Base") != base - or metadata.get("Adoption-Source") != source - ): - raise ReportError("snapshot does not bind the authorized first adoption") - else: - message = git(repo, "show", "-s", "--format=%B", current["codegen_sha"]).decode() - bindings = re.findall(r"^Generation metadata: ([A-Za-z0-9+/=]+)$", message, re.MULTILINE) - if len(bindings) != 1: - raise ReportError("invalid adoption generation binding") - envelope = json.loads(base64.b64decode(bindings[0], validate=True)) - record = envelope.get("record", {}) - if envelope.get("release_target") != "next" or record.get("public_baseline_commit") != base: - raise ReportError("new checkpoint must bind the old target base") - pin = json.loads(git(repo, "show", f"{base}:.castiron-ratchet.json"))["next_policy"] - sdk = pin["repository"].split("/")[1] - if record.get("target") != sdk or record.get("language") != sdk.removeprefix("openai-"): - raise ReportError("adoption generation SDK mismatch") - - -def prepare_public_snapshot( - source: Path, - source_base: str, - source_head: str, - destination: Path, - public_base: str, - branch: str, -) -> dict[str, Any]: - """Prepare public-only generated history; never push anything.""" - after = read_stats(source, source_head) - target_branch = ( - "next" if branch == "castiron/next" or branch.startswith("castiron/next/") else "main" - ) - adoption_source = None - if not git(destination, "ls-tree", public_base, "--", ".castiron.stats.yml"): - adoption_source = first_adoption_source( - destination, public_base, target_branch=target_branch, fetch=True - ) - else: - read_stats(source, source_base) - public = read_stats(destination, adoption_source or public_base) - baseline = resolve_checkpoint( - source, after, fetch=True, require_hash=True, target_branch=target_branch - ) - if adoption_source: - # Public source commit is already fetched into the private promotion clone. - adoption_head(source, public_base, adoption_source, source_head, baseline, public=False) - - codegen_branch = f"codegen/{branch}" - ref = f"refs/heads/{codegen_branch}" - git(destination, "check-ref-format", ref) - remote = git(destination, "ls-remote", "--refs", "origin", ref).decode().splitlines() - previous = require_sha(remote[0].split()[0]) if remote else "" - upgrade_legacy_parent = False - if previous: - git(destination, "fetch", "--quiet", "--no-tags", "origin", previous) - metadata = public_metadata(destination, previous) - # A pending legacy promotion has not been adopted on main. It may be - # retained as public history, but never reused to authorize the new - # snapshot: rebuild that from the verified private checkpoint below. - upgrade_legacy_parent = ( - target_branch == "main" - and "Target" not in metadata - and metadata["Branch"] == codegen_branch - ) - if not upgrade_legacy_parent: - validate_public_target( - destination, previous, metadata, target_branch=target_branch, fetch=True - ) - # Stats pin the snapshot, not the SDK branch that currently uses it. - # Another promotion may legitimately reuse this public commit; its - # original branch remains the locator used by resolve_public_baseline. - if hash_codegen_commit(destination, previous) != metadata["Hash"]: - raise ReportError("existing public codegen branch has invalid provenance") - - base_snapshot = public.get("public_codegen_sha") - if base_snapshot: - base_snapshot = resolve_public_baseline( - destination, - adoption_source or public_base, - fetch=True, - require_hash=False, - target_branch="main" if adoption_source else target_branch, - )["commit"] - elif previous: - try: - base_snapshot = resolve_public_baseline( - destination, - public_base, - fetch=True, - require_hash=False, - lineage=previous, - target_branch=target_branch, - )["commit"] - except ReportError: - base_snapshot = None - if base_snapshot is None: - old = resolve_checkpoint( - source, public, fetch=True, require_hash=False, target_branch=target_branch - ) - # Initial bootstrap is limited to paths already present on public main. - tree = copy_generated_tree(source, old["commit"], destination, public_base) - base_snapshot = create_public_snapshot( - destination, tree, old["generation_id"], old["hash"], codegen_branch, None - ) - - # Normal promotion has already applied the reviewed patch to the public - # index. Private HEAD can contain unchanged private-only paths, so it is - # not an approval boundary for publication. - approved_tree = git(destination, "write-tree").decode().strip() - tree = copy_generated_tree(source, baseline["commit"], destination, public_base, approved_tree) - parent = base_snapshot - if previous: - ancestors = git(destination, "rev-list", previous).decode().splitlines() - if base_snapshot in ancestors: - parent = previous - metadata = public_metadata(destination, previous) - if ( - not upgrade_legacy_parent - and metadata["Generation-ID"] == after["generation_id"] - and ( - not adoption_source - or ( - metadata.get("Adoption-Base") == public_base - and metadata.get("Adoption-Source") == adoption_source - ) - ) - and metadata["Hash"] == baseline["hash"] - and git(destination, "rev-parse", f"{previous}^{{tree}}").decode().strip() == tree - and base_snapshot in ancestors - ): - snapshot = previous - else: - snapshot = create_public_snapshot( - destination, - tree, - after["generation_id"], - baseline["hash"], - codegen_branch, - parent, - (public_base, adoption_source) if adoption_source else None, - ) - elif ( - public_metadata(destination, base_snapshot)["Generation-ID"] == after["generation_id"] - and git(destination, "rev-parse", f"{base_snapshot}^{{tree}}").decode().strip() == tree - ): - snapshot = base_snapshot - else: - snapshot = create_public_snapshot( - destination, - tree, - after["generation_id"], - baseline["hash"], - codegen_branch, - parent, - (public_base, adoption_source) if adoption_source else None, - ) - stats_path = destination / ".castiron.stats.yml" - current = git(source, "show", f"{require_sha(source_head)}:.castiron.stats.yml").decode("utf-8") - lines = [line for line in current.splitlines() if not line.startswith("public_codegen_sha:")] - stats_path.write_text("\n".join(lines) + f"\npublic_codegen_sha: {snapshot}\n") - git(destination, "add", "--", ".castiron.stats.yml") - return { - "commit": snapshot, - "base_commit": base_snapshot, - "branch": codegen_branch, - "previous": previous, - "hash": baseline["hash"], - } - - -def resolve_baseline( - repo: Path, revision: str, *, fetch: bool, require_hash: bool, target_branch: str = "main" -) -> dict[str, Any]: - return resolve_checkpoint( - repo, - read_stats(repo, revision), - fetch=fetch, - require_hash=require_hash, - target_branch=target_branch, - ) - - -def resolve_checkpoint( - repo: Path, - stats: dict[str, str], - *, - fetch: bool, - require_hash: bool, - target_branch: str = "main", -) -> dict[str, Any]: - if target_branch not in TARGET_BRANCHES: - raise ReportError("unsupported report target") - commit = stats["codegen_sha"] - if fetch: - git(repo, "fetch", "--quiet", "--no-tags", "origin", commit) - message = git(repo, "show", "-s", "--format=%B", commit).decode("utf-8") - match = re.search(r"^Generation metadata: ([A-Za-z0-9+/=]+)$", message, re.MULTILINE) - if not match: - raise ReportError("codegen checkpoint has no generation metadata") - try: - decoded: object = json.loads(base64.b64decode(match[1], validate=True)) - except (ValueError, TypeError) as exc: - raise ReportError("invalid codegen checkpoint metadata") from exc - if not isinstance(decoded, dict): - raise ReportError("invalid codegen checkpoint metadata") - metadata = cast(dict[str, Any], decoded) - namespace = "" - if "schema_version" in metadata: - if ( - set(metadata) != {"schema_version", "release_target", "record"} - or metadata["schema_version"] != "castiron-checkpoint/v2" - or metadata["release_target"] != "next" - or not isinstance(metadata["record"], dict) - ): - raise ReportError("invalid versioned checkpoint envelope") - if target_branch != "next": - raise ReportError("next checkpoint cannot authorize a main report") - metadata = cast(dict[str, Any], metadata["record"]) - namespace = "castiron/next/" - if {"release_target", "schema_version", "record"}.intersection(metadata): - raise ReportError("checkpoint target metadata must use the versioned envelope") - if metadata.get("generation_id") != stats["generation_id"]: - raise ReportError("codegen checkpoint generation ID does not match stats") - branch = metadata.get("source_branch") - if not isinstance(branch, str) or not branch: - raise ReportError("codegen checkpoint has no source branch") - if not namespace and (branch == "castiron/next" or branch.startswith("castiron/next/")): - raise ReportError("legacy checkpoint cannot use the reserved next namespace") - ref = f"refs/heads/codegen/{namespace}{branch}" - git(repo, "check-ref-format", ref) - tracking = f"refs/castiron/report/{hashlib.sha256(ref.encode()).hexdigest()}" - if fetch: - git(repo, "fetch", "--quiet", "--no-tags", "origin", f"+{ref}:{tracking}") - else: - tracking = f"refs/remotes/origin/codegen/{namespace}{branch}" - git(repo, "merge-base", "--is-ancestor", commit, tracking) - actual = hash_codegen_commit(repo, commit) - expected = stats.get("codegen_hash") - if expected is None and require_hash: - raise ReportError("head stats do not contain codegen_hash") - if expected is not None and actual != expected: - raise ReportError(f"codegen_hash mismatch: expected {expected}, got {actual}") - return { - "commit": commit, - "generation_id": stats["generation_id"], - "hash": actual, - "hash_recorded": expected is not None, - } - - -def numstats(repo: Path, generated: str, integrated: str) -> dict[bytes, tuple[str, str]]: - result: dict[bytes, tuple[str, str]] = {} - output = git( - repo, - "diff", - "--numstat", - "-z", - "--no-renames", - "--no-ext-diff", - "--no-textconv", - generated, - integrated, - "--", - ) - for record in output.split(b"\0"): - if record: - added, removed, path = record.split(b"\t", 2) - result[path] = (added.decode("ascii"), removed.decode("ascii")) - return result - - -def classify(g0: Entry | None, i0: Entry | None, g1: Entry | None, i1: Entry | None) -> str | None: - before = g0 is not None and g0 != i0 - after = g1 is not None and g1 != i1 - if g0 is not None and g1 is None and (before or i1 is not None): - return "no_longer_generated" - if g0 is None and g1 is not None and i0 is not None: - return "newly_generation_owned" - if not before and not after: - return None - if g0 is not None and g1 is not None and g0 != g1: - return "baseline_changed" - if not before: - return "newly_customized" - if not after: - return "removed" - return "unchanged" if i0 == i1 else "existing_changed" - - -def build_report( - repo: Path, - base: str, - head: str, - *, - fetch: bool = False, - require_head_hash: bool = False, - public: bool = False, - target_branch: str = "main", -) -> tuple[dict[str, Any], bytes]: - base, head = require_sha(base), require_sha(head) - comparison_base = git(repo, "merge-base", base, head).decode().strip() - adoption_source = None - if not git(repo, "ls-tree", comparison_base, "--", ".castiron.stats.yml"): - if comparison_base != base: - raise ReportError( - "first-adoption target advanced; regenerate against the protected base" - ) - adoption_source = first_adoption_source( - repo, comparison_base, target_branch=target_branch, fetch=fetch - ) - if public: - after = resolve_public_baseline( - repo, - head, - fetch=fetch, - require_hash=require_head_hash, - target_branch=target_branch, - ) - before = resolve_public_baseline( - repo, - adoption_source or comparison_base, - fetch=fetch, - require_hash=False, - lineage=after["commit"], - target_branch="main" if adoption_source else target_branch, - ) - else: - before = resolve_baseline( - repo, - adoption_source or comparison_base, - fetch=fetch, - require_hash=False, - target_branch="main" if adoption_source else target_branch, - ) - after = resolve_baseline( - repo, - head, - fetch=fetch, - require_hash=require_head_hash, - target_branch=target_branch, - ) - if adoption_source: - adoption_head(repo, comparison_base, adoption_source, head, after, public=public) - g0, i0 = ( - ({}, {}) - if adoption_source - else (tree_entries(repo, before["commit"]), tree_entries(repo, comparison_base)) - ) - g1, i1 = tree_entries(repo, after["commit"]), tree_entries(repo, head) - stats = numstats(repo, after["commit"], head) - files: list[dict[str, Any]] = [] - current_paths: list[str] = [] - for path in sorted(g0.keys() | g1.keys()): - category = classify(g0.get(path), i0.get(path), g1.get(path), i1.get(path)) - if category is None: - continue - custom_before = path in g0 and g0[path] != i0.get(path) - custom_after = path in g1 and g1[path] != i1.get(path) - added, removed = stats.get(path, ("0", "0")) if custom_after else ("0", "0") - files.append( - { - "path": path.decode("utf-8"), - "category": ( - ("first_adoption" if custom_after else "unchanged") - if adoption_source - else category - ), - "custom_before": None if adoption_source else custom_before, - "custom_after": custom_after, - "added": added, - "removed": removed, - } - ) - if custom_after: - current_paths.append(path.decode("utf-8")) - patch = b"" - if current_paths: - patch = git( - repo, - "--literal-pathspecs", - "diff", - "--binary", - "--full-index", - "--no-renames", - "--no-ext-diff", - "--no-textconv", - "--no-color", - after["commit"], - head, - "--", - *current_paths, - ) - return { - "schema_version": 1, - "status": "ok", - "public": public, - "target_base_sha": base, - "base_sha": comparison_base, - "head_sha": head, - "before": None if adoption_source else before, - "adoption_baseline": before if adoption_source else None, - "adoption_source": adoption_source, - "before_status": "legacy_generation_unavailable" if adoption_source else "verified", - "after": after, - "files": files, - "counts": { - "before": None if adoption_source else sum(f["custom_before"] for f in files), - "after": sum(f["custom_after"] for f in files), - **dict(collections.Counter(f["category"] for f in files)), - }, - }, patch - - -def render_report( - report: dict[str, Any], - run_url: str = "", - *, - repository: str = "", - run_id: int = 0, - run_attempt: int = 0, -) -> str: - head = require_sha(report["head_sha"]) - lines = [MARKER, "", "## Castiron custom code", ""] - branch = report.get("target_branch", "main") - if branch not in TARGET_BRANCHES: - raise ReportError("unsupported report target") - lines.extend([f"Evaluated {branch}: `{require_sha(report['target_base_sha'])}`.", ""]) - if "policy_sha" in report: - lines.extend([f"Trusted main policy: `{require_sha(report['policy_sha'])}`.", ""]) - if report.get("status") != "ok": - reason = html.escape(str(report.get("error", "Report could not be computed"))) - lines.extend([f"⚠️ Report unavailable for `{head[:12]}`.", "", reason]) - else: - files = report["files"] - counts: collections.Counter[str] = collections.Counter() - for file in files: - if file["category"] not in LABELS: - raise ReportError("unknown report category") - counts[file["category"]] += 1 - before = sum(bool(file["custom_before"]) for file in files) - after = sum(bool(file["custom_after"]) for file in files) - base = require_sha(report["base_sha"]) - newly_customized = sum( - bool(file["custom_after"]) and not file["custom_before"] for file in files - ) - removed = sum( - bool(file["custom_before"]) - and not file["custom_after"] - and file["category"] != "no_longer_generated" - for file in files - ) - if report.get("before_status") == "legacy_generation_unavailable": - lines.extend( - [ - "Legacy before-generation unavailable; no historical customization claim.", - "", - f"Current mixed files: {after}. Approved main reference: `{require_sha(report['adoption_source'])}`.", - ] - ) - elif newly_customized == 0: - remaining = ( - f"{after} mixed file{'s' if after != 1 else ''} remain{'s' if after == 1 else ''}" - ) - changed = counts["existing_changed"] - details = [f"{changed} existing customization{'s' if changed != 1 else ''} changed"] - if removed: - details.append(f"{removed} customizations removed") - if counts["baseline_changed"]: - details.append(f"{counts['baseline_changed']} generated baselines changed") - lines.extend( - [ - "✅ No new custom-code files detected.", - "", - remaining + "; " + "; ".join(details) + ".", - ] - ) - else: - lines.extend( - [ - f"**Mixed files: {before} → {after}**", - "", - ( - f"{newly_customized} newly customized · {removed} customizations removed · " - f"{counts['existing_changed']} existing customizations changed · {counts['baseline_changed']} generated baselines changed" - ), - ] - ) - verification = ( - "Current generation and approved main reference verified." - if report["before"] is None - else "Generated baselines verified." - ) - lines.extend(["", f"Compared `{base[:12]}` → `{head[:12]}`. {verification}"]) - if ( - not (report["before"] or report["adoption_baseline"])["hash_recorded"] - or not report["after"]["hash_recorded"] - ): - lines.extend( - [ - "", - "Legacy baseline: content hash computed from the pinned checkpoint; not yet recorded in that revision’s stats.", - ] - ) - interesting = [file for file in files if file["category"] != "unchanged"] - if interesting: - lines.extend(["", "| File | Result | Current custom patch |", "| --- | --- | --- |"]) - for file in interesting[:40]: - path = ( - html.escape(str(file["path"])) - .replace("|", "|") - .replace("\n", " ") - .replace("\r", " ") - ) - added, removed = str(file["added"]), str(file["removed"]) - if not re.fullmatch(r"\d+|-", added) or not re.fullmatch(r"\d+|-", removed): - raise ReportError("invalid line counts") - delta = f"+{added} / −{removed}" if file["custom_after"] else "None" - lines.append(f"| {path} | {LABELS[file['category']]} | {delta} |") - if len(interesting) > 40: - lines.extend( - [ - "", - f"{len(interesting) - 40} more affected files are listed in the full report.", - ] - ) - if counts["unchanged"]: - lines.extend( - [ - "", - f"
{counts['unchanged']} existing customizations unchanged", - "", - ] - ) - for file in [file for file in files if file["category"] == "unchanged"][:40]: - lines.append(f"- {html.escape(str(file['path']))}") - if counts["unchanged"] > 40: - lines.extend(["", f"{counts['unchanged'] - 40} more in the full report."]) - lines.extend(["", "
"]) - lines.extend( - [ - "", - "A changed generated baseline means this report cannot reliably identify which handwritten lines changed.", - ] - ) - target_base = require_sha(report["target_base_sha"]) - lines.extend(["", "
Inspect the custom-code diff", ""]) - if repository: - if not REPOSITORY.fullmatch(repository) or min(run_id, run_attempt) <= 0: - raise ReportError("invalid artifact download target") - artifact = f"castiron-custom-code-{run_id}-{run_attempt}" - lines.extend( - [ - "Download the exact patch produced by this run (requires repository access):", - "", - "```sh", - f"gh run download {run_id} --repo {repository} \\", - f" --name {artifact} --dir /tmp/{artifact}", - f"git apply --stat /tmp/{artifact}/custom-code.patch", - f"cat /tmp/{artifact}/custom-code.patch", - "```", - "", - ] - ) - lines.extend( - [ - "Or reproduce it from an SDK checkout containing the vendored reporter:", - "", - "```sh", - f"git fetch --no-tags origin {target_base} {head}", - "python3 scripts/castiron/custom_code_report.py report \\", - f" --base {target_base} \\", - f" --head {head} --target-branch {branch} --fetch --require-head-hash{' --public' if report.get('public') else ''} \\", - f" --out /tmp/castiron-custom-code-{head[:12]}", - f"cat /tmp/castiron-custom-code-{head[:12]}/custom-code.patch", - "```", - "", - "This is the **current full custom patch** for mixed files, not an attribution of only the handwritten lines changed by this PR.", - "", - "
", - ] - ) - if run_url: - lines.extend(["", f"[Full report and patch]({run_url})"]) - return "\n".join(lines) + "\n" - - -def api(method: str, path: str, payload: dict[str, Any] | None = None) -> Any: - token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") - if not token: - raise ReportError("GitHub token is required for comment publication") - args = ["gh", "api", "--hostname", "github.com", "--method", method, path] - if payload is not None: - args.extend(["--input", "-"]) - result = subprocess.run( - args, - input=json.dumps(payload).encode() if payload is not None else None, - capture_output=True, - check=False, - timeout=60, - ) - if result.returncode: - raise ReportError( - f"GitHub API request failed: {result.stderr.decode(errors='replace').strip()}" - ) - return json.loads(result.stdout) - - -def associated_pulls( - repository: str, run: dict[str, Any], *, branch_only: bool = False -) -> list[dict[str, Any]]: - root = f"repos/{repository}" - associated: list[dict[str, Any]] = ( - [] - if branch_only - else run["pull_requests"] - or api("GET", f"{root}/commits/{require_sha(run['head_sha'])}/pulls?per_page=100") - ) - if associated: - return associated - # Fork runs can be absent from both association endpoints. Discover candidates - # by branch; callers still verify the current PR head and target through GitHub. - source: dict[str, Any] = run.get("head_repository") or {} - if not ( - source.get("id") - and source.get("full_name") - and source.get("owner", {}).get("login") - and run.get("head_branch") - ): - return [] - head = f"{source['owner']['login']}:{run['head_branch']}" - for page in range(1, 101): - query = urlencode({"state": "open", "head": head, "per_page": 100, "page": page}) - pulls = api("GET", f"{root}/pulls?{query}") - associated.extend(pulls) - if len(pulls) < 100: - return associated - raise ReportError("too many pull requests for source branch") - - -def matches_source(pull: dict[str, Any], run: dict[str, Any]) -> bool: - """A shared commit is discovery metadata, not authority over another PR.""" - source: dict[str, Any] = run.get("head_repository") or {} - head: dict[str, Any] = pull.get("head") or {} - repository: dict[str, Any] = head.get("repo") or {} - return bool( - source.get("id") - and source.get("full_name") - and run.get("head_branch") - and repository.get("id") == source["id"] - and repository.get("full_name") == source["full_name"] - and head.get("ref") == run["head_branch"] - and head.get("sha") == run["head_sha"] - ) - - -def current_pull(repository: str, run: dict[str, Any]) -> tuple[int, str] | None: - """Resolve one live supported target; association metadata is discovery only.""" - current: list[tuple[int, str]] = [] - pulls = associated_pulls(repository, run) - for attempt in range(2): - for number in sorted({int(pr["number"]) for pr in pulls}): - if number <= 0: - raise ReportError("invalid associated pull request") - pull = api("GET", f"repos/{repository}/pulls/{number}") - if ( - pull["state"] == "open" - and matches_source(pull, run) - and pull["base"]["repo"]["full_name"] == repository - and pull["base"]["ref"] in TARGET_BRANCHES - ): - current.append((number, pull["base"]["ref"])) - if current or run["pull_requests"] or attempt: - break - # Nonempty commit associations can contain only unrelated PRs. - pulls = associated_pulls(repository, run, branch_only=True) - # Fallback discovery can find both targets for one head. Main's required - # checks take precedence over next's advisory report; keep same-target - # ambiguity fail-closed. Explicit run associations still limit discovery. - current = [pull for pull in current if pull[1] == "main"] or current - if len(current) > 1: - raise ReportError("workflow run has multiple current pull requests") - return current[0] if current else None - - -def publish_comment( - report: dict[str, Any], - repository: str, - number: int, - run_id: int, - run_attempt: int, - *, - artifact_run_id: int = 0, - artifact_run_attempt: int = 0, -) -> str: - if not REPOSITORY.fullmatch(repository) or min(number, run_id, run_attempt) <= 0: - raise ReportError("invalid GitHub publication target") - root = f"repos/{repository}" - pull = api("GET", f"{root}/pulls/{number}") - if ( - pull["state"] != "open" - or pull["head"]["sha"] != report["head_sha"] - or pull["base"]["ref"] != report.get("target_branch", "main") - or pull["base"]["ref"] not in TARGET_BRANCHES - or pull["base"]["repo"]["full_name"] != repository - ): - return "Skipped stale report" - run = api("GET", f"{root}/actions/runs/{run_id}") - if ( - run["event"] != "pull_request" - or run.get("path", "").split("@", 1)[0] != ".github/workflows/castiron-custom-code.yml" - or run["head_sha"] != report["head_sha"] - ): - raise ReportError("workflow run does not match report PR/head") - if run["run_attempt"] != run_attempt or not matches_source(pull, run): - return "Skipped stale report" - if current_pull(repository, run) != (number, pull["base"]["ref"]): - raise ReportError("workflow run does not match report PR/head") - artifact_run_id = artifact_run_id or run_id - artifact_run_attempt = artifact_run_attempt or run_attempt - body = render_report( - report, - f"https://github.com/{repository}/actions/runs/{artifact_run_id}", - repository=repository, - run_id=artifact_run_id, - run_attempt=artifact_run_attempt, - ) - body += f"\n\n" - found = None - for page in range(1, 101): - comments = api("GET", f"{root}/issues/{number}/comments?per_page=100&page={page}") - for comment in comments: - if comment["user"]["login"] == "github-actions[bot]" and comment["body"].startswith( - MARKER - ): - found = comment - break - if found is not None or len(comments) < 100: - break - if found is not None: - previous = RUN_MARKER.search(found["body"]) - if previous and tuple(map(int, previous.groups())) > (run_id, run_attempt): - return "Skipped stale report" - if found["body"] == body: - return str(found["html_url"]) - # The workflow serializes publishers; recheck after pagination before writing. - pull = api("GET", f"{root}/pulls/{number}") - if ( - pull["state"] != "open" - or pull["head"]["sha"] != report["head_sha"] - or pull["base"]["ref"] != report.get("target_branch", "main") - or pull["base"]["ref"] not in TARGET_BRANCHES - or pull["base"]["repo"]["full_name"] != repository - or not matches_source(pull, run) - ): - return "Skipped stale report" - if found is not None: - result = api("PATCH", f"{root}/issues/comments/{found['id']}", {"body": body}) - else: - result = api("POST", f"{root}/issues/{number}/comments", {"body": body}) - return str(result["html_url"]) - - -def write_report( - repo: Path, - base: str, - head: str, - out: Path, - *, - fetch: bool, - require_head_hash: bool, - public: bool, - target_branch: str = "main", - policy_sha: str | None = None, -) -> dict[str, Any]: - out.mkdir(parents=True, exist_ok=True) - try: - report, patch = build_report( - repo, - base, - head, - fetch=fetch, - require_head_hash=require_head_hash, - public=public, - target_branch=target_branch, - ) - except (ReportError, UnicodeError, KeyError, ValueError) as exc: - report = { - "schema_version": 1, - "status": "error", - "target_base_sha": require_sha(base), - "head_sha": require_sha(head), - "error": str(exc), - } - patch = b"" - report["target_branch"] = target_branch - if policy_sha is not None: - report["policy_sha"] = require_sha(policy_sha) - (out / "report.json").write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") - (out / "custom-code.patch").write_bytes(patch) - summary = render_report(report) - (out / "summary.md").write_text(summary) - sys.stdout.write(summary) - return report - - -def trusted_report( - repo: Path, repository: str, run_id: int, run_attempt: int, out: Path, *, base: str -) -> None: - """Recompute from GitHub-associated Git objects, never from PR-produced artifacts.""" - # Executable code and policy always come from the pinned main checkout. - # The target branch is independently resolved from live GitHub metadata. - policy_sha = require_sha(base) - if not REPOSITORY.fullmatch(repository) or min(run_id, run_attempt) <= 0: - raise ReportError("invalid GitHub report target") - root = f"repos/{repository}" - run = api("GET", f"{root}/actions/runs/{run_id}") - if ( - run["event"] != "pull_request" - or run.get("path", "").split("@", 1)[0] != ".github/workflows/castiron-custom-code.yml" - or run["status"] != "completed" - ): - raise ReportError("unexpected source workflow run") - if run["run_attempt"] != run_attempt: - return - head = require_sha(run["head_sha"]) - current = current_pull(repository, run) - if current is None: - return - number, branch = current - metadata = api("GET", root) - if metadata["default_branch"] != "main" or run["repository"]["full_name"] != repository: - raise ReportError("trusted reports require the repository's main default branch") - public = not metadata["private"] - base = ( - policy_sha - if branch == "main" - else require_sha(api("GET", f"{root}/git/ref/heads/{branch}")["object"]["sha"]) - ) - # This must be a new, bare repository: no PR worktree, hooks, configuration, - # submodules, or Python imports can affect the trusted reporter. - repo.mkdir() - git(repo, "init", "--quiet", "--bare") - git(repo, "remote", "add", "origin", f"https://github.com/{repository}.git") - git(repo, "fetch", "--quiet", "--no-tags", "origin", *dict.fromkeys((base, head, policy_sha))) - write_report( - repo, - base, - head, - out, - fetch=True, - require_head_hash=True, - public=public, - target_branch=branch, - policy_sha=policy_sha, - ) - (out / "context.json").write_text( - json.dumps({"pr": number, "repository": repository, "run": run_id, "attempt": run_attempt}) - + "\n" - ) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="command", required=True) - hashing = commands.add_parser("hash") - hashing.add_argument("--repo", type=Path, default=Path.cwd()) - hashing.add_argument("--commit", required=True) - reporting = commands.add_parser("report") - reporting.add_argument("--repo", type=Path, default=Path.cwd()) - reporting.add_argument("--base", required=True) - reporting.add_argument("--head", required=True) - reporting.add_argument("--out", type=Path, required=True) - reporting.add_argument("--fetch", action="store_true") - reporting.add_argument("--require-head-hash", action="store_true") - reporting.add_argument("--public", action="store_true") - reporting.add_argument("--target-branch", choices=sorted(TARGET_BRANCHES), default="main") - trusted = commands.add_parser("trusted-report") - trusted.add_argument("--repo", type=Path, required=True) - trusted.add_argument("--repository", required=True) - trusted.add_argument("--run-id", type=int, required=True) - trusted.add_argument("--run-attempt", type=int, required=True) - trusted.add_argument("--out", type=Path, required=True) - trusted.add_argument("--base", required=True, help="immutable main SHA of the trusted checkout") - selecting = commands.add_parser("promotion-base") - selecting.add_argument("--repo", type=Path, required=True) - selecting.add_argument("--base", required=True) - selecting.add_argument("--branch", required=True) - preparing = commands.add_parser("prepare-public") - preparing.add_argument("--source-repo", type=Path, required=True) - preparing.add_argument("--source-base", required=True) - preparing.add_argument("--source-head", required=True) - preparing.add_argument("--public-repo", type=Path, required=True) - preparing.add_argument("--public-base", required=True) - preparing.add_argument("--branch", required=True) - commenting = commands.add_parser("comment") - commenting.add_argument("--report", type=Path, required=True) - commenting.add_argument("--repository", required=True) - commenting.add_argument("--pr", type=int, required=True) - commenting.add_argument("--run-id", type=int, required=True) - commenting.add_argument("--run-attempt", type=int, required=True) - commenting.add_argument("--artifact-run-id", type=int, default=0) - commenting.add_argument("--artifact-run-attempt", type=int, default=0) - args = parser.parse_args() - try: - if args.command == "hash": - sys.stdout.write(hash_codegen_commit(args.repo, args.commit) + "\n") - elif args.command == "promotion-base": - base = require_sha(args.base) - if not git(args.repo, "ls-tree", base, "--", ".castiron.stats.yml"): - target = ( - "next" - if args.branch == "castiron/next" or args.branch.startswith("castiron/next/") - else "main" - ) - base = first_adoption_source(args.repo, base, target_branch=target, fetch=True) - sys.stdout.write(base + "\n") - elif args.command == "prepare-public": - sys.stdout.write( - json.dumps( - prepare_public_snapshot( - args.source_repo, - args.source_base, - args.source_head, - args.public_repo, - args.public_base, - args.branch, - ) - ) - + "\n" - ) - elif args.command == "trusted-report": - trusted_report( - args.repo, - args.repository, - args.run_id, - args.run_attempt, - args.out, - base=args.base, - ) - elif args.command == "comment": - if args.report.stat().st_size > 5_000_000: - raise ReportError("report artifact is too large") - report = json.loads(args.report.read_text()) - sys.stdout.write( - publish_comment( - report, - args.repository, - args.pr, - args.run_id, - args.run_attempt, - artifact_run_id=args.artifact_run_id, - artifact_run_attempt=args.artifact_run_attempt, - ) - + "\n" - ) - else: - report = write_report( - args.repo, - args.base, - args.head, - args.out, - fetch=args.fetch, - require_head_hash=args.require_head_hash, - public=args.public, - target_branch=args.target_branch, - ) - return 0 if report["status"] == "ok" else 1 - except (ReportError, OSError, subprocess.TimeoutExpired) as exc: - sys.stderr.write(f"Castiron custom-code report: {exc}\n") - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/castiron/custom_code_test_support.py b/scripts/castiron/custom_code_test_support.py deleted file mode 100644 index 812ec45ac..000000000 --- a/scripts/castiron/custom_code_test_support.py +++ /dev/null @@ -1,116 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -"""Small Git/checkpoint fixture shared by the offline Castiron tests.""" - -from __future__ import annotations - -import base64 -import json -import subprocess -import tempfile -import unittest -from pathlib import Path -from typing import Any, cast - -import custom_code_report as report - -GENERATION = "550e8400-e29b-41d4-a716-446655440000" - - -class GitTestCase(unittest.TestCase): - def setUp(self) -> None: # pyright: ignore[reportImplicitOverride] - self.temporary = tempfile.TemporaryDirectory() - self.addCleanup(self.temporary.cleanup) - self.repo = Path(self.temporary.name) - self.git("init", "-q", "-b", "main") - self.git("config", "user.name", "Castiron test") - self.git("config", "user.email", "castiron@example.test") - # Finish automatic maintenance before the temporary repository is removed. - self.git("config", "maintenance.autoDetach", "false") - self.git("config", "gc.autoDetach", "false") - - def git(self, *args: str) -> str: - return subprocess.run( - ["git", "-C", str(self.repo), *args], - check=True, - capture_output=True, - text=True, - ).stdout.strip() - - def write(self, path: str, body: str) -> None: - target = self.repo / path - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(body) - - def commit(self, message: str = "fixture") -> str: - self.git("add", "-A") - self.git("commit", "-q", "--allow-empty", "-m", message) - return self.git("rev-parse", "HEAD") - - def baseline(self) -> tuple[str, str]: - self.write("generated.py", "generated\n") - metadata = { - "generation_id": GENERATION, - "source_branch": "test", - "target": "openai-python", - "language": "python", - } - encoded = base64.b64encode(json.dumps(metadata).encode()).decode() - generated = self.commit(f"codegen\n\nGeneration metadata: {encoded}") - self.git("update-ref", "refs/remotes/origin/codegen/test", generated) - self.write( - ".castiron.stats.yml", - f"schema_version: 1\ngeneration_id: {GENERATION}\ncodegen_sha: {generated}\ncodegen_hash: {report.hash_codegen_commit(self.repo, generated)}\n", - ) - return generated, self.commit("integrated") - - -def source_run(head: str, event: str = "pull_request") -> dict[str, Any]: - return { - "id": 123, - "workflow_id": 456, - "event": event, - "head_sha": head, - "head_branch": "gh-readonly-queue/main/pr-7-example" if event == "merge_group" else "sdk", - "repository": {"full_name": "openai/example"}, - "head_repository": { - "id": 10, - "full_name": "contributor/example", - "owner": {"login": "contributor"}, - }, - "path": ".github/workflows/castiron-custom-code.yml", - "status": "completed", - "run_attempt": 1, - "pull_requests": [{"number": 3}] if event == "pull_request" else [], - "conclusion": "failure", # The candidate's result is deliberately ignored. - } - - -def source_head(sha: str) -> dict[str, Any]: - return {"sha": sha, "ref": "sdk", "repo": source_run(sha)["head_repository"]} - - -WORKFLOW = ( - Path(__file__).resolve().parents[2] / ".github/workflows/castiron-custom-code-comment.yml" -) - - -def run_publisher(step: str, payload: dict[str, Any]) -> list[dict[str, Any]]: - section = WORKFLOW.read_text().split(f"- name: {step}\n", 1)[1] - script = section.split(" script: |\n", 1)[1] - lines: list[str] = [] - for line in script.splitlines(): - if line and not line.startswith(" "): - break - lines.append(line[12:]) - return run_workflow_script("\n".join(lines), payload) - - -def run_workflow_script(script: str, payload: dict[str, Any]) -> list[dict[str, Any]]: - output = subprocess.run( - ["node", str(Path(__file__).with_name("fixtures") / "github_publisher.cjs")], - input=json.dumps({**payload, "script": script}), - text=True, - capture_output=True, - check=True, - ) - return cast(list[dict[str, Any]], json.loads(output.stdout)) diff --git a/scripts/castiron/fixtures/github_publisher.cjs b/scripts/castiron/fixtures/github_publisher.cjs deleted file mode 100644 index 58324d249..000000000 --- a/scripts/castiron/fixtures/github_publisher.cjs +++ /dev/null @@ -1,183 +0,0 @@ -// File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -// Execute the real workflow script with an offline GitHub API and capture writes. -const fs = require('node:fs'); -const data = JSON.parse(fs.readFileSync(0, 'utf8')); -const writes = []; -const github = { - rest: { - actions: { - getWorkflowRun: async ({ run_id }) => { - if (data.run_errors?.[run_id]) - throw Object.assign(new Error('Workflow run lookup failed'), { - status: data.run_errors[run_id], - }); - return { data: data.runs_by_id?.[run_id] || data.run || data.context?.payload?.workflow_run }; - }, - getWorkflow: async ({ workflow_id }) => ({ - data: { - id: workflow_id === 'castiron-custom-code-comment.yml' ? 789 : 456, - path: `.github/workflows/${workflow_id}`, - }, - }), - listWorkflowRuns: 'workflow-runs', - reRunWorkflow: async (value) => { - if (data.fail_rerun) throw new Error('Simulated dispatch failure'); - writes.push({ operation: 'rerun', ...value }); - }, - }, - pulls: { - get: async ({ pull_number }) => ({ - data: data.current_sequence?.shift() || data.pulls_by_number?.[pull_number] || data.current, - }), - list: 'pulls', - listReviews: 'reviews', - listFiles: 'files', - }, - git: { - getRef: async ({ ref }) => ({ - data: { - object: { - sha: ref.startsWith('pull/') - ? data.merge_sha - : ref === 'heads/main' && (data.main_sha_sequence?.length || data.main_sha) - ? data.main_sha_sequence?.shift() || data.main_sha - : ref === 'heads/next' && (data.target_sha_sequence?.length || data.target_sha) - ? data.target_sha_sequence?.shift() || data.target_sha - : data.current.base.sha, - }, - }, - }), - }, - repos: { - getContent: async ({ path, ref }) => { - if ( - !['scripts/castiron/custom_code_policy.py', 'scripts/castiron/custom_code_report.py'].includes(path) - ) - throw new Error('Unexpected trusted policy path'); - const content = Buffer.from(JSON.stringify({ ref, path })); - return { - data: { - type: 'file', - encoding: 'base64', - size: content.length, - content: content.toString('base64'), - ...(data.policy_file_override || {}), - }, - }; - }, - getCommit: async () => ({ data: data.merge_commit }), - compareCommitsWithBasehead: async ({ basehead }) => ({ - data: data.comparisons?.[basehead] || { - merge_base_commit: { sha: basehead.split('...')[0] }, - }, - }), - createCommitStatus: async (value) => { - writes.push(value); - data.statuses_by_sha ||= {}; - data.statuses_by_sha[value.sha] = [ - { ...value, creator: { login: 'github-actions[bot]' } }, - ...(data.statuses_by_sha[value.sha] || data.previous_statuses || []), - ]; - }, - listCommitStatusesForRef: 'statuses', - listPullRequestsAssociatedWithCommit: 'associations', - }, - issues: { - listComments: 'comments', - createComment: async (value) => writes.push({ operation: 'create', ...value }), - updateComment: async (value) => writes.push({ operation: 'update', ...value }), - }, - }, - graphql: async (_query, { branch, cursor, number }) => { - if (_query.includes('reviews(first:100')) { - const page = data.review_pages?.shift(); - if (page && page.expected_cursor !== cursor) throw new Error('Wrong review cursor'); - if (page?.after_read) Object.assign(data, page.after_read); - return { - repository: { - pullRequest: { - reviews: page || { - nodes: data.review_nodes_by_number?.[number] || data.review_nodes || [], - pageInfo: { hasNextPage: false, endCursor: null }, - }, - }, - }, - }; - } - if (data.forbid_queue) throw new Error('Unexpected queue discovery'); - if ((branch || 'next') !== data.current.base.ref) throw new Error('Wrong queue target'); - const page = data.queue_pages?.shift(); - if (page && page.expected_cursor !== cursor) throw new Error('Wrong queue cursor'); - return { - repository: { - mergeQueue: { - entries: page || { - nodes: - data.queue_nodes || (data.queue_absent ? [] : [{ headCommit: { oid: data.run.head_sha } }]), - pageInfo: { hasNextPage: false, endCursor: null }, - }, - }, - }, - }; - }, - paginate: async (method, params) => { - if (method === 'files') return data.files || [{ filename: '.castiron-ratchet.json' }]; - if (method === 'workflow-runs') return data.runs_by_sha?.[params.head_sha] || []; - if (method === 'reviews') return data.reviews_by_number?.[params.pull_number] || data.reviews || []; - if (method === 'statuses') return data.statuses_by_sha?.[params.ref] || data.previous_statuses || []; - if (method === 'comments') return data.comments_by_number?.[params.issue_number] || data.comments || []; - if (method === 'associations') return data.associated_pulls || []; - if (method === 'pulls' && params.head === 'contributor:sdk' && params.state === 'open') { - return data.fallback_pulls || []; - } - throw new Error(`Unexpected GitHub lookup: ${method}`); - }, -}; -const exec = { - getExecOutput: async (command, args) => { - if ( - command !== 'python3' || - args[0] !== '-I' || - require('node:path').basename(args[1]) !== 'custom_code_policy.py' || - args[2] !== 'review-required' || - args[3] !== '--repository' || - args[4] !== 'openai/example' || - args[5] !== '--repo' || - args[7] !== '--base' || - args[9] !== '--head' || - args[11] !== '--main' || - ![args[8], args[10], args[12]].every((value) => /^[0-9a-f]{40}$/.test(value)) - ) { - throw new Error('Unexpected trusted policy invocation'); - } - for (const name of ['custom_code_policy.py', 'custom_code_report.py']) { - const file = require('node:path').join(require('node:path').dirname(args[1]), name); - const source = JSON.parse(fs.readFileSync(file, 'utf8')); - if (source.ref !== args[12] || source.path !== `scripts/castiron/${name}`) - throw new Error('Executed stale or mismatched trusted policy'); - } - if (data.policy_error_sequence?.shift() || data.policy_error) - throw new Error('Trusted policy evaluation failed'); - const expectedBase = data.expected_policy_bases?.shift() || data.expected_policy_base; - if (expectedBase && args[8] !== expectedBase) throw new Error('Evaluated stale target policy'); - return { - stdout: JSON.stringify({ - requires_human: data.requires_human_by_main?.[args[12]] ?? data.requires_human ?? true, - }), - }; - }, -}; -const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor; -new AsyncFunction('github', 'context', 'process', 'require', 'exec', data.script)( - github, - data.context, - { env: { RUNNER_TEMP: require('node:os').tmpdir(), ...(data.env || {}) } }, - require, - exec, -) - .then(() => process.stdout.write(JSON.stringify(writes))) - .catch((error) => { - if (data.capture_writes) fs.writeFileSync(data.capture_writes, JSON.stringify(writes)); - console.error(error); - process.exitCode = 1; - }); diff --git a/scripts/castiron/generated-files.json b/scripts/castiron/generated-files.json index 9dfaab887..9f29d8fbd 100644 --- a/scripts/castiron/generated-files.json +++ b/scripts/castiron/generated-files.json @@ -1,14 +1,6 @@ { "schema_version": 1, "files": { - ".github/workflows/castiron-custom-code-comment.yml": { - "mode": "100644", - "sha256": "ec43a2286a8f61efc2bfade540b4ccf572aac51fe0d2796121700f64f5bfce9b" - }, - ".github/workflows/castiron-custom-code.yml": { - "mode": "100644", - "sha256": "b52dcb3feb424790aff1f6f7cff7f385bd921369257b8e59be2e3860ec1f4926" - }, "openai-java-core/src/main/kotlin/com/openai/client/OpenAIClient.kt": { "mode": "100644", "sha256": "0a107019cf0c4db425334816a0741aab1c63513639edfc239179e68b8fbd0981" @@ -15956,74 +15948,6 @@ "openai-java-core/src/test/kotlin/com/openai/services/blocking/webhooks/EventTypeServiceTest.kt": { "mode": "100644", "sha256": "2f623acbe84aca90d39c9f5e011086df7a8f6922495147972826c2d2931bbd54" - }, - "scripts/castiron/CUSTOM_CODE.md": { - "mode": "100644", - "sha256": "c2b12719937fa007d65bea7429962f2b674c531a8c284cc6e5e51c2a526ae351" - }, - "scripts/castiron/README.md": { - "mode": "100644", - "sha256": "4aff82cb988d01babde79902ae20de842fdd4b2b447d5f4cb3c5ce0a36d6fab8" - }, - "scripts/castiron/custom_code_budget.py": { - "mode": "100755", - "sha256": "b85ae4eab2a84dfbcd1f197f87077f686bc4f3347dbf74cca3abb1f99ab50b9a" - }, - "scripts/castiron/custom_code_policy.py": { - "mode": "100755", - "sha256": "890ecfc963b8a2461e7e801c9071735a975ed9876de89c80323da10960965295" - }, - "scripts/castiron/custom_code_report.py": { - "mode": "100755", - "sha256": "ddb15c9a162d26a6c9abf1b4fefcd80f2e685d9d255af729863258b7ecd08e2d" - }, - "scripts/castiron/custom_code_test_support.py": { - "mode": "100644", - "sha256": "b90c31b6b7347a3bc254b9fec228d5ccad3025306c60465a027a239420276671" - }, - "scripts/castiron/fixtures/github_publisher.cjs": { - "mode": "100644", - "sha256": "c84bd86518a70a5b7bbf877d8c13129debface0c220e5118510b0ef17d9ab707" - }, - "scripts/castiron/ruff.toml": { - "mode": "100644", - "sha256": "933d12c5582988097578a13689e1f7c877c36baf2c8c0e7123393e272ca023bd" - }, - "scripts/castiron/test_custom_code_budget.py": { - "mode": "100644", - "sha256": "7fdb3a9282978e6b1c0d84d73c43bb4adf96ff035663e77bc8e45f0fe082be7d" - }, - "scripts/castiron/test_custom_code_github.py": { - "mode": "100644", - "sha256": "815e8db60dd81f69895696813f6b612856eb899803e3e9b85df22ec688073b87" - }, - "scripts/castiron/test_custom_code_policy.py": { - "mode": "100644", - "sha256": "fd503013d604cf9659df95a3a47cecfc17da364397625d95ef6121aa790e6a05" - }, - "scripts/castiron/test_custom_code_policy_source.py": { - "mode": "100644", - "sha256": "dcb0be5e8157791dc02d1ae1d6b82cc703d4bc7b3c4797bc00bd9d2672ad13a2" - }, - "scripts/castiron/test_custom_code_publication.py": { - "mode": "100644", - "sha256": "4533c8c852d590446180715adbd00a63c12af29c53316d3e91d129890ccadbc6" - }, - "scripts/castiron/test_custom_code_report.py": { - "mode": "100644", - "sha256": "e1fbafb5242cac6128757339d78902b12e7658c82e5f157cc2ca7bef80347838" - }, - "scripts/castiron/test_custom_code_review_revocation.py": { - "mode": "100644", - "sha256": "d7f309e2324f48f62f9f7f90c2e82f31354e414a20e3ddb0da724b60adeff7a3" - }, - "scripts/castiron/test_custom_code_status_publication.py": { - "mode": "100644", - "sha256": "7398877d4ec209d0cd40e0d83f8e4482b6de5bef95f1711c9905d6a97a3791d3" - }, - "scripts/castiron/test_custom_code_strict.py": { - "mode": "100644", - "sha256": "20684147ec37f8233798a46aef8b618a2bbf2c8b507605a959d644fce163d73d" } } } diff --git a/scripts/castiron/ruff.toml b/scripts/castiron/ruff.toml deleted file mode 100644 index 1e8a30126..000000000 --- a/scripts/castiron/ruff.toml +++ /dev/null @@ -1,3 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -line-length = 100 -target-version = "py310" diff --git a/scripts/castiron/test_custom_code_budget.py b/scripts/castiron/test_custom_code_budget.py deleted file mode 100644 index 1290c5716..000000000 --- a/scripts/castiron/test_custom_code_budget.py +++ /dev/null @@ -1,430 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -# Regression tests for the custom-code budget. -from __future__ import annotations - -import json -import subprocess -import sys -import unittest -from pathlib import Path -from typing import Any -from unittest import mock - -import custom_code_budget as budget -from custom_code_test_support import GitTestCase - - -class GoRatchetTests(GitTestCase): - def prepare(self, *, inherited_ci: bool = False) -> str: - if inherited_ci: - self.write(".github/actions/check/main.go", "package main\n") - self.baseline() - self.policy() - return self.commit("activate Go ratchet") - - def policy(self, *, active: bool = True, limit: int = 2000) -> None: - value: dict[str, Any] = {"schema_version": 1, "max_custom_patch_lines": limit} - if active: - value["ratchet"] = "go" - self.write(budget.POLICY, json.dumps(value) + "\n") - - def evaluate(self, base: str, head: str, **kwargs: Any) -> dict[str, Any]: - return budget.evaluate(self.repo, base, head, public=False, **kwargs)[0] - - def test_handwritten_helpers_tests_comments_and_nested_modules_count(self) -> None: - base = self.prepare() - for path in ( - "helper.go", - "helper_test.go", - "nested/module/helper.go", - ".github/workflows/check.go", - ): - with self.subTest(path=path): - self.git("checkout", "--detach", "-q", base) - self.write(path, "// comment\n\npackage example\n// final line") - result = self.evaluate(base, self.commit()) - self.assertEqual(result["checks"]["budget"]["state"], "failure") - self.assertEqual(result["ratchet"]["delta"], 4) - self.assertEqual( - result["ratchet"]["after"]["handwritten_files"], [{"path": path, "lines": 4}] - ) - - def test_inherited_private_checkpoint_ci_is_handwritten(self) -> None: - base = self.prepare(inherited_ci=True) - self.write(".github/actions/check/main.go", "package main\n// added\n") - result = self.evaluate(base, self.commit()) - self.assertEqual(result["total"], 0) - self.assertEqual(result["ratchet"]["before"]["handwritten_lines"], 1) - self.assertEqual(result["ratchet"]["delta"], 1) - self.assertEqual( - result["ratchet"]["after"]["handwritten_files"], - [{"path": ".github/actions/check/main.go", "lines": 2}], - ) - self.assertEqual(result["checks"]["budget"]["state"], "failure") - - def test_patch_reduction_can_pay_for_handwritten_lines(self) -> None: - self.prepare() - self.write("generated.py", "generated\ncustom\ncustom\n") - base = self.commit() - self.write("generated.py", "generated\n") - self.write("helper.go", "package example\n") - result = self.evaluate(base, self.commit()) - self.assertEqual(result["ratchet"]["delta"], -1) - self.assertEqual(result["checks"]["budget"]["state"], "success") - - def test_stale_branch_cannot_reclaim_landed_reduction(self) -> None: - self.prepare() - self.write("helper.go", "package example\n// old allowance\n") - old = self.commit() - self.git("checkout", "-q", "-b", "stale") - head = self.commit() - self.git("checkout", "-q", "main") - self.write("helper.go", "package example\n") - base = self.commit() - self.assertEqual(self.git("merge-base", base, head), old) - result = self.evaluate(base, head) - self.assertEqual(result["ratchet"]["before"]["revision"], base) - self.assertEqual(result["ratchet"]["delta"], 1) - self.assertEqual(result["checks"]["budget"]["state"], "failure") - - def test_complete_merge_queue_candidate_is_counted(self) -> None: - base = self.prepare() - constituent = self.commit("empty constituent") - self.write("helper.go", "package example\n") - result = self.evaluate(base, self.commit("combined queue"), pull_heads=[constituent]) - self.assertEqual(result["checks"]["isolation"]["state"], "success") - self.assertEqual(result["checks"]["budget"]["state"], "failure") - - def test_ratchet_cannot_be_disabled(self) -> None: - base = self.prepare() - self.policy(active=False) - result = self.evaluate(base, self.commit()) - for name in ("budget", "isolation"): - self.assertEqual(result["checks"][name]["state"], "failure") - self.assertIn("cannot be disabled", result["checks"][name]["description"]) - - def test_old_ceiling_increase_cannot_restore_ratchet_headroom(self) -> None: - self.prepare() - self.policy(limit=3000) - base = self.commit("separately reviewed ceiling increase") - self.write("helper.go", "package example\n") - result = self.evaluate(base, self.commit()) - self.assertEqual(result["limit"], 3000) - self.assertEqual(result["checks"]["budget"]["state"], "failure") - - def test_fixed_patch_ceiling_remains_independent(self) -> None: - self.prepare() - self.policy(limit=1) - self.write("helper.go", "package example\n// two\n// three\n") - base = self.commit() - (self.repo / "helper.go").unlink() - self.write("generated.py", "generated\ncustom\ncustom\n") - result = self.evaluate(base, self.commit()) - self.assertEqual(result["ratchet"]["delta"], -1) - self.assertEqual(result["checks"]["budget"]["state"], "failure") - - def test_nontext_and_symlink_go_fail_closed(self) -> None: - base = self.prepare() - for content in (b"package example\0", b"\xff", None): - with self.subTest(content=content): - self.git("checkout", "--detach", "-q", base) - path = self.repo / "helper.go" - if content is None: - path.symlink_to("generated.py") - else: - path.write_bytes(content) - result = self.evaluate(base, self.commit()) - self.assertEqual(result["checks"]["budget"]["state"], "failure") - - def test_live_target_snapshot_hash_is_verified(self) -> None: - self.prepare() - self.git("checkout", "-q", "-b", "candidate") - head = self.commit() - self.git("checkout", "-q", "main") - stats = self.repo / ".castiron.stats.yml" - stats.write_text(stats.read_text().replace("codegen_hash: ", "codegen_hash: bad")) - result = self.evaluate(self.commit(), head) - self.assertEqual(result["checks"]["budget"]["state"], "failure") - - def test_activation_must_be_policy_only(self) -> None: - self.prepare() - self.policy(active=False) - base = self.commit() - self.policy() - head = self.commit() - result = self.evaluate(base, head) - self.assertTrue(all(check["state"] == "success" for check in result["checks"].values())) - self.write("helper.go", "package example\n") - result = self.evaluate(base, self.commit()) - self.assertEqual(result["checks"]["isolation"]["state"], "failure") - self.assertEqual(result["checks"]["budget"]["state"], "failure") - - -class BudgetTests(GitTestCase): - def setUp(self) -> None: # pyright: ignore[reportImplicitOverride] - super().setUp() - self.generated, _ = self.baseline() - self.policy(10) - self.base = self.commit("human-owned budget") - - def policy(self, limit: int) -> None: - self.write( - budget.POLICY, json.dumps({"schema_version": 1, "max_custom_patch_lines": limit}) + "\n" - ) - - def evaluate(self, head: str, base: str | None = None, **kwargs: Any) -> dict[str, Any]: - return budget.evaluate(self.repo, base or self.base, head, public=False, **kwargs)[0] - - def test_additions_and_deletions_do_not_cancel(self) -> None: - self.write("generated.py", "replacement\n") - result = self.evaluate(self.commit()) - self.assertEqual((result["additions"], result["deletions"], result["total"]), (1, 1, 2)) - self.assertEqual(result["checks"]["budget"]["state"], "success") - - def test_below_equal_and_above_limit(self) -> None: - for additions in (9, 10, 11): - with self.subTest(additions=additions): - self.write("generated.py", "generated\n" + "custom\n" * additions) - result = self.evaluate(self.commit()) - self.assertEqual(result["total"], additions) - self.assertEqual( - result["checks"]["budget"]["state"], "failure" if additions > 10 else "success" - ) - - def test_whole_generated_file_deletion_is_counted(self) -> None: - (self.repo / "generated.py").unlink() - result = self.evaluate(self.commit()) - self.assertEqual((result["additions"], result["deletions"]), (0, 1)) - self.assertEqual(result["mixed_files"], 1) - - def test_restoring_generated_content_removes_customization(self) -> None: - self.write("generated.py", "generated\ncustom\n") - customized = self.commit() - self.write("generated.py", "generated\n") - result = self.evaluate(self.commit(), base=customized) - self.assertEqual(result["total"], 0) - self.assertEqual(result["mixed_files"], 0) - - def test_handwritten_only_files_keep_existing_report_scope(self) -> None: - self.write("handwritten.py", "custom\n" * 100) - self.assertEqual(self.evaluate(self.commit())["total"], 0) - - def test_increase_is_isolated_but_does_not_apply_to_itself(self) -> None: - self.policy(20) - result = self.evaluate(self.commit()) - self.assertEqual(result["checks"]["isolation"]["state"], "success") - self.assertEqual(result["limit"], 10) - self.assertEqual(result["checked_limit"], 10) - - def test_entire_pr_must_be_budget_only_not_just_latest_commit(self) -> None: - self.write("generated.py", "generated\n" + "custom\n" * 11) - self.commit("SDK change first") - self.policy(100) - result = self.evaluate(self.commit("budget-only last commit")) - self.assertEqual(result["checks"]["isolation"]["state"], "failure") - self.assertIn("separate, budget-only PR", result["checks"]["isolation"]["description"]) - self.assertEqual(result["checks"]["budget"]["state"], "failure") - self.assertEqual(result["limit"], 10) - - def test_new_base_budget_is_used_for_stale_pr_branch(self) -> None: - self.git("checkout", "-q", "-b", "sdk", self.base) - self.write("generated.py", "generated\n" + "custom\n" * 11) - head = self.commit() - self.git("checkout", "-q", "main") - self.policy(20) - new_base = self.commit("separate approved increase") - result = self.evaluate(head, base=new_base) - self.assertEqual(result["limit"], 20) - self.assertEqual(result["checks"]["isolation"]["state"], "success") - self.assertEqual(result["checks"]["budget"]["state"], "success") - - def test_next_uses_main_policy_and_target_diff_independently(self) -> None: - self.git("checkout", "-q", "-b", "next", self.base) - self.policy(100) - target = self.commit("next carries a different policy") - self.write("generated.py", "generated\n" + "custom\n" * 11) - head = self.commit("next change") - result = self.evaluate(head, base=target, policy_sha=self.base) - self.assertEqual(result["base_sha"], target) - self.assertEqual(result["policy_sha"], self.base) - self.assertEqual(result["checks"]["isolation"]["state"], "success") - self.assertEqual(result["checks"]["budget"]["state"], "failure") - self.assertEqual(result["limit"], 10) - self.git("checkout", "-q", "main") - self.policy(20) - policy = self.commit("approved main policy") - result = self.evaluate(head, base=target, policy_sha=policy) - self.assertEqual(result["checks"]["budget"]["state"], "success") - self.policy(1) - policy = self.commit("main lowers policy after PR passed") - result = self.evaluate(head, base=target, policy_sha=policy, pull_heads=[head]) - self.assertEqual(result["checks"]["budget"]["state"], "failure") - self.assertEqual(result["limit"], 1) - - def test_decrease_must_fit_current_usage(self) -> None: - self.write("generated.py", "generated\ncustom\ncustom\n") - base = self.commit() - self.policy(1) - result = self.evaluate(self.commit(), base=base) - self.assertEqual(result["checks"]["isolation"]["state"], "success") - self.assertEqual(result["checks"]["budget"]["state"], "failure") - self.assertEqual(result["checked_limit"], 1) - - def test_missing_base_policy_fails_closed(self) -> None: - (self.repo / budget.POLICY).unlink() - base = self.commit() - self.policy(100) - result = self.evaluate(self.commit(), base=base) - self.assertTrue(all(c["state"] == "failure" for c in result["checks"].values())) - - def test_policy_deletion_rename_symlink_and_mode_change_fail(self) -> None: - for change in ("delete", "rename", "symlink", "executable"): - with self.subTest(change=change): - self.git("checkout", "--detach", "-q", self.base) - path = self.repo / budget.POLICY - if change == "delete": - path.unlink() - elif change == "rename": - path.rename(self.repo / "renamed.json") - elif change == "symlink": - path.unlink() - path.symlink_to("generated.py") - else: - path.chmod(0o755) - result = self.evaluate(self.commit()) - self.assertEqual(result["checks"]["isolation"]["state"], "failure") - - def test_invalid_policy_values_fail(self) -> None: - invalid = [ - "[]", - "{}", - "not json", - '{"schema_version":1,"max_custom_patch_lines":true}', - '{"schema_version":true,"max_custom_patch_lines":10}', - '{"schema_version":2,"max_custom_patch_lines":10}', - '{"schema_version":1,"max_custom_patch_lines":-1}', - '{"schema_version":1,"max_custom_patch_lines":10.5}', - '{"schema_version":1,"max_custom_patch_lines":null}', - '{"schema_version":1,"max_custom_patch_lines":10,"mode":"report-only"}', - '{"schema_version":1,"max_custom_patch_lines":10,"max_custom_patch_lines":999}', - " " * 4097, - ] - for contents in invalid: - with self.subTest(contents=contents[:100]): - self.write(budget.POLICY, contents) - result = self.evaluate(self.commit()) - self.assertEqual(result["checks"]["isolation"]["state"], "failure") - - def test_bad_snapshot_and_binary_change_fail_budget(self) -> None: - self.write("generated.py", "binary\0content") - result = self.evaluate(self.commit()) - self.assertIn("non-text", result["checks"]["budget"]["description"]) - self.write("generated.py", "generated\n") - path = self.repo / ".castiron.stats.yml" - path.write_text( - path.read_text().replace( - budget.report.hash_codegen_commit(self.repo, self.generated), "f" * 64 - ) - ) - result = self.evaluate(self.commit()) - self.assertIn("codegen_hash mismatch", result["checks"]["budget"]["description"]) - - def test_budget_uses_existing_reporter_with_strict_verification(self) -> None: - with mock.patch.object( - budget.report, "build_report", wraps=budget.report.build_report - ) as measured: - self.evaluate(self.base) - self.assertEqual( - measured.call_args.kwargs, - {"public": False, "fetch": False, "require_head_hash": True, "target_branch": "main"}, - ) - - def test_queue_isolates_prs_but_uses_main_budget_for_combined_tree(self) -> None: - self.git("checkout", "-q", "-b", "policy", self.base) - self.policy(100) - policy_head = self.commit() - self.git("checkout", "-q", "-b", "sdk", self.base) - self.write("generated.py", "generated\n" + "custom\n" * 11) - sdk_head = self.commit() - self.git("checkout", "-q", "-b", "queue", self.base) - self.git("merge", "--no-ff", "-m", "queue policy", policy_head) - self.git("merge", "--no-ff", "-m", "queue SDK", sdk_head) - result = self.evaluate(self.git("rev-parse", "HEAD"), pull_heads=[policy_head, sdk_head]) - self.assertEqual(result["checks"]["isolation"]["state"], "success") - self.assertEqual(result["checks"]["budget"]["state"], "failure") - self.assertEqual(result["limit"], 10) - - def test_queue_without_verified_members_fails(self) -> None: - self.assertEqual( - self.evaluate(self.base, pull_heads=[])["checks"]["isolation"]["state"], "failure" - ) - - def test_cli_executes_trusted_reporter_not_inspected_repo(self) -> None: - self.write( - "scripts/castiron/custom_code_report.py", 'raise RuntimeError("PR CODE EXECUTED")\n' - ) - self.write("sitecustomize.py", 'raise RuntimeError("PR IMPORTED")\n') - head = self.commit() - out = self.repo / "result" - command = [ - sys.executable, - "-I", - str(Path(budget.__file__).resolve()), - "check", - "--repo", - str(self.repo), - "--base", - self.base, - "--head", - head, - "--out", - str(out), - ] - completed = subprocess.run( - command, cwd=self.repo, capture_output=True, text=True, check=False - ) - self.assertEqual(completed.returncode, 0, completed.stderr + completed.stdout) - self.assertEqual(json.loads((out / "budget.json").read_text())["total"], 0) - - def test_summary_has_counts_revisions_and_no_longer_generated(self) -> None: - self.write("generated.py", "replacement\n") - result = self.evaluate(self.commit()) - out = self.repo / "report-output" - budget.write_result(out, result) - summary = (out / "summary.md").read_text() - self.assertIn("+1 / -1 = 2", summary) - self.assertIn(self.generated, summary) - self.assertIn(self.base, summary) - self.assertIn("human approving review", summary) - - -class InventoryTests(unittest.TestCase): - def test_literal_inventory_is_data_not_executable_code(self) -> None: - source = ( - b"raise AssertionError('must not execute')\n" - b"GENERATED_GITHUB_PATHS = frozenset({b'.github/workflows/generated.yml'})\n" - ) - self.assertEqual( - budget.checkpoint_github_inventory(source), - frozenset({b".github/workflows/generated.yml"}), - ) - self.assertEqual( - budget.checkpoint_github_inventory(b"GENERATED_GITHUB_PATHS = frozenset({})"), - frozenset(), - ) - - def test_inventory_rejects_missing_ambiguous_and_nonliteral_values(self) -> None: - for source in ( - b"pass", - b"GENERATED_GITHUB_PATHS = frozenset({})\nGENERATED_GITHUB_PATHS = frozenset({})", - b"GENERATED_GITHUB_PATHS = __import__('os').environ", - b"GENERATED_GITHUB_PATHS = frozenset(__import__('os').environ)", - b"GENERATED_GITHUB_PATHS = frozenset({'text'})", - b"GENERATED_GITHUB_PATHS = frozenset({b'not-github'})", - ): - with self.subTest(source=source), self.assertRaises(ValueError): - budget.checkpoint_github_inventory(source) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/castiron/test_custom_code_github.py b/scripts/castiron/test_custom_code_github.py deleted file mode 100644 index 8cf302c65..000000000 --- a/scripts/castiron/test_custom_code_github.py +++ /dev/null @@ -1,1048 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -"""Trusted GitHub evaluation with real local Git objects and an offline API.""" - -from __future__ import annotations - -import json -import os -import sys -import tempfile -import unittest -from pathlib import Path -from typing import Any -from unittest import mock - -import custom_code_budget as budget -import custom_code_report as report -from custom_code_test_support import GENERATION, GitTestCase, source_head, source_run - -REPOSITORY = "openai/example" - - -class GitHubAPI: - """Mutable remote state; unexpected requests fail instead of reaching GitHub.""" - - def __init__(self, base: str, head: str, event: str = "pull_request") -> None: - self.main = base - self.next = base - self.run = source_run(head, event) - self.pull: dict[str, Any] = { - "state": "open", - "head": source_head(head), - "base": {"sha": base, "ref": "main", "repo": {"full_name": REPOSITORY}}, - } - self.associations = [{"number": 3}] - self.branches = [{"number": 3}] - self.rules = [{"type": "merge_queue"}] - self.bodies: list[str] = [] - - def __call__(self, method: str, path: str, payload: dict[str, Any] | None = None) -> Any: - root = f"repos/{REPOSITORY}" - if method == "POST" and path == f"{root}/issues/3/comments" and payload: - self.bodies.append(payload["body"]) - return {"html_url": "published"} - responses = { - root: {"default_branch": "main", "private": False}, - f"{root}/actions/runs/123": self.run, - f"{root}/pulls/3": self.pull, - f"{root}/pulls/4": self.pull, - f"{root}/git/ref/heads/main": {"object": {"sha": self.main}}, - f"{root}/rules/branches/main?per_page=100&page=1": self.rules[:100], - f"{root}/rules/branches/main?per_page=100&page=2": self.rules[100:200], - f"{root}/git/ref/heads/next": {"object": {"sha": self.next}}, - f"{root}/rules/branches/next?per_page=100&page=1": self.rules[:100], - f"{root}/rules/branches/next?per_page=100&page=2": self.rules[100:200], - f"{root}/commits/{self.run['head_sha']}/pulls?per_page=100": self.associations, - f"{root}/pulls?state=open&head=contributor%3Asdk&per_page=100&page=1": self.branches, - f"{root}/issues/3/comments?per_page=100&page=1": [], - } - if method != "GET" or path not in responses: - raise AssertionError(f"Unexpected GitHub request: {method} {path}") - return responses[path] - - -class GitHubReportTests(GitTestCase): - def test_same_head_targets_preserve_required_main_discovery(self) -> None: - head = "b" * 40 - run = source_run(head, "pull_request") - run["pull_requests"] = [] - main: dict[str, Any] = { - "state": "open", - "head": source_head(head), - "base": {"ref": "main", "repo": {"full_name": REPOSITORY}}, - } - next_pull: dict[str, Any] = {**main, "base": {**main["base"], "ref": "next"}} - for numbers in ([3, 4], [4, 3]): - - def api_response(_method: str, path: str, numbers: list[int] = numbers) -> Any: - if "/commits/" in path: - return [{"number": number} for number in numbers] - return {"3": main, "4": next_pull}[path.rsplit("/", 1)[1]] - - with ( - self.subTest(numbers=numbers), - mock.patch.object(report, "api", side_effect=api_response), - ): - self.assertEqual(report.current_pull(REPOSITORY, run), (3, "main")) - run["pull_requests"] = [{"number": 4}] - self.assertEqual(report.current_pull(REPOSITORY, run), (4, "next")) - run["pull_requests"] = [] - next_pull["base"]["ref"] = "main" - with self.assertRaisesRegex(report.ReportError, "multiple current"): - report.current_pull(REPOSITORY, run) - next_pull["base"]["ref"] = "next" - - def test_trusted_report_recomputes_pr_output_in_a_bare_repository(self) -> None: - generated, _ = self.baseline() - content_hash = report.hash_codegen_commit(self.repo, generated) - snapshot = report.create_public_snapshot( - self.repo, - self.git("rev-parse", f"{generated}^{{tree}}"), - GENERATION, - content_hash, - "codegen/public-test", - None, - ) - self.git("branch", "codegen/public-test", snapshot) - stats = (self.repo / ".castiron.stats.yml").read_text() - self.write(".castiron.stats.yml", stats + f"public_codegen_sha: {snapshot}\n") - self.write(budget.POLICY, json.dumps({"schema_version": 1, "max_custom_patch_lines": 10})) - base = self.commit() - legitimate, _ = report.build_report(self.repo, base, base, require_head_hash=True) - self.write("generated.py", "generated\n# custom\n") - self.write( - "scripts/castiron/custom_code_report.py", - "raise RuntimeError('PR code ran')\n", - ) - self.write("report.json", json.dumps(legitimate)) - head = self.commit() - self.write( - ".castiron.stats.yml", - (self.repo / ".castiron.stats.yml").read_text().replace(content_hash, "0" * 64), - ) - broken = self.commit() - remote = self.repo / "public.git" - self.git("clone", "--bare", str(self.repo), str(remote)) - real_git = report.git - - def local_git(repo: Path, *args: str, input_bytes: bytes | None = None) -> bytes: - if args[:3] == ("remote", "add", "origin"): - self.assertEqual(args[3], f"https://github.com/{REPOSITORY}.git") - args = (*args[:3], str(remote)) - self.assertNotIn("checkout", args) - return real_git(repo, *args, input_bytes=input_bytes) - - for label, revision, total in ( - ("baseline", base, 0), - ("custom", head, 1), - ("fork", head, 1), - ("bad-hash", broken, None), - ("next", head, 1), - ("next-fork", head, 1), - ("next-bad-hash", broken, None), - ): - with self.subTest(label=label): - result: dict[str, Any] | None = None - api = GitHubAPI( - "d" * 40, revision - ) # PR metadata already differs from captured main. - if label.startswith("next"): - api.pull["base"]["ref"] = "next" - api.next = base - if "fork" in label: - api.run["pull_requests"] = [] - api.associations = [] - objects, out = self.repo / f"{label}.git", self.repo / f"{label}-report" - with ( - mock.patch.object(report, "api", side_effect=api), - mock.patch.object(budget.report, "api", side_effect=api), - mock.patch.object(report, "git", side_effect=local_git), - ): - report.trusted_report(objects, REPOSITORY, 123, 1, out, base=base) - self.assertEqual( - real_git(objects, "rev-parse", "--is-bare-repository"), b"true\n" - ) - self.assertFalse((objects / "scripts").exists()) - self.assertEqual(api.bodies, []) - measured = json.loads((out / "report.json").read_text()) - self.assertEqual( - (measured["target_base_sha"], measured["head_sha"]), (base, revision) - ) - # Advance main before both budget evaluation and publication. - api.main = api.next = api.pull["base"]["sha"] = "e" * 40 - if label.startswith("next"): - # Advisory reporting alone cannot authorize next without - # the trusted selector's protected-base pin receipt. - with self.assertRaisesRegex(ValueError, "selection"): - budget.github_evaluate( - objects, - REPOSITORY, - {"repository": {"full_name": REPOSITORY}, "workflow_run": api.run}, - base, - out, - ) - else: - result, _ = budget.github_evaluate( - objects, - REPOSITORY, - {"repository": {"full_name": REPOSITORY}, "workflow_run": api.run}, - base, - out, - selection={ - "schema_version": 1, - "target_branch": "main", - "base_sha": base, - "head_sha": revision, - "policy_sha": base, - }, - ) - self.assertEqual((result["base_sha"], result["head_sha"]), (base, revision)) - report.publish_comment( - measured, REPOSITORY, 3, 123, 1, artifact_run_id=9, artifact_run_attempt=3 - ) - self.assertEqual(len(api.bodies), 1) - body = api.bodies[0] - self.assertIn("castiron:run:v1:123:1", body) - self.assertIn(base, body) - self.assertIn("/actions/runs/9", body) - if label.startswith("next"): - self.assertIn( - "Report unavailable" if total is None else "Generated baselines verified", - body, - ) - continue - assert result is not None - if total is None: - self.assertEqual(result["checks"]["budget"]["state"], "failure") - self.assertIn("could not verify", result["checks"]["budget"]["description"]) - self.assertIn("codegen_hash mismatch", measured["error"]) - self.assertIn("Report unavailable", body) - self.assertNotIn("Generated baselines verified", body) - else: - self.assertEqual(result["total"], total) - self.assertEqual(result["checks"]["budget"]["state"], "success") - self.assertIn("Generated baselines verified", body) - self.assertIn("--name castiron-custom-code-9-3", body) - if total: - self.assertIn("1 newly customized", body) - self.assertIn("generated.py", body) - self.assertIn(b"+# custom", (out / "custom-code.patch").read_bytes()) - self.assertNotIn("No new custom-code files detected", body) - else: - self.assertIn("No new custom-code files detected", body) - - def test_fork_association_fallback_paginates_candidates(self) -> None: - run: dict[str, Any] = { - "head_sha": "a" * 40, - "pull_requests": [], - "head_repository": source_run("a" * 40)["head_repository"], - "head_branch": "fix/branch&other=value", - } - first = [{"number": number} for number in range(1, 101)] - with mock.patch.object(report, "api", side_effect=[[], first, [{"number": 101}]]) as api: - self.assertEqual(len(report.associated_pulls("openai/example", run)), 101) - self.assertEqual( - api.call_args.args, - ( - "GET", - "repos/openai/example/pulls?state=open&head=contributor%3Afix%2Fbranch%26other%3Dvalue&per_page=100&page=2", - ), - ) - - def test_trusted_report_rejects_invalid_or_stale_association_before_fetch(self) -> None: - cases: dict[str, tuple[dict[str, Any], dict[str, Any], bool]] = { - "wrong workflow": ({"path": "other.yml"}, {}, True), - "unfinished run": ({"status": "in_progress"}, {}, True), - "old attempt": ({"run_attempt": 2}, {}, False), - "closed PR": ({}, {"state": "closed"}, False), - "changed head": ({}, {"head": {"sha": "c" * 40}}, False), - "other branch": ( - {}, - {"base": {"ref": "other", "repo": {"full_name": REPOSITORY}}}, - False, - ), - "other repository": ( - {}, - {"base": {"ref": "main", "repo": {"full_name": "other/repo"}}}, - False, - ), - "multiple PRs": ({"pull_requests": [{"number": 3}, {"number": 4}]}, {}, True), - } - for fallback in (False, True): - for label, (run, pull, raises) in cases.items(): - with self.subTest(label=label, fallback=fallback): - api = GitHubAPI("b" * 40, "a" * 40) - api.run.update(run) - api.pull.update(pull) - if fallback: - api.branches = api.run["pull_requests"] - api.run["pull_requests"] = [] - api.associations = [] - with ( - mock.patch.object(report, "api", side_effect=api), - mock.patch.object(report, "git") as git, - ): - if raises: - with self.assertRaises(report.ReportError): - report.trusted_report( - self.repo / "objects", - REPOSITORY, - 123, - 1, - self.repo / "out", - base="b" * 40, - ) - else: - report.trusted_report( - self.repo / "objects", - REPOSITORY, - 123, - 1, - self.repo / "out", - base="b" * 40, - ) - git.assert_not_called() - self.assertFalse((self.repo / "out").exists()) - api = GitHubAPI("b" * 40, "a" * 40) - api.run["pull_requests"] = api.associations = api.branches = [] - with ( - mock.patch.object(report, "api", side_effect=api), - mock.patch.object(report, "git") as git, - ): - report.trusted_report( - self.repo / "objects", REPOSITORY, 123, 1, self.repo / "out", base="b" * 40 - ) - git.assert_not_called() - - def test_queue_membership_uses_synthetic_commit_not_original_pr_ancestry(self) -> None: - generated, _ = self.baseline() - self.git("branch", "codegen/test", generated) - self.write(budget.POLICY, json.dumps({"schema_version": 1, "max_custom_patch_lines": 10})) - base = self.commit("budget") - self.git("checkout", "-q", "-b", "sdk", base) - self.write("generated.py", "generated\ncustom\n") - pr_head = self.commit() - self.git("checkout", "-q", "-b", "queue", base) - self.git("merge", "--squash", "sdk") - queue_head = self.commit("synthetic queue commit") - self.assertNotEqual(self.git("merge-base", pr_head, queue_head), pr_head) - event = { - "repository": {"full_name": "openai/example"}, - "workflow_run": source_run(queue_head, "merge_group"), - } - original_git = report.git - - def local_fetch(repo: Path, *args: str, input_bytes: bytes | None = None) -> bytes: - if args[0] == "fetch": - args = tuple(str(self.repo) if arg == "origin" else arg for arg in args) - return original_git(repo, *args, input_bytes=input_bytes) - - with ( - tempfile.TemporaryDirectory() as temp, - mock.patch.object( - budget.report, - "api", - side_effect=[ - {"default_branch": "main", "private": True}, - source_run(queue_head, "merge_group"), - {"object": {"sha": base}}, - [{"type": "merge_queue"}], - ], - ), - mock.patch.object(budget, "queued_entries", return_value=[(pr_head, queue_head)]), - mock.patch.object( - budget.report, - "git", - side_effect=local_fetch, - ), - ): - repo = Path(temp) / "queue.git" - result, _ = budget.github_evaluate( - repo, - "openai/example", - event, - base, - selection={ - "schema_version": 1, - "target_branch": "main", - "base_sha": base, - "head_sha": queue_head, - "policy_sha": base, - }, - ) - self.assertEqual(result["head_sha"], queue_head) - self.assertEqual(result["checks"]["budget"]["state"], "success", result["checks"]) - self.assertEqual(result["total"], 1) - self.assertEqual( - [c["state"] for c in result["checks"].values()], ["success", "success"] - ) - - def test_next_queue_bootstrap_isolated_per_verified_constituent(self) -> None: - generated, _ = self.baseline() - self.git("branch", "codegen/test", generated) - policy = {"schema_version": 1, "max_custom_patch_lines": 10} - self.write(budget.POLICY, json.dumps(policy)) - base = self.commit("unpinned next budget") - self.git("checkout", "-q", "-b", "initialize", base) - self.write( - budget.POLICY, - json.dumps({**policy, "next_policy": {"repository": REPOSITORY, "main_sha": base}}), - ) - pin_head = self.commit("policy-only initialization") - self.git("checkout", "-q", "-b", "sdk", base) - self.write("generated.py", "generated\ncustom\n") - sdk_head = self.commit("unrelated SDK PR") - self.git("checkout", "-q", "-b", "queue", base) - self.git("merge", "--squash", "initialize") - pin_queue = self.commit("synthetic policy queue commit") - self.git("merge", "--squash", "sdk") - queue_head = self.commit("synthetic combined queue commit") - run = source_run(queue_head, "merge_group") - run["head_branch"] = "gh-readonly-queue/next/pr-7-example" - event = {"repository": {"full_name": REPOSITORY}, "workflow_run": run} - original_git = report.git - - def local_fetch(repo: Path, *args: str, input_bytes: bytes | None = None) -> bytes: - if args[0] == "fetch": - args = tuple(str(self.repo) if arg == "origin" else arg for arg in args) - return original_git(repo, *args, input_bytes=input_bytes) - - for entries, state in ( - ([(pin_head, pin_queue), (sdk_head, queue_head)], "success"), - ([(queue_head, queue_head)], "failure"), - ): - with ( - self.subTest(entries=entries), - tempfile.TemporaryDirectory() as temp, - mock.patch.object( - budget.report, - "api", - side_effect=[ - {"default_branch": "main", "private": True}, - run, - {"object": {"sha": base}}, - {"object": {"sha": base}}, - [{"type": "merge_queue"}], - ], - ), - mock.patch.object(budget, "queued_entries", return_value=entries), - mock.patch.object(budget.report, "git", side_effect=local_fetch), - ): - result, _ = budget.github_evaluate( - Path(temp) / "queue.git", - REPOSITORY, - event, - base, - selection={ - "schema_version": 1, - "target_branch": "next", - "base_sha": base, - "head_sha": queue_head, - "policy_sha": base, - "bootstrap": True, - }, - ) - self.assertEqual(result["checks"]["isolation"]["state"], state) - self.assertEqual(result["checks"]["budget"]["state"], "success") - self.assertEqual(result["limit"], 10) - self.assertEqual(result["total"], 1) - - def test_next_queue_uses_reviewed_main_policy_after_main_advances(self) -> None: - generated, _ = self.baseline() - self.git("branch", "codegen/test", generated) - self.write(budget.POLICY, json.dumps({"schema_version": 1, "max_custom_patch_lines": 10})) - approved_policy = self.commit("budget") - self.write( - budget.POLICY, - json.dumps( - { - "schema_version": 1, - "max_custom_patch_lines": 10, - "next_policy": {"repository": "openai/example", "main_sha": approved_policy}, - } - ), - ) - base = self.commit("reviewed next policy") - self.git("checkout", "-q", "-b", "sdk", base) - self.write("generated.py", "generated\ncustom\n") - pr_head = self.commit() - self.git("checkout", "-q", "-b", "queue", base) - self.git("merge", "--squash", "sdk") - queue_head = self.commit("synthetic queue commit") - self.assertNotEqual(self.git("merge-base", pr_head, queue_head), pr_head) - self.git("checkout", "-q", "main") - self.write(budget.POLICY, json.dumps({"schema_version": 1, "max_custom_patch_lines": 0})) - policy = self.commit("main tightens policy independently of next") - self.assertNotEqual(self.git("merge-base", policy, queue_head), policy) - run = source_run(queue_head, "merge_group") - run["head_branch"] = "gh-readonly-queue/next/pr-7-example" - event = { - "repository": {"full_name": "openai/example"}, - "workflow_run": run, - } - original_git = report.git - - def local_fetch(repo: Path, *args: str, input_bytes: bytes | None = None) -> bytes: - if args[0] == "fetch": - args = tuple(str(self.repo) if arg == "origin" else arg for arg in args) - return original_git(repo, *args, input_bytes=input_bytes) - - with ( - tempfile.TemporaryDirectory() as temp, - mock.patch.object( - budget.report, - "api", - side_effect=[ - {"default_branch": "main", "private": True}, - run, - {"object": {"sha": policy}}, - {"object": {"sha": base}}, - [{"type": "merge_queue"}], - ], - ), - mock.patch.object(budget, "queued_entries", return_value=[(pr_head, queue_head)]), - mock.patch.object( - budget.report, - "git", - side_effect=local_fetch, - ), - ): - repo = Path(temp) / "queue.git" - result, _ = budget.github_evaluate( - repo, - "openai/example", - event, - approved_policy, - selection={ - "target_branch": "next", - "base_sha": base, - "head_sha": queue_head, - "policy_sha": approved_policy, - "bootstrap": False, - "schema_version": 1, - }, - ) - self.assertEqual(result["head_sha"], queue_head) - self.assertEqual(result["checks"]["budget"]["state"], "success", result["checks"]) - self.assertEqual(result["limit"], 10) - self.assertEqual(result["base_sha"], base) - self.assertEqual(result["policy_sha"], approved_policy) - self.assertEqual(result["target_branch"], "next") - self.assertEqual(result["total"], 1) - self.assertEqual( - [c["state"] for c in result["checks"].values()], ["success", "success"] - ) - - def test_next_queue_bootstrap_with_separate_sdk_pr(self) -> None: - generated, _ = self.baseline() - self.git("branch", "codegen/test", generated) - self.write(budget.POLICY, json.dumps({"schema_version": 1, "max_custom_patch_lines": 10})) - approved_policy = self.commit("budget") - self.write( - budget.POLICY, - json.dumps( - { - "schema_version": 1, - "max_custom_patch_lines": 10, - "next_policy": {"repository": "openai/example", "main_sha": approved_policy}, - } - ), - ) - pin_head = self.commit("isolated initialization PR") - base = approved_policy - self.git("checkout", "-q", "-b", "sdk", base) - self.write("generated.py", "generated\ncustom\n") - pr_head = self.commit() - self.git("checkout", "-q", "-b", "queue", base) - self.git("merge", "--squash", "sdk") - self.commit("SDK queue entry") - self.git("merge", "--squash", pin_head) - queue_head = self.commit("synthetic queue commit") - self.assertNotEqual(self.git("merge-base", pr_head, queue_head), pr_head) - policy = approved_policy - run = source_run(queue_head, "merge_group") - run["head_branch"] = "gh-readonly-queue/next/pr-7-example" - event = { - "repository": {"full_name": "openai/example"}, - "workflow_run": run, - } - original_git = report.git - - def local_fetch(repo: Path, *args: str, input_bytes: bytes | None = None) -> bytes: - if args[0] == "fetch": - args = tuple(str(self.repo) if arg == "origin" else arg for arg in args) - return original_git(repo, *args, input_bytes=input_bytes) - - with ( - tempfile.TemporaryDirectory() as temp, - mock.patch.object( - budget.report, - "api", - side_effect=[ - {"default_branch": "main", "private": True}, - run, - {"object": {"sha": policy}}, - {"object": {"sha": base}}, - [{"type": "merge_queue"}], - ], - ), - mock.patch.object( - budget, - "queued_entries", - return_value=[(pin_head, queue_head), (pr_head, queue_head)], - ), - mock.patch.object( - budget.report, - "git", - side_effect=local_fetch, - ), - ): - repo = Path(temp) / "queue.git" - result, _ = budget.github_evaluate( - repo, - "openai/example", - event, - approved_policy, - selection={ - "target_branch": "next", - "base_sha": base, - "head_sha": queue_head, - "policy_sha": approved_policy, - "bootstrap": True, - "schema_version": 1, - }, - ) - self.assertEqual(result["head_sha"], queue_head) - self.assertEqual(result["checks"]["budget"]["state"], "success", result["checks"]) - self.assertEqual(result["limit"], 10) - self.assertEqual(result["base_sha"], base) - self.assertEqual(result["policy_sha"], approved_policy) - self.assertEqual(result["target_branch"], "next") - self.assertEqual(result["total"], 1) - self.assertEqual( - [c["state"] for c in result["checks"].values()], ["success", "success"] - ) - - def test_same_commit_discovery_is_bound_to_source_repository_and_branch( - self, - ) -> None: - run = source_run("b" * 40) - run["pull_requests"] = [] - own: dict[str, Any] = { - "state": "open", - "head": source_head(run["head_sha"]), - "base": {"ref": "main", "repo": {"full_name": REPOSITORY}}, - } - for foreign_head in ( - {**own["head"], "ref": "other"}, - {**own["head"], "repo": {"id": 11, "full_name": "victim/example"}}, - {**own["head"], "repo": None}, - ): - victim = {**own, "head": foreign_head} - - def discover(_method: str, path: str, victim: dict[str, Any] = victim) -> Any: - if "/commits/" in path: - return [{"number": 3}, {"number": 4}] - return {"3": own, "4": victim}[path.rsplit("/", 1)[1]] - - with ( - self.subTest(head=foreign_head), - mock.patch.object( - report, - "api", - side_effect=discover, - ), - ): - self.assertEqual(report.current_pull(REPOSITORY, run), (3, "main")) - - def test_foreign_only_associations_retry_source_branch(self) -> None: - run = source_run("b" * 40) - run["pull_requests"] = [] - own: dict[str, Any] = { - "state": "open", - "head": source_head("b" * 40), - "base": {"ref": "main", "repo": {"full_name": REPOSITORY}}, - } - other = {**own, "head": {**own["head"], "ref": "other"}} - branches = [{"number": 3}] - - def api(method: str, path: str) -> Any: - if "/commits/" in path: - return [{"number": 4}] - if "/pulls?" in path: - return branches - return {"3": own, "4": other, "5": own}[path.rsplit("/", 1)[1]] - - with mock.patch.object(report, "api", side_effect=api): - self.assertEqual(report.current_pull(REPOSITORY, run), (3, "main")) - run["pull_requests"] = [{"number": 4}] - self.assertIsNone(report.current_pull(REPOSITORY, run)) - run["pull_requests"] = [] - branches.append({"number": 5}) - with self.assertRaisesRegex(report.ReportError, "multiple current"): - report.current_pull(REPOSITORY, run) - - -class GitHubBudgetTests(unittest.TestCase): - def setUp(self) -> None: # pyright: ignore[reportImplicitOverride] - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - self.root = Path(temporary.name) - self.repo = self.root / "objects.git" - self.base, self.head = "a" * 40, "b" * 40 - self.api = GitHubAPI(self.base, self.head) - patch = mock.patch.object(budget.report, "api", side_effect=self.api) - patch.start() - self.addCleanup(patch.stop) - - def evaluate(self, *, trusted_report: Path | None = None) -> tuple[dict[str, Any], bytes]: - event = { - "repository": {"full_name": REPOSITORY}, - "workflow_run": source_run(self.head, self.api.run["event"]), - } - return budget.github_evaluate( - self.repo, - REPOSITORY, - event, - self.base, - trusted_report, - selection={ - "schema_version": 1, - "target_branch": "main", - "base_sha": self.base, - "head_sha": self.head, - "policy_sha": self.base, - }, - ) - - def test_main_rejects_missing_or_retargeted_selection(self) -> None: - for branch in (None, "next"): - selection = ( - None - if branch is None - else { - "schema_version": 1, - "target_branch": branch, - "base_sha": self.base, - "head_sha": self.head, - "policy_sha": self.base, - } - ) - with ( - self.subTest(branch=branch), - self.assertRaisesRegex(ValueError, "policy selection"), - ): - budget.github_evaluate( - self.repo, - REPOSITORY, - {"repository": {"full_name": REPOSITORY}, "workflow_run": self.api.run}, - self.base, - selection=selection, - ) - self.assertFalse(self.repo.exists()) - - def test_merge_queue_rule_is_required_before_passing(self) -> None: - self.api.run = source_run(self.head, "merge_group") - self.api.rules = [{"type": "required_status_checks"}] - with self.assertRaisesRegex(ValueError, "must require a merge queue"): - self.evaluate() - self.assertFalse(self.repo.exists()) - - def test_merge_queue_rule_on_later_page_is_honored(self) -> None: - self.api.rules = [{"type": "required_status_checks"}] * 100 + [{"type": "merge_queue"}] - with ( - mock.patch.object(budget.subprocess, "run"), - mock.patch.object(budget.report, "git"), - mock.patch.object(budget, "evaluate", return_value=({}, b"")) as evaluate, - ): - self.evaluate() - evaluate.assert_called_once() - - def test_merge_queue_rule_absent_after_full_page_is_rejected(self) -> None: - self.api.rules = [{"type": "required_status_checks"}] * 100 - with self.assertRaisesRegex(ValueError, "must require a merge queue"): - self.evaluate() - self.assertFalse(self.repo.exists()) - - def test_wrong_or_superseded_source_runs_fail_before_fetching(self) -> None: - for changes in ( - {"run_attempt": 2}, - {"status": "in_progress"}, - {"path": "other.yml"}, - {"head_sha": "c" * 40}, - {"repository": {"full_name": "wrong/repo"}}, - ): - with self.subTest(changes=changes): - self.api.run = {**source_run(self.head), **changes} - with self.assertRaisesRegex(ValueError, "source workflow run"): - self.evaluate() - self.assertFalse(self.repo.exists()) - - def test_reuses_only_matching_main_job_report(self) -> None: - self.api.main = self.api.pull["base"]["sha"] = "d" * 40 - self.repo.mkdir() - (self.root / "custom-code.patch").write_bytes(b"verified patch") - for stale in (None, "base", "head"): - with self.subTest(stale=stale): - measured = { - "target_base_sha": "c" * 40 if stale == "base" else self.base, - "head_sha": "c" * 40 if stale == "head" else self.head, - "target_branch": "main", - "policy_sha": self.base, - } - (self.root / "report.json").write_text(json.dumps(measured)) - with ( - mock.patch.object(budget.report, "git", return_value=b"true\n"), - mock.patch.object(budget, "evaluate", return_value=({}, b"")) as evaluate, - ): - if stale: - with self.assertRaisesRegex( - ValueError, - "trusted policy selection" - if stale == "base" - else "trusted report does not match", - ): - self.evaluate(trusted_report=self.root) - evaluate.assert_not_called() - else: - self.evaluate(trusted_report=self.root) - self.assertEqual(evaluate.call_args.args, (self.repo, self.base, self.head)) - self.assertEqual( - evaluate.call_args.kwargs["measurement"], (measured, b"verified patch") - ) - - def test_next_reuses_target_snapshot_but_uses_selected_main_policy(self) -> None: - self.api.pull["base"]["ref"] = "next" - target = "c" * 40 - self.api.next = target - self.repo.mkdir() - measured = { - "target_base_sha": target, - "target_branch": "next", - "policy_sha": self.base, - "head_sha": self.head, - } - (self.root / "report.json").write_text(json.dumps(measured)) - (self.root / "custom-code.patch").write_bytes(b"verified patch") - with ( - mock.patch.object(budget.report, "git", return_value=b"true\n"), - mock.patch.object(budget, "evaluate", return_value=({}, b"")) as evaluate, - mock.patch.object( - budget, - "read_policy", - return_value={"next_policy": {"repository": REPOSITORY, "main_sha": self.base}}, - ), - ): - budget.github_evaluate( - self.repo, - REPOSITORY, - {"repository": {"full_name": REPOSITORY}, "workflow_run": self.api.run}, - self.base, - self.root, - selection={ - "target_branch": "next", - "base_sha": target, - "head_sha": self.head, - "policy_sha": self.base, - "bootstrap": False, - "schema_version": 1, - }, - ) - self.assertEqual(evaluate.call_args.args, (self.repo, target, self.head)) - self.assertEqual(evaluate.call_args.kwargs["policy_sha"], self.base) - - def test_next_report_reuse_rejects_retargeting_and_policy_mismatch(self) -> None: - self.api.pull["base"]["ref"] = "next" - self.repo.mkdir() - self.api.next = "d" * 40 - for field, value in ( - ("target_branch", "main"), - ("policy_sha", "c" * 40), - ("head_sha", "c" * 40), - ): - measured = { - "target_base_sha": "d" * 40, - "head_sha": self.head, - "target_branch": "next", - "policy_sha": self.base, - field: value, - } - (self.root / "report.json").write_text(json.dumps(measured)) - with ( - self.subTest(field=field), - self.assertRaisesRegex(ValueError, "trusted report does not match"), - ): - budget.github_evaluate( - self.repo, - REPOSITORY, - {"repository": {"full_name": REPOSITORY}, "workflow_run": self.api.run}, - self.base, - self.root, - selection={ - "target_branch": "next", - "base_sha": self.api.next, - "head_sha": self.head, - "policy_sha": self.base, - "bootstrap": False, - "schema_version": 1, - }, - ) - - def test_next_queue_uses_target_rules_and_requires_policy_selection(self) -> None: - self.api.run = source_run(self.head, "merge_group") - self.api.run["head_branch"] = "gh-readonly-queue/next/pr-3-test" - self.api.rules = [] - with self.assertRaisesRegex(ValueError, "next must require a merge queue"): - self.evaluate() - self.api.rules = [{"type": "merge_queue"}] - self.api.main = "c" * 40 - with self.assertRaisesRegex(ValueError, "policy selection"): - self.evaluate() - self.assertFalse(self.repo.exists()) - - def test_next_selection_receipt_rejects_missing_unknown_or_boolean_version(self) -> None: - self.api.pull["base"]["ref"] = "next" - self.api.next = "d" * 40 - for version in (None, 2, True): - with ( - self.subTest(version=version), - self.assertRaisesRegex(ValueError, "policy selection"), - ): - budget.github_evaluate( - self.repo, - REPOSITORY, - {"repository": {"full_name": REPOSITORY}, "workflow_run": self.api.run}, - self.base, - selection={ - "schema_version": version, - "target_branch": "next", - "base_sha": self.api.next, - "head_sha": self.head, - "policy_sha": self.base, - "bootstrap": False, - }, - ) - self.assertFalse(self.repo.exists()) - - def test_queue_cannot_reuse_pr_measurement(self) -> None: - self.api.run = source_run(self.head, "merge_group") - with self.assertRaisesRegex(ValueError, "only PR runs can reuse"): - self.evaluate(trusted_report=self.root) - self.assertFalse(self.repo.exists()) - - def test_evaluation_error_emits_failure_outputs_and_diagnostic_summary(self) -> None: - event = self.root / "event.json" - event.write_text( - json.dumps({"repository": {"full_name": REPOSITORY}, "workflow_run": self.api.run}) - ) - output, result = self.root / "outputs", self.root / "result" - self.api.rules = [] # Real validation failure, before any Git objects are fetched. - args = [ - "budget", - "github", - "--repository", - REPOSITORY, - "--event-path", - str(event), - "--trusted-sha", - self.base, - "--repo", - str(self.repo), - "--out", - str(result), - ] - with ( - mock.patch.object(sys, "argv", args), - mock.patch.dict(os.environ, {"GITHUB_OUTPUT": str(output)}), - ): - self.assertEqual(budget.main(), 1) - self.assertIn("isolation=failure\nbudget=failure\n", output.read_text()) - self.assertIn("must require a merge queue", (result / "summary.md").read_text()) - - def test_queue_pagination(self) -> None: - pages = [ - { - "nodes": [ - { - "pullRequest": {"number": 1, "headRefOid": "a" * 40}, - "headCommit": {"oid": "c" * 40}, - } - ], - "pageInfo": {"hasNextPage": True, "endCursor": "cursor"}, - }, - { - "nodes": [ - { - "pullRequest": {"number": 2, "headRefOid": "b" * 40}, - "headCommit": {"oid": "d" * 40}, - } - ], - "pageInfo": {"hasNextPage": False, "endCursor": None}, - }, - ] - with mock.patch.object( - budget.report, - "api", - side_effect=[{"data": {"repository": {"mergeQueue": {"entries": p}}}} for p in pages], - ) as api: - self.assertEqual( - budget.queued_entries("openai/example", "main"), - [("a" * 40, "c" * 40), ("b" * 40, "d" * 40)], - ) - self.assertEqual(api.call_args_list[1].args[2]["variables"]["cursor"], "cursor") - - def test_queue_context_refuses_stale_checkout_before_fetching(self) -> None: - self.api.run = source_run(self.head, "merge_group") - self.api.main = "c" * 40 - with self.assertRaisesRegex(ValueError, "trusted checkout is stale"): - self.evaluate() - self.assertFalse(self.repo.exists()) - - def test_github_uses_fresh_bare_data_repo_and_checks_current_head(self) -> None: - self.api.run["pull_requests"] = self.api.associations = [] - self.api.pull["base"]["sha"] = "c" * 40 - with ( - mock.patch.object(budget.report, "git") as git, - mock.patch.object(budget, "evaluate", return_value=({}, b"")) as evaluate, - ): - self.evaluate() - self.assertTrue((self.repo / "HEAD").exists()) - self.assertFalse((self.repo / "src").exists()) - self.assertIn( - mock.call( - self.repo, "fetch", "--quiet", "--no-tags", "origin", self.base, self.head - ), - git.call_args_list, - ) - evaluate.assert_called_once_with( - self.repo, - self.base, - self.head, - public=True, - fetch=True, - pull_heads=None, - measurement=None, - policy_sha=self.base, - target_branch="main", - main_snapshot=None, - repository=REPOSITORY, - ) - - def test_stale_pull_and_wrong_target_fail_before_objects_created(self) -> None: - original = self.api.pull - self.api.run["pull_requests"] = self.api.associations = [] - for changes in ( - {"head": {"sha": "c" * 40}}, - {"base": {**original["base"], "repo": {"full_name": "wrong/repo"}}}, - {"base": {**original["base"], "ref": "other"}}, - {"state": "closed"}, - ): - with self.subTest(changes=changes): - self.api.pull = {**original, **changes} - with self.assertRaisesRegex(ValueError, "exactly one current PR"): - self.evaluate() - self.assertFalse(self.repo.exists()) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/castiron/test_custom_code_policy.py b/scripts/castiron/test_custom_code_policy.py deleted file mode 100644 index c68bb7072..000000000 --- a/scripts/castiron/test_custom_code_policy.py +++ /dev/null @@ -1,988 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -"""Current-main policy selection and review authorization regressions.""" - -from __future__ import annotations - -import io -import json -import sys -import tempfile -import unittest -import unittest.mock as mock # noqa: PLR0402 - keep unittest imports consistent -from collections.abc import Callable -from pathlib import Path -from typing import Any - -import custom_code_budget as budget -import custom_code_policy as selector -from custom_code_test_support import GitTestCase - -REPO = "openai/example" - - -class ReviewedPinTests(GitTestCase): - def setUp(self) -> None: # pyright: ignore[reportImplicitOverride] - super().setUp() - self.git("remote", "add", "origin", f"https://github.com/{REPO}.git") - self.policy = {"schema_version": 1, "max_custom_patch_lines": 4000} - self.write_policy(self.policy) - checker = self.repo / "scripts/castiron/custom_code_budget.py" - checker.parent.mkdir(parents=True) - checker.write_text("NEXT_POLICY_VERSION = 1\n") - checker.with_name("custom_code_policy.py").write_text( - 'PIN_REVIEW_POLICY = "effective-policy-v1"\n' - ) - self.main = self.commit("main policy") - self.git("update-ref", "refs/remotes/origin/main", self.main) - self.pin = {"repository": REPO, "main_sha": self.main} - self.write_policy({**self.policy, "next_policy": self.pin}) - self.base = self.commit("reviewed next pin") - self.git("checkout", "-q", "-b", "next", self.base) - (self.repo / "sdk.txt").write_text("candidate SDK bytes") - self.head = self.commit("sdk change") - - def test_review_command_uses_git_policies_and_rejects_stale_main(self) -> None: - git: Callable[..., bytes] = selector.report.git - - def local_fetch(repo: Path, *args: str, **kwargs: Any) -> bytes: - if args[0] == "fetch": - return git( - repo, - "fetch", - "--quiet", - str(self.repo), - f"{self.main}:refs/remotes/origin/main", - self.base, - self.head, - ) - return git(repo, *args, **kwargs) - - with ( - tempfile.TemporaryDirectory() as directory, - mock.patch.object(selector.report, "git", side_effect=local_fetch), - ): - for stale in (False, True): - with ( - self.subTest(stale=stale), - mock.patch("sys.stdout", new_callable=io.StringIO) as output, - ): - args = [ - "review-required", - "--repo", - str(Path(directory) / str(stale)), - "--repository", - REPO, - "--base", - self.base, - "--head", - self.head, - "--main", - self.base if stale else self.main, - ] - if stale: - with self.assertRaisesRegex(ValueError, "authenticated main changed"): - selector.main(args) - self.assertEqual(output.getvalue(), "") - else: - self.assertEqual(selector.main(args), 0) - self.assertEqual(json.loads(output.getvalue()), {"requires_human": False}) - - def write_policy(self, value: dict[str, Any]) -> None: - (self.repo / ".castiron-ratchet.json").write_text(json.dumps(value)) - - def select( - self, base: str | None = None, head: str | None = None, main: str | None = None, **kw: Any - ) -> dict[str, Any]: - return selector.select_policy( - self.repo, - REPO, - "next", - base or self.base, - head or self.head, - main or self.main, - **kw, - ) - - def review_api(self, reviews: list[dict[str, Any]]) -> Callable[[str, str], Any]: - def api(method: str, path: str) -> Any: - if "/commits/" in path: - return [{"number": 3}] - if "/reviews?" in path: - return reviews - return { - "number": 3, - "state": "open", - "head": {"sha": self.head}, - "base": {"ref": "next", "repo": {"full_name": REPO}}, - "user": {"login": "author"}, - } - - return api - - def test_policy_review_fork_fallback_keeps_live_exact_head_checks(self) -> None: - original = self.review_api([self.approved()]) - current = original("GET", "repos/openai/example/pulls/3") - cases: list[tuple[dict[str, Any], bool]] = [ - ({}, True), - ({"state": "closed"}, False), - ({"head": {"sha": "f" * 40}}, False), - ({"base": {"ref": "main", "repo": {"full_name": REPO}}}, False), - ({"base": {"ref": "next", "repo": {"full_name": "other/repo"}}}, False), - ] - for change, accepted in cases: - with self.subTest(change=change): - - def api(method: str, path: str, change: dict[str, Any] = change) -> Any: - if "/commits/" in path: - return [] - if "/pulls?" in path: - if path.endswith("page=1"): - return [{"head": {"sha": "a" * 40}}] * 100 - return [current] - if path.endswith("/pulls/3"): - return {**current, **change} - return original(method, path) - - with mock.patch.object(selector.report, "api", side_effect=api): - if accepted: - self.assertEqual(selector.require_human_policy_review(REPO, self.head), 3) - else: - with self.assertRaisesRegex(ValueError, "one open next PR"): - selector.require_human_policy_review(REPO, self.head) - - def approved(self, **kwargs: Any) -> dict[str, Any]: - return { - "id": 10, - "state": "APPROVED", - "commit_id": self.head, - "user": {"type": "User", "login": "reviewer"}, - "author_association": "MEMBER", - "body": selector.HUMAN_REVIEW_RULES["attestation_prefix"] + self.head, - **kwargs, - } - - def test_fork_policy_review_uses_the_discovered_pull(self) -> None: - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**self.policy, "next_policy": self.pin, "max_custom_patch_lines": 5000}) - self.head = self.commit("fork increases policy budget") - review = self.review_api([self.approved()]) - run: dict[str, Any] = { - "head_sha": self.head, - "head_repository": { - "id": 17, - "full_name": "forker/example", - "owner": {"login": "forker"}, - }, - "head_branch": "policy", - "pull_requests": [], - } - - def api(method: str, path: str) -> Any: - if "/commits/" in path: - return [] - if "/pulls?" in path: - self.assertIn("head=forker%3Apolicy", path) - return [{"number": 3}] - result = review(method, path) - if path == f"repos/{REPO}/pulls/3": - result["head"].update(ref="policy", repo=run["head_repository"]) - return result - - with mock.patch.object(selector.report, "api", side_effect=api): - current = selector.report.current_pull(REPO, run) - assert current is not None - number, branch = current - self.assertEqual(branch, "next") - result = self.select(base=self.main, pull_numbers={self.head: number}) - self.assertEqual(result["policy_reviews"], [{"number": 3, "head_sha": self.head}]) - with self.assertRaisesRegex(ValueError, "one open next PR"): - selector.require_human_policy_review(REPO, self.main, number) - - def test_only_base_pin_selects_checker(self) -> None: - (self.repo / "scripts/castiron/custom_code_budget.py").write_text( - 'raise RuntimeError("candidate code must never execute")\n' - ) - self.head = self.commit("malicious candidate code") - self.assertEqual(self.select()["policy_sha"], self.main) - self.assertEqual(self.select()["schema_version"], 1) - - def test_old_reviewed_pin_survives_main_advancing(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - (self.repo / "main-new.txt").write_text("later main") - later = self.commit("main later") - self.git("update-ref", "refs/remotes/origin/main", later) - self.assertEqual(self.select(main=later)["policy_sha"], self.main) - - def test_wrong_repository_and_unadopted_pin_rejected(self) -> None: - for pin in ( - {"repository": "other/repo", "main_sha": self.main}, - {"repository": REPO, "main_sha": self.head}, - ): - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**self.policy, "next_policy": pin}) - base = self.commit("bad pin") - with self.assertRaises((ValueError, selector.report.ReportError)): - self.select(base=base, head=base) - - def test_side_branch_merge_does_not_adopt_policy(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - (self.repo / "side").write_text("side") - side = self.commit("side policy") - self.git("checkout", "-q", "--detach", self.main) - (self.repo / "parent").write_text("main") - self.commit("main") - self.git("merge", "--no-ff", "-qm", "merge side", side) - main = self.git("rev-parse", "HEAD") - self.git("update-ref", "refs/remotes/origin/main", main) - self.write_policy({**self.policy, "next_policy": {"repository": REPO, "main_sha": side}}) - base = self.commit("next pins side") - with self.assertRaisesRegex(ValueError, "first-parent"): - self.select(base=base, head=base, main=main) - - def test_absent_pin_ordinary_change_fails(self) -> None: - with self.assertRaisesRegex(ValueError, "initialization"): - self.select(base=self.main, head=self.head) - - def test_absent_pin_preserves_value_error(self) -> None: - with self.assertRaisesRegex(ValueError, "trusted next base has no reviewed policy pin"): - selector.next_policy_sha(self.repo, REPO, self.main, self.main) - - def test_malformed_base_pin_cannot_bootstrap(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - self.write_policy({**self.policy, "next_policy": {}}) - base = self.commit("malformed") - with self.assertRaisesRegex(ValueError, "exactly"): - self.select(base=base, head=self.base) - - def test_bootstrap_requires_exact_policy_only_with_native_review(self) -> None: - self.head = self.base - with mock.patch.object(selector, "require_human_policy_review") as review: - result = self.select(base=self.main) - self.assertTrue(result["bootstrap"]) - self.assertEqual(result["policy_reviews"], []) - self.assertEqual(result["human_review_rules"], selector.HUMAN_REVIEW_RULES) - self.assertEqual(result["policy_sha"], self.main) - review.assert_not_called() - with self.assertRaisesRegex(ValueError, "policy-only"): - self.select(base=self.main, head=self.git("rev-parse", "next")) - - def test_queue_bootstrap_isolated_per_constituent(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - (self.repo / "unrelated.txt").write_text("independent SDK change") - sdk_head = self.commit("unrelated PR on unpinned next") - self.git("merge", "--no-ff", "-qm", "queue initialization and SDK", self.base) - group = self.git("rev-parse", "HEAD") - with mock.patch.object(selector, "require_human_policy_review", return_value=3) as review: - result = self.select(base=self.main, head=group, pull_heads=[self.base, sdk_head]) - self.assertTrue(result["bootstrap"]) - self.assertEqual(result["policy_sha"], self.main) - self.assertEqual(result["policy_reviews"], []) - review.assert_not_called() - for heads in (None, [group]): - with self.subTest(heads=heads), self.assertRaisesRegex(ValueError, "policy-only"): - self.select(base=self.main, head=group, pull_heads=heads) - with self.assertRaisesRegex(ValueError, "verified constituent"): - self.select(base=self.main, head=group, pull_heads=[]) - with self.assertRaisesRegex(ValueError, "verified policy-only constituent"): - self.select(base=self.main, head=group, pull_heads=[sdk_head]) - - def test_queue_pin_update_isolated_per_constituent(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - (self.repo / "main-new.txt").write_text("later main") - later = self.commit("main later") - self.git("update-ref", "refs/remotes/origin/main", later) - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**self.policy, "next_policy": {"repository": REPO, "main_sha": later}}) - pin_head = self.commit("isolated pin update") - self.git("merge", "--no-ff", "-qm", "queue group", self.head) - group = self.git("rev-parse", "HEAD") - with mock.patch.object(selector, "require_human_policy_review", return_value=3) as review: - result = self.select(head=group, main=later, pull_heads=[pin_head, self.head]) - self.assertEqual(result["policy_sha"], self.main) - self.assertEqual(result["policy_reviews"], []) - review.assert_not_called() - # A combined SDK/pin change in one PR is still forbidden. - with self.assertRaisesRegex(ValueError, "policy-only"): - self.select(head=group, main=later) - with self.assertRaisesRegex(ValueError, "policy-only"): - self.select(head=group, main=later, pull_heads=[group]) - - def advanced_pin_base(self) -> tuple[str, str]: - self.git("checkout", "-q", "--detach", self.main) - (self.repo / "later-main.txt").write_text("later main") - later = self.commit("later approved main") - self.git("update-ref", "refs/remotes/origin/main", later) - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**self.policy, "next_policy": {"repository": REPO, "main_sha": later}}) - return later, self.commit("reviewed pin update on next") - - def test_stale_sdk_branch_uses_current_protected_pin(self) -> None: - later, base = self.advanced_pin_base() - with mock.patch.object(selector, "require_human_policy_review") as review: - result = self.select(base=base, main=later) - self.assertEqual(result["policy_sha"], later) - self.assertEqual(result["policy_reviews"], []) - review.assert_not_called() - - def test_queue_stale_sdk_constituent_inherits_protected_pin(self) -> None: - later, base = self.advanced_pin_base() - self.git("merge", "--no-ff", "-qm", "queue SDK branch", self.head) - group = self.git("rev-parse", "HEAD") - with mock.patch.object(selector, "require_human_policy_review") as review: - result = self.select(base=base, head=group, main=later, pull_heads=[self.head]) - self.assertEqual(result["policy_sha"], later) - self.assertEqual(result["policy_reviews"], []) - review.assert_not_called() - - def test_stale_actual_pin_edit_still_requires_isolation(self) -> None: - later, base = self.advanced_pin_base() - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**self.policy, "next_policy": {"repository": REPO, "main_sha": later}}) - edited = self.commit("PR independently edits pin") - with mock.patch.object(selector, "require_human_policy_review", return_value=3) as review: - result = self.select(base=base, head=edited, main=later) - self.assertEqual(result["policy_reviews"], []) - review.assert_not_called() - (self.repo / "mixed-sdk.txt").write_text("SDK change") - mixed = self.commit("mix policy and SDK") - with self.assertRaisesRegex(ValueError, "policy-only"): - self.select(base=base, head=mixed, main=later) - with self.assertRaisesRegex(ValueError, "policy-only"): - self.select(base=base, head=mixed, main=later, pull_heads=[mixed]) - - def test_stale_sdk_budget_after_target_decrease_needs_no_review(self) -> None: - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**self.policy, "next_policy": self.pin, "max_custom_patch_lines": 3000}) - base = self.commit("lower protected next budget") - self.git("merge", "--no-ff", "-qm", "queue stale SDK branch", self.head) - group = self.git("rev-parse", "HEAD") - for head, heads in ((self.head, None), (group, [self.head])): - with ( - self.subTest(head=head), - mock.patch.object(selector, "require_human_policy_review") as review, - ): - result = self.select(base=base, head=head, pull_heads=heads) - self.assertEqual(result["policy_sha"], self.main) - self.assertEqual(result["base_sha"], base) - self.assertEqual(result["policy_reviews"], []) - self.assertEqual(budget.read_budget(self.repo, base), 3000) - review.assert_not_called() - - def test_actual_budget_increase_is_reviewed_even_if_target_increased_further(self) -> None: - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**self.policy, "next_policy": self.pin, "max_custom_patch_lines": 6000}) - base = self.commit("reviewed target budget increase") - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**self.policy, "next_policy": self.pin, "max_custom_patch_lines": 5000}) - edited = self.commit("PR actually raises budget") - for heads in (None, [edited]): - with ( - mock.patch.object( - selector, - "require_human_policy_review", - side_effect=ValueError("human required"), - ), - self.assertRaisesRegex(ValueError, "human"), - ): - self.select(base=base, head=edited, pull_heads=heads) - (self.repo / "mixed-sdk.txt").write_text("SDK change") - mixed = self.commit("mix SDK and actual budget increase") - with self.assertRaisesRegex(ValueError, "policy-only"): - self.select(base=base, head=mixed) - - def test_selector_outputs_transport_review_receipt_as_json(self) -> None: - event = self.repo / "event.json" - event.write_text("{}") - out, output = self.repo / "selection.json", self.repo / "outputs" - selected = { - "schema_version": 1, - "policy_reviews": [{"number": 3, "head_sha": self.head}], - "human_review_rules": selector.HUMAN_REVIEW_RULES, - "bootstrap": False, - } - args = [ - "budget", - "select-policy", - "--repo", - str(self.repo / "objects.git"), - "--repository", - REPO, - "--event-path", - str(event), - "--trusted-sha", - self.main, - "--out", - str(out), - ] - with ( - mock.patch.object(sys, "argv", args), - mock.patch.dict(selector.os.environ, {"GITHUB_OUTPUT": str(output)}), - mock.patch.object(selector, "select_github_policy", return_value=selected), - ): - self.assertEqual(selector.main(), 0) - values = dict(line.split("=", 1) for line in output.read_text().splitlines()) - self.assertEqual(json.loads(values["policy_reviews"]), selected["policy_reviews"]) - self.assertEqual(json.loads(values["human_review_rules"]), selector.HUMAN_REVIEW_RULES) - self.assertFalse(json.loads(values["bootstrap"])) - self.assertEqual(json.loads(out.read_text())["schema_version"], 1) - - def test_follow_up_comments_preserve_policy_approval(self) -> None: - approval = self.approved() - for state in ("COMMENTED", "PENDING"): - follow_up = self.approved(id=11, state=state, body="A non-blocking follow-up") - for reviews in ([approval, follow_up], [follow_up, approval]): - with ( - self.subTest(state=state, reviews=reviews), - mock.patch.object(selector.report, "api", side_effect=self.review_api(reviews)), - ): - self.assertEqual(selector.require_human_policy_review(REPO, self.head), 3) - - def test_comments_do_not_restore_revoked_policy_approval(self) -> None: - comment = self.approved(id=12, state="COMMENTED", body="A follow-up comment") - cases = [ - [self.approved(), self.approved(id=11, state="CHANGES_REQUESTED"), comment], - [self.approved(state="DISMISSED"), comment], - [self.approved(body="Attestation removed"), comment], - [comment], - ] - for reviews in cases: - with ( - self.subTest(reviews=reviews), - mock.patch.object(selector.report, "api", side_effect=self.review_api(reviews)), - self.assertRaisesRegex(ValueError, "human"), - ): - selector.require_human_policy_review(REPO, self.head) - - def test_forged_bot_stale_dismissed_untrusted_reviews_rejected(self) -> None: - self.git("checkout", "-q", "--detach", self.base) - self.write_policy( - { - **self.policy, - "next_policy": self.pin, - "max_custom_patch_lines": self.policy["max_custom_patch_lines"] + 1, - } - ) - self.head = self.commit("bootstrap pin with budget increase") - cases = [ - [], - [self.approved(body="")], - [self.approved(body="Looks good to me")], - [self.approved(body=selector.HUMAN_REVIEW_RULES["attestation_prefix"] + "f" * 40)], - [self.approved(state="COMMENTED", body="APPROVED by a human")], - [self.approved(user={"type": "Bot", "login": "reviewer[bot]"})], - [self.approved(commit_id="a" * 40)], - [self.approved(author_association="NONE")], - [ - self.approved( - user={"type": "User", "login": "dpiet-oai"}, - body="", - ) - ], - [ - self.approved( - user={"type": "User", "login": "markstuart-oai"}, - body="Reviewed `f4c5de3e28ea4c21c2a3d230d06341c3c25a0004`. No actionable findings in the two workflow changes. Adding `next` retains the existing CodeQL and exported-API jobs, uses the actual PR or merge-group comparison base, and leaves merge-group triggers and main-only release/credential guards intact.\n\nSource-only review; I did not execute candidate code or workflows. Current-head CodeQL and exported-API checks passed. Some hosted ecosystem and Node-version tests are still pending. Branch protection settings and the author's branch-creation claims were not independently verified; required CI and review gates still apply.", - ) - ], - [self.approved(body="Automated source review.")], - [self.approved(body="This does not substitute for human approval.")], - [self.approved(body="This does not constitute human approval.")], - [self.approved(body="This is not a human approving review.")], - [self.approved(), self.approved(id=11, state="DISMISSED")], - ] - for reviews in cases: - with ( - self.subTest(reviews=reviews), - mock.patch.object(selector.report, "api", side_effect=self.review_api(reviews)), - self.assertRaisesRegex(ValueError, "human"), - ): - self.select(base=self.main) - - def test_bootstrap_budget_increase_keeps_human_proof_for_pr_and_queue(self) -> None: - self.git("checkout", "-q", "--detach", self.base) - self.write_policy( - { - **self.policy, - "next_policy": self.pin, - "max_custom_patch_lines": self.policy["max_custom_patch_lines"] + 1, - } - ) - self.head = self.commit("bootstrap pin with budget increase") - for heads in (None, [self.head]): - with ( - mock.patch.object(selector.report, "api", side_effect=self.review_api([])), - self.assertRaisesRegex(ValueError, "human"), - ): - self.select(base=self.main, pull_heads=heads) - with mock.patch.object( - selector.report, "api", side_effect=self.review_api([self.approved()]) - ): - result = self.select(base=self.main, pull_heads=heads) - self.assertEqual(result["policy_reviews"], [{"number": 3, "head_sha": self.head}]) - - def test_authorless_review_does_not_block_valid_approval(self) -> None: - self.git("checkout", "-q", "--detach", self.base) - policy = selector.read_policy(self.repo, self.base) - self.write_policy( - {**policy, "max_custom_patch_lines": policy["max_custom_patch_lines"] + 1} - ) - self.head = self.commit("increase budget") - reviews = [self.approved(id=9, user=None), self.approved()] - with mock.patch.object(selector.report, "api", side_effect=self.review_api(reviews)): - self.assertEqual( - self.select()["policy_reviews"], - [{"number": 3, "head_sha": self.head}], - ) - - def test_budget_increase_without_pin_change_requires_human(self) -> None: - self.git("checkout", "-q", "--detach", self.base) - policy = selector.read_policy(self.repo, self.base) - self.write_policy( - {**policy, "max_custom_patch_lines": policy["max_custom_patch_lines"] + 1} - ) - self.head = self.commit("increase budget without changing pin") - for reviews in ([], [self.approved(body="Automated source review.")]): - with ( - mock.patch.object(selector.report, "api", side_effect=self.review_api(reviews)), - self.assertRaisesRegex(ValueError, "human"), - ): - self.select() - with mock.patch.object( - selector.report, "api", side_effect=self.review_api([self.approved()]) - ): - self.assertEqual(self.select()["policy_sha"], self.main) - - def test_changed_authenticated_main_ref_rejected(self) -> None: - self.git("update-ref", "refs/remotes/origin/main", self.head) - with self.assertRaisesRegex(ValueError, "changed"): - self.select() - - def test_go_ratchet_activation_requires_human_for_pr_and_queue(self) -> None: - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**selector.read_policy(self.repo, self.base), "ratchet": "go"}) - self.head = self.commit("activate Go ratchet without changing pin or ceiling") - for pull_heads in (None, [self.head]): - for reviews in ([], [self.approved(body="Automated source review.")]): - with ( - self.subTest(pull_heads=pull_heads, reviews=reviews), - mock.patch.object(selector.report, "api", side_effect=self.review_api(reviews)), - self.assertRaisesRegex(ValueError, "human"), - ): - self.select(pull_heads=pull_heads) - with mock.patch.object( - selector.report, "api", side_effect=self.review_api([self.approved()]) - ): - selected = self.select(pull_heads=pull_heads) - self.assertEqual(selected["policy_sha"], self.main) - self.assertEqual(selected["policy_reviews"], [{"number": 3, "head_sha": self.head}]) - - def test_wrong_origin_rejected(self) -> None: - self.git("remote", "set-url", "origin", "https://github.com/attacker/example.git") - with self.assertRaisesRegex(ValueError, "origin"): - self.select() - - def test_same_repository_github_url_spellings_are_accepted(self) -> None: - for origin in ( - f"https://github.com/{REPO}.git", - f"https://github.com/{REPO}", - f"git@github.com:{REPO}.git", - ): - with self.subTest(origin=origin): - self.git("remote", "set-url", "origin", origin) - self.assertEqual(self.select()["policy_sha"], self.main) - - def test_policy_update_uses_old_base_checker_and_requires_human(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - self.write_policy({**self.policy, "max_custom_patch_lines": 5000}) - newer = self.commit("new reviewed main policy") - self.git("update-ref", "refs/remotes/origin/main", newer) - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**self.policy, "next_policy": {"repository": REPO, "main_sha": newer}}) - self.head = self.commit("request new pin") - with mock.patch.object( - selector.report, "api", side_effect=self.review_api([self.approved()]) - ): - self.assertEqual(self.select(main=newer)["policy_sha"], self.main) - with ( - mock.patch.object(selector.report, "api", side_effect=self.review_api([])), - self.assertRaisesRegex(ValueError, "human"), - ): - self.select(main=newer) - - def test_pin_effective_policy_review_for_pr_and_queue(self) -> None: - cases = ( - ({**self.policy, "max_custom_patch_lines": 4000}, False), - ({**self.policy, "max_custom_patch_lines": 3999}, False), - ({**self.policy, "max_custom_patch_lines": 5000}, True), - ({**self.policy, "ratchet": "go"}, True), - ({"schema_version": 2, "max_custom_patch_lines": 0, "generated_files": "strict"}, True), - ) - for proposed, requires_human in cases: - self.git("checkout", "-q", "--detach", self.main) - self.write_policy(proposed) - newer = self.commit("adopt proposed effective policy") - self.git("update-ref", "refs/remotes/origin/main", newer) - for bootstrap in (False, True): - base = self.main if bootstrap else self.base - self.git("checkout", "-q", "--detach", base) - self.write_policy( - {**self.policy, "next_policy": {"repository": REPO, "main_sha": newer}} - ) - self.head = self.commit("request pin without changing local budget") - for heads in (None, [self.head]): - with self.subTest(proposed=proposed, bootstrap=bootstrap, heads=heads): - needs_proof = requires_human and not bootstrap - with mock.patch.object( - selector, - "require_human_policy_review", - side_effect=ValueError("human review missing"), - ) as review: - if needs_proof: - with self.assertRaisesRegex(ValueError, "human"): - self.select(base=base, main=newer, pull_heads=heads) - review.assert_called_once_with(REPO, self.head, None) - else: - result = self.select(base=base, main=newer, pull_heads=heads) - review.assert_not_called() - self.assertEqual(result["policy_reviews"], []) - if needs_proof: - with mock.patch.object( - selector.report, - "api", - side_effect=self.review_api([self.approved()]), - ): - result = self.select(base=base, main=newer, pull_heads=heads) - self.assertEqual( - result["policy_reviews"], [{"number": 3, "head_sha": self.head}] - ) - - def test_legacy_checker_pin_review_receipt_for_pr_and_queue(self) -> None: - declarations = ( - None, - 'PIN_REVIEW_POLICY = "unknown"\n', - 'PIN_REVIEW_POLICY = "effective-policy-v1"\n' * 2, - ) - for declaration in declarations: - self.git("checkout", "-q", "--detach", self.main) - checker = self.repo / "scripts/castiron/custom_code_policy.py" - if declaration is None: - checker.unlink() - else: - checker.write_text(declaration) - legacy = self.commit("legacy review contract") - checker.write_text('PIN_REVIEW_POLICY = "effective-policy-v1"\n') - newer = self.commit("modern review contract") - self.git("update-ref", "refs/remotes/origin/main", newer) - self.git("checkout", "-q", "--detach", legacy) - self.write_policy( - {**self.policy, "next_policy": {"repository": REPO, "main_sha": legacy}} - ) - base = self.commit("protected legacy checker") - self.write_policy( - {**self.policy, "next_policy": {"repository": REPO, "main_sha": newer}} - ) - self.head = self.commit("forward same-budget pin") - for heads in (None, [self.head]): - with self.subTest(declaration=declaration, heads=heads): - with ( - mock.patch.object(selector.report, "api", side_effect=self.review_api([])), - self.assertRaisesRegex(ValueError, "human"), - ): - self.select(base=base, main=newer, pull_heads=heads) - with mock.patch.object( - selector.report, "api", side_effect=self.review_api([self.approved()]) - ): - result = self.select(base=base, main=newer, pull_heads=heads) - self.assertEqual(result["policy_sha"], legacy) - self.assertEqual( - result["policy_reviews"], [{"number": 3, "head_sha": self.head}] - ) - - def test_rollback_pin_keeps_human_proof_for_pr_and_queue(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - (self.repo / "checker-fix.txt").write_text("reviewed checker fix") - newer = self.commit("newer main checker with unchanged policy") - self.git("update-ref", "refs/remotes/origin/main", newer) - self.git("checkout", "-q", "--detach", self.base) - self.write_policy({**self.policy, "next_policy": {"repository": REPO, "main_sha": newer}}) - advanced_base = self.commit("adopt newer checker") - self.write_policy({**self.policy, "next_policy": self.pin}) - self.head = self.commit("roll back checker without changing budget") - for heads in (None, [self.head]): - with ( - mock.patch.object(selector.report, "api", side_effect=self.review_api([])), - self.assertRaisesRegex(ValueError, "human"), - ): - self.select(base=advanced_base, main=newer, pull_heads=heads) - with mock.patch.object( - selector.report, "api", side_effect=self.review_api([self.approved()]) - ): - result = self.select(base=advanced_base, main=newer, pull_heads=heads) - self.assertEqual(result["policy_sha"], newer) - self.assertEqual(result["policy_reviews"], [{"number": 3, "head_sha": self.head}]) - - def test_stale_pin_compares_current_protected_policy_for_pr_and_queue(self) -> None: - for protected_policy in ( - self.policy, - {**self.policy, "max_custom_patch_lines": 3999}, - {"schema_version": 2, "max_custom_patch_lines": 0, "generated_files": "strict"}, - ): - self.git("checkout", "-q", "--detach", self.main) - (self.repo / "checker-fix.txt").write_text("intermediate checker") - intermediate = self.commit("intermediate main checker") - self.write_policy(protected_policy) - (self.repo / "checker-fix.txt").write_text("protected checker") - current = self.commit("current main checker") - (self.repo / "checker-fix.txt").write_text("forward checker") - forward = self.commit("forward main checker") - self.git("update-ref", "refs/remotes/origin/main", forward) - self.git("checkout", "-q", "--detach", self.base) - self.write_policy( - {**self.policy, "next_policy": {"repository": REPO, "main_sha": current}} - ) - protected_base = self.commit("advance protected next pin") - for proposed in (intermediate, current, forward): - self.git("checkout", "-q", "--detach", self.base) - self.write_policy( - {**self.policy, "next_policy": {"repository": REPO, "main_sha": proposed}} - ) - self.head = self.commit("stale policy-only PR") - for heads in (None, [self.head]): - with self.subTest(policy=protected_policy, proposed=proposed, heads=heads): - with mock.patch.object( - selector.report, "api", side_effect=self.review_api([]) - ): - if proposed == intermediate: - with self.assertRaisesRegex(ValueError, "human"): - self.select(base=protected_base, main=forward, pull_heads=heads) - else: - result = self.select( - base=protected_base, main=forward, pull_heads=heads - ) - self.assertEqual(result["policy_reviews"], []) - with mock.patch.object( - selector.report, "api", side_effect=self.review_api([self.approved()]) - ): - result = self.select( - base=protected_base, main=forward, pull_heads=heads - ) - self.assertEqual(result["policy_sha"], current) - self.assertEqual( - result["policy_reviews"], - [{"number": 3, "head_sha": self.head}] - if proposed == intermediate - else [], - ) - - def test_initial_older_pin_keeps_human_proof_for_pr_and_queue(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - (self.repo / "checker-fix.txt").write_text("reviewed bootstrap checker fix") - newer = self.commit("newer main checker with unchanged policy") - self.git("update-ref", "refs/remotes/origin/main", newer) - for pin_sha in (self.main, newer): - self.git("checkout", "-q", "--detach", self.main) - self.write_policy( - {**self.policy, "next_policy": {"repository": REPO, "main_sha": pin_sha}} - ) - self.head = self.commit("initial checker pin") - for heads in (None, [self.head]): - with mock.patch.object(selector.report, "api", side_effect=self.review_api([])): - if pin_sha == self.main: - with self.assertRaisesRegex(ValueError, "human"): - self.select(base=self.main, main=newer, pull_heads=heads) - else: - result = self.select(base=self.main, main=newer, pull_heads=heads) - self.assertEqual(result["policy_reviews"], []) - self.assertEqual(result["policy_sha"], newer) - with mock.patch.object( - selector.report, "api", side_effect=self.review_api([self.approved()]) - ): - result = self.select(base=self.main, main=newer, pull_heads=heads) - self.assertEqual(result["policy_sha"], newer) - self.assertTrue(result["bootstrap"]) - self.assertEqual( - result["policy_reviews"], - [{"number": 3, "head_sha": self.head}] if pin_sha == self.main else [], - ) - - def test_bootstrap_uses_active_main_policy_for_pr_and_queue(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - self.write_policy( - {"schema_version": 2, "max_custom_patch_lines": 0, "generated_files": "strict"} - ) - current = self.commit("main activates strict zero policy") - self.git("update-ref", "refs/remotes/origin/main", current) - for proposed in (self.main, current): - self.git("checkout", "-q", "--detach", self.main) - self.write_policy( - {**self.policy, "next_policy": {"repository": REPO, "main_sha": proposed}} - ) - self.head = self.commit("initial pin with unchanged legacy local ceiling") - for heads in (None, [self.head]): - with self.subTest(proposed=proposed, heads=heads): - with mock.patch.object(selector.report, "api", side_effect=self.review_api([])): - if proposed == self.main: - with self.assertRaisesRegex(ValueError, "human"): - self.select(base=self.main, main=current, pull_heads=heads) - else: - result = self.select(base=self.main, main=current, pull_heads=heads) - self.assertEqual(result["policy_reviews"], []) - self.assertEqual(result["policy_sha"], current) - with mock.patch.object( - selector.report, "api", side_effect=self.review_api([self.approved()]) - ): - result = self.select(base=self.main, main=current, pull_heads=heads) - self.assertTrue(result["bootstrap"]) - self.assertEqual(result["policy_sha"], current) - self.assertEqual( - result["policy_reviews"], - [{"number": 3, "head_sha": self.head}] if proposed == self.main else [], - ) - - def test_incapable_policy_is_not_executed(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - (self.repo / "scripts/castiron/custom_code_budget.py").write_text( - "raise RuntimeError('must not execute')\n" - ) - incapable = self.commit("old checker without pin protocol") - self.git("update-ref", "refs/remotes/origin/main", incapable) - self.write_policy( - {**self.policy, "next_policy": {"repository": REPO, "main_sha": incapable}} - ) - base = self.commit("pin incapable checker") - with self.assertRaisesRegex(ValueError, "does not support"): - self.select(base=base, head=base, main=incapable) - - def test_main_selection_unchanged(self) -> None: - result = selector.select_policy(self.repo, REPO, "main", self.main, self.head, self.main) - self.assertEqual(result["policy_sha"], self.main) - self.assertFalse(result["bootstrap"]) - - def prepare_snapshot(self) -> None: - self.git("checkout", "-q", "--detach", self.main) - self.base = self.main # unpinned legacy next - self.strict = { - "schema_version": 2, - "max_custom_patch_lines": 0, - "generated_files": "strict", - } - self.write_policy(self.strict) - self.write("sdk.txt", "approved main output\n") - self.write( - ".castiron.stats.yml", - "schema_version: 1\ngeneration_id: 550e8400-e29b-41d4-a716-446655440000\ncodegen_sha: " - + "a" * 40 - + "\npublic_codegen_sha: " - + "a" * 40 - + "\n", - ) - self.main = self.commit("strict main snapshot") - self.git("update-ref", "refs/remotes/origin/main", self.main) - self.write_policy( - {**self.strict, "next_policy": {"repository": REPO, "main_sha": self.main}} - ) - tree_commit = self.commit("adopt exact main snapshot") - self.head = self.git( - "commit-tree", - f"{tree_commit}^{{tree}}", - "-p", - self.base, - "-p", - self.main, - "-m", - "canonical snapshot", - ) - self.git("checkout", "--detach", "-q", self.head) - - def test_exact_main_snapshot_pr_and_queue(self) -> None: - self.prepare_snapshot() - for queue in (False, True): - with ( - self.subTest(queue=queue), - mock.patch.object( - selector.report, "api", side_effect=AssertionError("no human proof needed") - ), - ): - receipt = self.select(pull_heads=[self.head] if queue else None) - self.assertEqual(receipt["main_snapshot"], self.main) - self.assertEqual(receipt["policy_sha"], self.main) - self.assertEqual(receipt["policy_reviews"], []) - - def test_snapshot_rejects_additional_bytes_modes_or_metadata(self) -> None: - self.prepare_snapshot() - valid = self.head - for kind in ("content", "mode", "extra", "stats", "symlink"): - for queue in (False, True): - with self.subTest(kind=kind, queue=queue): - self.git("reset", "--hard", valid) - if kind == "content": - self.write("sdk.txt", "unreviewed output\n") - elif kind == "mode": - (self.repo / "sdk.txt").chmod(0o755) - elif kind == "extra": - self.write("extra.txt", "extra") - elif kind == "stats": - self.write(".castiron.stats.yml", "schema_version: 1\n") - else: - (self.repo / "sdk.txt").unlink() - (self.repo / "sdk.txt").symlink_to(".castiron.stats.yml") - self.head = self.commit("invalid adoption") - with self.assertRaises(ValueError): - self.select(pull_heads=[self.head] if queue else None) - - def test_snapshot_rejects_main_movement_and_nonancestor_target(self) -> None: - self.prepare_snapshot() - old_main = self.main - candidate = self.head - self.git("checkout", "-q", "--detach", old_main) - self.write("sdk.txt", "new approved main\n") - self.main = self.commit("main advances") - self.git("update-ref", "refs/remotes/origin/main", self.main) - with self.assertRaises(ValueError): - self.select(head=candidate) - self.main = old_main - self.git("update-ref", "refs/remotes/origin/main", self.main) - self.git("checkout", "-q", "--detach", self.base) - self.write("private-only.txt", "unmerged next history") - self.base = self.commit("divergent next") - with self.assertRaises(ValueError): - self.select(head=candidate) - - def test_snapshot_rejects_private_history_but_accepts_verified_queue_wrapper(self) -> None: - self.prepare_snapshot() - constituent = self.head - wrapper = self.git( - "commit-tree", - f"{constituent}^{{tree}}", - "-p", - self.base, - "-p", - constituent, - "-m", - "queue wrapper", - ) - self.assertEqual( - self.select(head=wrapper, pull_heads=[constituent])["main_snapshot"], self.main - ) - with self.assertRaises(ValueError): - self.select(head=wrapper) # A wrapper is not an independently verified PR head. - self.write("private.txt", "private-only historical bytes") - self.commit("private historical commit") - (self.repo / "private.txt").unlink() - reverted = self.commit("revert private historical bytes") - self.assertEqual( - self.git("rev-parse", f"{reverted}^{{tree}}"), - self.git("rev-parse", f"{constituent}^{{tree}}"), - ) - for heads in (None, [reverted]): - with self.subTest(queue=heads is not None), self.assertRaises(ValueError): - self.select(head=reverted, pull_heads=heads) - - def test_snapshot_rejects_multiple_queue_constituents(self) -> None: - self.prepare_snapshot() - with self.assertRaisesRegex(ValueError, "one exact"): - self.select(pull_heads=[self.head, self.head]) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/castiron/test_custom_code_policy_source.py b/scripts/castiron/test_custom_code_policy_source.py deleted file mode 100644 index 1b60637c4..000000000 --- a/scripts/castiron/test_custom_code_policy_source.py +++ /dev/null @@ -1,198 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -"""Current-main policy selection and review authorization regressions.""" - -from __future__ import annotations - -import copy -import json -import sys -import tempfile -import unittest -import unittest.mock as mock # noqa: PLR0402 - keep unittest imports consistent -from pathlib import Path -from typing import Any - -import custom_code_budget as budget -import custom_code_policy as selector - -REPO = "openai/example" - - -POLICY_SOURCE_ROOT = "repos/" + REPO -CANONICAL_POLICY_WORKFLOW = ".github/workflows/castiron-custom-code.yml" -CI_POLICY_WORKFLOW = ".github/workflows/ci.yml" - - -class PolicySourceWorkflowTests(unittest.TestCase): - def setUp(self) -> None: # pyright: ignore[reportImplicitOverride] - self.temp = tempfile.TemporaryDirectory() - self.addCleanup(self.temp.cleanup) - self.repo = Path(self.temp.name) / "objects" - self.workflow_run: dict[str, Any] = { - "id": 42, - "workflow_id": 456, - "repository": {"full_name": REPO}, - "head_sha": "a" * 40, - "status": "completed", - "event": "pull_request", - "run_attempt": 3, - "path": CANONICAL_POLICY_WORKFLOW + "@refs/pull/12/merge", - } - self.event: dict[str, Any] = { - "repository": {"full_name": REPO}, - "workflow_run": copy.deepcopy(self.workflow_run), - } - self.branch = "next" - self.select = self.start_patch( - mock.patch.object(selector, "select_policy", return_value={"selected": True}) - ) - self.start_patch(mock.patch.object(selector.report, "api", side_effect=self.api)) - self.start_patch( - mock.patch.object(selector.report, "current_pull", side_effect=self.current_pull) - ) - self.start_patch( - mock.patch.object( - selector.report, - "git", - side_effect=self.git_result, - ) - ) - - def current_pull(self, *args: Any) -> tuple[int, str]: - return 12, self.branch - - def git_result(self, repo: Path, *args: str) -> bytes: - return b"b" * 40 if args[0] == "rev-parse" else b"" - - def start_patch(self, patcher: Any) -> Any: - result = patcher.start() - self.addCleanup(patcher.stop) - return result - - def api(self, method: str, path: str, *args: Any) -> Any: - return { - POLICY_SOURCE_ROOT: {"default_branch": "main"}, - POLICY_SOURCE_ROOT + "/actions/runs/42": self.workflow_run, - POLICY_SOURCE_ROOT + "/actions/workflows/castiron-custom-code.yml": { - "id": 456, - "path": CANONICAL_POLICY_WORKFLOW, - }, - POLICY_SOURCE_ROOT + "/actions/workflows/ci.yml": { - "id": 456, - "path": CI_POLICY_WORKFLOW, - }, - POLICY_SOURCE_ROOT + "/git/ref/heads/main": {"object": {"sha": "c" * 40}}, - POLICY_SOURCE_ROOT + "/git/ref/heads/next": {"object": {"sha": "b" * 40}}, - }[path] - - def call(self, **kwargs: Any) -> dict[str, Any]: - return selector.select_github_policy(self.repo, REPO, self.event, "c" * 40, **kwargs) - - def use_ci(self) -> None: - self.workflow_run["path"] = CI_POLICY_WORKFLOW + "@refs/pull/12/merge" - - def test_lookalike_workflow_cannot_select_policy(self) -> None: - self.workflow_run["path"] = CANONICAL_POLICY_WORKFLOW + "@evil.yml" - self.workflow_run["workflow_id"] = 789 - with self.assertRaisesRegex(ValueError, "canonical workflow"): - self.call() - self.select.assert_not_called() - - def test_default_canonical_unchanged(self) -> None: - self.branch = "main" - self.assertEqual(self.call(), {"selected": True}) - - def test_explicit_ci_uses_same_base_pin_selector(self) -> None: - self.use_ci() - self.assertEqual(self.call(source_workflow=CI_POLICY_WORKFLOW), {"selected": True}) - self.select.assert_called_once_with( - self.repo, - REPO, - "next", - "b" * 40, - "a" * 40, - "c" * 40, - pull_heads=None, - pull_numbers={"a" * 40: 12}, - ) - - def test_candidate_event_cannot_select_ci_adapter(self) -> None: - self.use_ci() - self.event["source_workflow"] = CI_POLICY_WORKFLOW - with self.assertRaisesRegex(ValueError, "source run"): - self.call() - self.select.assert_not_called() - - def test_unknown_workflow_rejected(self) -> None: - with self.assertRaisesRegex(ValueError, "source workflow"): - self.call(source_workflow=".github/workflows/evil.yml") - self.select.assert_not_called() - - def test_ci_adapter_cannot_retarget_main(self) -> None: - self.use_ci() - self.branch = "main" - with self.assertRaisesRegex(ValueError, "restricted to next"): - self.call(source_workflow=CI_POLICY_WORKFLOW) - self.select.assert_not_called() - - def test_wrong_actual_workflow(self) -> None: - with self.assertRaisesRegex(ValueError, "source run"): - self.call(source_workflow=CI_POLICY_WORKFLOW) - self.select.assert_not_called() - - def test_stale_or_mismatched_authenticated_identity(self) -> None: - self.use_ci() - for field, value in [ - ("head_sha", "d" * 40), - ("run_attempt", 4), - ("status", "in_progress"), - ("event", "push"), - ("repository", {"full_name": "attacker/example"}), - ]: - with self.subTest(field=field): - old = self.workflow_run[field] - self.workflow_run[field] = value - with self.assertRaisesRegex(ValueError, "source run"): - self.call(source_workflow=CI_POLICY_WORKFLOW) - self.workflow_run[field] = old - self.select.assert_not_called() - - -class SelectorInterfaceTests(unittest.TestCase): - def test_review_rules_cli_exports_the_canonical_predicate(self) -> None: - import subprocess - - assert budget.__file__ is not None - completed = subprocess.run( - [sys.executable, "-I", budget.__file__, "review-rules"], - capture_output=True, - text=True, - check=True, - ) - self.assertEqual(json.loads(completed.stdout), selector.HUMAN_REVIEW_RULES) - - def test_budget_cli_forwards_to_adjacent_selector(self) -> None: - args = ["select-policy", "--repo", "objects", "--repository", REPO] - with ( - mock.patch.object(sys, "argv", ["budget", *args]), - mock.patch.object(budget.policy, "main", return_value=0) as select, - ): - self.assertEqual(budget.main(), 0) - select.assert_called_once_with(args) - - def test_isolated_cli_loads_only_adjacent_modules(self) -> None: - import subprocess - - for module in (budget, selector): - assert module.__file__ is not None - completed = subprocess.run( - [sys.executable, "-I", module.__file__, "select-policy", "--help"], - capture_output=True, - text=True, - check=True, - ) - self.assertIn("--source-workflow", completed.stdout) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/castiron/test_custom_code_publication.py b/scripts/castiron/test_custom_code_publication.py deleted file mode 100644 index 205fab507..000000000 --- a/scripts/castiron/test_custom_code_publication.py +++ /dev/null @@ -1,600 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -"""Run the trusted publishers against offline GitHub state.""" - -from __future__ import annotations - -import hashlib -import json -import shutil -import unittest -from pathlib import Path -from typing import Any -from unittest import mock - -import custom_code_report as report -from custom_code_test_support import ( - WORKFLOW, - GitTestCase, - run_publisher, - run_workflow_script, - source_head, - source_run, -) - - -class WorkflowTests(unittest.TestCase): - @unittest.skipUnless(shutil.which("node"), "GitHub Actions JavaScript runtime") - def test_workflow_path_filters_accept_refs_without_accepting_other_files(self) -> None: - compute = WORKFLOW.read_text().split("\n compute:\n", 1)[1] - condition = next( - line.removeprefix(" if: ") - for line in compute.splitlines() - if line.startswith(" if: ") - ) - expected_path = ".github/workflows/castiron-custom-code.yml" - cases = [ - (expected_path, True), - (expected_path + "@main", True), - (expected_path + "@refs/heads/next", True), - (expected_path + "@refs/pull/42/merge", True), - (".github/workflows/other.yml@main", False), - (expected_path + ".bak@main", False), - (expected_path + "/other@main", False), - ("other/" + expected_path + "@main", False), - ("", False), - ] - head = "a" * 40 - pull = { - "number": 3, - "state": "open", - "head": source_head(head), - "base": {"ref": "next", "repo": {"full_name": "openai/example"}}, - } - for path, accepted in cases: - for event in ("pull_request", "merge_group", "push"): - with self.subTest(path=path, event=event): - run = {**source_run(head), "id": 20, "path": path, "event": event} - payload = { - "context": { - "payload": {"workflow_run": run}, - "runId": 20, - "repo": {"owner": "openai", "repo": "example"}, - "serverUrl": "https://github.com", - }, - "current": pull, - } - # Evaluate the emitted job condition with its Actions builtins. - script = ( - "const fromJSON = JSON.parse;\n" - "const contains = (items, value) => items.includes(value);\n" - "const startsWith = (value, prefix) => value.startsWith(prefix);\n" - "github.event = context.payload;\n" - f"if ({condition}) await github.rest.issues.createComment({{}});" - ) - self.assertEqual( - bool(run_workflow_script(script, payload)), - accepted and event in ("pull_request", "merge_group"), - ) - self.assertEqual( - bool(run_publisher("Publish a trusted failure status", payload)), - accepted and event == "pull_request", - ) - - def test_candidate_tests_cannot_modify_credentialed_reporter_runner(self) -> None: - producer = (WORKFLOW.parent / "castiron-custom-code.yml").read_text() - tests, reporter = producer.split("\n report:\n", 1) - tests = tests.split("\n candidate-tests:\n", 1)[1] - self.assertIn("runs-on: ubuntu-latest", tests) - self.assertIn("persist-credentials: false", tests) - self.assertIn("python3 -m unittest discover", tests) - self.assertNotIn("GH_TOKEN", tests) - self.assertNotIn("upload-artifact@", tests) - self.assertNotIn("unittest", reporter) - self.assertIn("GH_TOKEN:", reporter) - self.assertNotIn("needs: candidate-tests", reporter) - - @unittest.skipUnless(shutil.which("node"), "GitHub Actions JavaScript runtime") - def test_trusted_failure_publisher_updates_one_current_comment(self) -> None: - head = "a" * 40 - run = {**source_run(head), "id": 20} - current = { - "number": 3, - "state": "open", - "head": source_head(head), - "base": {"ref": "main", "repo": {"full_name": "openai/example"}}, - } - previous = { - "id": 42, - "user": {"type": "Bot", "login": "github-actions[bot]"}, - "body": report.MARKER + "\n", - } - cases: dict[str, tuple[list[dict[str, Any]], dict[str, Any], str | None]] = { - "create": ([], current, "create"), - "update": ([previous], current, "update"), - "changed head": ([previous], {**current, "head": {"sha": "c" * 40}}, None), - "newer comment": ( - [{**previous, "body": report.MARKER + "\n"}], - current, - None, - ), - } - for fork in (False, True): - for name, (comments, pull, expected) in cases.items(): - with self.subTest(name=name, fork=fork): - event: dict[str, Any] = {**run, "pull_requests": []} if fork else run - writes = run_publisher( - "Publish a trusted failure status", - { - "context": { - "payload": {"workflow_run": event}, - "runId": 20, - "repo": {"owner": "openai", "repo": "example"}, - "serverUrl": "https://github.com", - }, - "current": pull, - "comments": comments, - "fallback_pulls": [{"number": 3}], - }, - ) - self.assertEqual( - [w["operation"] for w in writes], [expected] if expected else [] - ) - if expected: - self.assertIn("Report unavailable", writes[0]["body"]) - self.assertIn("castiron:run:v1:20:1", writes[0]["body"]) - - @unittest.skipUnless(shutil.which("node"), "GitHub Actions JavaScript runtime") - def test_failure_comments_require_one_supported_target(self) -> None: - head = "a" * 40 - - def pull(number: int, branch: str = "main", repo: str = "openai/example") -> dict[str, Any]: - return { - "number": number, - "state": "open", - "head": source_head(head), - "base": {"ref": branch, "repo": {"full_name": repo}}, - } - - cases: list[tuple[str, list[dict[str, Any]], list[int]]] = [ - ("retargeted", [pull(3, "release")], []), - ("other repository", [pull(3, repo="other/example")], []), - ("ambiguous main", [pull(3), pull(4)], []), - ("ambiguous next", [pull(3, "next"), pull(4, "next")], []), - ("prefer main", [pull(3, "next"), pull(4)], [4]), - ("one next", [pull(3, "next")], [3]), - ("ignore unsupported", [pull(3, "release"), pull(4)], [4]), - ] - for name, pulls, expected in cases: - for fallback in (False, True): - with self.subTest(name=name, fallback=fallback): - run = { - **source_run(head), - "id": 20, - "pull_requests": [] if fallback else pulls, - } - writes = run_publisher( - "Publish a trusted failure status", - { - "context": { - "payload": {"workflow_run": run}, - "runId": 20, - "repo": {"owner": "openai", "repo": "example"}, - "serverUrl": "https://github.com", - }, - "pulls_by_number": {str(pr["number"]): pr for pr in pulls}, - "fallback_pulls": pulls, - }, - ) - self.assertEqual([write["issue_number"] for write in writes], expected) - - def test_workflow_reports_all_branches_without_write_credentials(self) -> None: - workflows = Path(__file__).resolve().parents[2] / ".github/workflows" - producer = (workflows / "castiron-custom-code.yml").read_text() - publisher = (workflows / "castiron-custom-code-comment.yml").read_text() - self.assertIn("pull_request:", producer) - self.assertNotIn("CASTIRON_CUSTOM_CODE_BRANCHES", producer + publisher) - self.assertNotIn("pull-requests: write", producer) - self.assertNotIn("head.repo.full_name ==", producer) - self.assertIn("workflow_run:", publisher) - concurrency = publisher.split("\nconcurrency:\n", 1)[1].split("\njobs:\n", 1)[0] - self.assertIn("queue: max", concurrency) - self.assertIn("cancel-in-progress: false", concurrency) - self.assertIn("ref: ${{ github.workflow_sha }}", publisher) - self.assertNotIn("ref: ${{ github.event.pull_request.head.sha }}", publisher) - self.assertIn("persist-credentials: false", publisher) - self.assertIn("--report", publisher) - compute, comment = publisher.split("\n comment:\n", 1) - self.assertNotIn("pull-requests: write", compute) - self.assertIn("pull-requests: read", compute) - self.assertIn(" trusted-report ", compute) - self.assertIn("ref: main", compute) - self.assertIn('--base "$(git -C .castiron-policy-checker rev-parse HEAD)"', compute) - self.assertIn("trusted_sha=$(git -C .castiron-policy-checker rev-parse HEAD)", compute) - self.assertIn("merge_group:", producer) - self.assertNotIn("download-artifact@", compute) - self.assertNotIn("unittest", compute) - self.assertIn("needs: compute", comment) - self.assertIn("artifact-ids: ${{ needs.compute.outputs.artifact-id }}", comment) - self.assertNotIn("run-id: ${{ github.event.workflow_run.id }}", comment) - self.assertNotIn("git fetch", comment) - self.assertIn("--artifact-run-id", comment) - digest = hashlib.sha256( - (workflows.parents[1] / "scripts/castiron/custom_code_report.py").read_bytes() - ).hexdigest() - self.assertIn(f"REPORTER_SHA256: {digest}", producer) - - @unittest.skipUnless(shutil.which("node"), "GitHub Actions JavaScript runtime") - def test_review_signals_cannot_replace_pending_candidate_validation(self) -> None: - producer = (WORKFLOW.parent / "castiron-custom-code.yml").read_text() - group = producer.split("concurrency:\n", 1)[1].splitlines()[0].removeprefix(" group: ") - expression = group.replace("${{", "${").replace("}}", "}") - groups: list[str] = [] - for event in ("pull_request", "pull_request_review", "pull_request"): - payload = {"context": {"payload": {"pull_request": {"number": 42}}}} - script = ( - "github.event = context.payload;\n" - f"github.event_name = {json.dumps(event)};\n" - f"await github.rest.issues.createComment({{body: `{expression}`}});" - ) - groups.append(run_workflow_script(script, payload)[0]["body"]) - self.assertNotEqual(groups[0], groups[1]) - self.assertEqual(groups[0], groups[2]) - - @unittest.skipUnless(shutil.which("node"), "GitHub Actions JavaScript runtime") - def test_failure_publisher_binds_one_source_before_writing(self) -> None: - head = "a" * 40 - run: dict[str, Any] = {**source_run(head), "id": 20, "pull_requests": []} - own: dict[str, Any] = { - "number": 3, - "state": "open", - "head": source_head(head), - "base": {"ref": "main", "repo": {"full_name": "openai/example"}}, - } - previous = { - "id": 42, - "user": {"type": "Bot", "login": "github-actions[bot]"}, - "body": report.MARKER + "\n", - } - - def publish(**overrides: Any) -> list[dict[str, Any]]: - return run_publisher( - "Publish a trusted failure status", - { - "context": { - "payload": {"workflow_run": run}, - "runId": 30, - "repo": {"owner": "openai", "repo": "example"}, - "serverUrl": "https://github.com", - }, - "run": run, - "current": own, - "comments": [previous], - "comments_by_number": {3: [previous], 4: [{**previous, "id": 43}]}, - "associated_pulls": [{"number": 3}, {"number": 4}], - **overrides, - }, - ) - - for foreign_head in ( - {**own["head"], "ref": "victim"}, - {**own["head"], "repo": {"id": 11, "full_name": "victim/example"}}, - ): - victim = {**own, "number": 4, "head": foreign_head} - with self.subTest(head=foreign_head): - writes = publish(pulls_by_number={3: own, 4: victim}) - self.assertEqual([write["comment_id"] for write in writes], [42]) - writes = publish( - associated_pulls=[{"number": 4}], - fallback_pulls=[{"number": 3}], - pulls_by_number={3: own, 4: victim}, - ) - self.assertEqual([write["comment_id"] for write in writes], [42]) - self.assertEqual( - publish( - run={**run, "pull_requests": [{"number": 4}]}, - fallback_pulls=[{"number": 3}], - pulls_by_number={3: own, 4: victim}, - ), - [], - ) - # Even two equally authorized PRs are ambiguous: no write to either. - self.assertEqual(publish(), []) - for changed_run in ( - {**run, "run_attempt": 2}, - {**run, "status": "in_progress"}, - {**run, "head_repository": None}, - {**run, "repository": {"full_name": "other/repo"}}, - ): - with self.subTest(run=changed_run): - self.assertEqual(publish(run=changed_run), []) - changed = {**own, "head": {**own["head"], "ref": "other"}} - self.assertEqual( - publish(associated_pulls=[{"number": 3}], current_sequence=[own, changed]), [] - ) - - -class CommentTests(GitTestCase): - def test_comment_only_rerun_links_to_the_compute_artifact_attempt(self) -> None: - workflow = ( - Path(__file__).resolve().parents[2] - / ".github/workflows/castiron-custom-code-comment.yml" - ).read_text() - compute, comment = workflow.split("\n comment:\n", 1) - - def field(section: str, prefix: str) -> str: - return next( - line.removeprefix(prefix) - for line in section.splitlines() - if line.startswith(prefix) - ) - - def resolve(value: str, context: dict[str, str]) -> str: - for key, replacement in context.items(): - value = value.replace("${{ " + key + " }}", replacement) - self.assertNotIn("${{", value) - return value - - # A successful compute job's outputs survive a comment-only rerun. - compute_context = {"github.run_id": "9", "github.run_attempt": "3"} - uploaded_name = resolve(field(compute, " name: "), compute_context) - saved_attempt = resolve(field(compute, " artifact-run-attempt: "), compute_context) - comment_context = { - "github.run_id": "9", - "github.run_attempt": "4", - "needs.compute.outputs.artifact-run-attempt": saved_attempt, - } - artifact_attempt = resolve( - field(comment, " ARTIFACT_RUN_ATTEMPT: "), comment_context - ) - self.assertEqual(uploaded_name, "castiron-custom-code-9-3") - self.assertEqual(artifact_attempt, "3") - - _, base = self.baseline() - result, _ = report.build_report(self.repo, base, base) - pull: dict[str, Any] = { - "state": "open", - "head": source_head(base), - "base": {"sha": base, "ref": "main", "repo": {"full_name": "openai/example"}}, - } - run = { - **source_run(base), - "event": "pull_request", - "path": ".github/workflows/castiron-custom-code.yml", - "head_sha": base, - "run_attempt": 1, - "pull_requests": [{"number": 1}], - } - with mock.patch.object( - report, "api", side_effect=[pull, run, pull, [], pull, {"html_url": "published"}] - ) as api: - self.assertEqual( - report.publish_comment( - result, - "openai/example", - 1, - 2, - 1, - artifact_run_id=9, - artifact_run_attempt=int(artifact_attempt), - ), - "published", - ) - body = api.call_args.args[2]["body"] - self.assertIn(f"--name {uploaded_name}", body) - self.assertNotIn("--name castiron-custom-code-9-4", body) - self.assertIn("castiron:run:v1:2:1", body) - - def test_comment_updates_existing_bot_comment_and_skips_stale(self) -> None: - _, base = self.baseline() - result, _ = report.build_report(self.repo, base, base) - calls: list[tuple[str, str, object]] = [] - - def fake_api(method: str, path: str, payload: object = None) -> object: - calls.append((method, path, payload)) - if "/pulls/" in path: - return { - "state": "open", - "head": source_head(base), - "base": { - "sha": base, - "ref": "main", - "repo": {"full_name": "openai/example"}, - }, - } - if "/actions/runs/" in path: - return { - **source_run(base), - "event": "pull_request", - "path": ".github/workflows/castiron-custom-code.yml", - "head_sha": base, - "run_attempt": 1, - "pull_requests": [{"number": 1}], - } - if "/comments?" in path: - return [ - {"id": 7, "user": {"login": "someone"}, "body": report.MARKER}, - { - "id": 8, - "user": {"login": "github-actions[bot]"}, - "body": report.MARKER, - "html_url": "existing", - }, - ] - return {"html_url": "updated"} - - with mock.patch.object(report, "api", side_effect=fake_api): - self.assertEqual(report.publish_comment(result, "openai/example", 1, 2, 1), "updated") - self.assertEqual(calls[-1][:2], ("PATCH", "repos/openai/example/issues/comments/8")) - calls.clear() - result["head_sha"] = "f" * 40 - self.assertEqual( - report.publish_comment(result, "openai/example", 1, 2, 1), - "Skipped stale report", - ) - self.assertEqual(len(calls), 1) - - def test_comment_rejects_older_runs_attempts_and_wrong_pr(self) -> None: - _, base = self.baseline() - result, _ = report.build_report(self.repo, base, base) - pull: dict[str, Any] = { - "state": "open", - "head": source_head(base), - "base": {"sha": base, "ref": "main", "repo": {"full_name": "openai/example"}}, - } - run = { - **source_run(base), - "event": "pull_request", - "path": ".github/workflows/castiron-custom-code.yml", - "head_sha": base, - "run_attempt": 2, - "pull_requests": [{"number": 1}], - "head_repository": source_run(base)["head_repository"], - "head_branch": "sdk", - } - comment = { - "id": 8, - "user": {"login": "github-actions[bot]"}, - "body": report.MARKER + "\n", - "html_url": "existing", - } - with mock.patch.object(report, "api", side_effect=[pull, run]) as api: - self.assertEqual( - report.publish_comment(result, "openai/example", 1, 2, 1), - "Skipped stale report", - ) - self.assertEqual(api.call_count, 2) - with mock.patch.object(report, "api", side_effect=[pull, run, pull, [comment]]) as api: - self.assertEqual( - report.publish_comment(result, "openai/example", 1, 2, 2), - "Skipped stale report", - ) - self.assertEqual(api.call_count, 4) - with ( - mock.patch.object( - report, "api", side_effect=[pull, {**run, "pull_requests": []}, [], [], []] - ), - self.assertRaisesRegex(report.ReportError, "does not match report PR"), - ): - report.publish_comment(result, "openai/example", 1, 2, 2) - with ( - mock.patch.object(report, "api", side_effect=[pull, {**run, "path": "other.yml"}]), - self.assertRaisesRegex(report.ReportError, "does not match report PR"), - ): - report.publish_comment(result, "openai/example", 1, 2, 2) - with mock.patch.object( - report, - "api", - side_effect=[pull, {**run, "pull_requests": []}, [{"number": 1}], pull, [comment]], - ) as api: - self.assertEqual( - report.publish_comment(result, "openai/example", 1, 2, 2), - "Skipped stale report", - ) - self.assertIn(f"/commits/{base}/pulls", api.call_args_list[2].args[1]) - changed = {**pull, "head": source_head("f" * 40)} - with mock.patch.object(report, "api", side_effect=[pull, run, pull, [], changed]) as api: - self.assertEqual( - report.publish_comment(result, "openai/example", 1, 2, 2), - "Skipped stale report", - ) - self.assertEqual(api.call_count, 5) - for target in ( - {**pull["base"], "ref": "other"}, - {**pull["base"], "repo": {"full_name": "other/repo"}}, - ): - with mock.patch.object( - report, "api", side_effect=[pull, run, pull, [], {**pull, "base": target}] - ): - self.assertEqual( - report.publish_comment(result, "openai/example", 1, 2, 2), - "Skipped stale report", - ) - - def test_comment_uses_canonical_branch_fallback(self) -> None: - run = source_run("a" * 40) - run["pull_requests"] = [] - own: dict[str, Any] = { - "state": "open", - "head": source_head("a" * 40), - "base": {"ref": "main", "repo": {"full_name": "openai/example"}}, - } - foreign = {**own, "head": {**own["head"], "ref": "other"}} - branches = [{"number": 3}] - writes: list[str] = [] - measured = { - "schema_version": 1, - "status": "error", - "head_sha": "a" * 40, - "target_base_sha": "b" * 40, - "error": "synthetic failure", - } - - def api(method: str, path: str, payload: Any = None) -> Any: - if method == "POST": - writes.append(path) - return {"html_url": "published"} - if "/actions/runs/" in path: - return run - if "/commits/" in path: - return [{"number": 4}] - if "/pulls?" in path: - return branches - if "/comments?" in path: - return [] - return {"3": own, "4": foreign, "5": own}[path.rsplit("/", 1)[1]] - - with mock.patch.object(report, "api", side_effect=api): - self.assertEqual(report.current_pull("openai/example", run), (3, "main")) - self.assertEqual( - report.publish_comment(measured, "openai/example", 3, 123, 1), "published" - ) - self.assertEqual(writes, ["repos/openai/example/issues/3/comments"]) - writes.clear() - run["pull_requests"] = [{"number": 4}] - with self.assertRaisesRegex(report.ReportError, "does not match report PR"): - report.publish_comment(measured, "openai/example", 3, 123, 1) - run["pull_requests"] = [] - branches.append({"number": 5}) - with self.assertRaisesRegex(report.ReportError, "multiple current"): - report.publish_comment(measured, "openai/example", 3, 123, 1) - self.assertEqual(writes, []) - - def test_successful_comment_rejects_another_source_before_and_after_pagination( - self, - ) -> None: - head = "a" * 40 - run = source_run(head) - own: dict[str, Any] = { - "state": "open", - "head": source_head(head), - "base": {"ref": "main", "repo": {"full_name": "openai/example"}}, - } - measured = { - "schema_version": 1, - "status": "error", - "head_sha": head, - "target_base_sha": "b" * 40, - "error": "synthetic failure", - } - for foreign_head in ( - {**own["head"], "ref": "other"}, - {**own["head"], "repo": {**own["head"]["repo"], "id": 11}}, - {**own["head"], "repo": {**own["head"]["repo"], "full_name": "other/repo"}}, - {**own["head"], "repo": None}, - ): - foreign = {**own, "head": foreign_head} - cases: tuple[list[Any], ...] = ([foreign, run], [own, run, own, [], foreign]) - for states in cases: - with ( - self.subTest(head=foreign_head, reads=len(states)), - mock.patch.object(report, "api", side_effect=states) as api, - ): - self.assertEqual( - report.publish_comment(measured, "openai/example", 3, 123, 1), - "Skipped stale report", - ) - self.assertTrue(all(call.args[0] == "GET" for call in api.call_args_list)) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/castiron/test_custom_code_report.py b/scripts/castiron/test_custom_code_report.py deleted file mode 100644 index a2c9f4190..000000000 --- a/scripts/castiron/test_custom_code_report.py +++ /dev/null @@ -1,1591 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -from __future__ import annotations - -import base64 -import hashlib -import io -import json -import os -import struct -import subprocess -import unittest -from pathlib import Path -from typing import Any -from unittest import mock - -import custom_code_report as report -from custom_code_test_support import GENERATION, GitTestCase - - -class CustomCodeTests(GitTestCase): - def test_large_blob_batches_preserve_every_object_byte(self) -> None: - commands: list[bytes] = [] - expected: dict[str, bytes] = {} - entries: dict[bytes, report.Entry] = {} - for index in range(2048): - content = f"synthetic object {index}\n".encode() + b"x" * 1024 - oid = hashlib.sha1( - f"blob {len(content)}\0".encode() + content, usedforsecurity=False - ).hexdigest() - expected[oid] = content - entries[f"file-{index}".encode()] = report.Entry(b"100644", oid) - commands.extend([b"blob\n", f"data {len(content)}\n".encode(), content, b"\n"]) - report.git(self.repo, "fast-import", "--quiet", input_bytes=b"".join(commands)) - actual = report.read_blobs(self.repo, entries) - self.assertEqual(actual.keys(), expected.keys()) - for oid, content in expected.items(): - self.assertEqual(actual[oid], content) - - def test_git_preserves_authentication_without_inherited_repository_routing(self) -> None: - environment = { - "GIT_CONFIG_GLOBAL": "/ordinary/gitconfig", - "GIT_ASKPASS": "/ordinary/askpass", - "GIT_CONFIG_COUNT": "1", - "GIT_CONFIG_KEY_0": "credential.helper", - "GIT_CONFIG_VALUE_0": "example", - "GIT_DIR": "/wrong/repository", - "GIT_COMMON_DIR": "/wrong/common", - "GIT_INDEX_FILE": "/wrong/index", - } - with ( - mock.patch.dict(os.environ, environment), - mock.patch.object( - report.subprocess, - "run", - return_value=subprocess.CompletedProcess([], 0, stdout=b"ok"), - ) as run, - ): - self.assertEqual(report.git(self.repo, "status"), b"ok") - actual = run.call_args.kwargs["env"] - for name, value in environment.items(): - if name in {"GIT_DIR", "GIT_COMMON_DIR", "GIT_INDEX_FILE"}: - self.assertNotIn(name, actual) - else: - self.assertEqual(actual[name], value) - self.assertEqual(actual["GIT_NO_REPLACE_OBJECTS"], "1") - - @unittest.skipUnless(os.environ.get("CASTIRON_TEST_BIN"), "Castiron compiler contract test") - def test_rust_and_python_hashes_match(self) -> None: - self.write("a", "first\n") - self.write("nested/b", "second\n") - self.write(".github/workflows/ignored.yml", "ignored\n") - for change in ("initial", "mode", "bytes"): - if change == "mode": - (self.repo / "a").chmod(0o755) - elif change == "bytes": - self.write("nested/b", "changed\n") - commit = self.commit(change) - rust = subprocess.check_output( - [ - os.environ["CASTIRON_TEST_BIN"], - "sdk", - "codegen-hash", - "--repo", - str(self.repo), - "--commit", - commit, - ], - text=True, - ).strip() - self.assertEqual(rust, report.hash_codegen_commit(self.repo, commit)) - - def test_reporting_script_can_be_a_mixed_file(self) -> None: - path = "scripts/castiron/custom_code_report.py" - self.write(path, "# generated reporter\n") - _, base = self.baseline() - self.write(path, "# generated reporter\n# local customization\n") - head = self.commit() - result, _ = report.build_report(self.repo, base, head, require_head_hash=True) - changed = next(file for file in result["files"] if file["path"] == path) - self.assertEqual(changed["category"], "newly_customized") - - def test_hash_vector_and_exclusions(self) -> None: - self.write("a", "hello\n") - first = self.commit() - encoded = ( - report.DOMAIN - + struct.pack(">Q", 1) - + struct.pack(">Q", 1) - + b"a100644" - + struct.pack(">Q", 6) - + b"hello\n" - ) - expected = hashlib.sha256(encoded).hexdigest() - self.assertEqual(report.hash_codegen_commit(self.repo, first), expected) - self.write(".github/workflows/test.yml", "ignored\n") - self.write(".github/actions/test/action.yml", "ignored\n") - second = self.commit() - self.assertEqual(report.hash_codegen_commit(self.repo, second), expected) - self.assertEqual(set(report.tree_entries(self.repo, second)), {b"a"}) - self.assertEqual( - set(report.tree_entries(self.repo, second, include_workflows=True)), - {b"a", b".github/workflows/test.yml", b".github/actions/test/action.yml"}, - ) - os.chmod(self.repo / "a", 0o755) - self.assertNotEqual(report.hash_codegen_commit(self.repo, self.commit()), expected) - self.write(".github/CODEOWNERS", "not ignored\n") - self.assertNotEqual(report.hash_codegen_commit(self.repo, self.commit()), expected) - - def test_classifications(self) -> None: - a, b, c = (report.Entry(b"100644", value * 40) for value in "abc") - cases = [ - ((a, a, b, b), None), - ((a, a, a, b), "newly_customized"), - ((a, b, a, a), "removed"), - ((a, b, a, c), "existing_changed"), - ((a, b, a, b), "unchanged"), - ((a, b, c, b), "baseline_changed"), - ((a, b, None, b), "no_longer_generated"), - ((a, a, None, a), "no_longer_generated"), - ((a, a, None, None), None), - ((None, b, a, b), "newly_generation_owned"), - ((None, a, a, a), "newly_generation_owned"), - ((a, a, b, a), "baseline_changed"), - ((a, b, b, b), "baseline_changed"), - ((a, a, a, None), "newly_customized"), - ] - for values, expected in cases: - with self.subTest(expected=expected): - self.assertEqual(report.classify(*values), expected) - - def test_report_ignores_handwritten_files_and_shows_custom_patch(self) -> None: - generated, base = self.baseline() - self.write("generated.py", "generated\n# custom\n") - self.write("handwritten.py", "handwritten\n") - head = self.commit() - result, patch = report.build_report(self.repo, base, head, require_head_hash=True) - self.assertEqual(result["counts"]["newly_customized"], 1) - self.assertEqual(result["files"][0]["path"], "generated.py") - self.assertIn(b"+# custom", patch) - self.assertNotIn(b"handwritten", patch) - self.assertEqual(result["after"]["commit"], generated) - body = report.render_report(result, repository="openai/example", run_id=42, run_attempt=2) - self.assertIn("Inspect the custom-code diff", body) - self.assertIn("gh run download 42 --repo openai/example", body) - self.assertIn("--name castiron-custom-code-42-2", body) - self.assertIn(f"--head {head}", body) - self.assertIn("current full custom patch", body) - - def test_bad_hash_and_lost_checkpoint_fail(self) -> None: - generated, base = self.baseline() - stats = self.repo / ".castiron.stats.yml" - stats.write_text( - stats.read_text().replace(report.hash_codegen_commit(self.repo, generated), "f" * 64) - ) - with self.assertRaisesRegex(report.ReportError, "codegen_hash mismatch"): - report.build_report(self.repo, base, self.commit()) - self.git("update-ref", "-d", "refs/remotes/origin/codegen/test") - with self.assertRaises(report.ReportError): - report.resolve_baseline(self.repo, base, fetch=False, require_hash=True) - - def test_stats_symlink_is_rejected(self) -> None: - self.baseline() - path = self.repo / ".castiron.stats.yml" - original = path.read_text() - path.unlink() - path.symlink_to(original) - with self.assertRaisesRegex(report.ReportError, "regular Git file"): - report.read_stats(self.repo, self.commit()) - - def test_next_checkpoint_uses_its_namespaced_ref(self) -> None: - generated, base = self.baseline() - self.git("checkout", "--detach", generated) - metadata = { - "schema_version": "castiron-checkpoint/v2", - "release_target": "next", - "record": {"generation_id": GENERATION, "source_branch": "test"}, - } - encoded = base64.b64encode(json.dumps(metadata).encode()).decode() - checkpoint = self.commit(f"codegen\n\nGeneration metadata: {encoded}") - ref = "refs/remotes/origin/codegen/castiron/next/test" - self.git("update-ref", ref, checkpoint) - stats = report.read_stats(self.repo, base) | { - "codegen_sha": checkpoint, - "codegen_hash": report.hash_codegen_commit(self.repo, checkpoint), - } - result = report.resolve_checkpoint( - self.repo, stats, fetch=False, require_hash=True, target_branch="next" - ) - self.assertEqual(result["commit"], checkpoint) - self.git("update-ref", "refs/remotes/origin/codegen/test", checkpoint) - self.git("update-ref", "-d", ref) - with self.assertRaises(report.ReportError): - report.resolve_checkpoint( - self.repo, stats, fetch=False, require_hash=True, target_branch="next" - ) - - def test_next_checkpoint_cannot_authorize_main_reports(self) -> None: - import custom_code_budget as budget - - generated, base = self.baseline() - self.write( - ".castiron-ratchet.json", '{"schema_version": 1, "max_custom_patch_lines": 10000}\n' - ) - base = self.commit("policy") - self.git("checkout", "--detach", generated) - metadata = { - "schema_version": "castiron-checkpoint/v2", - "release_target": "next", - "record": {"generation_id": GENERATION, "source_branch": "test"}, - } - encoded = base64.b64encode(json.dumps(metadata).encode()).decode() - checkpoint = self.commit(f"codegen\n\nGeneration metadata: {encoded}") - self.git("update-ref", "refs/remotes/origin/codegen/castiron/next/test", checkpoint) - self.git("checkout", "--detach", base) - stats = report.read_stats(self.repo, base) | { - "codegen_sha": checkpoint, - "codegen_hash": report.hash_codegen_commit(self.repo, checkpoint), - } - self.write(".castiron.stats.yml", "".join(f"{k}: {v}\n" for k, v in stats.items())) - head = self.commit("next checkpoint") - for target, expected in (("main", "error"), ("next", "ok")): - with self.subTest(target=target): - result = report.write_report( - self.repo, - base, - head, - self.repo / "report-output", - fetch=False, - require_head_hash=True, - public=False, - target_branch=target, - ) - self.assertEqual(result["status"], expected) - measured, _ = budget.evaluate( - self.repo, - base, - head, - public=False, - target_branch=target, - ) - self.assertEqual(measured["target_branch"], target) - self.assertEqual( - measured["checks"]["budget"]["state"], - "success" if target == "next" else "failure", - ) - if target == "next": - output = self.repo / "budget-output" - budget.write_result(output, measured) - self.assertEqual( - json.loads((output / "budget.json").read_text())["target_branch"], "next" - ) - self.assertIn("Target next:", (output / "summary.md").read_text()) - snapshot = report.create_public_snapshot( - self.repo, - self.git("rev-parse", f"{checkpoint}^{{tree}}"), - GENERATION, - stats["codegen_hash"], - "codegen/castiron/next/test", - None, - ) - self.git("update-ref", "refs/remotes/origin/codegen/castiron/next/test", snapshot) - stats["public_codegen_sha"] = snapshot - self.write(".castiron.stats.yml", "".join(f"{k}: {v}\n" for k, v in stats.items())) - public_head = self.commit("public next snapshot") - with self.assertRaisesRegex(report.ReportError, "cannot authorize a main"): - report.build_report(self.repo, base, public_head, public=True) - accepted, _ = report.build_report( - self.repo, base, public_head, public=True, target_branch="next" - ) - self.assertEqual(accepted["status"], "ok") - - def test_public_next_namespace_root_remains_reserved(self) -> None: - generated, _ = self.baseline() - branch = "codegen/castiron/next" - snapshot = report.create_public_snapshot( - self.repo, - self.git("rev-parse", f"{generated}^{{tree}}"), - GENERATION, - report.hash_codegen_commit(self.repo, generated), - branch, - None, - ) - metadata = report.public_metadata(self.repo, snapshot) - self.assertEqual(metadata["Target"], "next") - for target in ("main", "next"): - with self.subTest(target=target): - metadata["Target"] = target - with self.assertRaisesRegex(report.ReportError, "cannot authorize a main"): - report.validate_public_target( - self.repo, snapshot, metadata, target_branch="main", fetch=False - ) - report.validate_public_target( - self.repo, snapshot, metadata, target_branch="next", fetch=False - ) - - def test_prepare_public_next_root_and_descendant(self) -> None: - generated, base = self.baseline() - self.git("branch", "codegen/test", generated) - self.git("checkout", "--detach", generated) - metadata = { - "schema_version": "castiron-checkpoint/v2", - "release_target": "next", - "record": {"generation_id": GENERATION, "source_branch": "test"}, - } - encoded = base64.b64encode(json.dumps(metadata).encode()).decode() - checkpoint = self.commit(f"codegen\n\nGeneration metadata: {encoded}") - self.git("branch", "codegen/castiron/next/test", checkpoint) - self.git("checkout", "--detach", base) - stats = report.read_stats(self.repo, base) | { - "codegen_sha": checkpoint, - "codegen_hash": report.hash_codegen_commit(self.repo, checkpoint), - } - self.write(".castiron.stats.yml", "".join(f"{k}: {v}\n" for k, v in stats.items())) - head = self.commit("next checkpoint") - self.git("remote", "add", "origin", str(self.repo)) - public = self.repo / "public" - self.git("clone", "--no-local", str(self.repo), str(public)) - report.git(public, "checkout", "--detach", base) - public_remote = self.repo / "public.git" - report.git(public, "init", "--bare", str(public_remote)) - report.git(public, "remote", "set-url", "origin", str(public_remote)) - report.git(public, "push", "origin", f"{base}:refs/heads/main") - for branch in ("castiron/next", "castiron/next/test"): - with self.subTest(branch=branch): - snapshot = report.prepare_public_snapshot( - self.repo, base, head, public, base, branch - ) - self.assertEqual(snapshot["branch"], f"codegen/{branch}") - public_metadata = report.public_metadata(public, snapshot["commit"]) - self.assertEqual(public_metadata["Target"], "next") - with self.assertRaisesRegex(report.ReportError, "cannot authorize a main"): - report.validate_public_target( - public, - snapshot["commit"], - public_metadata, - target_branch="main", - fetch=False, - ) - with self.assertRaises(report.ReportError): - report.prepare_public_snapshot(self.repo, base, head, public, base, "castiron/nextish") - - def test_legacy_public_snapshot_requires_trusted_main_adoption(self) -> None: - generated, base = self.baseline() - branch = "codegen/castiron/promotions/pr-42" - content_hash = report.hash_codegen_commit(self.repo, generated) - tree = self.git("rev-parse", f"{generated}^{{tree}}") - message = ( - "Castiron generated snapshot\n\n" - "Castiron-Public-Codegen-Version: 1\n" - f"Castiron-Public-Codegen-Generation-ID: {GENERATION}\n" - f"Castiron-Public-Codegen-Hash: {content_hash}\n" - f"Castiron-Public-Codegen-Branch: {branch}\n" - ) - legacy = self.git("commit-tree", tree, "-m", message) - self.git("update-ref", f"refs/remotes/origin/{branch}", legacy) - self.git("update-ref", "refs/remotes/origin/main", base) - stats = report.read_stats(self.repo, base) | {"public_codegen_sha": legacy} - self.write(".castiron.stats.yml", "".join(f"{k}: {v}\n" for k, v in stats.items())) - head = self.commit("untrusted legacy snapshot adoption") - with self.assertRaisesRegex(report.ReportError, "no verified main adoption"): - report.resolve_public_baseline(self.repo, head, fetch=False, require_hash=True) - report.resolve_public_baseline( - self.repo, head, fetch=False, require_hash=True, target_branch="next" - ) - # A side branch can add then remove the reference before merging. None - # of its intermediate stats were ever adopted on main's first-parent line. - base_tree = self.git("rev-parse", f"{base}^{{tree}}") - removed = self.git("commit-tree", base_tree, "-p", head, "-m", "remove reference") - self.write( - ".castiron.stats.yml", - self.git("show", f"{base}:.castiron.stats.yml") + "\n# main metadata retained\n", - ) - self.git("add", ".castiron.stats.yml") - merged = self.git( - "commit-tree", - self.git("write-tree"), - "-p", - base, - "-p", - removed, - "-m", - "merge clean tip with main metadata update", - ) - self.git("update-ref", "refs/remotes/origin/main", merged) - with self.assertRaisesRegex(report.ReportError, "no verified main adoption"): - report.resolve_public_baseline(self.repo, head, fetch=False, require_hash=True) - # A merge that actually adopts the snapshot on main is sufficient. - adopted = self.git( - "commit-tree", - self.git("rev-parse", f"{head}^{{tree}}"), - "-p", - merged, - "-p", - head, - "-m", - "adopt snapshot on main", - ) - self.git("update-ref", "refs/remotes/origin/main", adopted) - report.resolve_public_baseline(self.repo, head, fetch=False, require_hash=True) - for spelling in (f'"{legacy}"', f" {legacy}"): - with self.subTest(spelling=spelling): - self.write( - ".castiron.stats.yml", - "".join( - f"{key}: {spelling if key == 'public_codegen_sha' else value}\n" - for key, value in stats.items() - ), - ) - self.git("add", ".castiron.stats.yml") - formatted = self.git( - "commit-tree", self.git("write-tree"), "-p", base, "-m", "formatted adoption" - ) - self.git("update-ref", "refs/remotes/origin/main", formatted) - report.resolve_public_baseline(self.repo, formatted, fetch=False, require_hash=True) - - # Replacing a comment with the field preserves the SHA occurrence count. - original_stats = self.git("show", f"{base}:.castiron.stats.yml") + "\n" - self.write(".castiron.stats.yml", original_stats + f"# candidate: {legacy}\n") - self.git("add", ".castiron.stats.yml") - mentioned = self.git( - "commit-tree", self.git("write-tree"), "-p", base, "-m", "mention candidate" - ) - self.git("update-ref", "refs/remotes/origin/main", mentioned) - self.assertFalse(report.legacy_snapshot_on_main(self.repo, legacy, fetch=False)) - with self.assertRaisesRegex(report.ReportError, "no verified main adoption"): - report.resolve_public_baseline(self.repo, head, fetch=False, require_hash=True) - self.write(".castiron.stats.yml", original_stats + f"public_codegen_sha: {legacy}\n") - self.git("add", ".castiron.stats.yml") - adopted = self.git( - "commit-tree", self.git("write-tree"), "-p", mentioned, "-m", "adopt candidate" - ) - self.git("update-ref", "refs/remotes/origin/main", adopted) - report.resolve_public_baseline(self.repo, adopted, fetch=False, require_hash=True) - - # Explicit next metadata cannot be grandfathered, even with an old locator. - explicit_next = self.git( - "commit-tree", tree, "-m", message + "Castiron-Public-Codegen-Target: next\n" - ) - self.git("update-ref", f"refs/remotes/origin/{branch}", explicit_next) - stats["public_codegen_sha"] = explicit_next - self.write(".castiron.stats.yml", "".join(f"{k}: {v}\n" for k, v in stats.items())) - head = self.commit("explicit next target") - self.git("update-ref", "refs/remotes/origin/main", head) - with self.assertRaisesRegex(report.ReportError, "cannot authorize a main"): - report.resolve_public_baseline(self.repo, head, fetch=False, require_hash=True) - - def test_legacy_bootstrap_baseline_uses_verified_main_lineage(self) -> None: - generated, base = self.baseline() - tree = self.git("rev-parse", f"{generated}^{{tree}}") - content_hash = report.hash_codegen_commit(self.repo, generated) - branch = "codegen/castiron/promotions/pr-1" - - def legacy_snapshot(generation: str, parent: str | None = None) -> str: - message = ( - "Castiron generated snapshot\n\n" - "Castiron-Public-Codegen-Version: 1\n" - f"Castiron-Public-Codegen-Generation-ID: {generation}\n" - f"Castiron-Public-Codegen-Hash: {content_hash}\n" - f"Castiron-Public-Codegen-Branch: {branch}\n" - ) - return self.git("commit-tree", tree, *(["-p", parent] if parent else []), "-m", message) - - bootstrap = legacy_snapshot(GENERATION) - generation = "650e8400-e29b-41d4-a716-446655440000" - adopted = legacy_snapshot(generation, bootstrap) - self.git("update-ref", f"refs/remotes/origin/{branch}", adopted) - stats = report.read_stats(self.repo, base) | { - "generation_id": generation, - "public_codegen_sha": adopted, - } - self.write(".castiron.stats.yml", "".join(f"{k}: {v}\n" for k, v in stats.items())) - head = self.commit("adopt first public checkpoint") - self.git("update-ref", "refs/remotes/origin/main", head) - result, _ = report.build_report(self.repo, base, head, public=True, require_head_hash=True) - self.assertEqual(result["before"]["commit"], bootstrap) - self.assertEqual(result["after"]["commit"], adopted) - self.assertFalse(report.legacy_snapshot_on_main(self.repo, bootstrap, fetch=False)) - - # An unadopted legacy head cannot authorize its own bootstrap lineage. - with self.assertRaisesRegex(report.ReportError, "no verified main adoption"): - report.resolve_public_baseline( - self.repo, base, fetch=False, require_hash=False, lineage=bootstrap - ) - - def test_public_next_namespace_root_is_reserved(self) -> None: - generated, _ = self.baseline() - tree = self.git("rev-parse", f"{generated}^{{tree}}") - content_hash = report.hash_codegen_commit(self.repo, generated) - for branch, target in ( - ("codegen/castiron/next", "next"), - ("codegen/castiron/next/child", "next"), - ("codegen/castiron/next-door", "main"), - ): - with self.subTest(branch=branch): - snapshot = report.create_public_snapshot( - self.repo, tree, GENERATION, content_hash, branch, None - ) - metadata = report.public_metadata(self.repo, snapshot) - self.assertEqual(metadata["Target"], target) - # A misleading explicit target cannot authorize the reserved root. - metadata["Target"] = "main" - if target == "next": - with self.assertRaisesRegex(report.ReportError, "cannot authorize a main"): - report.validate_public_target( - self.repo, snapshot, metadata, target_branch="main", fetch=False - ) - else: - report.validate_public_target( - self.repo, snapshot, metadata, target_branch="main", fetch=False - ) - - def test_legacy_main_adoption_accepts_stats_formatting(self) -> None: - generated, base = self.baseline() - branch = "codegen/castiron/promotions/pr-42" - content_hash = report.hash_codegen_commit(self.repo, generated) - message = ( - "Castiron generated snapshot\n\n" - "Castiron-Public-Codegen-Version: 1\n" - f"Castiron-Public-Codegen-Generation-ID: {GENERATION}\n" - f"Castiron-Public-Codegen-Hash: {content_hash}\n" - f"Castiron-Public-Codegen-Branch: {branch}\n" - ) - snapshot = self.git( - "commit-tree", self.git("rev-parse", f"{generated}^{{tree}}"), "-m", message - ) - self.git("update-ref", f"refs/remotes/origin/{branch}", snapshot) - for value in (f"'{snapshot}'", f'"{snapshot}"', f" {snapshot}"): - with self.subTest(value=value): - self.git("checkout", "--detach", "-q", base) - stats = self.git("show", f"{base}:.castiron.stats.yml") - self.write(".castiron.stats.yml", stats + f"\n# pending snapshot: {snapshot}\n") - commented = self.commit("mention snapshot without adopting it") - self.git("update-ref", "refs/remotes/origin/main", commented) - self.assertFalse(report.legacy_snapshot_on_main(self.repo, snapshot, fetch=False)) - self.write(".castiron.stats.yml", stats + f"\npublic_codegen_sha: {value}\n") - adopted = self.commit("adopt formatted snapshot") - self.git("update-ref", "refs/remotes/origin/main", adopted) - resolved = report.resolve_public_baseline( - self.repo, adopted, fetch=False, require_hash=True - ) - self.assertEqual(resolved["commit"], snapshot) - - def test_legacy_bootstrap_parent_uses_main_adopted_lineage(self) -> None: - generated, base = self.baseline() - tree = self.git("rev-parse", f"{generated}^{{tree}}") - content_hash = report.hash_codegen_commit(self.repo, generated) - child_generation = "550e8400-e29b-41d4-a716-446655440001" - for parent_branch, parent_target, allowed in ( - ("codegen/castiron/bootstrap", "", True), - ("codegen/castiron/bootstrap", "next", False), - ("codegen/castiron/next", "", False), - ("codegen/castiron/next/bootstrap", "", False), - ): - with self.subTest(branch=parent_branch, target=parent_target): - for reserved in self.git( - "for-each-ref", - "--format=%(refname)", - "refs/remotes/origin/codegen/castiron/next", - ).splitlines(): - self.git("update-ref", "-d", reserved) - self.git("checkout", "--detach", "-q", base) - self.git("update-ref", "refs/remotes/origin/main", base) - parent_message = ( - "Castiron generated snapshot\n\n" - "Castiron-Public-Codegen-Version: 1\n" - f"Castiron-Public-Codegen-Generation-ID: {GENERATION}\n" - f"Castiron-Public-Codegen-Hash: {content_hash}\n" - f"Castiron-Public-Codegen-Branch: {parent_branch}\n" - ) - if parent_target: - parent_message += f"Castiron-Public-Codegen-Target: {parent_target}\n" - parent = self.git("commit-tree", tree, "-m", parent_message) - self.git("update-ref", f"refs/remotes/origin/{parent_branch}", parent) - child_branch = "codegen/castiron/promotions/test" - child_message = ( - "Castiron generated snapshot\n\n" - "Castiron-Public-Codegen-Version: 1\n" - f"Castiron-Public-Codegen-Generation-ID: {child_generation}\n" - f"Castiron-Public-Codegen-Hash: {content_hash}\n" - f"Castiron-Public-Codegen-Branch: {child_branch}\n" - ) - child = self.git("commit-tree", tree, "-p", parent, "-m", child_message) - self.git("update-ref", f"refs/remotes/origin/{child_branch}", child) - stats = report.read_stats(self.repo, base) | { - "generation_id": child_generation, - "public_codegen_sha": child, - } - self.write(".castiron.stats.yml", "".join(f"{k}: {v}\n" for k, v in stats.items())) - head = self.commit("adopt only the legacy descendant") - with self.assertRaisesRegex(report.ReportError, "no verified main adoption"): - report.build_report(self.repo, base, head, public=True, require_head_hash=True) - self.git("update-ref", "refs/remotes/origin/main", head) - if allowed: - result, _ = report.build_report( - self.repo, base, head, public=True, require_head_hash=True - ) - self.assertEqual(result["status"], "ok") - self.assertEqual( - report.resolve_public_baseline( - self.repo, base, fetch=False, require_hash=False, lineage=child - )["commit"], - parent, - ) - # Adoption of a descendant never authorizes a direct parent reference. - direct = report.read_stats(self.repo, base) | {"public_codegen_sha": parent} - self.write( - ".castiron.stats.yml", "".join(f"{k}: {v}\n" for k, v in direct.items()) - ) - direct_head = self.commit("reference an unadopted parent directly") - with self.assertRaisesRegex(report.ReportError, "no verified main adoption"): - report.resolve_public_baseline( - self.repo, direct_head, fetch=False, require_hash=True, lineage=child - ) - else: - with self.assertRaisesRegex(report.ReportError, "cannot authorize a main"): - report.build_report( - self.repo, base, head, public=True, require_head_hash=True - ) - - def test_legacy_checkpoint_cannot_use_next_namespace(self) -> None: - generated, base = self.baseline() - self.git("checkout", "--detach", generated) - metadata = {"generation_id": GENERATION, "source_branch": "castiron/next/test"} - encoded = base64.b64encode(json.dumps(metadata).encode()).decode() - checkpoint = self.commit(f"codegen\n\nGeneration metadata: {encoded}") - self.git("update-ref", "refs/remotes/origin/codegen/castiron/next/test", checkpoint) - stats = report.read_stats(self.repo, base) | { - "codegen_sha": checkpoint, - "codegen_hash": report.hash_codegen_commit(self.repo, checkpoint), - } - for target in ("main", "next"): - with ( - self.subTest(target=target), - self.assertRaisesRegex(report.ReportError, "reserved next"), - ): - report.resolve_checkpoint( - self.repo, stats, fetch=False, require_hash=True, target_branch=target - ) - - def test_invalid_checkpoint_envelopes_are_rejected(self) -> None: - _, base = self.baseline() - record = {"generation_id": GENERATION, "source_branch": "test"} - envelope = { - "schema_version": "castiron-checkpoint/v2", - "release_target": "next", - "record": record, - } - for metadata in ( - envelope | {"schema_version": "unknown"}, - envelope | {"release_target": "main"}, - envelope | {"extra": True}, - envelope | {"record": record | {"release_target": "next"}}, - record | {"release_target": "next"}, - ): - with self.subTest(metadata=metadata): - encoded = base64.b64encode(json.dumps(metadata).encode()).decode() - checkpoint = self.commit(f"codegen\n\nGeneration metadata: {encoded}") - stats = report.read_stats(self.repo, base) | {"codegen_sha": checkpoint} - with self.assertRaisesRegex(report.ReportError, "envelope"): - report.resolve_checkpoint( - self.repo, stats, fetch=False, require_hash=True, target_branch="next" - ) - - def test_public_snapshot_rejects_private_governance_and_unpublished_paths(self) -> None: - entry = report.Entry(b"100644", "a" * 40) - blocked = [ - b".castiron/private.json", - b".github/CODEOWNERS", - b".gitmodules", - b"CODEOWNERS", - b"SECURITY.md", - b"docs/SECURITY.md", - ] - for path in blocked: - with ( - self.subTest(path=path), - mock.patch.object( - report, "tree_entries", side_effect=[{path: entry}, {path: entry}] - ), - mock.patch.object(report, "read_blobs") as read, - ): - with self.assertRaisesRegex(report.ReportError, "private or governance"): - report.copy_generated_tree(self.repo, "a" * 40, self.repo, "b" * 40) - read.assert_not_called() - with ( - mock.patch.object(report, "tree_entries", side_effect=[{b"unreleased.py": entry}, {}]), - mock.patch.object(report, "read_blobs") as read, - ): - with self.assertRaisesRegex( - report.ReportError, "absent from the approved public SDK tree" - ): - report.copy_generated_tree(self.repo, "a" * 40, self.repo, "b" * 40) - read.assert_not_called() - - def test_zero_new_files_summary_reports_existing_changes(self) -> None: - _, clean = self.baseline() - self.write("generated.py", "generated\n# existing customization\n") - base = self.commit() - self.write("generated.py", "generated\n# existing customization updated\n") - result, _ = report.build_report(self.repo, base, self.commit()) - body = report.render_report(result) - self.assertIn("✅ No new custom-code files detected.", body) - self.assertIn("1 mixed file remains; 1 existing customization changed.", body) - self.assertNotIn("0 newly customized", body) - empty, _ = report.build_report(self.repo, clean, clean) - self.assertIn( - "0 mixed files remain; 0 existing customizations changed.", report.render_report(empty) - ) - - def test_new_mixed_file_headline_includes_changed_generated_baselines(self) -> None: - _, base = self.baseline() - result, _ = report.build_report(self.repo, base, base) - for category in ("baseline_changed", "newly_generation_owned"): - result["files"] = [ - { - "path": "generated.py", - "category": category, - "custom_before": False, - "custom_after": True, - "added": "1", - "removed": "1", - } - ] - body = report.render_report(result) - self.assertNotIn("No new custom-code files", body) - self.assertIn("1 newly customized", body) - - def test_removals_include_changed_baselines_but_not_handwritten_only_files(self) -> None: - _, base = self.baseline() - result, _ = report.build_report(self.repo, base, base) - result["files"] = [ - { - "path": category, - "category": category, - "custom_before": True, - "custom_after": False, - "added": "0", - "removed": "0", - } - for category in ("baseline_changed", "no_longer_generated") - ] - self.assertIn("1 customizations removed", report.render_report(result)) - result["files"].append( - { - "path": "new.py", - "category": "newly_customized", - "custom_before": False, - "custom_after": True, - "added": "1", - "removed": "0", - } - ) - self.assertIn("1 customizations removed", report.render_report(result)) - - def test_pending_legacy_promotion_is_upgraded_without_authorizing_legacy_reports(self) -> None: - generated, source_base = self.baseline() - self.git("branch", "codegen/test", generated) - private_remote = self.repo / "private.git" - self.git("clone", "--bare", str(self.repo), str(private_remote)) - self.git("remote", "add", "origin", str(private_remote)) - public = self.repo / "public" - public.mkdir() - report.git(public, "init", "-q", "-b", "main") - report.git(public, "config", "user.name", "Public test") - report.git(public, "config", "user.email", "public@example.test") - (public / "generated.py").write_text("generated\n") - (public / ".castiron.stats.yml").write_text((self.repo / ".castiron.stats.yml").read_text()) - report.git(public, "add", ".") - report.git(public, "commit", "-qm", "public baseline") - initial = report.git(public, "rev-parse", "HEAD").decode().strip() - branch = "castiron/promotions/legacy-retry" - codegen_branch = f"codegen/{branch}" - - def legacy_snapshot( - tree: str, generation: str, content_hash: str, parent: str | None - ) -> str: - message = ( - "Castiron generated snapshot\n\nCastiron-Public-Codegen-Version: 1\n" - f"Castiron-Public-Codegen-Generation-ID: {generation}\n" - f"Castiron-Public-Codegen-Hash: {content_hash}\n" - f"Castiron-Public-Codegen-Branch: {codegen_branch}\n" - ) - parents = ["-p", parent] if parent is not None else [] - return report.git(public, "commit-tree", tree, *parents, "-m", message).decode().strip() - - base_tree = report.copy_generated_tree(self.repo, generated, public, initial) - legacy_base = legacy_snapshot( - base_tree, GENERATION, report.hash_codegen_commit(self.repo, generated), None - ) - with (public / ".castiron.stats.yml").open("a") as stats: - stats.write(f"public_codegen_sha: {legacy_base}\n") - report.git(public, "add", ".castiron.stats.yml") - report.git(public, "commit", "-qm", "adopt legacy baseline") - public_base = report.git(public, "rev-parse", "HEAD").decode().strip() - public_remote = self.repo / "public.git" - report.git(public, "clone", "--bare", str(public), str(public_remote)) - report.git(public, "remote", "add", "origin", str(public_remote)) - report.git(public, "push", "origin", f"{legacy_base}:refs/heads/{codegen_branch}") - - generation = "650e8400-e29b-41d4-a716-446655440000" - self.git("checkout", "--detach", generated) - self.write("generated.py", "regenerated\n") - self.git("add", "generated.py") - metadata = base64.b64encode( - json.dumps({"generation_id": generation, "source_branch": "test"}).encode() - ).decode() - self.git("commit", "-qm", f"updated codegen\n\nGeneration metadata: {metadata}") - updated = self.git("rev-parse", "HEAD") - self.git("push", "origin", f"{updated}:refs/heads/codegen/test") - content_hash = report.hash_codegen_commit(self.repo, updated) - self.write( - ".castiron.stats.yml", - f"schema_version: 1\ngeneration_id: {generation}\ncodegen_sha: {updated}\ncodegen_hash: {content_hash}\n", - ) - self.git("add", ".castiron.stats.yml") - self.git("commit", "-qm", "updated integrated SDK") - source_head = self.git("rev-parse", "HEAD") - tree = report.copy_generated_tree(self.repo, updated, public, public_base) - pending = legacy_snapshot(tree, generation, content_hash, legacy_base) - report.git(public, "push", "origin", f"{pending}:refs/heads/{codegen_branch}") - self.assertTrue(report.legacy_snapshot_on_main(public, legacy_base, fetch=True)) - self.assertFalse(report.legacy_snapshot_on_main(public, pending, fetch=False)) - (public / "generated.py").write_text("regenerated\n") - report.git(public, "add", "generated.py") - - message = report.git(public, "show", "-s", "--format=%B", pending).decode() - for invalid_message, error in ( - (message + "Castiron-Public-Codegen-Target: next\n", "next checkpoint"), - (message.replace(content_hash, "f" * 64), "invalid provenance"), - ( - message.replace(codegen_branch, "codegen/another-promotion"), - "no verified main adoption", - ), - ): - with self.subTest(error=error): - invalid = ( - report.git( - public, "commit-tree", tree, "-p", legacy_base, "-m", invalid_message - ) - .decode() - .strip() - ) - report.git( - public, "push", "--force", "origin", f"{invalid}:refs/heads/{codegen_branch}" - ) - with self.assertRaisesRegex(report.ReportError, error): - report.prepare_public_snapshot( - self.repo, source_base, source_head, public, public_base, branch - ) - report.git(public, "push", "--force", "origin", f"{pending}:refs/heads/{codegen_branch}") - upgraded = report.prepare_public_snapshot( - self.repo, source_base, source_head, public, public_base, branch - ) - self.assertNotEqual(upgraded["commit"], pending) - self.assertEqual(upgraded["base_commit"], legacy_base) - self.assertEqual(upgraded["previous"], pending) - self.assertEqual( - report.git(public, "rev-parse", upgraded["commit"] + "^1").decode().strip(), pending - ) - self.assertEqual(report.public_metadata(public, upgraded["commit"])["Target"], "main") - self.assertEqual(report.hash_codegen_commit(public, upgraded["commit"]), content_hash) - with self.assertRaisesRegex(report.ReportError, "no verified main adoption"): - report.validate_public_target( - public, - pending, - report.public_metadata(public, pending), - target_branch="main", - fetch=False, - ) - report.git(public, "push", "origin", f"{upgraded['commit']}:refs/heads/{codegen_branch}") - report.git(public, "commit", "-qm", "upgraded public SDK") - head = report.git(public, "rev-parse", "HEAD").decode().strip() - result, _ = report.build_report( - public, public_base, head, fetch=True, require_head_hash=True, public=True - ) - self.assertEqual(result["after"]["generation_id"], generation) - - def test_public_snapshot_has_no_private_history_and_reports_without_private_remote( - self, - ) -> None: - self.write("private-only.txt", "must never be published\n") - private_ancestor = self.commit("private history") - (self.repo / "private-only.txt").unlink() - generated, base = self.baseline() - self.git("branch", "codegen/test", generated) - private_remote = self.repo / "private.git" - self.git("clone", "--bare", str(self.repo), str(private_remote)) - self.git("remote", "add", "origin", str(private_remote)) - - public = self.repo / "public" - public.mkdir() - report.git(public, "init", "-q", "-b", "main") - report.git(public, "config", "user.name", "Public test") - report.git(public, "config", "user.email", "public@example.test") - (public / "generated.py").write_text("generated\n") - (public / ".castiron.stats.yml").write_text( - (self.repo / ".castiron.stats.yml").read_text().split("codegen_hash:")[0] - ) - report.git(public, "add", ".") - report.git(public, "commit", "-qm", "already public") - public_base = report.git(public, "rev-parse", "HEAD").decode().strip() - public_remote = self.repo / "public.git" - report.git(public, "clone", "--bare", str(public), str(public_remote)) - report.git(public, "remote", "add", "origin", str(public_remote)) - snapshot = report.prepare_public_snapshot( - self.repo, base, base, public, public_base, "castiron/promotions/pr-1" - ) - self.assertEqual(snapshot["base_commit"], snapshot["commit"]) - self.assertNotEqual(snapshot["commit"], generated) - self.assertEqual(snapshot["base_commit"], snapshot["commit"]) - self.assertEqual(report.git(public, "rev-list", "--count", snapshot["commit"]), b"1\n") - self.assertEqual(report.hash_codegen_commit(public, snapshot["commit"]), snapshot["hash"]) - with self.assertRaises(report.ReportError): - report.git(public, "cat-file", "-e", private_ancestor) - self.assertNotIn( - b"private-only.txt", report.git(public, "ls-tree", "-r", snapshot["commit"]) - ) - report.git( - public, "push", "origin", f"{snapshot['commit']}:refs/heads/{snapshot['branch']}" - ) - (public / "generated.py").write_text("generated\n# custom\n") - report.git(public, "add", ".") - report.git(public, "commit", "-qm", "public customization") - public_head = report.git(public, "rev-parse", "HEAD").decode().strip() - result, patch = report.build_report( - public, public_base, public_head, fetch=True, require_head_hash=True, public=True - ) - self.assertEqual(result["counts"]["newly_customized"], 1) - self.assertIn(b"+# custom", patch) - self.assertIn(" --public", report.render_report(result)) - self.assertFalse(result["before"]["hash_recorded"]) - again = report.prepare_public_snapshot( - self.repo, base, base, public, public_base, "castiron/promotions/pr-1" - ) - self.assertEqual(again["base_commit"], snapshot["base_commit"]) - self.assertEqual(again["commit"], snapshot["commit"]) - self.assertEqual(again["base_commit"], snapshot["base_commit"]) - tree = report.git(public, "rev-parse", f"{snapshot['commit']}^{{tree}}").decode().strip() - message = report.git(public, "show", "-s", "--format=%B", snapshot["commit"]) - chained = ( - report.git( - public, - "commit-tree", - tree, - "-p", - snapshot["commit"], - "-F", - "-", - input_bytes=message, - ) - .decode() - .strip() - ) - report.git( - public, "push", "--force", "origin", f"{chained}:refs/heads/{snapshot['branch']}" - ) - reused = report.prepare_public_snapshot( - self.repo, base, base, public, public_base, "castiron/promotions/pr-1" - ) - self.assertEqual(reused["commit"], chained) - - # A fresh generation may introduce files. Its public parent must be a - # previously public snapshot, never the private checkpoint parent. - next_generation = "650e8400-e29b-41d4-a716-446655440000" - self.git("checkout", "--detach", generated) - self.write("generated.py", "regenerated\n") - self.write("new.py", "new generated file\n") - self.git("add", "--", "generated.py", "new.py") - metadata = base64.b64encode( - json.dumps( - { - "generation_id": next_generation, - "source_branch": "test", - } - ).encode() - ).decode() - self.git("commit", "-qm", f"next codegen\n\nGeneration metadata: {metadata}") - next_codegen = self.git("rev-parse", "HEAD") - self.git("push", "origin", f"{next_codegen}:refs/heads/codegen/test") - self.write( - ".castiron.stats.yml", - ( - f"schema_version: 1\ngeneration_id: {next_generation}\n" - f"codegen_sha: {next_codegen}\n" - f"codegen_hash: {report.hash_codegen_commit(self.repo, next_codegen)}\n" - ), - ) - self.git("add", "--", ".castiron.stats.yml") - self.git("commit", "-qm", "next integrated SDK") - next_head = self.git("rev-parse", "HEAD") - # An unreviewed path in private HEAD is not enough to publish it. - with self.assertRaisesRegex(report.ReportError, "approved public SDK tree"): - report.prepare_public_snapshot( - self.repo, base, next_head, public, public_base, "castiron/promotions/pr-1" - ) - # Model normal promotion applying its reviewed patch to the public index. - (public / "generated.py").write_text("regenerated\n") - (public / "new.py").write_text("new generated file\n") - report.git(public, "add", "--", "generated.py", "new.py") - advanced = report.prepare_public_snapshot( - self.repo, base, next_head, public, public_base, "castiron/promotions/pr-1" - ) - self.assertEqual( - report.git(public, "rev-parse", advanced["commit"] + "^1").decode().strip(), chained - ) - self.assertEqual(report.hash_codegen_commit(public, advanced["commit"]), advanced["hash"]) - with self.assertRaises(report.ReportError): - report.git(public, "cat-file", "-e", next_codegen) - report.git( - public, "push", "origin", f"{advanced['commit']}:refs/heads/{advanced['branch']}" - ) - (public / "generated.py").write_text("regenerated\n") - (public / "new.py").write_text("new generated file\n") - report.git(public, "add", "--", ".castiron.stats.yml", "generated.py", "new.py") - report.git(public, "commit", "-qm", "next public SDK") - next_public = report.git(public, "rev-parse", "HEAD").decode().strip() - compared, _ = report.build_report( - public, public_base, next_public, fetch=True, require_head_hash=True, public=True - ) - self.assertEqual(compared["before"]["generation_id"], GENERATION) - self.assertEqual(compared["after"]["generation_id"], next_generation) - - # Once A lands, handwritten-only promotion B may reuse A's exact - # snapshot. Its original branch is a locator, not exclusive ownership. - report.git(public, "push", "origin", f"{next_public}:refs/heads/main") - self.write("generated.py", "regenerated\n# handwritten change\n") - self.git("add", "--", "generated.py") - self.git("commit", "-qm", "handwritten change") - custom_head = self.git("rev-parse", "HEAD") - (public / "generated.py").write_text("regenerated\n# handwritten change\n") - report.git(public, "add", "--", "generated.py") - other_branch = "castiron/promotions/pr-2" - other = report.prepare_public_snapshot( - self.repo, next_head, custom_head, public, next_public, other_branch - ) - self.assertEqual(other["commit"], advanced["commit"]) - self.assertEqual( - report.public_metadata(public, other["commit"])["Branch"], advanced["branch"] - ) - report.git(public, "push", "origin", f"{other['commit']}:refs/heads/{other['branch']}") - retried = report.prepare_public_snapshot( - self.repo, next_head, custom_head, public, next_public, other_branch - ) - self.assertEqual(retried["commit"], other["commit"]) - - # A merge-queue commit also resolves the baseline from its own stats, - # independently of the name or parents of the integrated SDK commit. - merged_tree = report.git(public, "write-tree").decode().strip() - pr_head = ( - report.git( - public, - "commit-tree", - merged_tree, - "-p", - next_public, - "-m", - "public handwritten change", - ) - .decode() - .strip() - ) - merged = ( - report.git( - public, - "commit-tree", - merged_tree, - "-p", - next_public, - "-p", - pr_head, - "-m", - "merge queue", - ) - .decode() - .strip() - ) - merged_report, _ = report.build_report( - public, next_public, merged, fetch=True, require_head_hash=True, public=True - ) - self.assertEqual(merged_report["after"]["commit"], advanced["commit"]) - self.assertEqual(merged_report["counts"]["newly_customized"], 1) - - -ADOPTION_GENERATION = "550e8400-e29b-41d4-a716-446655440001" - - -class FirstAdoptionTests(GitTestCase): - def setUp(self) -> None: # pyright: ignore[reportImplicitOverride] - super().setUp() - self.repository = "openai/openai-java" - self.write(".stats.yml", "configured_endpoints: 278\n") - self.root = self.commit("legacy root") - self.git("remote", "add", "origin", "https://github.com/openai/openai-java.git") - self.policy: dict[str, Any] = {"schema_version": 1, "max_custom_patch_lines": 3000} - self.write(".castiron-ratchet.json", json.dumps(self.policy) + "\n") - self.g0 = self.generated("source\n", GENERATION, False) - self.public0 = report.create_public_snapshot( - self.repo, - self.git("rev-parse", f"{self.g0}^{{tree}}"), - GENERATION, - report.hash_codegen_commit(self.repo, self.g0), - "codegen/main", - None, - ) - self.git("update-ref", "refs/remotes/origin/codegen/main", self.public0) - self.write("generated.py", "source\n") - self.stats(self.g0, self.public0, GENERATION) - self.main = self.commit("approved main") - self.git("update-ref", "refs/remotes/origin/main", self.main) - self.git("checkout", "--detach", self.root) - self.pin: dict[str, Any] = { - **self.policy, - "next_policy": {"repository": self.repository, "main_sha": self.main}, - } - self.write(".castiron-ratchet.json", json.dumps(self.pin) + "\n") - self.base = self.commit("separate approved next policy") - self.git("update-ref", "refs/remotes/origin/next", self.base) - - def generated(self, text: str, generation: str, next_target: bool) -> str: - blob = ( - report.git(self.repo, "hash-object", "-w", "--stdin", input_bytes=text.encode()) - .decode() - .strip() - ) - tree = ( - report.git( - self.repo, - "mktree", - input_bytes=f"100644 blob {blob}\tgenerated.py\n".encode(), - ) - .decode() - .strip() - ) - record: dict[str, Any] = { - "generation_id": generation, - "source_branch": "test", - "target": self.repository.split("/")[1], - "language": self.repository.split("-")[-1], - } - if next_target: - record["public_baseline_commit"] = self.base - record = { - "schema_version": "castiron-checkpoint/v2", - "release_target": "next", - "record": record, - } - encoded = base64.b64encode(json.dumps(record).encode()).decode() - commit = ( - report.git( - self.repo, - "commit-tree", - tree, - input_bytes=f"codegen\n\nGeneration metadata: {encoded}\n".encode(), - ) - .decode() - .strip() - ) - namespace = "castiron/next/" if next_target else "" - self.git("update-ref", f"refs/remotes/origin/codegen/{namespace}test", commit) - return commit - - def stats(self, private: str, public: str, generation: str) -> None: - self.write( - ".castiron.stats.yml", - f"schema_version: 1\ngeneration_id: {generation}\ncodegen_sha: {private}\npublic_codegen_sha: {public}\ncodegen_hash: {report.hash_codegen_commit(self.repo, private)}\nconfig_hash: source-config\nopenapi_spec_hash: source-spec\n", - ) - - def seed(self) -> str: - # Synthetic two-parent private seed; production worker initialization is - # covered separately in the SDK worker candidate-flow suite. - self.git("read-tree", "--reset", "-u", self.main) - self.write(".castiron-ratchet.json", json.dumps(self.pin) + "\n") - self.git("add", "-A") - return self.git( - "commit-tree", - self.git("write-tree"), - "-p", - self.base, - "-p", - self.main, - "-m", - "approved main adoption fixture", - ) - - def head(self, *, public_receipt: bool = True, customize: bool = False) -> str: - seed = self.seed() - self.git("checkout", "--detach", seed) - self.g1 = self.generated("next generated\n", ADOPTION_GENERATION, True) - snap = report.create_public_snapshot( - self.repo, - self.git("rev-parse", f"{self.g1}^{{tree}}"), - ADOPTION_GENERATION, - report.hash_codegen_commit(self.repo, self.g1), - "codegen/castiron/next/new", - self.public0, - (self.base, self.main) if public_receipt else None, - ) - self.git("update-ref", "refs/remotes/origin/codegen/castiron/next/new", snap) - self.write( - "generated.py", - "next generated\n" + ("# customization\n" if customize else ""), - ) - self.stats(self.g1, snap, ADOPTION_GENERATION) - return self.commit("next candidate") - - def test_java_public_and_private_report_have_no_fake_before_generation(self) -> None: - head = self.head(customize=True) - for public in (False, True): - result, patch = report.build_report( - self.repo, - self.base, - head, - public=public, - target_branch="next", - require_head_hash=True, - ) - self.assertIsNone(result["before"]) - self.assertEqual(result["adoption_baseline"]["generation_id"], GENERATION) - self.assertEqual(result["files"][0]["added"], "1") - self.assertIn(b"# customization", patch) - self.assertIn("Legacy before-generation unavailable", report.render_report(result)) - - def test_cli_uses_same_protected_pin_path(self) -> None: - self.repository = "openai/openai-cli" - self.git("remote", "set-url", "origin", "https://github.com/openai/openai-cli.git") - self.pin["next_policy"]["repository"] = self.repository - self.write(".castiron-ratchet.json", json.dumps(self.pin) + "\n") - self.base = self.commit("CLI approved policy fixture") - head = self.head() - result, _ = report.build_report( - self.repo, - self.base, - head, - public=False, - target_branch="next", - require_head_hash=True, - ) - self.assertIsNone(result["before"]) - - def test_promotion_base_cli_recognizes_exact_next_branch(self) -> None: - original = report.git - - def offline_git(repo: Path, *args: str, input_bytes: bytes | None = None) -> bytes: - if args[0] == "fetch": - source, destination = args[-1].lstrip("+").split(":", 1) - return original( - repo, - "update-ref", - destination, - source.replace("refs/heads/", "refs/remotes/origin/"), - ) - return original(repo, *args, input_bytes=input_bytes) - - for base, branch, expected in ( - (self.base, "castiron/next", self.main), - (self.base, "castiron/next/child", self.main), - (self.base, "castiron/nextish", None), - (self.base, "castiron/next-door", None), - (self.base, "castiron/promotions/main", None), - (self.main, "castiron/promotions/main", self.main), - ): - with self.subTest(base=base, branch=branch): - output, errors = io.StringIO(), io.StringIO() - with ( - mock.patch.object(report, "git", side_effect=offline_git), - mock.patch.object( - report.sys, - "argv", - [ - "custom_code_report.py", - "promotion-base", - "--repo", - str(self.repo), - "--base", - base, - "--branch", - branch, - ], - ), - mock.patch.object(report.sys, "stdout", output), - mock.patch.object(report.sys, "stderr", errors), - ): - result = report.main() - if expected is None: - self.assertEqual(result, 1) - self.assertEqual(output.getvalue(), "") - self.assertIn("first adoption is next-only", errors.getvalue()) - else: - self.assertEqual(result, 0, errors.getvalue()) - self.assertEqual(output.getvalue(), expected + "\n") - - def test_promotion_prepares_new_snapshot_from_real_main_snapshot(self) -> None: - head = self.head() - original = report.git - - def offline_git(repo: Path, *args: str, input_bytes: bytes | None = None) -> bytes: - if args[0] == "fetch": - for arg in args: - if arg.startswith("+refs/") and ":" in arg: - source, destination = arg.lstrip("+").split(":", 1) - original( - repo, - "update-ref", - destination, - source.replace("refs/heads/", "refs/remotes/origin/"), - ) - return b"" - if args[0] == "ls-remote": - return b"" - return original(repo, *args, input_bytes=input_bytes) - - with mock.patch.object(report, "git", side_effect=offline_git): - prepared = report.prepare_public_snapshot( - self.repo, self.base, head, self.repo, self.base, "castiron/next/fresh" - ) - metadata = report.public_metadata(self.repo, prepared["commit"]) - self.assertEqual(metadata["Target"], "next") - self.assertEqual(metadata["Adoption-Base"], self.base) - self.assertEqual(metadata["Adoption-Source"], self.main) - self.assertEqual(prepared["base_commit"], self.public0) - self.assertEqual(self.git("rev-parse", prepared["commit"] + "^"), self.public0) - - def test_missing_receipt_rejected(self) -> None: - head = self.head(public_receipt=False) - with self.assertRaisesRegex(report.ReportError, "authorized first adoption"): - report.build_report(self.repo, self.base, head, public=True, target_branch="next") - - def test_wrong_sdk_record_rejected(self) -> None: - head = self.head() - self.repository = "openai/openai-cli" - wrong = self.generated("next generated\n", ADOPTION_GENERATION, True) - self.stats( - wrong, report.read_stats(self.repo, head)["public_codegen_sha"], ADOPTION_GENERATION - ) - head = self.commit("wrong SDK binding") - with self.assertRaisesRegex(report.ReportError, "SDK mismatch"): - report.build_report(self.repo, self.base, head, public=False, target_branch="next") - - def test_config_change_rejected(self) -> None: - self.head() - path = self.repo / ".castiron.stats.yml" - path.write_text(path.read_text().replace("source-config", "wrong-config")) - head = self.commit() - with self.assertRaisesRegex(report.ReportError, "authenticated main inputs"): - report.build_report(self.repo, self.base, head, public=False, target_branch="next") - - def test_existing_corrupt_deleted_stats_rejected(self) -> None: - self.write(".castiron.stats.yml", "corrupt: true\n") - corrupt = self.commit() - with self.assertRaisesRegex(report.ReportError, "existing/corrupt"): - report.first_adoption_source(self.repo, corrupt, target_branch="next", fetch=False) - self.git("rm", ".castiron.stats.yml") - deleted = self.commit() - with self.assertRaisesRegex(report.ReportError, "deleted"): - report.first_adoption_source(self.repo, deleted, target_branch="next", fetch=False) - - def test_candidate_cannot_select_source_or_remove_policy(self) -> None: - self.head() - self.write(".castiron-ratchet.json", json.dumps(self.policy) + "\n") - head = self.commit() - with self.assertRaisesRegex(report.ReportError, "protected target policy"): - report.build_report(self.repo, self.base, head, public=True, target_branch="next") - - def test_current_count_and_strict_zero_still_enforced(self) -> None: - import custom_code_budget as budget - - head = self.head(customize=True) - measurement = report.build_report( - self.repo, self.base, head, public=True, target_branch="next" - ) - evaluated, _ = budget.evaluate( - self.repo, - self.base, - head, - public=True, - policy_sha=self.main, - target_branch="next", - measurement=measurement, - ) - self.assertEqual(evaluated["total"], 1) - self.assertEqual(evaluated["checks"]["budget"]["state"], "success", evaluated) - self.write( - ".castiron-ratchet.json", - '{"schema_version":2,"max_custom_patch_lines":0,"generated_files":"strict"}\n', - ) - strict = self.commit() - evaluated, _ = budget.evaluate( - self.repo, - self.base, - head, - public=True, - policy_sha=strict, - target_branch="next", - measurement=measurement, - ) - self.assertEqual(evaluated["limit"], 0) - self.assertEqual(evaluated["checks"]["budget"]["state"], "failure") - self.assertIn("manifest", evaluated["checks"]["budget"]["description"]) - - def test_unapproved_source_pin_rejected(self) -> None: - self.pin["next_policy"]["main_sha"] = self.base - self.write(".castiron-ratchet.json", json.dumps(self.pin) + "\n") - invalid = self.commit() - with self.assertRaisesRegex(report.ReportError, "authenticated main history"): - report.first_adoption_source(self.repo, invalid, target_branch="next", fetch=False) - - def test_advanced_target_requires_fresh_adoption(self) -> None: - head = self.head() - self.git("checkout", "--detach", self.base) - self.write("legacy-note.txt", "another approved target change\n") - advanced = self.commit() - with self.assertRaisesRegex(report.ReportError, "target advanced"): - report.build_report(self.repo, advanced, head, public=True, target_branch="next") - - def test_main_target_rejected(self) -> None: - with self.assertRaisesRegex(report.ReportError, "next-only"): - report.first_adoption_source(self.repo, self.base, target_branch="main", fetch=False) - - def test_normal_public_main_report_unchanged(self) -> None: - self.git("checkout", "--detach", self.main) - self.write("generated.py", "source\n# customization\n") - head = self.commit() - result, _ = report.build_report( - self.repo, self.main, head, public=True, target_branch="main" - ) - self.assertIsNotNone(result["before"]) - self.assertEqual(result["before_status"], "verified") - - def promoted_public_head(self) -> tuple[Path, str, dict[str, Any]]: - # Exercise the separate public checkout and the actual single-parent - # commit-tree shape in sdk/promote.rs, not the private merge seed. - import subprocess - - destination = self.repo / "public-checkout" - head = self.head(customize=True) - subprocess.run( - ["git", "clone", "-q", "--no-hardlinks", str(self.repo), str(destination)], check=True - ) - - def public_git(*args: str) -> str: - return report.git(destination, *args).decode().strip() - - public_git("config", "user.name", "Castiron test") - public_git("config", "user.email", "castiron@example.test") - public_git("remote", "set-url", "origin", "https://github.com/" + self.repository + ".git") - for ref, commit in [ - ("main", self.main), - ("next", self.base), - ("codegen/main", self.public0), - ]: - public_git("update-ref", "refs/remotes/origin/" + ref, commit) - public_git("checkout", "--detach", self.base) - # Normal promotion applies the reviewed private patch to this index. - public_git("read-tree", "--reset", "-u", head) - original = report.git - - def offline_git(repo: Path, *args: str, input_bytes: bytes | None = None) -> bytes: - if args[0] == "fetch": - for arg in args: - if arg.startswith("+refs/") and ":" in arg: - source, target = arg.lstrip("+").split(":", 1) - original( - repo, - "update-ref", - target, - source.replace("refs/heads/", "refs/remotes/origin/"), - ) - return b"" - if args[0] == "ls-remote": - return b"" - return original(repo, *args, input_bytes=input_bytes) - - with mock.patch.object(report, "git", side_effect=offline_git): - prepared = report.prepare_public_snapshot( - self.repo, self.base, head, destination, self.base, "castiron/next/fresh" - ) - public_git("update-ref", "refs/remotes/origin/" + prepared["branch"], prepared["commit"]) - promoted = public_git( - "commit-tree", - public_git("write-tree"), - "-p", - self.base, - "-m", - "normal public promotion", - ) - self.assertEqual( - public_git("rev-list", "--parents", "-n", "1", promoted), promoted + " " + self.base - ) - with self.assertRaises(report.ReportError): - public_git("merge-base", "--is-ancestor", self.main, promoted) - # Private authorization still retains both parents. - self.git("merge-base", "--is-ancestor", self.main, head) - self.git("merge-base", "--is-ancestor", self.base, head) - return destination, promoted, prepared - - def test_normal_single_parent_public_promotion_reports_first_adoption(self) -> None: - destination, promoted, _ = self.promoted_public_head() - result, patch = report.build_report( - destination, - self.base, - promoted, - public=True, - target_branch="next", - require_head_hash=True, - ) - self.assertIsNone(result["before"]) - self.assertEqual(result["adoption_baseline"]["generation_id"], GENERATION) - self.assertIn(b"# customization", patch) - - def test_public_adoption_rejects_disconnected_snapshot_with_valid_receipt(self) -> None: - destination, promoted, prepared = self.promoted_public_head() - # The right receipt/hash alone does not authenticate approved-main lineage. - detached = report.create_public_snapshot( - destination, - report.git(destination, "rev-parse", prepared["commit"] + "^{tree}").decode().strip(), - ADOPTION_GENERATION, - prepared["hash"], - prepared["branch"], - None, - (self.base, self.main), - ) - report.git(destination, "update-ref", "refs/remotes/origin/" + prepared["branch"], detached) - report.git(destination, "checkout", "--detach", promoted) - stats = destination / ".castiron.stats.yml" - stats.write_text(stats.read_text().replace(prepared["commit"], detached)) - report.git(destination, "add", ".castiron.stats.yml") - invalid = ( - report.git( - destination, - "commit-tree", - report.git(destination, "write-tree").decode().strip(), - "-p", - self.base, - "-m", - "disconnected snapshot", - ) - .decode() - .strip() - ) - with self.assertRaises(report.ReportError): - report.build_report( - destination, - self.base, - invalid, - public=True, - target_branch="next", - require_head_hash=True, - ) - - def test_cli_normal_single_parent_public_promotion_reports_first_adoption(self) -> None: - self.repository = "openai/openai-cli" - self.git("remote", "set-url", "origin", "https://github.com/openai/openai-cli.git") - self.pin["next_policy"]["repository"] = self.repository - self.write(".castiron-ratchet.json", json.dumps(self.pin) + "\n") - self.base = self.commit("CLI approved next policy") - self.git("update-ref", "refs/remotes/origin/next", self.base) - destination, promoted, _ = self.promoted_public_head() - result, _ = report.build_report( - destination, - self.base, - promoted, - public=True, - target_branch="next", - require_head_hash=True, - ) - self.assertIsNone(result["before"]) - self.assertEqual(result["adoption_baseline"]["generation_id"], GENERATION) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/castiron/test_custom_code_review_revocation.py b/scripts/castiron/test_custom_code_review_revocation.py deleted file mode 100644 index f7285795a..000000000 --- a/scripts/castiron/test_custom_code_review_revocation.py +++ /dev/null @@ -1,986 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -"""Exercise review revocation and target freshness against offline GitHub state.""" - -from __future__ import annotations - -import json -import shutil -import subprocess -import tempfile -import unittest -from pathlib import Path -from typing import Any - -import custom_code_policy as policy -from custom_code_test_support import WORKFLOW, run_publisher, source_head, source_run - - -@unittest.skipUnless( - shutil.which("node"), "Node is needed to execute the review-revocation fixture" -) -class ReviewRevocationTests(unittest.TestCase): - def payload(self) -> dict[str, Any]: - head, base, queue = "a" * 40, "b" * 40, "c" * 40 - pull = { - "number": 3, - "user": {"login": "author"}, - "state": "open", - "head": source_head(head), - "base": {"ref": "next", "sha": base, "repo": {"full_name": "openai/example"}}, - } - run = { - **source_run(head), - "id": 20, - "actor": {"login": "outsider", "type": "User"}, - "event": "pull_request_review", - "status": "completed", - "pull_requests": [{"number": 3, "head": {"sha": head}}], - } - - def original(sha: str, event: str, identity: int) -> dict[str, Any]: - return {**run, "head_sha": sha, "id": identity, "event": event} - - return { - "run": run, - "policies_by_sha": { - base: {"schema_version": 1, "max_custom_patch_lines": 0}, - head: {"schema_version": 1, "max_custom_patch_lines": 1}, - }, - "previous_statuses": [ - { - "context": name, - "state": "success", - "creator": {"login": "github-actions[bot]"}, - "target_url": "https://github.com/openai/example/actions/runs/19", - } - for name in ( - "Castiron next / budget-only change", - "Castiron next / custom-code budget", - ) - ], - "current": pull, - "queue_nodes": [ - {"headCommit": {"oid": queue}, "pullRequest": {"number": 3, "headRefOid": head}} - ], - "runs_by_sha": { - head: [original(head, "pull_request", 12)], - queue: [original(queue, "merge_group", 14)], - }, - "context": { - "payload": {"workflow_run": dict(run)}, - "runId": 21, - "repo": {"owner": "openai", "repo": "example"}, - "serverUrl": "https://github.com", - }, - } - - def invoke(self, payload: dict[str, Any]) -> list[dict[str, Any]]: - return run_publisher("Re-evaluate current PR and queue after review changes", payload) - - def test_revokes_pr_and_queue_before_requesting_normal_re_evaluation(self) -> None: - result = self.invoke(self.payload()) - self.assertEqual([item.get("state") for item in result[:4]], ["pending"] * 4) - self.assertEqual({item["sha"] for item in result[:4]}, {"a" * 40, "c" * 40}) - self.assertTrue(all(item["context"].startswith("Castiron next /") for item in result[:4])) - self.assertEqual([item["run_id"] for item in result[4:]], [12, 14]) - self.assertNotIn("success", [item.get("state") for item in result]) - - def test_retries_policy_failure_when_main_advances_before_revocation(self) -> None: - payload = self.payload() - payload["main_sha_sequence"] = ["d" * 40, "e" * 40] - payload["main_sha"] = "e" * 40 - payload["policy_error_sequence"] = [True, False] - result = self.invoke(payload) - self.assertEqual([item.get("state") for item in result[:4]], ["pending"] * 4) - self.assertEqual({item["sha"] for item in result[:4]}, {"a" * 40, "c" * 40}) - self.assertEqual([item["run_id"] for item in result[4:]], [12, 14]) - - def test_uses_live_target_when_pull_base_snapshot_lags(self) -> None: - for requires_human in (False, True): - with self.subTest(requires_human=requires_human): - payload = self.payload() - payload["target_sha"] = "e" * 40 - payload["expected_policy_base"] = "e" * 40 - payload["requires_human"] = requires_human - result = self.invoke(payload) - if requires_human: - self.assertEqual([item.get("state") for item in result[:4]], ["pending"] * 4) - self.assertEqual([item["run_id"] for item in result[4:]], [12, 14]) - else: - self.assertEqual(result, []) - - def test_retries_when_live_target_advances_during_evaluation(self) -> None: - for requires_human in (False, True): - with self.subTest(requires_human=requires_human): - payload = self.payload() - payload["target_sha_sequence"] = ["d" * 40, "e" * 40] - payload["target_sha"] = "e" * 40 - payload["expected_policy_bases"] = ["d" * 40, "e" * 40] - payload["requires_human"] = requires_human - result = self.invoke(payload) - if requires_human: - self.assertEqual( - [item["run_id"] for item in result if "run_id" in item], [12, 14] - ) - self.assertTrue( - all(item.get("state") == "pending" for item in result if "state" in item) - ) - else: - self.assertEqual(result, []) - - def test_refreshes_executable_policy_when_main_changes_review_requirement(self) -> None: - payload = self.payload() - payload["main_sha_sequence"] = ["d" * 40, "e" * 40] - payload["main_sha"] = "e" * 40 - payload["requires_human_by_main"] = {"d" * 40: False, "e" * 40: True} - result = self.invoke(payload) - self.assertEqual([item.get("state") for item in result[:4]], ["pending"] * 4) - self.assertEqual({item["sha"] for item in result[:4]}, {"a" * 40, "c" * 40}) - self.assertEqual([item["run_id"] for item in result[4:]], [12, 14]) - - def test_rejects_invalid_trusted_policy_downloads(self) -> None: - for override in [ - {"type": "symlink"}, - {"encoding": "none"}, - {"size": 1048577}, - {"size": 0}, - {"content": ""}, - ]: - with self.subTest(override=override): - payload = self.payload() - payload["policy_file_override"] = override - with self.assertRaises(subprocess.CalledProcessError): - self.invoke(payload) - - def approval(self) -> dict[str, Any]: - return { - "fullDatabaseId": "101", - "state": "APPROVED", - "body": policy.HUMAN_REVIEW_RULES["attestation_prefix"] + "a" * 40, - "author": {"login": "reviewer", "__typename": "User"}, - "authorAssociation": "MEMBER", - "commit": {"oid": "a" * 40}, - "updatedAt": "2026-10-22T01:00:00Z", - } - - def recovery(self) -> dict[str, Any]: - payload = self.payload() - payload["review_nodes"] = [self.approval()] - payload["run"]["actor"] = {"login": "reviewer", "type": "User"} - for status in payload["previous_statuses"]: - status["state"] = "failure" - return payload - - def history(self, payload: dict[str, Any], writes: list[dict[str, Any]]) -> None: - payload["statuses_by_sha"] = { - sha: [ - {**item, "creator": {"login": "github-actions[bot]"}} - for item in writes - if item.get("sha") == sha - ] - for sha in ("a" * 40, "c" * 40) - } - payload["runs_by_id"] = { - "21": { - "id": 21, - "workflow_id": 789, - "status": "completed", - "conclusion": "success", - "repository": {"full_name": "openai/example"}, - "event": "workflow_run", - } - } - - def test_unrelated_comment_cannot_invalidate_valid_approval_or_request_recovery(self) -> None: - for state in ("success", "failure", "pending"): - payload = self.recovery() - payload["run"]["actor"]["login"] = "outsider" - payload["review_nodes"].append( - { - **self.approval(), - "fullDatabaseId": "102", - "author": {"login": "outsider", "__typename": "User"}, - "state": "COMMENTED", - "body": "hello", - } - ) - for status in payload["previous_statuses"]: - status["state"] = state - payload["forbid_queue"] = True - self.assertEqual(self.invoke(payload), []) - - def test_later_comment_preserves_green_pr_and_queue_statuses(self) -> None: - for state in ("COMMENTED", "PENDING"): - for paginated in (False, True): - with self.subTest(state=state, paginated=paginated): - payload = self.recovery() - approval = self.approval() - comment = { - **approval, - "fullDatabaseId": "102", - "state": state, - "body": "Follow-up discussion", - } - payload["review_nodes"] = [approval, comment] - if paginated: - payload.pop("review_nodes") - payload["review_pages"] = [ - { - "expected_cursor": None, - "nodes": [comment], - "pageInfo": {"hasNextPage": True, "endCursor": "more"}, - }, - { - "expected_cursor": "more", - "nodes": [approval], - "pageInfo": {"hasNextPage": False}, - }, - ] - for status in payload["previous_statuses"]: - status["state"] = "success" - self.assertEqual(self.invoke(payload), []) - - def test_ineligible_review_cannot_request_recovery(self) -> None: - for overrides in ( - {"state": "COMMENTED"}, - {"body": ""}, - {"authorAssociation": "CONTRIBUTOR"}, - {"commit": {"oid": "b" * 40}}, - {"author": {"login": "author", "__typename": "User"}}, - {"author": {"login": "reviewer", "__typename": "Bot"}}, - ): - with self.subTest(overrides=overrides): - payload = self.recovery() - payload["review_nodes"][0].update(overrides) - self.assertEqual(self.invoke(payload), []) - - def test_repeated_invalid_signals_do_not_rerun_already_revoked_heads(self) -> None: - payload = self.payload() - writes = self.invoke(payload) - self.history(payload, writes) - self.assertEqual(self.invoke(payload), []) - for statuses in payload["statuses_by_sha"].values(): - for status in statuses: - status["state"] = "failure" - self.assertEqual(self.invoke(payload), []) - - def test_approval_recovery_coalesces_and_edited_restoration_recovers_again(self) -> None: - payload = self.recovery() - writes = self.invoke(payload) - self.assertEqual(len(writes), 6) - self.assertTrue(all(len(item["description"]) <= 140 for item in writes[:4])) - self.history(payload, writes) - self.assertEqual(self.invoke(payload), []) - payload["review_nodes"][0]["updatedAt"] = "2026-10-22T01:02:00Z" - restored = self.invoke(payload) - self.assertEqual(len(restored), 6) - self.assertNotEqual(writes[0]["description"], restored[0]["description"]) - - def test_new_revocation_supersedes_old_recovery_even_with_equal_edit_timestamp(self) -> None: - payload = self.recovery() - approved = self.invoke(payload) - self.history(payload, approved) - for statuses in payload["statuses_by_sha"].values(): - statuses[:0] = [ - {**status, "state": "success", "description": "Successful evaluation"} - for status in statuses - ] - payload["review_nodes"][0]["body"] = "" - revoked = self.invoke(payload) - self.assertEqual(len(revoked), 6) - self.history(payload, revoked) - for sha, statuses in payload["statuses_by_sha"].items(): - statuses.extend( - {**item, "creator": {"login": "github-actions[bot]"}} - for item in approved - if item.get("sha") == sha - ) - payload["review_nodes"][0] = ( - self.approval() - ) # Same ID, body and second-precision update time. - self.assertEqual(len(self.invoke(payload)), 6) - - def test_full_review_id_handles_current_and_beyond_safe_integer_ids(self) -> None: - payload = self.recovery() - payload["review_nodes"][0].update(databaseId=None, fullDatabaseId="5614624241") - self.assertEqual(len(self.invoke(payload)), 6) - payload = self.payload() - payload["review_nodes"] = [ - {**self.approval(), "databaseId": None, "fullDatabaseId": "9007199254740992"}, - { - **self.approval(), - "databaseId": None, - "fullDatabaseId": "9007199254740993", - "state": "CHANGES_REQUESTED", - }, - ] - self.assertEqual(len(self.invoke(payload)), 6) - payload["review_nodes"][0]["fullDatabaseId"] = 9007199254740992 - with self.assertRaises(subprocess.CalledProcessError): - self.invoke(payload) - - def test_publisher_failure_supersedes_recovery_without_observed_revocation(self) -> None: - payload = self.recovery() - approved = self.invoke(payload) - self.history(payload, approved) - for statuses in payload["statuses_by_sha"].values(): - statuses[:0] = [ - {**status, "state": "failure", "description": "Approval missing during publication"} - for status in statuses - ] - # Publisher observed invalid body; review jobs only see its same-second restoration. - # No new review identity/timestamp and no review-handler revocation marker exists. - recovered = self.invoke(payload) - self.assertEqual(len(recovered), 6) - self.history(payload, recovered) - self.assertEqual(self.invoke(payload), []) - - def test_deleted_marker_writer_is_untrusted_but_service_errors_propagate(self) -> None: - payload = self.recovery() - approved = self.invoke(payload) - self.history(payload, approved) - payload["run_errors"] = {"21": 404} - self.assertEqual(len(self.invoke(payload)), 6) - for status in (403, 429, 500): - payload["run_errors"]["21"] = status - with self.subTest(status=status), self.assertRaises(subprocess.CalledProcessError): - self.invoke(payload) - - def test_failed_partial_dispatch_is_retried_before_signals_coalesce(self) -> None: - for authorized in (False, True): - for conclusion in ("failure", "cancelled", "timed_out"): - payload = self.recovery() if authorized else self.payload() - payload["fail_rerun"] = True - # The fixture persists actual pending writes before the simulated API error. - with tempfile.TemporaryDirectory() as directory: - capture = Path(directory) / "pending.json" - payload["capture_writes"] = str(capture) - with self.assertRaises(subprocess.CalledProcessError): - self.invoke(payload) - pending = json.loads(capture.read_text()) - self.assertEqual(len(pending), 4) - self.history(payload, pending) - payload.pop("capture_writes") - payload.pop("fail_rerun") - payload["runs_by_id"]["21"]["conclusion"] = conclusion - payload["context"]["runId"] = 22 - retried = self.invoke(payload) - self.assertEqual([item["run_id"] for item in retried if "run_id" in item], [12, 14]) - self.history(payload, retried) - payload["runs_by_id"]["22"] = {**payload["runs_by_id"]["21"], "id": 22} - self.assertEqual(self.invoke(payload), []) - - def test_untrusted_marker_cannot_suppress_recovery(self) -> None: - for field, value in ( - ("workflow_id", 999), - ("event", "pull_request"), - ("repository", {"full_name": "attacker/example"}), - ): - payload = self.recovery() - self.history(payload, self.invoke(payload)) - payload["runs_by_id"]["21"][field] = value - self.assertEqual(len(self.invoke(payload)), 6) - - def test_dismissal_edit_away_and_blocking_review_revoke_even_for_other_signal_actor( - self, - ) -> None: - for changed in ( - {"state": "DISMISSED"}, - {"body": ""}, - {"state": "CHANGES_REQUESTED", "fullDatabaseId": "102"}, - ): - payload = self.payload() - payload["review_nodes"] = [{**self.approval(), **changed}] - self.assertEqual(len(self.invoke(payload)), 6) - payload = self.payload() - payload["review_pages"] = [ - { - "expected_cursor": None, - "nodes": [ - {**self.approval(), "fullDatabaseId": "102", "state": "CHANGES_REQUESTED"} - ], - "pageInfo": {"hasNextPage": True, "endCursor": "more"}, - }, - { - "expected_cursor": "more", - "nodes": [self.approval()], - "pageInfo": {"hasNextPage": False}, - }, - ] - self.assertEqual(len(self.invoke(payload)), 6) - - def test_missing_authoritative_edit_time_rejects_recovery(self) -> None: - payload = self.recovery() - payload["review_nodes"][0]["updatedAt"] = None - with self.assertRaises(subprocess.CalledProcessError): - self.invoke(payload) - - def test_review_signal_requires_canonical_workflow_identity_before_writes(self) -> None: - for identity in (None, "456", 999): - for path in ( - ".github/workflows/castiron-custom-code.yml", - ".github/workflows/castiron-custom-code.yml@evil.yml", - ): - with self.subTest(identity=identity, path=path): - payload = self.payload() - payload["run"].update({"workflow_id": identity, "path": path}) - payload["context"]["payload"]["workflow_run"] = dict(payload["run"]) - self.assertEqual(self.invoke(payload), []) - - def test_rejects_forged_or_superseded_review_signals(self) -> None: - for field, value in [ - ("path", ".github/workflows/other.yml"), - ("run_attempt", 9), - ("head_sha", "d" * 40), - ("event", "pull_request"), - ]: - with self.subTest(field=field): - payload = self.payload() - payload["run"][field] = value - with self.assertRaises(subprocess.CalledProcessError): - self.invoke(payload) - payload = self.payload() - payload["run"]["repository"] = {"full_name": "attacker/example"} - with self.assertRaises(subprocess.CalledProcessError): - self.invoke(payload) - - def test_head_change_keeps_observed_statuses_revoked_without_stale_reruns(self) -> None: - payload = self.payload() - changed = {**payload["current"], "head": {"sha": "d" * 40}} - payload["current_sequence"] = [payload["current"], changed] - payload["current"] = changed - result = self.invoke(payload) - self.assertEqual(len(result), 4) - self.assertTrue(all(item["state"] == "pending" for item in result)) - self.assertNotIn("d" * 40, {item["sha"] for item in result}) - - def test_queue_change_retries_and_invalidates_new_snapshot(self) -> None: - payload = self.payload() - old = list(payload["queue_nodes"]) - payload["queue_nodes"].append( - { - "headCommit": {"oid": "d" * 40}, - "pullRequest": {"number": 4, "headRefOid": "e" * 40}, - } - ) - payload["runs_by_sha"]["d" * 40] = [ - { - **payload["runs_by_sha"]["c" * 40][0], - "head_sha": "d" * 40, - "id": 16, - } - ] - payload["queue_pages"] = [ - { - "expected_cursor": None, - "nodes": old, - "pageInfo": {"hasNextPage": False}, - } - ] - result = self.invoke(payload) - pending = [item for item in result if item.get("state") == "pending"] - self.assertEqual({item["sha"] for item in pending}, {"a" * 40, "c" * 40, "d" * 40}) - self.assertEqual([item["run_id"] for item in result if "run_id" in item], [12, 14, 16]) - self.assertTrue(all(item.get("state") == "pending" for item in result[:-3])) - - def test_empty_review_pr_list_uses_authenticated_fork_discovery(self) -> None: - for associated in ([], [{"number": 3}]): - payload = self.payload() - payload["run"].update( - { - "pull_requests": [], - "head_repository": source_run("a" * 40)["head_repository"], - "head_branch": "sdk", - } - ) - payload["associated_pulls"] = associated - payload["fallback_pulls"] = [{"number": 3}] - self.assertEqual(len(self.invoke(payload)), 6) - payload["current"]["head"]["sha"] = "d" * 40 - payload["merge_sha"] = "e" * 40 - payload["merge_commit"] = {"parents": []} - self.assertEqual(self.invoke(payload), []) - - def test_shared_head_revokes_the_run_source_pr(self) -> None: - payload = self.payload() - payload["run"]["pull_requests"] = [] - payload["run"]["head_repository"]["id"] = 101 - payload["current"]["head"].update( - ref="sdk", repo={"id": 101, "full_name": "contributor/example"} - ) - payload["associated_pulls"] = [{"number": number} for number in (3, 4, 5)] - payload["pulls_by_number"] = { - str(number): { - **payload["current"], - "number": number, - "head": {**payload["current"]["head"], **change}, - } - for number, change in ((4, {"ref": "other"}), (5, {"repo": {"id": 102}})) - } - result = self.invoke(payload) - self.assertEqual([item.get("state") for item in result[:4]], ["pending"] * 4) - self.assertEqual({item["sha"] for item in result[:4]}, {"a" * 40, "c" * 40}) - - def test_branch_only_merge_signal_requires_current_merge_and_base(self) -> None: - payload = self.payload() - payload["run"]["pull_requests"] = [] - payload["run"]["head_sha"] = payload["context"]["payload"]["workflow_run"]["head_sha"] = ( - "e" * 40 - ) - payload["fallback_pulls"] = [{"number": 3}] - payload["merge_commit"] = {"parents": [{"sha": "b" * 40}, {"sha": "a" * 40}]} - payload["merge_sha"] = "f" * 40 - self.assertEqual(self.invoke(payload), []) - - payload["merge_sha"] = "e" * 40 - self.assertEqual(len(self.invoke(payload)), 6) - payload["current"]["base"]["sha"] = "d" * 40 - payload["comparisons"] = { - f"{'d' * 40}...{'a' * 40}": {"merge_base_commit": {"sha": "b" * 40}} - } - self.assertEqual(self.invoke(payload), []) - - def test_merge_sha_signal_survives_base_advance_and_rejects_wrong_parents(self) -> None: - payload = self.payload() - payload["run"]["head_sha"] = payload["context"]["payload"]["workflow_run"]["head_sha"] = ( - "e" * 40 - ) - payload["current"]["base"]["sha"] = "d" * 40 - payload["comparisons"] = { - f"{'d' * 40}...{'a' * 40}": {"merge_base_commit": {"sha": "b" * 40}} - } - payload["merge_commit"] = {"parents": [{"sha": "b" * 40}, {"sha": "a" * 40}]} - self.assertEqual(len(self.invoke(payload)), 6) - payload["comparisons"][f"{'b' * 40}...{'d' * 40}"] = { - "merge_base_commit": {"sha": "f" * 40} - } - self.assertEqual(self.invoke(payload), []) - payload["merge_commit"]["parents"].reverse() - self.assertEqual(self.invoke(payload), []) - - def test_trusted_policy_decision_controls_revocation(self) -> None: - payload = self.payload() - payload["requires_human"] = False - self.assertEqual(self.invoke(payload), []) - payload["requires_human"] = True - self.assertEqual(len(self.invoke(payload)), 6) - - def test_policy_failure_or_malformed_decision_cannot_write_statuses(self) -> None: - for result in ({"policy_error": True}, {"requires_human": "false"}): - with self.subTest(result=result), tempfile.TemporaryDirectory() as directory: - payload = {**self.payload(), **result} - capture = Path(directory) / "writes.json" - payload["capture_writes"] = str(capture) - with self.assertRaises(subprocess.CalledProcessError): - self.invoke(payload) - self.assertEqual(json.loads(capture.read_text()), []) - - def test_revoked_go_ratchet_activation_invalidates_pr_and_queue_statuses(self) -> None: - for changed in ({"state": "DISMISSED"}, {"body": ""}): - with self.subTest(changed=changed): - payload = self.payload() - payload["policies_by_sha"]["a" * 40] = { - **payload["policies_by_sha"]["b" * 40], - "ratchet": "go", - } - payload["review_nodes"] = [{**self.approval(), **changed}] - writes = self.invoke(payload) - self.assertCountEqual( - [(item["sha"], item["state"]) for item in writes if "state" in item], - [("a" * 40, "pending")] * 2 + [("c" * 40, "pending")] * 2, - ) - self.assertCountEqual( - [item["run_id"] for item in writes if "run_id" in item], [12, 14] - ) - - def test_closed_or_superseded_review_is_a_no_op(self) -> None: - for changed in ({"state": "closed"}, {"head": {"sha": "f" * 40}}): - payload = self.payload() - payload["current"].update(changed) - self.assertEqual(self.invoke(payload), []) - - def test_null_queue_association_does_not_block_revocation(self) -> None: - payload = self.payload() - payload["queue_nodes"].insert(0, {"headCommit": {"oid": "f" * 40}, "pullRequest": None}) - result = self.invoke(payload) - self.assertEqual(len(result), 6) - self.assertNotIn("f" * 40, {item.get("sha") for item in result}) - payload["queue_nodes"] = payload["queue_nodes"][:1] - self.assertEqual(len(self.invoke(payload)), 3) - - def test_rerun_requires_the_reviewed_pr_not_just_shared_head(self) -> None: - payload = self.payload() - correct = payload["runs_by_sha"]["a" * 40][0] - main = { - **payload["current"], - "number": 4, - "base": {**payload["current"]["base"], "ref": "main"}, - } - payload["pulls_by_number"] = {"4": main} - other: dict[str, Any] = { - **correct, - "id": 30, - "pull_requests": [{"number": 4, "head": {"sha": "a" * 40}}], - } - payload["runs_by_sha"]["a" * 40].append(other) - result = self.invoke(payload) - self.assertEqual([item["run_id"] for item in result if "run_id" in item], [12, 14]) - # Empty fork associations may rerun only if canonical discovery selects this next PR. - fork: dict[str, Any] = { - **correct, - "pull_requests": [], - "head_repository": source_run("a" * 40)["head_repository"], - "head_branch": "sdk", - } - payload["fallback_pulls"] = [{"number": 3}] - payload["runs_by_sha"]["a" * 40] = [fork, other] - result = self.invoke(payload) - self.assertEqual([item["run_id"] for item in result if "run_id" in item], [12, 14]) - association_cases: tuple[list[dict[str, Any]], ...] = ( - [], - [*correct["pull_requests"], *other["pull_requests"]], - ) - for associations in association_cases: - payload["associated_pulls"] = [{"number": 3}, {"number": 4}] - payload["runs_by_sha"]["a" * 40] = [{**fork, "pull_requests": associations}, other] - result = self.invoke(payload) - self.assertEqual([item["run_id"] for item in result if "run_id" in item], [14]) - self.assertTrue(all(item["state"] == "pending" for item in result[:4])) - - def test_unprepared_queue_entry_still_revokes_pr_statuses(self) -> None: - payload = self.payload() - payload["queue_nodes"][0]["headCommit"] = None - result = self.invoke(payload) - self.assertEqual([item.get("state") for item in result], ["pending", "pending", None]) - self.assertEqual(result[-1]["run_id"], 12) - - def test_unrelated_review_does_not_restart_reporter(self) -> None: - payload = self.payload() - payload["files"] = [{"filename": "src/client.ts"}] - self.assertEqual(self.invoke(payload), []) - - def test_no_success_or_foreign_run_can_be_published(self) -> None: - payload = self.payload() - payload["runs_by_sha"]["a" * 40][0]["repository"] = {"full_name": "other/repo"} - result = self.invoke(payload) - self.assertEqual( - [item["run_id"] for item in result if item.get("operation") == "rerun"], [14] - ) - self.assertTrue(all(item.get("state") != "success" for item in result)) - - def main_advance_payload(self) -> dict[str, Any]: - payload = self.payload() - main = "d" * 40 - run = { - **source_run(main, "push"), - "head_branch": "main", - "head_repository": {"full_name": "openai/example"}, - } - payload["run"] = run - payload["main_sha"] = main - payload["context"]["payload"]["workflow_run"] = dict(run) - payload["runs_by_sha"]["c" * 40][0]["head_branch"] = "gh-readonly-queue/next/pr-3-example" - return payload - - def invoke_main_advance(self, payload: dict[str, Any]) -> list[dict[str, Any]]: - return run_publisher("Re-evaluate live next queue after main advances", payload) - - def test_main_advance_revokes_success_before_normal_queue_rerun(self) -> None: - result = self.invoke_main_advance(self.main_advance_payload()) - self.assertEqual([item.get("state") for item in result[:2]], ["pending", "pending"]) - self.assertEqual({item["sha"] for item in result[:2]}, {"c" * 40}) - self.assertTrue(all(item["context"].startswith("Castiron next /") for item in result[:2])) - self.assertEqual([item["run_id"] for item in result[2:]], [14]) - self.assertNotIn("success", [item.get("state") for item in result]) - - def test_old_main_push_does_not_invalidate_newer_qualified_candidates(self) -> None: - payload = self.main_advance_payload() - payload["main_sha"] = "e" * 40 - self.assertEqual(self.invoke_main_advance(payload), []) - - def test_main_advance_rejects_foreign_or_non_main_signals(self) -> None: - for field, value in ( - ("head_repository", {"full_name": "other/repo"}), - ("repository", {"full_name": "other/repo"}), - ("head_branch", "next"), - ("event", "pull_request"), - ): - with self.subTest(field=field): - payload = self.main_advance_payload() - payload["run"][field] = value - with self.assertRaises(subprocess.CalledProcessError): - self.invoke_main_advance(payload) - payload = self.main_advance_payload() - payload["run"]["workflow_id"] = 999 - self.assertEqual(self.invoke_main_advance(payload), []) - - def test_main_advance_invalidates_every_queue_page_before_dispatch(self) -> None: - payload = self.main_advance_payload() - payload["queue_pages"] = [ - { - "expected_cursor": None, - "nodes": [{"headCommit": None}, {"headCommit": {"oid": "c" * 40}}], - "pageInfo": {"hasNextPage": True, "endCursor": "second"}, - }, - { - "expected_cursor": "second", - "nodes": [{"headCommit": {"oid": "e" * 40}}, {"headCommit": None}], - "pageInfo": {"hasNextPage": False, "endCursor": None}, - }, - ] - result = self.invoke_main_advance(payload) - self.assertEqual([item.get("state") for item in result[:4]], ["pending"] * 4) - self.assertEqual({item["sha"] for item in result[:4]}, {"c" * 40, "e" * 40}) - self.assertEqual([item["run_id"] for item in result[4:]], [14]) - - def test_main_advance_still_rejects_malformed_non_null_heads(self) -> None: - for head in ({}, {"oid": None}, {"oid": "g" * 40}): - with self.subTest(head=head): - payload = self.main_advance_payload() - payload["queue_nodes"][0]["headCommit"] = head - with self.assertRaises(subprocess.CalledProcessError): - self.invoke_main_advance(payload) - - def test_main_advance_does_not_rerun_wrong_queue_or_foreign_workflows(self) -> None: - for field, value in ( - ("repository", {"full_name": "other/repo"}), - ("workflow_id", 999), - ("head_branch", "gh-readonly-queue/main/pr-3-example"), - ): - with self.subTest(field=field): - payload = self.main_advance_payload() - payload["runs_by_sha"]["c" * 40][0][field] = value - result = self.invoke_main_advance(payload) - self.assertEqual([item.get("state") for item in result], ["pending", "pending"]) - - def test_main_advance_leaves_newer_running_evaluation_to_publish(self) -> None: - payload = self.main_advance_payload() - runs = payload["runs_by_sha"]["c" * 40] - runs.append({**runs[0], "id": 15, "status": "in_progress"}) - result = self.invoke_main_advance(payload) - self.assertEqual([item.get("state") for item in result], ["pending", "pending"]) - - def test_main_advance_with_empty_queue_has_no_writes(self) -> None: - payload = self.main_advance_payload() - payload["queue_nodes"] = [] - payload["queue_absent"] = True - self.assertEqual(self.invoke_main_advance(payload), []) - - def test_signal_is_unprivileged_and_writers_serialize(self) -> None: - producer = (WORKFLOW.parent / "castiron-custom-code.yml").read_text() - self.assertIn("pull_request_review:\n types: [submitted, edited, dismissed]", producer) - signal = producer.split(" queue-signal:\n", 1)[1].split(" candidate-tests:\n", 1)[0] - self.assertIn("permissions: {}", signal) - self.assertIn("push:\n branches: [main]", producer) - main_change = ( - WORKFLOW.read_text().split(" main-change:\n", 1)[1].split(" compute:\n", 1)[0] - ) - self.assertNotIn("checkout@", main_change) - self.assertNotIn("download-artifact", main_change) - self.assertNotIn("checkout@", signal) - handler = WORKFLOW.read_text() - revoke = handler.split(" review-change:\n", 1)[1].split(" compute:\n", 1)[0] - self.assertNotIn("checkout@", revoke) - self.assertNotIn("ref: ${{ github.event", revoke) - self.assertIn("path.join(root, 'custom_code_policy.py'), 'review-required'", revoke) - self.assertNotIn("download-artifact", revoke) - self.assertEqual( - handler.count("group: castiron-budget-writes-${{ github.repository_id }}"), 3 - ) - - def test_direct_head_review_discovery_rejects_shared_commit_from_other_sources( - self, - ) -> None: - for field, value in ( - ("id", 99), - ("full_name", "other/example"), - ("ref", "other-branch"), - ): - with self.subTest(field=field): - payload = self.payload() - payload["run"]["pull_requests"] = [] - head = source_head("a" * 40) - if field == "ref": - head[field] = value - else: - head["repo"][field] = value - other = {**payload["current"], "number": 4, "head": head} - payload["pulls_by_number"] = {"4": other} - for discovery in ("associated_pulls", "fallback_pulls"): - payload["associated_pulls"] = [] - payload[discovery] = [{"number": 4}, {"number": 3}] - result = self.invoke(payload) - self.assertEqual(len(result), 6) - self.assertEqual( - [item["run_id"] for item in result if "run_id" in item], [12, 14] - ) - - def test_foreign_only_associations_retry_revocation_and_reporter_discovery(self) -> None: - for explicit in (False, True): - with self.subTest(explicit=explicit): - payload = self.payload() - payload["run"]["pull_requests"] = [{"number": 4}] if explicit else [] - payload["runs_by_sha"]["a" * 40][0]["pull_requests"] = [] - payload["associated_pulls"] = [{"number": 4}] - payload["fallback_pulls"] = [{"number": 3}] - own = payload["current"] - foreign: dict[str, Any] = { - **own, - "number": 4, - "head": {**own["head"], "ref": "other"}, - } - payload["pulls_by_number"] = {"4": foreign} - payload["review_nodes"] = [{**self.approval(), "state": "DISMISSED"}] - writes = self.invoke(payload) - self.assertEqual( - [w["state"] for w in writes if "state" in w], - [] if explicit else ["pending"] * 4, - ) - self.assertEqual( - [w["run_id"] for w in writes if "run_id" in w], [] if explicit else [12, 14] - ) - # A fallback reached through unrelated commit associations is still branch-only: - # an old synthetic merge must not gain the association-only exemption. - payload["run"]["pull_requests"] = [] - payload["run"]["head_sha"] = payload["context"]["payload"]["workflow_run"]["head_sha"] = ( - "e" * 40 - ) - payload["merge_sha"] = "d" * 40 - payload["merge_commit"] = {"parents": [{"sha": "b" * 40}, {"sha": "a" * 40}]} - payload["pulls_by_number"]["4"]["head"]["sha"] = "f" * 40 - self.assertEqual(self.invoke(payload), []) - - def test_merge_review_filters_foreign_sources_before_branch_fallback(self) -> None: - for field, value in (("ref", "other"), ("id", 99), ("full_name", "other/example")): - for explicit in (False, True): - with self.subTest(field=field, explicit=explicit): - payload = self.payload() - payload["run"]["head_sha"] = "e" * 40 - payload["context"]["payload"]["workflow_run"]["head_sha"] = "e" * 40 - payload["run"]["pull_requests"] = [{"number": 4}] if explicit else [] - payload["associated_pulls"] = [{"number": 4}] - payload["fallback_pulls"] = [{"number": 3}] - payload["merge_sha"] = "e" * 40 - payload["merge_commit"] = {"parents": [{"sha": "b" * 40}, {"sha": "a" * 40}]} - head = source_head("a" * 40) - if field == "ref": - head[field] = value - else: - head["repo"][field] = value - payload["pulls_by_number"] = { - "4": {**payload["current"], "number": 4, "head": head} - } - writes = self.invoke(payload) - self.assertEqual( - [w["state"] for w in writes if "state" in w], - [] if explicit else ["pending"] * 4, - ) - self.assertEqual( - [w["run_id"] for w in writes if "run_id" in w], - [] if explicit else [12, 14], - ) - # Filtering a foreign association must not exempt fallback from freshness. - payload["merge_sha"] = "f" * 40 - self.assertEqual(self.invoke(payload), []) - - def test_review_requires_complete_source_identity(self) -> None: - for overrides in ( - {"head_repository": None}, - {"head_repository": {"id": 10}}, - {"head_repository": {"full_name": "contributor/example"}}, - {"head_branch": None}, - {"head_branch": "refs/pull/3/merge"}, - ): - for merge in (False, True): - with self.subTest(overrides=overrides, merge=merge): - payload = self.payload() - if merge: - payload["run"]["head_sha"] = "e" * 40 - payload["context"]["payload"]["workflow_run"]["head_sha"] = "e" * 40 - payload["merge_commit"] = { - "parents": [{"sha": "b" * 40}, {"sha": "a" * 40}] - } - payload["run"].update(overrides) - self.assertEqual(self.invoke(payload), []) - - def test_merge_sha_signal_requires_exact_live_merge_and_parents(self) -> None: - payload = self.payload() - payload["run"]["head_sha"] = payload["context"]["payload"]["workflow_run"]["head_sha"] = ( - "e" * 40 - ) - payload["merge_sha"] = "e" * 40 - payload["merge_commit"] = {"parents": [{"sha": "b" * 40}, {"sha": "a" * 40}]} - self.assertEqual(len(self.invoke(payload)), 6) - payload["merge_commit"]["parents"].reverse() - self.assertEqual(self.invoke(payload), []) - - def test_historical_merge_revocation_after_base_advance(self) -> None: - for discovery in ("run", "commit", "branch"): - for revoked in ({"state": "DISMISSED"}, {"body": ""}): - with self.subTest(discovery=discovery, revoked=revoked): - payload = self.payload() - payload["review_nodes"] = [{**self.approval(), **revoked}] - payload["run"]["head_sha"] = "e" * 40 - payload["context"]["payload"]["workflow_run"]["head_sha"] = "e" * 40 - payload["current"]["base"]["sha"] = "f" * 40 - payload["comparisons"] = { - f"{'f' * 40}...{'a' * 40}": {"merge_base_commit": {"sha": "b" * 40}} - } - payload["merge_sha"] = "d" * 40 - payload["merge_commit"] = {"parents": [{"sha": "b" * 40}, {"sha": "a" * 40}]} - if discovery != "run": - payload["run"]["pull_requests"] = [] - payload["associated_pulls"] = ( - [{"number": 3}] if discovery == "commit" else [] - ) - payload["fallback_pulls"] = [{"number": 3}] - result = self.invoke(payload) - if discovery == "branch": - self.assertEqual(result, []) - else: - self.assertEqual( - [item.get("state") for item in result[:4]], ["pending"] * 4 - ) - self.assertEqual({item["sha"] for item in result[:4]}, {"a" * 40, "c" * 40}) - self.assertEqual([item["run_id"] for item in result[4:]], [12, 14]) - - def test_rerun_ignores_same_sha_main_from_another_source(self) -> None: - for changes in ( - {"ref": "other"}, - {"repo": {**source_head("a" * 40)["repo"], "id": 11}}, - {"repo": {**source_head("a" * 40)["repo"], "full_name": "other/repo"}}, - {"repo": None}, - ): - with self.subTest(changes=changes): - payload = self.payload() - reporter = payload["runs_by_sha"]["a" * 40][0] - reporter["pull_requests"] = [] - current = payload["current"] - main = { - **current, - "number": 4, - "head": {**current["head"], **changes}, - "base": {**current["base"], "ref": "main"}, - } - payload["pulls_by_number"] = {"4": main} - payload["associated_pulls"] = [{"number": 3}, {"number": 4}] - result = self.invoke(payload) - self.assertEqual([item["run_id"] for item in result if "run_id" in item], [12, 14]) - - def test_rerun_rejects_missing_reporter_source_identity(self) -> None: - for changes in ( - {"head_repository": None}, - {"head_repository": {"full_name": "contributor/example"}}, - {"head_repository": {"id": 10}}, - {"head_branch": ""}, - ): - with self.subTest(changes=changes): - payload = self.payload() - payload["runs_by_sha"]["a" * 40][0].update(changes) - result = self.invoke(payload) - self.assertEqual([item["run_id"] for item in result if "run_id" in item], [14]) - self.assertTrue(all(item["state"] == "pending" for item in result[:4])) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/castiron/test_custom_code_status_publication.py b/scripts/castiron/test_custom_code_status_publication.py deleted file mode 100644 index 0aca40221..000000000 --- a/scripts/castiron/test_custom_code_status_publication.py +++ /dev/null @@ -1,782 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -"""Execute the trusted workflow against offline GitHub state.""" - -from __future__ import annotations - -import json -import shutil -import unittest -from typing import Any - -import custom_code_policy as policy -from custom_code_test_support import run_publisher, source_head, source_run - - -@unittest.skipUnless(shutil.which("node"), "Node is needed to execute the status-publisher fixture") -class StatusPublisherTests(unittest.TestCase): - def publish( - self, - *, - event_name: str = "pull_request", - head_changed: bool = False, - base_changed: bool = False, - no_result: bool = False, - failed_budget: bool = False, - fallback_pulls: list[dict[str, int]] | None = None, - associated_pulls: list[dict[str, int]] | None = None, - run_overrides: dict[str, Any] | None = None, - previous_statuses: list[dict[str, Any]] | None = None, - branch: str = "main", - evaluated_branch: str | None = None, - policy_changed: bool = False, - authenticated_main_sha: str | None = None, - bootstrap: str = "false", - main_snapshot: str = "", - policy_sha: str | None = None, - selected_policy_matches: bool = True, - queue_absent: bool = False, - queue_pages: list[dict[str, Any]] | None = None, - retarget_during_publication: bool = False, - source_changes_during_publication: dict[str, Any] | None = None, - same_head_next_pr: bool = False, - policy_reviews: list[dict[str, Any]] | None = None, - reviews: list[dict[str, Any]] | None = None, - review_nodes: list[dict[str, Any]] | None = None, - review_pages: list[dict[str, Any]] | None = None, - policy_pull: dict[str, Any] | None = None, - missing_review_contract: bool = False, - review_contract: dict[str, Any] | None = None, - ) -> list[dict[str, Any]]: - base, head = "a" * 40, "b" * 40 - run = source_run(head, event_name) - if event_name == "merge_group": - run["head_branch"] = f"gh-readonly-queue/{branch}/pr-7-example" - payload: dict[str, Any] = { - "reviews": reviews or [], - "review_nodes": review_nodes - if review_nodes is not None - else [ - { - "fullDatabaseId": str(review["id"]), - "state": review["state"], - "body": review.get("body"), - "commit": {"oid": review.get("commit_id")}, - "author": { - "login": review["user"]["login"], - "__typename": review["user"]["type"], - }, - "authorAssociation": review.get("author_association"), - } - for review in reviews or [] - ], - "review_pages": review_pages, - "main_sha": "d" * 40 - if policy_changed - else (policy_sha or ("c" * 40 if branch == "main" and base_changed else base)), - "queue_absent": queue_absent, - "queue_pages": queue_pages, - "fallback_pulls": fallback_pulls, - "associated_pulls": associated_pulls, - "previous_statuses": previous_statuses or [], - "context": { - "eventName": "workflow_run", - "repo": {"owner": "openai", "repo": "example"}, - "serverUrl": "https://github.com", - "runId": 123, - "payload": { - "workflow_run": run, - }, - }, - "run": {**run, **(run_overrides or {})}, - "current": { - "number": 3, - "user": {"login": "author"}, - "state": "open", - "head": source_head("c" * 40 if head_changed else head), - "base": { - "sha": "c" * 40 if base_changed else base, - "ref": branch, - "repo": {"full_name": "openai/example"}, - }, - }, - "env": { - "POLICY_REVIEWS": json.dumps(policy_reviews or []), - "HUMAN_REVIEW_RULES": "" - if missing_review_contract - else json.dumps( - policy.HUMAN_REVIEW_RULES if review_contract is None else review_contract - ), - "SELECTED_BRANCH": branch, - "SELECTED_BOOTSTRAP": bootstrap, - "MAIN_SNAPSHOT": main_snapshot, - "SELECTED_BASE_SHA": base, - "SELECTED_HEAD_SHA": head, - "SELECTED_POLICY_SHA": (policy_sha or base) - if selected_policy_matches - else "f" * 40, - "TARGET_BRANCH": "" if no_result else (evaluated_branch or branch), - "POLICY_SHA": "" if no_result else (policy_sha or base), - "BASE_SHA": "" if no_result else base, - "HEAD_SHA": "" if no_result else head, - "ISOLATION_RESULT": "success", - "BUDGET_RESULT": "failure" if failed_budget else "success", - "PUBLISH_ATTEMPT": "1", - }, - } - payload["env"]["AUTHENTICATED_MAIN_SHA"] = ( - payload["main_sha"] if authenticated_main_sha is None else authenticated_main_sha - ) - if policy_pull: - payload["pulls_by_number"] = {policy_pull["number"]: policy_pull} - if same_head_next_pr: - current = payload["current"] - payload["pulls_by_number"] = { - 3: current, - 4: {**current, "number": 4, "base": {**current["base"], "ref": "next"}}, - } - if retarget_during_publication: - current = payload["current"] - payload["current_sequence"] = [ - current, - {**current, "base": {**current["base"], "ref": "other"}}, - ] - if source_changes_during_publication is not None: - current = payload["current"] - payload["current_sequence"] = [ - current, - {**current, "head": {**current["head"], **source_changes_during_publication}}, - ] - return run_publisher("Publish exact-head statuses after checking freshness", payload) - - def test_live_human_review_predicate_matches_selector(self) -> None: - head = "b" * 40 - approved = { - "id": 10, - "state": "APPROVED", - "commit_id": head, - "user": {"login": "human", "type": "User"}, - "author_association": "MEMBER", - "body": policy.HUMAN_REVIEW_RULES["attestation_prefix"] + head, - } - cases = [ - [approved], - [], - [{**approved, "state": "DISMISSED"}], - [approved, {**approved, "id": 11, "state": "CHANGES_REQUESTED"}], - [approved, {**approved, "id": 11, "state": "COMMENTED"}], - [{**approved, "commit_id": "f" * 40}], - [{**approved, "user": {"login": "author", "type": "User"}}], - [{**approved, "user": {"login": "bot", "type": "Bot"}}], - [{**approved, "author_association": "NONE"}], - ] - for body in [ - "", - "Looks good to me", - policy.HUMAN_REVIEW_RULES["attestation_prefix"] + "f" * 40, - "> " + policy.HUMAN_REVIEW_RULES["attestation_prefix"] + head, - "", - "Automated source review.", - "Source-only review; I did not execute candidate code or workflows.", - "This does not substitute for human approval.", - "This is not a human approving review.", - ]: - cases.append([{**approved, "body": body}]) - for reviews in cases: - with self.subTest(reviews=reviews): - expected = ( - "success" if policy.has_human_approval(reviews, "author", head) else "failure" - ) - writes = self.publish( - branch="next", - policy_reviews=[{"number": 3, "head_sha": head}], - reviews=reviews, - ) - self.assertEqual([w["state"] for w in writes], [expected, expected]) - - def test_final_publisher_preserves_approval_across_comments(self) -> None: - head = "b" * 40 - approval = { - "id": 10, - "state": "APPROVED", - "commit_id": head, - "user": {"login": "human", "type": "User"}, - "author_association": "MEMBER", - "body": policy.HUMAN_REVIEW_RULES["attestation_prefix"] + head, - } - comment = {**approval, "id": 11, "state": "COMMENTED", "body": "A follow-up comment"} - cases = [ - ([approval, comment], "success"), - ([comment, approval], "success"), - ([comment], "failure"), - ([{**approval, "state": "DISMISSED"}, comment], "failure"), - ([{**approval, "body": "Attestation removed"}, comment], "failure"), - ( - [ - approval, - {**approval, "id": 12, "state": "CHANGES_REQUESTED"}, - {**comment, "id": 13}, - ], - "failure", - ), - ( - [ - {**approval, "state": "CHANGES_REQUESTED"}, - {**approval, "id": 12}, - {**comment, "id": 13}, - ], - "success", - ), - ] - for reviews, expected in cases: - with self.subTest(reviews=reviews): - writes = self.publish( - branch="next", - policy_reviews=[{"number": 3, "head_sha": head}], - reviews=reviews, - ) - self.assertEqual([w["state"] for w in writes], [expected] * 2) - - def test_final_publisher_preserves_review_identity_precision(self) -> None: - head = "b" * 40 - approval = { - "id": 9007199254740992, - "state": "APPROVED", - "commit_id": head, - "user": {"login": "human", "type": "User"}, - "author_association": "MEMBER", - "body": policy.HUMAN_REVIEW_RULES["attestation_prefix"] + head, - } - replacement = {**approval, "id": 9007199254740993, "state": "CHANGES_REQUESTED"} - for reviews in ([approval, replacement], [replacement, approval]): - with self.subTest(reviews=reviews): - writes = self.publish( - branch="next", - policy_reviews=[{"number": 3, "head_sha": head}], - reviews=reviews, - ) - self.assertEqual([w["state"] for w in writes], ["failure"] * 2) - writes = self.publish( - branch="next", - policy_reviews=[{"number": 3, "head_sha": head}], - reviews=[{**replacement, "state": "APPROVED"}, {**approval, "state": "DISMISSED"}], - ) - self.assertEqual([w["state"] for w in writes], ["success"] * 2) - - def test_final_publisher_review_pages_and_malformed_identity(self) -> None: - head = "b" * 40 - approval = { - "fullDatabaseId": "9007199254740992", - "state": "APPROVED", - "commit": {"oid": head}, - "author": {"login": "human", "__typename": "User"}, - "authorAssociation": "MEMBER", - "body": policy.HUMAN_REVIEW_RULES["attestation_prefix"] + head, - } - - def publish(**kwargs: Any) -> list[str]: - return [ - w["state"] - for w in self.publish( - branch="next", - policy_reviews=[{"number": 3, "head_sha": head}], - **kwargs, - ) - ] - - self.assertEqual(publish(review_nodes=[approval]), ["success"] * 2) - for identity in (None, "", "1.5", "-1", "0", 9007199254740992): - with self.subTest(identity=identity): - self.assertEqual( - publish(review_nodes=[{**approval, "fullDatabaseId": identity}]), - ["failure"] * 2, - ) - pages: list[dict[str, Any]] = [ - { - "expected_cursor": None, - "nodes": [approval], - "pageInfo": {"hasNextPage": True, "endCursor": "page2"}, - }, - { - "expected_cursor": "page2", - "nodes": [{**approval, "fullDatabaseId": "9007199254740993", "state": "DISMISSED"}], - "pageInfo": {"hasNextPage": False, "endCursor": None}, - }, - ] - self.assertEqual(publish(review_pages=pages), ["failure"] * 2) - self.assertEqual( - publish( - review_pages=[{**pages[0], "pageInfo": {"hasNextPage": True, "endCursor": None}}] - ), - ["failure"] * 2, - ) - - def test_next_publication_binds_authenticated_main(self) -> None: - # The reviewed checker pin may remain old. The evaluation's main snapshot - # must still be current so an old publisher cannot undo invalidation. - for event in ("pull_request", "merge_group"): - for authenticated_main, expected in ( - ("a" * 40, "failure"), - ("d" * 40, "success"), - ("", "failure"), - ("invalid", "failure"), - ): - with self.subTest(event=event, authenticated_main=authenticated_main): - writes = self.publish( - event_name=event, - branch="next", - policy_changed=True, - authenticated_main_sha=authenticated_main, - ) - self.assertEqual([item["state"] for item in writes], [expected] * 2) - - def test_main_snapshot_publication_rechecks_live_main(self) -> None: - for event in ("pull_request", "merge_group"): - for moved in (False, True): - with self.subTest(event=event, moved=moved): - writes = self.publish( - event_name=event, - branch="next", - policy_sha="e" * 40, - main_snapshot="e" * 40, - policy_changed=moved, - ) - self.assertEqual( - [w["state"] for w in writes], - ["failure", "failure"] if moved else ["success", "success"], - ) - - def test_bootstrap_rechecks_authenticated_main(self) -> None: - for event in ("pull_request", "merge_group"): - for bootstrap, changed, expected in ( - ("true", False, "success"), - ("true", True, "failure"), - ("false", True, "success"), - ("", False, "failure"), - ("invalid", False, "failure"), - ): - with self.subTest(event=event, bootstrap=bootstrap, changed=changed): - writes = self.publish( - event_name=event, - branch="next", - bootstrap=bootstrap, - policy_changed=changed, - ) - self.assertEqual([item["state"] for item in writes], [expected] * 2) - - def test_freshness_is_rechecked_after_review_pagination(self) -> None: - head = "b" * 40 - approval = { - "fullDatabaseId": "1", - "state": "APPROVED", - "body": policy.HUMAN_REVIEW_RULES["attestation_prefix"] + head, - "author": {"login": "human", "__typename": "User"}, - "authorAssociation": "MEMBER", - "commit": {"oid": head}, - } - current: dict[str, Any] = { - "number": 3, - "user": {"login": "author"}, - "state": "open", - "head": source_head(head), - "base": {"ref": "next", "sha": "a" * 40, "repo": {"full_name": "openai/example"}}, - } - advanced = {**current, "base": {**current["base"], "sha": "c" * 40}} - changed_source = {**current, "head": {**current["head"], "ref": "other-source"}} - for event, mutation, expected in ( - ("pull_request", {}, ["success"] * 2), - ("merge_group", {}, ["success"] * 2), - ("pull_request", {"main_sha": "d" * 40}, ["failure"] * 2), - ("merge_group", {"main_sha": "d" * 40}, ["failure"] * 2), - ("pull_request", {"current": advanced}, ["failure"] * 2), - ("merge_group", {"current": advanced}, ["failure"] * 2), - ("merge_group", {"queue_absent": True}, ["failure"] * 2), - ("pull_request", {"current": changed_source}, []), - ): - with self.subTest(event=event, mutation=mutation): - result = self.publish( - event_name=event, - branch="next", - bootstrap="true", - policy_reviews=[{"number": 3, "head_sha": head}], - review_pages=[ - { - "expected_cursor": None, - "nodes": [], - "pageInfo": {"hasNextPage": True, "endCursor": "last"}, - }, - { - "expected_cursor": "last", - "nodes": [approval], - "pageInfo": {"hasNextPage": False, "endCursor": None}, - "after_read": mutation, - }, - ], - ) - self.assertEqual([item["state"] for item in result], expected) - - def test_queue_policy_approval_dismissal_after_selection_fails(self) -> None: - candidate = "e" * 40 - for queue_head in (source_head(candidate), {"sha": candidate}): - with self.subTest(queue_head=queue_head): - pull: dict[str, Any] = { - "number": 7, - "user": {"login": "author"}, - "state": "open", - "head": queue_head, - "base": {"ref": "next", "repo": {"full_name": "openai/example"}}, - } - approved = { - "id": 1, - "state": "APPROVED", - "commit_id": candidate, - "user": {"login": "human", "type": "User"}, - "author_association": "MEMBER", - "body": policy.HUMAN_REVIEW_RULES["attestation_prefix"] + candidate, - } - for state in ("APPROVED", "DISMISSED", "CHANGES_REQUESTED"): - with self.subTest(state=state): - writes = self.publish( - event_name="merge_group", - branch="next", - policy_pull=pull, - policy_reviews=[{"number": 7, "head_sha": candidate}], - reviews=[{**approved, "state": state}], - ) - self.assertEqual( - [w["state"] for w in writes], - ["success" if state == "APPROVED" else "failure"] * 2, - ) - # Ordinary automated approvals may keep the queue live, but cannot replace - # the independently required human policy approval. - writes = self.publish( - event_name="merge_group", - branch="next", - policy_pull=pull, - policy_reviews=[{"number": 7, "head_sha": candidate}], - reviews=[ - {**approved, "state": "DISMISSED"}, - { - **approved, - "id": 2, - "user": {"login": "automated", "type": "User"}, - "body": "", - }, - ], - ) - self.assertEqual([w["state"] for w in writes], ["failure", "failure"]) - - def test_queue_policy_constituent_changes_fail_closed(self) -> None: - candidate = "e" * 40 - for queue_head in (source_head(candidate), {"sha": candidate}): - with self.subTest(queue_head=queue_head): - pull: dict[str, Any] = { - "number": 7, - "user": {"login": "author"}, - "state": "open", - "head": queue_head, - "base": {"ref": "next", "repo": {"full_name": "openai/example"}}, - } - approved = { - "id": 1, - "state": "APPROVED", - "commit_id": candidate, - "user": {"login": "human", "type": "User"}, - "author_association": "MEMBER", - "body": policy.HUMAN_REVIEW_RULES["attestation_prefix"] + candidate, - } - for changed in [ - {**pull, "state": "closed"}, - {**pull, "head": {"sha": "f" * 40}}, - {**pull, "base": {**pull["base"], "ref": "main"}}, - ]: - with self.subTest(changed=changed): - writes = self.publish( - event_name="merge_group", - branch="next", - policy_pull=changed, - policy_reviews=[{"number": 7, "head_sha": candidate}], - reviews=[approved], - ) - self.assertEqual([w["state"] for w in writes], ["failure", "failure"]) - - def test_missing_policy_review_contract_fails_closed(self) -> None: - self.assertEqual( - [w["state"] for w in self.publish(branch="next", missing_review_contract=True)], - ["failure", "failure"], - ) - - def test_legacy_review_contract_fails_closed_only_on_next(self) -> None: - legacy = { - key: value - for key, value in policy.HUMAN_REVIEW_RULES.items() - if key != "attestation_prefix" - } - legacy["version"] = 1 - for branch, expected in (("next", "failure"), ("main", "success")): - with self.subTest(branch=branch): - self.assertEqual( - [w["state"] for w in self.publish(branch=branch, review_contract=legacy)], - [expected, expected], - ) - - def test_statuses_recheck_source_identity_immediately_before_writing(self) -> None: - source = source_head("b" * 40) - self.assertEqual(len(self.publish(source_changes_during_publication={})), 2) - for changes in ( - {"ref": "other"}, - {"repo": {**source["repo"], "id": 11}}, - {"repo": {**source["repo"], "full_name": "other/example"}}, - {"repo": None}, - ): - with self.subTest(changes=changes): - self.assertEqual(self.publish(source_changes_during_publication=changes), []) - - def test_statuses_reject_a_run_from_a_different_source(self) -> None: - for source in ( - {"head_branch": "other"}, - {"head_repository": {"id": 11, "full_name": "other/repo"}}, - {"head_repository": None}, - ): - with self.subTest(source=source): - self.assertEqual(self.publish(run_overrides=source), []) - - def test_statuses_attach_to_candidate_not_main(self) -> None: - for event in ("pull_request", "merge_group"): - with self.subTest(event=event): - results = self.publish(event_name=event) - self.assertEqual(len(results), 2) - self.assertTrue( - all(r["sha"] == "b" * 40 and r["state"] == "success" for r in results) - ) - - def test_next_cannot_overwrite_main_results(self) -> None: - for main_event in ("pull_request", "merge_group"): - main = self.publish(event_name=main_event, failed_budget=True) - self.assertEqual([r["state"] for r in main], ["success", "failure"]) - for event in ("pull_request", "merge_group"): - with self.subTest(main_event=main_event, event=event): - # Both evaluations use the same commit. A next success - # must not replace the main target's required failure. - result = self.publish(event_name=event, branch="next", previous_statuses=main) - self.assertEqual([r["state"] for r in result], ["success", "success"]) - self.assertTrue( - {r["context"] for r in result}.isdisjoint(r["context"] for r in main) - ) - - def test_foreign_only_associations_retry_source_branch(self) -> None: - foreign = { - "number": 4, - "state": "open", - "head": {**source_head("b" * 40), "ref": "other"}, - "base": {"ref": "main", "repo": {"full_name": "openai/example"}}, - } - for explicit in (False, True): - with self.subTest(explicit=explicit): - writes = self.publish( - run_overrides={"pull_requests": [{"number": 4}] if explicit else []}, - associated_pulls=[{"number": 4}], - fallback_pulls=[{"number": 3}], - policy_pull=foreign, - ) - self.assertEqual( - [w["state"] for w in writes], [] if explicit else ["success", "success"] - ) - - def test_same_head_next_pr_does_not_block_required_main_statuses(self) -> None: - for numbers in ([3, 4], [4, 3]): - with self.subTest(numbers=numbers): - results = self.publish( - run_overrides={"pull_requests": []}, - fallback_pulls=[{"number": number} for number in numbers], - same_head_next_pr=True, - failed_budget=True, - ) - self.assertEqual([r["state"] for r in results], ["success", "failure"]) - self.assertTrue(all(r["sha"] == "b" * 40 for r in results)) - next_results = self.publish( - branch="next", - run_overrides={"pull_requests": [{"number": 4}]}, - same_head_next_pr=True, - ) - self.assertEqual([r["state"] for r in next_results], ["success", "success"]) - self.assertTrue(all(r["context"].startswith("Castiron next /") for r in next_results)) - - def test_next_queue_approved_pin_survives_main_advancing(self) -> None: - for changed in (False, True): - with self.subTest(policy_changed=changed): - results = self.publish( - event_name="merge_group", branch="next", policy_changed=changed - ) - self.assertEqual([r["state"] for r in results], ["success", "success"]) - - def test_next_pin_receipt_and_target_freshness(self) -> None: - for event in ("pull_request", "merge_group"): - cases: tuple[dict[str, Any], ...] = ( - {"selected_policy_matches": False}, - {"no_result": True}, - {"base_changed": True}, - ) - for kwargs in cases: - with self.subTest(event=event, kwargs=kwargs): - results = self.publish(event_name=event, branch="next", **kwargs) - self.assertEqual([r["state"] for r in results], ["failure", "failure"]) - results = self.publish(event_name=event, branch="next", policy_sha="e" * 40) - self.assertEqual([r["state"] for r in results], ["success", "success"]) - results = self.publish(event_name="merge_group", branch="next", queue_absent=True) - self.assertEqual([r["state"] for r in results], ["failure", "failure"]) - - def test_main_statuses_bind_target_policy_and_queue_membership(self) -> None: - for event in ("pull_request", "merge_group"): - with self.subTest(event=event): - stale = self.publish(event_name=event, evaluated_branch="next") - self.assertEqual([r["state"] for r in stale], ["failure", "failure"]) - for policy_changed, queue_absent in ((True, False), (False, True)): - with self.subTest(policy_changed=policy_changed, queue_absent=queue_absent): - result = self.publish( - event_name="merge_group", - policy_changed=policy_changed, - queue_absent=queue_absent, - ) - self.assertEqual([r["state"] for r in result], ["failure", "failure"]) - self.assertEqual(self.publish(retarget_during_publication=True), []) - - def test_next_fork_and_unsupported_targets(self) -> None: - result = self.publish( - branch="next", run_overrides={"pull_requests": []}, fallback_pulls=[{"number": 3}] - ) - self.assertEqual([r["state"] for r in result], ["success", "success"]) - for event in ("pull_request", "merge_group"): - for branch in ("Next", "Main", "release", "next-other"): - with self.subTest(event=event, branch=branch): - self.assertEqual(self.publish(event_name=event, branch=branch), []) - - def test_main_queue_membership_paginates(self) -> None: - pages: list[dict[str, Any]] = [ - { - "expected_cursor": None, - "nodes": [], - "pageInfo": {"hasNextPage": True, "endCursor": "page2"}, - }, - { - "expected_cursor": "page2", - "nodes": [{"headCommit": {"oid": "b" * 40}}], - "pageInfo": {"hasNextPage": False, "endCursor": None}, - }, - ] - result = self.publish(event_name="merge_group", queue_pages=pages) - self.assertEqual([r["state"] for r in result], ["success", "success"]) - - def test_fork_statuses_with_no_commit_association(self) -> None: - options: dict[str, Any] = { - "run_overrides": {"pull_requests": []}, - "fallback_pulls": [{"number": 3}], - } - results = self.publish(**options) - self.assertEqual(len(results), 2) - self.assertTrue(all(r["sha"] == "b" * 40 and r["state"] == "success" for r in results)) - self.assertEqual(self.publish(**options, head_changed=True), []) - self.assertTrue( - all(r["state"] == "failure" for r in self.publish(**options, no_result=True)) - ) - self.assertEqual(self.publish(run_overrides={"pull_requests": []}, fallback_pulls=[]), []) - self.assertEqual( - self.publish( - run_overrides={"pull_requests": []}, - fallback_pulls=[{"number": 3}, {"number": 4}], - ), - [], - ) - - def test_stale_pr_head_is_not_published(self) -> None: - self.assertEqual(self.publish(head_changed=True), []) - - def test_pr_snapshot_survives_main_advancing(self) -> None: - results = self.publish(base_changed=True) - self.assertEqual(len(results), 2) - self.assertTrue(all(r["state"] == "success" for r in results)) - self.assertTrue(all("a" * 12 in r["description"] for r in results)) - - def test_queue_still_rejects_main_advancing(self) -> None: - results = self.publish(event_name="merge_group", base_changed=True) - self.assertEqual(len(results), 2) - self.assertTrue(all(r["state"] == "failure" for r in results)) - - def test_missing_evaluation_publishes_actionable_failures(self) -> None: - for event in ("pull_request", "merge_group"): - results = self.publish(event_name=event, no_result=True, base_changed=True) - self.assertEqual(len(results), 2) - for result in results: - self.assertEqual(result["state"], "failure") - self.assertIn("inspect the trusted run", result["description"]) - self.assertTrue(result["target_url"].endswith("/actions/runs/123")) - - def test_main_rejects_mismatched_selector_receipt(self) -> None: - results = self.publish(selected_policy_matches=False) - self.assertTrue(results) - self.assertTrue(all(result["state"] != "success" for result in results)) - - def test_lookalike_workflow_cannot_publish(self) -> None: - self.assertEqual( - self.publish( - run_overrides={ - "path": ".github/workflows/castiron-custom-code.yml@evil.yml", - "workflow_id": 789, - } - ), - [], - ) - - def test_superseded_or_wrong_source_run_cannot_publish(self) -> None: - for overrides in ( - {"run_attempt": 2}, - {"head_sha": "c" * 40}, - {"event": "push"}, - {"path": "other.yml"}, - ): - with self.subTest(overrides=overrides): - self.assertEqual(self.publish(run_overrides=overrides), []) - - def test_independent_check_failures_are_preserved(self) -> None: - results = self.publish(failed_budget=True) - self.assertEqual([r["state"] for r in results], ["success", "failure"]) - - def test_older_evaluation_cannot_overwrite_newer_failure(self) -> None: - for order in ("124:1:125:1", "123:2:125:1", "123:1:125:1", "123:1:123:2"): - with self.subTest(order=order): - previous = { - "context": "Castiron / custom-code budget", - "creator": {"login": "github-actions[bot]"}, - "state": "failure", - "description": f"Failed against newer main. [evaluation {order}]", - "target_url": f"https://github.com/openai/example/actions/runs/{order.split(':')[2]}", - } - self.assertEqual(self.publish(previous_statuses=[previous]), []) - - def test_publication_guard_allows_current_retry_and_ignores_unrelated_statuses( - self, - ) -> None: - previous = { - "context": "Castiron / custom-code budget", - "creator": {"login": "github-actions[bot]"}, - "description": "Failed. [evaluation 123:1:123:1]", - "target_url": "https://github.com/openai/example/actions/runs/123", - } - for overrides in ( - {}, # A partial publication can retry the same evaluation. - { - "description": "Failed. [evaluation 122:9:999:9]", - "target_url": "https://github.com/openai/example/actions/runs/999", - }, - {"description": "Legacy status without evaluation marker"}, - {"context": "Other check", "description": "[evaluation 999:9:999:9]"}, - {"creator": {"login": "someone"}, "description": "[evaluation 999:9:999:9]"}, - { - "description": "[evaluation 999:9:999:9]", - "target_url": "https://example.com/999", - }, - ): - with self.subTest(overrides=overrides): - results = self.publish(previous_statuses=[{**previous, **overrides}]) - self.assertEqual(len(results), 2) - self.assertTrue(all(r["state"] == "success" for r in results)) - self.assertTrue(all(len(r["description"]) <= 140 for r in results)) diff --git a/scripts/castiron/test_custom_code_strict.py b/scripts/castiron/test_custom_code_strict.py deleted file mode 100644 index b258213e6..000000000 --- a/scripts/castiron/test_custom_code_strict.py +++ /dev/null @@ -1,774 +0,0 @@ -# File generated from our OpenAPI spec by Castiron. See CONTRIBUTING.md for details. -# Regression tests for strict generated-file ownership. -from __future__ import annotations - -import hashlib -import json -import subprocess -import sys -import unittest -from pathlib import Path -from typing import Any, cast -from unittest import mock - -import custom_code_budget as budget -import custom_code_report as report -from custom_code_test_support import GitTestCase - - -class StrictTests(GitTestCase): - workflow = ".github/workflows/castiron-custom-code.yml" - tooling_dir = "scripts/castiron" - checker_path = f"{tooling_dir}/custom_code_budget.py" - manifest_path = f"{tooling_dir}/generated-files.json" - alternate_manifest_path = ( - "scripts/castiron/generated-files.json" - if tooling_dir == "generated/castiron" - else "generated/castiron/generated-files.json" - ) - - def setUp(self) -> None: # pyright: ignore[reportImplicitOverride] - super().setUp() - self.write("generated.py", "generated\n") - for path in budget.GENERATED_GITHUB_PATHS: - self.write(path.decode(), "workflow\n") - self.inventory(budget.GENERATED_GITHUB_PATHS) - self.manifest() - self.generated, _ = self.baseline() - self.write( - budget.POLICY, - json.dumps( - {"schema_version": 2, "generated_files": "strict", "max_custom_patch_lines": 0} - ), - ) - self.base = self.commit("enable strict ownership") - - def inventory(self, paths: frozenset[bytes]) -> None: - # Checkpoint code is data. A verifier must never execute this statement. - self.write( - self.checker_path, - "raise AssertionError('checkpoint code must not execute')\n" - f"GENERATED_GITHUB_PATHS = frozenset({set(paths)!r})\n", - ) - - def manifest(self, extra: tuple[str, ...] = ()) -> None: - files = { - p: { - "mode": "100644", - "sha256": hashlib.sha256((self.repo / p).read_bytes()).hexdigest(), - } - for p in ( - "generated.py", - *(path.decode() for path in budget.GENERATED_GITHUB_PATHS), - self.checker_path, - *extra, - ) - } - self.write(self.manifest_path, json.dumps({"schema_version": 1, "files": files})) - - def evaluate(self) -> dict[str, Any]: - return budget.evaluate(self.repo, self.base, self.commit(), public=False)[0] - - def test_handwritten_change_passes(self) -> None: - self.write("handwritten.py", "custom behavior\n") - self.assertEqual(self.evaluate()["checks"]["budget"]["state"], "success") - - def test_strict_main_checks_migration_target_and_candidate(self) -> None: - self.git("checkout", "-q", "-b", "next", self.base) - self.write(budget.POLICY, '{"schema_version":1,"max_custom_patch_lines":1000}') - target = self.commit("next still uses migration policy") - self.write(self.workflow, "manual workflow edit\n") - head = self.commit() - for pull_heads in (None, [head]): - with self.subTest(pull_heads=pull_heads): - result, _ = budget.evaluate( - self.repo, - target, - head, - public=False, - policy_sha=self.base, - pull_heads=pull_heads, - ) - self.assertEqual(result["total"], 0) - self.assertEqual(result["checks"]["isolation"]["state"], "success") - self.assertEqual(result["checks"]["budget"]["state"], "failure") - self.assertIn(self.workflow, result["strict_violations"][0]) - - def test_exact_main_snapshot_bootstrap_verifies_real_checkpoints(self) -> None: - self.assert_exact_main_snapshot_bootstrap() - - def test_exact_main_snapshot_bootstrap_preserves_go_ratchet(self) -> None: - self.assert_exact_main_snapshot_bootstrap(ratchet="go") - - def test_exact_main_snapshot_bootstrap_accepts_ssh_origin(self) -> None: - self.assert_exact_main_snapshot_bootstrap(origin="git@github.com:openai/example.git") - - def test_next_preparation_rejects_wrong_origin_before_network(self) -> None: - self.git("remote", "add", "origin", "https://github.com/attacker/example.git") - for origin in ( - "https://github.com/attacker/example.git", - "git@github.com:attacker/example.git", - "https://github.com/openai/example-internal.git", - "https://github.com.example.org/openai/example.git", - ): - with self.subTest(origin=origin), mock.patch.object(budget, "read_public_api") as api: - self.git("remote", "set-url", "origin", origin) - with self.assertRaisesRegex(ValueError, "public repository as origin"): - budget.prepare_next_baseline(self.repo, "openai/example", "codex/prepare") - api.assert_not_called() - - def assert_exact_main_snapshot_bootstrap( - self, ratchet: str | None = None, *, origin: str | None = None - ) -> None: - repository = "openai/example" - self.git("remote", "add", "origin", origin or f"https://github.com/{repository}.git") - - def public_snapshot(generated: str, parent: str | None) -> str: - stats = report.read_stats(self.repo, self.git("rev-parse", "HEAD")) - return report.create_public_snapshot( - self.repo, - self.git("rev-parse", f"{generated}^{{tree}}"), - stats["generation_id"], - report.hash_codegen_commit(self.repo, generated), - "codegen/test", - parent, - ) - - old_public = public_snapshot(self.generated, None) - stats_path = self.repo / ".castiron.stats.yml" - stats_path.write_text(stats_path.read_text() + f"public_codegen_sha: {old_public}\n") - self.write(budget.POLICY, '{"schema_version":1,"max_custom_patch_lines":4000}') - self.write("generated.py", "legacy custom edit\n") - legacy = self.commit("legacy unpinned next") - self.write("generated.py", "generated\n") - checker = self.repo / self.checker_path - checker.write_text(checker.read_text() + "NEXT_POLICY_VERSION = 1\n") - self.manifest() - # Generated checkpoints exclude integration metadata and repository policy. - stats_path.unlink() - (self.repo / budget.POLICY).unlink() - generated, _ = self.baseline() - current_public = public_snapshot(generated, old_public) - self.git("update-ref", "refs/remotes/origin/codegen/test", current_public) - stats_path.write_text(stats_path.read_text() + f"public_codegen_sha: {current_public}\n") - strict = {"schema_version": 2, "max_custom_patch_lines": 0, "generated_files": "strict"} - if ratchet is not None: - strict["ratchet"] = ratchet - self.write(budget.POLICY, json.dumps(strict)) - main = self.commit("verified strict main") - self.git("update-ref", "refs/remotes/origin/main", main) - self.git("update-ref", "refs/remotes/origin/next", legacy) - real_git = budget.report.git - - def offline_git(repo: Path, *args: str, **kwargs: Any) -> bytes: - # Only network transport is mocked; refs, trees, metadata and verification are real Git. - if args[0] == "fetch": - for spec in args: - if spec.startswith("+refs/heads/"): - source, destination = spec[1:].split(":", 1) - revision = ( - real_git( - repo, - "rev-parse", - source.replace("refs/heads/", "refs/remotes/origin/", 1), - ) - .decode() - .strip() - ) - real_git(repo, "update-ref", destination, revision) - return b"" - return cast(bytes, real_git(repo, *args, **kwargs)) - - def public_api(path: str) -> dict[str, Any]: - if path == f"repos/{repository}": - return {"private": False, "default_branch": "main", "full_name": repository} - return {"object": {"sha": main if path.endswith("/main") else legacy}} - - original_index = self.git("write-tree") - with ( - mock.patch.object(budget.report, "git", side_effect=offline_git), - mock.patch.object(budget, "read_public_api", side_effect=public_api), - ): - prepared = budget.prepare_next_baseline(self.repo, repository, "codex/adopt-main") - candidate = prepared["head_sha"] - self.assertEqual( - budget.read_policy(self.repo, candidate), - {**strict, "next_policy": {"repository": repository, "main_sha": main}}, - ) - self.assertEqual(self.git("rev-parse", "refs/heads/codex/adopt-main"), candidate) - self.assertEqual(self.git("write-tree"), original_index) - self.assertEqual(self.git("rev-parse", "HEAD"), main) - self.assertEqual( - self.git("show", f"{candidate}:.castiron.stats.yml"), - self.git("show", f"{main}:.castiron.stats.yml"), - ) - refs_read = 0 - - def moving_api(path: str) -> dict[str, Any]: - nonlocal refs_read - if path.endswith("/main"): - refs_read += 1 - if refs_read > 1: - return {"object": {"sha": legacy}} - return public_api(path) - - with ( - mock.patch.object(budget.report, "git", side_effect=offline_git), - mock.patch.object(budget, "read_public_api", side_effect=moving_api), - self.assertRaisesRegex(ValueError, "refs changed"), - ): - budget.prepare_next_baseline(self.repo, repository, "codex/stale-main") - self.assertEqual(self.git("for-each-ref", "refs/heads/codex/stale-main"), "") - self.git("checkout", "--detach", "-q", candidate) - wrapper = self.git( - "commit-tree", - f"{candidate}^{{tree}}", - "-p", - legacy, - "-p", - candidate, - "-m", - "verified queue wrapper", - ) - for heads in (None, [candidate]): - with self.subTest(queue=heads is not None): - result, _ = budget.evaluate( - self.repo, - legacy, - candidate if heads is None else wrapper, - public=True, - target_branch="next", - policy_sha=main, - main_snapshot=main, - repository=repository, - pull_heads=heads, - ) - self.assertEqual(result["total"], 0) - self.assertTrue( - all(c["state"] == "success" for c in result["checks"].values()), result - ) - # Exercise the real hosted evaluator with independent, initially empty - # object stores. The selector's origin/main ref is not shared with them. - from test_custom_code_github import GitHubAPI - - remote = self.repo / "bootstrap-remote.git" - self.git("clone", "--bare", str(self.repo), str(remote)) - real_git(remote, "update-ref", "refs/heads/main", main) - real_git(remote, "update-ref", "refs/heads/codegen/test", current_public) - real_git(remote, "update-ref", "refs/heads/queue", wrapper) - named_fetches: list[Path] = [] - - def transport(repo: Path, *args: str, **kwargs: Any) -> bytes: - if args[0] == "fetch": - self.assertIn("origin", args) - if "+refs/heads/main:refs/remotes/origin/main" in args: - named_fetches.append(repo) - args = tuple(str(remote) if arg == "origin" else arg for arg in args) - self.assertNotIn("checkout", args) - return cast(bytes, real_git(repo, *args, **kwargs)) - - for mode in ("pr", "reused-pr", "queue", "moved-main"): - with self.subTest(object_store=mode): - queue = mode in ("queue", "moved-main") - revision = wrapper if queue else candidate - hosted_api = GitHubAPI(main, revision, "merge_group" if queue else "pull_request") - hosted_api.next = legacy - hosted_api.pull["base"]["ref"] = "next" - if queue: - hosted_api.run["head_branch"] = "gh-readonly-queue/next/pr-3-test" - objects = self.repo / f"bootstrap-{mode}.git" - out = self.repo / f"bootstrap-{mode}-report" - receipt = { - "schema_version": 1, - "target_branch": "next", - "base_sha": legacy, - "head_sha": revision, - "policy_sha": main, - "bootstrap": True, - "main_snapshot": main, - } - with ( - mock.patch.object(report, "git", side_effect=transport), - mock.patch.object(budget.report, "git", side_effect=transport), - mock.patch.object(report, "api", side_effect=hosted_api), - mock.patch.object(budget.report, "api", side_effect=hosted_api), - mock.patch.object( - budget, "queued_entries", return_value=[(candidate, wrapper)] - ), - ): - if mode == "reused-pr": - report.trusted_report(objects, repository, 123, 1, out, base=main) - self.assertEqual( - real_git(objects, "for-each-ref", "refs/remotes/origin/main"), b"" - ) - if mode == "moved-main": - # API receipt remains captured at main while Git's named - # ref changes before fetch: the evaluator must fail closed. - real_git(remote, "update-ref", "refs/heads/main", legacy) - with self.assertRaisesRegex(ValueError, "authenticated main changed"): - budget.github_evaluate( - objects, - repository, - { - "repository": {"full_name": repository}, - "workflow_run": hosted_api.run, - }, - main, - selection=receipt, - ) - else: - result, _ = budget.github_evaluate( - objects, - repository, - { - "repository": {"full_name": repository}, - "workflow_run": hosted_api.run, - }, - main, - out if mode == "reused-pr" else None, - selection=receipt, - ) - self.assertTrue( - all(c["state"] == "success" for c in result["checks"].values()), result - ) - self.assertEqual(result["total"], 0) - self.assertEqual( - real_git(objects, "rev-parse", "refs/remotes/origin/main") - .decode() - .strip(), - main, - ) - self.assertIn(objects, named_fetches) - self.assertFalse((objects / "scripts").exists()) - self.write("generated.py", "unauthorized custom edit\n") - tampered = self.commit("tampered candidate") - result, _ = budget.evaluate( - self.repo, - legacy, - tampered, - public=True, - target_branch="next", - policy_sha=main, - main_snapshot=main, - repository=repository, - ) - self.assertEqual(result["checks"]["isolation"]["state"], "failure") - self.assertEqual(result["checks"]["budget"]["state"], "failure") - - def test_activation_accepts_stale_pr_without_policy_edits(self) -> None: - strict = (self.repo / budget.POLICY).read_text() - self.git("checkout", "-q", "-b", "migration", f"{self.base}^") - self.write(budget.POLICY, '{"schema_version":1,"max_custom_patch_lines":1000}') - migration = self.commit("migration policy before activation") - self.git("checkout", "-q", "-b", "stale", migration) - self.write("handwritten.py", "custom behavior\n") - stale = self.commit("PR predates strict activation") - self.git("checkout", "-q", "migration") - self.write(budget.POLICY, strict) - activated = self.commit("activate strict policy") - result, _ = budget.evaluate(self.repo, activated, stale, public=False) - self.assertTrue(all(c["state"] == "success" for c in result["checks"].values())) - self.git("merge", "--no-ff", "-m", "queue stale PR", stale) - result, _ = budget.evaluate( - self.repo, - activated, - self.git("rev-parse", "HEAD"), - public=False, - pull_heads=[stale], - ) - self.assertTrue(all(c["state"] == "success" for c in result["checks"].values())) - - def test_stale_policy_edit_cannot_downgrade_activated_base(self) -> None: - strict = (self.repo / budget.POLICY).read_text() - self.git("checkout", "-q", "-b", "migration", f"{self.base}^") - self.write(budget.POLICY, '{"schema_version":1,"max_custom_patch_lines":10}') - migration = self.commit("migration policy before activation") - self.git("checkout", "-q", "-b", "stale", migration) - self.write(budget.POLICY, '{"schema_version":1,"max_custom_patch_lines":1000}') - stale = self.commit("stale policy increase") - self.git("checkout", "-q", "migration") - self.write(budget.POLICY, strict) - activated = self.commit("activate strict policy") - result, _ = budget.evaluate(self.repo, activated, stale, public=False) - self.assertEqual(result["checks"]["isolation"]["state"], "failure") - self.assertIn("cannot be downgraded", result["checks"]["isolation"]["description"]) - - def test_workflow_edit_fails_despite_zero_legacy_patch(self) -> None: - self.write(self.workflow, "edited workflow\n") - result = self.evaluate() - self.assertEqual(result["total"], 0) - self.assertEqual(result["checks"]["budget"]["state"], "failure") - self.assertIn(self.workflow, result["strict_violations"][0]) - - def test_merge_queue_checks_combined_generated_ownership(self) -> None: - self.git("checkout", "-q", "-b", "handwritten", self.base) - self.write("handwritten.py", "custom behavior\n") - handwritten = self.commit() - self.git("checkout", "-q", "-b", "workflow", self.base) - self.write(self.workflow, "manual workflow edit\n") - workflow = self.commit() - self.git("checkout", "-q", "-b", "queue", self.base) - self.git("merge", "--no-ff", "-m", "queue handwritten", handwritten) - passing, _ = budget.evaluate( - self.repo, - self.base, - self.git("rev-parse", "HEAD"), - public=False, - pull_heads=[handwritten], - ) - self.assertTrue(all(c["state"] == "success" for c in passing["checks"].values())) - self.git("merge", "--no-ff", "-m", "queue workflow", workflow) - failing, _ = budget.evaluate( - self.repo, - self.base, - self.git("rev-parse", "HEAD"), - public=False, - pull_heads=[handwritten, workflow], - ) - self.assertEqual(failing["total"], 0) - self.assertEqual(failing["checks"]["isolation"]["state"], "success") - self.assertEqual(failing["checks"]["budget"]["state"], "failure") - self.assertIn(self.workflow, failing["strict_violations"][0]) - - def test_content_deletion_rename_mode_and_symlink_fail(self) -> None: - for mutation in ("edit", "delete", "rename", "mode", "symlink"): - with self.subTest(mutation=mutation): - self.git("reset", "--hard", self.base) - path = self.repo / "generated.py" - if mutation == "edit": - path.write_text("header removed and manually changed\n") - elif mutation == "delete": - path.unlink() - elif mutation == "rename": - path.rename(self.repo / "renamed.py") - elif mutation == "mode": - path.chmod(0o755) - else: - path.unlink() - path.symlink_to("handwritten.py") - self.assertEqual(self.evaluate()["checks"]["budget"]["state"], "failure") - - def test_candidate_cannot_rewrite_its_manifest_or_disable_strict(self) -> None: - self.write(self.workflow, "custom\n") - self.manifest() - self.write(budget.POLICY, json.dumps({"schema_version": 1, "max_custom_patch_lines": 1000})) - result = self.evaluate() - self.assertEqual(result["checks"]["isolation"]["state"], "failure") - self.assertEqual(result["checks"]["budget"]["state"], "failure") - - def test_missing_manifest_cannot_activate(self) -> None: - self.git("checkout", "-q", self.generated) - (self.repo / self.manifest_path).unlink() - checkpoint = self.commit(self.git("show", "-s", "--format=%B", self.generated)) - self.git("update-ref", "refs/remotes/origin/codegen/test", checkpoint) - self.git("checkout", "-q", "main") - stats = (self.repo / ".castiron.stats.yml").read_text() - self.write( - ".castiron.stats.yml", - stats.replace(self.generated, checkpoint).replace( - report.hash_codegen_commit(self.repo, self.generated), - report.hash_codegen_commit(self.repo, checkpoint), - ), - ) - self.assertEqual(self.evaluate()["checks"]["budget"]["state"], "failure") - - def test_legitimate_new_generated_checkpoint_passes(self) -> None: - self.git("checkout", "-q", self.generated) - self.write("generated.py", "new generated output\n") - self.write(self.workflow, "new generated workflow\n") - self.manifest() - checkpoint = self.commit(self.git("show", "-s", "--format=%B", self.generated)) - self.git("update-ref", "refs/remotes/origin/codegen/test", checkpoint) - self.git("checkout", "-q", "main") - for path in ("generated.py", self.workflow, self.manifest_path): - self.write(path, self.git("show", f"{checkpoint}:{path}") + "\n") - # Preserve exact manifest bytes (the fixture writes JSON without a newline). - self.write(self.manifest_path, self.git("show", f"{checkpoint}:{self.manifest_path}")) - stats = (self.repo / ".castiron.stats.yml").read_text() - self.write( - ".castiron.stats.yml", - stats.replace(self.generated, checkpoint).replace( - report.hash_codegen_commit(self.repo, self.generated), - report.hash_codegen_commit(self.repo, checkpoint), - ), - ) - self.assertEqual(self.evaluate()["checks"]["budget"]["state"], "success") - - def test_worktree_verification_catches_uncommitted_edits(self) -> None: - self.assertEqual( - budget.verify_generated(self.repo, self.generated, self.base, worktree=True), [] - ) - self.write(self.workflow, "uncommitted\n") - failures = budget.verify_generated(self.repo, self.generated, self.base, worktree=True) - self.assertEqual(len(failures), 1) - self.assertIn(self.workflow, failures[0]) - - def test_activation_is_policy_only_and_checks_workflows(self) -> None: - strict = (self.repo / budget.POLICY).read_text() - self.write(budget.POLICY, '{"schema_version":1,"max_custom_patch_lines":1000}') - migration = self.commit() - self.write(budget.POLICY, strict) - activation = self.commit() - result, _ = budget.evaluate(self.repo, migration, activation, public=False) - self.assertTrue(all(check["state"] == "success" for check in result["checks"].values())) - self.write(self.workflow, "custom workflow\n") - result, _ = budget.evaluate(self.repo, migration, self.commit(), public=False) - self.assertTrue(all(check["state"] == "failure" for check in result["checks"].values())) - - def test_manifest_must_cover_snapshot_and_match_its_bytes(self) -> None: - self.git("checkout", "-q", self.generated) - manifest = json.loads((self.repo / self.manifest_path).read_text()) - manifest["files"].pop("generated.py") - self.write(self.manifest_path, json.dumps(manifest)) - with self.assertRaisesRegex(ValueError, "cover.*exactly"): - budget.generated_manifest(self.repo, self.commit()) - self.manifest() - self.write("generated.py", "snapshot changed without updating manifest\n") - with self.assertRaisesRegex(ValueError, "disagrees"): - budget.generated_manifest(self.repo, self.commit()) - - def test_relocated_tooling_preserves_ownership_and_retirement_checks(self) -> None: - self.git("checkout", "--detach", "-q", self.generated) - old_manifest = self.manifest_path - new_manifest = self.alternate_manifest_path - old_checker = self.checker_path - new_checker = str(Path(new_manifest).with_name("custom_code_budget.py")) - manifest = json.loads((self.repo / old_manifest).read_text()) - manifest["files"][new_checker] = manifest["files"].pop(old_checker) - checker = (self.repo / old_checker).read_text() - self.write(new_checker, checker) - self.write(new_manifest, json.dumps(manifest)) - (self.repo / old_checker).unlink() - (self.repo / old_manifest).unlink() - relocated = self.commit("relocate generated tooling") - self.assertEqual( - budget.verify_generated(self.repo, relocated, relocated, previous=self.generated), [] - ) - ownership = budget.generated_manifest(self.repo, relocated) - self.assertIn(new_manifest.encode(), ownership) - self.assertIn(new_checker.encode(), ownership) - self.assertNotIn(old_manifest.encode(), ownership) - self.write(old_checker, checker) - self.write("generated.py", "handwritten modification\n") - failures = budget.verify_generated( - self.repo, relocated, self.commit(), previous=self.generated - ) - self.assertTrue( - any(old_checker in failure and "retired" in failure for failure in failures) - ) - self.assertTrue(any("generated.py: differs" in failure for failure in failures)) - # Candidate manifest edits cannot change the verified ownership contract. - self.write(new_manifest, '{"schema_version": 1, "files": {}}') - failures = budget.verify_generated(self.repo, relocated, self.commit()) - self.assertTrue( - any(new_manifest in failure and "differs" in failure for failure in failures) - ) - - def test_checkpoint_rejects_ambiguous_manifest_locations(self) -> None: - self.git("checkout", "--detach", "-q", self.generated) - self.write( - self.alternate_manifest_path, - (self.repo / self.manifest_path).read_text(), - ) - with self.assertRaisesRegex(ValueError, "ambiguous"): - budget.generated_manifest(self.repo, self.commit()) - - def test_relocated_manifest_requires_colocated_checker_inventory(self) -> None: - self.git("checkout", "--detach", "-q", self.generated) - self.write( - self.alternate_manifest_path, - (self.repo / self.manifest_path).read_text(), - ) - (self.repo / self.manifest_path).unlink() - with self.assertRaisesRegex(ValueError, "missing its generated GitHub inventory"): - budget.generated_manifest(self.repo, self.commit()) - - def test_manifest_preserves_inherited_sdk_owned_github_files(self) -> None: - self.git("checkout", "--detach", "-q", self.generated) - inherited = (".github/workflows/ci.yml", ".github/actions/setup-go/action.yml") - for path in inherited: - self.write(path, "SDK-owned configuration\n") - generated = self.commit("checkpoint preserves protected parent subtrees") - manifest = budget.generated_manifest(self.repo, generated) - for path in inherited: - self.assertNotIn(path.encode(), manifest) - self.write(path, "SDK-owned configuration update\n") - head = self.commit("handwritten configuration remains outside generated ownership") - self.assertEqual(budget.verify_generated(self.repo, generated, head), []) - - def test_private_manifest_checks_excluded_workflows_and_actions(self) -> None: - self.assertIn(self.workflow.encode(), budget.GENERATED_GITHUB_PATHS) - for path in (self.workflow, ".github/actions/generated/action.yml"): - for mutation in ("omit", "content", "mode"): - with ( - self.subTest(path=path, mutation=mutation), - mock.patch.object( - budget, - "GENERATED_GITHUB_PATHS", - budget.GENERATED_GITHUB_PATHS | {path.encode()}, - ), - ): - self.git("checkout", "--detach", "-q", self.generated) - self.write(path, "generated workflow or action\n") - self.inventory(budget.GENERATED_GITHUB_PATHS) - self.manifest((path,)) - manifest = json.loads((self.repo / self.manifest_path).read_text()) - manifest["files"][path] = { - "mode": "100644", - "sha256": hashlib.sha256((self.repo / path).read_bytes()).hexdigest(), - } - if mutation == "omit": - del manifest["files"][path] - elif mutation == "content": - self.write(path, "unrecorded change\n") - else: - (self.repo / path).chmod(0o755) - self.write(self.manifest_path, json.dumps(manifest)) - error = ( - "omits generated GitHub inventory path" - if mutation == "omit" - else "disagrees" - ) - with self.assertRaisesRegex(ValueError, error): - budget.generated_manifest(self.repo, self.commit()) - - def test_checkpoint_inventory_can_add_and_retire_generated_ci(self) -> None: - for path in (".github/workflows/new.yml", ".github/actions/new/action.yml"): - with self.subTest(path=path): - self.git("checkout", "--detach", "-q", self.generated) - self.write(path, "new generated CI\n") - self.inventory(budget.GENERATED_GITHUB_PATHS | {path.encode()}) - self.manifest((path,)) - newer = self.commit("new compiler inventory") - # The executing verifier still has the old inventory. - self.assertNotIn(path.encode(), budget.GENERATED_GITHUB_PATHS) - self.assertEqual( - budget.verify_generated(self.repo, newer, newer, previous=self.generated), [] - ) - (self.repo / path).unlink() - public = self.commit("public checkpoint omits private CI blob") - self.assertEqual( - budget.verify_generated(self.repo, public, newer, previous=self.generated), [] - ) - # A newer verifier must also read the prior inventory to enforce retirement. - self.assertEqual( - budget.verify_generated(self.repo, self.generated, self.base, previous=public), - [], - ) - self.git("checkout", "--detach", "-q", self.base) - self.write(path, "retired generated CI\n") - failures = budget.verify_generated( - self.repo, self.generated, self.commit(), previous=public - ) - self.assertTrue( - any(path in failure and "retired" in failure for failure in failures) - ) - - def test_manifest_cannot_claim_sdk_owned_workflows_or_actions(self) -> None: - for path in (".github/workflows/ci.yml", ".github/actions/setup-go/action.yml"): - for keep_blob in (True, False): - with self.subTest(path=path, keep_blob=keep_blob): - self.git("checkout", "--detach", "-q", self.generated) - self.write(path, "SDK-owned configuration\n") - manifest = json.loads((self.repo / self.manifest_path).read_text()) - manifest["files"][path] = { - "mode": "100644", - "sha256": hashlib.sha256((self.repo / path).read_bytes()).hexdigest(), - } - if not keep_blob: - (self.repo / path).unlink() - self.write(self.manifest_path, json.dumps(manifest)) - with self.assertRaisesRegex(ValueError, "SDK-owned GitHub path"): - budget.generated_manifest(self.repo, self.commit()) - - def test_public_snapshot_requires_every_inventory_identity(self) -> None: - for path in (self.workflow, ".github/actions/generated/action.yml"): - with self.subTest(path=path): - self.git("checkout", "--detach", "-q", self.generated) - self.write(path, "generated CI\n") - self.inventory(budget.GENERATED_GITHUB_PATHS | {path.encode()}) - self.manifest((path,)) - (self.repo / path).unlink() - manifest = json.loads((self.repo / self.manifest_path).read_text()) - del manifest["files"][path] - self.write(self.manifest_path, json.dumps(manifest)) - with self.assertRaisesRegex(ValueError, "omits generated GitHub inventory path"): - budget.generated_manifest( - self.repo, self.commit("incomplete public checkpoint") - ) - - def test_public_snapshot_keeps_strict_workflow_identity_without_private_blob(self) -> None: - self.git("checkout", "--detach", "-q", self.generated) - (self.repo / self.workflow).unlink() - public = self.commit("sanitized public checkpoint") - self.assertIn(self.workflow.encode(), budget.generated_manifest(self.repo, public)) - self.assertEqual(budget.verify_generated(self.repo, public, self.base), []) - self.git("checkout", "--detach", "-q", self.base) - self.write(self.workflow, "manual workflow edit\n") - violations = budget.verify_generated(self.repo, public, self.commit()) - self.assertEqual(len(violations), 1) - self.assertIn(self.workflow, violations[0]) - - def test_generated_deletion_requires_removing_retired_file(self) -> None: - self.git("checkout", "-q", self.generated) - (self.repo / "generated.py").unlink() - manifest = json.loads((self.repo / self.manifest_path).read_text()) - del manifest["files"]["generated.py"] - self.write(self.manifest_path, json.dumps(manifest)) - new = self.commit() - self.assertEqual(budget.verify_generated(self.repo, new, new, previous=self.generated), []) - self.write("generated.py", "retained handwritten replacement\n") - failures = budget.verify_generated(self.repo, new, self.commit(), previous=self.generated) - self.assertEqual(len(failures), 1) - self.assertIn("retired generated file", failures[0]) - - def test_generated_file_can_become_a_directory(self) -> None: - self.git("checkout", "-q", self.generated) - (self.repo / "generated.py").unlink() - self.write("generated.py/nested.py", "new generated file\n") - manifest = json.loads((self.repo / self.manifest_path).read_text()) - del manifest["files"]["generated.py"] - manifest["files"]["generated.py/nested.py"] = { - "mode": "100644", - "sha256": hashlib.sha256(b"new generated file\n").hexdigest(), - } - self.write(self.manifest_path, json.dumps(manifest)) - new = self.commit() - for worktree in (False, True): - with self.subTest(worktree=worktree): - self.assertEqual( - budget.verify_generated( - self.repo, new, new, previous=self.generated, worktree=worktree - ), - [], - ) - - def test_compiler_entrypoint_preserves_failed_worktree(self) -> None: - self.write(self.workflow, "uncommitted custom workflow\n") - result = subprocess.run( - [ - sys.executable, - "-I", - str(Path(budget.__file__)), - "verify-generated", - "--repo", - str(self.repo), - "--policy-ref", - self.base, - "--generated", - self.generated, - "--head", - self.base, - "--worktree", - ], - capture_output=True, - text=True, - check=False, - ) - self.assertEqual(result.returncode, 1) - self.assertIn(self.workflow, result.stderr) - self.assertEqual((self.repo / self.workflow).read_text(), "uncommitted custom workflow\n") - - -if __name__ == "__main__": - unittest.main() From 92ef9b4cde96d153b001b2fe0170e0b2a378823e Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Tue, 27 Oct 2026 17:11:35 +0000 Subject: [PATCH 5/6] Check generated files in ordinary SDK lint --- .github/workflows/ci.yml | 3 ++ AGENTS.md | 7 +++-- CONTRIBUTING.md | 13 +++++++-- docs/architecture/security-model.md | 4 +++ scripts/check_generated_files.py | 44 +++++++++++++++++++++++++++++ 5 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 scripts/check_generated_files.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd8adcac6..6df225fc3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,9 @@ jobs: with: persist-credentials: false + - name: Check generated files + run: python3 -I scripts/check_generated_files.py + - name: Set up Java uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 with: diff --git a/AGENTS.md b/AGENTS.md index 77765534b..dda831014 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,8 +17,11 @@ separately owned handwritten helpers and extension points. Regeneration replaces generator-owned files and deletes retired generated paths. Handwritten edits to generated files are overwritten; they are never replayed or merged into new output. Preserve separately owned handwritten files and resolve -ownership collisions explicitly. Enforce this rule through code review; there is -no customization budget or additional generated-file CI gate. +ownership collisions explicitly. SDK lint verifies the contents and Git file modes +listed in `scripts/castiron/generated-files.json`. Run +`python3 -I scripts/check_generated_files.py` locally. Fix failures by changing +the inputs/compiler and regenerating; never update hashes to bless a handwritten +patch. Review generated output and inventory changes together. ## Security requirements diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0aaf0d67c..70e8c54fe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,8 +33,17 @@ separately owned handwritten helpers and extension points. Regeneration replaces generated files and removes retired generated paths. Handwritten edits to generated output are overwritten, not carried forward. Separately owned handwritten files are preserved. Resolve ownership conflicts -explicitly before regeneration. Code review enforces this rule; customization -budgets and their CI checks are retired. +explicitly before regeneration. SDK lint checks generated-file contents and Git +file modes against `scripts/castiron/generated-files.json`. From the repository +root, run: + +```sh +python3 -I scripts/check_generated_files.py +``` + +If it fails, fix the generation inputs/compiler and regenerate the files and +inventory. Do not edit the hashes to accept a handwritten patch. Review generated +output and inventory changes together. ## Security expectations diff --git a/docs/architecture/security-model.md b/docs/architecture/security-model.md index 7917dcb05..d5a22a366 100644 --- a/docs/architecture/security-model.md +++ b/docs/architecture/security-model.md @@ -11,6 +11,10 @@ SECURITY.md remains the authority for private disclosure and reportability instructions; this document is the authority for architecture, trust boundaries, attacker capabilities, security objectives, and severity calibration. +SDK lint verifies generated-file contents and Git file modes against the +checked-in compiler inventory. The checker runs with the existing lint job +permissions; inventory and checker changes receive ordinary SDK code review. + ## Overview This repository publishes Java/Kotlin client libraries rather than a hosted diff --git a/scripts/check_generated_files.py b/scripts/check_generated_files.py new file mode 100644 index 000000000..6590424b3 --- /dev/null +++ b/scripts/check_generated_files.py @@ -0,0 +1,44 @@ +"""Verify compiler-owned files against their generated inventory.""" + +from __future__ import annotations + +import sys +import json +import hashlib +from pathlib import Path + + +def main() -> int: + root = Path(__file__).resolve().parents[1] + inventory = json.loads((root / "scripts/castiron/generated-files.json").read_text()) + if inventory["schema_version"] != 1 or not inventory["files"]: + sys.stderr.write("Unsupported or empty generated-file inventory.\n") + return 1 + + changed: list[str] = [] + for name, expected in inventory["files"].items(): + relative = Path(name) + path = root / relative + if ( + relative.is_absolute() + or ".." in relative.parts + or any(root.joinpath(*relative.parts[:i]).is_symlink() for i in range(1, len(relative.parts) + 1)) + or not path.is_file() + ): + changed.append(name) + continue + mode = "100755" if path.stat().st_mode & 0o111 else "100644" + if mode != expected["mode"] or hashlib.sha256(path.read_bytes()).hexdigest() != expected["sha256"]: + changed.append(name) + + if changed: + sys.stderr.write("Generated files differ from their inventory:\n") + for name in sorted(changed): + sys.stderr.write(f" {name!r}\n") + sys.stderr.write("Change the inputs/compiler and regenerate; do not hand-edit generated files or hashes.\n") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 998c66a58fb4849586c875a91c9c1616c2b8fbf1 Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Tue, 27 Oct 2026 18:07:49 +0000 Subject: [PATCH 6/6] docs: rely on AGENTS guidance for generated-file ownership --- .github/workflows/ci.yml | 3 -- AGENTS.md | 10 +++---- CONTRIBUTING.md | 17 ++++------- docs/architecture/security-model.md | 7 +++-- scripts/check_generated_files.py | 44 ----------------------------- 5 files changed, 15 insertions(+), 66 deletions(-) delete mode 100644 scripts/check_generated_files.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6df225fc3..dd8adcac6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,9 +39,6 @@ jobs: with: persist-credentials: false - - name: Check generated files - run: python3 -I scripts/check_generated_files.py - - name: Set up Java uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 with: diff --git a/AGENTS.md b/AGENTS.md index dda831014..951f43546 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,11 +17,11 @@ separately owned handwritten helpers and extension points. Regeneration replaces generator-owned files and deletes retired generated paths. Handwritten edits to generated files are overwritten; they are never replayed or merged into new output. Preserve separately owned handwritten files and resolve -ownership collisions explicitly. SDK lint verifies the contents and Git file modes -listed in `scripts/castiron/generated-files.json`. Run -`python3 -I scripts/check_generated_files.py` locally. Fix failures by changing -the inputs/compiler and regenerating; never update hashes to bless a handwritten -patch. Review generated output and inventory changes together. +ownership collisions explicitly. Follow this AGENTS.md rule and review generated output and inventory changes +together. Generated-file ownership currently relies on contributor guidance and +code review; there is no dedicated generated-file edit check or required gate. +Fix violations in the inputs/compiler and regenerate; never update inventory +hashes to accept a handwritten patch. ## Security requirements diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 70e8c54fe..2fddfb872 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,17 +33,12 @@ separately owned handwritten helpers and extension points. Regeneration replaces generated files and removes retired generated paths. Handwritten edits to generated output are overwritten, not carried forward. Separately owned handwritten files are preserved. Resolve ownership conflicts -explicitly before regeneration. SDK lint checks generated-file contents and Git -file modes against `scripts/castiron/generated-files.json`. From the repository -root, run: - -```sh -python3 -I scripts/check_generated_files.py -``` - -If it fails, fix the generation inputs/compiler and regenerate the files and -inventory. Do not edit the hashes to accept a handwritten patch. Review generated -output and inventory changes together. +explicitly before regeneration. Follow the ownership rules in `AGENTS.md` +and review generated output and inventory changes together. Generated-file +ownership currently relies on contributor guidance and code review, without a +dedicated generated-file edit check or required gate. Fix violations in the +generation inputs/compiler and regenerate; do not edit inventory hashes to +accept a handwritten patch. ## Security expectations diff --git a/docs/architecture/security-model.md b/docs/architecture/security-model.md index d5a22a366..2f71ec589 100644 --- a/docs/architecture/security-model.md +++ b/docs/architecture/security-model.md @@ -11,9 +11,10 @@ SECURITY.md remains the authority for private disclosure and reportability instructions; this document is the authority for architecture, trust boundaries, attacker capabilities, security objectives, and severity calibration. -SDK lint verifies generated-file contents and Git file modes against the -checked-in compiler inventory. The checker runs with the existing lint job -permissions; inventory and checker changes receive ordinary SDK code review. +Generated-file ownership currently relies on `AGENTS.md`, contributor guidance, +and ordinary SDK code review. There is no dedicated generated-file edit check or +required gate. Review generated output and inventory changes together; fix +violations in the inputs/compiler and regenerate. ## Overview diff --git a/scripts/check_generated_files.py b/scripts/check_generated_files.py deleted file mode 100644 index 6590424b3..000000000 --- a/scripts/check_generated_files.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Verify compiler-owned files against their generated inventory.""" - -from __future__ import annotations - -import sys -import json -import hashlib -from pathlib import Path - - -def main() -> int: - root = Path(__file__).resolve().parents[1] - inventory = json.loads((root / "scripts/castiron/generated-files.json").read_text()) - if inventory["schema_version"] != 1 or not inventory["files"]: - sys.stderr.write("Unsupported or empty generated-file inventory.\n") - return 1 - - changed: list[str] = [] - for name, expected in inventory["files"].items(): - relative = Path(name) - path = root / relative - if ( - relative.is_absolute() - or ".." in relative.parts - or any(root.joinpath(*relative.parts[:i]).is_symlink() for i in range(1, len(relative.parts) + 1)) - or not path.is_file() - ): - changed.append(name) - continue - mode = "100755" if path.stat().st_mode & 0o111 else "100644" - if mode != expected["mode"] or hashlib.sha256(path.read_bytes()).hexdigest() != expected["sha256"]: - changed.append(name) - - if changed: - sys.stderr.write("Generated files differ from their inventory:\n") - for name in sorted(changed): - sys.stderr.write(f" {name!r}\n") - sys.stderr.write("Change the inputs/compiler and regenerate; do not hand-edit generated files or hashes.\n") - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main())