From 5bcfd1bbe90c4fb9cf9f0b1637e7e3ac431dbd88 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Fri, 23 Oct 2026 02:44:17 +0000 Subject: [PATCH 1/9] feat(evals): add coding-task evaluation framework (#2084) --- package.json | 2 + scripts/agent-evals/README.md | 68 +- scripts/agent-evals/coding-compare.mts | 112 +++ scripts/agent-evals/coding-compare.ts | 61 ++ scripts/agent-evals/coding-digest.test.ts | 24 + scripts/agent-evals/coding-digest.ts | 28 + scripts/agent-evals/coding-fixtures.test.ts | 80 +++ scripts/agent-evals/coding-fixtures.ts | 669 ++++++++++++++++++ scripts/agent-evals/coding-grader.test.ts | 182 +++++ scripts/agent-evals/coding-grader.ts | 132 ++++ scripts/agent-evals/coding-metrics.test.ts | 305 ++++++++ scripts/agent-evals/coding-metrics.ts | 194 +++++ scripts/agent-evals/coding-outcomes.test.ts | 60 ++ scripts/agent-evals/coding-outcomes.ts | 54 ++ scripts/agent-evals/coding-pins.test.ts | 46 ++ scripts/agent-evals/coding-pins.ts | 34 + scripts/agent-evals/coding-pricing.test.ts | 53 ++ scripts/agent-evals/coding-pricing.ts | 74 ++ .../coding-reference-solutions.test.ts | 72 ++ scripts/agent-evals/coding-report.test.ts | 181 +++++ scripts/agent-evals/coding-report.ts | 211 ++++++ scripts/agent-evals/coding-run.test.ts | 163 +++++ scripts/agent-evals/coding-run.ts | 144 ++++ scripts/agent-evals/coding-side.mts | 135 ++++ scripts/agent-evals/coding-stats.test.ts | 62 ++ scripts/agent-evals/coding-stats.ts | 118 +++ scripts/agent-evals/coding-suite.ts | 53 ++ scripts/agent-evals/coding-workspace.test.ts | 102 +++ scripts/agent-evals/coding-workspace.ts | 49 ++ scripts/agent-evals/container-runtime.test.ts | 62 ++ scripts/agent-evals/container-runtime.ts | 141 ++++ scripts/agent-evals/grader-driver.mjs | 139 ++++ scripts/agent-evals/launcher.ts | 52 +- scripts/agent-evals/runner.ts | 9 + 34 files changed, 3862 insertions(+), 9 deletions(-) create mode 100644 scripts/agent-evals/coding-compare.mts create mode 100644 scripts/agent-evals/coding-compare.ts create mode 100644 scripts/agent-evals/coding-digest.test.ts create mode 100644 scripts/agent-evals/coding-digest.ts create mode 100644 scripts/agent-evals/coding-fixtures.test.ts create mode 100644 scripts/agent-evals/coding-fixtures.ts create mode 100644 scripts/agent-evals/coding-grader.test.ts create mode 100644 scripts/agent-evals/coding-grader.ts create mode 100644 scripts/agent-evals/coding-metrics.test.ts create mode 100644 scripts/agent-evals/coding-metrics.ts create mode 100644 scripts/agent-evals/coding-outcomes.test.ts create mode 100644 scripts/agent-evals/coding-outcomes.ts create mode 100644 scripts/agent-evals/coding-pins.test.ts create mode 100644 scripts/agent-evals/coding-pins.ts create mode 100644 scripts/agent-evals/coding-pricing.test.ts create mode 100644 scripts/agent-evals/coding-pricing.ts create mode 100644 scripts/agent-evals/coding-reference-solutions.test.ts create mode 100644 scripts/agent-evals/coding-report.test.ts create mode 100644 scripts/agent-evals/coding-report.ts create mode 100644 scripts/agent-evals/coding-run.test.ts create mode 100644 scripts/agent-evals/coding-run.ts create mode 100644 scripts/agent-evals/coding-side.mts create mode 100644 scripts/agent-evals/coding-stats.test.ts create mode 100644 scripts/agent-evals/coding-stats.ts create mode 100644 scripts/agent-evals/coding-suite.ts create mode 100644 scripts/agent-evals/coding-workspace.test.ts create mode 100644 scripts/agent-evals/coding-workspace.ts create mode 100644 scripts/agent-evals/container-runtime.test.ts create mode 100644 scripts/agent-evals/container-runtime.ts create mode 100644 scripts/agent-evals/grader-driver.mjs diff --git a/package.json b/package.json index 6ec1d9067f..1b4e098ad5 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,8 @@ "solheim-smoke:check-types": "tsc -p scripts/solheim-smoke/tsconfig.json", "test:agent-evals:unit": "node --import tsx --test scripts/agent-evals/*.test.ts", "agent-evals:check-types": "tsc -p scripts/agent-evals/tsconfig.json", + "coding-evals:side": "node --import tsx scripts/agent-evals/coding-side.mts", + "coding-evals:compare": "node --import tsx scripts/agent-evals/coding-compare.mts", "test:mutation-ci": "node --test scripts/stryker-diff.test.mjs", "lifecycle:model-check": "tsx scripts/check-task-lifecycle.ts && tsx scripts/check-task-store-concurrency.ts && tsx scripts/check-provider-handoff-scheduler.ts && pnpm cleanup-protocol:model-check && pnpm parser-scope:model-check && tsx scripts/check-completion-persistence.ts && tsx scripts/check-delegated-mode-readers.ts && pnpm tool-turn:model-check", "fanout-protocol:model-check": "tsx scripts/check-task-fanout-protocol.ts", diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index bd24b6cfba..afc55e6c23 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -92,13 +92,65 @@ pnpm agent-evals:check-types Both run offline. No check spawns VS Code or touches a provider. -## Follow-up milestones +## Coding evaluation framework + +The coding framework layers an evaluation on the substrate. It grades runs, +repeats them, and compares two exact commits. It stays independent from any +release process: a maintainer runs it locally or on demand. + +| File | Purpose | +| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `coding-fixtures.ts` | Five frozen coding fixtures. Each ships a known-good workspace, one prompt, a tool policy, limits, hidden acceptance tests, and two reference solutions. | +| `coding-suite.ts` | Suite identity `coding-basic-v1`, registration, and the suite content digest. | +| `coding-workspace.ts` | The immutable initial-state digest, fresh workspace builds, and fresh grading copies. | +| `coding-grader.ts` | The grader contract, version, driver invocation, and the local subprocess runtime. | +| `grader-driver.mjs` | The acceptance-test driver. Runs the workspace tests and writes one result JSON file. | +| `container-runtime.ts` | The default grading runtime. Builds and runs the locked-down container invocation. | +| `coding-outcomes.ts` | Outcome rules: `correct`, `incorrect`, `timed out`, `grading failure`, `infrastructure failure`. | +| `coding-pricing.ts` | The operator-maintained price table and the run cost estimate. | +| `coding-metrics.ts` | Per-run metrics: requests, usage, tool calls, cost, unavailable telemetry, cost per successful task. | +| `coding-run.ts` | One repetition: substrate run, final-workspace capture, grading, outcome, metrics. | +| `coding-pins.ts` | Pinned comparison inputs and the incompatibility check. | +| `coding-stats.ts` | Three repetitions, success rates with Wilson intervals, paired differences. | +| `coding-report.ts` | The self-contained JSON report and the Markdown renderer. | +| `coding-compare.ts` | The comparison orchestration and the side result shape. | +| `coding-side.mts` | One comparison side: all fixtures times N repetitions on this commit, graded, with pinned inputs. | +| `coding-compare.mts` | Two-commit comparison driver. Writes `comparison-report.json` and `comparison-report.md`. | + +### Grading + +- The grader is model-free and credential-free. It judges the final workspace state only. +- The final workspace is captured as bounded file data and rebuilt as a fresh copy. Hidden acceptance tests enter only that copy, so the agent can never read or change them, and one run can never inherit another run's files. +- The default runtime executes the driver in a disposable container with no network, no capabilities, no privilege escalation, a read-only root filesystem, and fixed CPU, memory, and process limits. No environment variable passes through. Only trusted repo code runs with the provider credential. Fixtures and extension code from outside pull requests never do. +- The grader has its own wall-clock limit. A grader that hangs, crashes, or loses its runner is a `grading failure`. It is never an `incorrect` solution. +- Each fixture ships two reference solutions. The known-correct solution must pass. The deliberately incorrect one passes the visible tests and must fail. Unit tests enforce both on every fixture. +- `createLocalSubprocessGradingRuntime` runs the same driver directly on the host. It does not give container isolation. It exists for unit tests and explicit local checks only and must never back a recorded comparison. + +### Comparisons + +A paired comparison pins both commit SHAs, the suite digest, the settings digest, the model identifier, the grader version, and the runner environment on both sides. Any difference in a pinned value marks the pair incompatible. The report records the values and flags the pair instead of dropping it. + +The default is three repetitions per fixture per commit. Reports show success rates with Wilson 95% intervals and paired differences. They make no significance claim: three repetitions can show large regressions, not small improvements. + +Runs with incomplete token or cost accounting are flagged and excluded from aggregate cost comparisons. Missing token or cost values stay missing. The report states the price-table identifier and date and how cache reads and writes are priced. Cost per successful task divides the total attempt cost by the number of `correct` runs. It is unavailable when no run is correct or a correct run lacks complete cost data. Duplicate-call detection and HTTP retry counts are reported as unavailable. + +### Commands -The substrate supports these planned steps. None of them exist in this -change: +```sh +# Unit tests and type checks. Offline. No VS Code, no provider, no containers. +pnpm test:agent-evals:unit +pnpm agent-evals:check-types + +# One comparison side against the real provider. Requires the credential +# environment variable and the container runtime for grading. +pnpm coding-evals:side --reps 3 --out /tmp/side-out + +# Full paired comparison of two commits. +pnpm coding-evals:compare --baseline --candidate --reps 3 --out /tmp/compare-out +``` + +### Follow-up milestones -1. Five frozen, real coding task fixtures with known-good workspaces. -2. Independent credential-free graders that judge workspace state only. -3. Paired exact trusted commit comparisons on the same fixtures. -4. Repeated runs with uncertainty reporting. -5. Advisory release integration aligned to the actual release workflows. +1. Release integration: trigger runs, store baselines, and promote them. See issue #2085. +2. Later suites: repository navigation, debugging, and cost stress, toward the 21-task set. +3. Optimization experiments: per-request context instrumentation, non-progress loop detection, tool-output filtering, compaction strategies, and dynamic tool-schema loading. diff --git a/scripts/agent-evals/coding-compare.mts b/scripts/agent-evals/coding-compare.mts new file mode 100644 index 0000000000..771c5c48b6 --- /dev/null +++ b/scripts/agent-evals/coding-compare.mts @@ -0,0 +1,112 @@ +// Paired comparison driver. Checks out both commits into temporary worktrees, +// runs one coding-evals side per commit with identical repetition counts, +// then writes the JSON and Markdown comparison reports. Both commits must +// contain this framework. The driver is trusted repo code and only runs +// trusted repo commits with the credential. +import { execFile, spawn } from "node:child_process" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { fileURLToPath } from "node:url" +import { promisify } from "node:util" +import { sideResultFileName, toSideResult, type SideResultFile } from "./coding-compare.ts" +import { buildComparisonReport, renderMarkdownReport } from "./coding-report.ts" + +const execFileAsync = promisify(execFile) +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") + +const readArgs = (): { + baseline: string + candidate: string + reps: number + out: string + credentialEnv: string + allowUnisolatedGrading: boolean +} => { + const args = process.argv.slice(2) + const read = (name: string): string | undefined => { + const index = args.indexOf(`--${name}`) + return index >= 0 ? args[index + 1] : undefined + } + const baseline = read("baseline") + const candidate = read("candidate") + const out = read("out") + if (!baseline || !candidate) throw new Error("Set --baseline and --candidate to commit SHAs") + if (!out) throw new Error("Set --out to a scratch directory") + const reps = Number(read("reps") ?? "3") + if (!Number.isInteger(reps) || reps < 1) throw new Error("--reps must be a positive integer") + return { + baseline, + candidate, + reps, + out: path.resolve(out), + credentialEnv: read("credential-env") ?? "SOLHEIM_API_KEY", + allowUnisolatedGrading: args.includes("--allow-unisolated-grading"), + } +} + +const runSide = async ( + side: "baseline" | "candidate", + commit: string, + inputs: ReturnType, + scratchDir: string, +): Promise => { + const worktree = path.join(scratchDir, side) + await execFileAsync("git", ["worktree", "add", "--detach", worktree, commit], { cwd: root, timeout: 120_000 }) + const sideOut = path.join(scratchDir, `${side}-out`) + await fs.mkdir(sideOut, { recursive: true }) + const exit = await new Promise((resolve, reject) => { + const child = spawn( + process.execPath, + [ + "--import", + "tsx", + "scripts/agent-evals/coding-side.mts", + "--reps", + String(inputs.reps), + "--out", + sideOut, + "--credential-env", + inputs.credentialEnv, + ...(inputs.allowUnisolatedGrading ? ["--allow-unisolated-grading"] : []), + ], + { + cwd: worktree, + env: { ...process.env, CODING_EVALS_SIDE: side }, + stdio: "inherit", + }, + ) + child.once("error", reject) + child.once("exit", (code) => resolve(code ?? 1)) + }) + if (exit !== 0) throw new Error(`${side} side failed with exit code ${exit}`) + return JSON.parse(await fs.readFile(path.join(sideOut, sideResultFileName(side)), "utf8")) as SideResultFile +} + +const inputs = readArgs() +const scratchDir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-evals-compare-")) +await fs.mkdir(inputs.out, { recursive: true }) +try { + const report = buildComparisonReport({ + baseline: toSideResult(await runSide("baseline", inputs.baseline, inputs, scratchDir)), + candidate: toSideResult(await runSide("candidate", inputs.candidate, inputs, scratchDir)), + repetitions: inputs.reps, + }) + await fs.writeFile(path.join(inputs.out, "comparison-report.json"), JSON.stringify(report, null, 2)) + await fs.writeFile(path.join(inputs.out, "comparison-report.md"), renderMarkdownReport(report)) + console.log( + report.compatibility.compatible + ? `comparison complete: compatible pair, report in ${inputs.out}` + : `comparison complete: INCOMPATIBLE pair (${report.compatibility.differences.join(", ")}), report in ${inputs.out}`, + ) +} finally { + await execFileAsync("git", ["worktree", "remove", "--force", path.join(scratchDir, "baseline")], { + cwd: root, + timeout: 30_000, + }).catch(() => {}) + await execFileAsync("git", ["worktree", "remove", "--force", path.join(scratchDir, "candidate")], { + cwd: root, + timeout: 30_000, + }).catch(() => {}) + await fs.rm(scratchDir, { recursive: true, force: true }).catch(() => {}) +} diff --git a/scripts/agent-evals/coding-compare.ts b/scripts/agent-evals/coding-compare.ts new file mode 100644 index 0000000000..56357f419d --- /dev/null +++ b/scripts/agent-evals/coding-compare.ts @@ -0,0 +1,61 @@ +import type { SidePins } from "./coding-pins.ts" +import { buildComparisonReport, renderMarkdownReport, type ComparisonReport, type SideResult } from "./coding-report.ts" +import { DEFAULT_REPETITIONS } from "./coding-stats.ts" +import type { CodingRunRecord } from "./coding-run.ts" + +export { buildComparisonReport, renderMarkdownReport } +export type { ComparisonReport, SideResult } + +export interface SideExecutionRequest { + side: "baseline" | "candidate" + commit: string + repetitions: number +} + +// Executes one side of the comparison. The live executor runs the side's own +// in-repo suite inside a worktree of that commit. Tests replace it with a +// synthetic executor. +export interface SideExecutor { + runSide(request: SideExecutionRequest): Promise +} + +export interface ComparisonRequest { + baselineCommit: string + candidateCommit: string + repetitions?: number + executor: SideExecutor +} + +// Paired comparison. Each side runs the same repetition count over its own +// frozen suite copy with pinned inputs. Pinned value differences surface in +// the report as an incompatible pair; they never abort the run, because the +// recorded values are the evidence a reviewer needs. +export async function runComparison(request: ComparisonRequest): Promise { + const repetitions = request.repetitions ?? DEFAULT_REPETITIONS + const baseline = await request.executor.runSide({ + side: "baseline", + commit: request.baselineCommit, + repetitions, + }) + const candidate = await request.executor.runSide({ + side: "candidate", + commit: request.candidateCommit, + repetitions, + }) + return buildComparisonReport({ baseline, candidate, repetitions }) +} + +// Serializes a side result for artifact storage. Full run records travel so +// each side's traces and grading verdicts stay inspectable. +export interface SideResultFile { + pins: SidePins + runs: CodingRunRecord[] +} + +// Reduces a side result file to the shape the report builder consumes. +export const toSideResult = (file: SideResultFile): SideResult => ({ + pins: file.pins, + runs: file.runs.map((record) => record.metrics), +}) + +export const sideResultFileName = (side: "baseline" | "candidate"): string => `${side}-results.json` diff --git a/scripts/agent-evals/coding-digest.test.ts b/scripts/agent-evals/coding-digest.test.ts new file mode 100644 index 0000000000..daedc1b1c3 --- /dev/null +++ b/scripts/agent-evals/coding-digest.test.ts @@ -0,0 +1,24 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { canonicalJson, digestValue, sha256Hex } from "./coding-digest.ts" + +describe("coding digest", () => { + it("produces stable digests independent of key order", () => { + const a = { b: 1, a: { d: 2, c: [1, 2] } } + const b = { a: { c: [1, 2], d: 2 }, b: 1 } + assert.equal(canonicalJson(a), canonicalJson(b)) + assert.equal(digestValue(a), digestValue(b)) + }) + + it("changes the digest when content changes", () => { + assert.notEqual(digestValue({ a: 1 }), digestValue({ a: 2 })) + }) + + it("keeps array order significant", () => { + assert.notEqual(digestValue([1, 2]), digestValue([2, 1])) + }) + + it("matches sha256 over the canonical form", () => { + assert.equal(digestValue({ x: "y" }), sha256Hex(canonicalJson({ x: "y" }))) + }) +}) diff --git a/scripts/agent-evals/coding-digest.ts b/scripts/agent-evals/coding-digest.ts new file mode 100644 index 0000000000..d66bbd19e2 --- /dev/null +++ b/scripts/agent-evals/coding-digest.ts @@ -0,0 +1,28 @@ +import { createHash } from "node:crypto" + +// Canonical JSON for digests. Object keys are sorted, array order is kept, and +// the output is stable across processes. Any content change changes the digest, +// so a digest pins content, not formatting. +export function canonicalJson(value: unknown): string { + return JSON.stringify(sortValue(value)) +} + +function sortValue(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortValue) + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value as Record) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([key, entry]) => [key, sortValue(entry)]), + ) + } + return value +} + +export function sha256Hex(text: string): string { + return createHash("sha256").update(text, "utf8").digest("hex") +} + +export function digestValue(value: unknown): string { + return sha256Hex(canonicalJson(value)) +} diff --git a/scripts/agent-evals/coding-fixtures.test.ts b/scripts/agent-evals/coding-fixtures.test.ts new file mode 100644 index 0000000000..d91d63b474 --- /dev/null +++ b/scripts/agent-evals/coding-fixtures.test.ts @@ -0,0 +1,80 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { configurationMatchesToolPolicy, validateFixture } from "./contract.ts" +import { CODING_FIXTURE_BUNDLES } from "./coding-fixtures.ts" +import { CODING_SUITE_ID, codingBundle, codingSuiteDigest } from "./coding-suite.ts" +import { GRADER_VERSION } from "./coding-grader.ts" + +const CREDENTIAL = "sk-test-credential-for-projection-check" + +describe("coding fixture suite", () => { + it("exposes exactly five fixtures", () => { + assert.equal(CODING_FIXTURE_BUNDLES.length, 5) + assert.equal(new Set(CODING_FIXTURE_BUNDLES.map((bundle) => bundle.fixture.identity.name)).size, 5) + }) + + it("passes the substrate contract for every fixture", () => { + for (const bundle of CODING_FIXTURE_BUNDLES) validateFixture(bundle.fixture) + }) + + it("builds configurations that match the declared tool policy", () => { + for (const bundle of CODING_FIXTURE_BUNDLES) { + const config = bundle.fixture.buildConfiguration(CREDENTIAL) as Record + assert.equal(configurationMatchesToolPolicy(config, bundle.fixture.toolPolicy), true) + } + }) + + it("keeps the credential out of the settings projection", () => { + for (const bundle of CODING_FIXTURE_BUNDLES) { + const projectionJson = JSON.stringify(bundle.fixture.settingsProjection) + assert.equal(projectionJson.includes(CREDENTIAL), false) + const configJson = JSON.stringify(bundle.fixture.buildConfiguration(CREDENTIAL)) + assert.equal(configJson.includes(CREDENTIAL), true) + } + }) + + it("ships hidden tests that are absent from the run workspace", () => { + for (const bundle of CODING_FIXTURE_BUNDLES) { + assert.ok(bundle.grading.hiddenTests.length > 0) + for (const hidden of bundle.grading.hiddenTests) + assert.equal( + bundle.fixture.workspace.files.some((file) => file.path === hidden.path), + false, + `hidden test leaked into workspace: ${hidden.path}`, + ) + } + }) + + it("ships two distinct reference solutions over existing workspace paths", () => { + for (const bundle of CODING_FIXTURE_BUNDLES) { + assert.ok(bundle.grading.correctSolution.length > 0) + const workspacePaths = new Set(bundle.fixture.workspace.files.map((file) => file.path)) + for (const file of [...bundle.grading.correctSolution, ...bundle.grading.incorrectSolution]) + assert.equal(workspacePaths.has(file.path), true, `solution path missing from workspace: ${file.path}`) + const sameFiles = bundle.grading.correctSolution.every((file) => { + const incorrect = bundle.grading.incorrectSolution.find((candidate) => candidate.path === file.path) + return incorrect === undefined || incorrect.content !== file.content + }) + assert.equal(sameFiles, true) + } + }) + + it("pins the grader identity with resources and an image", () => { + for (const bundle of CODING_FIXTURE_BUNDLES) { + assert.ok(bundle.grading.resources.timeoutMs > 0) + assert.ok(bundle.grading.resources.memoryMb > 0) + assert.ok(bundle.grading.resources.cpus > 0) + assert.ok(bundle.grading.resources.pidsLimit > 0) + assert.match(bundle.grading.resources.image, /^node:/) + } + }) + + it("produces a stable suite digest that tracks the grader version", () => { + assert.equal(codingSuiteDigest(), codingSuiteDigest()) + assert.equal(codingBundle("clamp-range").fixture.identity.name, "clamp-range") + assert.throws(() => codingBundle("unknown-fixture")) + assert.ok(codingSuiteDigest().length === 64) + assert.equal(CODING_SUITE_ID, "coding-basic-v1") + assert.equal(GRADER_VERSION, "coding-grader-v1") + }) +}) diff --git a/scripts/agent-evals/coding-fixtures.ts b/scripts/agent-evals/coding-fixtures.ts new file mode 100644 index 0000000000..f5990e9816 --- /dev/null +++ b/scripts/agent-evals/coding-fixtures.ts @@ -0,0 +1,669 @@ +import type { RooCodeSettings } from "@roo-code/types" +import type { AgentFixture, AgentFixtureWorkspaceFile } from "./contract.ts" +import type { GraderResources } from "./coding-grader.ts" +import { GRADER_IMAGE } from "./container-runtime.ts" + +// Suite provider. Mirrors the smoke fixture provider endpoint so a coding run +// uses the same trusted, preconfigured provider as the smoke run. +export const CODING_BASE_URL = "https://api.solheim.ai/v1" +export const CODING_MODEL_ID = "qwen3.8-27b" + +const CODING_ROLE_DEFINITION = "Complete one bounded coding task in this workspace. Work only inside the workspace." +const CODING_DISABLED_TOOLS = [ + "ask_followup_question", + "new_task", + "switch_mode", + "update_todo_list", + "skill", + "run_slash_command", +] as const + +export const CODING_GRADER_RESOURCES: GraderResources = { + timeoutMs: 30_000, + memoryMb: 512, + cpus: 1, + pidsLimit: 128, + image: GRADER_IMAGE, +} + +function buildCodingConfiguration(apiKey: string): RooCodeSettings { + return { + apiProvider: "openai", + openAiBaseUrl: CODING_BASE_URL, + openAiApiKey: apiKey, + openAiModelId: CODING_MODEL_ID, + openAiR1FormatEnabled: true, + openAiExtraBody: JSON.stringify({ chat_template_kwargs: { enable_thinking: false } }), + modelMaxTokens: 8192, + autoCondenseContext: false, + mode: "coding-eval", + customModes: [ + { + slug: "coding-eval", + name: "Coding eval", + roleDefinition: CODING_ROLE_DEFINITION, + groups: ["read", "edit", "command"], + }, + ], + disabledTools: [...CODING_DISABLED_TOOLS], + autoApprovalEnabled: true, + alwaysAllowReadOnly: true, + alwaysAllowReadOnlyOutsideWorkspace: false, + alwaysAllowWrite: true, + alwaysAllowWriteOutsideWorkspace: false, + alwaysAllowWriteProtected: false, + alwaysAllowExecute: true, + alwaysAllowMcp: false, + alwaysAllowModeSwitch: false, + alwaysAllowSubtasks: false, + alwaysAllowFollowupQuestions: false, + } +} + +// Explicit nonsecret mirror of the built configuration. Must stay in step +// with buildCodingConfiguration; the registry checks the builder against the +// tool policy and the trace digests the projection. +const codingSettingsProjection: Record = { + apiProvider: "openai", + openAiBaseUrl: CODING_BASE_URL, + openAiModelId: CODING_MODEL_ID, + openAiR1FormatEnabled: true, + openAiExtraBody: { chat_template_kwargs: { enable_thinking: false } }, + modelMaxTokens: 8192, + autoCondenseContext: false, + mode: "coding-eval", + customModes: [ + { + slug: "coding-eval", + name: "Coding eval", + roleDefinition: CODING_ROLE_DEFINITION, + groups: ["read", "edit", "command"], + }, + ], + disabledTools: [...CODING_DISABLED_TOOLS], + autoApprovalEnabled: true, + alwaysAllowReadOnly: true, + alwaysAllowReadOnlyOutsideWorkspace: false, + alwaysAllowWrite: true, + alwaysAllowWriteOutsideWorkspace: false, + alwaysAllowWriteProtected: false, + alwaysAllowExecute: true, + alwaysAllowMcp: false, + alwaysAllowModeSwitch: false, + alwaysAllowSubtasks: false, + alwaysAllowFollowupQuestions: false, +} + +export interface CodingFixtureBundle { + fixture: AgentFixture + grading: { + // Hidden acceptance tests. Copied into a fresh grading copy after the + // run. Never part of the workspace the agent sees. + hiddenTests: readonly AgentFixtureWorkspaceFile[] + // Reference solutions, each an overlay over the workspace files. The + // known-correct solution must pass the grader; the deliberately + // incorrect solution must fail it while passing the visible tests. + correctSolution: readonly AgentFixtureWorkspaceFile[] + incorrectSolution: readonly AgentFixtureWorkspaceFile[] + resources: GraderResources + } +} + +const workspacePackageJson = (name: string): AgentFixtureWorkspaceFile => ({ + path: "package.json", + content: `${JSON.stringify( + { + name: `${name}-workspace`, + private: true, + version: "1.0.0", + scripts: { test: "node --test test/" }, + }, + null, + 2, + )}\n`, +}) + +const codingLimits = { + activationTimeoutMs: 120_000, + taskStartTimeoutMs: 35_000, + taskRunTimeoutMs: 300_000, + usageDrainMs: 10_000, + teardownStepMs: 10_000, +} + +const bundle = ( + name: string, + version: string, + description: string, + prompt: string, + files: readonly AgentFixtureWorkspaceFile[], + hiddenTests: readonly AgentFixtureWorkspaceFile[], + correctSolution: readonly AgentFixtureWorkspaceFile[], + incorrectSolution: readonly AgentFixtureWorkspaceFile[], +): CodingFixtureBundle => ({ + fixture: { + identity: { name, version, description }, + prompt, + workspace: { files }, + toolPolicy: { + mode: "coding-eval", + roleDefinition: CODING_ROLE_DEFINITION, + groups: ["read", "edit", "command"], + disabledTools: [...CODING_DISABLED_TOOLS], + allow: { + readOnly: true, + readOnlyOutsideWorkspace: false, + write: true, + writeOutsideWorkspace: false, + writeProtected: false, + execute: true, + mcp: false, + modeSwitch: false, + subtasks: false, + followupQuestions: false, + }, + }, + limits: codingLimits, + settingsProjection: codingSettingsProjection, + buildConfiguration: buildCodingConfiguration, + }, + grading: { hiddenTests, correctSolution, incorrectSolution, resources: CODING_GRADER_RESOURCES }, +}) + +// --------------------------------------------------------------------------- +// Fixture 1: clamp-range +// --------------------------------------------------------------------------- + +const clampStub = `// Clamp a number into a closed range. +// +// clamp(value, min, max) returns value when min <= value <= max. A value +// below min returns min. A value above max returns max. When min is greater +// than max the range is empty and clamp returns min for every value. A NaN +// value returns NaN. + +function clamp(value, min, max) { + throw new Error("TODO: implement clamp") +} + +module.exports = { clamp } +` + +const clampVisibleTest = `const { test } = require("node:test") +const assert = require("node:assert") +const { clamp } = require("../src/clamp.js") + +test("keeps in-range values unchanged", () => { + assert.strictEqual(clamp(5, 0, 10), 5) +}) + +test("pulls values below the range down to min", () => { + assert.strictEqual(clamp(-1, 0, 10), 0) +}) + +test("pulls values above the range up to max", () => { + assert.strictEqual(clamp(11, 0, 10), 10) +}) +` + +const clampHiddenTest = `const { test } = require("node:test") +const assert = require("node:assert") +const { clamp } = require("../src/clamp.js") + +test("returns min for an empty range", () => { + assert.strictEqual(clamp(5, 10, 2), 10) +}) + +test("returns min for an empty range at the min boundary", () => { + assert.strictEqual(clamp(10, 10, 2), 10) +}) + +test("returns max for an empty range at the max boundary", () => { + assert.strictEqual(clamp(2, 10, 2), 10) +}) +` + +const clampCorrect = `// Clamp a number into a closed range. +function clamp(value, min, max) { + if (min > max) return min + if (value < min) return min + if (value > max) return max + return value +} + +module.exports = { clamp } +` + +const clampIncorrect = `function clamp(value, min, max) { + return Math.min(Math.max(value, min), max) +} + +module.exports = { clamp } +` + +// --------------------------------------------------------------------------- +// Fixture 2: slugify-text +// --------------------------------------------------------------------------- + +const slugifyStub = `// Convert text into a URL slug. +// +// slugify(text) lowercases the text, keeps latin letters and digits, and +// replaces every run of all other characters with exactly one dash. Leading +// and trailing runs produce no dashes. An empty result for empty input is a +// valid slug. + +function slugify(text) { + throw new Error("TODO: implement slugify") +} + +module.exports = { slugify } +` + +const slugifyVisibleTest = `const { test } = require("node:test") +const assert = require("node:assert") +const { slugify } = require("../src/slugify.js") + +test("joins words with one dash", () => { + assert.strictEqual(slugify("Hello World"), "hello-world") +}) + +test("collapses runs of separators", () => { + assert.strictEqual(slugify("A B"), "a-b") +}) + +test("keeps letters and digits", () => { + assert.strictEqual(slugify("abc123"), "abc123") +}) +` + +const slugifyHiddenTest = `const { test } = require("node:test") +const assert = require("node:assert") +const { slugify } = require("../src/slugify.js") + +test("treats punctuation as separators", () => { + assert.strictEqual(slugify("Hello, World!!"), "hello-world") +}) + +test("trims leading and trailing separators", () => { + assert.strictEqual(slugify(" --A-- "), "a") +}) + +test("returns an empty slug for empty input", () => { + assert.strictEqual(slugify(""), "") +}) +` + +const slugifyCorrect = `// Convert text into a URL slug. +function slugify(text) { + return text + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") +} + +module.exports = { slugify } +` + +const slugifyIncorrect = `function slugify(text) { + return text.toLowerCase().trim().replace(/\\s+/g, "-") +} + +module.exports = { slugify } +` + +// --------------------------------------------------------------------------- +// Fixture 3: parse-csv-line +// --------------------------------------------------------------------------- + +const csvStub = `// Parse one single-line CSV record. +// +// parseCsvLine(line) splits the line on commas. A field that starts with a +// double quote is quoted: it may contain commas and ends at the next double +// quote. Inside a quoted field, a doubled double-quote (\"\") stands for one +// literal double quote. Unquoted fields stay exactly as written, with no +// trimming. + +function parseCsvLine(line) { + throw new Error("TODO: implement parseCsvLine") +} + +module.exports = { parseCsvLine } +` + +const csvVisibleTest = `const { test } = require("node:test") +const assert = require("node:assert") +const { parseCsvLine } = require("../src/parse-csv-line.js") + +test("splits plain fields on commas", () => { + assert.deepStrictEqual(parseCsvLine("a,b,c"), ["a", "b", "c"]) +}) + +test("keeps commas inside quoted fields", () => { + assert.deepStrictEqual(parseCsvLine('"x,y",z'), ["x,y", "z"]) +}) + +test("returns a single field for a bare line", () => { + assert.deepStrictEqual(parseCsvLine("plain"), ["plain"]) +}) +` + +const csvHiddenTest = `const { test } = require("node:test") +const assert = require("node:assert") +const { parseCsvLine } = require("../src/parse-csv-line.js") + +test("unescapes doubled quotes inside quoted fields", () => { + assert.deepStrictEqual(parseCsvLine('"he said ""hi""",ok'), ['he said "hi"', "ok"]) +}) + +test("keeps a trailing empty field", () => { + assert.deepStrictEqual(parseCsvLine("a,"), ["a", ""]) +}) + +test("parses a quoted empty string", () => { + assert.deepStrictEqual(parseCsvLine('""'), [""]) +}) +` + +const csvCorrect = `function parseCsvLine(line) { + const fields = [] + let current = "" + let inQuotes = false + for (let index = 0; index < line.length; index++) { + const ch = line[index] + if (inQuotes) { + if (ch === '"') { + if (line[index + 1] === '"') { + current += '"' + index++ + } else { + inQuotes = false + } + } else { + current += ch + } + continue + } + if (ch === '"' && current.length === 0) { + inQuotes = true + continue + } + if (ch === ",") { + fields.push(current) + current = "" + continue + } + current += ch + } + fields.push(current) + return fields +} + +module.exports = { parseCsvLine } +` + +// Plausible but wrong: quote-aware, without doubled-quote unescaping. +const csvIncorrect = `function parseCsvLine(line) { + const fields = [] + let current = "" + let inQuotes = false + for (const ch of line) { + if (inQuotes) { + if (ch === '"') { + inQuotes = false + continue + } + current += ch + continue + } + if (ch === '"') { + inQuotes = true + continue + } + if (ch === ",") { + fields.push(current) + current = "" + continue + } + current += ch + } + fields.push(current) + return fields +} + +module.exports = { parseCsvLine } +` + +// --------------------------------------------------------------------------- +// Fixture 4: format-bytes +// --------------------------------------------------------------------------- + +const bytesStub = `// Format a byte count with SI decimal units. +// +// formatBytes(bytes) returns "0 B" for zero. Counts below 1000 return the +// integer with the unit "B". From 1000 upward it uses the units kB, MB, GB, +// and TB, dividing by 1000 at each step, always with exactly one decimal. + +function formatBytes(bytes) { + throw new Error("TODO: implement formatBytes") +} + +module.exports = { formatBytes } +` + +const bytesVisibleTest = `const { test } = require("node:test") +const assert = require("node:assert") +const { formatBytes } = require("../src/format-bytes.js") + +test("formats small counts in bytes", () => { + assert.strictEqual(formatBytes(500), "500 B") +}) + +test("formats one decimal kilobytes", () => { + assert.strictEqual(formatBytes(1500), "1.5 kB") +}) + +test("formats exact byte counts without decimals", () => { + assert.strictEqual(formatBytes(123), "123 B") +}) +` + +const bytesHiddenTest = `const { test } = require("node:test") +const assert = require("node:assert") +const { formatBytes } = require("../src/format-bytes.js") + +test("formats zero bytes", () => { + assert.strictEqual(formatBytes(0), "0 B") +}) + +test("uses decimal megabytes, not binary", () => { + assert.strictEqual(formatBytes(2500000), "2.5 MB") +}) + +test("uses decimal gigabytes, not binary", () => { + assert.strictEqual(formatBytes(1000000000), "1.0 GB") +}) + +test("keeps the last byte below the kilobyte boundary", () => { + assert.strictEqual(formatBytes(999), "999 B") +}) +` + +const bytesCorrect = `function formatBytes(bytes) { + if (bytes === 0) return "0 B" + if (bytes < 1000) return \`\${bytes} B\` + const units = ["kB", "MB", "GB", "TB"] + let value = bytes + let unit = -1 + do { + value /= 1000 + unit++ + } while (value >= 1000 && unit < units.length - 1) + return \`\${value.toFixed(1)} \${units[unit]}\` +} + +module.exports = { formatBytes } +` + +// Plausible but wrong: binary divisors and boundaries. +const bytesIncorrect = `function formatBytes(bytes) { + if (bytes === 0) return "0 B" + const units = ["B", "kB", "MB", "GB", "TB"] + let value = bytes + let unit = 0 + while (value >= 1024 && unit < units.length - 1) { + value /= 1024 + unit++ + } + return unit === 0 ? \`\${value} B\` : \`\${value.toFixed(1)} \${units[unit]}\` +} + +module.exports = { formatBytes } +` + +// --------------------------------------------------------------------------- +// Fixture 5: deep-get +// --------------------------------------------------------------------------- + +const deepGetStub = `// Read a value from an object by dot path. +// +// get(object, path) splits the path on dots and walks the object. Numeric +// segments index arrays. A missing key, a missing intermediate object, or a +// null intermediate returns undefined. get never throws on missing data. + +function get(object, path) { + throw new Error("TODO: implement get") +} + +module.exports = { get } +` + +const deepGetVisibleTest = `const { test } = require("node:test") +const assert = require("node:assert") +const { get } = require("../src/deep-get.js") + +test("reads nested objects", () => { + assert.strictEqual(get({ a: { b: 1 } }, "a.b"), 1) +}) + +test("indexes arrays with numeric segments", () => { + assert.strictEqual(get({ a: [10, 20] }, "a.1"), 20) +}) +` + +const deepGetHiddenTest = `const { test } = require("node:test") +const assert = require("node:assert") +const { get } = require("../src/deep-get.js") + +test("returns undefined for a missing path on an empty object", () => { + assert.strictEqual(get({}, "a.b"), undefined) +}) + +test("returns undefined past a non-object intermediate", () => { + assert.strictEqual(get({ a: 1 }, "a.b.c"), undefined) +}) + +test("walks through arrays and objects", () => { + assert.strictEqual(get({ a: [{ b: 3 }] }, "a.0.b"), 3) +}) + +test("returns undefined for an absent root key", () => { + assert.strictEqual(get({ a: { b: 2 } }, "x.y"), undefined) +}) +` + +const deepGetCorrect = `function get(object, path) { + let current = object + for (const segment of path.split(".")) { + if (current === null || current === undefined) return undefined + current = current[segment] + } + return current +} + +module.exports = { get } +` + +// Plausible but wrong: throws on missing intermediates. +const deepGetIncorrect = `function get(object, path) { + return path.split(".").reduce((value, key) => value[key], object) +} + +module.exports = { get } +` + +// --------------------------------------------------------------------------- +// Frozen suite definition. Fixture data is frozen: editing any prompt, file, +// test, or solution changes the suite digest and invalidates comparisons. +// --------------------------------------------------------------------------- + +export const CODING_FIXTURE_BUNDLES: readonly CodingFixtureBundle[] = [ + bundle( + "clamp-range", + "1.0.0", + "Implement clamp(value, min, max) with empty-range semantics; visible tests miss the empty-range case.", + "Implement the clamp function in src/clamp.js so the test suite passes. The header comment in src/clamp.js defines the exact behavior, including the empty range rule. Check your work with npm test. Do not modify the test files or package.json.", + [ + workspacePackageJson("clamp-range"), + { path: "src/clamp.js", content: clampStub }, + { path: "test/clamp.test.js", content: clampVisibleTest }, + ], + [{ path: "test/clamp.acceptance.test.js", content: clampHiddenTest }], + [{ path: "src/clamp.js", content: clampCorrect }], + [{ path: "src/clamp.js", content: clampIncorrect }], + ), + bundle( + "slugify-text", + "1.0.0", + "Implement slugify(text) with punctuation-as-separator semantics; visible tests miss punctuation runs.", + "Implement the slugify function in src/slugify.js so the test suite passes. The header comment in src/slugify.js defines the exact behavior. Check your work with npm test. Do not modify the test files or package.json.", + [ + workspacePackageJson("slugify-text"), + { path: "src/slugify.js", content: slugifyStub }, + { path: "test/slugify.test.js", content: slugifyVisibleTest }, + ], + [{ path: "test/slugify.acceptance.test.js", content: slugifyHiddenTest }], + [{ path: "src/slugify.js", content: slugifyCorrect }], + [{ path: "src/slugify.js", content: slugifyIncorrect }], + ), + bundle( + "parse-csv-line", + "1.0.0", + "Implement parseCsvLine(line) with doubled-quote unescaping; visible tests miss escaped quotes.", + "Implement the parseCsvLine function in src/parse-csv-line.js so the test suite passes. The header comment in src/parse-csv-line.js defines the quoting rules, including doubled quotes. Check your work with npm test. Do not modify the test files or package.json.", + [ + workspacePackageJson("parse-csv-line"), + { path: "src/parse-csv-line.js", content: csvStub }, + { path: "test/parse-csv-line.test.js", content: csvVisibleTest }, + ], + [{ path: "test/parse-csv-line.acceptance.test.js", content: csvHiddenTest }], + [{ path: "src/parse-csv-line.js", content: csvCorrect }], + [{ path: "src/parse-csv-line.js", content: csvIncorrect }], + ), + bundle( + "format-bytes", + "1.0.0", + "Implement formatBytes(bytes) with SI decimal units; visible tests miss zero and the MB and GB boundaries.", + "Implement the formatBytes function in src/format-bytes.js so the test suite passes. The header comment in src/format-bytes.js defines the exact units and rounding. Check your work with npm test. Do not modify the test files or package.json.", + [ + workspacePackageJson("format-bytes"), + { path: "src/format-bytes.js", content: bytesStub }, + { path: "test/format-bytes.test.js", content: bytesVisibleTest }, + ], + [{ path: "test/format-bytes.acceptance.test.js", content: bytesHiddenTest }], + [{ path: "src/format-bytes.js", content: bytesCorrect }], + [{ path: "src/format-bytes.js", content: bytesIncorrect }], + ), + bundle( + "deep-get", + "1.0.0", + "Implement get(object, path) that never throws on missing data; visible tests miss the missing-data paths.", + "Implement the get function in src/deep-get.js so the test suite passes. The header comment in src/deep-get.js defines the missing-data rules. Check your work with npm test. Do not modify the test files or package.json.", + [ + workspacePackageJson("deep-get"), + { path: "src/deep-get.js", content: deepGetStub }, + { path: "test/deep-get.test.js", content: deepGetVisibleTest }, + ], + [{ path: "test/deep-get.acceptance.test.js", content: deepGetHiddenTest }], + [{ path: "src/deep-get.js", content: deepGetCorrect }], + [{ path: "src/deep-get.js", content: deepGetIncorrect }], + ), +] diff --git a/scripts/agent-evals/coding-grader.test.ts b/scripts/agent-evals/coding-grader.test.ts new file mode 100644 index 0000000000..dd60644366 --- /dev/null +++ b/scripts/agent-evals/coding-grader.test.ts @@ -0,0 +1,182 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import { + createLocalSubprocessGradingRuntime, + runGrader, + type DriverCompleted, + type GraderJob, + type GraderVerdict, + type GradingRuntime, +} from "./coding-grader.ts" +import { verdictFromDriver } from "./container-runtime.ts" + +const tmpdir = (): Promise => fs.mkdtemp(path.join(os.tmpdir(), "coding-grader-test-")) + +const writeDriver = async (dir: string, script: string): Promise => { + const file = path.join(dir, "driver.mjs") + await fs.writeFile(file, script) + return file +} + +const driverCompleted = (overrides: Partial = {}): DriverCompleted => ({ + kind: "completed", + passed: true, + noTestsDiscovered: false, + testFiles: [], + passedCount: 3, + failedCount: 0, + failures: [], + durationMs: 1, + ...overrides, +}) + +const staticRuntime = (verdict: GraderVerdict): GradingRuntime => ({ + isolation: "container", + run: async () => verdict, +}) + +const job = async (scratch: string): Promise => { + const gradingDir = path.join(scratch, "grading-copy") + await fs.mkdir(gradingDir, { recursive: true }) + await fs.writeFile(path.join(gradingDir, "marker.txt"), "x") + return { + gradingDir, + driverPath: await writeDriver(scratch, "process.exit(0)"), + resources: { timeoutMs: 5_000, memoryMb: 256, cpus: 1, pidsLimit: 64, image: "node:22-bookworm" }, + } +} + +describe("grader verdict rules", () => { + it("maps driver results to verdicts", () => { + assert.equal(verdictFromDriver(driverCompleted(), "container").kind, "passed") + assert.equal( + verdictFromDriver( + driverCompleted({ passed: false, failedCount: 2, failures: [{ name: "x", file: "y" }] }), + "container", + ).kind, + "failed", + ) + assert.deepEqual(verdictFromDriver(driverCompleted({ noTestsDiscovered: true }), "container"), { + kind: "grading-failure", + isolation: "container", + reason: "noTestsDiscovered", + }) + assert.deepEqual(verdictFromDriver({ kind: "driver-error", message: "boom" }, "container"), { + kind: "grading-failure", + isolation: "container", + reason: "driverError", + }) + }) +}) + +describe("runGrader", () => { + it("builds a grading copy and removes it afterwards", async () => { + const scratch = await tmpdir() + try { + let sawGradingDir = false + const runtime: GradingRuntime = { + isolation: "container", + run: async (input) => { + sawGradingDir = await fs + .access(path.join(input.gradingDir, "marker.txt")) + .then(() => true) + .catch(() => false) + return { kind: "passed", isolation: "container", result: driverCompleted() } + }, + } + const verdict = await runGrader({ + finalFiles: [{ path: "marker.txt", content: "x" }], + hiddenTests: [], + resources: { timeoutMs: 1_000, memoryMb: 256, cpus: 1, pidsLimit: 64, image: "node:22-bookworm" }, + runtime, + scratchDir: scratch, + }) + assert.equal(verdict.kind, "passed") + assert.equal(sawGradingDir, true) + const gradingCopyGone = await fs + .access(path.join(scratch, "grading-copy")) + .then(() => false) + .catch(() => true) + assert.equal(gradingCopyGone, true) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) + + it("reports a failed grading copy as grading failure", async () => { + const scratch = await tmpdir() + try { + const verdict = await runGrader({ + finalFiles: [{ path: "../escape.txt", content: "x" }], + hiddenTests: [], + resources: { timeoutMs: 1_000, memoryMb: 256, cpus: 1, pidsLimit: 64, image: "node:22-bookworm" }, + runtime: staticRuntime({ kind: "passed", isolation: "container", result: driverCompleted() }), + scratchDir: scratch, + }) + assert.deepEqual(verdict, { + kind: "grading-failure", + isolation: "container", + reason: "gradingCopyFailed", + }) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) +}) + +describe("local subprocess runtime failure paths", () => { + it("reports a hung grader as graderTimeout", async () => { + const scratch = await tmpdir() + try { + const base = await job(scratch) + const driverPath = await writeDriver(scratch, "setInterval(() => {}, 1000)") + const verdict = await createLocalSubprocessGradingRuntime().run({ + ...base, + driverPath, + resources: { ...base.resources, timeoutMs: 300 }, + }) + assert.deepEqual(verdict, { + kind: "grading-failure", + isolation: "local-subprocess", + reason: "graderTimeout", + }) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) + + it("reports a crashed grader as graderCrashed", async () => { + const scratch = await tmpdir() + try { + const base = await job(scratch) + const driverPath = await writeDriver(scratch, "process.exit(3)") + const verdict = await createLocalSubprocessGradingRuntime().run({ ...base, driverPath }) + assert.deepEqual(verdict, { + kind: "grading-failure", + isolation: "local-subprocess", + reason: "graderCrashed", + }) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) + + it("reports a silent grader as graderResultMissing", async () => { + const scratch = await tmpdir() + try { + const base = await job(scratch) + const driverPath = await writeDriver(scratch, "process.exit(0)") + const verdict = await createLocalSubprocessGradingRuntime().run({ ...base, driverPath }) + assert.deepEqual(verdict, { + kind: "grading-failure", + isolation: "local-subprocess", + reason: "graderResultMissing", + }) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) +}) diff --git a/scripts/agent-evals/coding-grader.ts b/scripts/agent-evals/coding-grader.ts new file mode 100644 index 0000000000..3950bc9168 --- /dev/null +++ b/scripts/agent-evals/coding-grader.ts @@ -0,0 +1,132 @@ +import { spawn } from "node:child_process" +import { fileURLToPath } from "node:url" +import fs from "node:fs/promises" +import path from "node:path" +import type { AgentFixtureWorkspaceFile } from "./contract.ts" +import { buildGradingCopy } from "./coding-workspace.ts" +import { readDriverResult, verdictFromDriver } from "./container-runtime.ts" + +// The grader version is a pinned comparison input. Any change to the driver, +// the image, or the verdict rules must bump it. +export const GRADER_VERSION = "coding-grader-v1" + +export interface GraderResources { + timeoutMs: number + memoryMb: number + cpus: number + pidsLimit: number + image: string +} + +export interface DriverCompleted { + kind: "completed" + passed: boolean + noTestsDiscovered: boolean + testFiles: string[] + passedCount: number + failedCount: number + failures: { name: string; file: string }[] + durationMs: number +} + +export interface DriverError { + kind: "driver-error" + message: string +} + +export type DriverResult = DriverCompleted | DriverError + +export interface GraderJob { + // Fresh copy of the final workspace plus the hidden acceptance tests. + gradingDir: string + driverPath: string + resources: GraderResources +} + +export type GradingIsolation = "container" | "local-subprocess" + +export type GraderVerdict = + | { kind: "passed"; isolation: GradingIsolation; result: DriverCompleted } + | { kind: "failed"; isolation: GradingIsolation; result: DriverCompleted } + | { kind: "grading-failure"; isolation: GradingIsolation; reason: string } + +export interface GradingRuntime { + readonly isolation: GradingIsolation + run(job: GraderJob): Promise +} + +export const driverFilePath = (): string => fileURLToPath(new URL("./grader-driver.mjs", import.meta.url)) + +// Runs one grading job. The grader: +// - works on a fresh copy of the final workspace, never the original, +// - adds hidden acceptance tests only to that copy, +// - executes through the given runtime (disposable container by default), +// - has no credential and no network access in either runtime, +// - reports a hung, crashed, or over-limit grader as a grading failure. +export async function runGrader(inputs: { + finalFiles: readonly AgentFixtureWorkspaceFile[] + hiddenTests: readonly AgentFixtureWorkspaceFile[] + resources: GraderResources + runtime: GradingRuntime + scratchDir: string +}): Promise { + const gradingDir = path.join(inputs.scratchDir, "grading-copy") + try { + await buildGradingCopy(inputs.finalFiles, inputs.hiddenTests, gradingDir) + return await inputs.runtime.run({ + gradingDir, + driverPath: driverFilePath(), + resources: inputs.resources, + }) + } catch { + return { kind: "grading-failure", isolation: inputs.runtime.isolation, reason: "gradingCopyFailed" } + } finally { + await fs.rm(gradingDir, { recursive: true, force: true }) + } +} + +const INPROCESS_ENV_ALLOWLIST = ["PATH", "SYSTEMROOT", "WINDIR", "COMSPEC", "PATHEXT"] as const + +// Local subprocess runtime. It runs the same driver directly on the host with +// a minimal environment and the same wall-clock kill. It has NO container +// isolation: agent-written code runs with the host filesystem. It exists for +// unit tests and explicit local checks only. It does not give isolation equal +// to the container runtime and must never back a recorded comparison. +export function createLocalSubprocessGradingRuntime(nodeExecutable = process.execPath): GradingRuntime { + return { + isolation: "local-subprocess", + async run(job: GraderJob): Promise { + const outDir = path.join(path.dirname(job.gradingDir), "grading-out") + const resultPath = path.join(outDir, "result.json") + await fs.mkdir(outDir, { recursive: true }) + try { + const exit = await new Promise((resolve) => { + const env: NodeJS.ProcessEnv = {} + for (const name of INPROCESS_ENV_ALLOWLIST) { + const value = process.env[name] + if (value !== undefined) env[name] = value + } + const child = spawn(nodeExecutable, [job.driverPath, resultPath, "test"], { + cwd: job.gradingDir, + env, + stdio: "ignore", + timeout: job.resources.timeoutMs, + killSignal: "SIGKILL", + }) + child.once("error", () => resolve("error")) + child.once("close", (code, signal) => resolve(signal === "SIGKILL" ? "timeout" : (code ?? "error"))) + }) + if (exit === "timeout") + return { kind: "grading-failure", isolation: "local-subprocess", reason: "graderTimeout" } + if (exit === "error" || exit !== 0) + return { kind: "grading-failure", isolation: "local-subprocess", reason: "graderCrashed" } + const result = await readDriverResult(resultPath) + if (!result) + return { kind: "grading-failure", isolation: "local-subprocess", reason: "graderResultMissing" } + return verdictFromDriver(result, "local-subprocess") + } finally { + await fs.rm(outDir, { recursive: true, force: true }) + } + }, + } +} diff --git a/scripts/agent-evals/coding-metrics.test.ts b/scripts/agent-evals/coding-metrics.test.ts new file mode 100644 index 0000000000..9b20ff52dd --- /dev/null +++ b/scripts/agent-evals/coding-metrics.test.ts @@ -0,0 +1,305 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { buildRunMetrics, costPerSuccessfulTask, type CodingRunMetrics } from "./coding-metrics.ts" +import type { PriceTable } from "./coding-pricing.ts" +import type { AgentTrace, TraceAccounting } from "./trace.ts" +import { TRACE_KIND, TRACE_SCHEMA_VERSION, identityUnavailable } from "./trace.ts" +import type { ClassifiedOutcome } from "./coding-outcomes.ts" + +const correctOutcome: ClassifiedOutcome = { outcome: "correct", reason: "hiddenTestsPassed" } + +const accounting = (overrides: Partial = {}): TraceAccounting => ({ + requests: [], + requestCount: 0, + requestsMissingUsage: 0, + apiReqRetriedMarkers: 0, + usage: { available: false, snapshotsObserved: 0 }, + toolSnapshot: {}, + toolFailureEvents: 0, + toolFailuresByTool: {}, + completeness: { status: "complete", reasons: [] }, + ...overrides, +}) + +const trace = (overrides: Partial = {}): AgentTrace => ({ + traceSchemaVersion: TRACE_SCHEMA_VERSION, + kind: TRACE_KIND, + run: { + runId: "run-1", + fixture: { name: "clamp-range", version: "1.0.0" }, + startedAt: "2026-10-22T00:00:00.000Z", + finishedAt: "2026-10-22T00:01:00.000Z", + durationMs: 60_000, + identity: identityUnavailable(), + }, + settings: { available: true }, + execution: { + code: "COMPLETED", + stopReason: "taskCompleted", + activated: true, + taskAccepted: true, + sessionLost: false, + completionMessageSeen: true, + timings: { totalMs: 60_000 }, + }, + eventFlow: { + observed: 0, + malformed: 0, + foreignTaskEvents: 0, + bufferedBeforeAck: 0, + replayedAfterAck: 0, + eventBufferLimit: 256, + bufferOverflow: false, + droppedByOverflow: 0, + }, + accounting: accounting(), + teardown: { + status: "ok", + hostExited: true, + closeTaskSent: true, + escalatedSignals: [], + cleanup: "done", + failures: [], + }, + notes: [], + ...overrides, +}) + +const fullUsage = { + tokensIn: 1_000_000, + tokensOut: 100_000, + cacheWrites: { available: true, value: 200_000 } as const, + cacheReads: { available: true, value: 400_000 } as const, + cost: 0.42, + contextTokens: 50_000, + source: "taskCompleted" as const, + observedAtMs: 1, +} + +const testTable: PriceTable = { + id: "test-price-table", + date: "2026-01-01", + currency: "USD", + cachePricing: "test rates", + models: { + "qwen3.8-27b": { + inputPerMillion: 0.5, + outputPerMillion: 1.5, + cacheReadPerMillion: 0.125, + cacheWritePerMillion: 0.625, + }, + }, +} + +describe("run metrics", () => { + it("keeps missing token values missing and never writes zero", () => { + const metrics = buildRunMetrics({ + fixtureName: "clamp-range", + repIndex: 0, + trace: trace(), + outcome: correctOutcome, + modelIdentifier: { available: true, value: "qwen3.8-27b" }, + }) + assert.equal(metrics.usage.tokensIn, undefined) + assert.equal(metrics.cost.estimated.available, false) + if (!metrics.cost.estimated.available) assert.equal(metrics.cost.estimated.reason, "noUsageEventsObserved") + assert.equal(metrics.cost.accountingComplete, false) + }) + + it("estimates cost from reported usage with an explicitly configured price table", () => { + const metrics = buildRunMetrics({ + fixtureName: "clamp-range", + repIndex: 0, + trace: trace({ + accounting: accounting({ usage: { available: true, snapshotsObserved: 1, lastSnapshot: fullUsage } }), + }), + outcome: correctOutcome, + modelIdentifier: { available: true, value: "qwen3.8-27b" }, + priceTable: testTable, + }) + assert.equal(metrics.cost.estimated.available, true) + if (metrics.cost.estimated.available) { + // 1.0 * 0.5 + 0.1 * 1.5 + 0.4 * 0.125 + 0.2 * 0.625 = 0.825 + assert.ok(Math.abs(metrics.cost.estimated.usd - 0.825) < 1e-9) + assert.equal(metrics.cost.estimated.priceTableId, "test-price-table") + assert.ok(metrics.cost.estimated.cachePricing.length > 0) + } + assert.deepEqual(metrics.cost.providerReported, { available: true, value: 0.42 }) + assert.equal(metrics.cost.accountingComplete, true) + assert.equal(metrics.usage.cacheReads, 400_000) + assert.equal(metrics.usage.contextTokensEstimate, 50_000) + assert.equal(metrics.usage.labels.context, "extensionContextEstimate") + }) + + it("keeps cost unavailable while the shipped table has no rates", () => { + const metrics = buildRunMetrics({ + fixtureName: "clamp-range", + repIndex: 0, + trace: trace({ + accounting: accounting({ usage: { available: true, snapshotsObserved: 1, lastSnapshot: fullUsage } }), + }), + outcome: correctOutcome, + modelIdentifier: { available: true, value: "qwen3.8-27b" }, + }) + assert.equal(metrics.cost.estimated.available, false) + if (!metrics.cost.estimated.available) assert.equal(metrics.cost.estimated.reason, "modelMissingFromPriceTable") + assert.equal(metrics.cost.accountingComplete, false) + }) + + it("marks accounting incomplete when a cache count is missing", () => { + const metrics = buildRunMetrics({ + fixtureName: "clamp-range", + repIndex: 0, + trace: trace({ + accounting: accounting({ + usage: { + available: true, + snapshotsObserved: 1, + lastSnapshot: { ...fullUsage, cacheWrites: { available: false, reason: "notReported" } }, + }, + }), + }), + outcome: correctOutcome, + modelIdentifier: { available: true, value: "qwen3.8-27b" }, + priceTable: testTable, + }) + assert.equal(metrics.cost.estimated.available, false) + if (!metrics.cost.estimated.available) assert.equal(metrics.cost.estimated.reason, "cacheWritesNotReported") + assert.equal(metrics.cost.accountingComplete, false) + }) + + it("summarizes tool calls and failures from the tool snapshot", () => { + const metrics = buildRunMetrics({ + fixtureName: "clamp-range", + repIndex: 0, + trace: trace({ + accounting: accounting({ + toolSnapshot: { + snapshot: { read_file: { attempts: 4, failures: 1 }, apply_diff: { attempts: 2, failures: 0 } }, + }, + toolFailureEvents: 1, + toolFailuresByTool: { read_file: 1 }, + }), + }), + outcome: correctOutcome, + modelIdentifier: { available: true, value: "qwen3.8-27b" }, + }) + assert.deepEqual(metrics.toolCalls.totalAttempts, { available: true, value: 6 }) + assert.equal(metrics.toolCalls.totalFailures, 1) + assert.equal(metrics.toolCalls.failureEvents, 1) + assert.equal(metrics.toolCalls.byTool["apply_diff"]?.attempts, 2) + }) + + it("reports unsupported telemetry as unavailable", () => { + const metrics = buildRunMetrics({ + fixtureName: "clamp-range", + repIndex: 0, + trace: trace(), + outcome: correctOutcome, + modelIdentifier: { available: true, value: "qwen3.8-27b" }, + }) + assert.deepEqual(metrics.unavailableTelemetry, [ + { name: "duplicateCallDetection", reason: "notInstrumented" }, + { name: "httpRetryCount", reason: "apiReqRetriedMarkersCountsMessagesNotHttpRetries" }, + ]) + }) + + it("labels per-request prompt size rows", () => { + const metrics = buildRunMetrics({ + fixtureName: "clamp-range", + repIndex: 0, + trace: trace({ + accounting: accounting({ + requestCount: 2, + requests: [ + { + index: 0, + ts: 1, + apiProtocol: { available: false, reason: "notReported" }, + tokensIn: { available: true, value: 1_234 }, + tokensOut: { available: false, reason: "notReported" }, + cacheWrites: { available: false, reason: "notReported" }, + cacheReads: { available: false, reason: "notReported" }, + cost: { available: false, reason: "notReported" }, + streamingFailed: false, + firstObservedMs: 0, + lastUpdatedMs: 1, + messageCount: 1, + }, + ], + }), + }), + outcome: correctOutcome, + modelIdentifier: { available: true, value: "qwen3.8-27b" }, + }) + assert.deepEqual(metrics.requests.promptSize, [ + { + index: 0, + reportedInputTokens: { available: true, value: 1_234 }, + label: "providerReportedInputTokens", + }, + ]) + }) +}) + +const metricsRun = (overrides: Partial & { repIndex?: number }): CodingRunMetrics => ({ + fixtureName: "clamp-range", + repIndex: overrides.repIndex ?? 0, + outcome: correctOutcome, + requests: { count: 0, promptSize: [], requestsMissingUsage: 0 }, + usage: { labels: { context: "extensionContextEstimate" } }, + toolCalls: { byTool: {}, totalAttempts: { available: false, reason: "none" }, totalFailures: 0, failureEvents: 0 }, + cost: { + estimated: { available: false, reason: "noUsageEventsObserved" }, + providerReported: { available: false, reason: "noUsageEventsObserved" }, + accountingComplete: false, + }, + accounting: { status: "complete", reasons: [] }, + unavailableTelemetry: [], + ...overrides, +}) + +describe("cost per successful task", () => { + const completeCost = (usd: number) => ({ + estimated: { available: true, usd, priceTableId: "t", priceTableDate: "d", cachePricing: "c" } as const, + providerReported: { available: false, reason: "none" } as const, + accountingComplete: true, + }) + + it("divides the total attempt cost by the correct run count", () => { + const runs: CodingRunMetrics[] = [ + metricsRun({ repIndex: 0, cost: completeCost(0.5) }), + metricsRun({ + repIndex: 1, + outcome: { outcome: "incorrect", reason: "hiddenTestsFailed" }, + cost: completeCost(0.3), + }), + metricsRun({ repIndex: 2, cost: completeCost(0.4) }), + ] + const result = costPerSuccessfulTask(runs) + assert.equal(result.available, true) + assert.ok(Math.abs((result.usd ?? 0) - 0.6) < 1e-9) + assert.equal(result.correctRuns, 2) + assert.equal(result.includedRuns, 3) + assert.equal(result.excludedRuns, 0) + }) + + it("is unavailable with no correct runs", () => { + const result = costPerSuccessfulTask([ + metricsRun({ outcome: { outcome: "incorrect", reason: "x" }, cost: completeCost(1) }), + ]) + assert.equal(result.available, false) + assert.equal(result.reason, "noCorrectRuns") + }) + + it("is unavailable when a correct run lacks complete cost and flags the exclusion", () => { + const result = costPerSuccessfulTask([ + metricsRun({ repIndex: 0, cost: completeCost(1) }), + metricsRun({ repIndex: 1 }), + ]) + assert.equal(result.available, false) + assert.equal(result.reason, "correctRunCostIncomplete") + assert.equal(result.excludedRuns, 1) + assert.equal(result.policy, "excludeIncompleteFromAggregates-v1") + }) +}) diff --git a/scripts/agent-evals/coding-metrics.ts b/scripts/agent-evals/coding-metrics.ts new file mode 100644 index 0000000000..3b8898075e --- /dev/null +++ b/scripts/agent-evals/coding-metrics.ts @@ -0,0 +1,194 @@ +import type { AgentTrace, Measured } from "./trace.ts" +import type { GraderVerdict } from "./coding-grader.ts" +import type { ClassifiedOutcome } from "./coding-outcomes.ts" +import { estimateRunCost, type EstimatedCost, type PriceTable } from "./coding-pricing.ts" + +export const PARTIAL_DATA_POLICY = "excludeIncompleteFromAggregates-v1" + +export interface PromptSizeRow { + index: number + // Provider-reported input tokens for the request. + reportedInputTokens: Measured + label: "providerReportedInputTokens" +} + +export interface ToolCallSummary { + byTool: Record + totalAttempts: Measured + totalFailures: number + failureEvents: number +} + +export interface RunCostMetric { + estimated: EstimatedCost + // Provider-reported cumulative cost from extension events. Independent of + // the estimate and never used as a fallback estimate. + providerReported: Measured + accountingComplete: boolean +} + +export interface CodingRunMetrics { + fixtureName: string + repIndex: number + outcome: ClassifiedOutcome + executionCode?: string + stopReason?: string + durationMs?: number + requests: { count: number; promptSize: PromptSizeRow[]; requestsMissingUsage: number } + usage: { + tokensIn?: number + tokensOut?: number + cacheReads?: number + cacheWrites?: number + // Extension context estimate. Not a provider measurement. + contextTokensEstimate?: number + labels: { context: "extensionContextEstimate" } + } + toolCalls: ToolCallSummary + cost: RunCostMetric + accounting: { status: "complete" | "partial"; reasons: string[] } + grading?: { + kind: GraderVerdict["kind"] + isolation?: string + failedCount?: number + failures?: { name: string; file: string }[] + } + unavailableTelemetry: { name: string; reason: string }[] +} + +// Telemetry the substrate does not instrument. Reported as unavailable, never +// inferred from unrelated events. +export const UNSUPPORTED_TELEMETRY = [ + { name: "duplicateCallDetection", reason: "notInstrumented" }, + { name: "httpRetryCount", reason: "apiReqRetriedMarkersCountsMessagesNotHttpRetries" }, +] as const + +const measuredOrUndefined = (value: Measured): T | undefined => (value.available ? value.value : undefined) + +export function buildRunMetrics(input: { + fixtureName: string + repIndex: number + trace?: AgentTrace + grading?: GraderVerdict + outcome: ClassifiedOutcome + modelIdentifier: Measured + priceTable?: PriceTable +}): CodingRunMetrics { + const trace = input.trace + const accounting = trace?.accounting + const last = accounting?.usage.lastSnapshot + const estimated: EstimatedCost = last + ? estimateRunCost( + input.modelIdentifier.available ? input.modelIdentifier.value : "", + { available: true, value: last.tokensIn }, + { available: true, value: last.tokensOut }, + last.cacheReads, + last.cacheWrites, + input.priceTable, + ) + : { available: false, reason: last ? "modelIdentifierMissing" : "noUsageEventsObserved" } + const snapshot = accounting?.toolSnapshot.snapshot + const byTool: Record = {} + let totalAttempts = 0 + let totalFailures = 0 + for (const [tool, counts] of Object.entries(snapshot ?? {})) { + byTool[tool] = { attempts: counts.attempts, failures: counts.failures } + totalAttempts += counts.attempts + totalFailures += counts.failures + } + const cost: RunCostMetric = { + estimated, + providerReported: last + ? { available: true, value: last.cost } + : { available: false, reason: "noUsageEventsObserved" }, + accountingComplete: estimated.available, + } + return { + fixtureName: input.fixtureName, + repIndex: input.repIndex, + outcome: input.outcome, + executionCode: trace?.execution.code, + stopReason: trace?.execution.stopReason, + durationMs: trace?.run.durationMs, + requests: { + count: accounting?.requestCount ?? 0, + promptSize: (accounting?.requests ?? []).map((row) => ({ + index: row.index, + reportedInputTokens: row.tokensIn, + label: "providerReportedInputTokens", + })), + requestsMissingUsage: accounting?.requestsMissingUsage ?? 0, + }, + usage: { + tokensIn: last?.tokensIn, + tokensOut: last?.tokensOut, + cacheReads: measuredOrUndefined(last?.cacheReads ?? { available: false, reason: "noUsageEventsObserved" }), + cacheWrites: measuredOrUndefined( + last?.cacheWrites ?? { available: false, reason: "noUsageEventsObserved" }, + ), + contextTokensEstimate: last?.contextTokens, + labels: { context: "extensionContextEstimate" }, + }, + toolCalls: { + byTool, + totalAttempts: totalAttempts + ? { available: true, value: totalAttempts } + : { available: false, reason: "noToolSnapshotObserved" }, + totalFailures, + failureEvents: accounting?.toolFailureEvents ?? 0, + }, + cost, + accounting: { + status: accounting?.completeness.status ?? "partial", + reasons: accounting?.completeness.reasons ?? ["runTraceMissing"], + }, + grading: input.grading + ? { + kind: input.grading.kind, + isolation: input.grading.isolation, + failedCount: + input.grading.kind === "passed" || input.grading.kind === "failed" + ? input.grading.result.failedCount + : undefined, + failures: input.grading.kind === "failed" ? input.grading.result.failures.slice(0, 10) : undefined, + } + : undefined, + unavailableTelemetry: UNSUPPORTED_TELEMETRY.map((entry) => ({ ...entry })), + } +} + +export interface CostPerSuccessfulTask { + policy: typeof PARTIAL_DATA_POLICY + includedRuns: number + excludedRuns: number + correctRuns: number + available: boolean + reason?: string + usd?: number +} + +// Total estimated cost of all attempts with complete cost accounting, divided +// by the number of correct runs. Runs with incomplete cost accounting are +// flagged and excluded from aggregates. If no run is correct, or if any +// correct run lacks complete cost accounting, the value is unavailable. +export function costPerSuccessfulTask(runs: readonly CodingRunMetrics[]): CostPerSuccessfulTask { + const included = runs.filter((run) => run.cost.accountingComplete) + const excluded = runs.length - included.length + const correctRuns = runs.filter((run) => run.outcome.outcome === "correct").length + const base: Pick = { + policy: PARTIAL_DATA_POLICY, + includedRuns: included.length, + excludedRuns: excluded, + correctRuns, + } + if (correctRuns === 0) return { ...base, available: false, reason: "noCorrectRuns" } + const correctIncomplete = runs.filter( + (run) => run.outcome.outcome === "correct" && !run.cost.accountingComplete, + ).length + if (correctIncomplete > 0) return { ...base, available: false, reason: "correctRunCostIncomplete" } + const total = included.reduce((sum, run) => { + if (run.cost.estimated.available) return sum + run.cost.estimated.usd + return sum + }, 0) + return { ...base, available: true, usd: total / correctRuns } +} diff --git a/scripts/agent-evals/coding-outcomes.test.ts b/scripts/agent-evals/coding-outcomes.test.ts new file mode 100644 index 0000000000..6b6dbf57fa --- /dev/null +++ b/scripts/agent-evals/coding-outcomes.test.ts @@ -0,0 +1,60 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { classifyRunOutcome } from "./coding-outcomes.ts" + +const base = { + hasTrace: true, + executionCode: "COMPLETED" as const, + finalWorkspaceAvailable: true, +} + +describe("run outcome classification", () => { + it("grades a completed run by the hidden tests", () => { + assert.deepEqual(classifyRunOutcome({ ...base, grading: { kind: "passed" } }), { + outcome: "correct", + reason: "hiddenTestsPassed", + }) + // A completion message from the agent never makes a failing workspace correct. + assert.deepEqual(classifyRunOutcome({ ...base, grading: { kind: "failed" } }), { + outcome: "incorrect", + reason: "hiddenTestsFailed", + }) + }) + + it("grades a stopped run by the hidden tests", () => { + assert.equal( + classifyRunOutcome({ ...base, executionCode: "STOPPED", grading: { kind: "passed" } }).outcome, + "correct", + ) + assert.equal( + classifyRunOutcome({ ...base, executionCode: "STOPPED", grading: { kind: "failed" } }).outcome, + "incorrect", + ) + }) + + it("keeps grading failures out of incorrect", () => { + assert.deepEqual( + classifyRunOutcome({ ...base, grading: { kind: "grading-failure", reason: "graderTimeout" } }), + { outcome: "grading-failure", reason: "graderTimeout" }, + ) + }) + + it("reports timed out for the run-window expiry", () => { + assert.equal(classifyRunOutcome({ ...base, executionCode: "TASK_RUN_TIMEOUT" }).outcome, "timed-out") + assert.equal( + classifyRunOutcome({ ...base, executionCode: "TASK_RUN_TIMEOUT", grading: { kind: "passed" } }).outcome, + "timed-out", + ) + }) + + it("reports infrastructure failures for host, IPC, and provider faults", () => { + for (const code of ["SETUP_FAILED", "ACTIVATION_FAILED", "TASK_START_FAILED", "SESSION_LOST"] as const) + assert.equal(classifyRunOutcome({ ...base, executionCode: code }).outcome, "infrastructure-failure") + assert.equal(classifyRunOutcome({ ...base, hasTrace: false }).outcome, "infrastructure-failure") + assert.equal( + classifyRunOutcome({ ...base, grading: undefined, finalWorkspaceAvailable: false }).outcome, + "infrastructure-failure", + ) + assert.equal(classifyRunOutcome({ ...base, grading: undefined }).outcome, "infrastructure-failure") + }) +}) diff --git a/scripts/agent-evals/coding-outcomes.ts b/scripts/agent-evals/coding-outcomes.ts new file mode 100644 index 0000000000..74738edc02 --- /dev/null +++ b/scripts/agent-evals/coding-outcomes.ts @@ -0,0 +1,54 @@ +import type { AgentExecutionCode } from "./trace.ts" +import type { GraderVerdict } from "./coding-grader.ts" + +export type CodingRunOutcome = "correct" | "incorrect" | "timed-out" | "grading-failure" | "infrastructure-failure" + +export interface OutcomeInput { + hasTrace: boolean + executionCode?: AgentExecutionCode + finalWorkspaceAvailable: boolean + grading?: Pick & { reason?: string } +} + +export interface ClassifiedOutcome { + outcome: CodingRunOutcome + reason: string +} + +const INFRASTRUCTURE_CODES: readonly AgentExecutionCode[] = [ + "SETUP_FAILED", + "ACTIVATION_FAILED", + "TASK_START_FAILED", + "SESSION_LOST", +] + +// Outcome rules, in order: +// - A missing trace is infrastructure failure. Nothing about the task ran. +// - Setup, activation, start, and session-loss codes are infrastructure +// failure. VS Code, IPC, or the provider failed; the agent did not. +// - A run-window expiry is timed out. The agent hit an execution limit. +// - COMPLETED and STOPPED runs are graded. The hidden tests decide: +// passed is correct, failed is incorrect, including when the agent +// reported completion. +// - A grader that hung, crashed, or hit its limits is grading failure and is +// never counted as incorrect. +export function classifyRunOutcome(input: OutcomeInput): ClassifiedOutcome { + if (!input.hasTrace) return { outcome: "infrastructure-failure", reason: "runTraceMissing" } + const code = input.executionCode + if (code && INFRASTRUCTURE_CODES.includes(code)) + return { outcome: "infrastructure-failure", reason: `execution-${code}` } + if (code === "TASK_RUN_TIMEOUT") return { outcome: "timed-out", reason: "agentHitRunLimit" } + if (code !== "COMPLETED" && code !== "STOPPED") + return { outcome: "infrastructure-failure", reason: `execution-${code ?? "unknown"}` } + if (!input.finalWorkspaceAvailable) + return { outcome: "infrastructure-failure", reason: "finalWorkspaceUnavailable" } + if (!input.grading) return { outcome: "infrastructure-failure", reason: "graderNotRun" } + switch (input.grading.kind) { + case "passed": + return { outcome: "correct", reason: "hiddenTestsPassed" } + case "failed": + return { outcome: "incorrect", reason: "hiddenTestsFailed" } + case "grading-failure": + return { outcome: "grading-failure", reason: input.grading.reason ?? "gradingFailure" } + } +} diff --git a/scripts/agent-evals/coding-pins.test.ts b/scripts/agent-evals/coding-pins.test.ts new file mode 100644 index 0000000000..10a8bd2179 --- /dev/null +++ b/scripts/agent-evals/coding-pins.test.ts @@ -0,0 +1,46 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { comparePins, type SidePins } from "./coding-pins.ts" + +const pins = (side: "baseline" | "candidate", overrides: Partial = {}): SidePins => ({ + side, + commit: side === "baseline" ? "a".repeat(40) : "b".repeat(40), + suiteId: "coding-basic-v1", + suiteDigest: "d1".repeat(32), + settingsDigest: "d2".repeat(32), + modelIdentifier: "qwen3.8-27b", + graderVersion: "coding-grader-v1", + runnerEnvironment: { os: "linux", nodeVersion: "v22.23.1", vscodeTarget: "1.104.0" }, + ...overrides, +}) + +describe("pinned comparison inputs", () => { + it("accepts identical pins with different commits", () => { + const result = comparePins(pins("baseline"), pins("candidate")) + assert.equal(result.compatible, true) + assert.deepEqual(result.differences, []) + }) + + it("flags each differing pinned value", () => { + const cases: Partial[] = [ + { suiteId: "coding-basic-v2" }, + { suiteDigest: "zz".repeat(32) }, + { settingsDigest: "zz".repeat(32) }, + { modelIdentifier: "other-model" }, + { graderVersion: "coding-grader-v2" }, + { runnerEnvironment: { os: "darwin", nodeVersion: "v22.23.1", vscodeTarget: "1.104.0" } }, + { runnerEnvironment: { os: "linux", nodeVersion: "v24.0.0", vscodeTarget: "1.104.0" } }, + { runnerEnvironment: { os: "linux", nodeVersion: "v22.23.1", vscodeTarget: "1.100.0" } }, + ] + for (const overrides of cases) { + const result = comparePins(pins("baseline"), pins("candidate", overrides)) + assert.equal(result.compatible, false, `expected incompatibility for ${JSON.stringify(overrides)}`) + assert.ok(result.differences.length >= 1) + } + }) + + it("never compares the commit SHAs themselves", () => { + const result = comparePins(pins("baseline"), pins("candidate", { commit: "c".repeat(40) })) + assert.equal(result.compatible, true) + }) +}) diff --git a/scripts/agent-evals/coding-pins.ts b/scripts/agent-evals/coding-pins.ts new file mode 100644 index 0000000000..3433d11bfd --- /dev/null +++ b/scripts/agent-evals/coding-pins.ts @@ -0,0 +1,34 @@ +export interface SidePins { + side: "baseline" | "candidate" + commit: string + suiteId: string + suiteDigest: string + settingsDigest: string + modelIdentifier: string + graderVersion: string + runnerEnvironment: { + os: string + nodeVersion: string + vscodeTarget: string + } +} + +export interface PinComparison { + compatible: boolean + differences: string[] +} + +// Pinned comparison inputs. The two commit SHAs differ by design and are +// recorded, never compared. Every other pinned value must match on both +// sides; any difference marks the pair incompatible. +const COMPARED_FIELDS = ["suiteId", "suiteDigest", "settingsDigest", "modelIdentifier", "graderVersion"] as const + +export function comparePins(baseline: SidePins, candidate: SidePins): PinComparison { + const differences: string[] = [] + for (const field of COMPARED_FIELDS) if (baseline[field] !== candidate[field]) differences.push(field) + const envFields = ["os", "nodeVersion", "vscodeTarget"] as const + for (const field of envFields) + if (baseline.runnerEnvironment[field] !== candidate.runnerEnvironment[field]) + differences.push(`runnerEnvironment.${field}`) + return { compatible: differences.length === 0, differences } +} diff --git a/scripts/agent-evals/coding-pricing.test.ts b/scripts/agent-evals/coding-pricing.test.ts new file mode 100644 index 0000000000..d5fabe09d8 --- /dev/null +++ b/scripts/agent-evals/coding-pricing.test.ts @@ -0,0 +1,53 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { PRICE_TABLE, estimateRunCost } from "./coding-pricing.ts" +import { CODING_MODEL_ID } from "./coding-fixtures.ts" + +const reported = (value: number) => ({ available: true as const, value }) + +describe("shipped price table", () => { + it("contains no rates, so no placeholder price can look measured", () => { + assert.deepEqual(Object.keys(PRICE_TABLE.models), []) + }) + + it("keeps the suite model cost unavailable until real rates are configured", () => { + const cost = estimateRunCost( + CODING_MODEL_ID, + reported(1_000_000), + reported(100_000), + reported(400_000), + reported(200_000), + ) + assert.equal(cost.available, false) + if (!cost.available) assert.equal(cost.reason, "modelMissingFromPriceTable") + }) + + it("estimates from an explicitly configured table", () => { + const cost = estimateRunCost( + CODING_MODEL_ID, + reported(1_000_000), + reported(100_000), + reported(400_000), + reported(200_000), + { + id: "configured", + date: "2026-01-01", + currency: "USD", + cachePricing: "separate cache rates", + models: { + [CODING_MODEL_ID]: { + inputPerMillion: 1, + outputPerMillion: 2, + cacheReadPerMillion: 0.5, + cacheWritePerMillion: 0.25, + }, + }, + }, + ) + assert.equal(cost.available, true) + if (cost.available) { + assert.ok(Math.abs(cost.usd - (1 + 0.2 + 0.2 + 0.05)) < 1e-9) + assert.equal(cost.priceTableId, "configured") + } + }) +}) diff --git a/scripts/agent-evals/coding-pricing.ts b/scripts/agent-evals/coding-pricing.ts new file mode 100644 index 0000000000..74d81e001b --- /dev/null +++ b/scripts/agent-evals/coding-pricing.ts @@ -0,0 +1,74 @@ +import type { Measured } from "./trace.ts" + +export interface PriceTableEntry { + inputPerMillion: number + outputPerMillion: number + cacheReadPerMillion: number + cacheWritePerMillion: number +} + +export interface PriceTable { + id: string + date: string + currency: string + cachePricing: string + models: Record +} + +// Operator-maintained price table. The identifier and date travel with every +// estimated cost. Cache reads and writes are priced separately from input and +// output tokens. Update the entries; never reuse the identifier for new rates. +export const PRICE_TABLE: PriceTable = { + id: "agent-evals-prices-2026-10", + date: "2026-10-01", + currency: "USD", + cachePricing: + "cache reads and cache writes are priced per million tokens at their own rates; missing cache counts make the estimate unavailable", + // No rates are configured. Cost estimates stay unavailable until a + // maintainer records real rates here. Placeholder or guessed rates must + // never enter this table, because reports present estimates as measured + // cost. A rate change needs a new table identifier. + models: {}, +} + +export type EstimatedCost = + | { + available: true + usd: number + priceTableId: string + priceTableDate: string + cachePricing: string + } + | { available: false; reason: string } + +// Estimates one run's cost from the final cumulative usage snapshot. All four +// token counts must be reported. A missing count is never treated as zero. +export function estimateRunCost( + modelId: string, + tokensIn: Measured, + tokensOut: Measured, + cacheReads: Measured, + cacheWrites: Measured, + table: PriceTable = PRICE_TABLE, +): EstimatedCost { + const entry = table.models[modelId] + if (!entry) return { available: false, reason: "modelMissingFromPriceTable" } + // Every count must be reported. A missing count is never treated as zero. + if (!tokensIn.available) return { available: false, reason: "tokensInNotReported" } + if (!tokensOut.available) return { available: false, reason: "tokensOutNotReported" } + if (!cacheReads.available) return { available: false, reason: "cacheReadsNotReported" } + if (!cacheWrites.available) return { available: false, reason: "cacheWritesNotReported" } + const perMillion = (tokens: number, rate: number) => (tokens / 1_000_000) * rate + const usd = + perMillion(tokensIn.value, entry.inputPerMillion) + + perMillion(tokensOut.value, entry.outputPerMillion) + + perMillion(cacheReads.value, entry.cacheReadPerMillion) + + perMillion(cacheWrites.value, entry.cacheWritePerMillion) + return { + available: true, + usd, + priceTableId: table.id, + priceTableDate: table.date, + cachePricing: table.cachePricing, + } +} diff --git a/scripts/agent-evals/coding-reference-solutions.test.ts b/scripts/agent-evals/coding-reference-solutions.test.ts new file mode 100644 index 0000000000..a4034affa7 --- /dev/null +++ b/scripts/agent-evals/coding-reference-solutions.test.ts @@ -0,0 +1,72 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import { CODING_FIXTURE_BUNDLES } from "./coding-fixtures.ts" +import { createLocalSubprocessGradingRuntime, runGrader, GRADER_VERSION } from "./coding-grader.ts" +import { CODING_GRADER_RESOURCES } from "./coding-fixtures.ts" + +// Validates every grader against its two reference solutions. Runs the real +// driver and the real Node test runner. No VS Code, no provider, no network. +describe("coding graders against reference solutions", () => { + for (const bundle of CODING_FIXTURE_BUNDLES) { + const name = bundle.fixture.identity.name + + const gradeSolution = async (overlay: typeof bundle.grading.correctSolution) => { + const scratch = await fs.mkdtemp(path.join(os.tmpdir(), `coding-grade-${name}-`)) + try { + const files = bundle.fixture.workspace.files.map((file) => ({ ...file })) + for (const patch of overlay) { + const index = files.findIndex((file) => file.path === patch.path) + if (index >= 0) files[index] = { ...patch } + else files.push({ ...patch }) + } + return await runGrader({ + finalFiles: files, + hiddenTests: bundle.grading.hiddenTests, + resources: bundle.grading.resources, + runtime: createLocalSubprocessGradingRuntime(), + scratchDir: scratch, + }) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + } + + it(`[${name}] passes the known-correct solution`, async () => { + const verdict = await gradeSolution(bundle.grading.correctSolution) + assert.equal(verdict.kind, "passed") + if (verdict.kind === "passed") assert.ok(verdict.result.failedCount === 0) + }) + + it(`[${name}] fails the plausible incorrect solution on hidden tests only`, async () => { + const verdict = await gradeSolution(bundle.grading.incorrectSolution) + assert.equal(verdict.kind, "failed") + if (verdict.kind === "failed") { + assert.ok(verdict.result.failedCount > 0) + for (const failure of verdict.result.failures) + assert.match( + path.basename(failure.file), + /\.acceptance\.test\.js$/, + `unexpected visible-test failure: ${failure.name}`, + ) + } + }) + + it(`[${name}] passes visible tests even for the incorrect solution`, async () => { + const verdict = await gradeSolution(bundle.grading.incorrectSolution) + if (verdict.kind === "failed") { + const visibleFailures = verdict.result.failures.filter( + (failure) => !path.basename(failure.file).includes(".acceptance."), + ) + assert.equal(visibleFailures.length, 0) + } + }) + } + + it("pins one grader version across the suite", () => { + assert.equal(GRADER_VERSION, "coding-grader-v1") + assert.equal(CODING_GRADER_RESOURCES.image.startsWith("node:"), true) + }) +}) diff --git a/scripts/agent-evals/coding-report.test.ts b/scripts/agent-evals/coding-report.test.ts new file mode 100644 index 0000000000..e92aa27aa7 --- /dev/null +++ b/scripts/agent-evals/coding-report.test.ts @@ -0,0 +1,181 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { buildComparisonReport, renderMarkdownReport, type SideResult } from "./coding-report.ts" +import { runComparison, type SideExecutor } from "./coding-compare.ts" +import type { CodingRunMetrics } from "./coding-metrics.ts" +import type { SidePins } from "./coding-pins.ts" + +const pins = (side: "baseline" | "candidate", commit: string, overrides: Partial = {}): SidePins => ({ + side, + commit, + suiteId: "coding-basic-v1", + suiteDigest: "d1".repeat(32), + settingsDigest: "d2".repeat(32), + modelIdentifier: "qwen3.8-27b", + graderVersion: "coding-grader-v1", + runnerEnvironment: { os: "linux", nodeVersion: "v22.23.1", vscodeTarget: "1.104.0" }, + ...overrides, +}) + +const completeCost = (usd: number): CodingRunMetrics["cost"] => ({ + estimated: { available: true, usd, priceTableId: "t", priceTableDate: "d", cachePricing: "c" }, + providerReported: { available: false, reason: "none" }, + accountingComplete: true, +}) + +const metrics = ( + fixtureName: string, + repIndex: number, + outcome: CodingRunMetrics["outcome"]["outcome"], + usd: number | undefined, +): CodingRunMetrics => ({ + fixtureName, + repIndex, + outcome: { outcome, reason: outcome }, + durationMs: 1_000 + repIndex, + executionCode: "COMPLETED", + stopReason: "taskCompleted", + requests: { count: 0, promptSize: [], requestsMissingUsage: 0 }, + usage: { labels: { context: "extensionContextEstimate" } }, + toolCalls: { byTool: {}, totalAttempts: { available: false, reason: "none" }, totalFailures: 0, failureEvents: 0 }, + cost: + usd === undefined + ? { + estimated: { available: false, reason: "tokensInNotReported" }, + providerReported: { available: false, reason: "none" }, + accountingComplete: false, + } + : completeCost(usd), + accounting: { status: "complete", reasons: [] }, + unavailableTelemetry: [], +}) + +const sideResult = ( + side: "baseline" | "candidate", + commit: string, + runs: CodingRunMetrics[], + overrides?: Partial, +): SideResult => ({ + pins: pins(side, commit, overrides), + runs, +}) + +describe("comparison report", () => { + it("builds a self-contained JSON report over all suite fixtures", () => { + const baseline = sideResult("baseline", "a".repeat(40), [ + metrics("clamp-range", 0, "correct", 0.1), + metrics("clamp-range", 1, "correct", 0.1), + metrics("clamp-range", 2, "incorrect", 0.1), + ]) + const candidate = sideResult("candidate", "b".repeat(40), [ + metrics("clamp-range", 0, "correct", 0.05), + metrics("clamp-range", 1, "timed-out", undefined), + metrics("clamp-range", 2, "grading-failure", 0.05), + ]) + const report = buildComparisonReport({ baseline, candidate }) + assert.equal(report.reportVersion, 1) + assert.equal(report.fixtures.length, 5) + const row = report.fixtures.find((fixture) => fixture.fixtureName === "clamp-range")! + assert.equal(row.baseline.successRate.successes, 2) + assert.equal(row.candidate.successRate.successes, 1) + assert.deepEqual([...row.candidate.outcomes].sort(), ["correct", "grading-failure", "timed-out"].sort()) + assert.equal(row.pairedOutcomes.pairs, 3) + assert.ok(row.pairedDurations.pairs.length === 3) + // Missing candidate cost on rep 1 is a missing pair, never zero. + assert.equal(row.pairedCosts.missingPairs, 1) + // The report serializes cleanly: a self-contained result. + const parsed = JSON.parse(JSON.stringify(report)) as typeof report + assert.equal(parsed.fixtures.length, report.fixtures.length) + }) + + it("flags incompatible pairs when pinned inputs differ", () => { + const baseline = sideResult("baseline", "a".repeat(40), []) + const candidate = sideResult("candidate", "b".repeat(40), [], { modelIdentifier: "other-model" }) + const report = buildComparisonReport({ baseline, candidate }) + assert.equal(report.compatibility.compatible, false) + assert.deepEqual(report.compatibility.differences, ["modelIdentifier"]) + }) + + it("keeps grading failures out of the incorrect count and excludes incomplete cost runs", () => { + const baseline = sideResult("baseline", "a".repeat(40), [ + metrics("clamp-range", 0, "correct", 0.2), + metrics("clamp-range", 1, "grading-failure", undefined), + metrics("clamp-range", 2, "infrastructure-failure", undefined), + ]) + const report = buildComparisonReport({ baseline, candidate: sideResult("candidate", "b".repeat(40), []) }) + const summary = report.costSummary.baseline + assert.equal(summary.correctRuns, 1) + assert.equal(summary.excludedRuns, 2) + assert.equal(summary.available, true) + assert.equal(summary.usd, 0.2) + assert.equal(report.partialDataPolicy, "excludeIncompleteFromAggregates-v1") + }) + + it("renders the Markdown report with required sections and no significance claim", () => { + const baseline = sideResult("baseline", "a".repeat(40), [metrics("clamp-range", 0, "correct", 0.2)]) + const candidate = sideResult("candidate", "b".repeat(40), [metrics("clamp-range", 0, "incorrect", 0.3)]) + const report = buildComparisonReport({ baseline, candidate }) + const markdown = renderMarkdownReport(report) + assert.match(markdown, /# Coding eval comparison: coding-basic-v1/) + assert.match(markdown, /## Pinned inputs and compatibility/) + assert.match(markdown, /Baseline commit/) + assert.match(markdown, /## Method/) + assert.match(markdown, /cannot show small improvements/) + assert.match(markdown, /## Results per fixture/) + assert.match(markdown, /## Cost per successful task/) + assert.match(markdown, /unavailable|over 1 correct run/) + assert.match(markdown, /## Per-run appendix/) + assert.match(markdown, /baseline \| clamp-range \| 0 \| correct /) + }) + + it("marks an incompatible pair in the Markdown report", () => { + const baseline = sideResult("baseline", "a".repeat(40), []) + const candidate = sideResult("candidate", "b".repeat(40), [], { graderVersion: "coding-grader-v9" }) + const markdown = renderMarkdownReport(buildComparisonReport({ baseline, candidate })) + assert.match(markdown, /Incompatible pair/) + assert.match(markdown, /graderVersion/) + }) +}) + +describe("comparison orchestration", () => { + it("runs both sides with the same repetition count and merges results", async () => { + const requests: { side: string; repetitions: number }[] = [] + const executor: SideExecutor = { + runSide: async (request) => { + requests.push({ side: request.side, repetitions: request.repetitions }) + return request.side === "baseline" + ? sideResult("baseline", "a".repeat(40), [metrics("clamp-range", 0, "correct", 0.1)]) + : sideResult("candidate", "b".repeat(40), [metrics("clamp-range", 0, "correct", 0.2)]) + }, + } + const report = await runComparison({ + baselineCommit: "a".repeat(40), + candidateCommit: "b".repeat(40), + repetitions: 2, + executor, + }) + assert.deepEqual(requests, [ + { side: "baseline", repetitions: 2 }, + { side: "candidate", repetitions: 2 }, + ]) + assert.equal(report.compatibility.compatible, true) + assert.equal(report.repetitions, 2) + assert.equal(report.runs.filter((run) => run.side === "baseline").length, 1) + assert.equal(report.runs.filter((run) => run.side === "candidate").length, 1) + }) + + it("defaults to three repetitions", async () => { + const executor: SideExecutor = { + runSide: async (request) => { + assert.equal(request.repetitions, 3) + return sideResult(request.side, "c".repeat(40), []) + }, + } + const report = await runComparison({ + baselineCommit: "a".repeat(40), + candidateCommit: "b".repeat(40), + executor, + }) + assert.equal(report.repetitions, 3) + }) +}) diff --git a/scripts/agent-evals/coding-report.ts b/scripts/agent-evals/coding-report.ts new file mode 100644 index 0000000000..d4a8f80948 --- /dev/null +++ b/scripts/agent-evals/coding-report.ts @@ -0,0 +1,211 @@ +import type { CodingRunOutcome } from "./coding-outcomes.ts" +import type { CodingRunMetrics, CostPerSuccessfulTask, PARTIAL_DATA_POLICY } from "./coding-metrics.ts" +import { costPerSuccessfulTask } from "./coding-metrics.ts" +import type { SidePins, PinComparison } from "./coding-pins.ts" +import { comparePins } from "./coding-pins.ts" +import { + STATISTICAL_POLICY, + successRate, + pairedOutcomeDiffs, + pairedNumericDiffs, + type SuccessRate, + type PairedOutcomeDiff, + type PairedNumericDiff, +} from "./coding-stats.ts" +import { CODING_FIXTURE_BUNDLES } from "./coding-fixtures.ts" +import { initialStateDigest } from "./coding-workspace.ts" +import { CODING_SUITE_ID } from "./coding-suite.ts" + +export interface SideResult { + pins: SidePins + runs: CodingRunMetrics[] +} + +export interface FixtureComparisonRow { + fixtureName: string + fixtureVersion: string + initialStateDigest: string + baseline: SideFixtureStats + candidate: SideFixtureStats + pairedOutcomes: PairedOutcomeDiff + pairedDurations: PairedNumericDiff + pairedCosts: PairedNumericDiff +} + +export interface SideFixtureStats { + outcomes: CodingRunOutcome[] + successRate: SuccessRate +} + +export interface ComparisonReport { + reportVersion: 1 + generatedAt: string + suiteId: string + repetitions: number + statisticalPolicy: typeof STATISTICAL_POLICY + partialDataPolicy: typeof PARTIAL_DATA_POLICY + pins: { baseline: SidePins; candidate: SidePins } + compatibility: PinComparison + fixtures: FixtureComparisonRow[] + costSummary: { baseline: CostPerSuccessfulTask; candidate: CostPerSuccessfulTask } + runs: (CodingRunMetrics & { side: "baseline" | "candidate" })[] +} + +export interface ComparisonInputs { + baseline: SideResult + candidate: SideResult + repetitions?: number +} + +const byRepIndex = (a: CodingRunMetrics, b: CodingRunMetrics) => a.repIndex - b.repIndex + +export function buildComparisonReport(inputs: ComparisonInputs): ComparisonReport { + const repetitions = inputs.repetitions ?? STATISTICAL_POLICY.repetitionsDefault + const compatibility = comparePins(inputs.baseline.pins, inputs.candidate.pins) + const rows: FixtureComparisonRow[] = [] + for (const bundle of CODING_FIXTURE_BUNDLES) { + const name = bundle.fixture.identity.name + const baselineRuns = inputs.baseline.runs.filter((run) => run.fixtureName === name).sort(byRepIndex) + const candidateRuns = inputs.candidate.runs.filter((run) => run.fixtureName === name).sort(byRepIndex) + rows.push({ + fixtureName: name, + fixtureVersion: bundle.fixture.identity.version, + initialStateDigest: initialStateDigest(bundle.fixture), + baseline: { + outcomes: baselineRuns.map((run) => run.outcome.outcome), + successRate: successRate(baselineRuns.map((run) => run.outcome.outcome)), + }, + candidate: { + outcomes: candidateRuns.map((run) => run.outcome.outcome), + successRate: successRate(candidateRuns.map((run) => run.outcome.outcome)), + }, + pairedOutcomes: pairedOutcomeDiffs( + baselineRuns.map((run) => run.outcome.outcome), + candidateRuns.map((run) => run.outcome.outcome), + ), + pairedDurations: pairedNumericDiffs( + baselineRuns.map((run) => run.durationMs), + candidateRuns.map((run) => run.durationMs), + ), + pairedCosts: pairedNumericDiffs( + baselineRuns.map((run) => (run.cost.estimated.available ? run.cost.estimated.usd : undefined)), + candidateRuns.map((run) => (run.cost.estimated.available ? run.cost.estimated.usd : undefined)), + ), + }) + } + return { + reportVersion: 1, + generatedAt: new Date().toISOString(), + suiteId: CODING_SUITE_ID, + repetitions, + statisticalPolicy: STATISTICAL_POLICY, + partialDataPolicy: costPerSuccessfulTask([]).policy, + pins: { baseline: inputs.baseline.pins, candidate: inputs.candidate.pins }, + compatibility, + fixtures: rows, + costSummary: { + baseline: costPerSuccessfulTask(inputs.baseline.runs), + candidate: costPerSuccessfulTask(inputs.candidate.runs), + }, + runs: [ + ...inputs.baseline.runs.map((run) => ({ ...run, side: "baseline" as const })), + ...inputs.candidate.runs.map((run) => ({ ...run, side: "candidate" as const })), + ], + } +} + +const percent = (value: number): string => `${(value * 100).toFixed(0)}%` + +const rateCell = (stats: SideFixtureStats): string => { + const rate = stats.successRate + const counts = stats.outcomes.reduce>((acc, outcome) => { + acc[outcome] = (acc[outcome] ?? 0) + 1 + return acc + }, {}) + const breakdown = [ + `correct ${counts["correct"] ?? 0}`, + `incorrect ${counts["incorrect"] ?? 0}`, + `timed-out ${counts["timed-out"] ?? 0}`, + `grading-failure ${counts["grading-failure"] ?? 0}`, + `infrastructure-failure ${counts["infrastructure-failure"] ?? 0}`, + ].join(", ") + const interval = rate.interval ? ` [${percent(rate.interval.lower)}, ${percent(rate.interval.upper)}]` : "" + return `${rate.successes}/${rate.total}${interval} (${breakdown})` +} + +const costCell = (cost: CostPerSuccessfulTask): string => + cost.available + ? `$${cost.usd?.toFixed(4)} over ${cost.correctRuns} correct run(s), ${cost.excludedRuns} excluded` + : `unavailable (${cost.reason ?? "unavailable"}; ${cost.excludedRuns} excluded)` + +// Self-contained Markdown rendering of the JSON report. +export function renderMarkdownReport(report: ComparisonReport): string { + const lines: string[] = [] + lines.push(`# Coding eval comparison: ${report.suiteId}`) + lines.push("") + lines.push(`Generated: ${report.generatedAt}`) + lines.push("") + lines.push("## Pinned inputs and compatibility") + lines.push("") + lines.push(`- Baseline commit: \`${report.pins.baseline.commit}\``) + lines.push(`- Candidate commit: \`${report.pins.candidate.commit}\``) + lines.push(`- Suite: ${report.pins.baseline.suiteId} / ${report.pins.candidate.suiteId}`) + lines.push(`- Suite digest: \`${report.pins.baseline.suiteDigest}\` / \`${report.pins.candidate.suiteDigest}\``) + lines.push( + `- Settings digest: \`${report.pins.baseline.settingsDigest}\` / \`${report.pins.candidate.settingsDigest}\``, + ) + lines.push(`- Model: ${report.pins.baseline.modelIdentifier} / ${report.pins.candidate.modelIdentifier}`) + lines.push(`- Grader version: ${report.pins.baseline.graderVersion} / ${report.pins.candidate.graderVersion}`) + lines.push( + `- Runner environment: ${report.pins.baseline.runnerEnvironment.os}, node ${report.pins.baseline.runnerEnvironment.nodeVersion}, vscode ${report.pins.baseline.runnerEnvironment.vscodeTarget} / ${report.pins.candidate.runnerEnvironment.os}, node ${report.pins.candidate.runnerEnvironment.nodeVersion}, vscode ${report.pins.candidate.runnerEnvironment.vscodeTarget}`, + ) + lines.push("") + if (report.compatibility.compatible) lines.push("**Compatible pair.** All pinned inputs match.") + else + lines.push( + `**Incompatible pair.** Differing pinned inputs: ${report.compatibility.differences.join(", ")}. Treat the comparison as invalid.`, + ) + lines.push("") + lines.push("## Method") + lines.push("") + lines.push( + `- ${report.repetitions} repetitions per fixture per commit (default ${STATISTICAL_POLICY.repetitionsDefault}).`, + ) + lines.push(`- ${report.statisticalPolicy.method}.`) + lines.push(`- ${report.statisticalPolicy.significanceClaim}`) + lines.push( + `- Partial-data policy: ${report.partialDataPolicy}. Runs with incomplete cost accounting are flagged and excluded from cost aggregates.`, + ) + lines.push("") + lines.push("## Results per fixture") + lines.push("") + lines.push("| Fixture | Baseline | Candidate | Paired (improved/regressed/unchanged) |") + lines.push("| --- | --- | --- | --- |") + for (const row of report.fixtures) + lines.push( + `| ${row.fixtureName} | ${rateCell(row.baseline)} | ${rateCell(row.candidate)} | ${row.pairedOutcomes.improved}/${row.pairedOutcomes.regressed}/${row.pairedOutcomes.unchanged} |`, + ) + lines.push("") + lines.push("## Cost per successful task") + lines.push("") + lines.push(`- Baseline: ${costCell(report.costSummary.baseline)}`) + lines.push(`- Candidate: ${costCell(report.costSummary.candidate)}`) + lines.push(`- Price table: recorded per run with identifier and date; cache reads and writes priced separately.`) + lines.push("") + lines.push("## Unavailable telemetry") + lines.push("") + lines.push( + "Duplicate-call detection and HTTP retry counts are unsupported and reported as unavailable. They are never inferred from unrelated events.", + ) + lines.push("") + lines.push("## Per-run appendix") + lines.push("") + lines.push("| Side | Fixture | Rep | Outcome | Reason | Duration ms |") + lines.push("| --- | --- | --- | --- | --- | --- |") + for (const run of report.runs) + lines.push( + `| ${run.side} | ${run.fixtureName} | ${run.repIndex} | ${run.outcome.outcome} | ${run.outcome.reason} | ${run.durationMs ?? "unavailable"} |`, + ) + lines.push("") + return lines.join("\n") +} diff --git a/scripts/agent-evals/coding-run.test.ts b/scripts/agent-evals/coding-run.test.ts new file mode 100644 index 0000000000..dce16722b6 --- /dev/null +++ b/scripts/agent-evals/coding-run.test.ts @@ -0,0 +1,163 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import { RooCodeEventName, TaskCommandName, type TaskCommand, type TaskEvent } from "@roo-code/types" +import { runCodingFixtureOnce, withWorkspaceCapture } from "./coding-run.ts" +import { createLocalSubprocessGradingRuntime } from "./coding-grader.ts" +import type { AgentClient, AgentHost, AgentRuntime, FinalWorkspaceSnapshot } from "./runner.ts" +import type { AgentFixtureWorkspaceFile } from "./contract.ts" +import type { AgentTrace } from "./trace.ts" +import { codingBundle } from "./coding-suite.ts" + +// Deterministic clock: each pause advances time by 100 ms so the substrate's +// bounded windows expire without real waiting. +interface SyntheticOptions { + finalFiles: AgentFixtureWorkspaceFile[] + complete: boolean +} + +const syntheticRuntime = (options: SyntheticOptions): { runtime: AgentRuntime; traces: AgentTrace[] } => { + const traces: AgentTrace[] = [] + let listener: ((event: unknown) => void) | undefined + let fakeNow = 0 + const host: AgentHost = { + get exited() { + return true + }, + onExit: () => {}, + signal: () => {}, + } + const client: AgentClient = { + isReady: true, + onTaskEvent: (l) => { + listener = l + }, + onDisconnect: () => {}, + sendCommand: (command: TaskCommand) => { + if (command.commandName !== TaskCommandName.StartNewTask) return + const requestId = (command.data as { requestId: string }).requestId + const emit = (event: TaskEvent): void => listener?.(event) + emit({ + eventName: RooCodeEventName.TaskStartResponse, + payload: [{ requestId, success: true, taskId: "task-1" }], + }) + if (options.complete) + emit({ + eventName: RooCodeEventName.TaskCompleted, + payload: [ + "task-1", + { totalTokensIn: 1_000, totalTokensOut: 200, totalCost: 0.01, contextTokens: 5_000 }, + { apply_diff: { attempts: 1, failures: 0 } }, + { isSubtask: false }, + ], + }) + }, + disconnect: () => {}, + } + const runtime: AgentRuntime = { + launch: async () => host, + connect: () => client, + cleanup: async () => {}, + writeArtifact: async (fileName, value) => { + if (fileName === "trace.json") traces.push(value as AgentTrace) + }, + onInterrupt: () => () => {}, + now: () => fakeNow, + pause: async () => { + fakeNow += 100 + }, + readFinalWorkspace: async (): Promise => ({ + files: options.finalFiles, + truncated: false, + }), + } + return { runtime, traces } +} + +const finalFilesWith = (solution: AgentFixtureWorkspaceFile): AgentFixtureWorkspaceFile[] => { + const bundle = codingBundle("clamp-range") + return bundle.fixture.workspace.files.map((file) => (file.path === solution.path ? { ...solution } : { ...file })) +} + +describe("coding run orchestration", () => { + const scratch = async (): Promise => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-run-test-")) + process.once("exit", () => { + void fs.rm(dir, { recursive: true, force: true }) + }) + return dir + } + + it("grades a completed correct run as correct", async () => { + const bundle = codingBundle("clamp-range") + const { runtime, traces } = syntheticRuntime({ + finalFiles: finalFilesWith(bundle.grading.correctSolution[0]!), + complete: true, + }) + const record = await runCodingFixtureOnce({ + fixtureName: "clamp-range", + repIndex: 0, + credential: "sk-test", + runtime, + gradingRuntime: createLocalSubprocessGradingRuntime(), + scratchDir: await scratch(), + }) + assert.equal(record.outcome.outcome, "correct") + assert.equal(record.executionCode, "COMPLETED") + assert.equal(record.grading?.kind, "passed") + assert.equal(record.metrics.outcome.outcome, "correct") + // Usage lacks cache counts, so cost accounting stays incomplete. + assert.equal(record.metrics.cost.accountingComplete, false) + assert.equal(traces.length, 1) + }) + + it("grades a completed but wrong workspace as incorrect", async () => { + const bundle = codingBundle("clamp-range") + const incorrect = bundle.grading.incorrectSolution[0]! + const { runtime } = syntheticRuntime({ finalFiles: finalFilesWith(incorrect), complete: true }) + const record = await runCodingFixtureOnce({ + fixtureName: "clamp-range", + repIndex: 1, + credential: "sk-test", + runtime, + gradingRuntime: createLocalSubprocessGradingRuntime(), + scratchDir: await scratch(), + }) + // The agent completed; the hidden tests still fail. + assert.equal(record.outcome.outcome, "incorrect") + assert.equal(record.grading?.kind, "failed") + assert.ok((record.grading?.failedCount ?? 0) > 0) + }) + + it("reports a run-window expiry as timed out without grading", async () => { + const { runtime } = syntheticRuntime({ finalFiles: [], complete: false }) + const record = await runCodingFixtureOnce({ + fixtureName: "clamp-range", + repIndex: 2, + credential: "sk-test", + runtime, + gradingRuntime: createLocalSubprocessGradingRuntime(), + scratchDir: await scratch(), + }) + assert.equal(record.outcome.outcome, "timed-out") + assert.equal(record.grading, undefined) + assert.equal(record.executionCode, "TASK_RUN_TIMEOUT") + }) + + it("captures the workspace only through cleanup and never leaks scratch paths", async () => { + const bundle = codingBundle("clamp-range") + const inner = syntheticRuntime({ + finalFiles: finalFilesWith(bundle.grading.correctSolution[0]!), + complete: false, + }) + const captured = withWorkspaceCapture(inner.runtime) + assert.equal(captured.getSnapshot(), undefined) + await captured.runtime.cleanup() + const snapshot = captured.getSnapshot() + assert.ok(snapshot) + assert.equal(snapshot.truncated, false) + assert.ok(snapshot.files.length > 0) + }) +}) diff --git a/scripts/agent-evals/coding-run.ts b/scripts/agent-evals/coding-run.ts new file mode 100644 index 0000000000..3d7599268e --- /dev/null +++ b/scripts/agent-evals/coding-run.ts @@ -0,0 +1,144 @@ +import { runAgentFixture, type AgentRuntime, type FinalWorkspaceSnapshot } from "./runner.ts" +import { getFixture } from "./registry.ts" +import { ensureCodingSuiteRegistered, codingBundle } from "./coding-suite.ts" +import { runGrader, type GraderVerdict, type GradingRuntime } from "./coding-grader.ts" +import { classifyRunOutcome, type ClassifiedOutcome } from "./coding-outcomes.ts" +import { buildRunMetrics, type CodingRunMetrics } from "./coding-metrics.ts" +import { CODING_MODEL_ID } from "./coding-fixtures.ts" +import { measured, type AgentTrace, type Measured, type MeasuredString } from "./trace.ts" + +export interface CodingRunRecord { + fixtureName: string + fixtureVersion: string + repIndex: number + outcome: ClassifiedOutcome + executionCode?: string + trace?: AgentTrace + grading?: { + kind: GraderVerdict["kind"] + isolation?: string + reason?: string + failedCount?: number + failures?: { name: string; file: string }[] + } + finalWorkspaceTruncated?: boolean + metrics: CodingRunMetrics +} + +// Serializes a verdict for records and reports. +export const serializeVerdict = (verdict: GraderVerdict): CodingRunRecord["grading"] => { + switch (verdict.kind) { + case "passed": + case "failed": + return { + kind: verdict.kind, + isolation: verdict.isolation, + failedCount: verdict.result.failedCount, + failures: verdict.kind === "failed" ? verdict.result.failures.slice(0, 10) : undefined, + } + case "grading-failure": + return { kind: verdict.kind, isolation: verdict.isolation, reason: verdict.reason } + } +} + +const modelIdentifierFromProjection = (projection: Record): MeasuredString => + measured( + typeof projection["openAiModelId"] === "string" ? (projection["openAiModelId"] as string) : undefined, + "modelIdMissingInProjection", + ) + +// Wraps a runtime so the final workspace is captured inside cleanup, before +// the launcher removes its scratch directory. The captured file data feeds +// only the fresh grading copy. +export function withWorkspaceCapture(inner: AgentRuntime): { + runtime: AgentRuntime + getSnapshot: () => FinalWorkspaceSnapshot | undefined +} { + let snapshot: FinalWorkspaceSnapshot | undefined + // Local binding preserves the narrowing for the optional identity reader. + const readIdentity = inner.readIdentity?.bind(inner) + const runtime: AgentRuntime = { + launch: (files) => inner.launch(files), + connect: () => inner.connect(), + cleanup: async () => { + try { + snapshot = await inner.readFinalWorkspace?.() + } catch { + snapshot = { files: [], truncated: true } + } + await inner.cleanup() + }, + writeArtifact: (fileName, value) => inner.writeArtifact(fileName, value), + onInterrupt: (listener) => inner.onInterrupt(listener), + now: () => inner.now(), + pause: (ms) => inner.pause(ms), + readIdentity: readIdentity ? () => readIdentity() : undefined, + readFinalWorkspace: () => + inner.readFinalWorkspace?.() ?? Promise.reject(new Error("finalWorkspaceUnsupported")), + } + return { runtime, getSnapshot: () => snapshot } +} + +export interface CodingRunInputs { + fixtureName: string + repIndex: number + credential: string + runtime: AgentRuntime + gradingRuntime: GradingRuntime + scratchDir: string +} + +// One fixture repetition: substrate run, final-workspace capture, hidden-test +// grading, outcome classification, and metrics. The grader receives no +// credential. The record embeds the metadata-only trace. +export async function runCodingFixtureOnce(inputs: CodingRunInputs): Promise { + ensureCodingSuiteRegistered() + const bundle = codingBundle(inputs.fixtureName) + const { runtime, getSnapshot } = withWorkspaceCapture(inputs.runtime) + const result = await runAgentFixture(runtime, { + fixtureName: inputs.fixtureName, + credential: inputs.credential, + }) + const trace: AgentTrace = result.trace + const snapshot = getSnapshot() + const finalWorkspaceAvailable = snapshot !== undefined + let grading: GraderVerdict | undefined + const code = result.execution.code + if ((code === "COMPLETED" || code === "STOPPED") && snapshot) { + grading = await runGrader({ + finalFiles: snapshot.files, + hiddenTests: bundle.grading.hiddenTests, + resources: bundle.grading.resources, + runtime: inputs.gradingRuntime, + scratchDir: inputs.scratchDir, + }) + } + const outcome = classifyRunOutcome({ + hasTrace: true, + executionCode: code, + finalWorkspaceAvailable, + grading, + }) + const metrics = buildRunMetrics({ + fixtureName: inputs.fixtureName, + repIndex: inputs.repIndex, + trace, + grading, + outcome, + modelIdentifier: modelIdentifierFromProjection(bundle.fixture.settingsProjection), + }) + return { + fixtureName: inputs.fixtureName, + fixtureVersion: bundle.fixture.identity.version, + repIndex: inputs.repIndex, + outcome, + executionCode: code, + trace, + grading: grading ? serializeVerdict(grading) : undefined, + finalWorkspaceTruncated: snapshot?.truncated, + metrics, + } +} + +// The suite model identifier, for pinned inputs. +export const CODING_RUN_MODEL_ID = CODING_MODEL_ID diff --git a/scripts/agent-evals/coding-side.mts b/scripts/agent-evals/coding-side.mts new file mode 100644 index 0000000000..c4086e8252 --- /dev/null +++ b/scripts/agent-evals/coding-side.mts @@ -0,0 +1,135 @@ +// One comparison side. Runs every coding fixture for N repetitions on this +// commit through the substrate, grades each final workspace, and writes one +// side-results file with the pinned inputs. Trusted repo code only: the +// provider credential stays with this process and the extension host; the +// grader never receives it. +import fs from "node:fs/promises" +import path from "node:path" +import { fileURLToPath } from "node:url" +import { createHostRuntime } from "./launcher.ts" +import { createContainerGradingRuntime } from "./container-runtime.ts" +import { createLocalSubprocessGradingRuntime } from "./coding-grader.ts" +import { runCodingFixtureOnce, type CodingRunRecord } from "./coding-run.ts" +import { CODING_FIXTURE_BUNDLES } from "./coding-fixtures.ts" +import { CODING_SUITE_ID, codingSuiteDigest, ensureCodingSuiteRegistered } from "./coding-suite.ts" +import { GRADER_VERSION } from "./coding-grader.ts" +import { digestValue } from "./coding-digest.ts" +import type { SidePins } from "./coding-pins.ts" +import type { SideResultFile } from "./coding-compare.ts" +import { classifyRunOutcome } from "./coding-outcomes.ts" +import { buildRunMetrics } from "./coding-metrics.ts" + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") + +const readArgs = (): { reps: number; out: string; credentialEnv: string; allowUnisolatedGrading: boolean } => { + const args = process.argv.slice(2) + const read = (name: string): string | undefined => { + const index = args.indexOf(`--${name}`) + return index >= 0 ? args[index + 1] : undefined + } + const out = read("out") + if (!out) throw new Error("Set --out to a scratch directory") + return { + reps: Number(read("reps") ?? "3"), + out: path.resolve(out), + credentialEnv: read("credential-env") ?? "SOLHEIM_API_KEY", + allowUnisolatedGrading: args.includes("--allow-unisolated-grading"), + } +} + +const readVscodeTarget = async (): Promise => { + try { + const pkg = JSON.parse(await fs.readFile(path.join(root, "apps/vscode-e2e/package.json"), "utf8")) as { + devDependencies?: Record + } + return pkg.devDependencies?.["@types/vscode"] ?? "unavailable" + } catch { + return "unavailable" + } +} + +const readCommit = async (): Promise => { + const { execFile } = await import("node:child_process") + const { promisify } = await import("node:util") + try { + const { stdout } = await promisify(execFile)("git", ["rev-parse", "HEAD"], { cwd: root, timeout: 5_000 }) + return stdout.trim() + } catch { + return "commit-unavailable" + } +} + +const { reps, out, credentialEnv, allowUnisolatedGrading } = readArgs() +if (!Number.isInteger(reps) || reps < 1) throw new Error("--reps must be a positive integer") +await fs.mkdir(out, { recursive: true }) +ensureCodingSuiteRegistered() + +const credential = process.env[credentialEnv] ?? "" +const gradingRuntime = allowUnisolatedGrading ? createLocalSubprocessGradingRuntime() : createContainerGradingRuntime() + +const projection = CODING_FIXTURE_BUNDLES[0]?.fixture.settingsProjection ?? {} +const pins: SidePins = { + side: "baseline", + commit: await readCommit(), + suiteId: CODING_SUITE_ID, + suiteDigest: codingSuiteDigest(), + settingsDigest: digestValue(projection), + modelIdentifier: typeof projection["openAiModelId"] === "string" ? projection["openAiModelId"] : "unavailable", + graderVersion: GRADER_VERSION, + runnerEnvironment: { + os: process.platform, + nodeVersion: process.version, + vscodeTarget: await readVscodeTarget(), + }, +} + +const runs: CodingRunRecord[] = [] +for (const bundle of CODING_FIXTURE_BUNDLES) { + const fixtureName = bundle.fixture.identity.name + for (let repIndex = 0; repIndex < reps; repIndex++) { + const artifactsDir = path.join(out, "artifacts", `${fixtureName}-rep${repIndex}`) + await fs.mkdir(artifactsDir, { recursive: true }) + try { + const record = await runCodingFixtureOnce({ + fixtureName, + repIndex, + credential, + runtime: createHostRuntime({ root, artifactsDir }), + gradingRuntime, + scratchDir: artifactsDir, + }) + runs.push(record) + } catch { + // The substrate never throws for task failures. This guards wiring + // errors only, and records them as infrastructure failures. + const outcome = classifyRunOutcome({ hasTrace: false, finalWorkspaceAvailable: false }) + runs.push({ + fixtureName, + fixtureVersion: bundle.fixture.identity.version, + repIndex, + outcome, + metrics: buildRunMetrics({ + fixtureName, + repIndex, + outcome, + modelIdentifier: { available: false, reason: "runFailed" }, + }), + }) + } + } +} + +// Side identity must reflect the actual side label in a comparison context. +const sideLabel = (process.env["CODING_EVALS_SIDE"] as SidePins["side"] | undefined) ?? "baseline" +const result: SideResultFile = { pins: { ...pins, side: sideLabel }, runs } + +// Keep the trace of each run with its artifacts for inspection. +for (const record of runs) { + if (!record.trace) continue + const artifactDir = path.join(out, "artifacts", `${record.fixtureName}-rep${record.repIndex}`) + await fs.writeFile(path.join(artifactDir, "run-record.json"), JSON.stringify(record, null, 2)) +} + +await fs.writeFile(path.join(out, "side-results.json"), JSON.stringify(result, null, 2)) +const correct = runs.filter((record) => record.outcome.outcome === "correct").length +console.log(`coding-evals side complete: ${correct}/${runs.length} correct, suite ${pins.suiteDigest.slice(0, 12)}`) diff --git a/scripts/agent-evals/coding-stats.test.ts b/scripts/agent-evals/coding-stats.test.ts new file mode 100644 index 0000000000..c7e86dcd47 --- /dev/null +++ b/scripts/agent-evals/coding-stats.test.ts @@ -0,0 +1,62 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { + DEFAULT_REPETITIONS, + STATISTICAL_POLICY, + pairedNumericDiffs, + pairedOutcomeDiffs, + successRate, + wilsonInterval, +} from "./coding-stats.ts" + +describe("statistics", () => { + it("defaults to three repetitions per fixture per commit", () => { + assert.equal(DEFAULT_REPETITIONS, 3) + assert.equal(STATISTICAL_POLICY.repetitionsDefault, 3) + }) + + it("computes the Wilson interval without claiming significance", () => { + const perfect = wilsonInterval(3, 3) + assert.ok(perfect.lower > 0.29, `lower bound too low: ${perfect.lower}`) + assert.ok(perfect.upper <= 1) + const none = wilsonInterval(0, 3) + assert.equal(none.lower, 0) + assert.ok(none.upper < 0.71) + assert.deepEqual(wilsonInterval(1, 0), { lower: 0, upper: 0 }) + assert.match(STATISTICAL_POLICY.significanceClaim, /cannot show small improvements/) + assert.equal(STATISTICAL_POLICY.method.includes("wilson"), true) + }) + + it("summarizes success rates over outcomes", () => { + const rate = successRate(["correct", "correct", "incorrect"] as const) + assert.equal(rate.successes, 2) + assert.equal(rate.total, 3) + assert.ok(Math.abs(rate.rate - 2 / 3) < 1e-9) + assert.ok(rate.interval !== undefined) + assert.equal(successRate([]).interval, undefined) + }) + + it("pairs outcome differences by repetition index", () => { + const diff = pairedOutcomeDiffs( + ["correct", "incorrect", "correct", "correct"], + ["incorrect", "correct", "correct", "grading-failure"], + ) + assert.equal(diff.pairs, 4) + assert.equal(diff.improved, 1) + assert.equal(diff.regressed, 2) + assert.equal(diff.unchanged, 1) + }) + + it("pairs numeric deltas and counts missing pairs without zero filling", () => { + const diff = pairedNumericDiffs([1, 2, undefined, 4], [3, undefined, 5, 8]) + assert.equal(diff.pairs.length, 2) + assert.equal(diff.missingPairs, 2) + assert.deepEqual( + diff.pairs.map((pair) => pair.delta).sort((a, b) => a - b), + [2, 4], + ) + assert.ok(Math.abs((diff.meanDelta ?? 0) - 3) < 1e-9) + const empty = pairedNumericDiffs([], []) + assert.equal(empty.meanDelta, undefined) + }) +}) diff --git a/scripts/agent-evals/coding-stats.ts b/scripts/agent-evals/coding-stats.ts new file mode 100644 index 0000000000..5ac1befefd --- /dev/null +++ b/scripts/agent-evals/coding-stats.ts @@ -0,0 +1,118 @@ +import type { CodingRunOutcome } from "./coding-outcomes.ts" + +// Uncertainty constant for the Wilson score interval at the 95% level. The +// interval describes uncertainty. It is not a significance test. +export const UNCERTAINTY_Z = 1.96 + +export const DEFAULT_REPETITIONS = 3 + +export const STATISTICAL_POLICY = { + method: "success-rate-with-wilson-95-percent-interval-and-paired-deltas", + significanceClaim: "none. Three repetitions can show large regressions. They cannot show small improvements.", + repetitionsDefault: DEFAULT_REPETITIONS, +} as const + +export interface SuccessRate { + successes: number + total: number + rate: number + interval: { lower: number; upper: number } | undefined +} + +// Wilson score interval. n = 0 yields no interval. +export function wilsonInterval( + successes: number, + n: number, + z: number = UNCERTAINTY_Z, +): { lower: number; upper: number } { + if (n <= 0) return { lower: 0, upper: 0 } + const p = successes / n + const z2 = z * z + const denominator = 1 + z2 / n + const center = (p + z2 / (2 * n)) / denominator + const spread = (z / denominator) * Math.sqrt((p * (1 - p)) / n + z2 / (4 * n * n)) + return { lower: Math.max(0, center - spread), upper: Math.min(1, center + spread) } +} + +export function successRate(outcomes: readonly CodingRunOutcome[]): SuccessRate { + const total = outcomes.length + const successes = outcomes.filter((outcome) => outcome === "correct").length + return { + successes, + total, + rate: total === 0 ? 0 : successes / total, + interval: total === 0 ? undefined : wilsonInterval(successes, total), + } +} + +export interface PairedOutcomeDiff { + pairs: number + improved: number + regressed: number + unchanged: number + definition: string +} + +// Paired by repetition index. Improvement means the candidate run is correct +// where the baseline run is not. Regression is the reverse. No significance +// claim follows from these counts. +export function pairedOutcomeDiffs( + baseline: readonly CodingRunOutcome[], + candidate: readonly CodingRunOutcome[], +): PairedOutcomeDiff { + const pairs = Math.min(baseline.length, candidate.length) + let improved = 0 + let regressed = 0 + let unchanged = 0 + for (let index = 0; index < pairs; index++) { + const baseCorrect = baseline[index] === "correct" + const candCorrect = candidate[index] === "correct" + if (baseCorrect === candCorrect) unchanged++ + else if (candCorrect) improved++ + else regressed++ + } + return { + pairs, + improved, + regressed, + unchanged, + definition: "paired by repetition index; correct versus not correct", + } +} + +export interface PairedNumericDiff { + pairs: { baseline: number; candidate: number; delta: number }[] + missingPairs: number + meanDelta?: number + minDelta?: number + maxDelta?: number +} + +// Paired numeric deltas. A pair needs both sides reported; pairs with a +// missing value on either side are counted as missing, never as zero. +export function pairedNumericDiffs( + baseline: readonly (number | undefined)[], + candidate: readonly (number | undefined)[], +): PairedNumericDiff { + const pairs: { baseline: number; candidate: number; delta: number }[] = [] + let missingPairs = 0 + const count = Math.min(baseline.length, candidate.length) + for (let index = 0; index < count; index++) { + const base = baseline[index] + const cand = candidate[index] + if (base === undefined || cand === undefined) { + missingPairs++ + continue + } + pairs.push({ baseline: base, candidate: cand, delta: cand - base }) + } + if (pairs.length === 0) return { pairs, missingPairs } + const deltas = pairs.map((pair) => pair.delta) + return { + pairs, + missingPairs, + meanDelta: deltas.reduce((sum, delta) => sum + delta, 0) / deltas.length, + minDelta: Math.min(...deltas), + maxDelta: Math.max(...deltas), + } +} diff --git a/scripts/agent-evals/coding-suite.ts b/scripts/agent-evals/coding-suite.ts new file mode 100644 index 0000000000..ca7b10c452 --- /dev/null +++ b/scripts/agent-evals/coding-suite.ts @@ -0,0 +1,53 @@ +import { registerFixture, getFixture, listFixtures } from "./registry.ts" +import { validateFixture } from "./contract.ts" +import { digestValue } from "./coding-digest.ts" +import { CODING_FIXTURE_BUNDLES } from "./coding-fixtures.ts" +import { GRADER_VERSION } from "./coding-grader.ts" + +// Frozen suite identity. The digest pins the exact fixture content, hidden +// tests, reference solutions, and grader version. Any edit changes the digest +// and marks cross-commit comparisons incompatible. +export const CODING_SUITE_ID = "coding-basic-v1" + +let registered = false + +export function ensureCodingSuiteRegistered(): void { + if (registered) return + registered = true + for (const bundle of CODING_FIXTURE_BUNDLES) { + validateFixture(bundle.fixture) + registerFixture(bundle.fixture) + } +} + +export function codingBundle(fixtureName: string) { + const found = CODING_FIXTURE_BUNDLES.find((candidate) => candidate.fixture.identity.name === fixtureName) + if (!found) throw new Error(`unknown coding fixture: ${fixtureName}`) + return found +} + +// Suite digest. Canonical JSON over the frozen suite content and the grader +// identity. Computed from the in-repo data only, so the same digest holds on +// every machine. +export function codingSuiteDigest(): string { + return digestValue({ + kind: "agent-evals-coding-suite-v1", + suiteId: CODING_SUITE_ID, + graderVersion: GRADER_VERSION, + fixtures: CODING_FIXTURE_BUNDLES.map((bundle) => ({ + identity: bundle.fixture.identity, + prompt: bundle.fixture.prompt, + workspace: bundle.fixture.workspace, + toolPolicy: bundle.fixture.toolPolicy, + limits: bundle.fixture.limits, + settingsProjection: bundle.fixture.settingsProjection, + hiddenTests: bundle.grading.hiddenTests, + correctSolution: bundle.grading.correctSolution, + incorrectSolution: bundle.grading.incorrectSolution, + resources: bundle.grading.resources, + })), + }) +} + +// Re-exports for side runners and checks. +export { listFixtures } diff --git a/scripts/agent-evals/coding-workspace.test.ts b/scripts/agent-evals/coding-workspace.test.ts new file mode 100644 index 0000000000..aa3228b255 --- /dev/null +++ b/scripts/agent-evals/coding-workspace.test.ts @@ -0,0 +1,102 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import type { AgentFixture } from "./contract.ts" +import { buildGradingCopy, buildFreshWorkspace, initialStateDigest } from "./coding-workspace.ts" +import { CODING_FIXTURE_BUNDLES } from "./coding-fixtures.ts" + +const tmpdir = async (): Promise => fs.mkdtemp(path.join(os.tmpdir(), "coding-workspace-test-")) + +const anyBundle = (): (typeof CODING_FIXTURE_BUNDLES)[number] => CODING_FIXTURE_BUNDLES[0]! + +describe("initial state digest", () => { + it("is stable for the same fixture content", () => { + const fixture: AgentFixture = anyBundle().fixture + assert.equal(initialStateDigest(fixture), initialStateDigest(fixture)) + }) + + it("is independent of file declaration order", async () => { + const fixture: AgentFixture = anyBundle().fixture + const reordered: AgentFixture = { + ...fixture, + workspace: { files: [...fixture.workspace.files].reverse() }, + } + assert.equal(initialStateDigest(fixture), initialStateDigest(reordered)) + }) + + it("changes when any workspace file changes", () => { + const fixture: AgentFixture = anyBundle().fixture + const edited: AgentFixture = { + ...fixture, + workspace: { + files: fixture.workspace.files.map((file, index) => + index === 0 ? { ...file, content: `${file.content}\n` } : file, + ), + }, + } + assert.notEqual(initialStateDigest(fixture), initialStateDigest(edited)) + }) +}) + +describe("fresh workspace builder", () => { + it("writes the fixture files into a new directory", async () => { + const bundle = anyBundle() + const target = path.join(await tmpdir(), "workspace") + const result = await buildFreshWorkspace(bundle.fixture, target) + assert.equal(result.filesWritten, bundle.fixture.workspace.files.length) + for (const file of bundle.fixture.workspace.files) { + const written = await fs.readFile(path.join(target, file.path), "utf8") + assert.equal(written, file.content) + } + }) + + it("refuses to build into an existing directory", async () => { + const bundle = anyBundle() + const target = await tmpdir() + await assert.rejects(() => buildFreshWorkspace(bundle.fixture, target)) + }) + + it("keeps two builds independent", async () => { + const bundle = anyBundle() + const parent = await tmpdir() + const first = path.join(parent, "first") + const second = path.join(parent, "second") + await buildFreshWorkspace(bundle.fixture, first) + await buildFreshWorkspace(bundle.fixture, second) + // Mutating one build must not affect the other. + await fs.writeFile(path.join(first, bundle.fixture.workspace.files[0]!.path), "mutated") + const secondContent = await fs.readFile(path.join(second, bundle.fixture.workspace.files[0]!.path), "utf8") + assert.equal(secondContent, bundle.fixture.workspace.files[0]!.content) + }) +}) + +describe("grading copy", () => { + it("adds hidden tests only to the copy", async () => { + const bundle = anyBundle() + const hidden = bundle.grading.hiddenTests[0]! + const target = path.join(await tmpdir(), "grading") + await buildGradingCopy(bundle.fixture.workspace.files, bundle.grading.hiddenTests, target) + await fs.access(path.join(target, hidden.path)) + // The run workspace itself must never contain the hidden tests. + assert.equal( + bundle.fixture.workspace.files.some((file) => file.path === hidden.path), + false, + ) + }) + + it("lets hidden tests shadow an agent file with the same path", async () => { + const bundle = anyBundle() + const hidden = bundle.grading.hiddenTests[0]! + const target = path.join(await tmpdir(), "grading-shadow") + const tampered = [...bundle.fixture.workspace.files, { path: hidden.path, content: "// tampered" }] + await buildGradingCopy(tampered, bundle.grading.hiddenTests, target) + assert.equal(await fs.readFile(path.join(target, hidden.path), "utf8"), hidden.content) + }) + + it("rejects escaping paths", async () => { + const target = path.join(await tmpdir(), "grading-escape") + await assert.rejects(() => buildGradingCopy([{ path: "../escape.txt", content: "x" }], [], target)) + }) +}) diff --git a/scripts/agent-evals/coding-workspace.ts b/scripts/agent-evals/coding-workspace.ts new file mode 100644 index 0000000000..70182a2417 --- /dev/null +++ b/scripts/agent-evals/coding-workspace.ts @@ -0,0 +1,49 @@ +import fs from "node:fs/promises" +import path from "node:path" +import { validateWorkspaceFilePath, type AgentFixture, type AgentFixtureWorkspaceFile } from "./contract.ts" +import { digestValue } from "./coding-digest.ts" + +// Immutable initial-state digest. The digest pins the exact fixture workspace +// content, so every run of the same fixture version rebuilds byte-identical +// initial state and any fixture edit changes the pinned suite digest. +export function initialStateDigest(fixture: AgentFixture): string { + const sorted = [...fixture.workspace.files].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)) + return digestValue({ kind: "agent-evals-initial-workspace-v1", files: sorted }) +} + +// Builds the fixture workspace in a fresh directory. The directory must not +// exist yet, so no run can inherit files from an earlier run. +export async function buildFreshWorkspace(fixture: AgentFixture, targetDir: string): Promise<{ filesWritten: number }> { + await fs.mkdir(targetDir, { recursive: false }) + let filesWritten = 0 + for (const file of fixture.workspace.files) { + await writeContainedFile(targetDir, file) + filesWritten++ + } + return { filesWritten } +} + +// Writes one file inside root. The containment check mirrors the launcher's +// write-time check so grader-side copies cannot be tricked into escaping. +export async function writeContainedFile(root: string, file: AgentFixtureWorkspaceFile): Promise { + validateWorkspaceFilePath(file.path) + const rootReal = await fs.realpath(root) + const target = path.resolve(rootReal, file.path) + if (target !== rootReal && !target.startsWith(rootReal + path.sep)) + throw new Error("workspace file path escapes the workspace") + await fs.mkdir(path.dirname(target), { recursive: true }) + await fs.writeFile(target, file.content) + const written = await fs.realpath(target) + if (!written.startsWith(rootReal + path.sep)) throw new Error("workspace file escapes the workspace") +} + +// Builds a fresh grading copy of a final workspace. Hidden acceptance tests +// are added only to the copy, never to the workspace the agent used. +export async function buildGradingCopy( + finalFiles: readonly AgentFixtureWorkspaceFile[], + hiddenTests: readonly AgentFixtureWorkspaceFile[], + targetDir: string, +): Promise { + await fs.mkdir(targetDir, { recursive: false }) + for (const file of [...finalFiles, ...hiddenTests]) await writeContainedFile(targetDir, file) +} diff --git a/scripts/agent-evals/container-runtime.test.ts b/scripts/agent-evals/container-runtime.test.ts new file mode 100644 index 0000000000..afc27c96fa --- /dev/null +++ b/scripts/agent-evals/container-runtime.test.ts @@ -0,0 +1,62 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { buildContainerArgs, createContainerGradingRuntime, GRADER_IMAGE } from "./container-runtime.ts" +import type { GraderJob } from "./coding-grader.ts" + +const makeJob = (): GraderJob => ({ + gradingDir: "/tmp/evals/grading-copy", + driverPath: "/repo/scripts/agent-evals/grader-driver.mjs", + resources: { timeoutMs: 30_000, memoryMb: 512, cpus: 1, pidsLimit: 128, image: GRADER_IMAGE }, +}) + +describe("container grading runtime", () => { + it("builds a locked-down, network-less invocation", () => { + const args = buildContainerArgs(makeJob(), "/tmp/evals/grading-out") + const flagValue = (flag: string): string | undefined => { + const index = args.indexOf(flag) + return index === -1 ? undefined : args[index + 1] + } + assert.equal(flagValue("--network"), "none") + assert.equal(flagValue("--memory"), "512m") + assert.equal(flagValue("--memory-swap"), "512m") + assert.equal(flagValue("--cpus"), "1") + assert.equal(flagValue("--pids-limit"), "128") + assert.equal(flagValue("--cap-drop"), "ALL") + assert.equal(flagValue("--security-opt"), "no-new-privileges") + assert.ok(args.includes("--read-only")) + const tmpfs = flagValue("--tmpfs") + assert.match(tmpfs ?? "", /^\/tmp:rw/) + assert.match(tmpfs ?? "", /noexec/) + // The grading copy and driver mount read-only; only the result + // directory is writable. + assert.equal(flagValue("-v"), "/tmp/evals/grading-copy:/work:ro") + assert.ok(args.some((arg) => arg === "/tmp/evals/grading-out:/out:rw")) + assert.ok(args.some((arg) => arg.endsWith(":/driver/grader-driver.mjs:ro"))) + // The pinned image and driver command close the argv. + const imageIndex = args.indexOf(GRADER_IMAGE) + assert.ok(imageIndex > 0) + assert.deepEqual(args.slice(imageIndex), [ + GRADER_IMAGE, + "node", + "/driver/grader-driver.mjs", + "/out/result.json", + "test", + ]) + // No environment passes through, so no credential can leak in. + assert.equal(args.includes("-e"), false) + assert.equal(args.includes("--env"), false) + }) + + it("reports a missing container runtime as grading failure", async () => { + const runtime = createContainerGradingRuntime({ detectExecutable: async () => undefined }) + const verdict = await runtime.run(makeJob()) + assert.deepEqual(verdict, { kind: "grading-failure", isolation: "container", reason: "noContainerRuntime" }) + }) + + it("uses the detected executable and records container isolation", async () => { + // Detection seam returns undefined here; the executable contract is + // covered by the argv test above. This test pins the isolation label. + const runtime = createContainerGradingRuntime() + assert.equal(runtime.isolation, "container") + }) +}) diff --git a/scripts/agent-evals/container-runtime.ts b/scripts/agent-evals/container-runtime.ts new file mode 100644 index 0000000000..1d5a263326 --- /dev/null +++ b/scripts/agent-evals/container-runtime.ts @@ -0,0 +1,141 @@ +import { spawn } from "node:child_process" +import fs from "node:fs/promises" +import path from "node:path" +import type { GraderJob, GraderVerdict, GradingRuntime, DriverResult } from "./coding-grader.ts" + +// Pinned grader image. The image tag is part of the grader identity and is +// recorded in every pinned comparison through the grader version. +export const GRADER_IMAGE = "node:22-bookworm" + +export type ContainerExecutable = "docker" | "podman" + +export interface ContainerRuntimeOptions { + executable?: ContainerExecutable + // Test seam: detects the container binary instead of searching PATH. + detectExecutable?: () => Promise +} + +const defaultDetect = async (): Promise => { + for (const candidate of ["docker", "podman"] as const) { + const usable = await probeExecutable(candidate) + if (usable) return candidate + } + return undefined +} + +const probeExecutable = (candidate: ContainerExecutable): Promise => + new Promise((resolve) => { + try { + const child = spawn(candidate, ["--version"], { stdio: "ignore" }) + const timer = setTimeout(() => { + child.kill("SIGKILL") + resolve(false) + }, 5_000) + child.once("error", () => { + clearTimeout(timer) + resolve(false) + }) + child.once("exit", (code) => { + clearTimeout(timer) + resolve(code === 0) + }) + } catch { + resolve(false) + } + }) + +// Builds one `run` invocation. The container has no network, no Linux +// capabilities, no privilege escalation, a read-only root filesystem, fixed +// CPU, memory, and process limits, and a size-capped scratch tmpfs. The +// grading copy mounts read-only, so agent-written code cannot change the +// hidden tests at grading time. The driver mounts read-only from repo code. +// No environment variable passes through, so no credential can leak in. +export function buildContainerArgs(job: GraderJob, outDir: string): string[] { + return [ + "run", + "--rm", + "--network", + "none", + "--cpus", + String(job.resources.cpus), + "--memory", + `${job.resources.memoryMb}m`, + "--memory-swap", + `${job.resources.memoryMb}m`, + "--pids-limit", + String(job.resources.pidsLimit), + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--read-only", + "--tmpfs", + "/tmp:rw,noexec,nosuid,size=67108864", + "-v", + `${job.gradingDir}:/work:ro`, + "-v", + `${outDir}:/out:rw`, + "-v", + `${job.driverPath}:/driver/grader-driver.mjs:ro`, + "-w", + "/work", + job.resources.image, + "node", + "/driver/grader-driver.mjs", + "/out/result.json", + "test", + ] +} + +// Default grading runtime. Runs the driver in a disposable container. The +// host enforces the wall-clock limit and kills the container on expiry. +export function createContainerGradingRuntime(options: ContainerRuntimeOptions = {}): GradingRuntime { + const detect = options.detectExecutable ?? defaultDetect + return { + isolation: "container", + async run(job: GraderJob): Promise { + const executable = await detect() + if (!executable) return { kind: "grading-failure", isolation: "container", reason: "noContainerRuntime" } + const outDir = path.join(path.dirname(job.gradingDir), "grading-out") + await fs.mkdir(outDir, { recursive: true }) + try { + const args = buildContainerArgs(job, outDir) + const exit = await new Promise((resolve) => { + const child = spawn(executable, args, { + stdio: "ignore", + timeout: job.resources.timeoutMs, + killSignal: "SIGKILL", + }) + child.once("error", () => resolve("error")) + child.once("close", (code, signal) => resolve(signal === "SIGKILL" ? "timeout" : (code ?? "error"))) + }) + if (exit === "timeout") + return { kind: "grading-failure", isolation: "container", reason: "graderTimeout" } + if (exit === "error" || exit !== 0) + return { kind: "grading-failure", isolation: "container", reason: "graderCrashed" } + const result = await readDriverResult(path.join(outDir, "result.json")) + if (!result) return { kind: "grading-failure", isolation: "container", reason: "graderResultMissing" } + return verdictFromDriver(result, "container") + } catch { + return { kind: "grading-failure", isolation: "container", reason: "graderSetupFailed" } + } finally { + await fs.rm(outDir, { recursive: true, force: true }) + } + }, + } +} + +export async function readDriverResult(outPath: string): Promise { + try { + const parsed = JSON.parse(await fs.readFile(outPath, "utf8")) as DriverResult + return parsed.kind === "completed" || parsed.kind === "driver-error" ? parsed : undefined + } catch { + return undefined + } +} + +export function verdictFromDriver(result: DriverResult, isolation: GraderVerdict["isolation"]): GraderVerdict { + if (result.kind !== "completed") return { kind: "grading-failure", isolation, reason: "driverError" } + if (result.noTestsDiscovered) return { kind: "grading-failure", isolation, reason: "noTestsDiscovered" } + return result.passed ? { kind: "passed", isolation, result } : { kind: "failed", isolation, result } +} diff --git a/scripts/agent-evals/grader-driver.mjs b/scripts/agent-evals/grader-driver.mjs new file mode 100644 index 0000000000..203894323b --- /dev/null +++ b/scripts/agent-evals/grader-driver.mjs @@ -0,0 +1,139 @@ +// Acceptance-test driver. Runs the workspace test suite with the Node test +// runner and writes one result JSON file. Test failures are data, not exit +// codes: the driver exits 0 after writing any completed result. A nonzero exit +// or a missing result file means the grader itself failed, not the solution. +// The driver receives no credential and needs no network. +import { spawn } from "node:child_process" +import fs from "node:fs" +import path from "node:path" + +const outPath = process.argv[2] +if (!outPath) { + process.stderr.write("usage: grader-driver.mjs [test-dir]\n") + process.exit(2) +} +const testDir = process.argv[3] ?? "test" + +const writeResult = (result) => { + fs.mkdirSync(path.dirname(outPath), { recursive: true }) + fs.writeFileSync(outPath, JSON.stringify(result)) +} + +const TEST_EXTENSIONS = new Set([".js", ".mjs", ".cjs"]) + +const discoverTestFiles = (dir) => { + const files = [] + const visit = (current) => { + for (const entry of fs.readdirSync(current, { withFileTypes: true })) { + const full = path.join(current, entry.name) + if (entry.isDirectory()) visit(full) + else if (entry.isFile() && TEST_EXTENSIONS.has(path.extname(entry.name)) && entry.name.includes(".test.")) + files.push(full) + } + } + visit(dir) + return files.sort() +} + +const readCounts = (tap) => { + const read = (name) => { + const match = tap.match(new RegExp(`^# ${name} (\\d+)\\s*$`, "m")) + return match ? Number(match[1]) : undefined + } + return { passed: read("pass"), failed: read("fail"), cancelled: read("cancelled") } +} + +const readFailureNames = (tap) => { + const matches = [...tap.matchAll(/^(\s*)not ok \d+ - (.+)$/gm)].map((match) => ({ + indented: match[1].length > 0, + name: match[2], + })) + // Prefer indented entries: they name the failing tests inside the files. + // Top-level entries name the files and only appear when the whole file + // failed. + const specific = matches.filter((match) => match.indented) + const chosen = specific.length > 0 ? specific : matches + return chosen.slice(0, 50).map((match) => match.name) +} + +// One child per test file. Failure names stay attributable to their file, +// which is how the grader separates hidden-test failures from visible-test +// failures. +const runOneFile = (file) => + new Promise((resolve) => { + const child = spawn(process.execPath, ["--test", "--test-reporter", "tap", file], { + cwd: process.cwd(), + stdio: ["ignore", "pipe", "pipe"], + }) + let tap = "" + child.stdout.on("data", (chunk) => { + tap += String(chunk) + }) + child.stderr.on("data", (chunk) => { + tap += String(chunk) + }) + child.once("error", () => resolve({ ok: false, tap: "" })) + child.once("close", () => resolve({ ok: true, tap })) + }) + +try { + const files = fs.existsSync(testDir) ? discoverTestFiles(testDir) : [] + if (files.length === 0) { + writeResult({ + kind: "completed", + passed: false, + noTestsDiscovered: true, + testFiles: [], + passedCount: 0, + failedCount: 0, + failures: [], + durationMs: 0, + }) + process.exit(0) + } + const started = Date.now() + let passedCount = 0 + let failedCount = 0 + let runnerBroke = false + const failures = [] + for (const file of files) { + const outcome = await runOneFile(file) + if (!outcome.ok) { + runnerBroke = true + break + } + const counts = readCounts(outcome.tap) + if (counts.passed === undefined || counts.failed === undefined) { + // A file without a TAP summary means the runner never reported. + runnerBroke = true + break + } + passedCount += counts.passed + failedCount += counts.failed + if (counts.failed > 0 && failures.length < 50) + for (const name of readFailureNames(outcome.tap)) failures.push({ name, file }) + } + if (runnerBroke) { + writeResult({ kind: "driver-error", message: "testRunnerUnavailable" }) + process.exit(0) + } + writeResult({ + kind: "completed", + passed: failedCount === 0 && passedCount > 0, + noTestsDiscovered: false, + testFiles: files, + passedCount, + failedCount, + failures, + durationMs: Date.now() - started, + }) + process.exit(0) +} catch (error) { + // The driver itself crashed. Record it as a grader failure, never as a + // solution failure. + writeResult({ + kind: "driver-error", + message: error instanceof Error ? error.name : "driver-error", + }) + process.exit(0) +} diff --git a/scripts/agent-evals/launcher.ts b/scripts/agent-evals/launcher.ts index 1edd2917d2..e23f21f29c 100644 --- a/scripts/agent-evals/launcher.ts +++ b/scripts/agent-evals/launcher.ts @@ -7,7 +7,8 @@ import { downloadAndUnzipVSCode } from "@vscode/test-electron" import { IpcClient } from "@roo-code/ipc" import { IpcMessageType } from "@roo-code/types" import { buildChildEnvironment, buildLaunchArgs } from "./isolation.ts" -import type { AgentClient, AgentHost, AgentRuntime } from "./runner.ts" +import type { AgentClient, AgentHost, AgentRuntime, FinalWorkspaceSnapshot } from "./runner.ts" +import type { AgentFixtureWorkspaceFile } from "./contract.ts" import { identityUnavailable, measured, type MeasuredString, type TraceRunIdentity } from "./trace.ts" const execFileAsync = promisify(execFile) @@ -46,6 +47,51 @@ const readJsonField = async ( } } +const FINAL_WORKSPACE_MAX_FILES = 4_096 +const FINAL_WORKSPACE_MAX_FILE_BYTES = 1_048_576 +const FINAL_WORKSPACE_MAX_TOTAL_BYTES = 8_388_608 +const FINAL_WORKSPACE_SKIPPED_DIRS = new Set(["node_modules", ".git"]) + +// Bounded final-workspace capture. The walker skips dependency and VCS +// directories, caps file count and size, and reports truncation instead of +// reading unbounded agent-written trees. Paths stay relative and forward +// slashed, matching the fixture workspace file shape. +async function readWorkspaceFiles(workspaceRoot: string): Promise { + const files: AgentFixtureWorkspaceFile[] = [] + let truncated = false + let totalBytes = 0 + const visit = async (dir: string, prefix: string): Promise => { + const entries = await fs.readdir(dir, { withFileTypes: true }) + for (const entry of entries) { + if (files.length >= FINAL_WORKSPACE_MAX_FILES || totalBytes > FINAL_WORKSPACE_MAX_TOTAL_BYTES) { + truncated = true + return + } + const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name + const full = path.join(dir, entry.name) + if (entry.isDirectory()) { + if (FINAL_WORKSPACE_SKIPPED_DIRS.has(entry.name)) continue + await visit(full, relativePath) + continue + } + if (!entry.isFile()) continue + const stat = await fs.stat(full) + if ( + stat.size > FINAL_WORKSPACE_MAX_FILE_BYTES || + totalBytes + stat.size > FINAL_WORKSPACE_MAX_TOTAL_BYTES + ) { + truncated = true + continue + } + const content = await fs.readFile(full, "utf8") + totalBytes += stat.size + files.push({ path: relativePath, content }) + } + } + await visit(workspaceRoot, "") + return { files, truncated } +} + // The only place where VS Code is downloaded, spawned, isolated and torn // down. Both the smoke driver and future adapters get the same launcher, so // process and credential isolation cannot drift between entry points. @@ -185,5 +231,9 @@ export function createHostRuntime(options: HostRuntimeOptions): AgentRuntime { arch: fallback.arch, } }, + async readFinalWorkspace(): Promise { + if (!temp) throw new Error("workspace not created") + return readWorkspaceFiles(path.join(temp, "workspace")) + }, } } diff --git a/scripts/agent-evals/runner.ts b/scripts/agent-evals/runner.ts index 5c12961db3..df6836cfd6 100644 --- a/scripts/agent-evals/runner.ts +++ b/scripts/agent-evals/runner.ts @@ -37,6 +37,11 @@ export interface AgentClient { // The adapter owns processes, sockets, files and credentials. The controller // only sees these operations, so tests can drive it with synthetic IPC and a // deterministic clock. +export interface FinalWorkspaceSnapshot { + files: readonly AgentFixtureWorkspaceFile[] + truncated: boolean +} + export interface AgentRuntime { launch(workspaceFiles: readonly AgentFixtureWorkspaceFile[]): Promise connect(): AgentClient @@ -46,6 +51,10 @@ export interface AgentRuntime { now(): number pause(ms: number): Promise readIdentity?(): Promise + // Reads the final workspace as bounded file data. Graders consume it from a + // fresh copy; the launcher's scratch paths never leak. Capture must happen + // before cleanup removes the directory. + readFinalWorkspace?(): Promise } export interface AgentRunObservers { From 5a92a9b0f0ac388d5bff2e9933dfac74faadd26d Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Fri, 23 Oct 2026 03:55:33 +0000 Subject: [PATCH 2/9] fix(evals): secure coding comparisons and correct grading and cost accounting --- scripts/agent-evals/README.md | 31 +- scripts/agent-evals/coding-compare.mts | 118 ++++++-- scripts/agent-evals/coding-compare.ts | 22 ++ scripts/agent-evals/coding-fixtures.test.ts | 18 +- scripts/agent-evals/coding-grader.test.ts | 34 +++ scripts/agent-evals/coding-grader.ts | 35 ++- scripts/agent-evals/coding-metrics.test.ts | 58 ++++ scripts/agent-evals/coding-metrics.ts | 47 +-- scripts/agent-evals/coding-pins.test.ts | 6 +- scripts/agent-evals/coding-pins.ts | 13 +- .../coding-reference-solutions.test.ts | 2 +- scripts/agent-evals/coding-report.test.ts | 41 ++- scripts/agent-evals/coding-run.test.ts | 111 ++++++- scripts/agent-evals/coding-run.ts | 40 ++- scripts/agent-evals/coding-side.mts | 97 ++++-- scripts/agent-evals/coding-trust.test.ts | 98 ++++++ scripts/agent-evals/coding-trust.ts | 100 +++++++ scripts/agent-evals/container-runtime.test.ts | 260 +++++++++++++++- scripts/agent-evals/container-runtime.ts | 279 ++++++++++++++++-- scripts/agent-evals/fs-atomic.test.ts | 52 ++++ scripts/agent-evals/fs-atomic.ts | 26 ++ scripts/agent-evals/grader-driver.mjs | 8 +- scripts/agent-evals/launcher.ts | 8 +- scripts/agent-evals/runner.ts | 30 +- 24 files changed, 1383 insertions(+), 151 deletions(-) create mode 100644 scripts/agent-evals/coding-trust.test.ts create mode 100644 scripts/agent-evals/coding-trust.ts create mode 100644 scripts/agent-evals/fs-atomic.test.ts create mode 100644 scripts/agent-evals/fs-atomic.ts diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index afc55e6c23..da23438af4 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -114,25 +114,42 @@ release process: a maintainer runs it locally or on demand. | `coding-stats.ts` | Three repetitions, success rates with Wilson intervals, paired differences. | | `coding-report.ts` | The self-contained JSON report and the Markdown renderer. | | `coding-compare.ts` | The comparison orchestration and the side result shape. | -| `coding-side.mts` | One comparison side: all fixtures times N repetitions on this commit, graded, with pinned inputs. | +| `coding-side.mts` | One comparison side: all fixtures times N repetitions on the evaluated root, graded, with pinned inputs. | | `coding-compare.mts` | Two-commit comparison driver. Writes `comparison-report.json` and `comparison-report.md`. | +| `coding-trust.ts` | The comparison trust policy: approved full commit SHAs, resolution, the approval gate, and the side environment allowlist. | +| `fs-atomic.ts` | Atomic file publication through same-directory temp files and renames. | ### Grading - The grader is model-free and credential-free. It judges the final workspace state only. - The final workspace is captured as bounded file data and rebuilt as a fresh copy. Hidden acceptance tests enter only that copy, so the agent can never read or change them, and one run can never inherit another run's files. -- The default runtime executes the driver in a disposable container with no network, no capabilities, no privilege escalation, a read-only root filesystem, and fixed CPU, memory, and process limits. No environment variable passes through. Only trusted repo code runs with the provider credential. Fixtures and extension code from outside pull requests never do. -- The grader has its own wall-clock limit. A grader that hangs, crashes, or loses its runner is a `grading failure`. It is never an `incorrect` solution. +- The default runtime executes the driver in a uniquely named disposable container with no network, no capabilities, no privilege escalation, a read-only root filesystem, and fixed CPU, memory, and process limits. No environment variable passes through. Only trusted repo code runs with the provider credential. Fixtures and extension code from outside pull requests never do. +- The grader has its own wall-clock limit. On expiry the runtime removes the container by name and verifies the removal before it deletes any output. A grader that hangs, crashes, or loses its runner is a `grading failure`. It is never an `incorrect` solution. The image pull happens before the timed run and never consumes grading time. The runtime resolves the immutable grader image digest from the local engine and the side records it in the comparison pins; it never fabricates one. +- The driver's result file is untrusted data: it is written inside the container where agent-written test code executes. The host reads it with a size bound and validates shape, bounds, and internal consistency. An invalid or inconsistent result is a `grading failure`, never a verdict. - Each fixture ships two reference solutions. The known-correct solution must pass. The deliberately incorrect one passes the visible tests and must fail. Unit tests enforce both on every fixture. - `createLocalSubprocessGradingRuntime` runs the same driver directly on the host. It does not give container isolation. It exists for unit tests and explicit local checks only and must never back a recorded comparison. ### Comparisons -A paired comparison pins both commit SHAs, the suite digest, the settings digest, the model identifier, the grader version, and the runner environment on both sides. Any difference in a pinned value marks the pair incompatible. The report records the values and flags the pair instead of dropping it. +A paired comparison pins both commit SHAs, the suite digest, the settings digest, the model identifier, the grader version, the resolved grader image digest, and the runner environment on both sides. Any difference in a pinned value marks the pair incompatible. The report records the values and flags the pair instead of dropping it. The default is three repetitions per fixture per commit. Reports show success rates with Wilson 95% intervals and paired differences. They make no significance claim: three repetitions can show large regressions, not small improvements. -Runs with incomplete token or cost accounting are flagged and excluded from aggregate cost comparisons. Missing token or cost values stay missing. The report states the price-table identifier and date and how cache reads and writes are priced. Cost per successful task divides the total attempt cost by the number of `correct` runs. It is unavailable when no run is correct or a correct run lacks complete cost data. Duplicate-call detection and HTTP retry counts are reported as unavailable. +Runs with incomplete token or cost accounting are flagged and excluded from aggregate cost comparisons. Missing token or cost values stay missing. The report states the price-table identifier and date and how cache reads and writes are priced. Cost per successful task divides the total attempt cost by the number of `correct` runs. It is unavailable when no run is correct, when a correct run lacks complete cost data, or when any attempt, including a failed one, lacks complete cost data. Duplicate-call detection and HTTP retry counts are reported as unavailable. + +### Trust policy for comparisons + +A comparison runs the extension code of two commits with the provider credential. A commit runs only when its full 40-hex SHA is listed in an explicit trust policy file. Ref names are resolved first and checked against the policy before any worktree is created or any process starts. A policy entry is a full SHA; short SHAs and ref names are rejected. + +```json +{ "version": 1, "approvedCommits": [""] } +``` + +The comparison driver, fixtures, grader, and report builder all run from the trusted checkout that invokes `coding-evals:compare`. Only the extension host loads an evaluated worktree. The side process receives only the allowlisted environment plus the one named credential variable. Each side installs dependencies from its own worktree's frozen lockfile and builds its own exact commit, so each side runs its own compiled extension. Each side has its own wall clock (`--side-timeout-minutes`, default 120). + +### Output publication contract + +Artifacts are published atomically: content lands in a same-directory temp file and is renamed into place. A reader never sees a partial file. `side-results.json` appears only after every repetition of that side finished, so its existence means a complete side. A missing `side-results.json` means an interrupted side; consumers must treat it as a failed side and never pair partial output. The comparison reports are written atomically and only after both sides published complete results; a failed comparison leaves no report files. ### Commands @@ -145,8 +162,8 @@ pnpm agent-evals:check-types # environment variable and the container runtime for grading. pnpm coding-evals:side --reps 3 --out /tmp/side-out -# Full paired comparison of two commits. -pnpm coding-evals:compare --baseline --candidate --reps 3 --out /tmp/compare-out +# Full paired comparison of two approved commits. +pnpm coding-evals:compare --baseline --candidate --trust-file trust-policy.json --reps 3 --out /tmp/compare-out ``` ### Follow-up milestones diff --git a/scripts/agent-evals/coding-compare.mts b/scripts/agent-evals/coding-compare.mts index 771c5c48b6..89bd85fa64 100644 --- a/scripts/agent-evals/coding-compare.mts +++ b/scripts/agent-evals/coding-compare.mts @@ -1,20 +1,35 @@ -// Paired comparison driver. Checks out both commits into temporary worktrees, -// runs one coding-evals side per commit with identical repetition counts, -// then writes the JSON and Markdown comparison reports. Both commits must -// contain this framework. The driver is trusted repo code and only runs -// trusted repo commits with the credential. +// Paired comparison driver. Verifies both commits against an explicit trust +// policy, checks out each approved commit into a temporary worktree, prepares +// pinned dependencies and builds that exact commit, runs one trusted coding +// side per commit against its worktree with identical repetition counts, then +// writes the JSON and Markdown comparison reports. +// +// Trust model. The side script, fixtures, grader, and report builder all run +// from this trusted checkout. Only the extension host loads the evaluated +// worktree. The credential never leaves the trusted processes except through +// the one named provider variable. A commit runs only when its full SHA is +// listed in the operator's trust policy; refs are resolved and checked +// against the policy before any worktree or process exists. import { execFile, spawn } from "node:child_process" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" import { fileURLToPath } from "node:url" import { promisify } from "node:util" -import { sideResultFileName, toSideResult, type SideResultFile } from "./coding-compare.ts" +import { sideResultFileName, toSideResult, validateSideResultFile, type SideResultFile } from "./coding-compare.ts" import { buildComparisonReport, renderMarkdownReport } from "./coding-report.ts" +import { assertCommitsApproved, buildSideEnvironment, loadTrustPolicy, resolveCommit } from "./coding-trust.ts" +import { writeJsonAtomic, writeFileAtomic } from "./fs-atomic.ts" const execFileAsync = promisify(execFile) const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") +const INSTALL_TIMEOUT_MS = 10 * 60_000 +const BUILD_TIMEOUT_MS = 30 * 60_000 +const WORKTREE_TIMEOUT_MS = 120_000 +const SIDE_KILL_GRACE_MS = 30_000 +const DEFAULT_SIDE_TIMEOUT_MINUTES = 120 + const readArgs = (): { baseline: string candidate: string @@ -22,6 +37,8 @@ const readArgs = (): { out: string credentialEnv: string allowUnisolatedGrading: boolean + trustFile: string + sideTimeoutMs: number } => { const args = process.argv.slice(2) const read = (name: string): string | undefined => { @@ -31,10 +48,16 @@ const readArgs = (): { const baseline = read("baseline") const candidate = read("candidate") const out = read("out") + const trustFile = read("trust-file") if (!baseline || !candidate) throw new Error("Set --baseline and --candidate to commit SHAs") if (!out) throw new Error("Set --out to a scratch directory") + if (!trustFile) throw new Error("Set --trust-file to a policy file listing the approved full commit SHAs") const reps = Number(read("reps") ?? "3") if (!Number.isInteger(reps) || reps < 1) throw new Error("--reps must be a positive integer") + const sideTimeoutMinutes = Number(read("side-timeout-minutes") ?? String(DEFAULT_SIDE_TIMEOUT_MINUTES)) + if (!Number.isInteger(sideTimeoutMinutes) || sideTimeoutMinutes < 1) { + throw new Error("--side-timeout-minutes must be a positive integer") + } return { baseline, candidate, @@ -42,9 +65,26 @@ const readArgs = (): { out: path.resolve(out), credentialEnv: read("credential-env") ?? "SOLHEIM_API_KEY", allowUnisolatedGrading: args.includes("--allow-unisolated-grading"), + trustFile: path.resolve(trustFile), + sideTimeoutMs: sideTimeoutMinutes * 60_000, } } +const runBounded = async (command: string, args: readonly string[], cwd: string, timeoutMs: number): Promise => { + await execFileAsync(command, [...args], { cwd, timeout: timeoutMs, killSignal: "SIGKILL" }) +} + +// Prepares the worktree of one evaluated commit. Dependencies install from +// the worktree's own frozen lockfile, and that exact commit's extension and +// webview build from its own sources. No artifact crosses sides. +const prepareWorktree = async (worktree: string): Promise => { + await runBounded("pnpm", ["install", "--frozen-lockfile"], worktree, INSTALL_TIMEOUT_MS) + await runBounded("pnpm", ["build"], worktree, BUILD_TIMEOUT_MS) +} + +// Runs one trusted side against the evaluated worktree. The side script comes +// from this checkout, receives only the allowlisted environment plus the +// named credential variable, and must finish inside its own wall clock. const runSide = async ( side: "baseline" | "candidate", commit: string, @@ -52,48 +92,90 @@ const runSide = async ( scratchDir: string, ): Promise => { const worktree = path.join(scratchDir, side) - await execFileAsync("git", ["worktree", "add", "--detach", worktree, commit], { cwd: root, timeout: 120_000 }) + await runBounded("git", ["worktree", "add", "--detach", worktree, commit], root, WORKTREE_TIMEOUT_MS) + await prepareWorktree(worktree) const sideOut = path.join(scratchDir, `${side}-out`) await fs.mkdir(sideOut, { recursive: true }) - const exit = await new Promise((resolve, reject) => { + const exit = await new Promise((resolve) => { const child = spawn( process.execPath, [ "--import", "tsx", - "scripts/agent-evals/coding-side.mts", + path.join(root, "scripts/agent-evals/coding-side.mts"), "--reps", String(inputs.reps), "--out", sideOut, "--credential-env", inputs.credentialEnv, + "--evaluated-root", + worktree, ...(inputs.allowUnisolatedGrading ? ["--allow-unisolated-grading"] : []), ], { - cwd: worktree, - env: { ...process.env, CODING_EVALS_SIDE: side }, + cwd: root, + env: buildSideEnvironment(process.env, inputs.credentialEnv, side), stdio: "inherit", + detached: true, }, ) - child.once("error", reject) - child.once("exit", (code) => resolve(code ?? 1)) + let timedOut = false + const timer = setTimeout(() => { + timedOut = true + try { + if (child.pid) process.kill(-child.pid, "SIGTERM") + } catch { + child.kill("SIGTERM") + } + setTimeout(() => { + try { + if (child.pid) process.kill(-child.pid, "SIGKILL") + } catch { + child.kill("SIGKILL") + } + }, SIDE_KILL_GRACE_MS).unref() + }, inputs.sideTimeoutMs) + child.once("error", () => { + clearTimeout(timer) + resolve(1) + }) + child.once("exit", (code) => { + clearTimeout(timer) + resolve(timedOut ? 1 : (code ?? 1)) + }) }) if (exit !== 0) throw new Error(`${side} side failed with exit code ${exit}`) - return JSON.parse(await fs.readFile(path.join(sideOut, sideResultFileName(side)), "utf8")) as SideResultFile + const parsed: unknown = JSON.parse(await fs.readFile(path.join(sideOut, sideResultFileName(side)), "utf8")) + // The side output must describe exactly the approved commit, or the pair + // is invalid. + return validateSideResultFile(parsed, commit) } const inputs = readArgs() +if (process.env[inputs.credentialEnv] === undefined || process.env[inputs.credentialEnv]?.length === 0) { + throw new Error(`provider credential environment variable ${inputs.credentialEnv} is unset or empty`) +} +const policy = await loadTrustPolicy(inputs.trustFile) +// Trust gate. Both commits resolve to full SHAs and must be approved before +// any worktree is created or any code runs. +const baselineCommit = await resolveCommit(inputs.baseline, root) +const candidateCommit = await resolveCommit(inputs.candidate, root) +assertCommitsApproved(policy, [baselineCommit, candidateCommit]) + const scratchDir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-evals-compare-")) await fs.mkdir(inputs.out, { recursive: true }) try { const report = buildComparisonReport({ - baseline: toSideResult(await runSide("baseline", inputs.baseline, inputs, scratchDir)), - candidate: toSideResult(await runSide("candidate", inputs.candidate, inputs, scratchDir)), + baseline: toSideResult(await runSide("baseline", baselineCommit, inputs, scratchDir)), + candidate: toSideResult(await runSide("candidate", candidateCommit, inputs, scratchDir)), repetitions: inputs.reps, }) - await fs.writeFile(path.join(inputs.out, "comparison-report.json"), JSON.stringify(report, null, 2)) - await fs.writeFile(path.join(inputs.out, "comparison-report.md"), renderMarkdownReport(report)) + // The reports are the comparison's publication point. They appear + // atomically and only after both sides published complete results; any + // failure above leaves no report files behind. + await writeJsonAtomic(path.join(inputs.out, "comparison-report.json"), report) + await writeFileAtomic(path.join(inputs.out, "comparison-report.md"), renderMarkdownReport(report)) console.log( report.compatibility.compatible ? `comparison complete: compatible pair, report in ${inputs.out}` diff --git a/scripts/agent-evals/coding-compare.ts b/scripts/agent-evals/coding-compare.ts index 56357f419d..0a8b9cb04e 100644 --- a/scripts/agent-evals/coding-compare.ts +++ b/scripts/agent-evals/coding-compare.ts @@ -52,6 +52,28 @@ export interface SideResultFile { runs: CodingRunRecord[] } +// Validates one side result file before the report builder consumes it. The +// file is data, so its shape is checked, and it must record exactly the +// approved commit of its side. Anything else aborts the comparison instead of +// producing a report from partial output. +export function validateSideResultFile(parsed: unknown, expectedCommit: string): SideResultFile { + if (typeof parsed !== "object" || parsed === null) throw new Error("side result is not a JSON object") + const record = parsed as Record + const pins = record["pins"] + if (typeof pins !== "object" || pins === null) throw new Error("side result has no pins") + const pinsRecord = pins as Record + for (const field of ["side", "commit", "suiteId", "suiteDigest", "graderVersion"] as const) { + if (typeof pinsRecord[field] !== "string" || (pinsRecord[field] as string).length === 0) { + throw new Error(`side result pins lack ${field}`) + } + } + if (pinsRecord["commit"] !== expectedCommit) { + throw new Error(`side result records commit ${String(pinsRecord["commit"])} instead of ${expectedCommit}`) + } + if (!Array.isArray(record["runs"])) throw new Error("side result has no runs array") + return parsed as SideResultFile +} + // Reduces a side result file to the shape the report builder consumes. export const toSideResult = (file: SideResultFile): SideResult => ({ pins: file.pins, diff --git a/scripts/agent-evals/coding-fixtures.test.ts b/scripts/agent-evals/coding-fixtures.test.ts index d91d63b474..48aa0f7222 100644 --- a/scripts/agent-evals/coding-fixtures.test.ts +++ b/scripts/agent-evals/coding-fixtures.test.ts @@ -69,12 +69,28 @@ describe("coding fixture suite", () => { } }) + it("keeps the settings projection equal to the built configuration", () => { + // The suite and settings digests hash the projection, so the built + // configuration must never drift from it. The credential is removed + // and the serialized extra body is parsed before the comparison. + for (const bundle of CODING_FIXTURE_BUNDLES) { + const built: Record = { + ...(bundle.fixture.buildConfiguration(CREDENTIAL) as Record), + } + assert.equal(typeof built["openAiApiKey"], "string") + delete built["openAiApiKey"] + assert.equal(typeof built["openAiExtraBody"], "string") + built["openAiExtraBody"] = JSON.parse(built["openAiExtraBody"] as string) + assert.deepEqual(built, bundle.fixture.settingsProjection) + } + }) + it("produces a stable suite digest that tracks the grader version", () => { assert.equal(codingSuiteDigest(), codingSuiteDigest()) assert.equal(codingBundle("clamp-range").fixture.identity.name, "clamp-range") assert.throws(() => codingBundle("unknown-fixture")) assert.ok(codingSuiteDigest().length === 64) assert.equal(CODING_SUITE_ID, "coding-basic-v1") - assert.equal(GRADER_VERSION, "coding-grader-v1") + assert.equal(GRADER_VERSION, "coding-grader-v2") }) }) diff --git a/scripts/agent-evals/coding-grader.test.ts b/scripts/agent-evals/coding-grader.test.ts index dd60644366..ec2e946c13 100644 --- a/scripts/agent-evals/coding-grader.test.ts +++ b/scripts/agent-evals/coding-grader.test.ts @@ -28,6 +28,7 @@ const driverCompleted = (overrides: Partial = {}): DriverComple testFiles: [], passedCount: 3, failedCount: 0, + cancelledCount: 0, failures: [], durationMs: 1, ...overrides, @@ -125,6 +126,39 @@ describe("runGrader", () => { await fs.rm(scratch, { recursive: true, force: true }) } }) + + it("separates a grader runtime crash from a grading copy failure", async () => { + const scratch = await tmpdir() + try { + const runtime: GradingRuntime = { + isolation: "container", + run: async () => { + throw new Error("runtime exploded") + }, + } + const verdict = await runGrader({ + finalFiles: [{ path: "marker.txt", content: "x" }], + hiddenTests: [], + resources: { timeoutMs: 1_000, memoryMb: 256, cpus: 1, pidsLimit: 64, image: "node:22-bookworm" }, + runtime, + scratchDir: scratch, + }) + // A runtime exception must not be reported as a copy failure, and + // a failed cleanup must not replace the verdict. + assert.deepEqual(verdict, { + kind: "grading-failure", + isolation: "container", + reason: "graderRuntimeFailed", + }) + const gradingCopyGone = await fs + .access(path.join(scratch, "grading-copy")) + .then(() => false) + .catch(() => true) + assert.equal(gradingCopyGone, true) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) }) describe("local subprocess runtime failure paths", () => { diff --git a/scripts/agent-evals/coding-grader.ts b/scripts/agent-evals/coding-grader.ts index 3950bc9168..0f9800ee32 100644 --- a/scripts/agent-evals/coding-grader.ts +++ b/scripts/agent-evals/coding-grader.ts @@ -4,11 +4,12 @@ import fs from "node:fs/promises" import path from "node:path" import type { AgentFixtureWorkspaceFile } from "./contract.ts" import { buildGradingCopy } from "./coding-workspace.ts" -import { readDriverResult, verdictFromDriver } from "./container-runtime.ts" +import { loadDriverResult, verdictFromDriver } from "./container-runtime.ts" +import type { Measured } from "./trace.ts" // The grader version is a pinned comparison input. Any change to the driver, // the image, or the verdict rules must bump it. -export const GRADER_VERSION = "coding-grader-v1" +export const GRADER_VERSION = "coding-grader-v2" export interface GraderResources { timeoutMs: number @@ -25,6 +26,9 @@ export interface DriverCompleted { testFiles: string[] passedCount: number failedCount: number + // Tests the runner cancelled. A cancelled test never ran, so a passing + // verdict requires zero cancellations. + cancelledCount: number failures: { name: string; file: string }[] durationMs: number } @@ -53,6 +57,9 @@ export type GraderVerdict = export interface GradingRuntime { readonly isolation: GradingIsolation run(job: GraderJob): Promise + // Resolves the immutable grader image identity for the comparison pins. + // Optional: runtimes without an image report unavailability. + imageDigest?(): Promise> } export const driverFilePath = (): string => fileURLToPath(new URL("./grader-driver.mjs", import.meta.url)) @@ -71,18 +78,26 @@ export async function runGrader(inputs: { scratchDir: string }): Promise { const gradingDir = path.join(inputs.scratchDir, "grading-copy") + // Cleanup is best effort. A failed removal must never replace the verdict. + const removeCopy = (): Promise => fs.rm(gradingDir, { recursive: true, force: true }).catch(() => {}) + let verdict: GraderVerdict try { await buildGradingCopy(inputs.finalFiles, inputs.hiddenTests, gradingDir) - return await inputs.runtime.run({ + } catch { + await removeCopy() + return { kind: "grading-failure", isolation: inputs.runtime.isolation, reason: "gradingCopyFailed" } + } + try { + verdict = await inputs.runtime.run({ gradingDir, driverPath: driverFilePath(), resources: inputs.resources, }) } catch { - return { kind: "grading-failure", isolation: inputs.runtime.isolation, reason: "gradingCopyFailed" } - } finally { - await fs.rm(gradingDir, { recursive: true, force: true }) + verdict = { kind: "grading-failure", isolation: inputs.runtime.isolation, reason: "graderRuntimeFailed" } } + await removeCopy() + return verdict } const INPROCESS_ENV_ALLOWLIST = ["PATH", "SYSTEMROOT", "WINDIR", "COMSPEC", "PATHEXT"] as const @@ -120,10 +135,12 @@ export function createLocalSubprocessGradingRuntime(nodeExecutable = process.exe return { kind: "grading-failure", isolation: "local-subprocess", reason: "graderTimeout" } if (exit === "error" || exit !== 0) return { kind: "grading-failure", isolation: "local-subprocess", reason: "graderCrashed" } - const result = await readDriverResult(resultPath) - if (!result) + const loaded = await loadDriverResult(resultPath) + if (loaded === "missing") return { kind: "grading-failure", isolation: "local-subprocess", reason: "graderResultMissing" } - return verdictFromDriver(result, "local-subprocess") + if (loaded === "invalid") + return { kind: "grading-failure", isolation: "local-subprocess", reason: "graderResultInvalid" } + return verdictFromDriver(loaded, "local-subprocess") } finally { await fs.rm(outDir, { recursive: true, force: true }) } diff --git a/scripts/agent-evals/coding-metrics.test.ts b/scripts/agent-evals/coding-metrics.test.ts index 9b20ff52dd..f28d56ce07 100644 --- a/scripts/agent-evals/coding-metrics.test.ts +++ b/scripts/agent-evals/coding-metrics.test.ts @@ -168,6 +168,46 @@ describe("run metrics", () => { assert.equal(metrics.cost.accountingComplete, false) }) + it("marks accounting incomplete when trace completeness is partial even with a configured price table", () => { + // Usage is fully reported and the table prices the model, but some + // requests lack usage, so the trace accounting is partial. The cost + // estimate exists; the accounting is still incomplete. + const metrics = buildRunMetrics({ + fixtureName: "clamp-range", + repIndex: 0, + trace: trace({ + accounting: accounting({ + requestCount: 2, + requestsMissingUsage: 1, + usage: { available: true, snapshotsObserved: 1, lastSnapshot: fullUsage }, + completeness: { status: "partial", reasons: ["usageMissingForSomeRequests"] }, + }), + }), + outcome: correctOutcome, + modelIdentifier: { available: true, value: "qwen3.8-27b" }, + priceTable: testTable, + }) + assert.equal(metrics.cost.estimated.available, true) + assert.equal(metrics.accounting.status, "partial") + assert.equal(metrics.cost.accountingComplete, false) + }) + + it("reports a missing model identifier without an empty-string fallback", () => { + const metrics = buildRunMetrics({ + fixtureName: "clamp-range", + repIndex: 0, + trace: trace({ + accounting: accounting({ usage: { available: true, snapshotsObserved: 1, lastSnapshot: fullUsage } }), + }), + outcome: correctOutcome, + modelIdentifier: { available: false, reason: "modelIdMissingInProjection" }, + priceTable: testTable, + }) + assert.equal(metrics.cost.estimated.available, false) + if (!metrics.cost.estimated.available) assert.equal(metrics.cost.estimated.reason, "modelIdentifierMissing") + assert.equal(metrics.cost.accountingComplete, false) + }) + it("summarizes tool calls and failures from the tool snapshot", () => { const metrics = buildRunMetrics({ fixtureName: "clamp-range", @@ -302,4 +342,22 @@ describe("cost per successful task", () => { assert.equal(result.excludedRuns, 1) assert.equal(result.policy, "excludeIncompleteFromAggregates-v1") }) + + it("is unavailable when a failed attempt lacks complete cost accounting", () => { + // One success, one known-cost failure, one unknown-cost failure. The + // unknown attempt cost makes the total spend per success unknowable. + const result = costPerSuccessfulTask([ + metricsRun({ repIndex: 0, cost: completeCost(0.5) }), + metricsRun({ + repIndex: 1, + outcome: { outcome: "incorrect", reason: "hiddenTestsFailed" }, + cost: completeCost(0.3), + }), + metricsRun({ repIndex: 2, outcome: { outcome: "incorrect", reason: "hiddenTestsFailed" } }), + ]) + assert.equal(result.available, false) + assert.equal(result.reason, "failedAttemptCostIncomplete") + assert.equal(result.correctRuns, 1) + assert.equal(result.excludedRuns, 1) + }) }) diff --git a/scripts/agent-evals/coding-metrics.ts b/scripts/agent-evals/coding-metrics.ts index 3b8898075e..574cbbb8eb 100644 --- a/scripts/agent-evals/coding-metrics.ts +++ b/scripts/agent-evals/coding-metrics.ts @@ -77,16 +77,21 @@ export function buildRunMetrics(input: { const trace = input.trace const accounting = trace?.accounting const last = accounting?.usage.lastSnapshot - const estimated: EstimatedCost = last - ? estimateRunCost( - input.modelIdentifier.available ? input.modelIdentifier.value : "", - { available: true, value: last.tokensIn }, - { available: true, value: last.tokensOut }, - last.cacheReads, - last.cacheWrites, - input.priceTable, - ) - : { available: false, reason: last ? "modelIdentifierMissing" : "noUsageEventsObserved" } + // Usage completeness stays separate from price availability. The reasons + // name the first missing input: no usage snapshot, then no model + // identifier, then the price table decision. + const estimated: EstimatedCost = !last + ? { available: false, reason: "noUsageEventsObserved" } + : !input.modelIdentifier.available + ? { available: false, reason: "modelIdentifierMissing" } + : estimateRunCost( + input.modelIdentifier.value, + { available: true, value: last.tokensIn }, + { available: true, value: last.tokensOut }, + last.cacheReads, + last.cacheWrites, + input.priceTable, + ) const snapshot = accounting?.toolSnapshot.snapshot const byTool: Record = {} let totalAttempts = 0 @@ -101,7 +106,10 @@ export function buildRunMetrics(input: { providerReported: last ? { available: true, value: last.cost } : { available: false, reason: "noUsageEventsObserved" }, - accountingComplete: estimated.available, + // Complete requires both: the trace accounted for every request, and + // the price table could price the usage. Partial trace accounting with + // a configured price stays incomplete. + accountingComplete: estimated.available && accounting?.completeness.status === "complete", } return { fixtureName: input.fixtureName, @@ -167,10 +175,11 @@ export interface CostPerSuccessfulTask { usd?: number } -// Total estimated cost of all attempts with complete cost accounting, divided -// by the number of correct runs. Runs with incomplete cost accounting are -// flagged and excluded from aggregates. If no run is correct, or if any -// correct run lacks complete cost accounting, the value is unavailable. +// Total estimated cost of all attempts, divided by the number of correct +// runs. The value is unavailable when any attempt lacks complete cost +// accounting, including failed attempts: an unknown attempt cost makes the +// total spend per success unknowable, not merely smaller. Usage completeness +// and price availability both feed accountingComplete. export function costPerSuccessfulTask(runs: readonly CodingRunMetrics[]): CostPerSuccessfulTask { const included = runs.filter((run) => run.cost.accountingComplete) const excluded = runs.length - included.length @@ -182,10 +191,10 @@ export function costPerSuccessfulTask(runs: readonly CodingRunMetrics[]): CostPe correctRuns, } if (correctRuns === 0) return { ...base, available: false, reason: "noCorrectRuns" } - const correctIncomplete = runs.filter( - (run) => run.outcome.outcome === "correct" && !run.cost.accountingComplete, - ).length - if (correctIncomplete > 0) return { ...base, available: false, reason: "correctRunCostIncomplete" } + const correctIncomplete = runs.some((run) => run.outcome.outcome === "correct" && !run.cost.accountingComplete) + if (correctIncomplete) return { ...base, available: false, reason: "correctRunCostIncomplete" } + const failedIncomplete = runs.some((run) => run.outcome.outcome !== "correct" && !run.cost.accountingComplete) + if (failedIncomplete) return { ...base, available: false, reason: "failedAttemptCostIncomplete" } const total = included.reduce((sum, run) => { if (run.cost.estimated.available) return sum + run.cost.estimated.usd return sum diff --git a/scripts/agent-evals/coding-pins.test.ts b/scripts/agent-evals/coding-pins.test.ts index 10a8bd2179..88e9ed6ddf 100644 --- a/scripts/agent-evals/coding-pins.test.ts +++ b/scripts/agent-evals/coding-pins.test.ts @@ -9,7 +9,8 @@ const pins = (side: "baseline" | "candidate", overrides: Partial = {}) suiteDigest: "d1".repeat(32), settingsDigest: "d2".repeat(32), modelIdentifier: "qwen3.8-27b", - graderVersion: "coding-grader-v1", + graderVersion: "coding-grader-v2", + graderImageDigest: `node:22-bookworm@sha256:${"e".repeat(64)}`, runnerEnvironment: { os: "linux", nodeVersion: "v22.23.1", vscodeTarget: "1.104.0" }, ...overrides, }) @@ -27,7 +28,8 @@ describe("pinned comparison inputs", () => { { suiteDigest: "zz".repeat(32) }, { settingsDigest: "zz".repeat(32) }, { modelIdentifier: "other-model" }, - { graderVersion: "coding-grader-v2" }, + { graderVersion: "coding-grader-v9" }, + { graderImageDigest: "unavailable" }, { runnerEnvironment: { os: "darwin", nodeVersion: "v22.23.1", vscodeTarget: "1.104.0" } }, { runnerEnvironment: { os: "linux", nodeVersion: "v24.0.0", vscodeTarget: "1.104.0" } }, { runnerEnvironment: { os: "linux", nodeVersion: "v22.23.1", vscodeTarget: "1.100.0" } }, diff --git a/scripts/agent-evals/coding-pins.ts b/scripts/agent-evals/coding-pins.ts index 3433d11bfd..7449e0dff7 100644 --- a/scripts/agent-evals/coding-pins.ts +++ b/scripts/agent-evals/coding-pins.ts @@ -6,6 +6,10 @@ export interface SidePins { settingsDigest: string modelIdentifier: string graderVersion: string + // Immutable grader image identity resolved from the local engine, for + // example a repo digest. The value "unavailable" records that no identity + // was resolvable; it is never a fabricated digest. + graderImageDigest: string runnerEnvironment: { os: string nodeVersion: string @@ -21,7 +25,14 @@ export interface PinComparison { // Pinned comparison inputs. The two commit SHAs differ by design and are // recorded, never compared. Every other pinned value must match on both // sides; any difference marks the pair incompatible. -const COMPARED_FIELDS = ["suiteId", "suiteDigest", "settingsDigest", "modelIdentifier", "graderVersion"] as const +const COMPARED_FIELDS = [ + "suiteId", + "suiteDigest", + "settingsDigest", + "modelIdentifier", + "graderVersion", + "graderImageDigest", +] as const export function comparePins(baseline: SidePins, candidate: SidePins): PinComparison { const differences: string[] = [] diff --git a/scripts/agent-evals/coding-reference-solutions.test.ts b/scripts/agent-evals/coding-reference-solutions.test.ts index a4034affa7..0c89c339ac 100644 --- a/scripts/agent-evals/coding-reference-solutions.test.ts +++ b/scripts/agent-evals/coding-reference-solutions.test.ts @@ -66,7 +66,7 @@ describe("coding graders against reference solutions", () => { } it("pins one grader version across the suite", () => { - assert.equal(GRADER_VERSION, "coding-grader-v1") + assert.equal(GRADER_VERSION, "coding-grader-v2") assert.equal(CODING_GRADER_RESOURCES.image.startsWith("node:"), true) }) }) diff --git a/scripts/agent-evals/coding-report.test.ts b/scripts/agent-evals/coding-report.test.ts index e92aa27aa7..8cf8667a2c 100644 --- a/scripts/agent-evals/coding-report.test.ts +++ b/scripts/agent-evals/coding-report.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict" import { describe, it } from "node:test" import { buildComparisonReport, renderMarkdownReport, type SideResult } from "./coding-report.ts" -import { runComparison, type SideExecutor } from "./coding-compare.ts" +import { runComparison, validateSideResultFile, type SideExecutor } from "./coding-compare.ts" import type { CodingRunMetrics } from "./coding-metrics.ts" import type { SidePins } from "./coding-pins.ts" @@ -12,7 +12,8 @@ const pins = (side: "baseline" | "candidate", commit: string, overrides: Partial suiteDigest: "d1".repeat(32), settingsDigest: "d2".repeat(32), modelIdentifier: "qwen3.8-27b", - graderVersion: "coding-grader-v1", + graderVersion: "coding-grader-v2", + graderImageDigest: `node:22-bookworm@sha256:${"e".repeat(64)}`, runnerEnvironment: { os: "linux", nodeVersion: "v22.23.1", vscodeTarget: "1.104.0" }, ...overrides, }) @@ -96,7 +97,7 @@ describe("comparison report", () => { assert.deepEqual(report.compatibility.differences, ["modelIdentifier"]) }) - it("keeps grading failures out of the incorrect count and excludes incomplete cost runs", () => { + it("keeps grading failures out of the incorrect count and is unavailable with unknown attempt costs", () => { const baseline = sideResult("baseline", "a".repeat(40), [ metrics("clamp-range", 0, "correct", 0.2), metrics("clamp-range", 1, "grading-failure", undefined), @@ -106,8 +107,10 @@ describe("comparison report", () => { const summary = report.costSummary.baseline assert.equal(summary.correctRuns, 1) assert.equal(summary.excludedRuns, 2) - assert.equal(summary.available, true) - assert.equal(summary.usd, 0.2) + // The failed attempts have unknown costs, so the total spend per + // success is unknowable instead of merely smaller. + assert.equal(summary.available, false) + assert.equal(summary.reason, "failedAttemptCostIncomplete") assert.equal(report.partialDataPolicy, "excludeIncompleteFromAggregates-v1") }) @@ -123,7 +126,16 @@ describe("comparison report", () => { assert.match(markdown, /cannot show small improvements/) assert.match(markdown, /## Results per fixture/) assert.match(markdown, /## Cost per successful task/) - assert.match(markdown, /unavailable|over 1 correct run/) + // Each deterministic cost line is asserted on its own. An alternation + // would pass when only one line matched. + assert.ok( + markdown.includes("- Baseline: $0.2000 over 1 correct run(s), 0 excluded"), + `baseline cost line missing:\n${markdown}`, + ) + assert.ok( + markdown.includes("- Candidate: unavailable (noCorrectRuns; 0 excluded)"), + `candidate cost line missing:\n${markdown}`, + ) assert.match(markdown, /## Per-run appendix/) assert.match(markdown, /baseline \| clamp-range \| 0 \| correct /) }) @@ -179,3 +191,20 @@ describe("comparison orchestration", () => { assert.equal(report.repetitions, 3) }) }) + +describe("side result validation", () => { + it("accepts a result that records exactly the approved commit", () => { + const file = { pins: pins("baseline", "a".repeat(40)), runs: [] } + const validated = validateSideResultFile(JSON.parse(JSON.stringify(file)), "a".repeat(40)) + assert.equal(validated.pins.side, "baseline") + }) + + it("rejects a result that records a different commit, or has no shape", () => { + assert.throws(() => + validateSideResultFile({ pins: pins("baseline", "b".repeat(40)), runs: [] }, "a".repeat(40)), + ) + assert.throws(() => validateSideResultFile({ runs: [] }, "a".repeat(40))) + assert.throws(() => validateSideResultFile(null, "a".repeat(40))) + assert.throws(() => validateSideResultFile({ pins: pins("baseline", "a".repeat(40)) }, "a".repeat(40))) + }) +}) diff --git a/scripts/agent-evals/coding-run.test.ts b/scripts/agent-evals/coding-run.test.ts index dce16722b6..d26c627323 100644 --- a/scripts/agent-evals/coding-run.test.ts +++ b/scripts/agent-evals/coding-run.test.ts @@ -2,7 +2,7 @@ import assert from "node:assert/strict" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" -import { describe, it } from "node:test" +import { after, describe, it } from "node:test" import { RooCodeEventName, TaskCommandName, type TaskCommand, type TaskEvent } from "@roo-code/types" import { runCodingFixtureOnce, withWorkspaceCapture } from "./coding-run.ts" import { createLocalSubprocessGradingRuntime } from "./coding-grader.ts" @@ -16,6 +16,11 @@ import { codingBundle } from "./coding-suite.ts" interface SyntheticOptions { finalFiles: AgentFixtureWorkspaceFile[] complete: boolean + // Emits the model's final non-partial completion_result message instead + // of the accepted TaskCompleted event. + completeByMessageOnly?: boolean + // Rejects the final-workspace read, as a broken capture does. + failWorkspaceRead?: boolean } const syntheticRuntime = (options: SyntheticOptions): { runtime: AgentRuntime; traces: AgentTrace[] } => { @@ -43,6 +48,28 @@ const syntheticRuntime = (options: SyntheticOptions): { runtime: AgentRuntime; t eventName: RooCodeEventName.TaskStartResponse, payload: [{ requestId, success: true, taskId: "task-1" }], }) + if (options.completeByMessageOnly) { + // The model reports completion, but the accepted TaskCompleted + // event never fires. This is the headless completion path. + // The synthetic message satisfies the Message payload shape; + // the cast names the variant the schema accepts. + emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "task-1", + action: "created", + message: { + type: "say", + say: "completion_result", + ts: 1, + text: "Task finished.", + }, + }, + ], + } as unknown as TaskEvent) + return + } if (options.complete) emit({ eventName: RooCodeEventName.TaskCompleted, @@ -68,10 +95,10 @@ const syntheticRuntime = (options: SyntheticOptions): { runtime: AgentRuntime; t pause: async () => { fakeNow += 100 }, - readFinalWorkspace: async (): Promise => ({ - files: options.finalFiles, - truncated: false, - }), + readFinalWorkspace: async (): Promise => { + if (options.failWorkspaceRead) throw new Error("unreadable workspace") + return { files: options.finalFiles, truncated: false } + }, } return { runtime, traces } } @@ -82,11 +109,15 @@ const finalFilesWith = (solution: AgentFixtureWorkspaceFile): AgentFixtureWorksp } describe("coding run orchestration", () => { + // Every scratch directory is removed in the after hook, which completes + // before the test process exits. No async work rides on an exit handler. + const scratchDirs: string[] = [] + after(async () => { + for (const dir of scratchDirs) await fs.rm(dir, { recursive: true, force: true }) + }) const scratch = async (): Promise => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-run-test-")) - process.once("exit", () => { - void fs.rm(dir, { recursive: true, force: true }) - }) + scratchDirs.push(dir) return dir } @@ -160,4 +191,68 @@ describe("coding run orchestration", () => { assert.equal(snapshot.truncated, false) assert.ok(snapshot.files.length > 0) }) + + it("stops on the final completion message without an accepted TaskCompleted and grades independently", async () => { + const bundle = codingBundle("clamp-range") + const { runtime } = syntheticRuntime({ + finalFiles: finalFilesWith(bundle.grading.correctSolution[0]!), + complete: false, + completeByMessageOnly: true, + }) + const record = await runCodingFixtureOnce({ + fixtureName: "clamp-range", + repIndex: 3, + credential: "sk-test", + runtime, + gradingRuntime: createLocalSubprocessGradingRuntime(), + scratchDir: await scratch(), + }) + // The run stopped on the headless policy instead of waiting for the + // task-run window, never claimed an accepted completion, and the + // hidden tests still decide the outcome. + assert.equal(record.executionCode, "STOPPED") + assert.equal(record.outcome.outcome, "correct") + assert.equal(record.grading?.kind, "passed") + assert.equal(record.trace?.execution.stopReason, "adapterStopAfterCompletionMessage") + assert.equal(record.trace?.execution.completionMessageSeen, true) + }) + + it("reports a rejected final-workspace read as finalWorkspaceUnavailable without grading", async () => { + const { runtime } = syntheticRuntime({ finalFiles: [], complete: true, failWorkspaceRead: true }) + const record = await runCodingFixtureOnce({ + fixtureName: "clamp-range", + repIndex: 4, + credential: "sk-test", + runtime, + gradingRuntime: createLocalSubprocessGradingRuntime(), + scratchDir: await scratch(), + }) + // An infrastructure fault must never become an incorrect verdict from + // an empty workspace. + assert.equal(record.outcome.outcome, "infrastructure-failure") + assert.equal(record.outcome.reason, "finalWorkspaceUnavailable") + assert.equal(record.grading, undefined) + }) + + it("never rereads the workspace once a snapshot is captured", async () => { + const bundle = codingBundle("clamp-range") + const inner = syntheticRuntime({ + finalFiles: finalFilesWith(bundle.grading.correctSolution[0]!), + complete: false, + }) + let reads = 0 + const original = inner.runtime.readFinalWorkspace!.bind(inner.runtime) + inner.runtime.readFinalWorkspace = async () => { + reads++ + // A second cleanup finds the directory already removed. + return reads === 1 ? await original() : Promise.reject(new Error("removed")) + } + const captured = withWorkspaceCapture(inner.runtime) + await captured.runtime.cleanup() + const first = captured.getSnapshot() + assert.ok(first && first.files.length > 0) + // Repeated cleanup must not overwrite the captured snapshot. + await captured.runtime.cleanup() + assert.equal(captured.getSnapshot(), first) + }) }) diff --git a/scripts/agent-evals/coding-run.ts b/scripts/agent-evals/coding-run.ts index 3d7599268e..b09cc0e973 100644 --- a/scripts/agent-evals/coding-run.ts +++ b/scripts/agent-evals/coding-run.ts @@ -1,4 +1,10 @@ -import { runAgentFixture, type AgentRuntime, type FinalWorkspaceSnapshot } from "./runner.ts" +import { + runAgentFixture, + isFinalCompletionMessage, + type AgentRuntime, + type AgentRunObservers, + type FinalWorkspaceSnapshot, +} from "./runner.ts" import { getFixture } from "./registry.ts" import { ensureCodingSuiteRegistered, codingBundle } from "./coding-suite.ts" import { runGrader, type GraderVerdict, type GradingRuntime } from "./coding-grader.ts" @@ -47,9 +53,28 @@ const modelIdentifierFromProjection = (projection: Record): Mea "modelIdMissingInProjection", ) +// Headless stop policy. A headless extension host may never fire the accepted +// TaskCompleted event, so the run must not wait for the whole task-run window. +// The policy stops once the model's final non-partial completion_result +// message arrives for the accepted task. The substrate records the stop as +// adapterStopAfterCompletionMessage; it never claims an accepted completion. +// Grading below stays independent of the completion event. +export function stopOnFinalCompletionMessage(): AgentRunObservers { + let sawFinalCompletionMessage = false + return { + onAcceptedEvent: (event) => { + if (isFinalCompletionMessage(event)) sawFinalCompletionMessage = true + }, + shouldStop: () => sawFinalCompletionMessage, + } +} + // Wraps a runtime so the final workspace is captured inside cleanup, before // the launcher removes its scratch directory. The captured file data feeds -// only the fresh grading copy. +// only the fresh grading copy. A rejected read leaves the snapshot undefined, +// so a broken capture becomes an infrastructure failure, never an empty +// workspace scored against the agent. A captured snapshot is never reread, so +// repeated cleanup calls cannot overwrite it. export function withWorkspaceCapture(inner: AgentRuntime): { runtime: AgentRuntime getSnapshot: () => FinalWorkspaceSnapshot | undefined @@ -61,10 +86,12 @@ export function withWorkspaceCapture(inner: AgentRuntime): { launch: (files) => inner.launch(files), connect: () => inner.connect(), cleanup: async () => { - try { - snapshot = await inner.readFinalWorkspace?.() - } catch { - snapshot = { files: [], truncated: true } + if (snapshot === undefined) { + try { + snapshot = await inner.readFinalWorkspace?.() + } catch { + snapshot = undefined + } } await inner.cleanup() }, @@ -98,6 +125,7 @@ export async function runCodingFixtureOnce(inputs: CodingRunInputs): Promise { +const readArgs = (): { + reps: number + out: string + credentialEnv: string + allowUnisolatedGrading: boolean + evaluatedRoot: string +} => { const args = process.argv.slice(2) const read = (name: string): string | undefined => { const index = args.indexOf(`--${name}`) @@ -34,10 +42,25 @@ const readArgs = (): { reps: number; out: string; credentialEnv: string; allowUn out: path.resolve(out), credentialEnv: read("credential-env") ?? "SOLHEIM_API_KEY", allowUnisolatedGrading: args.includes("--allow-unisolated-grading"), + // Repository root of the extension under test. The trusted harness + // code drives the run; only the extension host loads this tree. + evaluatedRoot: path.resolve(read("evaluated-root") ?? thisRoot), } } -const readVscodeTarget = async (): Promise => { +// Side identity. The value must be one of the comparison labels; the default +// applies only when the variable is unset. Validation happens before any run +// spends provider budget or writes results. +const readSideLabel = (): SidePins["side"] => { + const raw = process.env["CODING_EVALS_SIDE"] + if (raw === undefined) return "baseline" + if (raw !== "baseline" && raw !== "candidate") { + throw new Error(`CODING_EVALS_SIDE must be "baseline" or "candidate", got "${raw}"`) + } + return raw +} + +const readVscodeTarget = async (root: string): Promise => { try { const pkg = JSON.parse(await fs.readFile(path.join(root, "apps/vscode-e2e/package.json"), "utf8")) as { devDependencies?: Record @@ -48,7 +71,7 @@ const readVscodeTarget = async (): Promise => { } } -const readCommit = async (): Promise => { +const readCommit = async (root: string): Promise => { const { execFile } = await import("node:child_process") const { promisify } = await import("node:util") try { @@ -59,29 +82,20 @@ const readCommit = async (): Promise => { } } -const { reps, out, credentialEnv, allowUnisolatedGrading } = readArgs() +const { reps, out, credentialEnv, allowUnisolatedGrading, evaluatedRoot } = readArgs() if (!Number.isInteger(reps) || reps < 1) throw new Error("--reps must be a positive integer") -await fs.mkdir(out, { recursive: true }) +const sideLabel = readSideLabel() +// Fail before any directory, run, or result file exists. An empty credential +// would otherwise produce a plausible file of infrastructure failures. +const credential = process.env[credentialEnv] +if (credential === undefined || credential.length === 0) { + throw new Error(`provider credential environment variable ${credentialEnv} is unset or empty`) +} ensureCodingSuiteRegistered() -const credential = process.env[credentialEnv] ?? "" const gradingRuntime = allowUnisolatedGrading ? createLocalSubprocessGradingRuntime() : createContainerGradingRuntime() const projection = CODING_FIXTURE_BUNDLES[0]?.fixture.settingsProjection ?? {} -const pins: SidePins = { - side: "baseline", - commit: await readCommit(), - suiteId: CODING_SUITE_ID, - suiteDigest: codingSuiteDigest(), - settingsDigest: digestValue(projection), - modelIdentifier: typeof projection["openAiModelId"] === "string" ? projection["openAiModelId"] : "unavailable", - graderVersion: GRADER_VERSION, - runnerEnvironment: { - os: process.platform, - nodeVersion: process.version, - vscodeTarget: await readVscodeTarget(), - }, -} const runs: CodingRunRecord[] = [] for (const bundle of CODING_FIXTURE_BUNDLES) { @@ -94,7 +108,11 @@ for (const bundle of CODING_FIXTURE_BUNDLES) { fixtureName, repIndex, credential, - runtime: createHostRuntime({ root, artifactsDir }), + runtime: createHostRuntime({ + root: evaluatedRoot, + artifactsDir, + vscodeCachePath: path.join(thisRoot, "apps/vscode-e2e/.vscode-test"), + }), gradingRuntime, scratchDir: artifactsDir, }) @@ -119,17 +137,38 @@ for (const bundle of CODING_FIXTURE_BUNDLES) { } } -// Side identity must reflect the actual side label in a comparison context. -const sideLabel = (process.env["CODING_EVALS_SIDE"] as SidePins["side"] | undefined) ?? "baseline" -const result: SideResultFile = { pins: { ...pins, side: sideLabel }, runs } +// Pins are recorded after the runs so they describe what actually executed: +// the evaluated commit and, when the container runtime resolved one, the +// immutable grader image identity. +const digest = gradingRuntime.imageDigest ? await gradingRuntime.imageDigest() : undefined +const pins: SidePins = { + side: sideLabel, + commit: await readCommit(evaluatedRoot), + suiteId: CODING_SUITE_ID, + suiteDigest: codingSuiteDigest(), + settingsDigest: digestValue(projection), + modelIdentifier: typeof projection["openAiModelId"] === "string" ? projection["openAiModelId"] : "unavailable", + graderVersion: GRADER_VERSION, + graderImageDigest: digest?.available === true ? digest.value : "unavailable", + runnerEnvironment: { + os: process.platform, + nodeVersion: process.version, + vscodeTarget: await readVscodeTarget(evaluatedRoot), + }, +} +const result: SideResultFile = { pins, runs } -// Keep the trace of each run with its artifacts for inspection. +// Per-run artifacts are published atomically as each record exists. for (const record of runs) { if (!record.trace) continue const artifactDir = path.join(out, "artifacts", `${record.fixtureName}-rep${record.repIndex}`) - await fs.writeFile(path.join(artifactDir, "run-record.json"), JSON.stringify(record, null, 2)) + await writeJsonAtomic(path.join(artifactDir, "run-record.json"), record) } -await fs.writeFile(path.join(out, "side-results.json"), JSON.stringify(result, null, 2)) +// Publication point. side-results.json appears atomically only after every +// repetition finished, so a reader that sees the file sees a complete side. +// A missing file means an interrupted side; consumers must treat it as a +// failed side and never pair partial output. +await writeJsonAtomic(path.join(out, "side-results.json"), result) const correct = runs.filter((record) => record.outcome.outcome === "correct").length console.log(`coding-evals side complete: ${correct}/${runs.length} correct, suite ${pins.suiteDigest.slice(0, 12)}`) diff --git a/scripts/agent-evals/coding-trust.test.ts b/scripts/agent-evals/coding-trust.test.ts new file mode 100644 index 0000000000..db7a96b5ae --- /dev/null +++ b/scripts/agent-evals/coding-trust.test.ts @@ -0,0 +1,98 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import { + assertCommitsApproved, + buildSideEnvironment, + loadTrustPolicy, + UntrustedCommitError, + type TrustPolicy, +} from "./coding-trust.ts" + +const sha = (c: string): string => c.repeat(40) + +const writePolicy = async (dir: string, content: unknown): Promise => { + const file = path.join(dir, "trust-policy.json") + await fs.writeFile(file, JSON.stringify(content)) + return file +} + +describe("trust policy", () => { + it("loads a policy of full commit SHAs", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) + try { + const file = await writePolicy(dir, { version: 1, approvedCommits: [sha("a"), sha("B")] }) + const policy = await loadTrustPolicy(file) + assert.deepEqual(policy, { version: 1, approvedCommits: [sha("a"), sha("b")] }) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("rejects missing files, wrong versions, empty lists, and non-SHA entries", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) + try { + await assert.rejects(loadTrustPolicy(path.join(dir, "absent.json")), /missing or unreadable/) + const bad: unknown[] = [ + { version: 2, approvedCommits: [sha("a")] }, + { version: 1, approvedCommits: [] }, + { version: 1, approvedCommits: ["main"] }, + { version: 1, approvedCommits: [sha("a").slice(0, 8)] }, + { version: 1, approvedCommits: [`${sha("a")}..${sha("b")}`] }, + { version: 1, approvedCommits: ["not-a-sha"] }, + ] + for (const content of bad) { + const file = await writePolicy(dir, content) + await assert.rejects(loadTrustPolicy(file)) + } + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("approves listed commits and rejects unapproved ones", () => { + const policy: TrustPolicy = { version: 1, approvedCommits: [sha("a")] } + assertCommitsApproved(policy, [sha("a")]) + // The gate is case insensitive on the approved side. + assertCommitsApproved(policy, [sha("A").toUpperCase()]) + assert.throws(() => assertCommitsApproved(policy, [sha("b"), sha("c")]), UntrustedCommitError) + try { + assertCommitsApproved(policy, [sha("b")]) + } catch (error) { + assert.ok(error instanceof UntrustedCommitError) + assert.match(error.message, /not approved by the trust policy/) + } + }) +}) + +describe("side environment allowlist", () => { + it("passes only the allowlisted variables, the credential, and the side label", () => { + const child = buildSideEnvironment( + { + PATH: "/usr/bin", + HOME: "/home/dev", + SOLHEIM_API_KEY: "sk-test", + DISPLAY: ":0", + AWS_SECRET_ACCESS_KEY: "leak-me", + OTHER_SECRET: "leak-me-too", + }, + "SOLHEIM_API_KEY", + "candidate", + ) + assert.deepEqual(child, { + PATH: "/usr/bin", + HOME: "/home/dev", + DISPLAY: ":0", + SOLHEIM_API_KEY: "sk-test", + CODING_EVALS_SIDE: "candidate", + }) + }) + + it("never invents a missing credential variable", () => { + const child = buildSideEnvironment({ PATH: "/usr/bin" }, "SOLHEIM_API_KEY", "baseline") + assert.equal(child["SOLHEIM_API_KEY"], undefined) + assert.equal(child["CODING_EVALS_SIDE"], "baseline") + }) +}) diff --git a/scripts/agent-evals/coding-trust.ts b/scripts/agent-evals/coding-trust.ts new file mode 100644 index 0000000000..d1280943a3 --- /dev/null +++ b/scripts/agent-evals/coding-trust.ts @@ -0,0 +1,100 @@ +import fs from "node:fs/promises" + +// Trust policy for paired comparisons. A comparison executes the extension +// code of two commits with the provider credential. Nothing about a ref name +// proves trust: a policy that resolved arbitrary refs and called them trusted +// would approve anything. The operator therefore lists approved full commit +// SHAs in an explicit policy file, and the driver rejects any other resolved +// commit before it creates a worktree or spawns a process. +export interface TrustPolicy { + version: 1 + approvedCommits: string[] +} + +// Git prints lowercase SHAs, but the policy accepts either case and the +// loader normalizes to lowercase. +const SHA_PATTERN = /^[0-9a-fA-F]{40}$/ + +export class UntrustedCommitError extends Error { + constructor(readonly commits: string[]) { + super(`commits not approved by the trust policy: ${commits.join(", ")}`) + this.name = "UntrustedCommitError" + } +} + +// Loads and validates one trust policy file. Every approved commit must be a +// full, lowercase-or-uppercase 40-hex SHA; short SHAs, ref names, and ranges +// are rejected. +export async function loadTrustPolicy(filePath: string): Promise { + let parsed: unknown + try { + parsed = JSON.parse(await fs.readFile(filePath, "utf8")) + } catch { + throw new Error(`trust policy file is missing or unreadable: ${filePath}`) + } + if (typeof parsed !== "object" || parsed === null) throw new Error("trust policy must be a JSON object") + const record = parsed as Record + if (record["version"] !== 1) throw new Error("trust policy version must be 1") + const commits = record["approvedCommits"] + if (!Array.isArray(commits) || commits.length === 0) { + throw new Error("trust policy must list at least one approved commit SHA") + } + const invalid = commits.filter((entry) => typeof entry !== "string" || !SHA_PATTERN.test(entry)) + if (invalid.length > 0) throw new Error("trust policy accepts only full 40-hex commit SHAs") + return { version: 1, approvedCommits: (commits as string[]).map((sha) => sha.toLowerCase()) } +} + +// Resolves one ref or SHA to a full commit SHA through the local repository. +export const resolveCommit = async (ref: string, root: string): Promise => { + const { execFile } = await import("node:child_process") + const { promisify } = await import("node:util") + const { stdout } = await promisify(execFile)("git", ["rev-parse", `${ref}^{commit}`], { + cwd: root, + timeout: 30_000, + }) + const sha = stdout.trim().toLowerCase() + if (!SHA_PATTERN.test(sha)) throw new Error(`cannot resolve a full commit SHA from ${ref}`) + return sha +} + +// Gate before any execution. Each resolved commit must appear in the policy. +export function assertCommitsApproved(policy: TrustPolicy, commits: readonly string[]): void { + const unapproved = commits.filter((sha) => !policy.approvedCommits.includes(sha.toLowerCase())) + if (unapproved.length > 0) throw new UntrustedCommitError(unapproved) +} + +// Environment allowlist for the side process. The comparison driver may hold +// unrelated secrets in its own environment; the side gets only what it needs +// to spawn git, node, and the extension host, plus the one named credential +// variable. +const SIDE_ENV_ALLOWLIST = [ + "PATH", + "HOME", + "LANG", + "LC_ALL", + "TMPDIR", + "TEMP", + "TMP", + "DISPLAY", + "XAUTHORITY", + "SYSTEMROOT", + "WINDIR", + "COMSPEC", + "PATHEXT", +] as const + +export function buildSideEnvironment( + env: NodeJS.ProcessEnv, + credentialEnv: string, + side: "baseline" | "candidate", +): NodeJS.ProcessEnv { + const child: NodeJS.ProcessEnv = {} + for (const name of SIDE_ENV_ALLOWLIST) { + const value = env[name] + if (value !== undefined) child[name] = value + } + const credential = env[credentialEnv] + if (credential !== undefined) child[credentialEnv] = credential + child["CODING_EVALS_SIDE"] = side + return child +} diff --git a/scripts/agent-evals/container-runtime.test.ts b/scripts/agent-evals/container-runtime.test.ts index afc27c96fa..931b3dadda 100644 --- a/scripts/agent-evals/container-runtime.test.ts +++ b/scripts/agent-evals/container-runtime.test.ts @@ -1,6 +1,15 @@ import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" import { describe, it } from "node:test" -import { buildContainerArgs, createContainerGradingRuntime, GRADER_IMAGE } from "./container-runtime.ts" +import { + buildContainerArgs, + createContainerGradingRuntime, + GRADER_IMAGE, + parseDriverResult, + type ContainerCli, +} from "./container-runtime.ts" import type { GraderJob } from "./coding-grader.ts" const makeJob = (): GraderJob => ({ @@ -9,6 +18,58 @@ const makeJob = (): GraderJob => ({ resources: { timeoutMs: 30_000, memoryMb: 512, cpus: 1, pidsLimit: 128, image: GRADER_IMAGE }, }) +const driverCompletedJson = (): string => + JSON.stringify({ + kind: "completed", + passed: true, + noTestsDiscovered: false, + testFiles: ["test/solution.test.js"], + passedCount: 3, + failedCount: 0, + cancelledCount: 0, + failures: [], + durationMs: 12, + }) + +// Locates the host outDir from the run argv, the way the real driver finds +// its result path through the /out mount. +const outDirFromArgs = (args: readonly string[]): string => { + const mount = args.find((arg) => arg.endsWith(":/out:rw")) + if (!mount) throw new Error("no /out mount in run argv") + return mount.slice(0, -":/out:rw".length) +} + +const recordingCli = ( + overrides: Partial = {}, +): ContainerCli & { calls: string[]; runArgs: string[][] } => { + const calls: string[] = [] + const runArgs: string[][] = [] + return { + calls, + runArgs, + pull: async (image) => { + calls.push(`pull ${image}`) + return "ok" + }, + run: async (args, timeoutMs) => { + runArgs.push([...args]) + calls.push(`run ${timeoutMs}`) + await fs.mkdir(outDirFromArgs(args), { recursive: true }) + await fs.writeFile(path.join(outDirFromArgs(args), "result.json"), driverCompletedJson()) + return 0 + }, + remove: async (name) => { + calls.push(`remove ${name}`) + return "removed" + }, + imageDigest: async () => { + calls.push("inspect") + return "node:22-bookworm@sha256:" + "a".repeat(64) + }, + ...overrides, + } +} + describe("container grading runtime", () => { it("builds a locked-down, network-less invocation", () => { const args = buildContainerArgs(makeJob(), "/tmp/evals/grading-out") @@ -47,6 +108,14 @@ describe("container grading runtime", () => { assert.equal(args.includes("--env"), false) }) + it("names the container when one is provided", () => { + const args = buildContainerArgs(makeJob(), "/tmp/evals/grading-out", "agent-evals-grader-1") + const nameIndex = args.indexOf("--name") + assert.ok(nameIndex > 0) + assert.equal(args[nameIndex + 1], "agent-evals-grader-1") + assert.equal(args[0], "run") + }) + it("reports a missing container runtime as grading failure", async () => { const runtime = createContainerGradingRuntime({ detectExecutable: async () => undefined }) const verdict = await runtime.run(makeJob()) @@ -59,4 +128,193 @@ describe("container grading runtime", () => { const runtime = createContainerGradingRuntime() assert.equal(runtime.isolation, "container") }) + + it("pulls the image outside the grading timeout and runs with a unique name", async () => { + const cli = recordingCli() + const runtime = createContainerGradingRuntime({ cli }) + const verdict = await runtime.run(makeJob()) + assert.equal(verdict.kind, "passed") + // The pull happens first, the image identity is resolved from the + // local engine, and the timed run comes after both. + assert.equal(cli.calls[0], `pull ${GRADER_IMAGE}`) + assert.equal(cli.calls.indexOf("run 30000"), 2) + assert.equal(cli.calls[1], "inspect") + // The run argv names a unique container. + const nameIndex = cli.runArgs[0]!.indexOf("--name") + assert.ok(nameIndex > 0) + assert.match(cli.runArgs[0]![nameIndex + 1] ?? "", /^agent-evals-grader-\d+-\d+-[0-9a-f]{8}$/) + // The immutable image identity is resolvable for the pins. + const digest = await runtime.imageDigest!() + assert.equal(digest.available, true) + assert.match(digest.available ? digest.value : "", /^node:22-bookworm@sha256:[0-9a-f]{64}$/) + }) + + it("removes the container by name on timeout and verifies the removal before deleting output", async () => { + const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "container-runtime-test-")) + try { + const cli = recordingCli({ + run: async () => "timeout", + remove: async (name) => { + cli.calls.push(`remove ${name}`) + // The verified removal runs before the output directory + // disappears. + await fs.access(path.join(scratch, "grading-out")).then( + () => {}, + () => { + throw new Error("output removed before container removal") + }, + ) + return "removed" + }, + }) + const runtime = createContainerGradingRuntime({ cli }) + const verdict = await runtime.run({ ...makeJob(), gradingDir: path.join(scratch, "grading-copy") }) + assert.deepEqual(verdict, { kind: "grading-failure", isolation: "container", reason: "graderTimeout" }) + assert.equal(cli.calls.filter((call) => call.startsWith("remove ")).length, 1) + assert.match(cli.calls.find((call) => call.startsWith("remove ")) ?? "", /^remove agent-evals-grader-/) + // The output directory is gone afterwards. + await fs.access(path.join(scratch, "grading-out")).then( + () => { + throw new Error("output directory survived") + }, + () => {}, + ) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) + + it("reports a failed timeout removal instead of a leaked container", async () => { + const cli = recordingCli({ + run: async () => "timeout", + remove: async () => "failed", + }) + const runtime = createContainerGradingRuntime({ cli }) + const verdict = await runtime.run(makeJob()) + assert.deepEqual(verdict, { + kind: "grading-failure", + isolation: "container", + reason: "graderContainerRemovalFailed", + }) + }) + + it("verifies removal after a crashed run", async () => { + const cli = recordingCli({ + run: async () => 3, + remove: async (name) => { + cli.calls.push(`remove ${name}`) + return "absent" + }, + }) + const runtime = createContainerGradingRuntime({ cli }) + const verdict = await runtime.run(makeJob()) + assert.deepEqual(verdict, { kind: "grading-failure", isolation: "container", reason: "graderCrashed" }) + assert.equal(cli.calls.filter((call) => call.startsWith("remove ")).length, 1) + }) + + it("reports a failed image pull before any container runs", async () => { + const cli = recordingCli({ pull: async () => "failed" }) + const runtime = createContainerGradingRuntime({ cli }) + const verdict = await runtime.run(makeJob()) + assert.deepEqual(verdict, { + kind: "grading-failure", + isolation: "container", + reason: "graderImagePullFailed", + }) + assert.deepEqual( + cli.calls.filter((call) => call.startsWith("run")), + [], + ) + }) + + it("reports an invalid driver result as graderResultInvalid", async () => { + const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "container-runtime-test-")) + try { + const outDir = path.join(scratch, "grading-out") + const cli = recordingCli({ + run: async () => { + await fs.mkdir(outDir, { recursive: true }) + await fs.writeFile(path.join(outDir, "result.json"), '{"kind":"completed","passed":true,"lie":1}') + return 0 + }, + }) + const runtime = createContainerGradingRuntime({ cli }) + const verdict = await runtime.run({ ...makeJob(), gradingDir: path.join(scratch, "grading-copy") }) + assert.deepEqual(verdict, { + kind: "grading-failure", + isolation: "container", + reason: "graderResultInvalid", + }) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) +}) + +describe("untrusted driver result validation", () => { + const valid = JSON.parse(driverCompletedJson()) as Record + const build = (overrides: Partial>): string => JSON.stringify({ ...valid, ...overrides }) + + it("accepts a well-formed completed result", () => { + const parsed = parseDriverResult(driverCompletedJson()) + assert.ok(parsed && parsed.kind === "completed") + assert.equal(parsed.cancelledCount, 0) + }) + + it("accepts a driver-error result", () => { + const parsed = parseDriverResult(JSON.stringify({ kind: "driver-error", message: "testRunnerUnavailable" })) + assert.ok(parsed && parsed.kind === "driver-error") + }) + + it("rejects malformed JSON and wrong shapes", () => { + assert.equal(parseDriverResult("{not json"), undefined) + assert.equal(parseDriverResult("42"), undefined) + assert.equal(parseDriverResult("null"), undefined) + assert.equal(parseDriverResult('{"kind":"other"}'), undefined) + }) + + it("rejects non-boolean flags, bad counts, and bad durations", () => { + assert.equal(parseDriverResult(build({ passed: "yes" })), undefined) + assert.equal(parseDriverResult(build({ noTestsDiscovered: 0 })), undefined) + assert.equal(parseDriverResult(build({ passedCount: -1 })), undefined) + assert.equal(parseDriverResult(build({ failedCount: 1.5 })), undefined) + assert.equal(parseDriverResult(build({ cancelledCount: "0" })), undefined) + assert.equal(parseDriverResult(build({ durationMs: -1 })), undefined) + assert.equal(parseDriverResult(build({ durationMs: Number.POSITIVE_INFINITY })), undefined) + }) + + it("rejects pass/fail inconsistency and cancelled passes", () => { + assert.equal(parseDriverResult(build({ passed: true, failedCount: 2 })), undefined) + assert.equal(parseDriverResult(build({ passed: true, passedCount: 0 })), undefined) + assert.equal(parseDriverResult(build({ passed: true, cancelledCount: 1 })), undefined) + assert.equal(parseDriverResult(build({ passed: false })), undefined) + }) + + it("rejects bad failure entries and failure counts that exceed failedCount", () => { + assert.equal(parseDriverResult(build({ failures: [{ name: "x" }] })), undefined) + assert.equal(parseDriverResult(build({ failures: [{ name: 1, file: "f" }] })), undefined) + assert.equal( + parseDriverResult( + build({ + failedCount: 1, + failures: [ + { name: "a", file: "f" }, + { name: "b", file: "f" }, + ], + }), + ), + undefined, + ) + }) + + it("rejects bad test file lists", () => { + assert.equal(parseDriverResult(build({ testFiles: [] })), undefined) + assert.equal(parseDriverResult(build({ testFiles: [""] })), undefined) + assert.equal(parseDriverResult(build({ testFiles: [42] })), undefined) + assert.equal(parseDriverResult(build({ testFiles: Array.from({ length: 1_001 }, () => "t.js") })), undefined) + }) + + it("rejects oversized payloads", () => { + assert.equal(parseDriverResult(`{"kind":"completed","pad":"${"x".repeat(70_000)}"}`), undefined) + }) }) diff --git a/scripts/agent-evals/container-runtime.ts b/scripts/agent-evals/container-runtime.ts index 1d5a263326..98da8ea0c6 100644 --- a/scripts/agent-evals/container-runtime.ts +++ b/scripts/agent-evals/container-runtime.ts @@ -1,10 +1,13 @@ import { spawn } from "node:child_process" +import { randomUUID } from "node:crypto" import fs from "node:fs/promises" import path from "node:path" import type { GraderJob, GraderVerdict, GradingRuntime, DriverResult } from "./coding-grader.ts" +import type { Measured } from "./trace.ts" -// Pinned grader image. The image tag is part of the grader identity and is -// recorded in every pinned comparison through the grader version. +// Pinned grader image tag. The digest resolved from the local engine travels +// in the comparison pins, so the recorded identity is immutable even though +// the tag alone is not. export const GRADER_IMAGE = "node:22-bookworm" export type ContainerExecutable = "docker" | "podman" @@ -13,6 +16,23 @@ export interface ContainerRuntimeOptions { executable?: ContainerExecutable // Test seam: detects the container binary instead of searching PATH. detectExecutable?: () => Promise + // Test seam: replaces the container CLI process calls. + cli?: ContainerCli +} + +// One container CLI. The default implementation spawns docker or podman. Tests +// replace it, so timeout cleanup is testable without a real engine. +export interface ContainerCli { + // Pulls the grader image. Runs outside the grading timeout. + pull(image: string): Promise<"ok" | "failed"> + // Runs one container. Resolves "timeout" when the CLI hit the wall clock. + run(args: readonly string[], timeoutMs: number): Promise + // Removes the named container. "absent" means the container is already + // gone, which counts as verified removal. + remove(name: string): Promise<"removed" | "absent" | "failed"> + // Resolves the immutable local image identity: the repo digest when one + // exists, otherwise the image ID. Never a fabricated value. + imageDigest(image: string): Promise } const defaultDetect = async (): Promise => { @@ -44,15 +64,89 @@ const probeExecutable = (candidate: ContainerExecutable): Promise => } }) +const exec = ( + file: string, + args: readonly string[], + timeoutMs: number, +): Promise<{ code: number | "timeout" | "error"; stdout: string; stderr: string }> => + new Promise((resolve) => { + try { + const child = spawn(file, [...args], { + stdio: ["ignore", "pipe", "pipe"], + timeout: timeoutMs, + killSignal: "SIGKILL", + }) + let stdout = "" + let stderr = "" + child.stdout?.on("data", (chunk) => { + stdout += String(chunk) + }) + child.stderr?.on("data", (chunk) => { + stderr += String(chunk) + }) + child.once("error", () => resolve({ code: "error", stdout, stderr })) + child.once("close", (code, signal) => + resolve({ code: signal === "SIGKILL" ? "timeout" : (code ?? "error"), stdout, stderr }), + ) + } catch { + resolve({ code: "error", stdout: "", stderr: "" }) + } + }) + +const IMAGE_PULL_TIMEOUT_MS = 10 * 60_000 +const CONTAINER_REMOVE_TIMEOUT_MS = 30_000 +const IMAGE_INSPECT_TIMEOUT_MS = 30_000 + +const containerGone = (stderr: string): boolean => /no such container/i.test(stderr) + +export function createDefaultContainerCli(executable: ContainerExecutable): ContainerCli { + return { + async pull(image) { + const outcome = await exec(executable, ["pull", "-q", image], IMAGE_PULL_TIMEOUT_MS) + return outcome.code === 0 ? "ok" : "failed" + }, + run(args, timeoutMs) { + return exec(executable, args, timeoutMs).then((outcome) => outcome.code) + }, + async remove(name) { + const outcome = await exec(executable, ["rm", "--force", name], CONTAINER_REMOVE_TIMEOUT_MS) + if (outcome.code === 0) return "removed" + if (outcome.code !== "timeout" && containerGone(outcome.stderr)) return "absent" + return "failed" + }, + async imageDigest(image) { + const repoDigest = await exec( + executable, + ["image", "inspect", "--format", "{{index .RepoDigests 0}}", image], + IMAGE_INSPECT_TIMEOUT_MS, + ) + if (repoDigest.code === 0) { + const digest = repoDigest.stdout.trim() + if (digest.length > 0) return digest + } + // A locally built image has no repo digest. The engine image ID is + // still an immutable local identity. + const imageId = await exec( + executable, + ["image", "inspect", "--format", "{{.Id}}", image], + IMAGE_INSPECT_TIMEOUT_MS, + ) + return imageId.code === 0 ? imageId.stdout.trim() || undefined : undefined + }, + } +} + // Builds one `run` invocation. The container has no network, no Linux // capabilities, no privilege escalation, a read-only root filesystem, fixed // CPU, memory, and process limits, and a size-capped scratch tmpfs. The // grading copy mounts read-only, so agent-written code cannot change the // hidden tests at grading time. The driver mounts read-only from repo code. // No environment variable passes through, so no credential can leak in. -export function buildContainerArgs(job: GraderJob, outDir: string): string[] { +export function buildContainerArgs(job: GraderJob, outDir: string, containerName?: string): string[] { + const leading: string[] = ["run"] + if (containerName !== undefined) leading.push("--name", containerName) return [ - "run", + ...leading, "--rm", "--network", "none", @@ -87,51 +181,176 @@ export function buildContainerArgs(job: GraderJob, outDir: string): string[] { ] } +// Unique per grading run, so concurrent or repeated runs never collide and a +// leftover container is always removable by name. +const containerNameFor = (): string => `agent-evals-grader-${process.pid}-${Date.now()}-${randomUUID().slice(0, 8)}` + // Default grading runtime. Runs the driver in a disposable container. The -// host enforces the wall-clock limit and kills the container on expiry. +// host enforces the wall-clock limit; on expiry it removes the container by +// name and verifies the removal before any output is deleted. A `timeout` +// spawn option alone would kill only the CLI client and leave the container +// running. First-run image pulls happen before the timed run, so grading +// time is never consumed by pulling the image. export function createContainerGradingRuntime(options: ContainerRuntimeOptions = {}): GradingRuntime { - const detect = options.detectExecutable ?? defaultDetect + let cli: ContainerCli | undefined + const resolveCli = async (): Promise => { + if (options.cli) return options.cli + if (!cli) { + const detect = options.detectExecutable ?? defaultDetect + const executable = options.executable ?? (await detect()) + if (!executable) return undefined + cli = createDefaultContainerCli(executable) + } + return cli + } + let resolvedDigest: string | undefined return { isolation: "container", async run(job: GraderJob): Promise { - const executable = await detect() - if (!executable) return { kind: "grading-failure", isolation: "container", reason: "noContainerRuntime" } + const failure = (reason: string): GraderVerdict => ({ + kind: "grading-failure", + isolation: "container", + reason, + }) + const selected = await resolveCli() + if (!selected) return failure("noContainerRuntime") + // The pull stays outside the grading timeout on purpose. + if ((await selected.pull(job.resources.image)) === "failed") return failure("graderImagePullFailed") + resolvedDigest = (await selected.imageDigest(job.resources.image)) ?? resolvedDigest const outDir = path.join(path.dirname(job.gradingDir), "grading-out") await fs.mkdir(outDir, { recursive: true }) + // The container runs as root with all capabilities dropped, so it + // cannot rely on CAP_DAC_OVERRIDE. The mount target must grant the + // write to any uid. + await fs.chmod(outDir, 0o777) + const name = containerNameFor() + const args = buildContainerArgs(job, outDir, name) + let exit: number | "timeout" | "error" try { - const args = buildContainerArgs(job, outDir) - const exit = await new Promise((resolve) => { - const child = spawn(executable, args, { - stdio: "ignore", - timeout: job.resources.timeoutMs, - killSignal: "SIGKILL", - }) - child.once("error", () => resolve("error")) - child.once("close", (code, signal) => resolve(signal === "SIGKILL" ? "timeout" : (code ?? "error"))) - }) - if (exit === "timeout") - return { kind: "grading-failure", isolation: "container", reason: "graderTimeout" } - if (exit === "error" || exit !== 0) - return { kind: "grading-failure", isolation: "container", reason: "graderCrashed" } - const result = await readDriverResult(path.join(outDir, "result.json")) - if (!result) return { kind: "grading-failure", isolation: "container", reason: "graderResultMissing" } - return verdictFromDriver(result, "container") + exit = await selected.run(args, job.resources.timeoutMs) } catch { - return { kind: "grading-failure", isolation: "container", reason: "graderSetupFailed" } + exit = "error" + } + if (exit === "timeout") { + // Verify the container is gone before the output mount is + // deleted. A failed removal leaves an unbounded process behind, + // so it must surface as a grading failure. + const removal = await selected.remove(name) + await fs.rm(outDir, { recursive: true, force: true }).catch(() => {}) + return removal === "failed" ? failure("graderContainerRemovalFailed") : failure("graderTimeout") + } + if (exit === "error" || exit !== 0) { + // A normally exited container removes itself through --rm; the + // explicit removal verifies it and cleans a killed client's + // leftover. + await selected.remove(name) + await fs.rm(outDir, { recursive: true, force: true }).catch(() => {}) + return failure("graderCrashed") + } + try { + const loaded = await loadDriverResult(path.join(outDir, "result.json")) + if (loaded === "missing") return failure("graderResultMissing") + if (loaded === "invalid") return failure("graderResultInvalid") + return verdictFromDriver(loaded, "container") } finally { - await fs.rm(outDir, { recursive: true, force: true }) + await fs.rm(outDir, { recursive: true, force: true }).catch(() => {}) } }, + async imageDigest(): Promise> { + if (resolvedDigest !== undefined) return { available: true, value: resolvedDigest } + const selected = await resolveCli() + if (!selected) return { available: false, reason: "noContainerRuntime" } + return { available: false, reason: "digestUnavailable" } + }, } } -export async function readDriverResult(outPath: string): Promise { +export const DRIVER_RESULT_MAX_BYTES = 65_536 +const MAX_TEST_FILES = 1_000 +const MAX_FAILURE_ENTRIES = 50 +const MAX_COUNT = 100_000 + +const isBoundedInt = (value: unknown, max: number): value is number => + typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= max + +const isBoundedString = (value: unknown, max: number): value is string => + typeof value === "string" && value.length >= 1 && value.length <= max + +// Validates one untrusted driver result. The result file is written inside +// the container where agent-written test code executes, so nothing about its +// content is trusted. Enforces shape, bounds, and internal consistency; +// anything else is rejected. +export function parseDriverResult(text: string): DriverResult | undefined { + if (text.length > DRIVER_RESULT_MAX_BYTES) return undefined + let parsed: unknown try { - const parsed = JSON.parse(await fs.readFile(outPath, "utf8")) as DriverResult - return parsed.kind === "completed" || parsed.kind === "driver-error" ? parsed : undefined + parsed = JSON.parse(text) } catch { return undefined } + if (typeof parsed !== "object" || parsed === null) return undefined + const record = parsed as Record + if (record["kind"] === "driver-error") { + if (!isBoundedString(record["message"], 512)) return undefined + return { kind: "driver-error", message: record["message"] } + } + if (record["kind"] !== "completed") return undefined + if (typeof record["passed"] !== "boolean") return undefined + if (typeof record["noTestsDiscovered"] !== "boolean") return undefined + const passedCount = record["passedCount"] + const failedCount = record["failedCount"] + const cancelledCount = record["cancelledCount"] + if (!isBoundedInt(passedCount, MAX_COUNT)) return undefined + if (!isBoundedInt(failedCount, MAX_COUNT)) return undefined + if (!isBoundedInt(cancelledCount, MAX_COUNT)) return undefined + const testFiles = record["testFiles"] + if (!Array.isArray(testFiles) || testFiles.length > MAX_TEST_FILES) return undefined + if (testFiles.some((entry) => !isBoundedString(entry, 4_096))) return undefined + if (!record["noTestsDiscovered"] && testFiles.length === 0) return undefined + const failures = record["failures"] + if (!Array.isArray(failures) || failures.length > MAX_FAILURE_ENTRIES) return undefined + for (const entry of failures) { + if (typeof entry !== "object" || entry === null) return undefined + const item = entry as Record + if (!isBoundedString(item["name"], 1_024)) return undefined + if (!isBoundedString(item["file"], 4_096)) return undefined + } + if (failures.length > failedCount) return undefined + const durationMs = record["durationMs"] + if (typeof durationMs !== "number" || !Number.isFinite(durationMs) || durationMs < 0) return undefined + // Consistency: a passing verdict requires zero failures, zero + // cancellations, and at least one passing test. + const expectedPassed = failedCount === 0 && cancelledCount === 0 && passedCount > 0 + if (record["passed"] !== expectedPassed) return undefined + return { + kind: "completed", + passed: record["passed"], + noTestsDiscovered: record["noTestsDiscovered"], + testFiles: testFiles as string[], + passedCount, + failedCount, + cancelledCount, + failures: failures as { name: string; file: string }[], + durationMs, + } +} + +export type LoadedDriverResult = DriverResult | "missing" | "invalid" + +// Reads one driver result file with a hard size bound. "missing" covers an +// absent or unreadable file; "invalid" covers any untrusted content that +// fails validation. +export async function loadDriverResult(outPath: string): Promise { + let text: string + try { + const stat = await fs.stat(outPath) + if (!stat.isFile() || stat.size > DRIVER_RESULT_MAX_BYTES) return "invalid" + text = await fs.readFile(outPath, "utf8") + } catch { + return "missing" + } + const parsed = parseDriverResult(text) + return parsed ?? "invalid" } export function verdictFromDriver(result: DriverResult, isolation: GraderVerdict["isolation"]): GraderVerdict { diff --git a/scripts/agent-evals/fs-atomic.test.ts b/scripts/agent-evals/fs-atomic.test.ts new file mode 100644 index 0000000000..c34d86f91b --- /dev/null +++ b/scripts/agent-evals/fs-atomic.test.ts @@ -0,0 +1,52 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import { writeFileAtomic, writeJsonAtomic } from "./fs-atomic.ts" + +describe("atomic file publication", () => { + it("publishes the content and leaves no temp files", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "fs-atomic-test-")) + try { + const target = path.join(dir, "report.json") + await writeFileAtomic(target, '{"a":1}') + assert.equal(await fs.readFile(target, "utf8"), '{"a":1}') + const entries = await fs.readdir(dir) + assert.deepEqual(entries, ["report.json"]) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("replaces an existing file in place", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "fs-atomic-test-")) + try { + const target = path.join(dir, "side-results.json") + await writeJsonAtomic(target, { version: 1 }) + await writeJsonAtomic(target, { version: 2 }) + assert.equal(await fs.readFile(target, "utf8"), JSON.stringify({ version: 2 }, null, 2)) + const entries = await fs.readdir(dir) + assert.deepEqual(entries, ["side-results.json"]) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("preserves the previous file and leaves no temp residue when a write fails", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "fs-atomic-test-")) + try { + const target = path.join(dir, "run-record.json") + await writeFileAtomic(target, "previous") + // The temp file is created before the rename; a rename into a + // missing directory fails and must clean its temp file while the + // original stays intact. + await assert.rejects(writeFileAtomic(path.join(dir, "missing", "run-record.json"), "next")) + assert.equal(await fs.readFile(target, "utf8"), "previous") + const entries = await fs.readdir(dir) + assert.deepEqual(entries, ["run-record.json"]) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/scripts/agent-evals/fs-atomic.ts b/scripts/agent-evals/fs-atomic.ts new file mode 100644 index 0000000000..7f627e1754 --- /dev/null +++ b/scripts/agent-evals/fs-atomic.ts @@ -0,0 +1,26 @@ +import { randomUUID } from "node:crypto" +import fs from "node:fs/promises" +import path from "node:path" + +// Publishes one file atomically. The content lands in a temp file in the +// target's own directory and is renamed into place, so readers never observe +// a partial write and a failed write never destroys the previous file. The +// temp name is unique per process and call, so concurrent writers cannot +// collide. +export async function writeFileAtomic(filePath: string, data: string): Promise { + const directory = path.dirname(filePath) + const temp = path.join(directory, `.${path.basename(filePath)}.${process.pid}.${randomUUID().slice(0, 8)}.tmp`) + try { + await fs.writeFile(temp, data) + await fs.rename(temp, filePath) + } catch (error) { + await fs.rm(temp, { force: true }).catch(() => {}) + throw error + } +} + +// Serializes one value to JSON and publishes it atomically. Serialization +// failures throw before any file is touched. +export async function writeJsonAtomic(filePath: string, value: unknown): Promise { + await writeFileAtomic(filePath, JSON.stringify(value, null, 2)) +} diff --git a/scripts/agent-evals/grader-driver.mjs b/scripts/agent-evals/grader-driver.mjs index 203894323b..9242597534 100644 --- a/scripts/agent-evals/grader-driver.mjs +++ b/scripts/agent-evals/grader-driver.mjs @@ -86,6 +86,7 @@ try { testFiles: [], passedCount: 0, failedCount: 0, + cancelledCount: 0, failures: [], durationMs: 0, }) @@ -94,6 +95,9 @@ try { const started = Date.now() let passedCount = 0 let failedCount = 0 + // Cancelled tests never ran. A passing verdict requires zero of them, so a + // solution cannot pass while a hidden test stayed pending. + let cancelledCount = 0 let runnerBroke = false const failures = [] for (const file of files) { @@ -110,6 +114,7 @@ try { } passedCount += counts.passed failedCount += counts.failed + cancelledCount += counts.cancelled ?? 0 if (counts.failed > 0 && failures.length < 50) for (const name of readFailureNames(outcome.tap)) failures.push({ name, file }) } @@ -119,11 +124,12 @@ try { } writeResult({ kind: "completed", - passed: failedCount === 0 && passedCount > 0, + passed: failedCount === 0 && cancelledCount === 0 && passedCount > 0, noTestsDiscovered: false, testFiles: files, passedCount, failedCount, + cancelledCount, failures, durationMs: Date.now() - started, }) diff --git a/scripts/agent-evals/launcher.ts b/scripts/agent-evals/launcher.ts index e23f21f29c..64f1c7dfc1 100644 --- a/scripts/agent-evals/launcher.ts +++ b/scripts/agent-evals/launcher.ts @@ -14,8 +14,14 @@ import { identityUnavailable, measured, type MeasuredString, type TraceRunIdenti const execFileAsync = promisify(execFile) export interface HostRuntimeOptions { + // Repository root of the evaluated extension. Reads of package metadata, + // commit identity, and the extension source all stay inside it. root: string artifactsDir: string + // Optional trusted cache for the downloaded VS Code build. Defaults to + // the evaluated root, so callers driving historical worktrees can keep + // one cache in the harness checkout. + vscodeCachePath?: string now?: () => number pause?: (ms: number) => Promise } @@ -123,7 +129,7 @@ export function createHostRuntime(options: HostRuntimeOptions): AgentRuntime { } const executable = await downloadAndUnzipVSCode({ version: pkg.devDependencies["@types/vscode"] ?? "1.100.0", - cachePath: path.join(root, "apps/vscode-e2e/.vscode-test"), + cachePath: options.vscodeCachePath ?? path.join(root, "apps/vscode-e2e/.vscode-test"), }) const args = buildLaunchArgs( workspace, diff --git a/scripts/agent-evals/runner.ts b/scripts/agent-evals/runner.ts index df6836cfd6..80ff42a87a 100644 --- a/scripts/agent-evals/runner.ts +++ b/scripts/agent-evals/runner.ts @@ -85,6 +85,24 @@ const EVENT_BUFFER_LIMIT = 256 type BufferedEvent = { kind: "parsed"; event: TaskEvent } +// True when the event carries the model's final, non-partial completion_result +// say message for the accepted task. This is model output, not the +// accepted-completion event. The headless stop policy in coding-run.ts uses +// the same predicate as the trace's completionMessageSeen refinement. +export const isFinalCompletionMessage = (event: TaskEvent): boolean => { + if (event.eventName !== RooCodeEventName.Message) return false + const message = event.payload[0]?.message + if (typeof message !== "object" || message === null) return false + const record = message as { type?: unknown; say?: unknown; partial?: unknown; text?: unknown } + return ( + record.type === "say" && + record.say === "completion_result" && + record.partial !== true && + typeof record.text === "string" && + record.text.trim().length > 0 + ) +} + const traceSettings = (fixtureProjection: Record, credential: string): AgentTrace["settings"] => { try { const projectionJson = JSON.stringify(fixtureProjection) @@ -175,17 +193,7 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp return } accountant.observe(event) - if (event.eventName === RooCodeEventName.Message) { - const message = event.payload[0].message - if ( - message.type === "say" && - message.say === "completion_result" && - message.partial !== true && - typeof message.text === "string" && - message.text.trim().length > 0 - ) - completionMessageSeen = true - } + if (isFinalCompletionMessage(event)) completionMessageSeen = true if (event.eventName === RooCodeEventName.TaskCompleted) completedSeen = true options.observers?.onAcceptedEvent?.(event, acceptedTaskId) } From ae3a9fb3731796bbde4bbc13bfaf09b9d507b87c Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 24 Oct 2026 01:29:01 +0000 Subject: [PATCH 3/9] fix(evals): enforce trust checks and reliable comparison cleanup and reports --- scripts/agent-evals/README.md | 22 +- scripts/agent-evals/coding-compare.mts | 109 +++++-- scripts/agent-evals/coding-compare.ts | 181 ++++++++++- scripts/agent-evals/coding-outcomes.test.ts | 18 ++ scripts/agent-evals/coding-outcomes.ts | 7 + scripts/agent-evals/coding-report.test.ts | 291 +++++++++++++++++- scripts/agent-evals/coding-run.test.ts | 116 ++++++- scripts/agent-evals/coding-run.ts | 6 +- scripts/agent-evals/coding-side.mts | 58 ++-- scripts/agent-evals/coding-side.test.ts | 178 +++++++++++ scripts/agent-evals/coding-trust.test.ts | 60 +++- scripts/agent-evals/coding-trust.ts | 50 ++- scripts/agent-evals/container-runtime.test.ts | 28 ++ scripts/agent-evals/container-runtime.ts | 15 +- scripts/agent-evals/fs-atomic.test.ts | 22 ++ scripts/agent-evals/runner.test.ts | 94 +++++- scripts/agent-evals/runner.ts | 8 +- .../agent-evals/side-process-guard.test.ts | 266 ++++++++++++++++ scripts/agent-evals/side-process-guard.ts | 70 +++++ scripts/solheim-smoke/run.test.ts | 4 +- 20 files changed, 1510 insertions(+), 93 deletions(-) create mode 100644 scripts/agent-evals/coding-side.test.ts create mode 100644 scripts/agent-evals/side-process-guard.test.ts create mode 100644 scripts/agent-evals/side-process-guard.ts diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index da23438af4..d73b8dcf52 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -80,8 +80,10 @@ them: message still appears in `completionMessageSeen`. - `SESSION_LOST` covers disconnects and interrupts. A manual stop after completion output stays `SESSION_LOST`. -- The usage drain runs only after a `taskCompleted` event. It stays inside the - bounded drain window. +- The usage drain runs after a `taskCompleted` event and after an observer stop + that followed the final `completion_result` message. It stays inside the + bounded drain window. A plain stop, a run timeout, and a session loss get no + drain. ## Checks @@ -147,9 +149,17 @@ A comparison runs the extension code of two commits with the provider credential The comparison driver, fixtures, grader, and report builder all run from the trusted checkout that invokes `coding-evals:compare`. Only the extension host loads an evaluated worktree. The side process receives only the allowlisted environment plus the one named credential variable. Each side installs dependencies from its own worktree's frozen lockfile and builds its own exact commit, so each side runs its own compiled extension. Each side has its own wall clock (`--side-timeout-minutes`, default 120). +The side script runs the same gate on its own inputs. It requires `--trust-file`, resolves the full commit of its `--evaluated-root`, and rejects an unapproved or unreadable commit before it creates a host or reads the credential. The `--credential-env` name must come from the supported provider variable allowlist; an arbitrary name can never forward an unrelated parent secret. Worktree install and build children also receive only an allowlisted environment, so the driver's secrets never reach them. + +Every driver child, preparation steps and the side alike, runs detached in its own process group and is tracked by one guard. A SIGINT or SIGTERM that reaches only the driver terminates the tracked group through a bounded SIGTERM-then-SIGKILL escalation, and the worktree cleanup waits for that escalation before it removes any tree. + ### Output publication contract -Artifacts are published atomically: content lands in a same-directory temp file and is renamed into place. A reader never sees a partial file. `side-results.json` appears only after every repetition of that side finished, so its existence means a complete side. A missing `side-results.json` means an interrupted side; consumers must treat it as a failed side and never pair partial output. The comparison reports are written atomically and only after both sides published complete results; a failed comparison leaves no report files. +Artifacts are published atomically: content lands in a same-directory temp file and is renamed into place. A reader never sees a partial file. + +A side run invalidates its output directory first. Before the first repetition executes, the side removes any previous `side-results.json` and the whole `artifacts` tree, so stale results from an earlier rerun can never survive. `side-results.json` is the completion marker: it appears only after every repetition of the current invocation finished, and consumers must require it and treat its absence as a failed side. A side rerun that cannot remove a stale artifact fails before any repetition runs. + +The comparison reports are published as one generation. `comparison-report.json` and `comparison-report.md` carry the report's `generatedAt` stamp, and `comparison-report.complete` holds that stamp; it is written last. Consumers must require the marker and match its stamp against the report files. `readPublishedComparisonReport` enforces this: it returns the report only when the marker exists and its stamp matches both files. The marker is removed before publication starts, so a crash between the two file publications leaves a detectable incomplete generation instead of a mixed one. A thrown failure rolls both files back; a failed comparison leaves no report files. ### Commands @@ -158,9 +168,9 @@ Artifacts are published atomically: content lands in a same-directory temp file pnpm test:agent-evals:unit pnpm agent-evals:check-types -# One comparison side against the real provider. Requires the credential -# environment variable and the container runtime for grading. -pnpm coding-evals:side --reps 3 --out /tmp/side-out +# One comparison side against the real provider. Requires the trust policy, +# the credential environment variable, and the container runtime for grading. +pnpm coding-evals:side --reps 3 --out /tmp/side-out --trust-file trust-policy.json # Full paired comparison of two approved commits. pnpm coding-evals:compare --baseline --candidate --trust-file trust-policy.json --reps 3 --out /tmp/compare-out diff --git a/scripts/agent-evals/coding-compare.mts b/scripts/agent-evals/coding-compare.mts index 89bd85fa64..cf3863f041 100644 --- a/scripts/agent-evals/coding-compare.mts +++ b/scripts/agent-evals/coding-compare.mts @@ -9,17 +9,38 @@ // worktree. The credential never leaves the trusted processes except through // the one named provider variable. A commit runs only when its full SHA is // listed in the operator's trust policy; refs are resolved and checked -// against the policy before any worktree or process exists. +// against the policy before any worktree or process exists. The side process +// re-checks the root it was handed against the same policy. +// +// Process containment. Install, build, worktree, and side children receive +// only an allowlisted environment, so the driver's secrets never reach them. +// Each runs detached in its own process group under one guard, so SIGINT and +// SIGTERM aimed at the driver alone terminate the tracked group with a bounded +// SIGTERM-then-SIGKILL escalation before the scratch cleanup runs, and an +// interrupt leaves no orphan host behind. import { execFile, spawn } from "node:child_process" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" import { fileURLToPath } from "node:url" import { promisify } from "node:util" -import { sideResultFileName, toSideResult, validateSideResultFile, type SideResultFile } from "./coding-compare.ts" +import { + publishComparisonReport, + runBoundedPreparation, + sideResultFileName, + toSideResult, + validateSideResultFile, + type SideResultFile, +} from "./coding-compare.ts" import { buildComparisonReport, renderMarkdownReport } from "./coding-report.ts" -import { assertCommitsApproved, buildSideEnvironment, loadTrustPolicy, resolveCommit } from "./coding-trust.ts" -import { writeJsonAtomic, writeFileAtomic } from "./fs-atomic.ts" +import { + assertCommitsApproved, + assertCredentialEnvAllowed, + buildSideEnvironment, + loadTrustPolicy, + resolveCommit, +} from "./coding-trust.ts" +import { createSideProcessGuard, type SideKillOutcome } from "./side-process-guard.ts" const execFileAsync = promisify(execFile) const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") @@ -30,6 +51,22 @@ const WORKTREE_TIMEOUT_MS = 120_000 const SIDE_KILL_GRACE_MS = 30_000 const DEFAULT_SIDE_TIMEOUT_MINUTES = 120 +// Interrupt state. The handlers are registered before any child is spawned. +// A signal terminates the tracked child group at once; the main flow observes +// the child exit, the cleanup awaits the escalation before it removes the +// worktrees, and the driver exits with the conventional signal exit code. +const sideGuard = createSideProcessGuard(SIDE_KILL_GRACE_MS) +let interruptSignal: NodeJS.Signals | undefined +let interruptKill: Promise | undefined +const isInterrupted = (): boolean => interruptSignal !== undefined +const onInterruptSignal = (signal: NodeJS.Signals): void => { + if (interruptSignal !== undefined) return + interruptSignal = signal + interruptKill = sideGuard.interrupt() +} +process.on("SIGINT", () => onInterruptSignal("SIGINT")) +process.on("SIGTERM", () => onInterruptSignal("SIGTERM")) + const readArgs = (): { baseline: string candidate: string @@ -58,42 +95,59 @@ const readArgs = (): { if (!Number.isInteger(sideTimeoutMinutes) || sideTimeoutMinutes < 1) { throw new Error("--side-timeout-minutes must be a positive integer") } + const credentialEnv = read("credential-env") ?? "SOLHEIM_API_KEY" + // The variable name must be one the framework supports. An arbitrary name + // could forward an unrelated parent secret to the side. + assertCredentialEnvAllowed(credentialEnv) return { baseline, candidate, reps, out: path.resolve(out), - credentialEnv: read("credential-env") ?? "SOLHEIM_API_KEY", + credentialEnv, allowUnisolatedGrading: args.includes("--allow-unisolated-grading"), trustFile: path.resolve(trustFile), sideTimeoutMs: sideTimeoutMinutes * 60_000, } } -const runBounded = async (command: string, args: readonly string[], cwd: string, timeoutMs: number): Promise => { - await execFileAsync(command, [...args], { cwd, timeout: timeoutMs, killSignal: "SIGKILL" }) -} - // Prepares the worktree of one evaluated commit. Dependencies install from // the worktree's own frozen lockfile, and that exact commit's extension and // webview build from its own sources. No artifact crosses sides. const prepareWorktree = async (worktree: string): Promise => { - await runBounded("pnpm", ["install", "--frozen-lockfile"], worktree, INSTALL_TIMEOUT_MS) - await runBounded("pnpm", ["build"], worktree, BUILD_TIMEOUT_MS) + await runBoundedPreparation( + sideGuard, + isInterrupted, + "pnpm", + ["install", "--frozen-lockfile"], + worktree, + INSTALL_TIMEOUT_MS, + ) + await runBoundedPreparation(sideGuard, isInterrupted, "pnpm", ["build"], worktree, BUILD_TIMEOUT_MS) } // Runs one trusted side against the evaluated worktree. The side script comes // from this checkout, receives only the allowlisted environment plus the -// named credential variable, and must finish inside its own wall clock. +// named credential variable, re-checks the worktree commit against the trust +// policy, and must finish inside its own wall clock. const runSide = async ( side: "baseline" | "candidate", commit: string, inputs: ReturnType, scratchDir: string, ): Promise => { + if (isInterrupted()) throw new Error(`interrupted before the ${side} side started`) const worktree = path.join(scratchDir, side) - await runBounded("git", ["worktree", "add", "--detach", worktree, commit], root, WORKTREE_TIMEOUT_MS) + await runBoundedPreparation( + sideGuard, + isInterrupted, + "git", + ["worktree", "add", "--detach", worktree, commit], + root, + WORKTREE_TIMEOUT_MS, + ) await prepareWorktree(worktree) + if (isInterrupted()) throw new Error(`interrupted before the ${side} side started`) const sideOut = path.join(scratchDir, `${side}-out`) await fs.mkdir(sideOut, { recursive: true }) const exit = await new Promise((resolve) => { @@ -107,6 +161,8 @@ const runSide = async ( String(inputs.reps), "--out", sideOut, + "--trust-file", + inputs.trustFile, "--credential-env", inputs.credentialEnv, "--evaluated-root", @@ -120,6 +176,7 @@ const runSide = async ( detached: true, }, ) + sideGuard.track(child.pid) let timedOut = false const timer = setTimeout(() => { timedOut = true @@ -138,18 +195,20 @@ const runSide = async ( }, inputs.sideTimeoutMs) child.once("error", () => { clearTimeout(timer) + sideGuard.track(undefined) resolve(1) }) child.once("exit", (code) => { clearTimeout(timer) + sideGuard.track(undefined) resolve(timedOut ? 1 : (code ?? 1)) }) }) if (exit !== 0) throw new Error(`${side} side failed with exit code ${exit}`) - const parsed: unknown = JSON.parse(await fs.readFile(path.join(sideOut, sideResultFileName(side)), "utf8")) - // The side output must describe exactly the approved commit, or the pair - // is invalid. - return validateSideResultFile(parsed, commit) + const parsed: unknown = JSON.parse(await fs.readFile(path.join(sideOut, sideResultFileName()), "utf8")) + // The side output must describe exactly this side and its approved commit, + // or the pair is invalid. + return validateSideResultFile(parsed, side, commit) } const inputs = readArgs() @@ -171,17 +230,19 @@ try { candidate: toSideResult(await runSide("candidate", candidateCommit, inputs, scratchDir)), repetitions: inputs.reps, }) - // The reports are the comparison's publication point. They appear - // atomically and only after both sides published complete results; any - // failure above leaves no report files behind. - await writeJsonAtomic(path.join(inputs.out, "comparison-report.json"), report) - await writeFileAtomic(path.join(inputs.out, "comparison-report.md"), renderMarkdownReport(report)) + // The reports are the comparison's publication point. The pair is + // published as one generation with a completion marker; any failure above + // or during publication leaves no complete report generation behind. + await publishComparisonReport(inputs.out, report) console.log( report.compatibility.compatible ? `comparison complete: compatible pair, report in ${inputs.out}` : `comparison complete: INCOMPATIBLE pair (${report.compatibility.differences.join(", ")}), report in ${inputs.out}`, ) } finally { + // An interrupt escalation must finish before the worktrees go away, so + // no child is still writing while its tree is deleted. + await interruptKill await execFileAsync("git", ["worktree", "remove", "--force", path.join(scratchDir, "baseline")], { cwd: root, timeout: 30_000, @@ -192,3 +253,7 @@ try { }).catch(() => {}) await fs.rm(scratchDir, { recursive: true, force: true }).catch(() => {}) } +// An interrupt terminated the side group above, and the cleanup in the +// finally block has completed. Exit with the conventional signal code. +if (interruptSignal === "SIGINT") process.exit(130) +if (interruptSignal === "SIGTERM") process.exit(143) diff --git a/scripts/agent-evals/coding-compare.ts b/scripts/agent-evals/coding-compare.ts index 0a8b9cb04e..921b901283 100644 --- a/scripts/agent-evals/coding-compare.ts +++ b/scripts/agent-evals/coding-compare.ts @@ -1,7 +1,13 @@ +import { spawn } from "node:child_process" +import fs from "node:fs/promises" +import path from "node:path" import type { SidePins } from "./coding-pins.ts" import { buildComparisonReport, renderMarkdownReport, type ComparisonReport, type SideResult } from "./coding-report.ts" import { DEFAULT_REPETITIONS } from "./coding-stats.ts" import type { CodingRunRecord } from "./coding-run.ts" +import { buildWorktreeEnvironment } from "./coding-trust.ts" +import type { SideProcessGuard } from "./side-process-guard.ts" +import { writeFileAtomic, writeJsonAtomic } from "./fs-atomic.ts" export { buildComparisonReport, renderMarkdownReport } export type { ComparisonReport, SideResult } @@ -45,6 +51,63 @@ export async function runComparison(request: ComparisonRequest): Promise boolean, + command: string, + args: readonly string[], + cwd: string, + timeoutMs: number, +): Promise => { + if (isInterrupted()) throw new Error(`interrupted before ${command} started`) + await new Promise((resolve, reject) => { + const child = spawn(command, [...args], { + cwd, + stdio: ["ignore", "ignore", "pipe"], + detached: true, + env: buildWorktreeEnvironment(process.env), + }) + guard.track(child.pid) + let stderr = "" + child.stderr?.on("data", (chunk: Buffer) => { + if (stderr.length >= STDERR_LIMIT_BYTES) return + stderr += chunk.toString() + }) + let timedOut = false + const timer = setTimeout(() => { + timedOut = true + try { + if (child.pid) process.kill(-child.pid, "SIGKILL") + } catch { + child.kill("SIGKILL") + } + }, timeoutMs) + child.once("error", (error) => { + clearTimeout(timer) + guard.track(undefined) + reject(error) + }) + child.once("exit", (code, signal) => { + clearTimeout(timer) + guard.track(undefined) + if (code === 0) { + resolve() + return + } + const reason = timedOut ? "its time bound expired" : code === null ? `signal ${signal}` : `code ${code}` + reject(new Error([`${command} failed: ${reason}`, stderr].filter(Boolean).join("\n"))) + }) + }) +} + // Serializes a side result for artifact storage. Full run records travel so // each side's traces and grading verdicts stay inspectable. export interface SideResultFile { @@ -54,23 +117,58 @@ export interface SideResultFile { // Validates one side result file before the report builder consumes it. The // file is data, so its shape is checked, and it must record exactly the -// approved commit of its side. Anything else aborts the comparison instead of -// producing a report from partial output. -export function validateSideResultFile(parsed: unknown, expectedCommit: string): SideResultFile { +// expected side label and the approved commit of that side. The pins must be +// complete, and every run must carry object metrics. Anything else aborts the +// comparison instead of producing a report from partial output. +const PIN_STRING_FIELDS = [ + "side", + "commit", + "suiteId", + "suiteDigest", + "settingsDigest", + "modelIdentifier", + "graderVersion", + "graderImageDigest", +] as const + +export function validateSideResultFile( + parsed: unknown, + expectedSide: SidePins["side"], + expectedCommit: string, +): SideResultFile { if (typeof parsed !== "object" || parsed === null) throw new Error("side result is not a JSON object") const record = parsed as Record const pins = record["pins"] if (typeof pins !== "object" || pins === null) throw new Error("side result has no pins") const pinsRecord = pins as Record - for (const field of ["side", "commit", "suiteId", "suiteDigest", "graderVersion"] as const) { + for (const field of PIN_STRING_FIELDS) { if (typeof pinsRecord[field] !== "string" || (pinsRecord[field] as string).length === 0) { throw new Error(`side result pins lack ${field}`) } } + if (pinsRecord["side"] !== expectedSide) { + throw new Error(`side result records side ${String(pinsRecord["side"])} instead of ${expectedSide}`) + } if (pinsRecord["commit"] !== expectedCommit) { throw new Error(`side result records commit ${String(pinsRecord["commit"])} instead of ${expectedCommit}`) } - if (!Array.isArray(record["runs"])) throw new Error("side result has no runs array") + const runnerEnvironment = pinsRecord["runnerEnvironment"] + if (typeof runnerEnvironment !== "object" || runnerEnvironment === null) { + throw new Error("side result pins lack runnerEnvironment") + } + for (const field of ["os", "nodeVersion", "vscodeTarget"] as const) { + const value = (runnerEnvironment as Record)[field] + if (typeof value !== "string" || value.length === 0) { + throw new Error(`side result runnerEnvironment lacks ${field}`) + } + } + const runs = record["runs"] + if (!Array.isArray(runs)) throw new Error("side result has no runs array") + for (const run of runs) { + if (typeof run !== "object" || run === null) throw new Error("side result run is not an object") + const metrics = (run as Record)["metrics"] + if (typeof metrics !== "object" || metrics === null) throw new Error("side result run lacks metrics") + } return parsed as SideResultFile } @@ -80,4 +178,75 @@ export const toSideResult = (file: SideResultFile): SideResult => ({ runs: file.runs.map((record) => record.metrics), }) -export const sideResultFileName = (side: "baseline" | "candidate"): string => `${side}-results.json` +// The one shared side result file name. The side writer and the comparison +// reader must agree on it. The name carries no side label, because each side +// writes into its own output directory. +export const sideResultFileName = (): string => "side-results.json" + +const COMPARISON_REPORT_JSON = "comparison-report.json" +const COMPARISON_REPORT_MD = "comparison-report.md" +const COMPARISON_REPORT_COMPLETE = "comparison-report.complete" + +// Publishes the JSON and Markdown report pair as one generation. The +// completion marker of any previous generation is removed first, so a crash +// between the two publications leaves a detectable incomplete generation +// instead of a mixed one. A thrown failure rolls both files back, so a failed +// comparison leaves no report files behind. Consumers must require the +// completion marker and match its generation against the report files. +export async function publishComparisonReport(outDir: string, report: ComparisonReport): Promise { + const generation = report.generatedAt + const jsonPath = path.join(outDir, COMPARISON_REPORT_JSON) + const markdownPath = path.join(outDir, COMPARISON_REPORT_MD) + const markerPath = path.join(outDir, COMPARISON_REPORT_COMPLETE) + await fs.rm(markerPath, { force: true }) + try { + await writeJsonAtomic(jsonPath, { ...report, generation }) + await writeFileAtomic(markdownPath, renderMarkdownReport(report)) + await writeFileAtomic(markerPath, `${generation}\n`) + } catch (error) { + await fs.rm(jsonPath, { force: true }).catch(() => {}) + await fs.rm(markdownPath, { force: true }).catch(() => {}) + await fs.rm(markerPath, { force: true }).catch(() => {}) + throw error + } +} + +// Reads the generation a complete publication carries. undefined means the +// publication is absent, partial, or crashed between its files. +export const publishedComparisonGeneration = async (outDir: string): Promise => { + try { + const generation = (await fs.readFile(path.join(outDir, COMPARISON_REPORT_COMPLETE), "utf8")).trim() + return generation.length > 0 ? generation : undefined + } catch { + return undefined + } +} + +// Reads one published comparison report. The completion marker is required +// and its generation stamp must match the JSON report and the Markdown +// render, so a partial or mixed generation is never consumed. undefined means +// no complete publication exists. +export async function readPublishedComparisonReport(outDir: string): Promise { + const generation = await publishedComparisonGeneration(outDir) + if (generation === undefined) return undefined + const parsed: unknown = JSON.parse(await fs.readFile(path.join(outDir, COMPARISON_REPORT_JSON), "utf8")) + if (typeof parsed !== "object" || parsed === null) throw new Error("comparison report is not a JSON object") + if ((parsed as Record)["generation"] !== generation) { + throw new Error("comparison report generation does not match the completion marker") + } + const markdown = await fs.readFile(path.join(outDir, COMPARISON_REPORT_MD), "utf8") + if (!markdown.includes(generation)) { + throw new Error("comparison report Markdown does not match the completion marker") + } + return parsed as ComparisonReport +} + +// Removes the previous side generation before a side executes. After this +// point a crash can no longer present stale results as fresh output, because +// consumers require side-results.json, and it exists only after the current +// invocation completes. A stale file that cannot be removed fails the side +// before any run spends budget. +export async function invalidateSideArtifacts(outDir: string): Promise { + await fs.rm(path.join(outDir, "artifacts"), { recursive: true, force: true }) + await fs.rm(path.join(outDir, sideResultFileName()), { force: true }) +} diff --git a/scripts/agent-evals/coding-outcomes.test.ts b/scripts/agent-evals/coding-outcomes.test.ts index 6b6dbf57fa..1a247ba859 100644 --- a/scripts/agent-evals/coding-outcomes.test.ts +++ b/scripts/agent-evals/coding-outcomes.test.ts @@ -57,4 +57,22 @@ describe("run outcome classification", () => { ) assert.equal(classifyRunOutcome({ ...base, grading: undefined }).outcome, "infrastructure-failure") }) + + it("reports a truncated final workspace as infrastructure failure without grading", () => { + // A truncated capture is incomplete. Even a passing verdict must not + // turn missing files into a correct answer. + assert.deepEqual(classifyRunOutcome({ ...base, finalWorkspaceTruncated: true, grading: { kind: "passed" } }), { + outcome: "infrastructure-failure", + reason: "finalWorkspaceTruncated", + }) + assert.deepEqual(classifyRunOutcome({ ...base, finalWorkspaceTruncated: true }), { + outcome: "infrastructure-failure", + reason: "finalWorkspaceTruncated", + }) + // A complete capture keeps grading. + assert.deepEqual(classifyRunOutcome({ ...base, finalWorkspaceTruncated: false, grading: { kind: "passed" } }), { + outcome: "correct", + reason: "hiddenTestsPassed", + }) + }) }) diff --git a/scripts/agent-evals/coding-outcomes.ts b/scripts/agent-evals/coding-outcomes.ts index 74738edc02..aca87ba7c4 100644 --- a/scripts/agent-evals/coding-outcomes.ts +++ b/scripts/agent-evals/coding-outcomes.ts @@ -7,6 +7,9 @@ export interface OutcomeInput { hasTrace: boolean executionCode?: AgentExecutionCode finalWorkspaceAvailable: boolean + // True when the captured workspace was incomplete. A truncated capture is + // a harness fault, never agent work, so it can never grade as incorrect. + finalWorkspaceTruncated?: boolean grading?: Pick & { reason?: string } } @@ -30,6 +33,8 @@ const INFRASTRUCTURE_CODES: readonly AgentExecutionCode[] = [ // - COMPLETED and STOPPED runs are graded. The hidden tests decide: // passed is correct, failed is incorrect, including when the agent // reported completion. +// - A truncated final workspace is infrastructure failure. The capture is +// incomplete, so grading it could turn missing files into a wrong answer. // - A grader that hung, crashed, or hit its limits is grading failure and is // never counted as incorrect. export function classifyRunOutcome(input: OutcomeInput): ClassifiedOutcome { @@ -42,6 +47,8 @@ export function classifyRunOutcome(input: OutcomeInput): ClassifiedOutcome { return { outcome: "infrastructure-failure", reason: `execution-${code ?? "unknown"}` } if (!input.finalWorkspaceAvailable) return { outcome: "infrastructure-failure", reason: "finalWorkspaceUnavailable" } + if (input.finalWorkspaceTruncated === true) + return { outcome: "infrastructure-failure", reason: "finalWorkspaceTruncated" } if (!input.grading) return { outcome: "infrastructure-failure", reason: "graderNotRun" } switch (input.grading.kind) { case "passed": diff --git a/scripts/agent-evals/coding-report.test.ts b/scripts/agent-evals/coding-report.test.ts index 8cf8667a2c..07a7617a72 100644 --- a/scripts/agent-evals/coding-report.test.ts +++ b/scripts/agent-evals/coding-report.test.ts @@ -1,7 +1,19 @@ import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" import { describe, it } from "node:test" import { buildComparisonReport, renderMarkdownReport, type SideResult } from "./coding-report.ts" -import { runComparison, validateSideResultFile, type SideExecutor } from "./coding-compare.ts" +import { + invalidateSideArtifacts, + publishComparisonReport, + publishedComparisonGeneration, + readPublishedComparisonReport, + runComparison, + validateSideResultFile, + type SideExecutor, +} from "./coding-compare.ts" +import { writeFileAtomic, writeJsonAtomic } from "./fs-atomic.ts" import type { CodingRunMetrics } from "./coding-metrics.ts" import type { SidePins } from "./coding-pins.ts" @@ -193,18 +205,277 @@ describe("comparison orchestration", () => { }) describe("side result validation", () => { - it("accepts a result that records exactly the approved commit", () => { - const file = { pins: pins("baseline", "a".repeat(40)), runs: [] } - const validated = validateSideResultFile(JSON.parse(JSON.stringify(file)), "a".repeat(40)) + const validatedFile = (side: "baseline" | "candidate", commit: string, overrides: Record = {}) => + JSON.parse(JSON.stringify({ pins: pins(side, commit), runs: [], ...overrides })) + + it("accepts a result that records exactly the expected side and commit", () => { + const validated = validateSideResultFile(validatedFile("baseline", "a".repeat(40)), "baseline", "a".repeat(40)) assert.equal(validated.pins.side, "baseline") + assert.equal(validated.pins.commit, "a".repeat(40)) + }) + + it("rejects a result that records the wrong side or a different commit", () => { + assert.throws( + () => validateSideResultFile(validatedFile("candidate", "a".repeat(40)), "baseline", "a".repeat(40)), + /records side candidate instead of baseline/, + ) + assert.throws( + () => validateSideResultFile(validatedFile("baseline", "b".repeat(40)), "baseline", "a".repeat(40)), + /records commit .* instead of/, + ) + }) + + it("rejects incomplete pins with the intended error", () => { + for (const field of [ + "side", + "commit", + "suiteId", + "suiteDigest", + "settingsDigest", + "modelIdentifier", + "graderVersion", + "graderImageDigest", + ]) { + const file = validatedFile("baseline", "a".repeat(40)) + delete (file.pins as Record)[field] + assert.throws( + () => validateSideResultFile(file, "baseline", "a".repeat(40)), + new RegExp(`pins lack ${field}`), + ) + } + assert.throws( + () => + validateSideResultFile( + validatedFile("baseline", "a".repeat(40), { pins: { side: "baseline" } }), + "baseline", + "a".repeat(40), + ), + /pins lack commit/, + ) + }) + + it("rejects incomplete runnerEnvironment pins", () => { + const file = validatedFile("baseline", "a".repeat(40)) + delete (file.pins as Record)["runnerEnvironment"] + assert.throws(() => validateSideResultFile(file, "baseline", "a".repeat(40)), /pins lack runnerEnvironment/) + const partial = validatedFile("baseline", "a".repeat(40)) + ;(partial.pins as Record)["runnerEnvironment"] = { os: "linux", nodeVersion: "v22" } + assert.throws( + () => validateSideResultFile(partial, "baseline", "a".repeat(40)), + /runnerEnvironment lacks vscodeTarget/, + ) + }) + + it("rejects runs without object metrics", () => { + assert.throws( + () => + validateSideResultFile( + validatedFile("baseline", "a".repeat(40), { runs: [{ fixtureName: "clamp-range", repIndex: 0 }] }), + "baseline", + "a".repeat(40), + ), + /run lacks metrics/, + ) + assert.throws( + () => + validateSideResultFile( + validatedFile("baseline", "a".repeat(40), { runs: ["nope"] }), + "baseline", + "a".repeat(40), + ), + /run is not an object/, + ) }) - it("rejects a result that records a different commit, or has no shape", () => { - assert.throws(() => - validateSideResultFile({ pins: pins("baseline", "b".repeat(40)), runs: [] }, "a".repeat(40)), + it("rejects missing shape with the intended error", () => { + assert.throws(() => validateSideResultFile({ runs: [] }, "baseline", "a".repeat(40)), /has no pins/) + assert.throws(() => validateSideResultFile(null, "baseline", "a".repeat(40)), /not a JSON object/) + assert.throws( + () => validateSideResultFile({ pins: pins("baseline", "a".repeat(40)) }, "baseline", "a".repeat(40)), + /has no runs array/, ) - assert.throws(() => validateSideResultFile({ runs: [] }, "a".repeat(40))) - assert.throws(() => validateSideResultFile(null, "a".repeat(40))) - assert.throws(() => validateSideResultFile({ pins: pins("baseline", "a".repeat(40)) }, "a".repeat(40))) + }) +}) + +describe("comparison report publication", () => { + const report = buildComparisonReport({ + baseline: sideResult("baseline", "a".repeat(40), [metrics("clamp-range", 0, "correct", 0.1)]), + candidate: sideResult("candidate", "b".repeat(40), [metrics("clamp-range", 0, "correct", 0.2)]), + }) + + it("publishes the pair with a matching completion marker", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "report-publish-test-")) + try { + await publishComparisonReport(dir, report) + const generation = await publishedComparisonGeneration(dir) + assert.equal(generation, report.generatedAt) + const json = JSON.parse(await fs.readFile(path.join(dir, "comparison-report.json"), "utf8")) as { + generation?: string + } + assert.equal(json.generation, report.generatedAt) + const markdown = await fs.readFile(path.join(dir, "comparison-report.md"), "utf8") + assert.ok(markdown.includes(report.generatedAt)) + // The publication leaves no temp residue. + assert.deepEqual((await fs.readdir(dir)).sort(), [ + "comparison-report.complete", + "comparison-report.json", + "comparison-report.md", + ]) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("replaces a prior report set and updates the marker to the new generation", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "report-publish-test-")) + try { + // An existing prior report set. + await writeFileAtomic(path.join(dir, "comparison-report.json"), "old") + await writeFileAtomic(path.join(dir, "comparison-report.md"), "old md") + await writeFileAtomic(path.join(dir, "comparison-report.complete"), "2020-01-01T00:00:00.000Z\n") + assert.equal(await publishedComparisonGeneration(dir), "2020-01-01T00:00:00.000Z") + await publishComparisonReport(dir, report) + assert.equal(await publishedComparisonGeneration(dir), report.generatedAt) + const markdown = await fs.readFile(path.join(dir, "comparison-report.md"), "utf8") + assert.ok(markdown.includes(report.generatedAt)) + assert.ok(!markdown.includes("old md")) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("rolls the pair back when the Markdown publication fails", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "report-publish-test-")) + try { + // The Markdown target is a non-empty directory, so its atomic write + // fails after the JSON publication. + const markdownPath = path.join(dir, "comparison-report.md") + await fs.mkdir(markdownPath) + await fs.writeFile(path.join(markdownPath, "occupied"), "x") + await assert.rejects(publishComparisonReport(dir, report)) + // The JSON and marker are rolled back. No mixed generation and no + // temp residue remains. + assert.equal(await publishedComparisonGeneration(dir), undefined) + assert.equal( + await fs.access(path.join(dir, "comparison-report.json")).then( + () => true, + () => false, + ), + false, + ) + assert.deepEqual(await fs.readdir(dir), ["comparison-report.md"]) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("treats a crash between publications as an incomplete generation", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "report-publish-test-")) + try { + // Both files exist, but the process died before the marker. + await writeFileAtomic(path.join(dir, "comparison-report.json"), "partial") + await writeFileAtomic(path.join(dir, "comparison-report.md"), "partial md") + assert.equal(await publishedComparisonGeneration(dir), undefined) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +}) + +describe("comparison report consumption", () => { + const report = buildComparisonReport({ + baseline: sideResult("baseline", "a".repeat(40), [metrics("clamp-range", 0, "correct", 0.1)]), + candidate: sideResult("candidate", "b".repeat(40), [metrics("clamp-range", 0, "correct", 0.2)]), + }) + + it("reads back the report only when the marker matches both files", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "report-consume-test-")) + try { + await publishComparisonReport(dir, report) + const read = await readPublishedComparisonReport(dir) + assert.ok(read !== undefined) + assert.equal(read.generatedAt, report.generatedAt) + assert.equal(read.compatibility.compatible, report.compatibility.compatible) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("reports no publication when the marker is absent", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "report-consume-test-")) + try { + // Both report files exist, but the process died before the marker. + await writeFileAtomic(path.join(dir, "comparison-report.json"), "{}") + await writeFileAtomic(path.join(dir, "comparison-report.md"), "partial md") + assert.equal(await readPublishedComparisonReport(dir), undefined) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("refuses a report whose generation differs from the marker", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "report-consume-test-")) + try { + await publishComparisonReport(dir, report) + await writeFileAtomic( + path.join(dir, "comparison-report.json"), + JSON.stringify({ ...report, generation: "2020-01-01T00:00:00.000Z" }), + ) + await assert.rejects(readPublishedComparisonReport(dir), /does not match the completion marker/) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("refuses a Markdown render that differs from the marker", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "report-consume-test-")) + try { + await publishComparisonReport(dir, report) + await writeFileAtomic(path.join(dir, "comparison-report.md"), "stale render without the stamp") + await assert.rejects(readPublishedComparisonReport(dir), /does not match the completion marker/) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("refuses a pair that lost its Markdown file", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "report-consume-test-")) + try { + await publishComparisonReport(dir, report) + await fs.rm(path.join(dir, "comparison-report.md")) + await assert.rejects(readPublishedComparisonReport(dir)) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +}) + +describe("side artifact invalidation", () => { + it("removes stale side results and run records before a rerun", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-invalidate-test-")) + try { + await writeJsonAtomic(path.join(dir, "side-results.json"), { stale: true }) + const staleRecordDir = path.join(dir, "artifacts", "clamp-range-rep4") + await fs.mkdir(staleRecordDir, { recursive: true }) + await writeJsonAtomic(path.join(staleRecordDir, "run-record.json"), { stale: true }) + await invalidateSideArtifacts(dir) + assert.deepEqual(await fs.readdir(dir), []) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("fails closed when the stale side result cannot be removed", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-invalidate-test-")) + try { + // The stale result path is a non-empty directory, so the removal + // fails and the side must abort before any run executes. + const stalePath = path.join(dir, "side-results.json") + await fs.mkdir(stalePath) + await fs.writeFile(path.join(stalePath, "occupied"), "x") + await assert.rejects(invalidateSideArtifacts(dir)) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } }) }) diff --git a/scripts/agent-evals/coding-run.test.ts b/scripts/agent-evals/coding-run.test.ts index d26c627323..cbef1e6956 100644 --- a/scripts/agent-evals/coding-run.test.ts +++ b/scripts/agent-evals/coding-run.test.ts @@ -4,7 +4,7 @@ import os from "node:os" import path from "node:path" import { after, describe, it } from "node:test" import { RooCodeEventName, TaskCommandName, type TaskCommand, type TaskEvent } from "@roo-code/types" -import { runCodingFixtureOnce, withWorkspaceCapture } from "./coding-run.ts" +import { runCodingFixtureOnce, stopOnFinalCompletionMessage, withWorkspaceCapture } from "./coding-run.ts" import { createLocalSubprocessGradingRuntime } from "./coding-grader.ts" import type { AgentClient, AgentHost, AgentRuntime, FinalWorkspaceSnapshot } from "./runner.ts" import type { AgentFixtureWorkspaceFile } from "./contract.ts" @@ -19,8 +19,14 @@ interface SyntheticOptions { // Emits the model's final non-partial completion_result message instead // of the accepted TaskCompleted event. completeByMessageOnly?: boolean + // Emits only a partial completion_result message. The headless stop + // policy must never treat it as final. + completeByPartialMessageOnly?: boolean // Rejects the final-workspace read, as a broken capture does. failWorkspaceRead?: boolean + // Reports the final-workspace capture as truncated, as a bounded walker + // over an oversized tree does. + truncateWorkspaceRead?: boolean } const syntheticRuntime = (options: SyntheticOptions): { runtime: AgentRuntime; traces: AgentTrace[] } => { @@ -48,11 +54,13 @@ const syntheticRuntime = (options: SyntheticOptions): { runtime: AgentRuntime; t eventName: RooCodeEventName.TaskStartResponse, payload: [{ requestId, success: true, taskId: "task-1" }], }) - if (options.completeByMessageOnly) { + if (options.completeByMessageOnly || options.completeByPartialMessageOnly) { // The model reports completion, but the accepted TaskCompleted - // event never fires. This is the headless completion path. - // The synthetic message satisfies the Message payload shape; - // the cast names the variant the schema accepts. + // event never fires. This is the headless completion path. The + // partial variant streams the message with partial: true, which + // the stop policy must reject. The synthetic message satisfies + // the Message payload shape; the cast names the variant the + // schema accepts. emit({ eventName: RooCodeEventName.Message, payload: [ @@ -64,6 +72,7 @@ const syntheticRuntime = (options: SyntheticOptions): { runtime: AgentRuntime; t say: "completion_result", ts: 1, text: "Task finished.", + ...(options.completeByPartialMessageOnly ? { partial: true } : {}), }, }, ], @@ -97,7 +106,7 @@ const syntheticRuntime = (options: SyntheticOptions): { runtime: AgentRuntime; t }, readFinalWorkspace: async (): Promise => { if (options.failWorkspaceRead) throw new Error("unreadable workspace") - return { files: options.finalFiles, truncated: false } + return { files: options.finalFiles, truncated: options.truncateWorkspaceRead === true } }, } return { runtime, traces } @@ -255,4 +264,99 @@ describe("coding run orchestration", () => { await captured.runtime.cleanup() assert.equal(captured.getSnapshot(), first) }) + + const completionSayEvent = (message: Record): TaskEvent => + ({ + eventName: RooCodeEventName.Message, + payload: [{ taskId: "task-1", action: "created", message: { ts: 1, ...message } }], + }) as unknown as TaskEvent + + it("stops on the final non-partial completion_result say message", () => { + const observer = stopOnFinalCompletionMessage() + observer.onAcceptedEvent!(completionSayEvent({ type: "say", say: "completion_result", text: "done" }), "task-1") + assert.equal(observer.shouldStop!(), true) + }) + + it("never stops on partial, blank, or wrong-say completion messages", () => { + const rejected = [ + { name: "partial", message: { type: "say", say: "completion_result", text: "working", partial: true } }, + { name: "blank", message: { type: "say", say: "completion_result", text: " \n\t " } }, + { name: "wrongSay", message: { type: "say", say: "text", text: "progress note" } }, + ] + for (const { name, message } of rejected) { + const observer = stopOnFinalCompletionMessage() + observer.onAcceptedEvent!(completionSayEvent(message), "task-1") + assert.equal(observer.shouldStop!(), false, `observer must not stop on a ${name} completion message`) + } + }) + + it( + "skips grading and reports infrastructure failure when the snapshot is truncated", + { timeout: 30_000 }, + async () => { + const bundle = codingBundle("clamp-range") + const { runtime } = syntheticRuntime({ + finalFiles: finalFilesWith(bundle.grading.correctSolution[0]!), + complete: true, + truncateWorkspaceRead: true, + }) + const record = await runCodingFixtureOnce({ + fixtureName: "clamp-range", + repIndex: 5, + credential: "sk-test", + runtime, + gradingRuntime: createLocalSubprocessGradingRuntime(), + scratchDir: await scratch(), + }) + // A truncated capture is incomplete harness data. Grading it could + // score missing files as a wrong answer, so it never grades. + assert.equal(record.outcome.outcome, "infrastructure-failure") + assert.equal(record.outcome.reason, "finalWorkspaceTruncated") + assert.equal(record.grading, undefined) + assert.equal(record.finalWorkspaceTruncated, true) + }, + ) + + it("keeps grading a complete snapshot", { timeout: 30_000 }, async () => { + const bundle = codingBundle("clamp-range") + const { runtime } = syntheticRuntime({ + finalFiles: finalFilesWith(bundle.grading.correctSolution[0]!), + complete: true, + truncateWorkspaceRead: false, + }) + const record = await runCodingFixtureOnce({ + fixtureName: "clamp-range", + repIndex: 6, + credential: "sk-test", + runtime, + gradingRuntime: createLocalSubprocessGradingRuntime(), + scratchDir: await scratch(), + }) + assert.equal(record.outcome.outcome, "correct") + assert.equal(record.grading?.kind, "passed") + assert.equal(record.finalWorkspaceTruncated, false) + }) + + it("times out a run that only ever streams a partial completion message", { timeout: 30_000 }, async () => { + const { runtime } = syntheticRuntime({ + finalFiles: [], + complete: false, + completeByPartialMessageOnly: true, + }) + const record = await runCodingFixtureOnce({ + fixtureName: "clamp-range", + repIndex: 7, + credential: "sk-test", + runtime, + gradingRuntime: createLocalSubprocessGradingRuntime(), + scratchDir: await scratch(), + }) + // The partial message never satisfies the stop policy, so the run + // window expires instead of stopping early. A timeout is never graded. + assert.equal(record.executionCode, "TASK_RUN_TIMEOUT") + assert.equal(record.outcome.outcome, "timed-out") + assert.equal(record.grading, undefined) + assert.equal(record.trace?.execution.stopReason, "runTimeout") + assert.equal(record.trace?.execution.completionMessageSeen, false) + }) }) diff --git a/scripts/agent-evals/coding-run.ts b/scripts/agent-evals/coding-run.ts index b09cc0e973..07914237d2 100644 --- a/scripts/agent-evals/coding-run.ts +++ b/scripts/agent-evals/coding-run.ts @@ -132,7 +132,10 @@ export async function runCodingFixtureOnce(inputs: CodingRunInputs): Promise { const args = process.argv.slice(2) const read = (name: string): string | undefined => { @@ -37,14 +41,21 @@ const readArgs = (): { } const out = read("out") if (!out) throw new Error("Set --out to a scratch directory") + const trustFile = read("trust-file") + if (!trustFile) throw new Error("Set --trust-file to the policy file listing the approved commit for this side") + const credentialEnv = read("credential-env") ?? "SOLHEIM_API_KEY" + // The variable name must be one the framework supports. An arbitrary name + // could forward an unrelated parent secret to the evaluated extension. + assertCredentialEnvAllowed(credentialEnv) return { reps: Number(read("reps") ?? "3"), out: path.resolve(out), - credentialEnv: read("credential-env") ?? "SOLHEIM_API_KEY", + credentialEnv, allowUnisolatedGrading: args.includes("--allow-unisolated-grading"), // Repository root of the extension under test. The trusted harness // code drives the run; only the extension host loads this tree. evaluatedRoot: path.resolve(read("evaluated-root") ?? thisRoot), + trustFile: path.resolve(trustFile), } } @@ -71,20 +82,18 @@ const readVscodeTarget = async (root: string): Promise => { } } -const readCommit = async (root: string): Promise => { - const { execFile } = await import("node:child_process") - const { promisify } = await import("node:util") - try { - const { stdout } = await promisify(execFile)("git", ["rev-parse", "HEAD"], { cwd: root, timeout: 5_000 }) - return stdout.trim() - } catch { - return "commit-unavailable" - } -} - -const { reps, out, credentialEnv, allowUnisolatedGrading, evaluatedRoot } = readArgs() +const { reps, out, credentialEnv, allowUnisolatedGrading, evaluatedRoot, trustFile } = readArgs() if (!Number.isInteger(reps) || reps < 1) throw new Error("--reps must be a positive integer") const sideLabel = readSideLabel() +// Trust gate. The evaluated root is loaded by the extension host and receives +// the credential, so its resolved full commit must be approved before any +// directory is prepared, any host is created, or the credential is read. The +// comparison driver checks its resolved commits first; this second check binds +// the root that was actually passed in. The commit of an unapproved root is +// unreadable or foreign, and both fail closed here. +const policy = await loadTrustPolicy(trustFile) +const evaluatedCommit = await resolveCommit("HEAD", evaluatedRoot) +assertCommitsApproved(policy, [evaluatedCommit]) // Fail before any directory, run, or result file exists. An empty credential // would otherwise produce a plausible file of infrastructure failures. const credential = process.env[credentialEnv] @@ -95,6 +104,12 @@ ensureCodingSuiteRegistered() const gradingRuntime = allowUnisolatedGrading ? createLocalSubprocessGradingRuntime() : createContainerGradingRuntime() +// Invalidate the previous generation before execution. After this point a +// crash can never leave stale side results or run records behind: consumers +// require side-results.json, and it exists only after this invocation +// completes. +await invalidateSideArtifacts(out) + const projection = CODING_FIXTURE_BUNDLES[0]?.fixture.settingsProjection ?? {} const runs: CodingRunRecord[] = [] @@ -138,12 +153,12 @@ for (const bundle of CODING_FIXTURE_BUNDLES) { } // Pins are recorded after the runs so they describe what actually executed: -// the evaluated commit and, when the container runtime resolved one, the -// immutable grader image identity. +// the approved evaluated commit and, when the container runtime resolved one, +// the immutable grader image identity. const digest = gradingRuntime.imageDigest ? await gradingRuntime.imageDigest() : undefined const pins: SidePins = { side: sideLabel, - commit: await readCommit(evaluatedRoot), + commit: evaluatedCommit, suiteId: CODING_SUITE_ID, suiteDigest: codingSuiteDigest(), settingsDigest: digestValue(projection), @@ -165,10 +180,11 @@ for (const record of runs) { await writeJsonAtomic(path.join(artifactDir, "run-record.json"), record) } -// Publication point. side-results.json appears atomically only after every -// repetition finished, so a reader that sees the file sees a complete side. -// A missing file means an interrupted side; consumers must treat it as a -// failed side and never pair partial output. +// Publication point and completion marker. side-results.json appears +// atomically only after every repetition of this invocation finished, so a +// reader that sees the file sees a complete side of the current run. A +// missing file means an interrupted side; consumers must treat it as a failed +// side and never pair partial output. await writeJsonAtomic(path.join(out, "side-results.json"), result) const correct = runs.filter((record) => record.outcome.outcome === "correct").length console.log(`coding-evals side complete: ${correct}/${runs.length} correct, suite ${pins.suiteDigest.slice(0, 12)}`) diff --git a/scripts/agent-evals/coding-side.test.ts b/scripts/agent-evals/coding-side.test.ts new file mode 100644 index 0000000000..fccf0c995a --- /dev/null +++ b/scripts/agent-evals/coding-side.test.ts @@ -0,0 +1,178 @@ +import assert from "node:assert/strict" +import { execFile } from "node:child_process" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { fileURLToPath } from "node:url" +import { promisify } from "node:util" +import { describe, it } from "node:test" + +const execFileAsync = promisify(execFile) +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") + +interface RunOutcome { + code: number + stdout: string + stderr: string +} + +// Spawns the real side script with a minimal, credential-free environment. +// tsx resolves from the repository the same way the package scripts resolve +// it. The trust gate runs before any host creation, so these checks never +// download VS Code and never touch a provider. +const runSideScript = async (args: string[], env: NodeJS.ProcessEnv): Promise => { + try { + const { stdout } = await execFileAsync( + process.execPath, + ["--import", "tsx", path.join(repoRoot, "scripts/agent-evals/coding-side.mts"), ...args], + { + cwd: repoRoot, + timeout: 120_000, + killSignal: "SIGKILL", + env, + }, + ) + return { code: 0, stdout, stderr: "" } + } catch (error) { + const failure = error as { code?: number | string; stdout?: string; stderr?: string; killed?: boolean } + if (failure.killed) throw new Error("side script hung and hit the test kill timer") + return { + code: typeof failure.code === "number" ? failure.code : 1, + stdout: failure.stdout ?? "", + stderr: failure.stderr ?? "", + } + } +} + +const minimalEnv = (extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv => ({ + PATH: process.env["PATH"] ?? "/usr/bin:/bin", + HOME: process.env["HOME"] ?? os.homedir(), + ...extra, +}) + +const writePolicy = async (dir: string, approvedCommits: string[]): Promise => { + const file = path.join(dir, "trust-policy.json") + await fs.writeFile(file, JSON.stringify({ version: 1, approvedCommits })) + return file +} + +// One empty fixture commit in a throwaway repository. It stands in for an +// evaluated root; no fixture code from it ever runs in these checks. +const makeFixtureCommit = async (dir: string): Promise<{ repo: string; commit: string }> => { + const repo = path.join(dir, "fixture-repo") + await fs.mkdir(repo) + await execFileAsync("git", ["init", "-q"], { cwd: repo }) + await execFileAsync( + "git", + [ + "-c", + "user.email=fixture@example.test", + "-c", + "user.name=fixture", + // The operator's git config may force signed commits, and the + // throwaway fixture has no key or tty. + "-c", + "commit.gpgsign=false", + "commit", + "--allow-empty", + "-q", + "-m", + "fixture", + ], + { cwd: repo }, + ) + const { stdout } = await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: repo }) + return { repo, commit: stdout.trim() } +} + +describe("coding side trust gate", () => { + it("rejects an unapproved evaluated root before any host launch", { timeout: 180_000 }, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) + try { + const { repo, commit } = await makeFixtureCommit(dir) + // The policy approves a different commit, so the fixture root is + // untrusted. + const trustFile = await writePolicy(dir, ["b".repeat(40)]) + const outcome = await runSideScript( + ["--reps", "1", "--out", path.join(dir, "out"), "--trust-file", trustFile, "--evaluated-root", repo], + minimalEnv({ SOLHEIM_API_KEY: "sk-test" }), + ) + assert.notEqual(outcome.code, 0) + assert.match(outcome.stderr, /commits not approved by the trust policy/) + assert.ok(outcome.stderr.includes(commit.slice(0, 8))) + // The rejection happens before the run loop: nothing completed and + // no results file exists. + assert.ok(!outcome.stdout.includes("coding-evals side complete")) + await assert.rejects(fs.access(path.join(dir, "out", "side-results.json"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("rejects an unrelated secret variable before any process starts", { timeout: 180_000 }, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) + try { + const { repo, commit } = await makeFixtureCommit(dir) + // The root is approved; the credential variable name is not. + const trustFile = await writePolicy(dir, [commit]) + const outcome = await runSideScript( + [ + "--reps", + "1", + "--out", + path.join(dir, "out"), + "--trust-file", + trustFile, + "--evaluated-root", + repo, + "--credential-env", + "AWS_SECRET_ACCESS_KEY", + ], + minimalEnv({ AWS_SECRET_ACCESS_KEY: "leak-me" }), + ) + assert.notEqual(outcome.code, 0) + assert.match(outcome.stderr, /credential environment variable must be one of/) + assert.ok(!outcome.stdout.includes("coding-evals side complete")) + await assert.rejects(fs.access(path.join(dir, "out", "side-results.json"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it( + "accepts an approved root and stops at the credential check without launching a host", + { timeout: 180_000 }, + async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) + try { + const { repo, commit } = await makeFixtureCommit(dir) + const trustFile = await writePolicy(dir, [commit]) + // No credential variable is set. The gate must pass and the run + // must fail closed at the credential check, before any host or + // container work. + const outcome = await runSideScript( + [ + "--reps", + "1", + "--out", + path.join(dir, "out"), + "--trust-file", + trustFile, + "--evaluated-root", + repo, + ], + minimalEnv(), + ) + assert.notEqual(outcome.code, 0) + assert.match( + outcome.stderr, + /provider credential environment variable SOLHEIM_API_KEY is unset or empty/, + ) + assert.ok(!outcome.stdout.includes("coding-evals side complete")) + await assert.rejects(fs.access(path.join(dir, "out", "side-results.json"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }, + ) +}) diff --git a/scripts/agent-evals/coding-trust.test.ts b/scripts/agent-evals/coding-trust.test.ts index db7a96b5ae..c30f55ab66 100644 --- a/scripts/agent-evals/coding-trust.test.ts +++ b/scripts/agent-evals/coding-trust.test.ts @@ -5,7 +5,9 @@ import path from "node:path" import { describe, it } from "node:test" import { assertCommitsApproved, + assertCredentialEnvAllowed, buildSideEnvironment, + buildWorktreeEnvironment, loadTrustPolicy, UntrustedCommitError, type TrustPolicy, @@ -58,12 +60,18 @@ describe("trust policy", () => { // The gate is case insensitive on the approved side. assertCommitsApproved(policy, [sha("A").toUpperCase()]) assert.throws(() => assertCommitsApproved(policy, [sha("b"), sha("c")]), UntrustedCommitError) - try { - assertCommitsApproved(policy, [sha("b")]) - } catch (error) { - assert.ok(error instanceof UntrustedCommitError) - assert.match(error.message, /not approved by the trust policy/) - } + // The single-commit rejection carries the exact rejected SHA. A silent + // pass here would approve an untrusted commit, so the assertion names + // the intended error instead of catching anything. + assert.throws( + () => assertCommitsApproved(policy, [sha("b")]), + (error: unknown) => { + assert.ok(error instanceof UntrustedCommitError) + assert.match(error.message, /not approved by the trust policy/) + assert.deepEqual(error.commits, [sha("b")]) + return true + }, + ) }) }) @@ -96,3 +104,43 @@ describe("side environment allowlist", () => { assert.equal(child["CODING_EVALS_SIDE"], "baseline") }) }) + +describe("credential environment allowlist", () => { + it("accepts the supported provider variable", () => { + assert.doesNotThrow(() => assertCredentialEnvAllowed("SOLHEIM_API_KEY")) + }) + + it("rejects unrelated secret variables before any process starts", () => { + for (const name of ["AWS_SECRET_ACCESS_KEY", "OPENAI_API_KEY", "GITHUB_TOKEN", "PATH"]) { + assert.throws(() => assertCredentialEnvAllowed(name), /credential environment variable must be one of/) + } + }) +}) + +describe("worktree process environment", () => { + it("passes the install and build allowlist and drops secrets", () => { + const child = buildWorktreeEnvironment({ + PATH: "/usr/bin", + HOME: "/home/dev", + PNPM_HOME: "/home/dev/.local/share/pnpm", + XDG_DATA_HOME: "/home/dev/.local/share", + npm_config_registry: "https://registry.example", + SOLHEIM_API_KEY: "sk-test", + AWS_SECRET_ACCESS_KEY: "leak-me", + OTHER_SECRET: "leak-me-too", + NODE_OPTIONS: "--require /tmp/never", + }) + assert.deepEqual(child, { + PATH: "/usr/bin", + HOME: "/home/dev", + PNPM_HOME: "/home/dev/.local/share/pnpm", + XDG_DATA_HOME: "/home/dev/.local/share", + npm_config_registry: "https://registry.example", + }) + }) + + it("never carries the provider credential into install or build", () => { + const child = buildWorktreeEnvironment({ PATH: "/usr/bin", SOLHEIM_API_KEY: "sk-test" }) + assert.equal(child["SOLHEIM_API_KEY"], undefined) + }) +}) diff --git a/scripts/agent-evals/coding-trust.ts b/scripts/agent-evals/coding-trust.ts index d1280943a3..6ff10a36e6 100644 --- a/scripts/agent-evals/coding-trust.ts +++ b/scripts/agent-evals/coding-trust.ts @@ -63,11 +63,23 @@ export function assertCommitsApproved(policy: TrustPolicy, commits: readonly str if (unapproved.length > 0) throw new UntrustedCommitError(unapproved) } -// Environment allowlist for the side process. The comparison driver may hold -// unrelated secrets in its own environment; the side gets only what it needs -// to spawn git, node, and the extension host, plus the one named credential -// variable. -const SIDE_ENV_ALLOWLIST = [ +// Provider credential variables this framework supports. The side script and +// the comparison driver accept only these names, so the tooling can never be +// pointed at an unrelated parent secret by naming an arbitrary variable. +export const CREDENTIAL_ENV_ALLOWLIST = ["SOLHEIM_API_KEY"] as const + +export function assertCredentialEnvAllowed(name: string): void { + if (!(CREDENTIAL_ENV_ALLOWLIST as readonly string[]).includes(name)) { + throw new Error(`credential environment variable must be one of: ${CREDENTIAL_ENV_ALLOWLIST.join(", ")}`) + } +} + +// Environment allowlist shared by every child this framework spawns. The +// comparison driver may hold unrelated secrets in its own environment; the +// children get only what they need to spawn git, node, and the extension +// host. Credentials never travel through this list; the side adds the one +// named credential variable separately. +const PROCESS_ENV_ALLOWLIST = [ "PATH", "HOME", "LANG", @@ -83,13 +95,15 @@ const SIDE_ENV_ALLOWLIST = [ "PATHEXT", ] as const +// Environment for the side process: the shared allowlist plus the one named +// credential variable and the side label. export function buildSideEnvironment( env: NodeJS.ProcessEnv, credentialEnv: string, side: "baseline" | "candidate", ): NodeJS.ProcessEnv { const child: NodeJS.ProcessEnv = {} - for (const name of SIDE_ENV_ALLOWLIST) { + for (const name of PROCESS_ENV_ALLOWLIST) { const value = env[name] if (value !== undefined) child[name] = value } @@ -98,3 +112,27 @@ export function buildSideEnvironment( child["CODING_EVALS_SIDE"] = side return child } + +// Environment for the worktree install and build processes. These children +// inherit neither the driver's secrets nor the provider credential. They get +// the shared process locations plus the explicitly approved pnpm variables a +// frozen-lockfile install and a build need to locate their store, cache, and +// registry. +const WORKTREE_ENV_ALLOWLIST = [ + ...PROCESS_ENV_ALLOWLIST, + "PNPM_HOME", + "XDG_DATA_HOME", + "XDG_CACHE_HOME", + "XDG_CONFIG_HOME", + "XDG_STATE_HOME", + "npm_config_registry", +] as const + +export function buildWorktreeEnvironment(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + const child: NodeJS.ProcessEnv = {} + for (const name of WORKTREE_ENV_ALLOWLIST) { + const value = env[name] + if (value !== undefined) child[name] = value + } + return child +} diff --git a/scripts/agent-evals/container-runtime.test.ts b/scripts/agent-evals/container-runtime.test.ts index 931b3dadda..891da30572 100644 --- a/scripts/agent-evals/container-runtime.test.ts +++ b/scripts/agent-evals/container-runtime.test.ts @@ -6,6 +6,8 @@ import { describe, it } from "node:test" import { buildContainerArgs, createContainerGradingRuntime, + EXEC_OUTPUT_MAX_BYTES, + exec, GRADER_IMAGE, parseDriverResult, type ContainerCli, @@ -318,3 +320,29 @@ describe("untrusted driver result validation", () => { assert.equal(parseDriverResult(`{"kind":"completed","pad":"${"x".repeat(70_000)}"}`), undefined) }) }) + +describe("bounded cli output capture", () => { + it("caps oversized stdout and stderr at the shared bound and keeps the head", { timeout: 30_000 }, async () => { + const outcome = await exec( + process.execPath, + ["-e", "process.stdout.write('a'.repeat(1000000)); process.stderr.write('b'.repeat(1000000))"], + 30_000, + ) + assert.equal(outcome.code, 0) + assert.equal(outcome.stdout.length, EXEC_OUTPUT_MAX_BYTES) + assert.equal(outcome.stderr.length, EXEC_OUTPUT_MAX_BYTES) + assert.ok(outcome.stdout.startsWith("a")) + assert.ok(outcome.stderr.startsWith("b")) + }) + + it("keeps short diagnostic output intact", { timeout: 30_000 }, async () => { + const outcome = await exec( + process.execPath, + ["-e", "process.stdout.write('sha256:digest'); process.stderr.write('no such container: gone')"], + 30_000, + ) + assert.equal(outcome.code, 0) + assert.equal(outcome.stdout, "sha256:digest") + assert.equal(outcome.stderr, "no such container: gone") + }) +}) diff --git a/scripts/agent-evals/container-runtime.ts b/scripts/agent-evals/container-runtime.ts index 98da8ea0c6..23f981e2ec 100644 --- a/scripts/agent-evals/container-runtime.ts +++ b/scripts/agent-evals/container-runtime.ts @@ -64,7 +64,16 @@ const probeExecutable = (candidate: ContainerExecutable): Promise => } }) -const exec = ( +// Shared bound for one CLI call's captured stdout and stderr. Digest, pull, +// and removal diagnostics are far smaller than this cap; the cap only bounds +// a pathological output stream, so captured output can never grow unbounded. +export const EXEC_OUTPUT_MAX_BYTES = 262_144 + +const appendBounded = (current: string, chunk: string): string => + current.length >= EXEC_OUTPUT_MAX_BYTES ? current : current + chunk.slice(0, EXEC_OUTPUT_MAX_BYTES - current.length) + +// Exported as a test seam. Production callers go through ContainerCli. +export const exec = ( file: string, args: readonly string[], timeoutMs: number, @@ -79,10 +88,10 @@ const exec = ( let stdout = "" let stderr = "" child.stdout?.on("data", (chunk) => { - stdout += String(chunk) + stdout = appendBounded(stdout, String(chunk)) }) child.stderr?.on("data", (chunk) => { - stderr += String(chunk) + stderr = appendBounded(stderr, String(chunk)) }) child.once("error", () => resolve({ code: "error", stdout, stderr })) child.once("close", (code, signal) => diff --git a/scripts/agent-evals/fs-atomic.test.ts b/scripts/agent-evals/fs-atomic.test.ts index c34d86f91b..c9e8415541 100644 --- a/scripts/agent-evals/fs-atomic.test.ts +++ b/scripts/agent-evals/fs-atomic.test.ts @@ -49,4 +49,26 @@ describe("atomic file publication", () => { await fs.rm(dir, { recursive: true, force: true }) } }) + + it("preserves the previous file when the rename fails after a successful temp write", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "fs-atomic-test-")) + try { + const previous = path.join(dir, "report.json") + await writeFileAtomic(previous, "previous") + // The target path is a non-empty directory. The temp file is + // written successfully into the target's own directory, and only + // the rename fails. + const blocked = path.join(dir, "blocked-target") + await fs.mkdir(blocked) + await fs.writeFile(path.join(blocked, "occupied"), "x") + await assert.rejects(writeFileAtomic(blocked, "next")) + // The previous publication stays intact and no temp residue or + // partial content remains. + assert.equal(await fs.readFile(previous, "utf8"), "previous") + assert.deepEqual((await fs.readdir(dir)).sort(), ["blocked-target", "report.json"]) + assert.deepEqual(await fs.readdir(blocked), ["occupied"]) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) }) diff --git a/scripts/agent-evals/runner.test.ts b/scripts/agent-evals/runner.test.ts index d65e56a144..0b8422bf05 100644 --- a/scripts/agent-evals/runner.test.ts +++ b/scripts/agent-evals/runner.test.ts @@ -845,9 +845,10 @@ describe("agent fixture controller", () => { }) assert.equal(trace.accounting.completeness.status, "complete") assert.deepEqual(trace.accounting.completeness.reasons, []) - // The drain is bound to the accepted-completion event. Without it, no - // drain window runs. - assert.equal(trace.execution.timings.usageDrainMs, undefined) + // The stop after the final completion message gets the same bounded + // usage drain a confirmed completion gets. The fixture drain window + // is 5_000 ms. + assert.equal(trace.execution.timings.usageDrainMs, 5_000) const serialized = JSON.stringify(trace) assert.ok(serialized.includes("pr1909-smoke-topology-synthetic")) assert.ok(!serialized.includes(CREDENTIAL)) @@ -1050,4 +1051,91 @@ describe("agent fixture controller", () => { assert.equal(result.execution.sessionLost, false) assert.equal(result.teardown.status, "ok") }) + + it("drains bounded usage after an observer stop on the final completion message", async () => { + registerTestFixture("controller-stop-drain", { + limits: { + activationTimeoutMs: 120_000, + taskStartTimeoutMs: 35_000, + taskRunTimeoutMs: 120_000, + usageDrainMs: 500, + teardownStepMs: 5_000, + }, + }) + let lateSent = false + const f = harness("no-events", { + onPause: (count) => { + if (count === 1) + f.emit({ + eventName: RooCodeEventName.Message, + payload: [ + { + taskId: "root", + action: "created", + message: { + ts: 1, + type: "say", + say: "completion_result", + text: "synthetic final answer", + }, + }, + ], + }) + // A usage snapshot lands inside the drain window that follows + // the stop. The bounded drain must capture it. + if (count === 2 && !lateSent) { + lateSent = true + f.emit({ + eventName: RooCodeEventName.TaskTokenUsageUpdated, + payload: [ + "root", + { totalTokensIn: 40, totalTokensOut: 20, totalCost: 0, contextTokens: 30 }, + {}, + ], + }) + } + }, + }) + let completionOutput = false + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-stop-drain", + credential: CREDENTIAL, + observers: { + onAcceptedEvent: (event) => { + if ( + event.eventName === RooCodeEventName.Message && + event.payload[0].message.say === "completion_result" + ) + completionOutput = true + }, + shouldStop: () => completionOutput, + }, + }) + assert.equal(result.execution.code, "STOPPED") + assert.equal(result.execution.stopReason, "adapterStopAfterCompletionMessage") + const trace = writtenTrace(f) + // The stop path drains for the fixture's bounded window and captures + // the late usage snapshot. + assert.equal(trace.execution.timings.usageDrainMs, 500) + assert.equal(trace.accounting.usage.snapshotsObserved, 1) + assert.equal(trace.accounting.usage.lastSnapshot?.source, "taskTokenUsageUpdated") + assert.equal(trace.accounting.usage.lastSnapshot?.tokensIn, 40) + assert.equal(trace.accounting.completeness.status, "complete") + assert.deepEqual(trace.accounting.completeness.reasons, []) + }) + + it("skips the usage drain for a plain observer stop without the completion message", async () => { + registerTestFixture("controller-plain-stop") + const f = harness("success") + const result = await runAgentFixture(f.runtime, { + fixtureName: "controller-plain-stop", + credential: CREDENTIAL, + observers: { shouldStop: () => true }, + }) + assert.equal(result.execution.code, "STOPPED") + assert.equal(result.execution.stopReason, "adapterStop") + assert.equal(result.execution.completionMessageSeen, false) + // Without the final completion message, no drain window runs. + assert.equal(writtenTrace(f).execution.timings.usageDrainMs, undefined) + }) }) diff --git a/scripts/agent-evals/runner.ts b/scripts/agent-evals/runner.ts index 80ff42a87a..cdf5bd7a29 100644 --- a/scripts/agent-evals/runner.ts +++ b/scripts/agent-evals/runner.ts @@ -302,7 +302,12 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp ) runSessionLost = sessionLost timings.taskRunMs = runtime.now() - runStart - if (completedSeen && !runSessionLost) { + const stopped = !runSessionLost && !completedSeen && options.observers?.shouldStop?.() === true + // The model's final completion message already produced + // reportable usage, so a stop issued after it gets the + // same bounded drain a confirmed completion gets. A + // plain stop, a timeout, and a session loss get none. + if ((completedSeen || (stopped && completionMessageSeen)) && !runSessionLost) { // Fixed bound; ends early only on session loss. const drainStart = runtime.now() await until(() => false, fixture.limits.usageDrainMs) @@ -312,7 +317,6 @@ export async function runAgentFixture(runtime: AgentRuntime, options: AgentRunOp // accounting must carry the caveat. lostDuringDrain = sessionLost } - const stopped = !runSessionLost && !completedSeen && options.observers?.shouldStop?.() === true if (completedSeen) { // A confirmed accepted completion outranks a stop // request or a loss observed in the same window. diff --git a/scripts/agent-evals/side-process-guard.test.ts b/scripts/agent-evals/side-process-guard.test.ts new file mode 100644 index 0000000000..bd5b8106e7 --- /dev/null +++ b/scripts/agent-evals/side-process-guard.test.ts @@ -0,0 +1,266 @@ +import assert from "node:assert/strict" +import { spawn } from "node:child_process" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import { runBoundedPreparation } from "./coding-compare.ts" +import { createSideProcessGuard, type SideKillOutcome } from "./side-process-guard.ts" + +// Credential-free process fixtures. Each child is a bare node process in its +// own detached group, the same topology the comparison driver creates. +const spawnDetachedChild = (script: string) => { + const child = spawn(process.execPath, ["-e", script], { stdio: ["ignore", "pipe", "ignore"], detached: true }) + const spawned = new Promise((resolve, reject) => { + child.once("spawn", resolve) + child.once("error", reject) + }) + const exited = new Promise((resolve) => child.once("exit", () => resolve())) + // Resolves when the child reports readiness, so a signal cannot land + // before the child finished its own setup. + const ready = new Promise((resolve, reject) => { + child.stdout?.once("data", () => resolve()) + child.once("error", reject) + child.once("exit", () => reject(new Error("child exited before reporting readiness"))) + }) + // Tests that never await readiness still keep the rejection handled. + ready.catch(() => {}) + return { child, spawned, ready, exited } +} + +describe("side process guard", () => { + it("resolves without work when nothing is tracked", async () => { + const guard = createSideProcessGuard(1_000) + assert.equal(await guard.interrupt(), "none") + // Repeated interrupts are no-ops, never a double kill. + assert.equal(await guard.interrupt(), "none") + }) + + it("terminates a detached child group on SIGTERM", { timeout: 30_000 }, async () => { + const guard = createSideProcessGuard(10_000) + const { child, spawned, exited } = spawnDetachedChild("setInterval(() => {}, 1000000)") + try { + await spawned + guard.track(child.pid) + assert.equal(await guard.interrupt(), "terminated") + // The detached child must actually be gone, not orphaned. + await Promise.race([ + exited, + new Promise((_, reject) => setTimeout(() => reject(new Error("child survived the interrupt")), 20_000)), + ]) + } finally { + child.kill("SIGKILL") + } + }) + + it("escalates to SIGKILL when the child ignores SIGTERM", { timeout: 30_000 }, async () => { + const guard = createSideProcessGuard(300) + const { child, ready, exited } = spawnDetachedChild( + "process.on('SIGTERM', () => {}); process.stdout.write('ready'); setInterval(() => {}, 1000000)", + ) + try { + // The signal must not land before the child installed its ignore + // handler. + await ready + guard.track(child.pid) + assert.equal(await guard.interrupt(), "killed") + await Promise.race([ + exited, + new Promise((_, reject) => setTimeout(() => reject(new Error("child survived SIGKILL")), 20_000)), + ]) + } finally { + child.kill("SIGKILL") + } + }) + + it("runs the escalation once across repeated calls", { timeout: 30_000 }, async () => { + const guard = createSideProcessGuard(10_000) + const { child, spawned, exited } = spawnDetachedChild("setInterval(() => {}, 1000000)") + try { + await spawned + guard.track(child.pid) + const first = guard.interrupt() + // A second signal must not start a second kill sequence. + assert.equal(await guard.interrupt(), "none") + assert.equal(await first, "terminated") + await exited + } finally { + child.kill("SIGKILL") + } + }) + + it("keeps the escalation running when the tracked child exits mid-wait", { timeout: 30_000 }, async () => { + // The runner untracks the direct child as soon as it settles. The + // group id is snapshotted at interrupt time, so that untrack cannot + // cut the wait short while a group member still writes. + const guard = createSideProcessGuard(500) + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "guard-escalation-test-")) + const grandchildHeartbeat = path.join(dir, "grandchild-heartbeat") + const grandchildPidPath = path.join(dir, "grandchild-pid") + const grandchildReadyPath = path.join(dir, "grandchild-ready") + const grandchildScript = `const fs = require('node:fs'); process.on('SIGTERM', () => {}); fs.writeFileSync(${JSON.stringify(grandchildReadyPath)}, 'ready'); setInterval(() => fs.appendFileSync(${JSON.stringify(grandchildHeartbeat)}, 'x'), 50)` + const fixture = spawnDetachedChild( + `const fs = require('node:fs'); + const { spawn } = require('node:child_process'); + const grandchild = spawn(process.execPath, ['-e', ${JSON.stringify(grandchildScript)}], { stdio: 'ignore' }); + fs.writeFileSync(${JSON.stringify(grandchildPidPath)}, String(grandchild.pid)); + process.stdout.write('ready'); + setTimeout(() => process.exit(0), 100);`, + ) + try { + // The signal must not land before the grandchild installed its + // ignore handler, so both fixture processes report readiness. + await Promise.all([fixture.ready, waitForFile(grandchildReadyPath)]) + const childPid = fixture.child.pid + assert.ok(childPid !== undefined) + guard.track(childPid) + const interrupt = guard.interrupt() + // The direct child settles and the runner untracks it while the + // grandchild, which ignores SIGTERM, still lives. + await fixture.exited + guard.track(undefined) + assert.equal(await interrupt, "killed") + // The SIGKILL escalation reached the surviving group member: its + // heartbeat went silent. + await assertHeartbeatStopped(grandchildHeartbeat) + } finally { + fixture.child.kill("SIGKILL") + const grandchildPid = await fs.readFile(grandchildPidPath, "utf8").then(Number, () => undefined) + if (grandchildPid !== undefined) { + try { + process.kill(grandchildPid, "SIGKILL") + } catch { + /* Already gone. */ + } + } + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) + } + }) +}) + +const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)) + +// Confirms the fixture writer stopped appending, so no child is still +// writing when the driver's cleanup would run. +const assertHeartbeatStopped = async (filePath: string): Promise => { + const sizeBefore = await fs.stat(filePath).then( + (entry) => entry.size, + () => -1, + ) + await delay(400) + const sizeAfter = await fs.stat(filePath).then( + (entry) => entry.size, + () => -1, + ) + assert.equal(sizeAfter, sizeBefore) +} + +// Polls until the file exists, so a signal cannot land before the fixture +// children finished their own setup. +const waitForFile = async (filePath: string): Promise => { + const deadline = Date.now() + 10_000 + while (Date.now() < deadline) { + if ( + await fs.access(filePath).then( + () => true, + () => false, + ) + ) + return + await new Promise((resolve) => setTimeout(resolve, 10)) + } + throw new Error(`fixture file never appeared: ${filePath}`) +} + +describe("bounded preparation children", () => { + // Regression: a SIGTERM aimed at the driver process alone used to leave + // install and build children running, because only the coding side was + // tracked. The test replays the driver wiring credential-free: one guard, + // one handler that arms one escalation, and a cleanup that waits for it. + it("stops a tracked preparation group when only the parent gets SIGTERM", { timeout: 30_000 }, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "bounded-prep-test-")) + const guard = createSideProcessGuard(2_000) + let interruptSignal: NodeJS.Signals | undefined + let interruptKill: Promise | undefined + const onInterruptSignal = (signal: NodeJS.Signals): void => { + if (interruptSignal !== undefined) return + interruptSignal = signal + interruptKill = guard.interrupt() + } + process.on("SIGTERM", onInterruptSignal) + // The preparation child leads its own group and spawns a grandchild + // into it, the same topology a real pnpm install creates. Both keep + // writing a heartbeat file, so a surviving member cannot hide. + const readyPath = path.join(dir, "ready") + const childPidPath = path.join(dir, "child-pid") + const childHeartbeat = path.join(dir, "child-heartbeat") + const grandchildHeartbeat = path.join(dir, "grandchild-heartbeat") + const grandchildScript = `const fs = require('node:fs'); setInterval(() => fs.appendFileSync(${JSON.stringify(grandchildHeartbeat)}, 'x'), 50)` + const childScript = [ + "const fs = require('node:fs')", + "const { spawn } = require('node:child_process')", + `fs.writeFileSync(${JSON.stringify(childPidPath)}, String(process.pid))`, + `const grandchild = spawn(process.execPath, ['-e', ${JSON.stringify(grandchildScript)}], { stdio: 'ignore' })`, + `fs.writeFileSync(${JSON.stringify(readyPath)}, 'ready')`, + `setInterval(() => fs.appendFileSync(${JSON.stringify(childHeartbeat)}, 'x'), 50)`, + ].join("; ") + const bounded = runBoundedPreparation( + guard, + () => interruptSignal !== undefined, + process.execPath, + ["-e", childScript], + dir, + 60_000, + ) + try { + await waitForFile(readyPath) + // The signal targets this test process alone. The child group + // receives nothing from this kill call. + process.kill(process.pid, "SIGTERM") + await assert.rejects(bounded) + // The escalation finishes before any cleanup runs, and the whole + // group, grandchild included, is gone by then. + await interruptKill + assert.equal(interruptSignal, "SIGTERM") + // The whole group went silent before the cleanup ran, so nothing + // survives to write into the removed tree. + await assertHeartbeatStopped(childHeartbeat) + await assertHeartbeatStopped(grandchildHeartbeat) + await fs.rm(dir, { recursive: true, force: true }) + } finally { + process.removeListener("SIGTERM", onInterruptSignal) + const childPid = await fs.readFile(childPidPath, "utf8").then(Number, () => undefined) + if (childPid !== undefined) { + try { + process.kill(-childPid, "SIGKILL") + } catch { + /* The group is already gone. */ + } + } + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) + } + }) + + it("refuses to start a preparation child after the interrupt", { timeout: 30_000 }, async () => { + const guard = createSideProcessGuard(100) + // A signal that landed between children tracked nothing; the next + // spawn must refuse instead of escaping the stop. + assert.equal(await guard.interrupt(), "none") + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "bounded-prep-test-")) + try { + await assert.rejects( + runBoundedPreparation( + guard, + () => true, + process.execPath, + ["-e", "setInterval(() => {}, 1000000)"], + dir, + 1_000, + ), + /interrupted before/, + ) + } finally { + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) + } + }) +}) diff --git a/scripts/agent-evals/side-process-guard.ts b/scripts/agent-evals/side-process-guard.ts new file mode 100644 index 0000000000..8745273b8e --- /dev/null +++ b/scripts/agent-evals/side-process-guard.ts @@ -0,0 +1,70 @@ +// Interrupt handling for the comparison driver's detached children. Each +// bounded child, the preparation steps and the coding side alike, runs in its +// own process group, so a terminal signal aimed at the driver alone never +// reaches it. When the driver receives SIGINT or SIGTERM, this guard +// terminates the tracked group: SIGTERM first, a bounded grace, then SIGKILL +// for anything that remained. Repeated calls are a no-op, so several signals +// can share one guard without double cleanup. +export type SideKillOutcome = "terminated" | "killed" | "none" + +export interface SideProcessGuard { + // Tracks the active detached child process id. undefined clears the + // tracking, for example after the child exited on its own. + track(pid: number | undefined): void + // Terminates the tracked process group. Resolves after the group is gone + // ("terminated"), after the SIGKILL escalation fired ("killed"), or at + // once when nothing is tracked or a kill already ran ("none"). + interrupt(): Promise +} + +const groupAlive = (pid: number): boolean => { + try { + process.kill(-pid, 0) + return true + } catch { + return false + } +} + +const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)) + +export const createSideProcessGuard = (graceMs: number): SideProcessGuard => { + let pid: number | undefined + let interrupted = false + const signalGroup = (group: number, signal: NodeJS.Signals): void => { + try { + process.kill(-group, signal) + } catch { + // The group may already be gone; fall back to the direct child. + try { + process.kill(group, signal) + } catch { + /* The child already exited. */ + } + } + } + return { + track: (childPid) => { + pid = childPid + }, + interrupt: async (): Promise => { + // The group id is snapshotted, so an untrack that races the + // escalation, for example the tracked child's own exit handler, + // cannot cut the wait short while descendants still run. + const group = pid + if (interrupted || group === undefined) { + interrupted = true + return "none" + } + interrupted = true + signalGroup(group, "SIGTERM") + const deadline = Date.now() + graceMs + while (Date.now() < deadline) { + if (!groupAlive(group)) return "terminated" + await delay(25) + } + signalGroup(group, "SIGKILL") + return "killed" + }, + } +} diff --git a/scripts/solheim-smoke/run.test.ts b/scripts/solheim-smoke/run.test.ts index 9be74b3d75..61d399b69e 100644 --- a/scripts/solheim-smoke/run.test.ts +++ b/scripts/solheim-smoke/run.test.ts @@ -210,7 +210,9 @@ describe("provider smoke orchestration", () => { const f = fixture("success", { stuck: true }) const verdict = await runSmoke(f.runtime, "private-test-key") assert.equal(verdict.code, "TEARDOWN_FAILED") - assert.equal(verdict.durationMs, 15_000) + // The success scenario stops after the final completion message, so + // the bounded usage drain runs: 5_000 drain + 3 x 5_000 teardown. + assert.equal(verdict.durationMs, 20_000) assert.deepEqual(f.signals, ["SIGTERM", "SIGKILL"]) assert.equal(verdict.gates.childExited, false) assert.equal(f.cleaned, false) From e2e9f1fe558da79a5b6ab7d79d34e8d308d12c89 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 24 Oct 2026 03:49:49 +0000 Subject: [PATCH 4/9] fix(evals): pass test fixture paths as arguments instead of embedding code --- .../agent-evals/side-process-guard.test.ts | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/scripts/agent-evals/side-process-guard.test.ts b/scripts/agent-evals/side-process-guard.test.ts index bd5b8106e7..adba5a2b64 100644 --- a/scripts/agent-evals/side-process-guard.test.ts +++ b/scripts/agent-evals/side-process-guard.test.ts @@ -9,8 +9,11 @@ import { createSideProcessGuard, type SideKillOutcome } from "./side-process-gua // Credential-free process fixtures. Each child is a bare node process in its // own detached group, the same topology the comparison driver creates. -const spawnDetachedChild = (script: string) => { - const child = spawn(process.execPath, ["-e", script], { stdio: ["ignore", "pipe", "ignore"], detached: true }) +const spawnDetachedChild = (script: string, args: readonly string[] = []) => { + const child = spawn(process.execPath, ["-e", script, ...args], { + stdio: ["ignore", "pipe", "ignore"], + detached: true, + }) const spawned = new Promise((resolve, reject) => { child.once("spawn", resolve) child.once("error", reject) @@ -98,14 +101,19 @@ describe("side process guard", () => { const grandchildHeartbeat = path.join(dir, "grandchild-heartbeat") const grandchildPidPath = path.join(dir, "grandchild-pid") const grandchildReadyPath = path.join(dir, "grandchild-ready") - const grandchildScript = `const fs = require('node:fs'); process.on('SIGTERM', () => {}); fs.writeFileSync(${JSON.stringify(grandchildReadyPath)}, 'ready'); setInterval(() => fs.appendFileSync(${JSON.stringify(grandchildHeartbeat)}, 'x'), 50)` + // Fixture sources stay static. Host paths and the grandchild source + // travel as argv, so no host value is embedded in generated code. + const grandchildScript = + "const fs = require('node:fs'); process.on('SIGTERM', () => {}); const [heartbeatPath, readyPath] = process.argv.slice(1); fs.writeFileSync(readyPath, 'ready'); setInterval(() => fs.appendFileSync(heartbeatPath, 'x'), 50)" const fixture = spawnDetachedChild( `const fs = require('node:fs'); const { spawn } = require('node:child_process'); - const grandchild = spawn(process.execPath, ['-e', ${JSON.stringify(grandchildScript)}], { stdio: 'ignore' }); - fs.writeFileSync(${JSON.stringify(grandchildPidPath)}, String(grandchild.pid)); + const [grandchildPidPath, grandchildReadyPath, grandchildHeartbeatPath, grandchildScript] = process.argv.slice(1); + const grandchild = spawn(process.execPath, ['-e', grandchildScript, grandchildHeartbeatPath, grandchildReadyPath], { stdio: 'ignore' }); + fs.writeFileSync(grandchildPidPath, String(grandchild.pid)); process.stdout.write('ready'); setTimeout(() => process.exit(0), 100);`, + [grandchildPidPath, grandchildReadyPath, grandchildHeartbeat, grandchildScript], ) try { // The signal must not land before the grandchild installed its @@ -195,20 +203,27 @@ describe("bounded preparation children", () => { const childPidPath = path.join(dir, "child-pid") const childHeartbeat = path.join(dir, "child-heartbeat") const grandchildHeartbeat = path.join(dir, "grandchild-heartbeat") - const grandchildScript = `const fs = require('node:fs'); setInterval(() => fs.appendFileSync(${JSON.stringify(grandchildHeartbeat)}, 'x'), 50)` + // Fixture sources stay static. Host paths and the grandchild source + // travel as argv, so no host value is embedded in generated code. + const grandchildScript = + "const fs = require('node:fs'); const [heartbeatPath] = process.argv.slice(1); setInterval(() => fs.appendFileSync(heartbeatPath, 'x'), 50)" const childScript = [ "const fs = require('node:fs')", "const { spawn } = require('node:child_process')", - `fs.writeFileSync(${JSON.stringify(childPidPath)}, String(process.pid))`, - `const grandchild = spawn(process.execPath, ['-e', ${JSON.stringify(grandchildScript)}], { stdio: 'ignore' })`, - `fs.writeFileSync(${JSON.stringify(readyPath)}, 'ready')`, - `setInterval(() => fs.appendFileSync(${JSON.stringify(childHeartbeat)}, 'x'), 50)`, + "const [childPidPath, readyPath, childHeartbeatPath, grandchildHeartbeatPath, grandchildScript] = process.argv.slice(1)", + "fs.writeFileSync(childPidPath, String(process.pid))", + "const grandchild = spawn(process.execPath, ['-e', grandchildScript, grandchildHeartbeatPath], { stdio: 'ignore' })", + "fs.writeFileSync(readyPath, 'ready')", + "setInterval(() => fs.appendFileSync(childHeartbeatPath, 'x'), 50)", ].join("; ") + // Security regression guard: generated fixture code must not embed + // any host path. Every host value travels as argv instead. + assert.ok(!childScript.includes(dir)) const bounded = runBoundedPreparation( guard, () => interruptSignal !== undefined, process.execPath, - ["-e", childScript], + ["-e", childScript, childPidPath, readyPath, childHeartbeat, grandchildHeartbeat, grandchildScript], dir, 60_000, ) From 85c0bb446321a44957ba3fde2c9765a75f5bbdde Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 24 Oct 2026 04:20:08 +0000 Subject: [PATCH 5/9] fix(evals): enforce isolated grading and safe interruption and result writes --- scripts/agent-evals/README.md | 4 + .../agent-evals/coding-compare-script.test.ts | 85 +++++++++ scripts/agent-evals/coding-compare.mts | 9 +- scripts/agent-evals/coding-fixtures.test.ts | 2 +- scripts/agent-evals/coding-grader.ts | 2 +- .../coding-reference-solutions.test.ts | 2 +- scripts/agent-evals/coding-report.test.ts | 11 ++ scripts/agent-evals/coding-side-flow.test.ts | 178 ++++++++++++++++++ scripts/agent-evals/coding-side-flow.ts | 81 ++++++++ scripts/agent-evals/coding-side.mts | 161 ++++++++-------- scripts/agent-evals/coding-side.test.ts | 31 +++ scripts/agent-evals/container-runtime.test.ts | 63 ++++++- scripts/agent-evals/container-runtime.ts | 14 +- scripts/agent-evals/grader-driver.mjs | 22 ++- scripts/agent-evals/grader-driver.test.ts | 80 ++++++++ .../agent-evals/side-process-guard.test.ts | 61 +++++- scripts/agent-evals/side-process-guard.ts | 42 ++++- 17 files changed, 747 insertions(+), 101 deletions(-) create mode 100644 scripts/agent-evals/coding-compare-script.test.ts create mode 100644 scripts/agent-evals/coding-side-flow.test.ts create mode 100644 scripts/agent-evals/coding-side-flow.ts create mode 100644 scripts/agent-evals/grader-driver.test.ts diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index d73b8dcf52..3c1a32feb8 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -130,6 +130,8 @@ release process: a maintainer runs it locally or on demand. - The driver's result file is untrusted data: it is written inside the container where agent-written test code executes. The host reads it with a size bound and validates shape, bounds, and internal consistency. An invalid or inconsistent result is a `grading failure`, never a verdict. - Each fixture ships two reference solutions. The known-correct solution must pass. The deliberately incorrect one passes the visible tests and must fail. Unit tests enforce both on every fixture. - `createLocalSubprocessGradingRuntime` runs the same driver directly on the host. It does not give container isolation. It exists for unit tests and explicit local checks only and must never back a recorded comparison. +- The live `coding-evals:side` and `coding-evals:compare` entry points reject `--allow-unisolated-grading` instead of ignoring it. A recorded comparison always grades in the container runtime. +- The driver publishes its result file atomically: the content lands in a unique same-directory temp file and is renamed into place. A failed write or rename removes the temp file, and the host treats a missing or invalid result as a grading failure. ### Comparisons @@ -159,6 +161,8 @@ Artifacts are published atomically: content lands in a same-directory temp file A side run invalidates its output directory first. Before the first repetition executes, the side removes any previous `side-results.json` and the whole `artifacts` tree, so stale results from an earlier rerun can never survive. `side-results.json` is the completion marker: it appears only after every repetition of the current invocation finished, and consumers must require it and treat its absence as a failed side. A side rerun that cannot remove a stale artifact fails before any repetition runs. +The side process arms one flag when it receives SIGINT or SIGTERM. The flag starts no further repetition and blocks the completion marker, so an interrupted side exits with the conventional signal code and publishes nothing. + The comparison reports are published as one generation. `comparison-report.json` and `comparison-report.md` carry the report's `generatedAt` stamp, and `comparison-report.complete` holds that stamp; it is written last. Consumers must require the marker and match its stamp against the report files. `readPublishedComparisonReport` enforces this: it returns the report only when the marker exists and its stamp matches both files. The marker is removed before publication starts, so a crash between the two file publications leaves a detectable incomplete generation instead of a mixed one. A thrown failure rolls both files back; a failed comparison leaves no report files. ### Commands diff --git a/scripts/agent-evals/coding-compare-script.test.ts b/scripts/agent-evals/coding-compare-script.test.ts new file mode 100644 index 0000000000..05b68288d5 --- /dev/null +++ b/scripts/agent-evals/coding-compare-script.test.ts @@ -0,0 +1,85 @@ +import assert from "node:assert/strict" +import { execFile } from "node:child_process" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { fileURLToPath } from "node:url" +import { promisify } from "node:util" +import { describe, it } from "node:test" + +const execFileAsync = promisify(execFile) +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") + +interface RunOutcome { + code: number + stdout: string + stderr: string +} + +// Spawns the real comparison driver with a minimal, credential-free +// environment. The argument gate runs before the trust policy loads, so the +// rejection happens before any worktree, child, or report exists. +const runCompareScript = async (args: string[], env: NodeJS.ProcessEnv): Promise => { + try { + const { stdout } = await execFileAsync( + process.execPath, + ["--import", "tsx", path.join(repoRoot, "scripts/agent-evals/coding-compare.mts"), ...args], + { + cwd: repoRoot, + timeout: 120_000, + killSignal: "SIGKILL", + env, + }, + ) + return { code: 0, stdout, stderr: "" } + } catch (error) { + const failure = error as { code?: number | string; stdout?: string; stderr?: string; killed?: boolean } + if (failure.killed) throw new Error("compare script hung and hit the test kill timer") + return { + code: typeof failure.code === "number" ? failure.code : 1, + stdout: failure.stdout ?? "", + stderr: failure.stderr ?? "", + } + } +} + +const minimalEnv = (): NodeJS.ProcessEnv => ({ + PATH: process.env["PATH"] ?? "/usr/bin:/bin", + HOME: process.env["HOME"] ?? os.homedir(), +}) + +describe("coding compare isolation gate", () => { + it("rejects --allow-unisolated-grading before any worktree or report exists", { timeout: 180_000 }, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-compare-test-")) + try { + const trustFile = path.join(dir, "trust-policy.json") + await fs.writeFile(trustFile, JSON.stringify({ version: 1, approvedCommits: [] })) + const out = path.join(dir, "out") + const outcome = await runCompareScript( + [ + "--baseline", + "a".repeat(40), + "--candidate", + "b".repeat(40), + "--reps", + "1", + "--out", + out, + "--trust-file", + trustFile, + "--allow-unisolated-grading", + ], + minimalEnv(), + ) + assert.notEqual(outcome.code, 0) + assert.match(outcome.stderr, /--allow-unisolated-grading is not accepted here/) + // The rejection happens before any report generation exists, so + // the comparison can never publish host-graded artifacts. + for (const name of ["comparison-report.json", "comparison-report.md", "comparison-report.complete"]) { + await assert.rejects(fs.access(path.join(out, name))) + } + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/scripts/agent-evals/coding-compare.mts b/scripts/agent-evals/coding-compare.mts index cf3863f041..e201fbc431 100644 --- a/scripts/agent-evals/coding-compare.mts +++ b/scripts/agent-evals/coding-compare.mts @@ -73,11 +73,16 @@ const readArgs = (): { reps: number out: string credentialEnv: string - allowUnisolatedGrading: boolean trustFile: string sideTimeoutMs: number } => { const args = process.argv.slice(2) + // The local subprocess runtime has no container isolation: agent-written + // grading code would execute on the host filesystem. A comparison always + // grades in the container runtime, so the dangerous option is rejected + // loudly here and never forwarded to a side. + if (args.includes("--allow-unisolated-grading")) + throw new Error("--allow-unisolated-grading is not accepted here: a comparison grades in the container runtime") const read = (name: string): string | undefined => { const index = args.indexOf(`--${name}`) return index >= 0 ? args[index + 1] : undefined @@ -105,7 +110,6 @@ const readArgs = (): { reps, out: path.resolve(out), credentialEnv, - allowUnisolatedGrading: args.includes("--allow-unisolated-grading"), trustFile: path.resolve(trustFile), sideTimeoutMs: sideTimeoutMinutes * 60_000, } @@ -167,7 +171,6 @@ const runSide = async ( inputs.credentialEnv, "--evaluated-root", worktree, - ...(inputs.allowUnisolatedGrading ? ["--allow-unisolated-grading"] : []), ], { cwd: root, diff --git a/scripts/agent-evals/coding-fixtures.test.ts b/scripts/agent-evals/coding-fixtures.test.ts index 48aa0f7222..66f2daafa3 100644 --- a/scripts/agent-evals/coding-fixtures.test.ts +++ b/scripts/agent-evals/coding-fixtures.test.ts @@ -91,6 +91,6 @@ describe("coding fixture suite", () => { assert.throws(() => codingBundle("unknown-fixture")) assert.ok(codingSuiteDigest().length === 64) assert.equal(CODING_SUITE_ID, "coding-basic-v1") - assert.equal(GRADER_VERSION, "coding-grader-v2") + assert.equal(GRADER_VERSION, "coding-grader-v3") }) }) diff --git a/scripts/agent-evals/coding-grader.ts b/scripts/agent-evals/coding-grader.ts index 0f9800ee32..33039f4426 100644 --- a/scripts/agent-evals/coding-grader.ts +++ b/scripts/agent-evals/coding-grader.ts @@ -9,7 +9,7 @@ import type { Measured } from "./trace.ts" // The grader version is a pinned comparison input. Any change to the driver, // the image, or the verdict rules must bump it. -export const GRADER_VERSION = "coding-grader-v2" +export const GRADER_VERSION = "coding-grader-v3" export interface GraderResources { timeoutMs: number diff --git a/scripts/agent-evals/coding-reference-solutions.test.ts b/scripts/agent-evals/coding-reference-solutions.test.ts index 0c89c339ac..1593b4c245 100644 --- a/scripts/agent-evals/coding-reference-solutions.test.ts +++ b/scripts/agent-evals/coding-reference-solutions.test.ts @@ -66,7 +66,7 @@ describe("coding graders against reference solutions", () => { } it("pins one grader version across the suite", () => { - assert.equal(GRADER_VERSION, "coding-grader-v2") + assert.equal(GRADER_VERSION, "coding-grader-v3") assert.equal(CODING_GRADER_RESOURCES.image.startsWith("node:"), true) }) }) diff --git a/scripts/agent-evals/coding-report.test.ts b/scripts/agent-evals/coding-report.test.ts index 07a7617a72..7dee41eba0 100644 --- a/scripts/agent-evals/coding-report.test.ts +++ b/scripts/agent-evals/coding-report.test.ts @@ -214,6 +214,17 @@ describe("side result validation", () => { assert.equal(validated.pins.commit, "a".repeat(40)) }) + it("accepts a well-formed run with object metrics and keeps it", () => { + const run = { fixtureName: "clamp-range", repIndex: 0, metrics: metrics("clamp-range", 0, "correct", 0.1) } + const validated = validateSideResultFile( + validatedFile("baseline", "a".repeat(40), { runs: [run] }), + "baseline", + "a".repeat(40), + ) + assert.equal(validated.runs.length, 1) + assert.deepEqual(validated.runs[0], run) + }) + it("rejects a result that records the wrong side or a different commit", () => { assert.throws( () => validateSideResultFile(validatedFile("candidate", "a".repeat(40)), "baseline", "a".repeat(40)), diff --git a/scripts/agent-evals/coding-side-flow.test.ts b/scripts/agent-evals/coding-side-flow.test.ts new file mode 100644 index 0000000000..85a13c3f8f --- /dev/null +++ b/scripts/agent-evals/coding-side-flow.test.ts @@ -0,0 +1,178 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import { executeSideFlow, type SideRepetition } from "./coding-side-flow.ts" +import type { CodingRunRecord } from "./coding-run.ts" +import { buildRunMetrics } from "./coding-metrics.ts" +import type { AgentTrace } from "./trace.ts" +import type { SidePins } from "./coding-pins.ts" + +const plan = (count: number, fixtureName = "clamp-range"): SideRepetition[] => + Array.from({ length: count }, (_, repIndex) => ({ fixtureName, fixtureVersion: "1.0.0", repIndex })) + +const record = (repetition: SideRepetition, withTrace = false): CodingRunRecord => ({ + fixtureName: repetition.fixtureName, + fixtureVersion: repetition.fixtureVersion, + repIndex: repetition.repIndex, + outcome: { outcome: "correct", reason: "hiddenTestsPassed" }, + // The flow only checks trace presence before it writes the artifact, so + // an empty object stands in for the metadata-only trace here. + trace: withTrace ? ({} as AgentTrace) : undefined, + metrics: buildRunMetrics({ + fixtureName: repetition.fixtureName, + repIndex: repetition.repIndex, + outcome: { outcome: "correct", reason: "hiddenTestsPassed" }, + modelIdentifier: { available: true, value: "test-model" }, + }), +}) + +const pins = (): SidePins => ({ + side: "candidate", + commit: "a".repeat(40), + suiteId: "coding-basic-v1", + suiteDigest: "d1".repeat(32), + settingsDigest: "d2".repeat(32), + modelIdentifier: "test-model", + graderVersion: "coding-grader-v2", + graderImageDigest: "unavailable", + runnerEnvironment: { os: "linux", nodeVersion: process.version, vscodeTarget: "test" }, +}) + +const flow = (inputs: { + repetitions: readonly SideRepetition[] + out: string + isInterrupted: () => boolean + runRepetition: (repetition: SideRepetition) => Promise +}) => + executeSideFlow({ + ...inputs, + buildResult: (runs) => ({ pins: pins(), runs }), + }) + +describe("side execution flow", () => { + it("runs every repetition and publishes the completion marker last", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-flow-test-")) + try { + const executed: string[] = [] + const repetitions = plan(2) + const outcome = await flow({ + repetitions, + out: dir, + isInterrupted: () => false, + runRepetition: async (repetition) => { + executed.push(`${repetition.fixtureName}#${repetition.repIndex}`) + return record(repetition, true) + }, + }) + assert.equal(outcome.completed, true) + assert.deepEqual(executed, ["clamp-range#0", "clamp-range#1"]) + // The marker exists and records every run. + const marker = JSON.parse(await fs.readFile(path.join(dir, "side-results.json"), "utf8")) as { + runs: { repIndex: number }[] + } + assert.deepEqual( + marker.runs.map((run) => run.repIndex), + [0, 1], + ) + // Per-run artifacts exist for the records with a trace. + const artifact = JSON.parse( + await fs.readFile(path.join(dir, "artifacts", "clamp-range-rep0", "run-record.json"), "utf8"), + ) as { repIndex: number } + assert.equal(artifact.repIndex, 0) + // The atomic publication leaves no temp residue. + assert.deepEqual((await fs.readdir(path.join(dir, "artifacts", "clamp-range-rep0"))).sort(), [ + "run-record.json", + ]) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("stops before the next repetition and publishes nothing when interrupted mid-flow", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-flow-test-")) + try { + const repetitions = plan(3) + let interrupted = false + const executed: number[] = [] + const outcome = await flow({ + repetitions, + out: dir, + isInterrupted: () => interrupted, + runRepetition: async (repetition) => { + executed.push(repetition.repIndex) + // The signal lands during the first repetition. + interrupted = true + return record(repetition, true) + }, + }) + assert.equal(outcome.completed, false) + // Exactly the in-flight repetition ran; no later repetition started. + assert.deepEqual(executed, [0]) + assert.equal(outcome.runs.length, 1) + // The interrupted side publishes neither artifacts nor the marker. + await assert.rejects(fs.access(path.join(dir, "side-results.json"))) + await assert.rejects(fs.access(path.join(dir, "artifacts"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("does not publish when the interrupt lands after the last repetition", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-flow-test-")) + try { + const repetitions = plan(2) + let completedRepetitions = 0 + const outcome = await flow({ + repetitions, + out: dir, + // The flag is read between the loop and the publication, so it + // goes live once every repetition finished. + isInterrupted: () => completedRepetitions === repetitions.length, + runRepetition: async (repetition) => { + completedRepetitions += 1 + return record(repetition, false) + }, + }) + assert.equal(outcome.completed, false) + assert.equal(outcome.runs.length, 2) + await assert.rejects(fs.access(path.join(dir, "side-results.json"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("keeps running after a wiring error and records it as an infrastructure failure", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-flow-test-")) + try { + const repetitions = plan(2) + const executed: number[] = [] + const outcome = await flow({ + repetitions, + out: dir, + isInterrupted: () => false, + runRepetition: async (repetition) => { + executed.push(repetition.repIndex) + if (repetition.repIndex === 0) throw new Error("wiring broke") + return record(repetition, false) + }, + }) + assert.equal(outcome.completed, true) + assert.deepEqual(executed, [0, 1]) + const failed = outcome.runs[0] + assert.ok(failed) + assert.deepEqual(failed.outcome, { outcome: "infrastructure-failure", reason: "runTraceMissing" }) + assert.equal(failed.fixtureName, "clamp-range") + assert.equal(failed.repIndex, 0) + // A record without a trace writes no artifact; the marker still + // lists both runs. + const marker = JSON.parse(await fs.readFile(path.join(dir, "side-results.json"), "utf8")) as { + runs: unknown[] + } + assert.equal(marker.runs.length, 2) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/scripts/agent-evals/coding-side-flow.ts b/scripts/agent-evals/coding-side-flow.ts new file mode 100644 index 0000000000..cba0deaf5a --- /dev/null +++ b/scripts/agent-evals/coding-side-flow.ts @@ -0,0 +1,81 @@ +// The side execution loop: one repetition plan, interruption gates before +// each repetition and before publication, and the completion-marker +// publication. The live side wires its fixture runner into it; tests inject +// synthetic repetitions, so the gating and publication order is testable +// without a credential, a host, or a provider. +import fs from "node:fs/promises" +import path from "node:path" +import type { SideResultFile } from "./coding-compare.ts" +import type { CodingRunRecord } from "./coding-run.ts" +import { classifyRunOutcome } from "./coding-outcomes.ts" +import { buildRunMetrics } from "./coding-metrics.ts" +import { writeJsonAtomic } from "./fs-atomic.ts" + +export interface SideRepetition { + fixtureName: string + fixtureVersion: string + repIndex: number +} + +export interface SideFlowInputs { + // The full repetition plan of this invocation, in execution order. + repetitions: readonly SideRepetition[] + out: string + // True once a terminal signal arrived. The loop checks it before each + // repetition and again before publication. + isInterrupted: () => boolean + // Runs one repetition. The substrate never throws for task failures; a + // throw is a wiring error and becomes an infrastructure-failure record. + runRepetition: (repetition: SideRepetition) => Promise + // Builds the side result that publication writes as the completion + // marker. May be async, so the pins can describe what actually executed. + buildResult: (runs: CodingRunRecord[]) => SideResultFile | Promise +} + +export interface SideFlowOutcome { + runs: CodingRunRecord[] + // True when every repetition finished and the completion marker was + // published. False when an interrupt stopped the side; no marker and no + // artifacts exist in that case. + completed: boolean +} + +export async function executeSideFlow(inputs: SideFlowInputs): Promise { + const runs: CodingRunRecord[] = [] + for (const repetition of inputs.repetitions) { + // An interrupt stops the loop before the next repetition starts. The + // in-flight repetition is left to the substrate's own teardown. + if (inputs.isInterrupted()) return { runs, completed: false } + try { + runs.push(await inputs.runRepetition(repetition)) + } catch { + // The substrate never throws for task failures. This guards wiring + // errors only, and records them as infrastructure failures. + const outcome = classifyRunOutcome({ hasTrace: false, finalWorkspaceAvailable: false }) + runs.push({ + fixtureName: repetition.fixtureName, + fixtureVersion: repetition.fixtureVersion, + repIndex: repetition.repIndex, + outcome, + metrics: buildRunMetrics({ + fixtureName: repetition.fixtureName, + repIndex: repetition.repIndex, + outcome, + modelIdentifier: { available: false, reason: "runFailed" }, + }), + }) + } + } + // The completion marker must never exist for an interrupted side, so the + // publication stays gated even when the loop itself finished. + if (inputs.isInterrupted()) return { runs, completed: false } + // Per-run artifacts are published atomically as each record exists. + for (const record of runs) { + if (!record.trace) continue + const artifactDir = path.join(inputs.out, "artifacts", `${record.fixtureName}-rep${record.repIndex}`) + await fs.mkdir(artifactDir, { recursive: true }) + await writeJsonAtomic(path.join(artifactDir, "run-record.json"), record) + } + await writeJsonAtomic(path.join(inputs.out, "side-results.json"), await inputs.buildResult(runs)) + return { runs, completed: true } +} diff --git a/scripts/agent-evals/coding-side.mts b/scripts/agent-evals/coding-side.mts index 8b6675d688..e07b52b113 100644 --- a/scripts/agent-evals/coding-side.mts +++ b/scripts/agent-evals/coding-side.mts @@ -10,19 +10,16 @@ import path from "node:path" import { fileURLToPath } from "node:url" import { createHostRuntime } from "./launcher.ts" import { createContainerGradingRuntime } from "./container-runtime.ts" -import { createLocalSubprocessGradingRuntime } from "./coding-grader.ts" -import { runCodingFixtureOnce, type CodingRunRecord } from "./coding-run.ts" +import { runCodingFixtureOnce } from "./coding-run.ts" import { CODING_FIXTURE_BUNDLES } from "./coding-fixtures.ts" import { CODING_SUITE_ID, codingSuiteDigest, ensureCodingSuiteRegistered } from "./coding-suite.ts" import { GRADER_VERSION } from "./coding-grader.ts" import { digestValue } from "./coding-digest.ts" import type { SidePins } from "./coding-pins.ts" import { invalidateSideArtifacts } from "./coding-compare.ts" -import type { SideResultFile } from "./coding-compare.ts" import { assertCommitsApproved, assertCredentialEnvAllowed, loadTrustPolicy, resolveCommit } from "./coding-trust.ts" -import { classifyRunOutcome } from "./coding-outcomes.ts" -import { buildRunMetrics } from "./coding-metrics.ts" -import { writeJsonAtomic } from "./fs-atomic.ts" +import { executeSideFlow, type SideRepetition } from "./coding-side-flow.ts" +import { createInterruptFlag } from "./side-process-guard.ts" const thisRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") @@ -30,11 +27,16 @@ const readArgs = (): { reps: number out: string credentialEnv: string - allowUnisolatedGrading: boolean evaluatedRoot: string trustFile: string } => { const args = process.argv.slice(2) + // The local subprocess runtime has no container isolation: agent-written + // grading code would execute on the host filesystem. A live side always + // grades in the container runtime, so the dangerous option is rejected + // loudly instead of ignored. + if (args.includes("--allow-unisolated-grading")) + throw new Error("--allow-unisolated-grading is not accepted here: a live side grades in the container runtime") const read = (name: string): string | undefined => { const index = args.indexOf(`--${name}`) return index >= 0 ? args[index + 1] : undefined @@ -51,7 +53,6 @@ const readArgs = (): { reps: Number(read("reps") ?? "3"), out: path.resolve(out), credentialEnv, - allowUnisolatedGrading: args.includes("--allow-unisolated-grading"), // Repository root of the extension under test. The trusted harness // code drives the run; only the extension host loads this tree. evaluatedRoot: path.resolve(read("evaluated-root") ?? thisRoot), @@ -82,7 +83,7 @@ const readVscodeTarget = async (root: string): Promise => { } } -const { reps, out, credentialEnv, allowUnisolatedGrading, evaluatedRoot, trustFile } = readArgs() +const { reps, out, credentialEnv, evaluatedRoot, trustFile } = readArgs() if (!Number.isInteger(reps) || reps < 1) throw new Error("--reps must be a positive integer") const sideLabel = readSideLabel() // Trust gate. The evaluated root is loaded by the extension host and receives @@ -102,7 +103,19 @@ if (credential === undefined || credential.length === 0) { } ensureCodingSuiteRegistered() -const gradingRuntime = allowUnisolatedGrading ? createLocalSubprocessGradingRuntime() : createContainerGradingRuntime() +// Side-level interrupt. A SIGINT or SIGTERM that reaches this process arms +// one flag at most once. The flow checks it before each repetition and +// before publication, so an interrupted side starts no new repetition and +// never publishes the completion marker. The substrate registers its own +// host teardown listener per run; the two listeners stay independent and +// the flag never consumes their signals. +const interrupt = createInterruptFlag() +process.on("SIGINT", () => interrupt.signal("SIGINT")) +process.on("SIGTERM", () => interrupt.signal("SIGTERM")) + +// Live sides always grade in the container runtime. The local subprocess +// runtime stays available to unit tests only. +const gradingRuntime = createContainerGradingRuntime() // Invalidate the previous generation before execution. After this point a // crash can never leave stale side results or run records behind: consumers @@ -112,79 +125,69 @@ await invalidateSideArtifacts(out) const projection = CODING_FIXTURE_BUNDLES[0]?.fixture.settingsProjection ?? {} -const runs: CodingRunRecord[] = [] -for (const bundle of CODING_FIXTURE_BUNDLES) { - const fixtureName = bundle.fixture.identity.name - for (let repIndex = 0; repIndex < reps; repIndex++) { - const artifactsDir = path.join(out, "artifacts", `${fixtureName}-rep${repIndex}`) - await fs.mkdir(artifactsDir, { recursive: true }) - try { - const record = await runCodingFixtureOnce({ - fixtureName, - repIndex, - credential, - runtime: createHostRuntime({ - root: evaluatedRoot, - artifactsDir, - vscodeCachePath: path.join(thisRoot, "apps/vscode-e2e/.vscode-test"), - }), - gradingRuntime, - scratchDir: artifactsDir, - }) - runs.push(record) - } catch { - // The substrate never throws for task failures. This guards wiring - // errors only, and records them as infrastructure failures. - const outcome = classifyRunOutcome({ hasTrace: false, finalWorkspaceAvailable: false }) - runs.push({ - fixtureName, - fixtureVersion: bundle.fixture.identity.version, - repIndex, - outcome, - metrics: buildRunMetrics({ - fixtureName, - repIndex, - outcome, - modelIdentifier: { available: false, reason: "runFailed" }, - }), - }) - } +const repetitions: SideRepetition[] = CODING_FIXTURE_BUNDLES.flatMap((bundle) => + Array.from({ length: reps }, (_, repIndex) => ({ + fixtureName: bundle.fixture.identity.name, + fixtureVersion: bundle.fixture.identity.version, + repIndex, + })), +) + +// Pins are recorded when the side publishes, so they describe what actually +// executed: the approved evaluated commit and, when the container runtime +// resolved one, the immutable grader image identity. +const buildPins = async (): Promise => { + const digest = gradingRuntime.imageDigest ? await gradingRuntime.imageDigest() : undefined + return { + side: sideLabel, + commit: evaluatedCommit, + suiteId: CODING_SUITE_ID, + suiteDigest: codingSuiteDigest(), + settingsDigest: digestValue(projection), + modelIdentifier: typeof projection["openAiModelId"] === "string" ? projection["openAiModelId"] : "unavailable", + graderVersion: GRADER_VERSION, + graderImageDigest: digest?.available === true ? digest.value : "unavailable", + runnerEnvironment: { + os: process.platform, + nodeVersion: process.version, + vscodeTarget: await readVscodeTarget(evaluatedRoot), + }, } } -// Pins are recorded after the runs so they describe what actually executed: -// the approved evaluated commit and, when the container runtime resolved one, -// the immutable grader image identity. -const digest = gradingRuntime.imageDigest ? await gradingRuntime.imageDigest() : undefined -const pins: SidePins = { - side: sideLabel, - commit: evaluatedCommit, - suiteId: CODING_SUITE_ID, - suiteDigest: codingSuiteDigest(), - settingsDigest: digestValue(projection), - modelIdentifier: typeof projection["openAiModelId"] === "string" ? projection["openAiModelId"] : "unavailable", - graderVersion: GRADER_VERSION, - graderImageDigest: digest?.available === true ? digest.value : "unavailable", - runnerEnvironment: { - os: process.platform, - nodeVersion: process.version, - vscodeTarget: await readVscodeTarget(evaluatedRoot), +const { runs, completed } = await executeSideFlow({ + repetitions, + out, + isInterrupted: interrupt.isInterrupted, + runRepetition: async ({ fixtureName, repIndex }) => { + const artifactsDir = path.join(out, "artifacts", `${fixtureName}-rep${repIndex}`) + await fs.mkdir(artifactsDir, { recursive: true }) + return runCodingFixtureOnce({ + fixtureName, + repIndex, + credential, + runtime: createHostRuntime({ + root: evaluatedRoot, + artifactsDir, + vscodeCachePath: path.join(thisRoot, "apps/vscode-e2e/.vscode-test"), + }), + gradingRuntime, + scratchDir: artifactsDir, + }) }, -} -const result: SideResultFile = { pins, runs } + // Publication point and completion marker. side-results.json appears + // atomically only after every repetition of this invocation finished, so + // a reader that sees the file sees a complete side of the current run. A + // missing file means an interrupted side; consumers must treat it as a + // failed side and never pair partial output. + buildResult: async (runs) => ({ pins: await buildPins(), runs }), +}) -// Per-run artifacts are published atomically as each record exists. -for (const record of runs) { - if (!record.trace) continue - const artifactDir = path.join(out, "artifacts", `${record.fixtureName}-rep${record.repIndex}`) - await writeJsonAtomic(path.join(artifactDir, "run-record.json"), record) +if (!completed) { + // An interrupted side exits with the conventional signal code and without + // side-results.json, so consumers treat it as a failed side. + console.error(`coding-evals side interrupted by ${interrupt.name() ?? "signal"}; side-results.json not published`) + process.exit(interrupt.exitCode()) } - -// Publication point and completion marker. side-results.json appears -// atomically only after every repetition of this invocation finished, so a -// reader that sees the file sees a complete side of the current run. A -// missing file means an interrupted side; consumers must treat it as a failed -// side and never pair partial output. -await writeJsonAtomic(path.join(out, "side-results.json"), result) const correct = runs.filter((record) => record.outcome.outcome === "correct").length -console.log(`coding-evals side complete: ${correct}/${runs.length} correct, suite ${pins.suiteDigest.slice(0, 12)}`) +console.log(`coding-evals side complete: ${correct}/${runs.length} correct, suite ${codingSuiteDigest().slice(0, 12)}`) diff --git a/scripts/agent-evals/coding-side.test.ts b/scripts/agent-evals/coding-side.test.ts index fccf0c995a..402c753a7c 100644 --- a/scripts/agent-evals/coding-side.test.ts +++ b/scripts/agent-evals/coding-side.test.ts @@ -176,3 +176,34 @@ describe("coding side trust gate", () => { }, ) }) + +describe("coding side isolation gate", () => { + it("rejects --allow-unisolated-grading before any run or artifact exists", { timeout: 180_000 }, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) + try { + // The argument gate runs before the trust policy loads, so no + // fixture commit is needed and no credential is read. + const trustFile = await writePolicy(dir, []) + const outcome = await runSideScript( + [ + "--reps", + "1", + "--out", + path.join(dir, "out"), + "--trust-file", + trustFile, + "--allow-unisolated-grading", + ], + minimalEnv({ SOLHEIM_API_KEY: "sk-test" }), + ) + assert.notEqual(outcome.code, 0) + assert.match(outcome.stderr, /--allow-unisolated-grading is not accepted here/) + assert.ok(!outcome.stdout.includes("coding-evals side complete")) + // The rejection happens before any directory or result file + // exists, so the side can never publish host-graded artifacts. + await assert.rejects(fs.access(path.join(dir, "out", "side-results.json"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/scripts/agent-evals/container-runtime.test.ts b/scripts/agent-evals/container-runtime.test.ts index 891da30572..1746ae172d 100644 --- a/scripts/agent-evals/container-runtime.test.ts +++ b/scripts/agent-evals/container-runtime.test.ts @@ -131,16 +131,14 @@ describe("container grading runtime", () => { assert.equal(runtime.isolation, "container") }) - it("pulls the image outside the grading timeout and runs with a unique name", async () => { + it("skips the pull when the image digest already resolves locally", async () => { const cli = recordingCli() const runtime = createContainerGradingRuntime({ cli }) const verdict = await runtime.run(makeJob()) assert.equal(verdict.kind, "passed") - // The pull happens first, the image identity is resolved from the - // local engine, and the timed run comes after both. - assert.equal(cli.calls[0], `pull ${GRADER_IMAGE}`) - assert.equal(cli.calls.indexOf("run 30000"), 2) - assert.equal(cli.calls[1], "inspect") + // The digest lookup decides, so a cached image never pulls again: the + // inspect comes first and the timed run follows it directly. + assert.deepEqual(cli.calls, ["inspect", "run 30000"]) // The run argv names a unique container. const nameIndex = cli.runArgs[0]!.indexOf("--name") assert.ok(nameIndex > 0) @@ -151,6 +149,45 @@ describe("container grading runtime", () => { assert.match(digest.available ? digest.value : "", /^node:22-bookworm@sha256:[0-9a-f]{64}$/) }) + it("pulls when the local digest is absent, then resolves the identity again", async () => { + let lookups = 0 + const cli = recordingCli({ + imageDigest: async () => { + lookups += 1 + cli.calls.push("inspect") + return lookups === 1 ? undefined : "node:22-bookworm@sha256:" + "a".repeat(64) + }, + }) + const runtime = createContainerGradingRuntime({ cli }) + const verdict = await runtime.run(makeJob()) + assert.equal(verdict.kind, "passed") + // Inspect, pull, inspect again, and only then the timed run. + assert.deepEqual(cli.calls, ["inspect", `pull ${GRADER_IMAGE}`, "inspect", "run 30000"]) + const digest = await runtime.imageDigest!() + assert.equal(digest.available, true) + assert.equal(digest.available ? digest.value : "", "node:22-bookworm@sha256:" + "a".repeat(64)) + }) + + it("preserves the resolved digest when a later lookup stays unavailable", async () => { + let lookups = 0 + const cli = recordingCli({ + imageDigest: async () => { + lookups += 1 + cli.calls.push("inspect") + return lookups <= 2 ? "node:22-bookworm@sha256:" + "b".repeat(64) : undefined + }, + }) + const runtime = createContainerGradingRuntime({ cli }) + await runtime.run(makeJob()) + // The second run finds no digest, pulls, and still finds none; the + // runtime keeps the identity the first run resolved and never + // fabricates a new one. + await runtime.run(makeJob()) + const digest = await runtime.imageDigest!() + assert.equal(digest.available, true) + assert.equal(digest.available ? digest.value : "", "node:22-bookworm@sha256:" + "b".repeat(64)) + }) + it("removes the container by name on timeout and verifies the removal before deleting output", async () => { const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "container-runtime-test-")) try { @@ -215,7 +252,17 @@ describe("container grading runtime", () => { }) it("reports a failed image pull before any container runs", async () => { - const cli = recordingCli({ pull: async () => "failed" }) + const cli = recordingCli({ + // The image is absent locally, so the runtime attempts the pull. + imageDigest: async () => { + cli.calls.push("inspect") + return undefined + }, + pull: async (image) => { + cli.calls.push(`pull ${image}`) + return "failed" + }, + }) const runtime = createContainerGradingRuntime({ cli }) const verdict = await runtime.run(makeJob()) assert.deepEqual(verdict, { @@ -223,6 +270,8 @@ describe("container grading runtime", () => { isolation: "container", reason: "graderImagePullFailed", }) + // The pull failure aborts before any timed run. + assert.deepEqual(cli.calls, ["inspect", `pull ${GRADER_IMAGE}`]) assert.deepEqual( cli.calls.filter((call) => call.startsWith("run")), [], diff --git a/scripts/agent-evals/container-runtime.ts b/scripts/agent-evals/container-runtime.ts index 23f981e2ec..1cb5e8dd16 100644 --- a/scripts/agent-evals/container-runtime.ts +++ b/scripts/agent-evals/container-runtime.ts @@ -223,9 +223,17 @@ export function createContainerGradingRuntime(options: ContainerRuntimeOptions = }) const selected = await resolveCli() if (!selected) return failure("noContainerRuntime") - // The pull stays outside the grading timeout on purpose. - if ((await selected.pull(job.resources.image)) === "failed") return failure("graderImagePullFailed") - resolvedDigest = (await selected.imageDigest(job.resources.image)) ?? resolvedDigest + // The digest lookup decides whether a pull is needed, so a cached + // image never pulls again, and the pull stays outside the grading + // timeout on purpose. The identity is resolved again after a pull. + let digest = await selected.imageDigest(job.resources.image) + if (digest === undefined) { + if ((await selected.pull(job.resources.image)) === "failed") return failure("graderImagePullFailed") + digest = await selected.imageDigest(job.resources.image) + } + // A lookup that still finds nothing preserves a digest resolved by + // an earlier run of this runtime; it never fabricates one. + resolvedDigest = digest ?? resolvedDigest const outDir = path.join(path.dirname(job.gradingDir), "grading-out") await fs.mkdir(outDir, { recursive: true }) // The container runs as root with all capabilities dropped, so it diff --git a/scripts/agent-evals/grader-driver.mjs b/scripts/agent-evals/grader-driver.mjs index 9242597534..f4709f861a 100644 --- a/scripts/agent-evals/grader-driver.mjs +++ b/scripts/agent-evals/grader-driver.mjs @@ -4,6 +4,7 @@ // or a missing result file means the grader itself failed, not the solution. // The driver receives no credential and needs no network. import { spawn } from "node:child_process" +import { randomUUID } from "node:crypto" import fs from "node:fs" import path from "node:path" @@ -14,9 +15,28 @@ if (!outPath) { } const testDir = process.argv[3] ?? "test" +// The result file is the host's only view of the grading outcome, so its +// publication is atomic: the content lands in a unique same-directory temp +// file and is renamed into place. A reader never observes a partial write, +// and a failed write or rename removes the temp file before the error +// propagates. const writeResult = (result) => { fs.mkdirSync(path.dirname(outPath), { recursive: true }) - fs.writeFileSync(outPath, JSON.stringify(result)) + const temp = path.join( + path.dirname(outPath), + `.${path.basename(outPath)}.${process.pid}.${randomUUID().slice(0, 8)}.tmp`, + ) + try { + fs.writeFileSync(temp, JSON.stringify(result)) + fs.renameSync(temp, outPath) + } catch (error) { + try { + fs.rmSync(temp, { force: true }) + } catch { + /* The temp file is already gone. */ + } + throw error + } } const TEST_EXTENSIONS = new Set([".js", ".mjs", ".cjs"]) diff --git a/scripts/agent-evals/grader-driver.test.ts b/scripts/agent-evals/grader-driver.test.ts new file mode 100644 index 0000000000..835d5bd81f --- /dev/null +++ b/scripts/agent-evals/grader-driver.test.ts @@ -0,0 +1,80 @@ +import assert from "node:assert/strict" +import { spawn } from "node:child_process" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import { driverFilePath } from "./coding-grader.ts" + +// Runs the real driver against a real workspace with the real filesystem. +// The driver is the grading publication boundary, so these checks exercise +// the actual temp-write-and-rename behavior, not a stub. +const runDriver = async (args: readonly string[], cwd: string): Promise => { + // This test file itself runs under the node:test runner, whose + // NODE_TEST_CONTEXT would leak into the driver and then into its inner + // `node --test` children, switching them into runner-child mode with no + // TAP summary. The driver gets an environment without that variable. + const { NODE_TEST_CONTEXT: _runnerContext, ...driverEnv } = process.env + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [driverFilePath(), ...args], { cwd, env: driverEnv, stdio: "ignore" }) + child.once("error", reject) + child.once("close", (code) => resolve(code ?? -1)) + }) +} + +const passingTest = "import assert from 'node:assert/strict'\nassert.equal(1 + 1, 2)\n" + +const writeWorkspace = async (scratch: string): Promise => { + const workspace = path.join(scratch, "workspace") + await fs.mkdir(path.join(workspace, "test"), { recursive: true }) + await fs.writeFile(path.join(workspace, "test", "passing.test.mjs"), passingTest) + return workspace +} + +describe("grader driver result publication", () => { + it( + "publishes the completed result through a same-directory rename with no temp residue", + { timeout: 60_000 }, + async () => { + const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "grader-driver-test-")) + try { + const workspace = await writeWorkspace(scratch) + const outDir = path.join(scratch, "out") + const exit = await runDriver([path.join(outDir, "result.json"), "test"], workspace) + assert.equal(exit, 0) + const result = JSON.parse(await fs.readFile(path.join(outDir, "result.json"), "utf8")) as { + kind: string + passed: boolean + passedCount: number + } + assert.equal(result.kind, "completed") + assert.equal(result.passed, true) + assert.equal(result.passedCount, 1) + // The atomic publication leaves exactly the result file behind. + assert.deepEqual((await fs.readdir(outDir)).sort(), ["result.json"]) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }, + ) + + it("removes the temp file when the rename into place fails", { timeout: 60_000 }, async () => { + const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "grader-driver-test-")) + try { + const workspace = await writeWorkspace(scratch) + // A directory at the target path makes the rename fail after the + // temp write succeeded, so the cleanup path is the one under test. + const blocked = path.join(scratch, "blocked") + await fs.mkdir(blocked) + const exit = await runDriver([blocked, "test"], workspace) + assert.notEqual(exit, 0) + // Both publication attempts cleaned their temp files. + const residue = (await fs.readdir(scratch)).filter((name) => name.includes(".tmp")) + assert.deepEqual(residue, []) + // The blocked target is untouched. + assert.equal((await fs.stat(blocked)).isDirectory(), true) + } finally { + await fs.rm(scratch, { recursive: true, force: true }) + } + }) +}) diff --git a/scripts/agent-evals/side-process-guard.test.ts b/scripts/agent-evals/side-process-guard.test.ts index adba5a2b64..c5620d49bc 100644 --- a/scripts/agent-evals/side-process-guard.test.ts +++ b/scripts/agent-evals/side-process-guard.test.ts @@ -5,7 +5,7 @@ import os from "node:os" import path from "node:path" import { describe, it } from "node:test" import { runBoundedPreparation } from "./coding-compare.ts" -import { createSideProcessGuard, type SideKillOutcome } from "./side-process-guard.ts" +import { createInterruptFlag, createSideProcessGuard, groupAlive, type SideKillOutcome } from "./side-process-guard.ts" // Credential-free process fixtures. Each child is a bare node process in its // own detached group, the same topology the comparison driver creates. @@ -279,3 +279,62 @@ describe("bounded preparation children", () => { } }) }) + +describe("group liveness probe", () => { + it("reports a live process group as alive", { timeout: 30_000 }, async () => { + const { child, spawned } = spawnDetachedChild("setInterval(() => {}, 1000000)") + try { + await spawned + assert.ok(child.pid !== undefined) + assert.equal(groupAlive(child.pid), true) + } finally { + child.kill("SIGKILL") + } + }) + + it("reports a gone process group as gone", { timeout: 30_000 }, async () => { + const { child, spawned, exited } = spawnDetachedChild("process.exit(0)") + await spawned + await exited + assert.ok(child.pid !== undefined) + assert.equal(groupAlive(child.pid), false) + }) + + it("treats a refused probe as an alive group so the escalation continues", () => { + const throwCode = (code: string) => () => { + throw Object.assign(new Error(`probe refused: ${code}`), { code }) + } + // EPERM means the group exists but the probe was not allowed. + assert.equal(groupAlive(123_456, throwCode("EPERM")), true) + // Any other error is not proof of absence either. + assert.equal( + groupAlive(123_456, () => { + throw new Error("probe failed") + }), + true, + ) + // Only ESRCH proves the group is gone. + assert.equal(groupAlive(123_456, throwCode("ESRCH")), false) + }) +}) + +describe("side interrupt flag", () => { + it("arms at most once and keeps the first signal", () => { + const flag = createInterruptFlag() + assert.equal(flag.isInterrupted(), false) + assert.equal(flag.exitCode(), 1) + flag.signal("SIGINT") + // A later signal must not un-arm or replace the first one. + flag.signal("SIGTERM") + assert.equal(flag.isInterrupted(), true) + assert.equal(flag.name(), "SIGINT") + assert.equal(flag.exitCode(), 130) + }) + + it("maps SIGTERM to its conventional exit code", () => { + const flag = createInterruptFlag() + flag.signal("SIGTERM") + assert.equal(flag.name(), "SIGTERM") + assert.equal(flag.exitCode(), 143) + }) +}) diff --git a/scripts/agent-evals/side-process-guard.ts b/scripts/agent-evals/side-process-guard.ts index 8745273b8e..cd7897bf48 100644 --- a/scripts/agent-evals/side-process-guard.ts +++ b/scripts/agent-evals/side-process-guard.ts @@ -17,12 +17,46 @@ export interface SideProcessGuard { interrupt(): Promise } -const groupAlive = (pid: number): boolean => { +// Probes one process group. Only ESRCH proves the group is gone. EPERM and +// any other error mean the probe was refused for a group that still exists, +// so the group counts as alive and the SIGKILL escalation still fires. The +// probe parameter is a test seam; production callers use process.kill. +export const groupAlive = ( + pid: number, + probe: (pid: number, signal?: NodeJS.Signals | number) => void = process.kill, +): boolean => { try { - process.kill(-pid, 0) + probe(-pid, 0) return true - } catch { - return false + } catch (error) { + return (error as NodeJS.ErrnoException).code !== "ESRCH" + } +} + +// One side-level interrupt flag. The handler arms it at most once, so +// repeated signals cannot un-arm or re-arm it. Listeners that share the +// process, the substrate's host teardown for example, observe their own +// handlers and never consume this flag. +export interface InterruptFlag { + // Arms the flag. Later calls are no-ops. + signal(name: NodeJS.Signals): void + // True once any signal arrived. + isInterrupted(): boolean + // The observed signal name, for diagnostics. + name(): NodeJS.Signals | undefined + // The conventional shell exit code for the observed signal. + exitCode(): number +} + +export const createInterruptFlag = (): InterruptFlag => { + let signal: NodeJS.Signals | undefined + return { + signal: (name) => { + if (signal === undefined) signal = name + }, + isInterrupted: () => signal !== undefined, + name: () => signal, + exitCode: () => (signal === "SIGINT" ? 130 : signal === "SIGTERM" ? 143 : 1), } } From c1430ace6f99aede45e1f65e6ca96515eacb8599 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Mon, 26 Oct 2026 02:55:24 +0000 Subject: [PATCH 6/9] fix(agent-evals): reject dirty checkouts and clear stale grader digests --- scripts/agent-evals/README.md | 2 +- .../agent-evals/coding-compare-script.test.ts | 43 ++++++ scripts/agent-evals/coding-side-flow.test.ts | 56 ++++++++ scripts/agent-evals/coding-side-flow.ts | 15 ++- scripts/agent-evals/coding-side.mts | 16 ++- scripts/agent-evals/coding-side.test.ts | 42 ++++++ scripts/agent-evals/coding-trust.test.ts | 124 ++++++++++++++++++ scripts/agent-evals/coding-trust.ts | 34 +++++ scripts/agent-evals/container-runtime.test.ts | 23 ++-- scripts/agent-evals/container-runtime.ts | 11 +- 10 files changed, 349 insertions(+), 17 deletions(-) diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index 3c1a32feb8..c337ad7716 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -151,7 +151,7 @@ A comparison runs the extension code of two commits with the provider credential The comparison driver, fixtures, grader, and report builder all run from the trusted checkout that invokes `coding-evals:compare`. Only the extension host loads an evaluated worktree. The side process receives only the allowlisted environment plus the one named credential variable. Each side installs dependencies from its own worktree's frozen lockfile and builds its own exact commit, so each side runs its own compiled extension. Each side has its own wall clock (`--side-timeout-minutes`, default 120). -The side script runs the same gate on its own inputs. It requires `--trust-file`, resolves the full commit of its `--evaluated-root`, and rejects an unapproved or unreadable commit before it creates a host or reads the credential. The `--credential-env` name must come from the supported provider variable allowlist; an arbitrary name can never forward an unrelated parent secret. Worktree install and build children also receive only an allowlisted environment, so the driver's secrets never reach them. +The side script runs the same gate on its own inputs. It requires `--trust-file`, resolves the full commit of its `--evaluated-root`, and rejects an unapproved or unreadable commit before it creates a host or reads the credential. It also rejects an evaluated root whose tracked or untracked files differ from that commit, because the approval describes the commit and not the working tree. The check has one trust limit: ignored files, such as dependency installs and build outputs, stay outside commit identity and outside the check, so the approval never attests them. A recorded comparison closes that gap by loading only a fresh detached worktree built from the approved commit. The `--credential-env` name must come from the supported provider variable allowlist; an arbitrary name can never forward an unrelated parent secret. Worktree install and build children also receive only an allowlisted environment, so the driver's secrets never reach them. Every driver child, preparation steps and the side alike, runs detached in its own process group and is tracked by one guard. A SIGINT or SIGTERM that reaches only the driver terminates the tracked group through a bounded SIGTERM-then-SIGKILL escalation, and the worktree cleanup waits for that escalation before it removes any tree. diff --git a/scripts/agent-evals/coding-compare-script.test.ts b/scripts/agent-evals/coding-compare-script.test.ts index 05b68288d5..ff45865fc4 100644 --- a/scripts/agent-evals/coding-compare-script.test.ts +++ b/scripts/agent-evals/coding-compare-script.test.ts @@ -82,4 +82,47 @@ describe("coding compare isolation gate", () => { await fs.rm(dir, { recursive: true, force: true }) } }) + + it( + "runs past the isolation gate without the flag and fails at the approved-commit check", + { timeout: 180_000 }, + async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-compare-test-")) + try { + const trustFile = path.join(dir, "trust-policy.json") + // The empty policy approves nothing, so the trust gate must reject + // the run. The placeholder credential only proves the failure is + // not the credential check; no real credential enters this test. + await fs.writeFile(trustFile, JSON.stringify({ version: 1, approvedCommits: [] })) + const out = path.join(dir, "out") + const env = { ...minimalEnv(), SOLHEIM_API_KEY: "not-a-real-credential" } + const outcome = await runCompareScript( + [ + "--baseline", + "a".repeat(40), + "--candidate", + "b".repeat(40), + "--reps", + "1", + "--out", + out, + "--trust-file", + trustFile, + ], + env, + ) + assert.notEqual(outcome.code, 0) + // The failure is the approved-commit gate, not the argument gate. + assert.doesNotMatch(outcome.stderr, /--allow-unisolated-grading is not accepted here/) + assert.match(outcome.stderr, /trust policy must list at least one approved commit SHA/) + // The rejection happens before any report generation exists, so + // the comparison never publishes anything. + for (const name of ["comparison-report.json", "comparison-report.md", "comparison-report.complete"]) { + await assert.rejects(fs.access(path.join(out, name))) + } + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }, + ) }) diff --git a/scripts/agent-evals/coding-side-flow.test.ts b/scripts/agent-evals/coding-side-flow.test.ts index 85a13c3f8f..ddb44c5082 100644 --- a/scripts/agent-evals/coding-side-flow.test.ts +++ b/scripts/agent-evals/coding-side-flow.test.ts @@ -143,6 +143,62 @@ describe("side execution flow", () => { } }) + it("prints one bounded diagnostic with the fixture name and repetition index when a repetition throws", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-flow-test-")) + const lines: string[] = [] + const originalError = console.error + console.error = (line: string) => lines.push(String(line)) + try { + const outcome = await flow({ + repetitions: plan(1), + out: dir, + isInterrupted: () => false, + runRepetition: async () => { + throw new Error("wiring broke") + }, + }) + assert.equal(outcome.completed, true) + const failed = outcome.runs[0] + assert.ok(failed) + assert.deepEqual(failed.outcome, { outcome: "infrastructure-failure", reason: "runTraceMissing" }) + assert.equal(lines.length, 1) + assert.match( + lines[0] ?? "", + /^coding-evals repetition failed: fixture clamp-range rep 0: Error: wiring broke$/, + ) + } finally { + console.error = originalError + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("prints a safe fallback for a non-Error throw and never prints the thrown value", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-flow-test-")) + const lines: string[] = [] + const originalError = console.error + console.error = (line: string) => lines.push(String(line)) + try { + const outcome = await flow({ + repetitions: plan(1), + out: dir, + isInterrupted: () => false, + runRepetition: async () => { + throw { credential: "not-a-real-credential" } + }, + }) + const failed = outcome.runs[0] + assert.ok(failed) + assert.deepEqual(failed.outcome, { outcome: "infrastructure-failure", reason: "runTraceMissing" }) + assert.equal(lines.length, 1) + assert.match(lines[0] ?? "", /non-Error value thrown$/) + // The thrown value itself never reaches the diagnostic. + assert.ok(!lines[0]?.includes("not-a-real-credential")) + } finally { + console.error = originalError + await fs.rm(dir, { recursive: true, force: true }) + } + }) + it("keeps running after a wiring error and records it as an infrastructure failure", async () => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-flow-test-")) try { diff --git a/scripts/agent-evals/coding-side-flow.ts b/scripts/agent-evals/coding-side-flow.ts index cba0deaf5a..8c0057b3f7 100644 --- a/scripts/agent-evals/coding-side-flow.ts +++ b/scripts/agent-evals/coding-side-flow.ts @@ -40,6 +40,12 @@ export interface SideFlowOutcome { completed: boolean } +// One bounded diagnostic line per failed repetition. Only the error name and +// message travel to stderr; a thrown non-Error value can hold anything, so it +// is never printed. +const describeRepetitionError = (error: unknown): string => + error instanceof Error ? `${error.name}: ${error.message.slice(0, 500)}` : "non-Error value thrown" + export async function executeSideFlow(inputs: SideFlowInputs): Promise { const runs: CodingRunRecord[] = [] for (const repetition of inputs.repetitions) { @@ -48,9 +54,14 @@ export async function executeSideFlow(inputs: SideFlowInputs): Promise { + it( + "rejects a dirty approved evaluated root before any host launch or publication", + { timeout: 180_000 }, + async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) + try { + const { repo, commit } = await makeFixtureCommit(dir) + const trustFile = await writePolicy(dir, [commit]) + // The approved checkout is modified after approval: one staged + // tracked change and one untracked file. The commit identity is + // still approved, but it no longer describes the tree. + await fs.writeFile(path.join(repo, "tracked.txt"), "changed") + await execFileAsync("git", ["add", "tracked.txt"], { cwd: repo }) + await fs.writeFile(path.join(repo, "untracked.txt"), "untracked") + // The fake credential only proves the failure is not the + // credential check: the cleanliness gate must stop the side first. + const outcome = await runSideScript( + [ + "--reps", + "1", + "--out", + path.join(dir, "out"), + "--trust-file", + trustFile, + "--evaluated-root", + repo, + ], + minimalEnv({ SOLHEIM_API_KEY: "sk-test" }), + ) + assert.notEqual(outcome.code, 0) + assert.match(outcome.stderr, /tracked or untracked changes/) + assert.ok(!outcome.stdout.includes("coding-evals side complete")) + // The rejection happens before the publication point: no side + // results and no artifacts exist. + await assert.rejects(fs.access(path.join(dir, "out", "side-results.json"))) + await assert.rejects(fs.access(path.join(dir, "out", "artifacts"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }, + ) + it("rejects an unapproved evaluated root before any host launch", { timeout: 180_000 }, async () => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) try { diff --git a/scripts/agent-evals/coding-trust.test.ts b/scripts/agent-evals/coding-trust.test.ts index c30f55ab66..9081c034a9 100644 --- a/scripts/agent-evals/coding-trust.test.ts +++ b/scripts/agent-evals/coding-trust.test.ts @@ -1,20 +1,53 @@ import assert from "node:assert/strict" +import { execFile } from "node:child_process" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" +import { promisify } from "node:util" import { describe, it } from "node:test" import { assertCommitsApproved, assertCredentialEnvAllowed, + assertEvaluatedTreeClean, buildSideEnvironment, buildWorktreeEnvironment, + DirtyEvaluatedTreeError, loadTrustPolicy, UntrustedCommitError, type TrustPolicy, } from "./coding-trust.ts" +const execFileAsync = promisify(execFile) const sha = (c: string): string => c.repeat(40) +// One throwaway repository with one empty commit. It stands in for an +// evaluated root; no code from it ever runs in these checks. +const makeRepo = async (dir: string): Promise => { + const repo = path.join(dir, "repo") + await fs.mkdir(repo) + await execFileAsync("git", ["init", "-q"], { cwd: repo }) + await execFileAsync( + "git", + [ + "-c", + "user.email=fixture@example.test", + "-c", + "user.name=fixture", + // The operator's git config may force signed commits, and the + // throwaway fixture has no key or tty. + "-c", + "commit.gpgsign=false", + "commit", + "--allow-empty", + "-q", + "-m", + "fixture", + ], + { cwd: repo }, + ) + return repo +} + const writePolicy = async (dir: string, content: unknown): Promise => { const file = path.join(dir, "trust-policy.json") await fs.writeFile(file, JSON.stringify(content)) @@ -117,6 +150,97 @@ describe("credential environment allowlist", () => { }) }) +describe("evaluated tree cleanliness gate", () => { + it("accepts a clean tree", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) + try { + const repo = await makeRepo(dir) + await assert.doesNotReject(assertEvaluatedTreeClean(repo)) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("accepts a tree with only ignored build artifacts", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) + try { + const repo = await makeRepo(dir) + await fs.writeFile(path.join(repo, ".gitignore"), "dist\n") + await execFileAsync("git", ["add", ".gitignore"], { cwd: repo }) + await execFileAsync( + "git", + [ + "-c", + "user.email=fixture@example.test", + "-c", + "user.name=fixture", + "-c", + "commit.gpgsign=false", + "commit", + "-q", + "-m", + "ignore", + ], + { cwd: repo }, + ) + // Ignored files sit outside commit identity: dependency installs + // and build outputs never make the tree dirty, and the approval + // never attests them either. + await fs.mkdir(path.join(repo, "dist")) + await fs.writeFile(path.join(repo, "dist", "run.sh"), "#!/bin/sh\n") + await fs.chmod(path.join(repo, "dist", "run.sh"), 0o755) + await assert.doesNotReject(assertEvaluatedTreeClean(repo)) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("rejects a tracked modification after the commit was approved", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) + try { + const repo = await makeRepo(dir) + await fs.writeFile(path.join(repo, "tracked.txt"), "approved\n") + await execFileAsync("git", ["add", "tracked.txt"], { cwd: repo }) + await execFileAsync( + "git", + [ + "-c", + "user.email=fixture@example.test", + "-c", + "user.name=fixture", + "-c", + "commit.gpgsign=false", + "commit", + "-q", + "-m", + "tracked", + ], + { cwd: repo }, + ) + await fs.writeFile(path.join(repo, "tracked.txt"), "changed\n") + await assert.rejects(assertEvaluatedTreeClean(repo), (error: unknown) => { + assert.ok(error instanceof DirtyEvaluatedTreeError) + assert.match(error.message, /tracked or untracked changes/) + assert.ok(error.entries.some((entry) => entry.includes("tracked.txt"))) + return true + }) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("rejects an untracked file", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) + try { + const repo = await makeRepo(dir) + await fs.writeFile(path.join(repo, "untracked.txt"), "x") + await assert.rejects(assertEvaluatedTreeClean(repo), DirtyEvaluatedTreeError) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) +}) + describe("worktree process environment", () => { it("passes the install and build allowlist and drops secrets", () => { const child = buildWorktreeEnvironment({ diff --git a/scripts/agent-evals/coding-trust.ts b/scripts/agent-evals/coding-trust.ts index 6ff10a36e6..1b797c68e0 100644 --- a/scripts/agent-evals/coding-trust.ts +++ b/scripts/agent-evals/coding-trust.ts @@ -63,6 +63,40 @@ export function assertCommitsApproved(policy: TrustPolicy, commits: readonly str if (unapproved.length > 0) throw new UntrustedCommitError(unapproved) } +// A tree whose tracked or untracked files differ from its commit executes +// content the approved commit does not contain, so the approval would not +// describe the code the extension host loads. +export class DirtyEvaluatedTreeError extends Error { + constructor(readonly entries: string[]) { + const shown = entries.slice(0, 10) + const more = entries.length > shown.length ? `, and ${entries.length - shown.length} more` : "" + super( + `evaluated root has tracked or untracked changes; the approved commit does not describe the files the host would load: ${shown.join(", ")}${more}`, + ) + this.name = "DirtyEvaluatedTreeError" + } +} + +// Gate before any host launch. The evaluated root must match its approved +// commit exactly, so a modified checkout can never run under that commit's +// approval. Ignored files, such as dependency installs and build outputs, +// stay outside commit identity and outside this check: the approval does not +// attest them. The comparison driver closes that gap by loading only a fresh +// detached worktree built from the approved commit. +export async function assertEvaluatedTreeClean(root: string): Promise { + const { execFile } = await import("node:child_process") + const { promisify } = await import("node:util") + const { stdout } = await promisify(execFile)("git", ["status", "--porcelain"], { + cwd: root, + timeout: 30_000, + }) + const entries = stdout + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + if (entries.length > 0) throw new DirtyEvaluatedTreeError(entries) +} + // Provider credential variables this framework supports. The side script and // the comparison driver accept only these names, so the tooling can never be // pointed at an unrelated parent secret by naming an arbitrary variable. diff --git a/scripts/agent-evals/container-runtime.test.ts b/scripts/agent-evals/container-runtime.test.ts index 1746ae172d..7552474f95 100644 --- a/scripts/agent-evals/container-runtime.test.ts +++ b/scripts/agent-evals/container-runtime.test.ts @@ -168,24 +168,31 @@ describe("container grading runtime", () => { assert.equal(digest.available ? digest.value : "", "node:22-bookworm@sha256:" + "a".repeat(64)) }) - it("preserves the resolved digest when a later lookup stays unavailable", async () => { + it("never reuses a previous digest when the current lookup stays unavailable", async () => { let lookups = 0 const cli = recordingCli({ imageDigest: async () => { lookups += 1 cli.calls.push("inspect") - return lookups <= 2 ? "node:22-bookworm@sha256:" + "b".repeat(64) : undefined + // Only the first run's lookup finds the image; every later + // lookup, before and after a pull, stays unavailable. + return lookups === 1 ? "node:22-bookworm@sha256:" + "b".repeat(64) : undefined }, }) const runtime = createContainerGradingRuntime({ cli }) + // The first run resolves the identity and records it for the pins. await runtime.run(makeJob()) - // The second run finds no digest, pulls, and still finds none; the - // runtime keeps the identity the first run resolved and never - // fabricates a new one. + const first = await runtime.imageDigest!() + assert.equal(first.available, true) + assert.equal(first.available ? first.value : "", "node:22-bookworm@sha256:" + "b".repeat(64)) + // The second run finds no digest before or after the pull. The + // runtime must report the identity as unavailable instead of reusing + // the digest of the image the first run graded with. await runtime.run(makeJob()) - const digest = await runtime.imageDigest!() - assert.equal(digest.available, true) - assert.equal(digest.available ? digest.value : "", "node:22-bookworm@sha256:" + "b".repeat(64)) + // The missed lookups sent the second run to the pull path. + assert.equal(cli.calls.filter((call) => call.startsWith("pull")).length, 1) + const second = await runtime.imageDigest!() + assert.equal(second.available, false) }) it("removes the container by name on timeout and verifies the removal before deleting output", async () => { diff --git a/scripts/agent-evals/container-runtime.ts b/scripts/agent-evals/container-runtime.ts index 1cb5e8dd16..b09efef745 100644 --- a/scripts/agent-evals/container-runtime.ts +++ b/scripts/agent-evals/container-runtime.ts @@ -221,6 +221,10 @@ export function createContainerGradingRuntime(options: ContainerRuntimeOptions = isolation: "container", reason, }) + // Each run records only its own lookup result. A digest an earlier + // run resolved must never stand in for the identity of the image + // this run grades with. + resolvedDigest = undefined const selected = await resolveCli() if (!selected) return failure("noContainerRuntime") // The digest lookup decides whether a pull is needed, so a cached @@ -231,9 +235,10 @@ export function createContainerGradingRuntime(options: ContainerRuntimeOptions = if ((await selected.pull(job.resources.image)) === "failed") return failure("graderImagePullFailed") digest = await selected.imageDigest(job.resources.image) } - // A lookup that still finds nothing preserves a digest resolved by - // an earlier run of this runtime; it never fabricates one. - resolvedDigest = digest ?? resolvedDigest + // A lookup that still finds nothing leaves the digest unset, and + // the pins record the identity as unavailable instead of reusing + // one from an earlier run. The runtime never fabricates a value. + resolvedDigest = digest const outDir = path.join(path.dirname(job.gradingDir), "grading-out") await fs.mkdir(outDir, { recursive: true }) // The container runs as root with all capabilities dropped, so it From 2c3c523ca5d7173e32b23f419929a990b28908ca Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Mon, 26 Oct 2026 22:35:58 +0000 Subject: [PATCH 7/9] fix(agent-evals): prevent child startup after interruption and preserve signal exit codes --- .../agent-evals/coding-compare-driver.test.ts | 314 ++++++++++++++++++ .../agent-evals/coding-compare-script.test.ts | 157 ++++++++- scripts/agent-evals/coding-compare.mts | 188 +++-------- scripts/agent-evals/coding-compare.ts | 183 +++++++++- 4 files changed, 703 insertions(+), 139 deletions(-) create mode 100644 scripts/agent-evals/coding-compare-driver.test.ts diff --git a/scripts/agent-evals/coding-compare-driver.test.ts b/scripts/agent-evals/coding-compare-driver.test.ts new file mode 100644 index 0000000000..fcf7a1f28d --- /dev/null +++ b/scripts/agent-evals/coding-compare-driver.test.ts @@ -0,0 +1,314 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { fileURLToPath } from "node:url" +import { describe, it } from "node:test" +import { + executeComparisonFlow, + publishedComparisonGeneration, + runCompareSide, + type CompareSideWiring, + type ComparisonFlowWiring, + type SideResultFile, +} from "./coding-compare.ts" +import { buildRunMetrics } from "./coding-metrics.ts" +import type { SidePins } from "./coding-pins.ts" +import { createInterruptFlag, type InterruptFlag } from "./side-process-guard.ts" + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") +const COMMIT = "b".repeat(40) + +// The guard records every tracked group, so a test can prove the side child +// was never spawned and nothing was left tracked after a real child exited. +const trackingGuard = (tracked: (number | undefined)[]): CompareSideWiring["guard"] => ({ + track: (pid) => { + tracked.push(pid) + }, + interrupt: async () => "none", +}) + +const sideWiring = (inputs: { + flag: InterruptFlag + tracked: (number | undefined)[] + sideScript: string + createSideOutDir: (dir: string) => Promise +}): CompareSideWiring => ({ + guard: trackingGuard(inputs.tracked), + isInterrupted: inputs.flag.isInterrupted, + root: repoRoot, + sideScript: inputs.sideScript, + reps: 1, + trustFile: path.join(repoRoot, "unused-in-unit-tests.json"), + credentialEnv: "SOLHEIM_API_KEY", + sideTimeoutMs: 60_000, + killGraceMs: 100, + prepareWorktree: async () => {}, + createSideOutDir: inputs.createSideOutDir, +}) + +const flowWiring = (inputs: { + flag: InterruptFlag + out: string + scratchDir: string + runSide: (side: "baseline" | "candidate") => Promise +}): ComparisonFlowWiring => ({ + out: inputs.out, + scratchDir: inputs.scratchDir, + root: repoRoot, + reps: 1, + runSide: inputs.runSide, + interruptedSignal: inputs.flag.name, + interruptKill: async () => {}, +}) + +const pins = (side: SidePins["side"], commit: string): SidePins => ({ + side, + commit, + suiteId: "coding-basic-v1", + suiteDigest: "d1".repeat(32), + settingsDigest: "d2".repeat(32), + modelIdentifier: "test-model", + graderVersion: "coding-grader-v2", + graderImageDigest: "unavailable", + runnerEnvironment: { os: "linux", nodeVersion: process.version, vscodeTarget: "test" }, +}) + +const runMetrics = (fixtureName: string, repIndex: number) => + buildRunMetrics({ + fixtureName, + repIndex, + outcome: { outcome: "correct", reason: "hiddenTestsPassed" }, + modelIdentifier: { available: true, value: "test-model" }, + }) + +const record = (side: SidePins["side"], commit: string): SideResultFile => ({ + pins: pins(side, commit), + runs: [ + { + fixtureName: "clamp-range", + fixtureVersion: "1.0.0", + repIndex: 0, + outcome: { outcome: "correct", reason: "hiddenTestsPassed" }, + trace: undefined, + metrics: runMetrics("clamp-range", 0), + }, + ], +}) + +// A side script fixture that writes one valid result file under --out and +// exits with the given code. The commit travels in the source, because the +// spawn passes no commit argument and the pins must record the approved one. +const sideScriptFixture = (commit: string, exitCode: number): string => + [ + `const fs = require("node:fs")`, + `const path = require("node:path")`, + `const read = (name) => {`, + ` const args = process.argv.slice(2)`, + ` const index = args.indexOf("--" + name)`, + ` return index >= 0 ? args[index + 1] : undefined`, + `}`, + `const pins = {`, + ` side: process.env.CODING_EVALS_SIDE,`, + ` commit: ${JSON.stringify(commit)},`, + ` suiteId: "coding-basic-v1",`, + ` suiteDigest: "d1".repeat(32),`, + ` settingsDigest: "d2".repeat(32),`, + ` modelIdentifier: "fixture-model",`, + ` graderVersion: "fixture-grader-v1",`, + ` graderImageDigest: "unavailable",`, + ` runnerEnvironment: { os: "fixture-os", nodeVersion: process.version, vscodeTarget: "fixture" },`, + `}`, + `const out = read("out")`, + `fs.mkdirSync(out, { recursive: true })`, + `fs.writeFileSync(path.join(out, "side-results.json"), JSON.stringify({`, + ` pins,`, + ` runs: [{ fixtureName: "fixture", fixtureVersion: "1.0.0", repIndex: 0, outcome: { outcome: "correct", reason: "hiddenTestsPassed" }, metrics: {} }],`, + `}))`, + `process.exit(${exitCode})`, + ].join("\n") + +describe("compare side interruption gate", () => { + it("refuses to spawn the side when the interrupt lands during the output directory creation", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "compare-side-test-")) + try { + const flag = createInterruptFlag() + const tracked: (number | undefined)[] = [] + const sideOut = path.join(dir, "candidate-out") + const wiring = sideWiring({ + flag, + tracked, + sideScript: "unused-because-no-spawn-may-happen.cjs", + // The signal lands inside the last await before the spawn: the + // window where the guard holds no tracked group and a finished + // escalation would never revisit the new child. + createSideOutDir: async () => { + flag.signal("SIGINT") + }, + }) + await assert.rejects( + runCompareSide("candidate", COMMIT, dir, wiring), + /interrupted before the candidate side started/, + ) + // No child was ever spawned or tracked, so no repetition can be + // running and nothing is left to kill. + assert.deepEqual(tracked, []) + await assert.rejects(fs.access(sideOut)) + } finally { + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) + } + }) + + it("runs the side, tracks it, and reads its result when nothing interrupts", { timeout: 60_000 }, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "compare-side-test-")) + try { + const flag = createInterruptFlag() + const tracked: (number | undefined)[] = [] + const fixture = path.join(dir, "fixture-side.cjs") + await fs.writeFile(fixture, sideScriptFixture(COMMIT, 0)) + const wiring = sideWiring({ + flag, + tracked, + sideScript: fixture, + createSideOutDir: async (sideOut) => { + await fs.mkdir(sideOut, { recursive: true }) + }, + }) + const result = await runCompareSide("baseline", COMMIT, dir, wiring) + assert.equal(result.pins.side, "baseline") + assert.equal(result.pins.commit, COMMIT) + assert.equal(result.runs.length, 1) + // The real child was tracked and left nothing tracked behind. + assert.equal(tracked.length >= 1, true) + assert.equal(tracked.at(-1), undefined) + } finally { + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) + } + }) + + it("fails the side and untracks it when the side process exits nonzero", { timeout: 60_000 }, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "compare-side-test-")) + try { + const flag = createInterruptFlag() + const tracked: (number | undefined)[] = [] + const fixture = path.join(dir, "fixture-side.cjs") + await fs.writeFile(fixture, sideScriptFixture(COMMIT, 3)) + const wiring = sideWiring({ + flag, + tracked, + sideScript: fixture, + createSideOutDir: async (sideOut) => { + await fs.mkdir(sideOut, { recursive: true }) + }, + }) + await assert.rejects( + runCompareSide("baseline", COMMIT, dir, wiring), + /baseline side failed with exit code 3/, + ) + assert.equal(tracked.at(-1), undefined) + } finally { + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) + } + }) +}) + +describe("comparison flow outcome and cleanup", () => { + // Both signals replay the throwing interrupt path: the signal lands + // mid-side and the side still ends in a failure. + const interruptedRun = async (signal: NodeJS.Signals) => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "compare-flow-test-")) + try { + const flag = createInterruptFlag() + const sides: ("baseline" | "candidate")[] = [] + const outcome = await executeComparisonFlow( + flowWiring({ + flag, + out: path.join(dir, "out"), + scratchDir: dir, + runSide: async (side) => { + sides.push(side) + if (side === "baseline") { + flag.signal(signal) + throw new Error("baseline side failed with exit code 1") + } + return record(side, COMMIT) + }, + }), + ) + return { outcome, sides, dir } + } finally { + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) + } + } + + it("reports SIGINT and finishes cleanup when a side fails under the interrupt", async () => { + const { outcome, sides, dir } = await interruptedRun("SIGINT") + assert.deepEqual(outcome, { status: "interrupted", signal: "SIGINT" }) + // The interrupted side stopped the flow before the candidate started. + assert.deepEqual(sides, ["baseline"]) + // The cleanup completed before the outcome resolved. + await assert.rejects(fs.access(dir)) + }) + + it("reports SIGTERM and finishes cleanup when a side fails under the interrupt", async () => { + const { outcome, sides, dir } = await interruptedRun("SIGTERM") + assert.deepEqual(outcome, { status: "interrupted", signal: "SIGTERM" }) + assert.deepEqual(sides, ["baseline"]) + await assert.rejects(fs.access(dir)) + }) + + it("keeps a non-interrupt failure a failure and cleans up anyway", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "compare-flow-test-")) + try { + const flag = createInterruptFlag() + await assert.rejects( + executeComparisonFlow( + flowWiring({ + flag, + out: path.join(dir, "out"), + scratchDir: dir, + runSide: async () => { + throw new Error("candidate side failed with exit code 7") + }, + }), + ), + /candidate side failed with exit code 7/, + ) + assert.equal(flag.isInterrupted(), false) + await assert.rejects(fs.access(dir)) + } finally { + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) + } + }) + + it("completes, publishes, and returns the report when both sides succeed", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "compare-flow-test-")) + try { + const out = path.join(dir, "out") + await fs.mkdir(out, { recursive: true }) + const baselineCommit = "a".repeat(40) + const candidateCommit = "c".repeat(40) + const flag = createInterruptFlag() + const sides: ("baseline" | "candidate")[] = [] + const outcome = await executeComparisonFlow( + flowWiring({ + flag, + out, + scratchDir: path.join(dir, "scratch"), + runSide: async (side) => { + sides.push(side) + return record(side, side === "baseline" ? baselineCommit : candidateCommit) + }, + }), + ) + assert.deepEqual(sides, ["baseline", "candidate"]) + assert.equal(outcome.status, "completed") + if (outcome.status !== "completed") return + assert.equal(outcome.report.pins.baseline.commit, baselineCommit) + assert.equal(outcome.report.pins.candidate.commit, candidateCommit) + assert.ok((await publishedComparisonGeneration(out)) !== undefined) + } finally { + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) + } + }) +}) diff --git a/scripts/agent-evals/coding-compare-script.test.ts b/scripts/agent-evals/coding-compare-script.test.ts index ff45865fc4..411dc20729 100644 --- a/scripts/agent-evals/coding-compare-script.test.ts +++ b/scripts/agent-evals/coding-compare-script.test.ts @@ -1,11 +1,12 @@ import assert from "node:assert/strict" -import { execFile } from "node:child_process" +import { execFile, execFileSync, spawn } from "node:child_process" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" import { fileURLToPath } from "node:url" import { promisify } from "node:util" import { describe, it } from "node:test" +import { groupAlive } from "./side-process-guard.ts" const execFileAsync = promisify(execFile) const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") @@ -126,3 +127,157 @@ describe("coding compare isolation gate", () => { }, ) }) + +const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)) + +const waitForFile = async (filePath: string, timeoutMs: number): Promise => { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if ( + await fs.access(filePath).then( + () => true, + () => false, + ) + ) + return + await delay(20) + } + throw new Error(`fixture file never appeared: ${filePath}`) +} + +// Polls until the group answers ESRCH, so the test proves the group is gone +// instead of hoping the kill landed. +const assertGroupGone = async (groupPid: number): Promise => { + const deadline = Date.now() + 10_000 + while (Date.now() < deadline) { + if (!groupAlive(groupPid)) return + await delay(50) + } + throw new Error(`worktree preparation group ${groupPid} survived the driver exit`) +} + +interface SignalRun { + exitCode: number | null + stderr: string + scratchParent: string + worktreeGroupPid: number +} + +// Spawns the real comparison driver with a PATH shim. The shim turns +// `git worktree add` into a detached sleeper that records its group id, so +// the test can aim one signal at the driver alone while a preparation child +// runs. Every other git invocation proxies to the real git, and the trust +// gates pass with the HEAD commit. The credential is a placeholder; nothing +// here contacts a provider. +const runDriverUntilSignal = async (signal: NodeJS.Signals): Promise => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-compare-signal-test-")) + const realGit = execFileSync("sh", ["-c", "command -v git"], { encoding: "utf8" }).trim() + const shimDir = path.join(dir, "bin") + await fs.mkdir(shimDir) + const gitShim = [ + "#!/bin/sh", + 'if [ "$1" = "worktree" ]; then', + ' if [ "$2" = "add" ]; then', + ' printf \'%s\\n\' "$$" > "$TMPDIR/worktree-add-pid"', + " exec sleep 30", + " fi", + ' if [ "$2" = "remove" ]; then', + " exit 0", + " fi", + "fi", + `exec '${realGit}' "$@"`, + ].join("\n") + await fs.writeFile(path.join(shimDir, "git"), gitShim, { mode: 0o755 }) + const head = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repoRoot, encoding: "utf8" }).trim() + const trustFile = path.join(dir, "trust-policy.json") + await fs.writeFile(trustFile, JSON.stringify({ version: 1, approvedCommits: [head] })) + const child = spawn( + process.execPath, + [ + "--import", + "tsx", + path.join(repoRoot, "scripts/agent-evals/coding-compare.mts"), + "--baseline", + head, + "--candidate", + head, + "--reps", + "1", + "--out", + path.join(dir, "out"), + "--trust-file", + trustFile, + ], + { + cwd: repoRoot, + env: { + PATH: `${shimDir}:${process.env["PATH"] ?? ""}`, + HOME: process.env["HOME"] ?? os.homedir(), + TMPDIR: dir, + SOLHEIM_API_KEY: "not-a-real-credential", + }, + stdio: ["ignore", "ignore", "pipe"], + }, + ) + let stderr = "" + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString() + }) + const pidFile = path.join(dir, "worktree-add-pid") + await waitForFile(pidFile, 60_000) + // One signal aimed at the driver alone. The detached preparation group + // receives nothing from this kill call. + child.kill(signal) + const exit = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve, reject) => { + const timer = setTimeout(() => { + child.kill("SIGKILL") + reject(new Error(`driver did not exit after ${signal}: ${stderr}`)) + }, 60_000) + child.once("exit", (code, exitSignal) => { + clearTimeout(timer) + resolve({ code, signal: exitSignal }) + }) + child.once("error", (error) => { + clearTimeout(timer) + reject(error) + }) + }) + const groupPid = Number(await fs.readFile(pidFile, "utf8")) + return { exitCode: exit.code, stderr, scratchParent: dir, worktreeGroupPid: groupPid } +} + +describe("compare driver signal exit", () => { + const expectSignalExit = async (signal: NodeJS.Signals, expectedCode: number): Promise => { + const run = await runDriverUntilSignal(signal) + try { + // The throwing interrupt path exits with the conventional signal + // code after the cleanup, not with an unhandled-rejection 1. + assert.equal(run.exitCode, expectedCode) + // The detached worktree preparation group did not survive. + await assertGroupGone(run.worktreeGroupPid) + // The cleanup removed the scratch generation before the exit. + const leftovers = (await fs.readdir(run.scratchParent)).filter((entry) => + entry.startsWith("coding-evals-compare-"), + ) + assert.deepEqual(leftovers, []) + } finally { + await fs.rm(run.scratchParent, { recursive: true, force: true }).catch(() => {}) + } + } + + it( + "exits 130 after cleanup when SIGINT reaches the driver during worktree preparation", + { timeout: 180_000 }, + async () => { + await expectSignalExit("SIGINT", 130) + }, + ) + + it( + "exits 143 after cleanup when SIGTERM reaches the driver during worktree preparation", + { timeout: 180_000 }, + async () => { + await expectSignalExit("SIGTERM", 143) + }, + ) +}) diff --git a/scripts/agent-evals/coding-compare.mts b/scripts/agent-evals/coding-compare.mts index e201fbc431..226dccfbc8 100644 --- a/scripts/agent-evals/coding-compare.mts +++ b/scripts/agent-evals/coding-compare.mts @@ -18,31 +18,14 @@ // SIGTERM aimed at the driver alone terminate the tracked group with a bounded // SIGTERM-then-SIGKILL escalation before the scratch cleanup runs, and an // interrupt leaves no orphan host behind. -import { execFile, spawn } from "node:child_process" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" import { fileURLToPath } from "node:url" -import { promisify } from "node:util" -import { - publishComparisonReport, - runBoundedPreparation, - sideResultFileName, - toSideResult, - validateSideResultFile, - type SideResultFile, -} from "./coding-compare.ts" -import { buildComparisonReport, renderMarkdownReport } from "./coding-report.ts" -import { - assertCommitsApproved, - assertCredentialEnvAllowed, - buildSideEnvironment, - loadTrustPolicy, - resolveCommit, -} from "./coding-trust.ts" -import { createSideProcessGuard, type SideKillOutcome } from "./side-process-guard.ts" +import { executeComparisonFlow, runBoundedPreparation, runCompareSide } from "./coding-compare.ts" +import { assertCommitsApproved, assertCredentialEnvAllowed, loadTrustPolicy, resolveCommit } from "./coding-trust.ts" +import { createInterruptFlag, createSideProcessGuard, type SideKillOutcome } from "./side-process-guard.ts" -const execFileAsync = promisify(execFile) const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") const INSTALL_TIMEOUT_MS = 10 * 60_000 @@ -52,16 +35,16 @@ const SIDE_KILL_GRACE_MS = 30_000 const DEFAULT_SIDE_TIMEOUT_MINUTES = 120 // Interrupt state. The handlers are registered before any child is spawned. -// A signal terminates the tracked child group at once; the main flow observes -// the child exit, the cleanup awaits the escalation before it removes the -// worktrees, and the driver exits with the conventional signal exit code. +// A signal arms the flag and starts the guard escalation at once; the flow +// observes the flag at its async boundaries, the cleanup awaits the +// escalation before it removes the worktrees, and the driver exits with the +// conventional signal exit code after that cleanup. const sideGuard = createSideProcessGuard(SIDE_KILL_GRACE_MS) -let interruptSignal: NodeJS.Signals | undefined +const interruptFlag = createInterruptFlag() let interruptKill: Promise | undefined -const isInterrupted = (): boolean => interruptSignal !== undefined const onInterruptSignal = (signal: NodeJS.Signals): void => { - if (interruptSignal !== undefined) return - interruptSignal = signal + if (interruptFlag.isInterrupted()) return + interruptFlag.signal(signal) interruptKill = sideGuard.interrupt() } process.on("SIGINT", () => onInterruptSignal("SIGINT")) @@ -121,97 +104,27 @@ const readArgs = (): { const prepareWorktree = async (worktree: string): Promise => { await runBoundedPreparation( sideGuard, - isInterrupted, + interruptFlag.isInterrupted, "pnpm", ["install", "--frozen-lockfile"], worktree, INSTALL_TIMEOUT_MS, ) - await runBoundedPreparation(sideGuard, isInterrupted, "pnpm", ["build"], worktree, BUILD_TIMEOUT_MS) + await runBoundedPreparation(sideGuard, interruptFlag.isInterrupted, "pnpm", ["build"], worktree, BUILD_TIMEOUT_MS) } -// Runs one trusted side against the evaluated worktree. The side script comes -// from this checkout, receives only the allowlisted environment plus the -// named credential variable, re-checks the worktree commit against the trust -// policy, and must finish inside its own wall clock. -const runSide = async ( - side: "baseline" | "candidate", - commit: string, - inputs: ReturnType, - scratchDir: string, -): Promise => { - if (isInterrupted()) throw new Error(`interrupted before the ${side} side started`) - const worktree = path.join(scratchDir, side) +// One side's full preparation: the detached worktree add of that exact +// commit, then the install and build inside it. +const prepareSideWorktree = async (commit: string, worktree: string): Promise => { await runBoundedPreparation( sideGuard, - isInterrupted, + interruptFlag.isInterrupted, "git", ["worktree", "add", "--detach", worktree, commit], root, WORKTREE_TIMEOUT_MS, ) await prepareWorktree(worktree) - if (isInterrupted()) throw new Error(`interrupted before the ${side} side started`) - const sideOut = path.join(scratchDir, `${side}-out`) - await fs.mkdir(sideOut, { recursive: true }) - const exit = await new Promise((resolve) => { - const child = spawn( - process.execPath, - [ - "--import", - "tsx", - path.join(root, "scripts/agent-evals/coding-side.mts"), - "--reps", - String(inputs.reps), - "--out", - sideOut, - "--trust-file", - inputs.trustFile, - "--credential-env", - inputs.credentialEnv, - "--evaluated-root", - worktree, - ], - { - cwd: root, - env: buildSideEnvironment(process.env, inputs.credentialEnv, side), - stdio: "inherit", - detached: true, - }, - ) - sideGuard.track(child.pid) - let timedOut = false - const timer = setTimeout(() => { - timedOut = true - try { - if (child.pid) process.kill(-child.pid, "SIGTERM") - } catch { - child.kill("SIGTERM") - } - setTimeout(() => { - try { - if (child.pid) process.kill(-child.pid, "SIGKILL") - } catch { - child.kill("SIGKILL") - } - }, SIDE_KILL_GRACE_MS).unref() - }, inputs.sideTimeoutMs) - child.once("error", () => { - clearTimeout(timer) - sideGuard.track(undefined) - resolve(1) - }) - child.once("exit", (code) => { - clearTimeout(timer) - sideGuard.track(undefined) - resolve(timedOut ? 1 : (code ?? 1)) - }) - }) - if (exit !== 0) throw new Error(`${side} side failed with exit code ${exit}`) - const parsed: unknown = JSON.parse(await fs.readFile(path.join(sideOut, sideResultFileName()), "utf8")) - // The side output must describe exactly this side and its approved commit, - // or the pair is invalid. - return validateSideResultFile(parsed, side, commit) } const inputs = readArgs() @@ -227,36 +140,39 @@ assertCommitsApproved(policy, [baselineCommit, candidateCommit]) const scratchDir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-evals-compare-")) await fs.mkdir(inputs.out, { recursive: true }) -try { - const report = buildComparisonReport({ - baseline: toSideResult(await runSide("baseline", baselineCommit, inputs, scratchDir)), - candidate: toSideResult(await runSide("candidate", candidateCommit, inputs, scratchDir)), - repetitions: inputs.reps, - }) - // The reports are the comparison's publication point. The pair is - // published as one generation with a completion marker; any failure above - // or during publication leaves no complete report generation behind. - await publishComparisonReport(inputs.out, report) - console.log( - report.compatibility.compatible - ? `comparison complete: compatible pair, report in ${inputs.out}` - : `comparison complete: INCOMPATIBLE pair (${report.compatibility.differences.join(", ")}), report in ${inputs.out}`, - ) -} finally { - // An interrupt escalation must finish before the worktrees go away, so - // no child is still writing while its tree is deleted. - await interruptKill - await execFileAsync("git", ["worktree", "remove", "--force", path.join(scratchDir, "baseline")], { - cwd: root, - timeout: 30_000, - }).catch(() => {}) - await execFileAsync("git", ["worktree", "remove", "--force", path.join(scratchDir, "candidate")], { - cwd: root, - timeout: 30_000, - }).catch(() => {}) - await fs.rm(scratchDir, { recursive: true, force: true }).catch(() => {}) -} -// An interrupt terminated the side group above, and the cleanup in the -// finally block has completed. Exit with the conventional signal code. -if (interruptSignal === "SIGINT") process.exit(130) -if (interruptSignal === "SIGTERM") process.exit(143) +// Both sides run inside the flow. The flow cleanup finishes the interrupt +// escalation, removes the worktrees, and removes the scratch directory +// before any exit, so no child is still writing while its tree is deleted. +const outcome = await executeComparisonFlow({ + out: inputs.out, + scratchDir, + root, + reps: inputs.reps, + runSide: (side) => + runCompareSide(side, side === "baseline" ? baselineCommit : candidateCommit, scratchDir, { + guard: sideGuard, + isInterrupted: interruptFlag.isInterrupted, + root, + sideScript: path.join(root, "scripts/agent-evals/coding-side.mts"), + reps: inputs.reps, + trustFile: inputs.trustFile, + credentialEnv: inputs.credentialEnv, + sideTimeoutMs: inputs.sideTimeoutMs, + killGraceMs: SIDE_KILL_GRACE_MS, + prepareWorktree: prepareSideWorktree, + createSideOutDir: async (dir) => { + await fs.mkdir(dir, { recursive: true }) + }, + }), + interruptedSignal: interruptFlag.name, + interruptKill: () => interruptKill ?? Promise.resolve(), +}) +// An interrupt stopped the comparison and the flow cleanup has completed. +// Exit with the conventional signal code on the normal and the throwing +// path alike. +if (outcome.status === "interrupted") process.exit(interruptFlag.exitCode()) +console.log( + outcome.report.compatibility.compatible + ? `comparison complete: compatible pair, report in ${inputs.out}` + : `comparison complete: INCOMPATIBLE pair (${outcome.report.compatibility.differences.join(", ")}), report in ${inputs.out}`, +) diff --git a/scripts/agent-evals/coding-compare.ts b/scripts/agent-evals/coding-compare.ts index 921b901283..2f8c278d37 100644 --- a/scripts/agent-evals/coding-compare.ts +++ b/scripts/agent-evals/coding-compare.ts @@ -1,11 +1,12 @@ -import { spawn } from "node:child_process" +import { execFile, spawn } from "node:child_process" import fs from "node:fs/promises" import path from "node:path" +import { promisify } from "node:util" import type { SidePins } from "./coding-pins.ts" import { buildComparisonReport, renderMarkdownReport, type ComparisonReport, type SideResult } from "./coding-report.ts" import { DEFAULT_REPETITIONS } from "./coding-stats.ts" import type { CodingRunRecord } from "./coding-run.ts" -import { buildWorktreeEnvironment } from "./coding-trust.ts" +import { buildSideEnvironment, buildWorktreeEnvironment } from "./coding-trust.ts" import type { SideProcessGuard } from "./side-process-guard.ts" import { writeFileAtomic, writeJsonAtomic } from "./fs-atomic.ts" @@ -108,6 +109,184 @@ export const runBoundedPreparation = async ( }) } +// The worktree removals in the flow cleanup are best effort and bounded. +const WORKTREE_REMOVE_TIMEOUT_MS = 30_000 + +// Everything one side execution needs from the driver process. The guard and +// the interrupt view are shared with the driver's signal handler; the two +// async steps are seams, so tests can place a signal inside the last boundary +// before the spawn without a repository, a toolchain, or a credential. +export interface CompareSideWiring { + guard: SideProcessGuard + isInterrupted: () => boolean + // Trusted checkout root: the spawn cwd of the side process. + root: string + // The side entry script of this checkout. Production points at + // scripts/agent-evals/coding-side.mts; tests substitute a fixture. + sideScript: string + reps: number + trustFile: string + credentialEnv: string + sideTimeoutMs: number + killGraceMs: number + // Prepares the side's worktree: the detached worktree add of this exact + // commit, the frozen dependency install, and the build of that commit. + prepareWorktree: (commit: string, worktree: string) => Promise + // Creates the side output directory. This is the last async boundary + // before the spawn; tests arm the interrupt inside it. + createSideOutDir: (dir: string) => Promise +} + +// Runs one trusted side against the evaluated worktree. The side script comes +// from this checkout, receives only the allowlisted environment plus the named +// credential variable, re-checks the worktree commit against the trust +// policy, and must finish inside its own wall clock. +export const runCompareSide = async ( + side: "baseline" | "candidate", + commit: string, + scratchDir: string, + wiring: CompareSideWiring, +): Promise => { + if (wiring.isInterrupted()) throw new Error(`interrupted before the ${side} side started`) + const worktree = path.join(scratchDir, side) + await wiring.prepareWorktree(commit, worktree) + if (wiring.isInterrupted()) throw new Error(`interrupted before the ${side} side started`) + const sideOut = path.join(scratchDir, `${side}-out`) + // The directory creation is the last await before the spawn. A signal + // during it arms the guard while nothing is tracked, and the guard never + // revisits a finished escalation, so the side must be refused here rather + // than started into an already-interrupted driver. + await wiring.createSideOutDir(sideOut) + if (wiring.isInterrupted()) throw new Error(`interrupted before the ${side} side started`) + // From this refusal check to the guard track there is no await, so a + // signal cannot slip between them and miss the new child. + const exit = await new Promise((resolve) => { + const child = spawn( + process.execPath, + [ + "--import", + "tsx", + wiring.sideScript, + "--reps", + String(wiring.reps), + "--out", + sideOut, + "--trust-file", + wiring.trustFile, + "--credential-env", + wiring.credentialEnv, + "--evaluated-root", + worktree, + ], + { + cwd: wiring.root, + env: buildSideEnvironment(process.env, wiring.credentialEnv, side), + stdio: "inherit", + detached: true, + }, + ) + wiring.guard.track(child.pid) + let timedOut = false + const timer = setTimeout(() => { + timedOut = true + try { + if (child.pid) process.kill(-child.pid, "SIGTERM") + } catch { + child.kill("SIGTERM") + } + setTimeout(() => { + try { + if (child.pid) process.kill(-child.pid, "SIGKILL") + } catch { + child.kill("SIGKILL") + } + }, wiring.killGraceMs).unref() + }, wiring.sideTimeoutMs) + child.once("error", () => { + clearTimeout(timer) + wiring.guard.track(undefined) + resolve(1) + }) + child.once("exit", (code) => { + clearTimeout(timer) + wiring.guard.track(undefined) + resolve(timedOut ? 1 : (code ?? 1)) + }) + }) + if (exit !== 0) throw new Error(`${side} side failed with exit code ${exit}`) + const parsed: unknown = JSON.parse(await fs.readFile(path.join(sideOut, sideResultFileName()), "utf8")) + // The side output must describe exactly this side and its approved commit, + // or the pair is invalid. + return validateSideResultFile(parsed, side, commit) +} + +// The comparison body after the trust gates: both sides, one report +// publication, and the cleanup that must finish before the driver exits. +export interface ComparisonFlowWiring { + out: string + scratchDir: string + root: string + reps: number + // Runs one side to its validated result. The driver wires its own side + // runner; tests substitute synthetic sides. + runSide: (side: "baseline" | "candidate") => Promise + // The observed terminal signal, or undefined. The interrupt outcome and + // the final exit code read this one view. + interruptedSignal: () => NodeJS.Signals | undefined + // The guard's kill escalation, created when the signal arrived. Cleanup + // awaits it so no tracked child outlives the removed worktrees. + interruptKill: () => Promise +} + +export type ComparisonFlowOutcome = + { status: "completed"; report: ComparisonReport } | { status: "interrupted"; signal: "SIGINT" | "SIGTERM" } + +// Runs the comparison and resolves to its outcome after the cleanup. An +// interrupt resolves to its signal no matter how the body ended, so the +// caller exits with the conventional code on the normal and the throwing +// path alike; any other failure stays a thrown failure. +export const executeComparisonFlow = async (wiring: ComparisonFlowWiring): Promise => { + let completed: ComparisonReport | undefined + let failure: unknown + try { + const report = buildComparisonReport({ + baseline: toSideResult(await wiring.runSide("baseline")), + candidate: toSideResult(await wiring.runSide("candidate")), + repetitions: wiring.reps, + }) + // The reports are the comparison's publication point. The pair is + // published as one generation with a completion marker; any failure + // above or during publication leaves no complete report generation + // behind. + await publishComparisonReport(wiring.out, report) + completed = report + } catch (error) { + // Deferred, not swallowed: the cleanup below runs first, then the + // failure rethrows unless the run was interrupted. + failure = error + } finally { + // An interrupt escalation must finish before the worktrees go away, + // so no child is still writing while its tree is deleted. + await wiring.interruptKill() + await execFileAsync("git", ["worktree", "remove", "--force", path.join(wiring.scratchDir, "baseline")], { + cwd: wiring.root, + timeout: WORKTREE_REMOVE_TIMEOUT_MS, + }).catch(() => {}) + await execFileAsync("git", ["worktree", "remove", "--force", path.join(wiring.scratchDir, "candidate")], { + cwd: wiring.root, + timeout: WORKTREE_REMOVE_TIMEOUT_MS, + }).catch(() => {}) + await fs.rm(wiring.scratchDir, { recursive: true, force: true }).catch(() => {}) + } + const signal = wiring.interruptedSignal() + if (signal === "SIGINT" || signal === "SIGTERM") return { status: "interrupted", signal } + if (failure !== undefined) throw failure + if (completed === undefined) throw new Error("comparison ended without a report or a failure") + return { status: "completed", report: completed } +} + +const execFileAsync = promisify(execFile) + // Serializes a side result for artifact storage. Full run records travel so // each side's traces and grading verdicts stay inspectable. export interface SideResultFile { From 9aa015af9b72db506714c3a7cb6a73f7d53d280f Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Tue, 27 Oct 2026 00:33:52 +0000 Subject: [PATCH 8/9] fix(agent-evals): detect dirty checkouts despite Git settings that hide changes --- scripts/agent-evals/coding-trust.test.ts | 171 +++++++++++++++++++++-- scripts/agent-evals/coding-trust.ts | 15 +- 2 files changed, 174 insertions(+), 12 deletions(-) diff --git a/scripts/agent-evals/coding-trust.test.ts b/scripts/agent-evals/coding-trust.test.ts index 9081c034a9..4dae732ec7 100644 --- a/scripts/agent-evals/coding-trust.test.ts +++ b/scripts/agent-evals/coding-trust.test.ts @@ -21,11 +21,25 @@ const execFileAsync = promisify(execFile) const sha = (c: string): string => c.repeat(40) // One throwaway repository with one empty commit. It stands in for an -// evaluated root; no code from it ever runs in these checks. -const makeRepo = async (dir: string): Promise => { - const repo = path.join(dir, "repo") +// evaluated root; no code from it ever runs in these checks. The commit +// identity lives in the repository's local config only. +const makeRepo = async (dir: string, name = "repo"): Promise => { + const repo = path.join(dir, name) await fs.mkdir(repo) await execFileAsync("git", ["init", "-q"], { cwd: repo }) + // The operator's git config may force signed commits, and the + // throwaway fixture has no key or tty. + await execFileAsync("git", ["config", "commit.gpgsign", "false"], { cwd: repo }) + await execFileAsync("git", ["config", "user.email", "fixture@example.test"], { cwd: repo }) + await execFileAsync("git", ["config", "user.name", "fixture"], { cwd: repo }) + await execFileAsync("git", ["commit", "--allow-empty", "-q", "-m", "fixture"], { cwd: repo }) + return repo +} + +const commitAll = async (repo: string, message: string): Promise => { + await execFileAsync("git", ["add", "-A"], { cwd: repo }) + // Submodule checkouts are clones without the fixture's local config, so + // the flags travel with the command instead of any stored config. await execFileAsync( "git", [ @@ -33,19 +47,15 @@ const makeRepo = async (dir: string): Promise => { "user.email=fixture@example.test", "-c", "user.name=fixture", - // The operator's git config may force signed commits, and the - // throwaway fixture has no key or tty. "-c", "commit.gpgsign=false", "commit", - "--allow-empty", "-q", "-m", - "fixture", + message, ], { cwd: repo }, ) - return repo } const writePolicy = async (dir: string, content: unknown): Promise => { @@ -54,6 +64,21 @@ const writePolicy = async (dir: string, content: unknown): Promise => { return file } +// A superproject with one clean submodule checked out from a throwaway local +// origin. Nothing from it runs in these checks. `name` keeps the on-disk +// repositories of separate cases apart inside one temporary directory. +const makeSuperWithSubmodule = async (dir: string, name: string): Promise<{ superRepo: string; sub: string }> => { + const origin = await makeRepo(dir, `${name}-origin`) + await fs.writeFile(path.join(origin, "lib.txt"), "library\n") + await commitAll(origin, "library") + const superRepo = await makeRepo(dir, `${name}-super`) + await execFileAsync("git", ["-c", "protocol.file.allow=always", "submodule", "add", "-q", origin, "sub"], { + cwd: superRepo, + }) + await commitAll(superRepo, "submodule") + return { superRepo, sub: path.join(superRepo, "sub") } +} + describe("trust policy", () => { it("loads a policy of full commit SHAs", async () => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) @@ -239,6 +264,136 @@ describe("evaluated tree cleanliness gate", () => { await fs.rm(dir, { recursive: true, force: true }) } }) + + it("rejects untracked files when the caller config tells status to hide them", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) + try { + // status.showUntrackedFiles=no makes a plain status report an + // empty result; the gate must not trust that setting. + const cases = [ + { + name: "root", + makeDirty: async (repo: string) => { + await fs.writeFile(path.join(repo, "untracked.txt"), "x") + }, + }, + { + name: "nested", + makeDirty: async (repo: string) => { + await fs.mkdir(path.join(repo, "nested")) + await fs.writeFile(path.join(repo, "nested", "untracked.txt"), "x") + }, + }, + ] + for (const [index, { name, makeDirty }] of cases.entries()) { + const repo = await makeRepo(dir, `hide-untracked-${index}`) + await execFileAsync("git", ["config", "status.showUntrackedFiles", "no"], { cwd: repo }) + await makeDirty(repo) + await assert.rejects(assertEvaluatedTreeClean(repo), (error: unknown) => { + assert.ok(error instanceof DirtyEvaluatedTreeError) + assert.ok( + error.entries.some((entry) => entry.includes("untracked.txt")), + name, + ) + return true + }) + } + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("rejects staged changes under the same hiding config", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) + try { + const cases = [ + { + name: "staged new file", + stage: async (repo: string) => { + await fs.writeFile(path.join(repo, "staged.txt"), "x") + await execFileAsync("git", ["add", "staged.txt"], { cwd: repo }) + }, + }, + { + name: "staged modification", + stage: async (repo: string) => { + await fs.writeFile(path.join(repo, "tracked.txt"), "approved\n") + await commitAll(repo, "tracked") + await fs.writeFile(path.join(repo, "tracked.txt"), "changed\n") + await execFileAsync("git", ["add", "tracked.txt"], { cwd: repo }) + }, + }, + ] + for (const [index, { name, stage }] of cases.entries()) { + const repo = await makeRepo(dir, `staged-${index}`) + await execFileAsync("git", ["config", "status.showUntrackedFiles", "no"], { cwd: repo }) + await stage(repo) + await assert.rejects(assertEvaluatedTreeClean(repo), (error: unknown) => { + assert.ok(error instanceof DirtyEvaluatedTreeError, name) + return true + }) + } + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("accepts a clean superproject whose submodule the caller config ignores", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) + try { + const { superRepo } = await makeSuperWithSubmodule(dir, "clean") + // submodule..ignore=all must not turn the explicit + // --ignore-submodules=none scan into a false rejection. + await execFileAsync("git", ["config", "submodule.sub.ignore", "all"], { cwd: superRepo }) + await assert.doesNotReject(assertEvaluatedTreeClean(superRepo)) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("rejects dirty submodules the caller config ignores", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-trust-test-")) + try { + const cases = [ + { + name: "modified submodule content", + makeDirty: async (sub: string) => { + await fs.writeFile(path.join(sub, "lib.txt"), "changed\n") + }, + }, + { + name: "untracked submodule content", + makeDirty: async (sub: string) => { + await fs.writeFile(path.join(sub, "extra.txt"), "x") + }, + }, + { + name: "submodule checked out past the approved commit", + makeDirty: async (sub: string) => { + await fs.writeFile(path.join(sub, "lib.txt"), "advanced\n") + await commitAll(sub, "advance") + }, + }, + ] + for (const [index, { name, makeDirty }] of cases.entries()) { + const { superRepo, sub } = await makeSuperWithSubmodule(dir, `dirty-${index}`) + // ignore=all hides every submodule change from a plain + // status, so only the gate's explicit flag sees the dirt. + await execFileAsync("git", ["config", "submodule.sub.ignore", "all"], { cwd: superRepo }) + await makeDirty(sub) + await assert.rejects(assertEvaluatedTreeClean(superRepo), (error: unknown) => { + assert.ok(error instanceof DirtyEvaluatedTreeError) + assert.ok( + error.entries.some((entry) => entry.includes("sub")), + name, + ) + return true + }) + } + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) }) describe("worktree process environment", () => { diff --git a/scripts/agent-evals/coding-trust.ts b/scripts/agent-evals/coding-trust.ts index 1b797c68e0..463c8126a1 100644 --- a/scripts/agent-evals/coding-trust.ts +++ b/scripts/agent-evals/coding-trust.ts @@ -86,10 +86,17 @@ export class DirtyEvaluatedTreeError extends Error { export async function assertEvaluatedTreeClean(root: string): Promise { const { execFile } = await import("node:child_process") const { promisify } = await import("node:util") - const { stdout } = await promisify(execFile)("git", ["status", "--porcelain"], { - cwd: root, - timeout: 30_000, - }) + // The flags override the caller's git configuration: status.showUntrackedFiles + // would otherwise hide untracked files and submodule..ignore would + // otherwise hide submodule dirtiness, so a modified checkout could pass. + const { stdout } = await promisify(execFile)( + "git", + ["status", "--porcelain", "--untracked-files=all", "--ignore-submodules=none"], + { + cwd: root, + timeout: 30_000, + }, + ) const entries = stdout .split("\n") .map((line) => line.trim()) From 7da7344da5c444357400aa5fa50fd3183f35c002 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Tue, 27 Oct 2026 01:38:52 +0000 Subject: [PATCH 9/9] fix(agent-evals): isolate side builds and invalidate stale results before artifacts --- scripts/agent-evals/README.md | 10 +- .../agent-evals/coding-compare-driver.test.ts | 19 +- .../agent-evals/coding-compare-script.test.ts | 235 +++++++++---- scripts/agent-evals/coding-compare.mts | 50 ++- scripts/agent-evals/coding-compare.ts | 28 +- scripts/agent-evals/coding-report.test.ts | 54 +++ scripts/agent-evals/coding-side.mts | 222 ++++++++---- scripts/agent-evals/coding-side.test.ts | 331 +++++++++++++++--- scripts/agent-evals/coding-trust.ts | 11 +- 9 files changed, 723 insertions(+), 237 deletions(-) diff --git a/scripts/agent-evals/README.md b/scripts/agent-evals/README.md index c337ad7716..03b0a008de 100644 --- a/scripts/agent-evals/README.md +++ b/scripts/agent-evals/README.md @@ -116,7 +116,7 @@ release process: a maintainer runs it locally or on demand. | `coding-stats.ts` | Three repetitions, success rates with Wilson intervals, paired differences. | | `coding-report.ts` | The self-contained JSON report and the Markdown renderer. | | `coding-compare.ts` | The comparison orchestration and the side result shape. | -| `coding-side.mts` | One comparison side: all fixtures times N repetitions on the evaluated root, graded, with pinned inputs. | +| `coding-side.mts` | One comparison side: rebuilds the evaluated tree, then all fixtures times N repetitions on it, graded, with pinned inputs. | | `coding-compare.mts` | Two-commit comparison driver. Writes `comparison-report.json` and `comparison-report.md`. | | `coding-trust.ts` | The comparison trust policy: approved full commit SHAs, resolution, the approval gate, and the side environment allowlist. | | `fs-atomic.ts` | Atomic file publication through same-directory temp files and renames. | @@ -149,17 +149,17 @@ A comparison runs the extension code of two commits with the provider credential { "version": 1, "approvedCommits": [""] } ``` -The comparison driver, fixtures, grader, and report builder all run from the trusted checkout that invokes `coding-evals:compare`. Only the extension host loads an evaluated worktree. The side process receives only the allowlisted environment plus the one named credential variable. Each side installs dependencies from its own worktree's frozen lockfile and builds its own exact commit, so each side runs its own compiled extension. Each side has its own wall clock (`--side-timeout-minutes`, default 120). +The comparison driver, fixtures, grader, and report builder all run from the trusted checkout that invokes `coding-evals:compare`. Only the extension host loads an evaluated worktree. The side process receives only the allowlisted environment plus the one named credential variable. The driver creates each side's fresh detached worktree of the approved commit. Each side then constructs its own fresh detached worktree of that same commit, installs the frozen lockfile dependencies and builds inside it, and launches every host against it, so each side runs only its own compiled extension of the approved commit. The install and the build consume the side's own wall clock (`--side-timeout-minutes`, default 120). -The side script runs the same gate on its own inputs. It requires `--trust-file`, resolves the full commit of its `--evaluated-root`, and rejects an unapproved or unreadable commit before it creates a host or reads the credential. It also rejects an evaluated root whose tracked or untracked files differ from that commit, because the approval describes the commit and not the working tree. The check has one trust limit: ignored files, such as dependency installs and build outputs, stay outside commit identity and outside the check, so the approval never attests them. A recorded comparison closes that gap by loading only a fresh detached worktree built from the approved commit. The `--credential-env` name must come from the supported provider variable allowlist; an arbitrary name can never forward an unrelated parent secret. Worktree install and build children also receive only an allowlisted environment, so the driver's secrets never reach them. +The side script runs the same gate on its own inputs. It requires `--trust-file`, resolves the full commit of its `--evaluated-root`, and rejects an unapproved or unreadable commit before it creates a host or reads the credential. It never executes a caller-prepared tree. After the gate it constructs a fresh detached worktree of the approved commit in private scratch space, verifies that the worktree resolves to the approved commit and holds no extra content, and then installs the dependencies from the frozen lockfile and builds inside that worktree with credential-free, allowlisted children. The host loads only build output this side produced from the approved commit, so dirty files, untracked files, ignored dependencies, stale build output, and symlinks in the caller's tree can never execute. The side removes its worktree on completion, on failure, and on interruption. The `--credential-env` name must come from the supported provider variable allowlist; an arbitrary name can never forward an unrelated parent secret. Worktree install and build children also receive only an allowlisted environment, so the driver's secrets never reach them. -Every driver child, preparation steps and the side alike, runs detached in its own process group and is tracked by one guard. A SIGINT or SIGTERM that reaches only the driver terminates the tracked group through a bounded SIGTERM-then-SIGKILL escalation, and the worktree cleanup waits for that escalation before it removes any tree. +Every driver child, the side included, runs detached in its own process group and is tracked by one guard. The driver's guard tracks the side; the side's guard tracks its own worktree, install, and build. A SIGINT or SIGTERM that reaches only the driver terminates the side group through a bounded SIGTERM-then-SIGKILL escalation. The side starts the same escalation for its preparation children and waits for it before it removes its worktree or exits. The driver's worktree cleanup also waits for the driver's escalation before it removes any tree. ### Output publication contract Artifacts are published atomically: content lands in a same-directory temp file and is renamed into place. A reader never sees a partial file. -A side run invalidates its output directory first. Before the first repetition executes, the side removes any previous `side-results.json` and the whole `artifacts` tree, so stale results from an earlier rerun can never survive. `side-results.json` is the completion marker: it appears only after every repetition of the current invocation finished, and consumers must require it and treat its absence as a failed side. A side rerun that cannot remove a stale artifact fails before any repetition runs. +A side run invalidates its output directory first. Before the first repetition executes, the side removes any previous `side-results.json` and then the whole `artifacts` tree, so stale results from an earlier rerun can never survive. The authoritative result goes first, so an error or an interruption during invalidation leaves the generation incomplete, and consumers fail closed until the new atomic publication. `side-results.json` is the completion marker: it appears only after every repetition of the current invocation finished, and consumers must require it and treat its absence as a failed side. A side rerun that cannot remove a stale artifact fails before any repetition runs. The side process arms one flag when it receives SIGINT or SIGTERM. The flag starts no further repetition and blocks the completion marker, so an interrupted side exits with the conventional signal code and publishes nothing. diff --git a/scripts/agent-evals/coding-compare-driver.test.ts b/scripts/agent-evals/coding-compare-driver.test.ts index fcf7a1f28d..5f05e4ba39 100644 --- a/scripts/agent-evals/coding-compare-driver.test.ts +++ b/scripts/agent-evals/coding-compare-driver.test.ts @@ -235,26 +235,33 @@ describe("comparison flow outcome and cleanup", () => { }, }), ) - return { outcome, sides, dir } + // Observed before this helper's finally: a removal here would + // mask a flow cleanup that left the scratch directory behind. + const scratchDirExists = await fs.access(dir).then( + () => true, + () => false, + ) + return { outcome, sides, scratchDirExists } } finally { await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) } } it("reports SIGINT and finishes cleanup when a side fails under the interrupt", async () => { - const { outcome, sides, dir } = await interruptedRun("SIGINT") + const { outcome, sides, scratchDirExists } = await interruptedRun("SIGINT") assert.deepEqual(outcome, { status: "interrupted", signal: "SIGINT" }) // The interrupted side stopped the flow before the candidate started. assert.deepEqual(sides, ["baseline"]) - // The cleanup completed before the outcome resolved. - await assert.rejects(fs.access(dir)) + // The observation happened before the helper's own teardown, so a + // false value proves the flow's cleanup removed the directory. + assert.equal(scratchDirExists, false) }) it("reports SIGTERM and finishes cleanup when a side fails under the interrupt", async () => { - const { outcome, sides, dir } = await interruptedRun("SIGTERM") + const { outcome, sides, scratchDirExists } = await interruptedRun("SIGTERM") assert.deepEqual(outcome, { status: "interrupted", signal: "SIGTERM" }) assert.deepEqual(sides, ["baseline"]) - await assert.rejects(fs.access(dir)) + assert.equal(scratchDirExists, false) }) it("keeps a non-interrupt failure a failure and cleans up anyway", async () => { diff --git a/scripts/agent-evals/coding-compare-script.test.ts b/scripts/agent-evals/coding-compare-script.test.ts index 411dc20729..6f3217b6d5 100644 --- a/scripts/agent-evals/coding-compare-script.test.ts +++ b/scripts/agent-evals/coding-compare-script.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict" -import { execFile, execFileSync, spawn } from "node:child_process" +import { execFile, execFileSync, spawn, type ChildProcess } from "node:child_process" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" @@ -163,87 +163,149 @@ interface SignalRun { worktreeGroupPid: number } +interface DriverRunOptions { + // Test seam: bounds the wait for the preparation group's pid file. + pidFileTimeoutMs?: number + // Test seam: bounds the wait for the driver exit after the signal. + exitTimeoutMs?: number + // Test seam: throws after the preparation group recorded itself, so the + // failure cleanup runs with a live driver and a live sleeper group. + failAfterPidFile?: boolean + // Test seam: reports what the failure cleanup observed, so a test can + // prove the group kill and the directory removal without holding state + // the helper already removed. + onFailureCleanup?: (observed: { scratchParent: string; groupPid: number | undefined }) => void +} + +// Reads the group id the shim recorded. Only a plausible group id counts: +// a missing, negative, zero, or tiny value would aim the kill at unrelated +// process groups, so the cleanup skips it instead. +const recordedGroupPid = async (pidFile: string): Promise => { + const raw = await fs.readFile(pidFile, "utf8").catch(() => undefined) + if (raw === undefined) return undefined + const pid = Number(raw.trim()) + return Number.isInteger(pid) && pid > 1 ? pid : undefined +} + +// Terminates the spawned driver and waits for its exit: SIGTERM first, a +// bounded wait, then SIGKILL. The wait proves the process is gone before +// the failure path removes the temporary directory. +const terminateDriver = async (child: ChildProcess): Promise => { + if (child.exitCode !== null || child.signalCode !== null) return + await new Promise((resolve) => { + const timer = setTimeout(() => child.kill("SIGKILL"), 10_000) + timer.unref() + child.once("exit", () => { + clearTimeout(timer) + resolve() + }) + if (!child.kill("SIGTERM")) resolve() + }) +} + // Spawns the real comparison driver with a PATH shim. The shim turns // `git worktree add` into a detached sleeper that records its group id, so // the test can aim one signal at the driver alone while a preparation child // runs. Every other git invocation proxies to the real git, and the trust // gates pass with the HEAD commit. The credential is a placeholder; nothing // here contacts a provider. -const runDriverUntilSignal = async (signal: NodeJS.Signals): Promise => { +const runDriverUntilSignal = async (signal: NodeJS.Signals, options: DriverRunOptions = {}): Promise => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-compare-signal-test-")) - const realGit = execFileSync("sh", ["-c", "command -v git"], { encoding: "utf8" }).trim() - const shimDir = path.join(dir, "bin") - await fs.mkdir(shimDir) - const gitShim = [ - "#!/bin/sh", - 'if [ "$1" = "worktree" ]; then', - ' if [ "$2" = "add" ]; then', - ' printf \'%s\\n\' "$$" > "$TMPDIR/worktree-add-pid"', - " exec sleep 30", - " fi", - ' if [ "$2" = "remove" ]; then', - " exit 0", - " fi", - "fi", - `exec '${realGit}' "$@"`, - ].join("\n") - await fs.writeFile(path.join(shimDir, "git"), gitShim, { mode: 0o755 }) - const head = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repoRoot, encoding: "utf8" }).trim() - const trustFile = path.join(dir, "trust-policy.json") - await fs.writeFile(trustFile, JSON.stringify({ version: 1, approvedCommits: [head] })) - const child = spawn( - process.execPath, - [ - "--import", - "tsx", - path.join(repoRoot, "scripts/agent-evals/coding-compare.mts"), - "--baseline", - head, - "--candidate", - head, - "--reps", - "1", - "--out", - path.join(dir, "out"), - "--trust-file", - trustFile, - ], - { - cwd: repoRoot, - env: { - PATH: `${shimDir}:${process.env["PATH"] ?? ""}`, - HOME: process.env["HOME"] ?? os.homedir(), - TMPDIR: dir, - SOLHEIM_API_KEY: "not-a-real-credential", - }, - stdio: ["ignore", "ignore", "pipe"], - }, - ) - let stderr = "" - child.stderr?.on("data", (chunk: Buffer) => { - stderr += chunk.toString() - }) const pidFile = path.join(dir, "worktree-add-pid") - await waitForFile(pidFile, 60_000) - // One signal aimed at the driver alone. The detached preparation group - // receives nothing from this kill call. - child.kill(signal) - const exit = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve, reject) => { - const timer = setTimeout(() => { - child.kill("SIGKILL") - reject(new Error(`driver did not exit after ${signal}: ${stderr}`)) - }, 60_000) - child.once("exit", (code, exitSignal) => { - clearTimeout(timer) - resolve({ code, signal: exitSignal }) + let child: ChildProcess | undefined + try { + const realGit = execFileSync("sh", ["-c", "command -v git"], { encoding: "utf8" }).trim() + const shimDir = path.join(dir, "bin") + await fs.mkdir(shimDir) + const gitShim = [ + "#!/bin/sh", + 'if [ "$1" = "worktree" ]; then', + ' if [ "$2" = "add" ]; then', + ' printf \'%s\\n\' "$$" > "$TMPDIR/worktree-add-pid"', + " exec sleep 30", + " fi", + ' if [ "$2" = "remove" ]; then', + " exit 0", + " fi", + "fi", + `exec '${realGit}' "$@"`, + ].join("\n") + await fs.writeFile(path.join(shimDir, "git"), gitShim, { mode: 0o755 }) + const head = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repoRoot, encoding: "utf8" }).trim() + const trustFile = path.join(dir, "trust-policy.json") + await fs.writeFile(trustFile, JSON.stringify({ version: 1, approvedCommits: [head] })) + child = spawn( + process.execPath, + [ + "--import", + "tsx", + path.join(repoRoot, "scripts/agent-evals/coding-compare.mts"), + "--baseline", + head, + "--candidate", + head, + "--reps", + "1", + "--out", + path.join(dir, "out"), + "--trust-file", + trustFile, + ], + { + cwd: repoRoot, + env: { + PATH: `${shimDir}:${process.env["PATH"] ?? ""}`, + HOME: process.env["HOME"] ?? os.homedir(), + TMPDIR: dir, + SOLHEIM_API_KEY: "not-a-real-credential", + }, + stdio: ["ignore", "ignore", "pipe"], + }, + ) + let stderr = "" + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString() }) - child.once("error", (error) => { - clearTimeout(timer) - reject(error) + await waitForFile(pidFile, options.pidFileTimeoutMs ?? 60_000) + if (options.failAfterPidFile) throw new Error("injected failure after the preparation group recorded itself") + // One signal aimed at the driver alone. The detached preparation group + // receives nothing from this kill call. + child.kill(signal) + const exit = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve, reject) => { + const timer = setTimeout(() => { + child?.kill("SIGKILL") + reject(new Error(`driver did not exit after ${signal}: ${stderr}`)) + }, options.exitTimeoutMs ?? 60_000) + child?.once("exit", (code, exitSignal) => { + clearTimeout(timer) + resolve({ code, signal: exitSignal }) + }) + child?.once("error", (error) => { + clearTimeout(timer) + reject(error) + }) }) - }) - const groupPid = Number(await fs.readFile(pidFile, "utf8")) - return { exitCode: exit.code, stderr, scratchParent: dir, worktreeGroupPid: groupPid } + const groupPid = Number(await fs.readFile(pidFile, "utf8")) + return { exitCode: exit.code, stderr, scratchParent: dir, worktreeGroupPid: groupPid } + } catch (error) { + // A failure here used to leak both the detached preparation group and + // the temporary directory. The recorded group dies first, but only + // when the shim discovered it: an unknown or implausible pid must + // never aim a negative-pid kill at unrelated groups. The driver is + // then terminated and waited for, and the directory is removed. + const groupPid = await recordedGroupPid(pidFile) + if (groupPid !== undefined) { + try { + process.kill(-groupPid, "SIGKILL") + } catch { + /* The group already exited. */ + } + } + if (child) await terminateDriver(child) + options.onFailureCleanup?.({ scratchParent: dir, groupPid }) + await fs.rm(dir, { recursive: true, force: true }).catch(() => {}) + throw error + } } describe("compare driver signal exit", () => { @@ -280,4 +342,33 @@ describe("compare driver signal exit", () => { await expectSignalExit("SIGTERM", 143) }, ) + + it( + "kills the preparation group and removes the scratch directory when a run fails mid-wait", + { timeout: 180_000 }, + async () => { + let observed: { scratchParent: string; groupPid: number | undefined } | undefined + await assert.rejects( + runDriverUntilSignal("SIGINT", { + failAfterPidFile: true, + onFailureCleanup: (info) => { + observed = info + }, + }), + /injected failure after the preparation group recorded itself/, + ) + assert.ok(observed) + // The shim had discovered its group, so the cleanup had a real + // group id and the sleeper cannot have survived. + assert.equal(typeof observed.groupPid, "number") + await assertGroupGone(observed.groupPid as number) + // The driver was terminated and waited for before the scratch + // directory was removed. + const scratchParentExists = await fs.access(observed.scratchParent).then( + () => true, + () => false, + ) + assert.equal(scratchParentExists, false) + }, + ) }) diff --git a/scripts/agent-evals/coding-compare.mts b/scripts/agent-evals/coding-compare.mts index 226dccfbc8..ccb01f450f 100644 --- a/scripts/agent-evals/coding-compare.mts +++ b/scripts/agent-evals/coding-compare.mts @@ -1,8 +1,10 @@ // Paired comparison driver. Verifies both commits against an explicit trust -// policy, checks out each approved commit into a temporary worktree, prepares -// pinned dependencies and builds that exact commit, runs one trusted coding -// side per commit against its worktree with identical repetition counts, then -// writes the JSON and Markdown comparison reports. +// policy, checks out each approved commit into a fresh detached temporary +// worktree, and runs one trusted coding side per commit with identical +// repetition counts. Each side then constructs its own fresh detached +// worktree of that exact commit, installs the frozen dependencies, and +// builds inside it before any host launch. The driver writes the JSON and +// Markdown comparison reports. // // Trust model. The side script, fixtures, grader, and report builder all run // from this trusted checkout. Only the extension host loads the evaluated @@ -10,14 +12,17 @@ // the one named provider variable. A commit runs only when its full SHA is // listed in the operator's trust policy; refs are resolved and checked // against the policy before any worktree or process exists. The side process -// re-checks the root it was handed against the same policy. +// re-checks the commit it was handed against the same policy, constructs a +// fresh detached worktree of it, and installs and builds inside that +// worktree before any launch. // -// Process containment. Install, build, worktree, and side children receive -// only an allowlisted environment, so the driver's secrets never reach them. -// Each runs detached in its own process group under one guard, so SIGINT and -// SIGTERM aimed at the driver alone terminate the tracked group with a bounded -// SIGTERM-then-SIGKILL escalation before the scratch cleanup runs, and an -// interrupt leaves no orphan host behind. +// Process containment. Worktree, side, and the sides' install and build +// children receive only an allowlisted environment, so the driver's secrets +// never reach them. Each runs detached in its own process group under a +// guard, so SIGINT and SIGTERM aimed at the driver alone terminate the side +// group with a bounded SIGTERM-then-SIGKILL escalation before the scratch +// cleanup runs, the side runs the same escalation for its own children, and +// an interrupt leaves no orphan host behind. import fs from "node:fs/promises" import os from "node:os" import path from "node:path" @@ -28,8 +33,6 @@ import { createInterruptFlag, createSideProcessGuard, type SideKillOutcome } fro const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") -const INSTALL_TIMEOUT_MS = 10 * 60_000 -const BUILD_TIMEOUT_MS = 30 * 60_000 const WORKTREE_TIMEOUT_MS = 120_000 const SIDE_KILL_GRACE_MS = 30_000 const DEFAULT_SIDE_TIMEOUT_MINUTES = 120 @@ -98,23 +101,11 @@ const readArgs = (): { } } -// Prepares the worktree of one evaluated commit. Dependencies install from -// the worktree's own frozen lockfile, and that exact commit's extension and -// webview build from its own sources. No artifact crosses sides. -const prepareWorktree = async (worktree: string): Promise => { - await runBoundedPreparation( - sideGuard, - interruptFlag.isInterrupted, - "pnpm", - ["install", "--frozen-lockfile"], - worktree, - INSTALL_TIMEOUT_MS, - ) - await runBoundedPreparation(sideGuard, interruptFlag.isInterrupted, "pnpm", ["build"], worktree, BUILD_TIMEOUT_MS) -} - // One side's full preparation: the detached worktree add of that exact -// commit, then the install and build inside it. +// commit. The side derives its own fresh worktree of this commit, installs +// the frozen dependencies, and builds there before any host launch, so the +// executed build output is always the side's own product of the approved +// commit. const prepareSideWorktree = async (commit: string, worktree: string): Promise => { await runBoundedPreparation( sideGuard, @@ -124,7 +115,6 @@ const prepareSideWorktree = async (commit: string, worktree: string): Promise Promise // Creates the side output directory. This is the last async boundary // before the spawn; tests arm the interrupt inside it. @@ -420,12 +421,19 @@ export async function readPublishedComparisonReport(outDir: string): Promise { - await fs.rm(path.join(outDir, "artifacts"), { recursive: true, force: true }) - await fs.rm(path.join(outDir, sideResultFileName()), { force: true }) +// Removes the previous side generation before a side executes. The +// authoritative result is removed first, so every stop after that point, an +// error or an interruption alike, leaves the generation incomplete: consumers +// require side-results.json, and it exists only after the new atomic +// publication completes. A result that cannot be removed, or an artifacts +// tree that cannot be removed after it, fails the side before any run spends +// budget. +export type SideArtifactRemover = (target: string, options: { force?: boolean; recursive?: boolean }) => Promise + +export async function invalidateSideArtifacts( + outDir: string, + remove: SideArtifactRemover = (target, options) => fs.rm(target, options), +): Promise { + await remove(path.join(outDir, sideResultFileName()), { force: true }) + await remove(path.join(outDir, "artifacts"), { recursive: true, force: true }) } diff --git a/scripts/agent-evals/coding-report.test.ts b/scripts/agent-evals/coding-report.test.ts index 7dee41eba0..13f1e3c96a 100644 --- a/scripts/agent-evals/coding-report.test.ts +++ b/scripts/agent-evals/coding-report.test.ts @@ -11,6 +11,7 @@ import { readPublishedComparisonReport, runComparison, validateSideResultFile, + type SideArtifactRemover, type SideExecutor, } from "./coding-compare.ts" import { writeFileAtomic, writeJsonAtomic } from "./fs-atomic.ts" @@ -489,4 +490,57 @@ describe("side artifact invalidation", () => { await fs.rm(dir, { recursive: true, force: true }) } }) + + it("removes the authoritative side result before the artifacts tree", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-invalidate-test-")) + try { + await writeJsonAtomic(path.join(dir, "side-results.json"), { stale: true }) + await fs.mkdir(path.join(dir, "artifacts", "clamp-range-rep4"), { recursive: true }) + const removed: string[] = [] + const remove: SideArtifactRemover = async (target, options) => { + removed.push(path.relative(dir, target)) + await fs.rm(target, options) + } + await invalidateSideArtifacts(dir, remove) + assert.deepEqual(removed, ["side-results.json", "artifacts"]) + assert.deepEqual(await fs.readdir(dir), []) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("fails closed when artifact removal fails after the result is gone", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-invalidate-test-")) + try { + await writeJsonAtomic(path.join(dir, "side-results.json"), { stale: true }) + const remove: SideArtifactRemover = (target, options) => + target === path.join(dir, "artifacts") + ? Promise.reject(new Error("artifact removal blocked")) + : fs.rm(target, options) + await assert.rejects(invalidateSideArtifacts(dir, remove), /artifact removal blocked/) + // The authoritative result is already gone, so the generation is + // incomplete and every consumer fails closed until the new atomic + // publication. + await assert.rejects(fs.access(path.join(dir, "side-results.json"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it("leaves publication working after invalidation", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "side-invalidate-test-")) + try { + await writeJsonAtomic(path.join(dir, "side-results.json"), { stale: true }) + await fs.mkdir(path.join(dir, "artifacts", "old"), { recursive: true }) + await invalidateSideArtifacts(dir) + assert.deepEqual(await fs.readdir(dir), []) + // The new generation publishes atomically into the invalidated + // directory and is complete again. + await writeJsonAtomic(path.join(dir, "side-results.json"), { pins: { fresh: true } }) + const parsed = JSON.parse(await fs.readFile(path.join(dir, "side-results.json"), "utf8")) as unknown + assert.deepEqual(parsed, { pins: { fresh: true } }) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) }) diff --git a/scripts/agent-evals/coding-side.mts b/scripts/agent-evals/coding-side.mts index 52b53747f7..07326eaffb 100644 --- a/scripts/agent-evals/coding-side.mts +++ b/scripts/agent-evals/coding-side.mts @@ -2,12 +2,22 @@ // commit through the substrate, grades each final workspace, and writes one // side-results file with the pinned inputs. Trusted repo code only: the // provider credential stays with this process and the extension host; the -// grader never receives it. The extension under test comes from the evaluated -// root, which must be an approved commit and a tree that matches that commit -// exactly before any host is created or any credential is read. +// grader never receives it. +// +// The extension under test is never a caller-prepared tree. The side resolves +// the evaluated root's HEAD commit, requires its approval, and then +// constructs a fresh detached worktree of that exact commit in private +// scratch space. The frozen install, the build, and every host launch run +// inside that worktree, so the executed content is only this side's own +// product of the approved commit. Dirty files, untracked files, ignored +// dependencies, stale build output, and symlinks in the caller's tree can +// never reach it. +import { execFile } from "node:child_process" import fs from "node:fs/promises" +import os from "node:os" import path from "node:path" import { fileURLToPath } from "node:url" +import { promisify } from "node:util" import { createHostRuntime } from "./launcher.ts" import { createContainerGradingRuntime } from "./container-runtime.ts" import { runCodingFixtureOnce } from "./coding-run.ts" @@ -16,7 +26,7 @@ import { CODING_SUITE_ID, codingSuiteDigest, ensureCodingSuiteRegistered } from import { GRADER_VERSION } from "./coding-grader.ts" import { digestValue } from "./coding-digest.ts" import type { SidePins } from "./coding-pins.ts" -import { invalidateSideArtifacts } from "./coding-compare.ts" +import { invalidateSideArtifacts, runBoundedPreparation } from "./coding-compare.ts" import { assertCommitsApproved, assertCredentialEnvAllowed, @@ -25,10 +35,21 @@ import { resolveCommit, } from "./coding-trust.ts" import { executeSideFlow, type SideRepetition } from "./coding-side-flow.ts" -import { createInterruptFlag } from "./side-process-guard.ts" +import { createInterruptFlag, createSideProcessGuard } from "./side-process-guard.ts" + +const execFileAsync = promisify(execFile) const thisRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") +// The side owns the worktree construction, the frozen install, and the build +// of the approved commit. All steps consume the side's own wall clock, and a +// step past its bound is killed as a group like an expired run. +const WORKTREE_TIMEOUT_MS = 120_000 +const INSTALL_TIMEOUT_MS = 10 * 60_000 +const BUILD_TIMEOUT_MS = 30 * 60_000 +const WORKTREE_REMOVE_TIMEOUT_MS = 30_000 +const SIDE_KILL_GRACE_MS = 30_000 + const readArgs = (): { reps: number out: string @@ -59,8 +80,9 @@ const readArgs = (): { reps: Number(read("reps") ?? "3"), out: path.resolve(out), credentialEnv, - // Repository root of the extension under test. The trusted harness - // code drives the run; only the extension host loads this tree. + // Repository whose HEAD commit this side evaluates. The trusted + // harness code drives the run; only the extension host loads the + // fresh worktree this side builds from the approved commit. evaluatedRoot: path.resolve(read("evaluated-root") ?? thisRoot), trustFile: path.resolve(trustFile), } @@ -92,19 +114,15 @@ const readVscodeTarget = async (root: string): Promise => { const { reps, out, credentialEnv, evaluatedRoot, trustFile } = readArgs() if (!Number.isInteger(reps) || reps < 1) throw new Error("--reps must be a positive integer") const sideLabel = readSideLabel() -// Trust gate. The evaluated root is loaded by the extension host and receives -// the credential, so its resolved full commit must be approved before any -// directory is prepared, any host is created, or the credential is read. The -// comparison driver checks its resolved commits first; this second check binds -// the root that was actually passed in. The commit of an unapproved root is -// unreadable or foreign, and both fail closed here. +// Trust gate. The evaluated root names the repository whose HEAD commit this +// side evaluates, so that commit must be approved before any directory is +// prepared, any host is created, or the credential is read. The comparison +// driver checks its resolved commits first; this second check binds the root +// that was actually passed in. The commit of an unapproved root is unreadable +// or foreign, and both fail closed here. const policy = await loadTrustPolicy(trustFile) const evaluatedCommit = await resolveCommit("HEAD", evaluatedRoot) assertCommitsApproved(policy, [evaluatedCommit]) -// The approval covers the commit only, so the tree the host loads must also -// match that commit. A tracked or untracked change makes the executed files -// differ from the approved content, and the side stops here. -await assertEvaluatedTreeClean(evaluatedRoot) // Fail before any directory, run, or result file exists. An empty credential // would otherwise produce a plausible file of infrastructure failures. const credential = process.env[credentialEnv] @@ -114,14 +132,21 @@ if (credential === undefined || credential.length === 0) { ensureCodingSuiteRegistered() // Side-level interrupt. A SIGINT or SIGTERM that reaches this process arms -// one flag at most once. The flow checks it before each repetition and -// before publication, so an interrupted side starts no new repetition and -// never publishes the completion marker. The substrate registers its own -// host teardown listener per run; the two listeners stay independent and -// the flag never consumes their signals. +// one flag at most once and starts the preparation guard's escalation, so +// a bounded install or build never outlives the side. The flow checks the +// flag before each repetition and before publication, so an interrupted +// side starts no new repetition and never publishes the completion marker. +// The substrate registers its own host teardown listener per run; the +// listeners stay independent and the flag never consumes their signals. const interrupt = createInterruptFlag() -process.on("SIGINT", () => interrupt.signal("SIGINT")) -process.on("SIGTERM", () => interrupt.signal("SIGTERM")) +const prepGuard = createSideProcessGuard(SIDE_KILL_GRACE_MS) +let prepSettled: Promise = Promise.resolve() +const onSideInterrupt = (signal: NodeJS.Signals): void => { + interrupt.signal(signal) + prepSettled = prepGuard.interrupt() +} +process.on("SIGINT", () => onSideInterrupt("SIGINT")) +process.on("SIGTERM", () => onSideInterrupt("SIGTERM")) // Live sides always grade in the container runtime. The local subprocess // runtime stays available to unit tests only. @@ -133,6 +158,25 @@ const gradingRuntime = createContainerGradingRuntime() // completes. await invalidateSideArtifacts(out) +// Private scratch space for the fresh evaluated tree. The caller's root +// contributes only the approved commit object; nothing from its working +// tree, its ignored dependencies, or its build output is copied, linked, or +// executed. +const sideScratch = await fs.mkdtemp(path.join(os.tmpdir(), "coding-evals-side-")) +const evaluatedWorktree = path.join(sideScratch, "worktree") + +// Removal is best effort and bounded, like the comparison flow cleanup. The +// escalation of an in-flight preparation child settles before any tree is +// removed, so no child is still writing while its tree is deleted. +const removePreparedTree = async (): Promise => { + await prepSettled + await execFileAsync("git", ["worktree", "remove", "--force", evaluatedWorktree], { + cwd: evaluatedRoot, + timeout: WORKTREE_REMOVE_TIMEOUT_MS, + }).catch(() => {}) + await fs.rm(sideScratch, { recursive: true, force: true }).catch(() => {}) +} + const projection = CODING_FIXTURE_BUNDLES[0]?.fixture.settingsProjection ?? {} const repetitions: SideRepetition[] = CODING_FIXTURE_BUNDLES.flatMap((bundle) => @@ -160,44 +204,102 @@ const buildPins = async (): Promise => { runnerEnvironment: { os: process.platform, nodeVersion: process.version, - vscodeTarget: await readVscodeTarget(evaluatedRoot), + vscodeTarget: await readVscodeTarget(evaluatedWorktree), }, } } -const { runs, completed } = await executeSideFlow({ - repetitions, - out, - isInterrupted: interrupt.isInterrupted, - runRepetition: async ({ fixtureName, repIndex }) => { - const artifactsDir = path.join(out, "artifacts", `${fixtureName}-rep${repIndex}`) - await fs.mkdir(artifactsDir, { recursive: true }) - return runCodingFixtureOnce({ - fixtureName, - repIndex, - credential, - runtime: createHostRuntime({ - root: evaluatedRoot, - artifactsDir, - vscodeCachePath: path.join(thisRoot, "apps/vscode-e2e/.vscode-test"), - }), - gradingRuntime, - scratchDir: artifactsDir, - }) - }, - // Publication point and completion marker. side-results.json appears - // atomically only after every repetition of this invocation finished, so - // a reader that sees the file sees a complete side of the current run. A - // missing file means an interrupted side; consumers must treat it as a - // failed side and never pair partial output. - buildResult: async (runs) => ({ pins: await buildPins(), runs }), -}) - -if (!completed) { - // An interrupted side exits with the conventional signal code and without - // side-results.json, so consumers treat it as a failed side. - console.error(`coding-evals side interrupted by ${interrupt.name() ?? "signal"}; side-results.json not published`) - process.exit(interrupt.exitCode()) +// Everything after the gates runs under one cleanup: the prepared tree is +// removed on completion, on failure, and on interruption alike, and an +// interrupted side exits with the conventional signal code after that +// removal. A thrown failure is deferred until the removal finished, so no +// error path leaks the worktree. +let interruptedExitCode: number | undefined +let failure: unknown +try { + await runBoundedPreparation( + prepGuard, + interrupt.isInterrupted, + "git", + ["worktree", "add", "--detach", evaluatedWorktree, evaluatedCommit], + evaluatedRoot, + WORKTREE_TIMEOUT_MS, + ) + // The approval names one commit, so the constructed tree must resolve to + // exactly that commit and hold no other content. + const preparedCommit = await resolveCommit("HEAD", evaluatedWorktree) + if (preparedCommit !== evaluatedCommit) { + throw new Error(`prepared evaluated worktree is at ${preparedCommit}, not the approved ${evaluatedCommit}`) + } + await assertEvaluatedTreeClean(evaluatedWorktree) + // The install and the build run inside the fresh worktree, so the entry + // the host loads is always this side's own product of the approved + // commit. The preparation children receive only the allowlisted + // environment, so the credential never reaches them. + await runBoundedPreparation( + prepGuard, + interrupt.isInterrupted, + "pnpm", + ["install", "--frozen-lockfile"], + evaluatedWorktree, + INSTALL_TIMEOUT_MS, + ) + await runBoundedPreparation( + prepGuard, + interrupt.isInterrupted, + "pnpm", + ["build"], + evaluatedWorktree, + BUILD_TIMEOUT_MS, + ) + + const { runs, completed } = await executeSideFlow({ + repetitions, + out, + isInterrupted: interrupt.isInterrupted, + runRepetition: async ({ fixtureName, repIndex }) => { + const artifactsDir = path.join(out, "artifacts", `${fixtureName}-rep${repIndex}`) + await fs.mkdir(artifactsDir, { recursive: true }) + return runCodingFixtureOnce({ + fixtureName, + repIndex, + credential, + runtime: createHostRuntime({ + root: evaluatedWorktree, + artifactsDir, + vscodeCachePath: path.join(thisRoot, "apps/vscode-e2e/.vscode-test"), + }), + gradingRuntime, + scratchDir: artifactsDir, + }) + }, + // Publication point and completion marker. side-results.json appears + // atomically only after every repetition of this invocation finished, + // so a reader that sees the file sees a complete side of the current + // run. A missing file means an interrupted side; consumers must treat + // it as a failed side and never pair partial output. + buildResult: async (runs) => ({ pins: await buildPins(), runs }), + }) + + if (!completed) { + console.error( + `coding-evals side interrupted by ${interrupt.name() ?? "signal"}; side-results.json not published`, + ) + interruptedExitCode = interrupt.exitCode() + } else { + const correct = runs.filter((record) => record.outcome.outcome === "correct").length + console.log( + `coding-evals side complete: ${correct}/${runs.length} correct, suite ${codingSuiteDigest().slice(0, 12)}`, + ) + } +} catch (error) { + if (interrupt.isInterrupted()) { + console.error("coding-evals side interrupted during preparation; side-results.json not published") + interruptedExitCode = interrupt.exitCode() + } else { + failure = error + } } -const correct = runs.filter((record) => record.outcome.outcome === "correct").length -console.log(`coding-evals side complete: ${correct}/${runs.length} correct, suite ${codingSuiteDigest().slice(0, 12)}`) +await removePreparedTree() +if (failure !== undefined) throw failure +if (interruptedExitCode !== undefined) process.exit(interruptedExitCode) diff --git a/scripts/agent-evals/coding-side.test.ts b/scripts/agent-evals/coding-side.test.ts index 418e79f4b0..f00b5d47d8 100644 --- a/scripts/agent-evals/coding-side.test.ts +++ b/scripts/agent-evals/coding-side.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict" -import { execFile } from "node:child_process" +import { execFile, spawn } from "node:child_process" import fs from "node:fs/promises" import os from "node:os" import path from "node:path" @@ -56,12 +56,21 @@ const writePolicy = async (dir: string, approvedCommits: string[]): Promise => { +// One fixture commit in a throwaway repository. It stands in for an +// evaluated root; no fixture code from it ever runs in these checks. The +// optional files become part of the committed tree. +const makeFixtureCommit = async ( + dir: string, + files: Record = {}, +): Promise<{ repo: string; commit: string }> => { const repo = path.join(dir, "fixture-repo") await fs.mkdir(repo) + for (const [name, content] of Object.entries(files)) { + await fs.mkdir(path.dirname(path.join(repo, name)), { recursive: true }) + await fs.writeFile(path.join(repo, name), content) + } await execFileAsync("git", ["init", "-q"], { cwd: repo }) + await execFileAsync("git", ["add", "-A"], { cwd: repo }) await execFileAsync( "git", [ @@ -86,48 +95,6 @@ const makeFixtureCommit = async (dir: string): Promise<{ repo: string; commit: s } describe("coding side trust gate", () => { - it( - "rejects a dirty approved evaluated root before any host launch or publication", - { timeout: 180_000 }, - async () => { - const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) - try { - const { repo, commit } = await makeFixtureCommit(dir) - const trustFile = await writePolicy(dir, [commit]) - // The approved checkout is modified after approval: one staged - // tracked change and one untracked file. The commit identity is - // still approved, but it no longer describes the tree. - await fs.writeFile(path.join(repo, "tracked.txt"), "changed") - await execFileAsync("git", ["add", "tracked.txt"], { cwd: repo }) - await fs.writeFile(path.join(repo, "untracked.txt"), "untracked") - // The fake credential only proves the failure is not the - // credential check: the cleanliness gate must stop the side first. - const outcome = await runSideScript( - [ - "--reps", - "1", - "--out", - path.join(dir, "out"), - "--trust-file", - trustFile, - "--evaluated-root", - repo, - ], - minimalEnv({ SOLHEIM_API_KEY: "sk-test" }), - ) - assert.notEqual(outcome.code, 0) - assert.match(outcome.stderr, /tracked or untracked changes/) - assert.ok(!outcome.stdout.includes("coding-evals side complete")) - // The rejection happens before the publication point: no side - // results and no artifacts exist. - await assert.rejects(fs.access(path.join(dir, "out", "side-results.json"))) - await assert.rejects(fs.access(path.join(dir, "out", "artifacts"))) - } finally { - await fs.rm(dir, { recursive: true, force: true }) - } - }, - ) - it("rejects an unapproved evaluated root before any host launch", { timeout: 180_000 }, async () => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) try { @@ -190,8 +157,8 @@ describe("coding side trust gate", () => { const { repo, commit } = await makeFixtureCommit(dir) const trustFile = await writePolicy(dir, [commit]) // No credential variable is set. The gate must pass and the run - // must fail closed at the credential check, before any host or - // container work. + // must fail closed at the credential check, before any worktree + // or container work. const outcome = await runSideScript( [ "--reps", @@ -249,3 +216,271 @@ describe("coding side isolation gate", () => { } }) }) + +// The install behavior of the shimmed toolchain: "succeed" runs the benign +// build, "fail" exits nonzero after the install starts, and "hang" blocks so +// the test can interrupt the side mid-preparation. +type InstallBehavior = "succeed" | "fail" | "hang" + +// A pnpm shim that records each invocation, its working directory, and its +// environment, so a test can prove the side prepared and built a fresh tree +// without the credential. The build step writes the benign entry the real +// toolchain would produce, relative to the tree it actually ran in. +const writePnpmShim = async (dir: string, install: InstallBehavior): Promise => { + const shimDir = path.join(dir, "bin") + await fs.mkdir(shimDir) + const log = (name: string): string => JSON.stringify(path.join(dir, name)) + const lines = [ + "#!/bin/sh", + `printf '%s\\n' "$*" >> ${log("pnpm-args.log")}`, + 'case "$1" in', + " install)", + ` pwd > ${log("pnpm-install-cwd.log")}`, + ` env > ${log("pnpm-env.log")}`, + ] as string[] + if (install === "hang") lines.push(" sleep 30") + if (install === "fail") lines.push(" exit 1") + lines.push( + " exit 0", + " ;;", + " build)", + ` pwd > ${log("pnpm-build-cwd.log")}`, + ` git rev-parse HEAD > ${log("pnpm-build-head.log")}`, + " mkdir -p src/dist", + " printf 'benign extension entry\\n' > src/dist/extension.js", + " exit 0", + " ;;", + "esac", + "exit 1", + ) + await fs.writeFile(path.join(shimDir, "pnpm"), lines.join("\n"), { mode: 0o755 }) + return shimDir +} + +const preparedEnv = (dir: string, shimDir: string): NodeJS.ProcessEnv => + minimalEnv({ SOLHEIM_API_KEY: "sk-test", PATH: `${shimDir}:${process.env["PATH"] ?? ""}` }) + +const preparedArgs = (dir: string, repo: string, trustFile: string): string[] => [ + "--reps", + "1", + "--out", + path.join(dir, "out"), + "--trust-file", + trustFile, + "--evaluated-root", + repo, +] + +const runPreparedSide = async (dir: string, repo: string, trustFile: string): Promise => { + const shimDir = await writePnpmShim(dir, "succeed") + return runSideScript(preparedArgs(dir, repo, trustFile), preparedEnv(dir, shimDir)) +} + +const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)) + +const waitForFile = async (filePath: string, timeoutMs: number): Promise => { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if ( + await fs.access(filePath).then( + () => true, + () => false, + ) + ) + return + await delay(20) + } + throw new Error(`fixture file never appeared: ${filePath}`) +} + +// The repository must hold no linked worktree: its only entry is the main +// tree itself. +const assertSingleWorktree = async (repo: string): Promise => { + const { stdout } = await execFileAsync("git", ["worktree", "list", "--porcelain"], { cwd: repo }) + const listed = stdout.split("\n").filter((line) => line.startsWith("worktree ")) + assert.equal(listed.length, 1) +} + +// The install and the build must have run in one fresh worktree of the +// approved commit, never in the caller tree. Returns the worktree path, so +// the caller can prove its removal. +const assertPreparationRanInFreshTree = async (dir: string, repo: string, commit: string): Promise => { + const args = (await fs.readFile(path.join(dir, "pnpm-args.log"), "utf8")).trim().split("\n") + assert.deepEqual(args, ["install --frozen-lockfile", "build"]) + const installCwd = (await fs.readFile(path.join(dir, "pnpm-install-cwd.log"), "utf8")).trim() + const buildCwd = (await fs.readFile(path.join(dir, "pnpm-build-cwd.log"), "utf8")).trim() + const buildHead = (await fs.readFile(path.join(dir, "pnpm-build-head.log"), "utf8")).trim() + assert.equal(installCwd, buildCwd) + assert.notEqual(installCwd, repo) + assert.ok(path.basename(path.dirname(installCwd)).startsWith("coding-evals-side-")) + assert.equal(buildHead, commit) + return installCwd +} + +interface SideResultsFile { + pins: { commit: string } + runs: { outcome: { outcome: string } }[] +} + +// The launch fails deterministically because the fixture has no host entry, +// so every repetition is a non-correct record and the side still publishes. +const assertPublishedFailedRuns = async (dir: string, commit: string): Promise => { + const results = JSON.parse(await fs.readFile(path.join(dir, "out", "side-results.json"), "utf8")) as SideResultsFile + assert.equal(results.pins.commit, commit) + assert.ok(results.runs.length >= 1) + for (const run of results.runs) assert.notEqual(run.outcome.outcome, "correct") +} + +describe("coding side evaluated tree preparation", () => { + it( + "builds a fresh detached worktree of the approved commit and never loads the caller tree", + { timeout: 240_000 }, + async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) + try { + const { repo, commit } = await makeFixtureCommit(dir, { + ".gitignore": "src/dist\nnode_modules\n", + "tracked.txt": "committed", + }) + const trustFile = await writePolicy(dir, [commit]) + // Everything planted below stays outside the approval. The + // tracked change and the untracked file make the caller tree + // dirty. The ignored entry and the ignored dependency bin are + // executable artifacts a real build may never overwrite. If + // any of them ran, its marker file would exist. + await fs.writeFile(path.join(repo, "tracked.txt"), "tampered") + await fs.writeFile(path.join(repo, "untracked.txt"), "untracked") + const plantedEntry = `require("node:fs").writeFileSync(${JSON.stringify(path.join(repo, "planted-entry-marker"))}, "executed")\n` + await fs.mkdir(path.join(repo, "src/dist"), { recursive: true }) + await fs.writeFile(path.join(repo, "src/dist/extension.js"), plantedEntry) + await fs.mkdir(path.join(repo, "node_modules/.bin"), { recursive: true }) + await fs.writeFile( + path.join(repo, "node_modules/.bin/evil"), + `#!/bin/sh\nprintf executed > ${JSON.stringify(path.join(repo, "planted-bin-marker"))}\n`, + { mode: 0o755 }, + ) + const outcome = await runPreparedSide(dir, repo, trustFile) + assert.equal(outcome.code, 0) + assert.ok(outcome.stdout.includes("coding-evals side complete")) + // No planted artifact executed, and the side never wrote into + // the caller tree. + await assert.rejects(fs.access(path.join(repo, "planted-entry-marker"))) + await assert.rejects(fs.access(path.join(repo, "planted-bin-marker"))) + assert.equal(await fs.readFile(path.join(repo, "src/dist/extension.js"), "utf8"), plantedEntry) + await fs.access(path.join(repo, "node_modules/.bin/evil")) + // The install and the build ran in one fresh worktree of the + // approved commit, never in the caller tree. + const worktree = await assertPreparationRanInFreshTree(dir, repo, commit) + // Cleanup on the normal path: the worktree and its scratch are + // gone, and the caller repository holds no linked worktree. + await assert.rejects(fs.access(worktree)) + await assert.rejects(fs.access(path.dirname(worktree))) + await assertSingleWorktree(repo) + // The pins record the approved commit, and the preparation + // children never saw the credential. + await assertPublishedFailedRuns(dir, commit) + const envLog = await fs.readFile(path.join(dir, "pnpm-env.log"), "utf8") + assert.ok(!envLog.includes("SOLHEIM_API_KEY=")) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }, + ) + + it( + "accepts a clean approved checkout and builds it in a fresh worktree before the first run", + { timeout: 240_000 }, + async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) + try { + const { repo, commit } = await makeFixtureCommit(dir, { ".gitignore": "src/dist\n" }) + const trustFile = await writePolicy(dir, [commit]) + const outcome = await runPreparedSide(dir, repo, trustFile) + assert.equal(outcome.code, 0) + assert.ok(outcome.stdout.includes("coding-evals side complete")) + const worktree = await assertPreparationRanInFreshTree(dir, repo, commit) + await assert.rejects(fs.access(worktree)) + await assertSingleWorktree(repo) + await assertPublishedFailedRuns(dir, commit) + const envLog = await fs.readFile(path.join(dir, "pnpm-env.log"), "utf8") + assert.ok(!envLog.includes("SOLHEIM_API_KEY=")) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }, + ) + + it("removes its worktree when preparation fails", { timeout: 240_000 }, async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) + try { + const { repo, commit } = await makeFixtureCommit(dir, { ".gitignore": "src/dist\nnode_modules\n" }) + const trustFile = await writePolicy(dir, [commit]) + const shimDir = await writePnpmShim(dir, "fail") + const outcome = await runSideScript(preparedArgs(dir, repo, trustFile), preparedEnv(dir, shimDir)) + assert.notEqual(outcome.code, 0) + assert.match(outcome.stderr, /pnpm failed: code 1/) + // The failing install names the worktree it ran in; that tree and + // its scratch must be gone, and the repository must hold no linked + // worktree. + const worktree = (await fs.readFile(path.join(dir, "pnpm-install-cwd.log"), "utf8")).trim() + await assert.rejects(fs.access(worktree)) + await assert.rejects(fs.access(path.dirname(worktree))) + await assertSingleWorktree(repo) + // The build never ran and no results exist. + await assert.rejects(fs.access(path.join(repo, "src/dist"))) + await assert.rejects(fs.access(path.join(dir, "out", "side-results.json"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + it( + "removes its worktree when interrupted during preparation and exits with the signal code", + { timeout: 240_000 }, + async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "coding-side-test-")) + try { + const { repo, commit } = await makeFixtureCommit(dir, { ".gitignore": "src/dist\nnode_modules\n" }) + const trustFile = await writePolicy(dir, [commit]) + const shimDir = await writePnpmShim(dir, "hang") + let stderr = "" + const child = spawn( + process.execPath, + [ + "--import", + "tsx", + path.join(repoRoot, "scripts/agent-evals/coding-side.mts"), + ...preparedArgs(dir, repo, trustFile), + ], + { + cwd: repoRoot, + env: preparedEnv(dir, shimDir), + stdio: ["ignore", "ignore", "pipe"], + }, + ) + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString() + }) + const exitCode = new Promise((resolve, reject) => { + child.once("error", reject) + child.once("exit", (code) => resolve(code ?? 1)) + }) + // The install is in flight when the signal arrives. A SIGINT that + // reaches only the side must stop the preparation group, remove + // the worktree, and exit with the conventional code. + await waitForFile(path.join(dir, "pnpm-install-cwd.log"), 60_000) + child.kill("SIGINT") + assert.equal(await exitCode, 130) + assert.match(stderr, /interrupted during preparation/) + const worktree = (await fs.readFile(path.join(dir, "pnpm-install-cwd.log"), "utf8")).trim() + await assert.rejects(fs.access(worktree)) + await assert.rejects(fs.access(path.dirname(worktree))) + await assertSingleWorktree(repo) + // An interrupted side never publishes the completion marker. + await assert.rejects(fs.access(path.join(dir, "out", "side-results.json"))) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }, + ) +}) diff --git a/scripts/agent-evals/coding-trust.ts b/scripts/agent-evals/coding-trust.ts index 463c8126a1..fb55cccfd8 100644 --- a/scripts/agent-evals/coding-trust.ts +++ b/scripts/agent-evals/coding-trust.ts @@ -77,12 +77,11 @@ export class DirtyEvaluatedTreeError extends Error { } } -// Gate before any host launch. The evaluated root must match its approved -// commit exactly, so a modified checkout can never run under that commit's -// approval. Ignored files, such as dependency installs and build outputs, -// stay outside commit identity and outside this check: the approval does not -// attest them. The comparison driver closes that gap by loading only a fresh -// detached worktree built from the approved commit. +// Gate before any host launch. The executed tree must match its approved +// commit exactly, so modified or planted content can never run under that +// commit's approval. The side runs this check against the fresh detached +// worktree it constructed from the approved commit. The caller's own tree is +// never executed at all, so its ignored files need no approval. export async function assertEvaluatedTreeClean(root: string): Promise { const { execFile } = await import("node:child_process") const { promisify } = await import("node:util")