diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index 8bf1b8af..f4a4a9ff 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -127,3 +127,51 @@ Deferred lint/test guardrails surfaced during reviews. Promote to a `CExxx` rule that environment injects the criterion. Until then CE030 stays scoped to the four top-level models; the `command_pattern`/`exclude_pattern` contract this PR changed is documented in the Field descriptions and TASK_DEFINITION_GUIDE regardless. + +## From the evalboard Path-to-GA de-tag / mature-passes fix (4e5bbc4…dd5f7e9) — TS-side guards deferred + +Context: the CExxx harness is a **Python** AST runner over `src/coder_eval/`, so none +of the invariants below are mechanizable in it. Each would need a TypeScript lint +harness (eslint config + custom rules) that `evalboard/` does not have today — +standing one up for three call sites fails the KISS/YAGNI gate. Deferring rather +than dropping; promote if a fourth TS-side invariant appears, and stand up the +harness once for all of them. + +- [ ] **"Every consumer of `RunOverviewTask.matureSkipped` must decide explicitly + whether a carry-forward row counts."** Four consumers now, and they deliberately + DISAGREE: `lib/trends.ts` and `app/runs/[id]/run-view.tsx` count a mature skip as + a pass; `lib/overview.ts::buildTagTaskRows` excludes it from both terms + (`/path-to-ga` is a GA-readiness page). A new consumer silently inheriting either + convention is a real hazard. Guard shape: flag a file that reads `.matureSkipped` + without a nearby comment naming its convention — weak, hence the deferral. Closed + for now by unit tests that assert the exclusion from BOTH numerator and denominator + (`lib/__tests__/overview.test.ts` → `describe("buildTagTaskRows")`). + +- [ ] **`taskCarriesRepoTag` is the single repo-provenance tag predicate — but one + duplicate survives.** `app/runs/[id]/run-view.tsx:283` still inlines its exact body + (`(tag) => t.tags.includes(tag) || t.skill === tag`). NOT adopted deliberately: + `run-view.tsx` is `"use client"` and `lib/overview.ts` imports `next/cache` plus the + blob readers, so importing the predicate there would drag server-only code into the + client bundle. Fixing it properly means extracting the predicate into a + dependency-free module (e.g. `lib/tags.ts`) — worth doing next time either file is + touched, not worth a standalone change. A lint rule ("no inline `tags.includes(x) || + skill === x`") would catch future copies. + +- [ ] **The de-tag rule fails CLOSED on a newest run that loads fine but stamps no + `tags`** (`lib/overview.ts::buildTagTaskRows`): every tagged task would read as + de-tagged and the table would empty, rendering an empty state indistinguishable from + a genuine full de-tagging. Its sibling failure mode (`overview == null`, a transient + blob read failure) IS guarded, with exactly this rationale. Currently unreachable — + 0 of ~116k date-shaped non-ad-hoc task rows in `runs-remote/` lack `tags`, and the + six zero-tag runs found are all ad-hoc (filtered upstream by id shape + `meta.adhoc`) + — so the barrier is two upstream filters rather than a check at the seam. Left + unguarded on purpose: a `if (taggedInRun.size === 0) skip the de-tag signal` guard + would also mask a real, total de-tagging. Revisit if the pipeline ever stops + stamping tags, or if a non-ad-hoc run legitimately carries zero tagged rows. + +- [ ] **Discriminating-test discipline for predicate narrowings.** Two tests in this + change passed for the wrong reason — a downstream rule (the de-tag drop) masked the + mutation they claimed to catch — and the plan leaned on a `grep` acceptance criterion + that CI never runs. Both were found by mutation-testing the suite and fixed. No + mechanizable guard; the durable lesson is: when a test names a narrowing, construct + the fixture so the row SURVIVES every other rule, or the assertion proves nothing. diff --git a/evalboard/app/path-to-ga/__tests__/task-table.test.tsx b/evalboard/app/path-to-ga/__tests__/task-table.test.tsx new file mode 100644 index 00000000..954586fc --- /dev/null +++ b/evalboard/app/path-to-ga/__tests__/task-table.test.tsx @@ -0,0 +1,117 @@ +import { describe, expect, test } from "vitest"; +import { render, screen } from "@testing-library/react"; +import type { TagTaskRow } from "@/lib/overview"; +import { TagTaskTable } from "../task-table"; + +// TagTaskTable is a pure props-in/JSX-out component — no router hooks, so unlike +// run-view.render.test.tsx this needs no next/navigation stub. + +function row(overrides: Partial = {}): TagTaskRow { + return { + taskId: "skill-flow-coded-agent", + skill: "uipath-maestro-flow", + appearances: 20, + matureSkips: 0, + passRate: 90, + latestStatus: "SUCCESS", + latestScore: 1.0, + latestRunId: "2026-07-31_04-38-51", + latestMatureSkipped: false, + ...overrides, + }; +} + +function renderTable( + rows: TagTaskRow[], + harness: string | null = null, +) { + return render( + , + ); +} + +describe("TagTaskTable", () => { + test("shows a Mature pill instead of Passed when the latest run skipped the task", () => { + renderTable([row({ latestMatureSkipped: true })]); + expect(screen.getByText("Mature")).toBeInTheDocument(); + expect(screen.queryByText("Passed")).not.toBeInTheDocument(); + }); + + test("shows Passed and the numeric score for an ordinary executed row", () => { + renderTable([row({ latestScore: 0.75 })]); + expect(screen.getByText("Passed")).toBeInTheDocument(); + expect(screen.getByText("0.75")).toBeInTheDocument(); + }); + + test("dashes out the latest score on a mature row", () => { + // 1.0 on a carry-forward row is inherited, not measured — showing it + // beside a Mature pill would read as a fresh result. + renderTable([row({ latestMatureSkipped: true, latestScore: 1.0 })]); + expect(screen.queryByText("1.00")).not.toBeInTheDocument(); + // Exactly one cell dashes — the score. Every other column on the default + // row is populated, so this pins WHICH cell went un-measured. + expect(screen.getAllByText("—")).toHaveLength(1); + }); + + test("annotates Appearances with the mature count, and only when non-zero", () => { + renderTable([row({ appearances: 24, matureSkips: 3 })]); + expect(screen.getByText("(3 mature)")).toBeInTheDocument(); + // The raw count stays plain beside it (getByText matches an element's own + // direct text nodes, so this is the cell's "24", not "24 (3 mature)"). + expect(screen.getByText("24")).toBeInTheDocument(); + }); + + test("no mature annotation when nothing was skipped", () => { + renderTable([row({ appearances: 24, matureSkips: 0 })]); + expect(screen.queryByText(/mature\)/)).not.toBeInTheDocument(); + }); + + test("renders an em dash for an unmeasured pass rate", () => { + // Every appearance was a carry-forward → nothing executed → no rate. + // Must not read as NaN% or a measured 0%. `latestMatureSkipped` is true + // by construction here: buildTagTaskRows reads latest* off one of the + // counted appearances, so matureSkips === appearances forces it — the + // score dashes too, hence two dashes rather than one. + renderTable([ + row({ + appearances: 4, + matureSkips: 4, + passRate: null, + latestMatureSkipped: true, + }), + ]); + expect(screen.queryByText(/NaN/)).not.toBeInTheDocument(); + expect(screen.queryByText("0%")).not.toBeInTheDocument(); + expect(screen.getAllByText("—")).toHaveLength(2); + }); + + test("Last seen shows the date half of the latest run id", () => { + renderTable([row({ latestRunId: "2026-07-16_04-24-15" })]); + expect(screen.getByText("2026-07-16")).toBeInTheDocument(); + }); + + test("empty rows render the empty state naming the tag and window", () => { + // Newly reachable: a tag whose every task was de-tagged yields [] where + // it previously yielded stale rows. + renderTable([]); + expect( + screen.getByText(/No tasks tagged path-to-ga in the last 30d\./), + ).toBeInTheDocument(); + }); + + test("the pooled-across-harnesses note tracks the harness prop", () => { + const { unmount } = renderTable([row()], null); + expect(screen.getByText(/pooled across harnesses/)).toBeInTheDocument(); + unmount(); + + renderTable([row()], "claude-code"); + expect( + screen.queryByText(/pooled across harnesses/), + ).not.toBeInTheDocument(); + }); +}); diff --git a/evalboard/app/path-to-ga/page.tsx b/evalboard/app/path-to-ga/page.tsx index 1432ddb5..86443952 100644 --- a/evalboard/app/path-to-ga/page.tsx +++ b/evalboard/app/path-to-ga/page.tsx @@ -1,18 +1,14 @@ -import Link from "next/link"; import { getOverview, getTagTaskBreakdown, listRecentHarnesses, } from "@/lib/overview"; import { parseHarnessScope } from "@/lib/harness"; -import { humanizeTaskId } from "@/lib/format"; -import { passClass } from "@/lib/pass-rate"; import { HarnessSelector } from "../_components/harness-selector"; import { harnessShortLabel } from "../_components/harness-badge"; import { type Window } from "@/lib/reviews-types"; import { DailySuccessChart } from "../_overview/daily-chart"; -import { TableScroll } from "../_components/scroll-table"; -import { StatusPill } from "@/lib/pills"; +import { TagTaskTable } from "./task-table"; export const dynamic = "force-dynamic"; @@ -98,10 +94,22 @@ export default async function PathToGaPage({ {taskRows.length}
- distinct task{taskRows.length === 1 ? "" : "s"} + distinct task{taskRows.length === 1 ? "" : "s"} still + tagged
+ {/* The tile above and the chart below keep their original + mature-blind, union-over-the-window semantics (they feed the + front page and every tag-filtered view); the table does not. + Say so, rather than let the two silently disagree. */} +

+ The rate above and the chart cover every run that carried a{" "} + {TAG} task at the time it + ran, counting mature carry-forwards as passes. The table + below is narrower: only tasks still carrying the tag, scored + on runs that actually executed. +

{runsInWindow > 0 ? ( -
-
-

- Tasks -

- {/* Unscoped, a task's appearances span harnesses, so its rate - pools regimes that aren't strictly comparable. Say so - rather than let the number read as one harness's. */} - {!harness && ( - - pooled across harnesses · pick one above to separate - them - - )} -
- - - - - - - - - - - - - - {taskRows.map((r) => ( - - - - - - - - - ))} - {taskRows.length === 0 && ( - - - - )} - -
- Task - - Skill - - Appearances - - Pass rate - - Latest status - - Latest score -
- - {humanizeTaskId(r.taskId)} - -
- {r.taskId} -
- 		- {r.skill ?? "—"} - - {r.appearances} - - - {r.passRate.toFixed(0)}% - - - - - {r.latestScore != null - ? r.latestScore.toFixed(2) - : "—"} -
- No tasks tagged {TAG} in the last{" "} - {WINDOW}. -
- -
+ ); } diff --git a/evalboard/app/path-to-ga/task-table.tsx b/evalboard/app/path-to-ga/task-table.tsx new file mode 100644 index 00000000..cc224c82 --- /dev/null +++ b/evalboard/app/path-to-ga/task-table.tsx @@ -0,0 +1,186 @@ +import Link from "next/link"; +import type { TagTaskRow } from "@/lib/overview"; +import { fmtRunDate, humanizeTaskId } from "@/lib/format"; +import { passClass } from "@/lib/pass-rate"; +import { MATURE_TOOLTIP, MaturePill, StatusPill } from "@/lib/pills"; +import { TableScroll } from "../_components/scroll-table"; +import { type Window } from "@/lib/reviews-types"; + +// The Path-to-GA task table. Split out of page.tsx (which stays the async IO +// shell) purely so it is render-testable in jsdom — mirrors the +// app/runs/[id]/page.tsx + run-view.tsx split. No "use client": this holds no +// state and no handlers, and a server component may render the "use client" +// TableScroll as a child. +// +// Every row here is scored on runs that ACTUALLY EXECUTED (see +// lib/overview.ts::buildTagTaskRows), which is narrower than the headline tile +// and chart above it — hence the caveat paragraph in page.tsx. A mature +// carry-forward's inherited status/score are dashed out rather than shown as if +// they were measured (same idiom as app/trends/trends-view.tsx; note +// app/runs/[id]/task-grid.tsx deliberately still shows the carried-forward 1.00 +// beside its own MaturePill, so the two surfaces differ). Not extracted into a +// shared helper — the column shapes differ, so it would be a wrapper around a +// ternary. +function passRateTooltip(r: TagTaskRow): string { + const executed = r.appearances - r.matureSkips; + if (executed === 0) { + return ( + `Not executed once in this window — all ${r.appearances} appearance` + + `${r.appearances === 1 ? "" : "s"} were mature carry-forwards, so ` + + "there is no measured pass rate." + ); + } + return ( + `Measured over ${executed} executed appearance` + + `${executed === 1 ? "" : "s"}` + + (r.matureSkips > 0 + ? ` (${r.matureSkips} mature carry-forward${r.matureSkips === 1 ? "" : "s"} excluded).` + : ".") + ); +} +export function TagTaskTable({ + rows, + tag, + window, + harness, +}: { + rows: TagTaskRow[]; + tag: string; + window: Window; + harness: string | null; +}) { + return ( +
+
+

Tasks

+ {/* Unscoped, a task's appearances span harnesses, so its rate + pools regimes that aren't strictly comparable — and the row + then mixes scopes: Appearances/Pass rate are pooled, while + Last seen and the two Latest columns describe ONE run (and + maturity is per-harness pipeline state, so a Mature pill can + legitimately sit beside a middling pooled rate). Say both, + rather than let either read as one harness's. */} + {!harness && ( + + pooled across harnesses · Last seen and Latest describe a + single run · pick one above to separate them + + )} +
+ + + + + + + + + + + + + + + {rows.map((r) => ( + + + + + {/* No conditional dimming: under harness rotation + "newest" differs per harness, so a relative + highlight would mislead. */} + + {/* The denominator is `appearances - matureSkips`, + which is NOT the Appearances shown two cells + left — a mature task is re-validated about + weekly, so a 24-appearance row can rest on a + single executed run and still tint green. + Name the sample size on hover so a confident + percentage can't hide a tiny denominator. */} + + + + + ))} + {rows.length === 0 && ( + + + + )} + +
TaskSkill + Appearances + + Last seen + + Pass rate + + Latest status + + Latest score +
+ + {humanizeTaskId(r.taskId)} + +
+ {r.taskId} +
+ 		+ {r.skill ?? "—"} + + {r.appearances} + {r.matureSkips > 0 && ( + + {" "} + ({r.matureSkips} mature) + + )} + + {fmtRunDate(r.latestRunId)} + + + {r.passRate != null + ? `${r.passRate.toFixed(0)}%` + : "—"} + + + {r.latestMatureSkipped ? ( + + ) : ( + + )} + + {r.latestMatureSkipped || + r.latestScore == null + ? "—" + : r.latestScore.toFixed(2)} +
+ No tasks tagged {tag} in the last {window}. +
+ +
+ ); +} diff --git a/evalboard/lib/__tests__/format.test.ts b/evalboard/lib/__tests__/format.test.ts index 137a625d..868a6a08 100644 --- a/evalboard/lib/__tests__/format.test.ts +++ b/evalboard/lib/__tests__/format.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "vitest"; -import { fmtRunTime, fmtTimestamp } from "../format"; +import { fmtRunDate, fmtRunTime, fmtTimestamp } from "../format"; describe("fmtRunTime", () => { test("reformats a daily-pipeline run id into a readable timestamp", () => { @@ -23,6 +23,20 @@ describe("fmtRunTime", () => { }); }); +describe("fmtRunDate", () => { + test("keeps only the date half of a daily-pipeline run id", () => { + expect(fmtRunDate("2026-07-16_04-24-15")).toBe("2026-07-16"); + }); + + test("returns ad-hoc run ids verbatim rather than splitting on the underscore", () => { + // A bare .split("_")[0] would hand back "adhoc-2026-07-25" / "codex". + expect(fmtRunDate("adhoc-2026-07-25_09-19-36")).toBe( + "adhoc-2026-07-25_09-19-36", + ); + expect(fmtRunDate("codex_skills_full_v2")).toBe("codex_skills_full_v2"); + }); +}); + describe("fmtTimestamp", () => { test("formats an ISO start_time into the fmtRunTime shape", () => { // run.json start_time carries microseconds and no timezone; the literal diff --git a/evalboard/lib/__tests__/overview.test.ts b/evalboard/lib/__tests__/overview.test.ts index 2f8b9440..ba49d3de 100644 --- a/evalboard/lib/__tests__/overview.test.ts +++ b/evalboard/lib/__tests__/overview.test.ts @@ -1,10 +1,13 @@ import { describe, expect, test, vi } from "vitest"; import { buildAdhocRows, + buildTagTaskRows, collectPipelineRuns, projectRunRow, scopeRunTasks, summarizeListing, + taskCarriesRepoTag, + taskMatchesTag, turnBudgetRateForTasks, type PerRun, type RunListingRow, @@ -491,6 +494,326 @@ describe("buildAdhocRows", () => { }); }); +// buildTagTaskRows drives /path-to-ga. Two behaviours are asserted here that no +// other test covers: a task de-tagged upstream (present in a newer run WITHOUT +// the tag) must disappear, and a mature carry-forward must leave both the +// numerator AND the denominator of passRate — the page-local divergence from +// trends.ts, which counts a carry-forward as a pass. +describe("buildTagTaskRows", () => { + const TAG = "path-to-ga"; + + function perRun(id: string, tasks: RunOverviewTask[]): PerRun { + return { + id, + overview: { + id, + tasks, + totalCostUsd: null, + taskDurationSeconds: null, + componentShas: [], + }, + reviewTagCounts: {}, + reviewTagsByTask: {}, + adhoc: false, + title: null, + }; + } + + test("keeps a task still tagged in the newest run", () => { + const rows = buildTagTaskRows( + [ + perRun("r1", [task({ taskId: "a", tags: [TAG] })]), + perRun("r2", [task({ taskId: "a", tags: [TAG] })]), + ], + TAG, + ); + expect(rows.map((r) => r.taskId)).toEqual(["a"]); + expect(rows[0].appearances).toBe(2); + expect(rows[0].latestRunId).toBe("r2"); + }); + + test("drops a task present-but-untagged in a newer run (the de-tag bug)", () => { + // Models ipe-drive-to-slack: tagged in r1, still running in the newer r2 + // but with the tag removed from its YAML. That is proof of de-tagging, so + // the row must vanish rather than linger until r1 ages out of the window. + const rows = buildTagTaskRows( + [ + perRun("r1", [task({ taskId: "detagged", tags: [TAG] })]), + perRun("r2", [task({ taskId: "detagged", tags: ["other"] })]), + ], + TAG, + ); + expect(rows).toEqual([]); + }); + + test("keeps a task that simply stopped appearing, dated to its newest tagged run", () => { + // Retired / renamed / skip:true is unknowable from run data, so the row + // stays and latestRunId is what the page renders as "Last seen". + const rows = buildTagTaskRows( + [ + perRun("r1", [task({ taskId: "gone", tags: [TAG] })]), + perRun("r2", [task({ taskId: "other", tags: [TAG] })]), + ], + TAG, + ); + expect(rows.map((r) => r.taskId)).toEqual(["gone", "other"]); + expect(rows[0].latestRunId).toBe("r1"); + }); + + test("one untagged replicate in the newest run is not a de-tag", () => { + // A replicated task has several rows per run; the de-tag check collapses + // with any-row semantics, so a single untagged replicate cannot drop it. + const rows = buildTagTaskRows( + [ + perRun("r1", [ + task({ taskId: "a", tags: [TAG] }), + task({ taskId: "a", tags: [] }), + ]), + ], + TAG, + ); + expect(rows.map((r) => r.taskId)).toEqual(["a"]); + // Only the tagged row accumulates. + expect(rows[0].appearances).toBe(1); + }); + + test("matureSkips counts carry-forwards; appearances still includes them", () => { + const rows = buildTagTaskRows( + [ + perRun("r1", [task({ taskId: "a", tags: [TAG] })]), + perRun("r2", [ + task({ taskId: "a", tags: [TAG], matureSkipped: true }), + ]), + ], + TAG, + ); + expect(rows[0].appearances).toBe(2); + expect(rows[0].matureSkips).toBe(1); + }); + + test("passRate excludes mature skips from both numerator and denominator", () => { + // 4 appearances, 1 mature skip, 2 executed passes out of 3 executed rows + // → 66.67%. The old mature-inclusive rule would have said 75% (3/4). + const rows = buildTagTaskRows( + [ + perRun("r1", [task({ taskId: "a", tags: [TAG] })]), + perRun("r2", [ + task({ taskId: "a", tags: [TAG], status: "FAILURE" }), + ]), + perRun("r3", [task({ taskId: "a", tags: [TAG] })]), + perRun("r4", [ + task({ taskId: "a", tags: [TAG], matureSkipped: true }), + ]), + ], + TAG, + ); + expect(rows[0].appearances).toBe(4); + expect(rows[0].matureSkips).toBe(1); + expect(rows[0].passRate).toBeCloseTo(66.6667, 3); + }); + + test("passRate is null when every tagged appearance was a mature skip", () => { + // Nothing was measured, so the page must show "—" rather than a + // measured-looking 100% (or a divide-by-zero NaN). + const rows = buildTagTaskRows( + [ + perRun("r1", [ + task({ taskId: "a", tags: [TAG], matureSkipped: true }), + ]), + perRun("r2", [ + task({ taskId: "a", tags: [TAG], matureSkipped: true }), + ]), + ], + TAG, + ); + expect(rows[0].matureSkips).toBe(2); + expect(rows[0].passRate).toBeNull(); + // Producer invariant the table's rendering relies on: latest* is read + // off one of the counted appearances, so "nothing executed" necessarily + // means the latest appearance was a skip. The table dashes BOTH cells on + // the strength of this — it can never show a measured-looking score + // beside an unmeasured rate. + expect(rows[0].latestMatureSkipped).toBe(true); + }); + + test("latestMatureSkipped reflects the newest tagged appearance", () => { + const skippedLatest = buildTagTaskRows( + [ + perRun("r1", [task({ taskId: "a", tags: [TAG] })]), + perRun("r2", [ + task({ taskId: "a", tags: [TAG], matureSkipped: true }), + ]), + ], + TAG, + ); + expect(skippedLatest[0].latestMatureSkipped).toBe(true); + + const executedLatest = buildTagTaskRows( + [ + perRun("r1", [ + task({ taskId: "a", tags: [TAG], matureSkipped: true }), + ]), + perRun("r2", [task({ taskId: "a", tags: [TAG] })]), + ], + TAG, + ); + expect(executedLatest[0].latestMatureSkipped).toBe(false); + }); + + test("a tag matched via skill is never dropped", () => { + // taskCarriesRepoTag's first clause: every run containing the task + // matches, so `tagged` is always true. Using a raw tags.includes() here + // would wrongly drop every skill-tag row. + const rows = buildTagTaskRows( + [ + perRun("r1", [task({ taskId: "a", skill: TAG })]), + perRun("r2", [task({ taskId: "a", skill: TAG })]), + ], + TAG, + ); + expect(rows.map((r) => r.taskId)).toEqual(["a"]); + }); + + test("a task carrying the tag only as a review tag does not appear", () => { + // Review tags are a post-hoc, effectively disjoint namespace; this page + // reports repo-declared tags only. + const r = perRun("r1", [task({ taskId: "a", tags: [] })]); + const rows = buildTagTaskRows( + [{ ...r, reviewTagsByTask: { a: [TAG] } }], + TAG, + ); + expect(rows).toEqual([]); + }); + + test("a review-tagged older appearance is not folded into a kept row", () => { + // Discriminates the accumulate path specifically: the row IS kept (its + // newest run carries the repo tag), so only `appearances` reveals which + // predicate accumulated. Widening the accumulate path back to + // taskMatchesTag would count r1's review-tag-only row too and report 2. + const older = perRun("r1", [task({ taskId: "a", tags: [] })]); + const rows = buildTagTaskRows( + [ + { ...older, reviewTagsByTask: { a: [TAG] } }, + perRun("r2", [task({ taskId: "a", tags: [TAG] })]), + ], + TAG, + ); + expect(rows).toHaveLength(1); + expect(rows[0].appearances).toBe(1); + }); + + test("a task that only recently GAINED the tag is kept", () => { + // Mirror image of the de-tag case, and the reason the newest-appearance + // rule is first-write-wins rather than "tagged in every appearance": a + // task tagged on day 20 of a 30-day window is present-but-untagged in the + // older runs, and must not read as de-tagged. + const rows = buildTagTaskRows( + [ + perRun("r1", [task({ taskId: "a", tags: [] })]), + perRun("r2", [task({ taskId: "a", tags: [TAG] })]), + ], + TAG, + ); + expect(rows.map((r) => r.taskId)).toEqual(["a"]); + expect(rows[0].appearances).toBe(1); + expect(rows[0].latestRunId).toBe("r2"); + }); + + test("a newest run with a null overview neither adds nor drops anything", () => { + // A transient blob failure on the newest run must not read as a de-tag of + // every row. + const broken: PerRun = { + id: "r9", + overview: null, + reviewTagCounts: {}, + reviewTagsByTask: {}, + adhoc: false, + title: null, + }; + const rows = buildTagTaskRows( + [broken, perRun("r1", [task({ taskId: "a", tags: [TAG] })])], + TAG, + ); + expect(rows.map((r) => r.taskId)).toEqual(["a"]); + expect(rows[0].appearances).toBe(1); + expect(rows[0].latestRunId).toBe("r1"); + }); + + test("empty input returns an empty list", () => { + expect(buildTagTaskRows([], TAG)).toEqual([]); + }); + + test("rows come back sorted by taskId", () => { + const rows = buildTagTaskRows( + [ + perRun("r1", [ + task({ taskId: "c", tags: [TAG] }), + task({ taskId: "a", tags: [TAG] }), + task({ taskId: "b", tags: [TAG] }), + ]), + ], + TAG, + ); + expect(rows.map((r) => r.taskId)).toEqual(["a", "b", "c"]); + }); + + test("all three latest* fields come off the same row on replicate disagreement", () => { + // The newest run has an executed replicate and a carried-forward one. + // First-row-wins decides, and the Mature pill must never sit beside a + // measured score from the other replicate. + const rows = buildTagTaskRows( + [ + perRun("r1", [ + task({ + taskId: "a", + tags: [TAG], + matureSkipped: true, + status: "SUCCESS", + weightedScore: 1.0, + }), + task({ + taskId: "a", + tags: [TAG], + status: "FAILURE", + weightedScore: 0.25, + }), + ]), + ], + TAG, + ); + expect(rows[0].latestMatureSkipped).toBe(true); + expect(rows[0].latestStatus).toBe("SUCCESS"); + expect(rows[0].latestScore).toBe(1.0); + // Both replicates are tagged, so `appearances` counts ROWS (2), not runs + // (1) — the semantics the interface comment promises. + expect(rows[0].appearances).toBe(2); + expect(rows[0].matureSkips).toBe(1); + }); +}); + +// taskMatchesTag was rebuilt on top of the extracted taskCarriesRepoTag so +// buildTagTaskRows could reuse the repo-provenance half. scopeRunTasks and the +// front-page rails still go through taskMatchesTag and legitimately filter on +// review tags, so the extraction has to be behaviour-preserving. +describe("taskCarriesRepoTag / taskMatchesTag", () => { + test("taskCarriesRepoTag matches skill and YAML tags, not review tags", () => { + expect(taskCarriesRepoTag(task({ skill: "x" }), "x")).toBe(true); + expect(taskCarriesRepoTag(task({ tags: ["x"] }), "x")).toBe(true); + expect(taskCarriesRepoTag(task({ tags: ["y"] }), "x")).toBe(false); + }); + + test("taskMatchesTag still matches via skill, tags, or a review tag", () => { + expect(taskMatchesTag(task({ skill: "x" }), {}, "x")).toBe(true); + expect(taskMatchesTag(task({ tags: ["x"] }), {}, "x")).toBe(true); + expect( + taskMatchesTag(task({ taskId: "t" }), { t: ["x"] }, "x"), + ).toBe(true); + expect(taskMatchesTag(task({ taskId: "t", tags: ["y"] }), {}, "x")).toBe( + false, + ); + }); +}); + // projectRunRow is the single definition of "does this run count, and with which // tasks" — the summary tiles (getWindowRollup) and the paged run table // (getRunListing) both go through it. They used to be one loop; if they ever diff --git a/evalboard/lib/__tests__/pricing-parity.test.ts b/evalboard/lib/__tests__/pricing-parity.test.ts index 7e5e1a67..927ca370 100644 --- a/evalboard/lib/__tests__/pricing-parity.test.ts +++ b/evalboard/lib/__tests__/pricing-parity.test.ts @@ -71,20 +71,24 @@ describe("pricing.ts ↔ pricing.py parity", () => { }); // Python-priced models we deliberately do NOT mirror to the frontend: heavy - // frontier Claude/GPT variants the evalboard never runs, so pricing them here - // adds nothing. Kept explicit (not a blanket "ignore extras") so a NEW model - // added to pricing.py that ISN'T here and ISN'T in PRICING breaks the build — - // catching a real litellm-relevant omission (e.g. the Bedrock open-weight ids - // that previously rendered "—" for cost). + // frontier variants no harness runs, so pricing them here adds nothing. Kept + // explicit (not a blanket "ignore extras") so a NEW model added to pricing.py + // that ISN'T here and ISN'T in PRICING breaks the build — catching a real + // litellm-relevant omission (e.g. the Bedrock open-weight ids that previously + // rendered "—" for cost). + // + // KEEP THIS SET HONEST. It silences the drift guard, so a stale entry hides a + // live bug rather than a non-issue: `claude-sonnet-5`, `gpt-5.6-sol`, + // `gpt-5.6-terra` and `gpt-5.6-luna` sat here under "the evalboard never runs + // them" while appearing ~32k / ~2k / ~17k / ~2k times in `runs-remote/`, so + // every one of those runs rendered "—" for cost with nothing failing. Before + // adding an id, grep the corpus for it — absence from run data is the ONLY + // justification, and it expires the moment a harness adopts the model. const DELIBERATELY_UNMIRRORED = new Set([ - "claude-sonnet-5", "gpt-5.4-mini", "gpt-5.4-nano", "gpt-5.4-pro", "gpt-5.5-pro", - "gpt-5.6-sol", - "gpt-5.6-terra", - "gpt-5.6-luna", // OpenRouter open-weight models: priced in pricing.py only for the Python // max_usd static fallback. The evalboard deliberately does NOT statically // price them — OpenRouter routes per-request, so it shows the captured diff --git a/evalboard/lib/__tests__/pricing.test.ts b/evalboard/lib/__tests__/pricing.test.ts index 66302cf1..7c6ea53c 100644 --- a/evalboard/lib/__tests__/pricing.test.ts +++ b/evalboard/lib/__tests__/pricing.test.ts @@ -35,8 +35,16 @@ describe("resolvePricing", () => { expect(resolvePricing("__proto__")).toBeNull(); }); - test("knows the current default opus id", () => { - expect(resolvePricing("claude-opus-4-8")?.outputPerMTok).toBe(75); + test("knows the current default opus id, at the repriced tier", () => { + // Opus 4.5 REPRICED the family from $15/$75 to $5/$25 per Mtok. The two + // generations differ 3x, so pin the boundary: an id on the wrong side of + // it triples (or thirds) every Opus cost the board renders, which reads + // as a plausible number rather than an obvious error. + // pricing-parity.test.ts is the authority on the rates themselves; this + // asserts the split survives an edit to the table. + expect(resolvePricing("claude-opus-4-8")?.outputPerMTok).toBe(25); + expect(resolvePricing("claude-opus-5")?.outputPerMTok).toBe(25); + expect(resolvePricing("claude-opus-4-1")?.outputPerMTok).toBe(75); }); test("strips LiteLLM/Bedrock routing + region prefixes (recorded model_used is qualified)", () => { diff --git a/evalboard/lib/format.ts b/evalboard/lib/format.ts index d801c526..a1f6241f 100644 --- a/evalboard/lib/format.ts +++ b/evalboard/lib/format.ts @@ -17,6 +17,15 @@ export function fmtRunTime(id: string): string { return `${d} · ${t.replace(/-/g, ":")}`; } +// Date-only form of fmtRunTime, for columns that need to answer "how old is +// this" rather than "which run exactly". Non-date-shaped ids pass through +// (reusing fmtRunTime's guard rather than a bare split, which would hand back +// "codex" for an ad-hoc id). +export function fmtRunDate(id: string): string { + if (!DAILY_RUN_ID_RE.test(id)) return id; + return id.split("_")[0]; +} + // Format a run.json ISO timestamp (`start_time`, "YYYY-MM-DDTHH:MM:SS[.ffffff]") // into the same "YYYY-MM-DD · HH:MM:SS" shape fmtRunTime renders for date-shaped // run ids — so an ad-hoc run, whose id carries no date, shows a comparable diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index e02215d6..7540f1ef 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -515,13 +515,21 @@ function loadWindowData(window: Window): Promise { return loadWindowDataInner(window); } +// Repo-provenance half of taskMatchesTag: the tag as the task's own YAML +// declared it, stamped into run.json at execution time. This is the ONLY half +// whose absence in a newer run proves the tag was removed — review tags are +// post-hoc annotations, and an unreviewed run carries none. Split out so +// buildTagTaskRows can use exactly this half without inlining a second copy. +export function taskCarriesRepoTag(task: RunOverviewTask, tag: string): boolean { + return task.skill === tag || task.tags.includes(tag); +} + export function taskMatchesTag( task: RunOverviewTask, reviewTagsByTask: Record, tag: string, ): boolean { - if (task.skill === tag) return true; - if (task.tags.includes(tag)) return true; + if (taskCarriesRepoTag(task, tag)) return true; const rt = reviewTagsByTask[task.taskId]; return rt ? rt.includes(tag) : false; } @@ -624,76 +632,168 @@ export async function getOverview( export interface TagTaskRow { taskId: string; skill: string | null; - // How many runs in the window carried this task under the tag. + // Tagged task ROWS in the window, not distinct runs: a replicated task + // contributes one per replicate. Unchanged from the previous behaviour + // (the column header stays "Appearances") — the de-tag check below is the + // only place that collapses to one sample per run. Includes rows the + // nightly skipped as mature and carried forward. appearances: number; - passRate: number; // 0-100 across those appearances + // Of `appearances`, how many were mature carry-forwards (not executed). + matureSkips: number; + // 0-100 over EXECUTED appearances only (appearances - matureSkips). + // null when nothing in the window actually ran, so the UI shows "—" + // rather than a measured-looking 0% or 100%. + passRate: number | null; latestStatus: string | null; latestScore: number | null; latestRunId: string; + // True when the newest tagged ROW — the same row latestStatus and + // latestScore come from — was a mature carry-forward, so those two are + // inherited, not measured. + latestMatureSkipped: boolean; } -// Per-task breakdown for a single tag, windowed like getOverview but grouped -// by task instead of by run. One row per distinct task_id carrying the tag -// anywhere in the window; "latest" fields come from the newest run the task -// appeared in (runs are walked newest-first, so the first occurrence wins). -export async function getTagTaskBreakdown( - window: Window, - tag: string, - harness: string | null = null, -): Promise { - const perRun = (await loadWindowData(window)).filter( - (r) => - !r.adhoc && - (harness == null || - normalizeHarness(r.overview?.harness) === harness), - ); +// Per-task breakdown for a single tag, windowed like getOverview but grouped by +// task instead of by run. Pure over the PerRun[] it is handed (the caller does +// the fetching and the adhoc/harness filtering), so it unit-tests without +// touching the blob store — mirrors lib/trends.ts::aggregate. +// +// DE-TAGGING. A run.json task row carries `tags` as a historical stamp written +// at execution time; the board never consults the skills repo. So a task +// de-tagged upstream keeps rendering for as long as a run that predates the +// removal stays in the window. The rule here: a task is dropped when it appears +// in a NEWER run in the window whose rows for it do not carry the tag — that is +// proof the tag was removed, not a heuristic. A task that merely stopped +// appearing (retired, renamed, `skip: true`) is unknowable and therefore KEPT, +// with `latestRunId` doubling as "last seen" so its age is visible. +// +// The signal comes from taskCarriesRepoTag, NOT taskMatchesTag: review tags +// (review_index.json) are post-hoc annotations from a separate namespace, so an +// as-yet-unreviewed newest run — the normal case — would otherwise read as a +// de-tag. The same predicate is used to accumulate, so the narrowing is +// symmetric: a task pulled onto this page only by a review tag does not appear. +// +// The "did any row carry the tag this run" collapse is required because a +// replicated task has several rows per run, and one untagged replicate must not +// read as a de-tag. +// +// MATURITY — a DELIBERATE, page-local divergence. lib/trends.ts:158-171 (and +// app/runs/[id]/run-view.tsx) count a mature carry-forward as a pass and exclude +// it only from the cost/duration averages. Here it is excluded from BOTH the +// numerator and the denominator of `passRate`, because /path-to-ga is a +// GA-readiness page and must report MEASURED passes. That difference is +// intentional — do not "harmonise" this with trends.ts. +export function buildTagTaskRows(perRun: PerRun[], tag: string): TagTaskRow[] { + // Run ids are date-shaped, so a lexical sort is chronological — the same + // assumption trends.ts:90 and the previous implementation already make. const sorted = [...perRun].sort((a, b) => b.id.localeCompare(a.id)); interface Acc { skill: string | null; - statuses: (string | null)[]; + appearances: number; + matureSkips: number; + executedPasses: number; latestRunId: string; latestStatus: string | null; latestScore: number | null; + latestMatureSkipped: boolean; } const byTask = new Map(); - - for (const { id, overview, reviewTagsByTask } of sorted) { + // taskId -> did its NEWEST appearance in the window carry the tag. First + // write wins because the walk is newest-first, so a task that only gained + // the tag recently reads as tagged (and one that lost it reads as untagged) + // regardless of what the older runs say. + const newestTagged = new Map(); + + for (const { id, overview } of sorted) { + // A run whose run.json failed to load (loadPerRunForId downgrades to a + // null overview) must contribute neither an appearance nor a de-tag + // signal — otherwise a transient blob failure on the newest run would + // drop every row. if (!overview) continue; + + const seenInRun = new Set(); + const taggedInRun = new Set(); for (const t of overview.tasks) { - if (!taskMatchesTag(t, reviewTagsByTask, tag)) continue; + seenInRun.add(t.taskId); + if (taskCarriesRepoTag(t, tag)) taggedInRun.add(t.taskId); + } + for (const taskId of seenInRun) { + if (!newestTagged.has(taskId)) { + newestTagged.set(taskId, taggedInRun.has(taskId)); + } + } + + for (const t of overview.tasks) { + if (!taskCarriesRepoTag(t, tag)) continue; let entry = byTask.get(t.taskId); if (!entry) { + // All three latest* fields come off ONE row — the first tagged + // row of the newest-first walk — so the Mature pill and the + // dashed-out score always describe the same sample. entry = { skill: t.skill, - statuses: [], + appearances: 0, + matureSkips: 0, + executedPasses: 0, latestRunId: id, latestStatus: t.status, latestScore: t.weightedScore, + latestMatureSkipped: t.matureSkipped ?? false, }; byTask.set(t.taskId, entry); } - entry.statuses.push(t.status); + entry.appearances += 1; + if (t.matureSkipped) { + entry.matureSkips += 1; + } else if (t.status === "SUCCESS") { + entry.executedPasses += 1; + } } } const rows: TagTaskRow[] = []; for (const [taskId, e] of byTask) { - const appearances = e.statuses.length; - const passed = e.statuses.filter((s) => s === "SUCCESS").length; + // Provably de-tagged: the task is still running, and its newest run does + // not carry the tag. (A task in byTask always has a newestTagged entry — + // both are written from the same non-null-overview iteration — so the + // `?? true` only satisfies Map.get's `| undefined`; it is not a real + // "unknown ⇒ keep" case.) + if (!(newestTagged.get(taskId) ?? true)) continue; + const executed = e.appearances - e.matureSkips; rows.push({ taskId, skill: e.skill, - appearances, - passRate: appearances ? (passed / appearances) * 100 : 0, + appearances: e.appearances, + matureSkips: e.matureSkips, + passRate: executed > 0 ? (e.executedPasses / executed) * 100 : null, latestStatus: e.latestStatus, latestScore: e.latestScore, latestRunId: e.latestRunId, + latestMatureSkipped: e.latestMatureSkipped, }); } return rows.sort((a, b) => a.taskId.localeCompare(b.taskId)); } +// IO wrapper around buildTagTaskRows: fetch the window, drop ad-hoc runs and +// (optionally) scope to one harness, then aggregate. Harness scoping happens +// HERE, before the pure function sees the runs, so a newer run on a different +// harness cannot de-tag a row in a harness-scoped view. +export async function getTagTaskBreakdown( + window: Window, + tag: string, + harness: string | null = null, +): Promise { + const perRun = (await loadWindowData(window)).filter( + (r) => + !r.adhoc && + (harness == null || + normalizeHarness(r.overview?.harness) === harness), + ); + return buildTagTaskRows(perRun, tag); +} + // The slice of a run that the active tag/q filter selects: which tasks count, // and the cost/duration summed over exactly those. null means the run has // nothing matching and drops out entirely. diff --git a/evalboard/lib/pricing.ts b/evalboard/lib/pricing.ts index ad6d0293..5a122b3d 100644 --- a/evalboard/lib/pricing.ts +++ b/evalboard/lib/pricing.ts @@ -19,28 +19,49 @@ export interface Pricing { // build on drift — this hand-copied mirror is otherwise guarded only by a // comment. Not part of the consumer API; use resolvePricing() instead. export const PRICING: Record = { - "claude-opus-4-8": p(15, 75, 18.75, 1.5), - "claude-opus-4-7": p(15, 75, 18.75, 1.5), - "claude-opus-4-6": p(15, 75, 18.75, 1.5), - "claude-opus-4-6-20250514": p(15, 75, 18.75, 1.5), - "claude-opus-4-5-20251101": p(15, 75, 18.75, 1.5), + // Claude. Opus 4.5 and later are priced at the POST-repricing $5/$25 rates, + // not Opus 4.1's $15/$75 — the two generations differ 3x, so an undated + // alias must never inherit the older tier. Undated aliases each need their + // own key: resolvePricing's fallback only strips a trailing date (dated → + // undated), it cannot invent one. + "claude-fable-5": p(10, 50, 12.5, 1), + "claude-opus-5": p(5, 25, 6.25, 0.5), + "claude-opus-4-8": p(5, 25, 6.25, 0.5), + "claude-opus-4-7": p(5, 25, 6.25, 0.5), + "claude-opus-4-6": p(5, 25, 6.25, 0.5), + "claude-opus-4-5": p(5, 25, 6.25, 0.5), + "claude-opus-4-5-20251101": p(5, 25, 6.25, 0.5), + "claude-opus-4-1": p(15, 75, 18.75, 1.5), + "claude-opus-4": p(15, 75, 18.75, 1.5), "claude-opus-4-20250514": p(15, 75, 18.75, 1.5), + "claude-sonnet-5": p(3, 15, 3.75, 0.3), "claude-sonnet-4-6": p(3, 15, 3.75, 0.3), - "claude-sonnet-4-6-20250514": p(3, 15, 3.75, 0.3), + "claude-sonnet-4-5": p(3, 15, 3.75, 0.3), "claude-sonnet-4-5-20250929": p(3, 15, 3.75, 0.3), "claude-sonnet-4-20250514": p(3, 15, 3.75, 0.3), - "claude-haiku-4-5-20251001": p(0.8, 4, 1, 0.08), + "claude-haiku-4-5": p(1, 5, 1.25, 0.1), + "claude-haiku-4-5-20251001": p(1, 5, 1.25, 0.1), + "claude-haiku-3-5": p(0.8, 4, 1, 0.08), "claude-3-7-sonnet-20250219": p(3, 15, 3.75, 0.3), "claude-3-5-sonnet-20241022": p(3, 15, 3.75, 0.3), "claude-3-5-sonnet-20240620": p(3, 15, 3.75, 0.3), "claude-3-opus-20240229": p(15, 75, 18.75, 1.5), "claude-3-sonnet-20240229": p(3, 15, 3.75, 0.3), "claude-3-haiku-20240307": p(0.25, 1.25, 0.3, 0.03), + // OpenAI (CodexAgent). "gpt-5-codex": p(1.25, 10, 1.25, 0.125), "gpt-5": p(1.25, 10, 1.25, 0.125), + "gpt-5.1-codex-max": p(1.25, 10, 1.25, 0.125), + "gpt-5.1-codex": p(1.25, 10, 1.25, 0.125), + "gpt-5.1-codex-mini": p(0.25, 2, 0.25, 0.025), + "codex-mini-latest": p(1.5, 6, 1.5, 0.375), "gpt-5.3-codex": p(1.75, 14, 1.75, 0.175), + "gpt-5.2-codex": p(1.75, 14, 1.75, 0.175), "gpt-5.4": p(2.5, 15, 2.5, 0.25), "gpt-5.5": p(5, 30, 5, 0.5), + "gpt-5.6-sol": p(5, 30, 5, 0.5), + "gpt-5.6-terra": p(2.5, 15, 2.5, 0.25), + "gpt-5.6-luna": p(1, 6, 1, 0.1), // Google Gemini (AntigravityAgent). Gemini bills no separate cache-write // fee (cache_write == input, effectively unused); cache_read is the cached- // input rate. Pro's >200K-token tier is higher — this flat rate reads low @@ -48,8 +69,12 @@ export const PRICING: Record = { "gemini-3-pro-preview": p(2, 12, 2, 0.2), "gemini-3.1-pro-preview": p(2, 12, 2, 0.2), "gemini-3.1-pro-preview-customtools": p(2, 12, 2, 0.2), + "gemini-3.6-flash": p(1.5, 7.5, 1.5, 0.15), "gemini-3.5-flash": p(1.5, 9, 1.5, 0.15), - "gemini-3-flash-preview": p(1.5, 9, 1.5, 0.15), + "gemini-3.5-flash-lite": p(0.3, 2.5, 0.3, 0.03), + "gemini-3.1-flash-lite": p(0.25, 1.5, 0.25, 0.025), + "gemini-3.1-flash-lite-preview": p(0.25, 1.5, 0.25, 0.025), + "gemini-3-flash-preview": p(0.5, 3, 0.5, 0.05), // OpenRouter open-weight models (litellm backend) are DELIBERATELY NOT priced // here. OpenRouter routes per-request, so a static headline rate is wrong (the // billed rate depends on the provider it landed on), and there is no per-bucket