From 5b8bb81b32c1135487400c0153f8b617734dd997 Mon Sep 17 00:00:00 2001 From: Deep Kumar Singh Kushwah Date: Wed, 15 Jul 2026 02:42:07 +0530 Subject: [PATCH] =?UTF-8?q?feat(match):=20confidence=20calibration,=20hone?= =?UTF-8?q?sty=20metrics,=20sharp=20disambiguation=20(D2/D6/G1)=20?= =?UTF-8?q?=E2=80=94=20Gate=204?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Close out the matching engine (TRACKER step 4.5): - Calibration: confidence thresholds live in core (CONFIDENCE_THRESHOLDS); a new `node eval/dist/run.js --calibrate` measures precision at the `high` cutoff on the eval set and records eval/calibration.json (precision@high 1.000, n=71). - Honesty metrics + gates: the scorecard now reports highConfidenceCorrect, ambiguityHonesty, and poisonRate; thresholds.json gates them (=1.0 / ≥0.90 / ≤0.05). Current run: 1.000 / 1.000 / 0.000. - Ambiguity protocol (D6/G1): the `disambiguation` question is now built from the DIFFERENCES between the tied leaders — "Which one — BillingForm (shows 'Card number'), or ProfileForm (shows 'Display name')?" — so the caller can answer with a resolving term. Declined reasons stay machine-readable (no-signal / not-our-app / …). 40 core + 3 vision + 102 parser tests pass; eval green with the new gates. **Gate 4 passes — matching engine complete (M4).** Co-Authored-By: Claude Opus 4.8 --- TRACKER.md | 10 ++-- eval/calibration.json | 11 +++++ eval/src/checks.ts | 22 ++++++++- eval/src/golden.ts | 24 ++++++++++ eval/src/run.ts | 75 ++++++++++++++++++++++++++++++ eval/thresholds.json | 5 +- packages/core/src/matching.test.ts | 9 ++++ packages/core/src/query.ts | 20 ++++++-- packages/core/src/result.ts | 16 +++++-- 9 files changed, 178 insertions(+), 14 deletions(-) create mode 100644 eval/calibration.json diff --git a/TRACKER.md b/TRACKER.md index 8ee6efe..2e3be31 100644 --- a/TRACKER.md +++ b/TRACKER.md @@ -4,10 +4,10 @@ ## Status -- **Current phase:** 4 — Matching engine -- **Next step:** 4.5 — Confidence calibration & ambiguity protocol (last step for Gate 4) -- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.5, 4.1–4.4, 4.6 (4.5 deferred — Gate 4 pending it) -- **Gates passed:** Gate 0 (CI + red-path, #5/#6) · Gate 1 (precision 1.000, recall 0.895, zero poison) · Gate 2 (C1 instance attribution 1.000 · B1 4-level handler chains · C6 store writers↔readers · A9 portals — scorecard 137/0/0, precision & recall 1.000) · Gate 3 (B3 action effects · B4 routers · B6 cyclic journeys terminate · B7/B8 form & non-JSX events · G5 flag/role conditions — precision & recall 1.000) +- **Current phase:** 5 — Context bundle & agent interface +- **Next step:** 5.1 — `resolveContext` orchestrator +- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.5, 4.1–4.6 +- **Gates passed:** Gate 0 (CI + red-path, #5/#6) · Gate 1 (precision 1.000, recall 0.895, zero poison) · Gate 2 (C1 instance attribution 1.000 · B1 4-level handler chains · C6 store writers↔readers · A9 portals — scorecard 137/0/0, precision & recall 1.000) · Gate 3 (B3 action effects · B4 routers · B6 cyclic journeys terminate · B7/B8 form & non-JSX events · G5 flag/role conditions — precision & recall 1.000) · Gate 4 (A4 rarity · A10 fuzzy/OCR · A1 structural · A6 subtree · E3 vision annotations · E2 aliases · G4 corrections — high-conf correct 1.000, ambiguity honesty 1.000, poison rate 0.000) ## What CodeRadar is @@ -245,7 +245,7 @@ The heart of the project. C1 and B1 live here. - **Ephemeral only** (G7): images processed in memory, never written to the graph or disk; document in the package README. **Accept:** interface unit-tested against recorded extraction outputs (no live API in CI); annotation weighting verified on `e3-annotated-screenshot` fixture (pre-extracted terms + regions checked in). -### [ ] 4.5 Confidence calibration & ambiguity protocol +### [x] 4.5 Confidence calibration & ambiguity protocol **Failure modes:** D2, D6, G1 **Build:** - Score → confidence mapping calibrated on the eval set (thresholds chosen so measured precision at `high` ≥ 0.95 — recorded in `eval/calibration.json`, regenerated by an eval subcommand). diff --git a/eval/calibration.json b/eval/calibration.json new file mode 100644 index 0000000..f5c81c6 --- /dev/null +++ b/eval/calibration.json @@ -0,0 +1,11 @@ +{ + "generatedAt": "2026-07-14T21:10:13.631Z", + "thresholds": { + "high": 0.8, + "medium": 0.5 + }, + "measuredPrecisionAtHigh": 1, + "empiricalHighFloor": 1, + "sampleSize": 71, + "note": "The matcher's confidenceFromScore uses these thresholds. measuredPrecisionAtHigh is the fraction of high-confidence eval answers that are correct; empiricalHighFloor is the lowest score at which precision still holds >= 0.95." +} diff --git a/eval/src/checks.ts b/eval/src/checks.ts index 47d7891..6755acc 100644 --- a/eval/src/checks.ts +++ b/eval/src/checks.ts @@ -7,7 +7,7 @@ import { traceLineage, } from "@coderadar/core"; -import type { CheckResult, FixtureResult, Golden } from "./golden.js"; +import type { CheckResult, FixtureResult, Golden, QueryOutcome } from "./golden.js"; export function runChecks( fixture: string, @@ -17,6 +17,7 @@ export function runChecks( ): FixtureResult { const checks: CheckResult[] = []; const attribution = { truePositives: 0, falsePositives: 0, falseNegatives: 0 }; + const queryOutcomes: QueryOutcome[] = []; const finalize = ( kind: CheckResult["kind"], @@ -249,9 +250,26 @@ export function runChecks( } } finalize("queries", id, passed, query.expectedFail, detail); + + if (query.expectedFail === undefined) { + const top = result.candidates[0]; + queryOutcomes.push({ + terms: query.terms, + expectedStatus: query.status, + gotStatus: result.status, + ...(top !== undefined + ? { + top: top.value.component.name, + confidence: top.confidence.level, + score: top.confidence.score, + } + : {}), + correct: passed, + }); + } } - return { fixture, failureMode: golden.failureMode, checks, attribution }; + return { fixture, failureMode: golden.failureMode, checks, attribution, queries: queryOutcomes }; } /** Endpoints reached from a definition or a specific instance. Null if the target is missing. */ diff --git a/eval/src/golden.ts b/eval/src/golden.ts index 808fc5b..c2de573 100644 --- a/eval/src/golden.ts +++ b/eval/src/golden.ts @@ -148,12 +148,27 @@ export interface CheckResult { detail?: string; } +/** Per-query outcome, for confidence calibration and honesty metrics (step 4.5). */ +export interface QueryOutcome { + terms: string[]; + expectedStatus: "ok" | "ambiguous" | "declined"; + gotStatus: "ok" | "ambiguous" | "declined"; + /** Top candidate name + its confidence (ok results). */ + top?: string; + confidence?: "high" | "medium" | "low"; + score?: number; + /** ok: top === golden.top; otherwise gotStatus === expectedStatus. */ + correct: boolean; +} + export interface FixtureResult { fixture: string; failureMode: string; checks: CheckResult[]; /** Attribution tallies for lineage precision/recall. */ attribution: { truePositives: number; falsePositives: number; falseNegatives: number }; + /** Query outcomes (populated for fixtures with `queries`). */ + queries: QueryOutcome[]; } export interface Scorecard { @@ -168,6 +183,12 @@ export interface Scorecard { lineagePrecision: number | null; lineageRecall: number | null; matchAccuracy: number | null; + /** Fraction of high-confidence `ok` answers that are correct (step 4.5). */ + highConfidenceCorrect: number | null; + /** Fraction of genuinely-ambiguous queries the matcher returned ambiguous. */ + ambiguityHonesty: number | null; + /** Fraction of `ok` answers that are confidently wrong (the poison rate). */ + poisonRate: number | null; }; } @@ -178,4 +199,7 @@ export interface Thresholds { minLineagePrecision?: number; minLineageRecall?: number; minMatchAccuracy?: number; + minHighConfidenceCorrect?: number; + minAmbiguityHonesty?: number; + maxPoisonRate?: number; } diff --git a/eval/src/run.ts b/eval/src/run.ts index 050fe60..811dc75 100644 --- a/eval/src/run.ts +++ b/eval/src/run.ts @@ -14,6 +14,7 @@ import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { CONFIDENCE_THRESHOLDS } from "@coderadar/core"; import { resolveHookEdges, scanReact } from "@coderadar/parser-react"; import { parse as parseYaml } from "yaml"; @@ -61,10 +62,50 @@ function main(): void { ); } + if (process.argv.includes("--calibrate")) calibrate(results); + print(scorecard); gate(scorecard); } +/** + * Measure the score → confidence calibration on the eval set and record it in + * eval/calibration.json: the precision at the matcher's `high` cutoff, plus the + * lowest score threshold at which precision still holds ≥ 0.95 (step 4.5). + */ +function calibrate(results: FixtureResult[]): void { + const ok = results + .flatMap((r) => r.queries) + .filter((o) => o.gotStatus === "ok" && o.score !== undefined); + const precisionAbove = (t: number): number => { + const bucket = ok.filter((o) => (o.score ?? 0) >= t); + return bucket.length > 0 ? bucket.filter((o) => o.correct).length / bucket.length : 1; + }; + const scores = [...new Set(ok.map((o) => o.score ?? 0))].sort((a, b) => b - a); + let empiricalHighFloor = 1; + for (const t of scores) { + if (precisionAbove(t) >= 0.95) empiricalHighFloor = t; + else break; + } + const calibration = { + generatedAt: new Date().toISOString(), + thresholds: CONFIDENCE_THRESHOLDS, + measuredPrecisionAtHigh: round(precisionAbove(CONFIDENCE_THRESHOLDS.high)), + empiricalHighFloor: round(empiricalHighFloor), + sampleSize: ok.length, + note: "The matcher's confidenceFromScore uses these thresholds. measuredPrecisionAtHigh is the fraction of high-confidence eval answers that are correct; empiricalHighFloor is the lowest score at which precision still holds >= 0.95.", + }; + fs.writeFileSync( + path.join(evalDir, "calibration.json"), + `${JSON.stringify(calibration, null, 2)}\n`, + ); + console.log( + `\nWrote calibration.json — high=${CONFIDENCE_THRESHOLDS.high}, precision@high=${round( + precisionAbove(CONFIDENCE_THRESHOLDS.high), + )}, floor=${round(empiricalHighFloor)} (n=${ok.length})`, + ); +} + function buildScorecard(results: FixtureResult[]): Scorecard { let pass = 0; let fail = 0; @@ -92,6 +133,13 @@ function buildScorecard(results: FixtureResult[]): Scorecard { const attributed = attr.truePositives + attr.falsePositives; const golden = attr.truePositives + attr.falseNegatives; + + // Confidence & honesty metrics (step 4.5) over every non-xfail query outcome. + const outcomes = results.flatMap((r) => r.queries); + const okAnswers = outcomes.filter((o) => o.gotStatus === "ok"); + const highOk = okAnswers.filter((o) => o.confidence === "high"); + const ambiguous = outcomes.filter((o) => o.expectedStatus === "ambiguous"); + return { generatedAt: new Date().toISOString(), commitSha: gitSha(), @@ -104,6 +152,16 @@ function buildScorecard(results: FixtureResult[]): Scorecard { lineagePrecision: attributed > 0 ? round(attr.truePositives / attributed) : null, lineageRecall: golden > 0 ? round(attr.truePositives / golden) : null, matchAccuracy: queryTotal > 0 ? round(queryPass / queryTotal) : null, + highConfidenceCorrect: + highOk.length > 0 ? round(highOk.filter((o) => o.correct).length / highOk.length) : null, + ambiguityHonesty: + ambiguous.length > 0 + ? round(ambiguous.filter((o) => o.gotStatus === "ambiguous").length / ambiguous.length) + : null, + poisonRate: + okAnswers.length > 0 + ? round(okAnswers.filter((o) => !o.correct).length / okAnswers.length) + : null, }, }; } @@ -124,6 +182,9 @@ function print(scorecard: Scorecard): void { console.log( `lineage precision=${fmt(s.lineagePrecision)} recall=${fmt(s.lineageRecall)} · match accuracy=${fmt(s.matchAccuracy)}`, ); + console.log( + `high-conf correct=${fmt(s.highConfidenceCorrect)} · ambiguity honesty=${fmt(s.ambiguityHonesty)} · poison rate=${fmt(s.poisonRate)}`, + ); } function gate(scorecard: Scorecard): void { @@ -139,6 +200,20 @@ function gate(scorecard: Scorecard): void { checkFloor(violations, "lineagePrecision", s.lineagePrecision, thresholds.minLineagePrecision); checkFloor(violations, "lineageRecall", s.lineageRecall, thresholds.minLineageRecall); checkFloor(violations, "matchAccuracy", s.matchAccuracy, thresholds.minMatchAccuracy); + checkFloor( + violations, + "highConfidenceCorrect", + s.highConfidenceCorrect, + thresholds.minHighConfidenceCorrect, + ); + checkFloor(violations, "ambiguityHonesty", s.ambiguityHonesty, thresholds.minAmbiguityHonesty); + if ( + thresholds.maxPoisonRate !== undefined && + s.poisonRate !== null && + s.poisonRate > thresholds.maxPoisonRate + ) { + violations.push(`poisonRate ${s.poisonRate} > max ${thresholds.maxPoisonRate}`); + } if (violations.length > 0) { console.error(`\nEVAL GATE FAILED:\n ${violations.join("\n ")}`); diff --git a/eval/thresholds.json b/eval/thresholds.json index 6233934..ded3b1d 100644 --- a/eval/thresholds.json +++ b/eval/thresholds.json @@ -3,5 +3,8 @@ "maxUnexpectedPass": 0, "minMatchAccuracy": 1, "minLineagePrecision": 0.95, - "minLineageRecall": 0.95 + "minLineageRecall": 0.95, + "minHighConfidenceCorrect": 1, + "minAmbiguityHonesty": 0.9, + "maxPoisonRate": 0.05 } diff --git a/packages/core/src/matching.test.ts b/packages/core/src/matching.test.ts index 5206023..29537c0 100644 --- a/packages/core/src/matching.test.ts +++ b/packages/core/src/matching.test.ts @@ -67,6 +67,15 @@ describe("matchComponentsByText scorer (TRACKER 4.1, A4/A10)", () => { expect(result.candidates[0]?.value.component.name).toBe("BillingForm"); }); + it("the disambiguation question names each leader's distinctive text (D6)", () => { + const result = matchComponentsByText(graph(forms), ["Save"]); + expect(result.status).toBe("ambiguous"); + // Built from the DIFFERENCES: each tied candidate's unique text. + expect(result.disambiguation).toContain("Card number"); + expect(result.disambiguation).toContain("Notifications"); + expect(result.disambiguation).toContain("Display name"); + }); + it("respects word order — 'Order deleted' ≠ 'Delete order'", () => { const g = graph([ component("Toast", ["Order deleted"]), diff --git a/packages/core/src/query.ts b/packages/core/src/query.ts index 9b4cead..17e11de 100644 --- a/packages/core/src/query.ts +++ b/packages/core/src/query.ts @@ -348,11 +348,25 @@ export function matchComponents( const tied = top === undefined ? [] : winners.filter((s) => Math.abs(s.score - top.score) < 1e-9); if (tied.length > 1) { - const names = tied.map((s) => s.match.component.name).join(", "); + // A concrete question built from the DIFFERENCES between the leaders: each + // gets the first text unique to it, so the caller can answer with a term + // that resolves the tie (D6/G1). + const options = tied.slice(0, 3).map((s) => { + const elsewhere = new Set( + tied + .filter((o) => o !== s) + .flatMap((o) => o.match.component.renderedText.map((r) => normalizeText(r.text))), + ); + const distinctive = s.match.component.renderedText.find( + (r) => normalizeText(r.text).length > 0 && !elsewhere.has(normalizeText(r.text)), + ); + return distinctive !== undefined + ? `${s.match.component.name} (shows "${distinctive.text}")` + : s.match.component.name; + }); return ambiguous( candidates, - `Several components match equally well (${names}). ` + - `Which page is the screenshot from, or what other distinctive text is visible?`, + `Which one — ${options.join(", or ")}? Add a term unique to the one you mean.`, ); } return ok(candidates); diff --git a/packages/core/src/result.ts b/packages/core/src/result.ts index 2312078..6d35296 100644 --- a/packages/core/src/result.ts +++ b/packages/core/src/result.ts @@ -44,13 +44,23 @@ export function declined(reason: DeclineReason): QueryResult { } /** - * Map a 0–1 score to a confidence level. - * Provisional thresholds — replaced by measured calibration in Phase 4.5. + * Score → confidence thresholds, calibrated on the eval set (TRACKER step 4.5): + * every `high` answer in the eval set is correct at these cutoffs. The eval + * `--calibrate` subcommand measures precision at these levels and records it in + * eval/calibration.json. */ +export const CONFIDENCE_THRESHOLDS = { high: 0.8, medium: 0.5 } as const; + +/** Map a 0–1 score to a calibrated confidence level. */ export function confidenceFromScore(score: number): Confidence { const clamped = Math.max(0, Math.min(1, score)); return { score: clamped, - level: clamped >= 0.8 ? "high" : clamped >= 0.5 ? "medium" : "low", + level: + clamped >= CONFIDENCE_THRESHOLDS.high + ? "high" + : clamped >= CONFIDENCE_THRESHOLDS.medium + ? "medium" + : "low", }; }