feat(match): confidence calibration, honesty metrics, disambiguation (D2/D6/G1) — Gate 4

TRACKER.md

@@ -4,10 +4,10 @@
 
 ## Status
 
-- **Current phase:** 4 — Matching engine
-- **Next step:** 4.5 — Confidence calibration & ambiguity protocol (last step for Gate 4)
-- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.5, 4.1–4.4, 4.6 (4.5 deferred — Gate 4 pending it)
-- **Gates passed:** Gate 0 (CI + red-path, #5/#6) · Gate 1 (precision 1.000, recall 0.895, zero poison) · Gate 2 (C1 instance attribution 1.000 · B1 4-level handler chains · C6 store writers↔readers · A9 portals — scorecard 137/0/0, precision & recall 1.000) · Gate 3 (B3 action effects · B4 routers · B6 cyclic journeys terminate · B7/B8 form & non-JSX events · G5 flag/role conditions — precision & recall 1.000)
+- **Current phase:** 5 — Context bundle & agent interface
+- **Next step:** 5.1 — `resolveContext` orchestrator
+- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.5, 4.1–4.6
+- **Gates passed:** Gate 0 (CI + red-path, #5/#6) · Gate 1 (precision 1.000, recall 0.895, zero poison) · Gate 2 (C1 instance attribution 1.000 · B1 4-level handler chains · C6 store writers↔readers · A9 portals — scorecard 137/0/0, precision & recall 1.000) · Gate 3 (B3 action effects · B4 routers · B6 cyclic journeys terminate · B7/B8 form & non-JSX events · G5 flag/role conditions — precision & recall 1.000) · Gate 4 (A4 rarity · A10 fuzzy/OCR · A1 structural · A6 subtree · E3 vision annotations · E2 aliases · G4 corrections — high-conf correct 1.000, ambiguity honesty 1.000, poison rate 0.000)
 
 ## What CodeRadar is
 
@@ -245,7 +245,7 @@ The heart of the project. C1 and B1 live here.
 - **Ephemeral only** (G7): images processed in memory, never written to the graph or disk; document in the package README.
 **Accept:** interface unit-tested against recorded extraction outputs (no live API in CI); annotation weighting verified on `e3-annotated-screenshot` fixture (pre-extracted terms + regions checked in).
 
-### [ ] 4.5 Confidence calibration & ambiguity protocol
+### [x] 4.5 Confidence calibration & ambiguity protocol
 **Failure modes:** D2, D6, G1
 **Build:**
 - Score → confidence mapping calibrated on the eval set (thresholds chosen so measured precision at `high` ≥ 0.95 — recorded in `eval/calibration.json`, regenerated by an eval subcommand).

eval/calibration.json

@@ -0,0 +1,11 @@
+{
+  "generatedAt": "2026-07-14T21:10:13.631Z",
+  "thresholds": {
+    "high": 0.8,
+    "medium": 0.5
+  },
+  "measuredPrecisionAtHigh": 1,
+  "empiricalHighFloor": 1,
+  "sampleSize": 71,
+  "note": "The matcher's confidenceFromScore uses these thresholds. measuredPrecisionAtHigh is the fraction of high-confidence eval answers that are correct; empiricalHighFloor is the lowest score at which precision still holds >= 0.95."
+}

eval/src/checks.ts

@@ -7,7 +7,7 @@ import {
   traceLineage,
 } from "@coderadar/core";
 
-import type { CheckResult, FixtureResult, Golden } from "./golden.js";
+import type { CheckResult, FixtureResult, Golden, QueryOutcome } from "./golden.js";
 
 export function runChecks(
   fixture: string,
@@ -17,6 +17,7 @@ export function runChecks(
 ): FixtureResult {
   const checks: CheckResult[] = [];
   const attribution = { truePositives: 0, falsePositives: 0, falseNegatives: 0 };
+  const queryOutcomes: QueryOutcome[] = [];
 
   const finalize = (
     kind: CheckResult["kind"],
@@ -249,9 +250,26 @@ export function runChecks(
       }
     }
     finalize("queries", id, passed, query.expectedFail, detail);
+
+    if (query.expectedFail === undefined) {
+      const top = result.candidates[0];
+      queryOutcomes.push({
+        terms: query.terms,
+        expectedStatus: query.status,
+        gotStatus: result.status,
+        ...(top !== undefined
+          ? {
+              top: top.value.component.name,
+              confidence: top.confidence.level,
+              score: top.confidence.score,
+            }
+          : {}),
+        correct: passed,
+      });
+    }
   }
 
-  return { fixture, failureMode: golden.failureMode, checks, attribution };
+  return { fixture, failureMode: golden.failureMode, checks, attribution, queries: queryOutcomes };
 }
 
 /** Endpoints reached from a definition or a specific instance. Null if the target is missing. */

eval/src/golden.ts

@@ -148,12 +148,27 @@ export interface CheckResult {
   detail?: string;
 }
 
+/** Per-query outcome, for confidence calibration and honesty metrics (step 4.5). */
+export interface QueryOutcome {
+  terms: string[];
+  expectedStatus: "ok" | "ambiguous" | "declined";
+  gotStatus: "ok" | "ambiguous" | "declined";
+  /** Top candidate name + its confidence (ok results). */
+  top?: string;
+  confidence?: "high" | "medium" | "low";
+  score?: number;
+  /** ok: top === golden.top; otherwise gotStatus === expectedStatus. */
+  correct: boolean;
+}
+
 export interface FixtureResult {
   fixture: string;
   failureMode: string;
   checks: CheckResult[];
   /** Attribution tallies for lineage precision/recall. */
   attribution: { truePositives: number; falsePositives: number; falseNegatives: number };
+  /** Query outcomes (populated for fixtures with `queries`). */
+  queries: QueryOutcome[];
 }
 
 export interface Scorecard {
@@ -168,6 +183,12 @@ export interface Scorecard {
     lineagePrecision: number | null;
     lineageRecall: number | null;
     matchAccuracy: number | null;
+    /** Fraction of high-confidence `ok` answers that are correct (step 4.5). */
+    highConfidenceCorrect: number | null;
+    /** Fraction of genuinely-ambiguous queries the matcher returned ambiguous. */
+    ambiguityHonesty: number | null;
+    /** Fraction of `ok` answers that are confidently wrong (the poison rate). */
+    poisonRate: number | null;
   };
 }
 
@@ -178,4 +199,7 @@ export interface Thresholds {
   minLineagePrecision?: number;
   minLineageRecall?: number;
   minMatchAccuracy?: number;
+  minHighConfidenceCorrect?: number;
+  minAmbiguityHonesty?: number;
+  maxPoisonRate?: number;
 }

eval/src/run.ts

@@ -14,6 +14,7 @@ import fs from "node:fs";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 
+import { CONFIDENCE_THRESHOLDS } from "@coderadar/core";
 import { resolveHookEdges, scanReact } from "@coderadar/parser-react";
 import { parse as parseYaml } from "yaml";
 
@@ -61,10 +62,50 @@ function main(): void {
     );
   }
 
+  if (process.argv.includes("--calibrate")) calibrate(results);
+
   print(scorecard);
   gate(scorecard);
 }
 
+/**
+ * Measure the score → confidence calibration on the eval set and record it in
+ * eval/calibration.json: the precision at the matcher's `high` cutoff, plus the
+ * lowest score threshold at which precision still holds ≥ 0.95 (step 4.5).
+ */
+function calibrate(results: FixtureResult[]): void {
+  const ok = results
+    .flatMap((r) => r.queries)
+    .filter((o) => o.gotStatus === "ok" && o.score !== undefined);
+  const precisionAbove = (t: number): number => {
+    const bucket = ok.filter((o) => (o.score ?? 0) >= t);
+    return bucket.length > 0 ? bucket.filter((o) => o.correct).length / bucket.length : 1;
+  };
+  const scores = [...new Set(ok.map((o) => o.score ?? 0))].sort((a, b) => b - a);
+  let empiricalHighFloor = 1;
+  for (const t of scores) {
+    if (precisionAbove(t) >= 0.95) empiricalHighFloor = t;
+    else break;
+  }
+  const calibration = {
+    generatedAt: new Date().toISOString(),
+    thresholds: CONFIDENCE_THRESHOLDS,
+    measuredPrecisionAtHigh: round(precisionAbove(CONFIDENCE_THRESHOLDS.high)),
+    empiricalHighFloor: round(empiricalHighFloor),
+    sampleSize: ok.length,
+    note: "The matcher's confidenceFromScore uses these thresholds. measuredPrecisionAtHigh is the fraction of high-confidence eval answers that are correct; empiricalHighFloor is the lowest score at which precision still holds >= 0.95.",
+  };
+  fs.writeFileSync(
+    path.join(evalDir, "calibration.json"),
+    `${JSON.stringify(calibration, null, 2)}\n`,
+  );
+  console.log(
+    `\nWrote calibration.json — high=${CONFIDENCE_THRESHOLDS.high}, precision@high=${round(
+      precisionAbove(CONFIDENCE_THRESHOLDS.high),
+    )}, floor=${round(empiricalHighFloor)} (n=${ok.length})`,
+  );
+}
+
 function buildScorecard(results: FixtureResult[]): Scorecard {
   let pass = 0;
   let fail = 0;
@@ -92,6 +133,13 @@ function buildScorecard(results: FixtureResult[]): Scorecard {
 
   const attributed = attr.truePositives + attr.falsePositives;
   const golden = attr.truePositives + attr.falseNegatives;
+
+  // Confidence & honesty metrics (step 4.5) over every non-xfail query outcome.
+  const outcomes = results.flatMap((r) => r.queries);
+  const okAnswers = outcomes.filter((o) => o.gotStatus === "ok");
+  const highOk = okAnswers.filter((o) => o.confidence === "high");
+  const ambiguous = outcomes.filter((o) => o.expectedStatus === "ambiguous");
+
   return {
     generatedAt: new Date().toISOString(),
     commitSha: gitSha(),
@@ -104,6 +152,16 @@ function buildScorecard(results: FixtureResult[]): Scorecard {
       lineagePrecision: attributed > 0 ? round(attr.truePositives / attributed) : null,
       lineageRecall: golden > 0 ? round(attr.truePositives / golden) : null,
       matchAccuracy: queryTotal > 0 ? round(queryPass / queryTotal) : null,
+      highConfidenceCorrect:
+        highOk.length > 0 ? round(highOk.filter((o) => o.correct).length / highOk.length) : null,
+      ambiguityHonesty:
+        ambiguous.length > 0
+          ? round(ambiguous.filter((o) => o.gotStatus === "ambiguous").length / ambiguous.length)
+          : null,
+      poisonRate:
+        okAnswers.length > 0
+          ? round(okAnswers.filter((o) => !o.correct).length / okAnswers.length)
+          : null,
     },
   };
 }
@@ -124,6 +182,9 @@ function print(scorecard: Scorecard): void {
   console.log(
     `lineage precision=${fmt(s.lineagePrecision)} recall=${fmt(s.lineageRecall)} · match accuracy=${fmt(s.matchAccuracy)}`,
   );
+  console.log(
+    `high-conf correct=${fmt(s.highConfidenceCorrect)} · ambiguity honesty=${fmt(s.ambiguityHonesty)} · poison rate=${fmt(s.poisonRate)}`,
+  );
 }
 
 function gate(scorecard: Scorecard): void {
@@ -139,6 +200,20 @@ function gate(scorecard: Scorecard): void {
   checkFloor(violations, "lineagePrecision", s.lineagePrecision, thresholds.minLineagePrecision);
   checkFloor(violations, "lineageRecall", s.lineageRecall, thresholds.minLineageRecall);
   checkFloor(violations, "matchAccuracy", s.matchAccuracy, thresholds.minMatchAccuracy);
+  checkFloor(
+    violations,
+    "highConfidenceCorrect",
+    s.highConfidenceCorrect,
+    thresholds.minHighConfidenceCorrect,
+  );
+  checkFloor(violations, "ambiguityHonesty", s.ambiguityHonesty, thresholds.minAmbiguityHonesty);
+  if (
+    thresholds.maxPoisonRate !== undefined &&
+    s.poisonRate !== null &&
+    s.poisonRate > thresholds.maxPoisonRate
+  ) {
+    violations.push(`poisonRate ${s.poisonRate} > max ${thresholds.maxPoisonRate}`);
+  }
 
   if (violations.length > 0) {
     console.error(`\nEVAL GATE FAILED:\n  ${violations.join("\n  ")}`);

eval/thresholds.json

@@ -3,5 +3,8 @@
   "maxUnexpectedPass": 0,
   "minMatchAccuracy": 1,
   "minLineagePrecision": 0.95,
-  "minLineageRecall": 0.95
+  "minLineageRecall": 0.95,
+  "minHighConfidenceCorrect": 1,
+  "minAmbiguityHonesty": 0.9,
+  "maxPoisonRate": 0.05
 }

packages/core/src/matching.test.ts

@@ -67,6 +67,15 @@ describe("matchComponentsByText scorer (TRACKER 4.1, A4/A10)", () => {
     expect(result.candidates[0]?.value.component.name).toBe("BillingForm");
   });
 
+  it("the disambiguation question names each leader's distinctive text (D6)", () => {
+    const result = matchComponentsByText(graph(forms), ["Save"]);
+    expect(result.status).toBe("ambiguous");
+    // Built from the DIFFERENCES: each tied candidate's unique text.
+    expect(result.disambiguation).toContain("Card number");
+    expect(result.disambiguation).toContain("Notifications");
+    expect(result.disambiguation).toContain("Display name");
+  });
+
   it("respects word order — 'Order deleted' ≠ 'Delete order'", () => {
     const g = graph([
       component("Toast", ["Order deleted"]),

packages/core/src/query.ts

@@ -348,11 +348,25 @@ export function matchComponents(
   const tied =
     top === undefined ? [] : winners.filter((s) => Math.abs(s.score - top.score) < 1e-9);
   if (tied.length > 1) {
-    const names = tied.map((s) => s.match.component.name).join(", ");
+    // A concrete question built from the DIFFERENCES between the leaders: each
+    // gets the first text unique to it, so the caller can answer with a term
+    // that resolves the tie (D6/G1).
+    const options = tied.slice(0, 3).map((s) => {
+      const elsewhere = new Set(
+        tied
+          .filter((o) => o !== s)
+          .flatMap((o) => o.match.component.renderedText.map((r) => normalizeText(r.text))),
+      );
+      const distinctive = s.match.component.renderedText.find(
+        (r) => normalizeText(r.text).length > 0 && !elsewhere.has(normalizeText(r.text)),
+      );
+      return distinctive !== undefined
+        ? `${s.match.component.name} (shows "${distinctive.text}")`
+        : s.match.component.name;
+    });
     return ambiguous(
       candidates,
-      `Several components match equally well (${names}). ` +
-        `Which page is the screenshot from, or what other distinctive text is visible?`,
+      `Which one — ${options.join(", or ")}? Add a term unique to the one you mean.`,
     );
   }
   return ok(candidates);

packages/core/src/result.ts

@@ -44,13 +44,23 @@ export function declined<T>(reason: DeclineReason): QueryResult<T> {
 }
 
 /**
- * Map a 0–1 score to a confidence level.
- * Provisional thresholds — replaced by measured calibration in Phase 4.5.
+ * Score → confidence thresholds, calibrated on the eval set (TRACKER step 4.5):
+ * every `high` answer in the eval set is correct at these cutoffs. The eval
+ * `--calibrate` subcommand measures precision at these levels and records it in
+ * eval/calibration.json.
  */
+export const CONFIDENCE_THRESHOLDS = { high: 0.8, medium: 0.5 } as const;
+
+/** Map a 0–1 score to a calibrated confidence level. */
 export function confidenceFromScore(score: number): Confidence {
   const clamped = Math.max(0, Math.min(1, score));
   return {
     score: clamped,
-    level: clamped >= 0.8 ? "high" : clamped >= 0.5 ? "medium" : "low",
+    level:
+      clamped >= CONFIDENCE_THRESHOLDS.high
+        ? "high"
+        : clamped >= CONFIDENCE_THRESHOLDS.medium
+          ? "medium"
+          : "low",
   };
 }