diff --git a/TRACKER.md b/TRACKER.md index c297727..26b81ea 100644 --- a/TRACKER.md +++ b/TRACKER.md @@ -5,8 +5,8 @@ ## Status - **Current phase:** 4 — Matching engine -- **Next step:** 4.1 — Term matching v2 -- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.5 +- **Next step:** 4.2 — Structural matching +- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.5, 4.1 - **Gates passed:** Gate 0 (CI + red-path, #5/#6) · Gate 1 (precision 1.000, recall 0.895, zero poison) · Gate 2 (C1 instance attribution 1.000 · B1 4-level handler chains · C6 store writers↔readers · A9 portals — scorecard 137/0/0, precision & recall 1.000) · Gate 3 (B3 action effects · B4 routers · B6 cyclic journeys terminate · B7/B8 form & non-JSX events · G5 flag/role conditions — precision & recall 1.000) ## What CodeRadar is @@ -218,7 +218,7 @@ The heart of the project. C1 and B1 live here. ## Phase 4 — Matching engine -### [ ] 4.1 Term matching v2 +### [x] 4.1 Term matching v2 **Failure modes:** A4, A7 (matching half), A10 **Build:** replace `matchComponentsByText` with a scorer over instances: - Normalization (from 1.5) both sides; fuzzy token match (edit distance ≤ 1 per token, token-set overlap) for OCR noise. diff --git a/eval/fixtures/a10-ocr-noise/app/InvoiceDashboard.tsx b/eval/fixtures/a10-ocr-noise/app/InvoiceDashboard.tsx new file mode 100644 index 0000000..8db7e5c --- /dev/null +++ b/eval/fixtures/a10-ocr-noise/app/InvoiceDashboard.tsx @@ -0,0 +1,8 @@ +export function InvoiceDashboard() { + return ( +
+

Invoice Dashboard

+

Outstanding balances by customer

+
+ ); +} diff --git a/eval/fixtures/a10-ocr-noise/app/ReconciliationReport.tsx b/eval/fixtures/a10-ocr-noise/app/ReconciliationReport.tsx new file mode 100644 index 0000000..87d7bac --- /dev/null +++ b/eval/fixtures/a10-ocr-noise/app/ReconciliationReport.tsx @@ -0,0 +1,8 @@ +export function ReconciliationReport() { + return ( +
+

Account Reconciliation

+

Match transactions against statements

+
+ ); +} diff --git a/eval/fixtures/a10-ocr-noise/app/SettingsPanel.tsx b/eval/fixtures/a10-ocr-noise/app/SettingsPanel.tsx new file mode 100644 index 0000000..333c094 --- /dev/null +++ b/eval/fixtures/a10-ocr-noise/app/SettingsPanel.tsx @@ -0,0 +1,8 @@ +export function SettingsPanel() { + return ( +
+

Settings

+

Preferences and account options

+
+ ); +} diff --git a/eval/fixtures/a10-ocr-noise/app/ShipmentTracker.tsx b/eval/fixtures/a10-ocr-noise/app/ShipmentTracker.tsx new file mode 100644 index 0000000..1d73c4e --- /dev/null +++ b/eval/fixtures/a10-ocr-noise/app/ShipmentTracker.tsx @@ -0,0 +1,8 @@ +export function ShipmentTracker() { + return ( +
+

Shipment Tracking

+

Delivery status and carrier updates

+
+ ); +} diff --git a/eval/fixtures/a10-ocr-noise/golden.json b/eval/fixtures/a10-ocr-noise/golden.json new file mode 100644 index 0000000..8eb24dd --- /dev/null +++ b/eval/fixtures/a10-ocr-noise/golden.json @@ -0,0 +1,18 @@ +{ + "failureMode": "A10", + "note": "OCR noise: screenshot text arrives misspelled ('Acount Reconcilliation', 'Invoise Dashbord'). Fuzzy token matching — a small, length-scaled edit distance on long distinctive tokens — still lands the right component in the top 3 (here, top 1). Short common tokens must still match exactly so noise doesn't collide unrelated components.", + "expect": { + "components": [ + { "name": "ReconciliationReport", "instances": 0 }, + { "name": "InvoiceDashboard", "instances": 0 }, + { "name": "ShipmentTracker", "instances": 0 }, + { "name": "SettingsPanel", "instances": 0 } + ], + "queries": [ + { "terms": ["Acount Reconcilliation"], "status": "ok", "top": "ReconciliationReport", "topK": 3 }, + { "terms": ["Invoise Dashbord"], "status": "ok", "top": "InvoiceDashboard", "topK": 3 }, + { "terms": ["Shipmnt Tracking"], "status": "ok", "top": "ShipmentTracker", "topK": 3 }, + { "terms": ["Reconciliation"], "status": "ok", "top": "ReconciliationReport" } + ] + } +} diff --git a/eval/src/checks.ts b/eval/src/checks.ts index 5e7cf51..0c5cf7f 100644 --- a/eval/src/checks.ts +++ b/eval/src/checks.ts @@ -224,9 +224,15 @@ export function runChecks(fixture: string, golden: Golden, graph: LineageGraph): if (!passed) { detail = `expected status ${query.status}, got ${result.status}`; } else if (query.status === "ok" && query.top !== undefined) { - const top = result.candidates[0]?.value.component.name; - passed = top === query.top; - if (!passed) detail = `expected top ${query.top}, got ${top ?? "none"}`; + const k = query.topK ?? 1; + const topNames = result.candidates.slice(0, k).map((c) => c.value.component.name); + passed = topNames.includes(query.top); + if (!passed) { + detail = + k > 1 + ? `expected ${query.top} in top ${k}, got [${topNames.join(", ")}]` + : `expected top ${query.top}, got ${topNames[0] ?? "none"}`; + } } finalize("queries", id, passed, query.expectedFail, detail); } diff --git a/eval/src/golden.ts b/eval/src/golden.ts index dd511a6..dc87240 100644 --- a/eval/src/golden.ts +++ b/eval/src/golden.ts @@ -38,6 +38,11 @@ export interface GoldenQuery { status: "ok" | "ambiguous" | "declined"; /** Required top-1 component name when status is "ok". */ top?: string; + /** + * When set, `top` need only appear within the first `topK` candidates rather + * than at rank 1 — the honest bar for OCR-noisy input (failure mode A10). + */ + topK?: number; expectedFail?: string; } diff --git a/packages/core/src/matching.test.ts b/packages/core/src/matching.test.ts new file mode 100644 index 0000000..e010ee3 --- /dev/null +++ b/packages/core/src/matching.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from "vitest"; + +import { matchComponentsByText } from "./query.js"; +import { editDistance, fuzzyTokenMatch } from "./text.js"; +import { type ComponentNode, type LineageGraph, nodeId } from "./types.js"; + +const loc = (file: string) => ({ file, line: 1, endLine: 1 }); + +function component(name: string, texts: string[]): ComponentNode { + return { + id: nodeId("component", `${name}.tsx`, name), + kind: "component", + name, + loc: loc(`${name}.tsx`), + exportName: name, + props: [], + renderedText: texts.map((text) => ({ text, source: "jsx" as const })), + rendersComponents: [], + }; +} + +function graph(components: ComponentNode[]): LineageGraph { + return { + version: 2, + root: "/app", + generatedAt: "2026-01-01T00:00:00.000Z", + generator: "test", + nodes: components, + edges: [], + }; +} + +describe("editDistance / fuzzyTokenMatch (TRACKER 4.1)", () => { + it("bounds the edit distance and abandons past the budget", () => { + expect(editDistance("kitten", "sitting", 3)).toBe(3); + expect(editDistance("abcdef", "uvwxyz", 2)).toBe(3); // max + 1 sentinel + }); + + it("tolerates OCR slips on long tokens but keeps short tokens strict", () => { + expect(fuzzyTokenMatch("reconciliation", "reconcilliation")).toBe(true); + expect(fuzzyTokenMatch("dashboard", "dashbord")).toBe(true); + expect(fuzzyTokenMatch("save", "safe")).toBe(false); // short → exact only + }); +}); + +describe("matchComponentsByText scorer (TRACKER 4.1, A4/A10)", () => { + const forms = [ + component("SettingsForm", ["Save", "Notifications"]), + component("ProfileForm", ["Save", "Display name"]), + component("BillingForm", ["Save", "Card number"]), + ]; + + it("a lone generic term is honestly ambiguous, not a coin flip", () => { + const result = matchComponentsByText(graph(forms), ["Save"]); + expect(result.status).toBe("ambiguous"); + expect(result.disambiguation).toBeDefined(); + }); + + it("a distinctive term breaks the tie via rarity weighting", () => { + const result = matchComponentsByText(graph(forms), ["Save", "Card number"]); + expect(result.status).toBe("ok"); + expect(result.candidates[0]?.value.component.name).toBe("BillingForm"); + }); + + it("respects word order — 'Order deleted' ≠ 'Delete order'", () => { + const g = graph([ + component("Toast", ["Order deleted"]), + component("DeleteButton", ["Delete order"]), + ]); + expect(matchComponentsByText(g, ["Order deleted"]).candidates[0]?.value.component.name).toBe( + "Toast", + ); + expect(matchComponentsByText(g, ["Delete order"]).candidates[0]?.value.component.name).toBe( + "DeleteButton", + ); + }); + + it("matches OCR-noisy distinctive text", () => { + const g = graph([ + component("ReconciliationReport", ["Account Reconciliation"]), + component("InvoiceDashboard", ["Invoice Dashboard"]), + ]); + const result = matchComponentsByText(g, ["Acount Reconcilliation"]); + expect(result.status).toBe("ok"); + expect(result.candidates[0]?.value.component.name).toBe("ReconciliationReport"); + }); + + it("declines when nothing matches", () => { + expect(matchComponentsByText(graph(forms), ["Purchase history"]).status).toBe("declined"); + }); +}); diff --git a/packages/core/src/query.ts b/packages/core/src/query.ts index a433f52..ace2690 100644 --- a/packages/core/src/query.ts +++ b/packages/core/src/query.ts @@ -14,7 +14,7 @@ import { ok, type QueryResult, } from "./result.js"; -import { normalizeText, textMatches } from "./text.js"; +import { fuzzyTokenMatch, normalizeText, textMatches, tokenize } from "./text.js"; import type { ComponentNode, DataSourceNode, @@ -25,6 +25,7 @@ import type { LineageEdge, LineageGraph, LineageNode, + RenderedText, RouteNode, SourceLocation, StateNode, @@ -38,56 +39,108 @@ export interface ComponentMatch { } /** - * Rank components by overlap between `terms` (words/phrases read off a - * screenshot or ticket) and each component's statically rendered text. + * Rank components by how well `terms` (words/phrases read off a screenshot or + * ticket) match each component's statically rendered text (TRACKER step 4.1). * - * Returns `ambiguous` when several components tie at the top score, - * `declined("no-signal")` when nothing matches at all. + * Three ideas beyond raw overlap: + * - **Rarity weighting** — a term's weight is its inverse document frequency + * across the graph, so "Save" (everywhere) counts for almost nothing while + * "Reconciliation" (one component) dominates. + * - **Fuzzy tokens** — long tokens match within a small edit distance, so OCR + * slips ("Reconcilliation") still land (failure mode A10). + * - **Combination bonus** — several distinct terms co-occurring in one + * component outrank the same terms scattered across many. + * + * Returns `ambiguous` when the leaders tie on rarity-weighted score (a lone + * generic term is honestly ambiguous), `declined("no-signal")` on no match. */ export function matchComponentsByText( graph: LineageGraph, terms: string[], ): QueryResult { - const needles = terms.map((t) => normalizeText(t)).filter((t) => t.length > 1); - if (needles.length === 0) return declined("no-signal"); + const queryTerms = terms.map((t) => normalizeText(t)).filter((t) => t.length > 1); + if (queryTerms.length === 0) return declined("no-signal"); const instancesByDefinition = groupInstances(graph); - const scored: Array<{ match: ComponentMatch; score: number; evidence: Evidence[] }> = []; - - for (const node of graph.nodes) { - if (node.kind !== "component") continue; - const matchedText: string[] = []; - const evidence: Evidence[] = []; - for (const needle of needles) { - const hit = node.renderedText.find((entry) => - textMatches(normalizeText(entry.text), needle), - ); - if (hit !== undefined) { - matchedText.push(hit.text.toLowerCase()); - const provenance = - hit.source === "i18n" - ? ` (i18n key ${hit.key ?? "?"}, locale ${hit.locale ?? "?"})` - : hit.branch !== undefined - ? ` (renders only when ${hit.branch})` - : ""; - evidence.push({ - kind: "text-match", - detail: `"${needle}" matched rendered text "${hit.text}"${provenance}`, - loc: node.loc, - }); + const components = graph.nodes.filter((n): n is ComponentNode => n.kind === "component"); + + // Document frequency per token, for rarity (IDF) weighting. + const documentFrequency = new Map(); + for (const component of components) { + const tokens = new Set(); + for (const entry of component.renderedText) for (const t of tokenize(entry.text)) tokens.add(t); + for (const t of tokens) documentFrequency.set(t, (documentFrequency.get(t) ?? 0) + 1); + } + const total = components.length || 1; + const idf = (token: string): number => { + let df = documentFrequency.get(token); + if (df === undefined) { + // Not an exact token — charge the rarity of the closest fuzzy match. + for (const [candidate, count] of documentFrequency) { + if (fuzzyTokenMatch(candidate, token)) df = Math.max(df ?? 0, count); } } - if (matchedText.length > 0) { - scored.push({ - match: { - component: node, - instances: instancesByDefinition.get(node.id) ?? [], - matchedText, - }, - score: matchedText.length / needles.length, - evidence, + return Math.log((total + 1) / ((df ?? 0.5) + 0.5)); + }; + + // The rendered-text entry a phrase term matches: directly (exact/substring/ + // wildcard) or as an ordered, fuzzy, contiguous run of its tokens — order + // matters, so "Order deleted" does not match "Delete order", but OCR slips + // in any single token still land. Returns the entry (for provenance) or null. + const matchingEntry = (term: string, component: ComponentNode): RenderedText | null => { + const termTokens = tokenize(term); + if (termTokens.length === 0) return null; + for (const entry of component.renderedText) { + if (textMatches(normalizeText(entry.text), term)) return entry; + if (containsPhrase(tokenize(entry.text), termTokens)) return entry; + } + return null; + }; + + const termWeight = (term: string): number => + tokenize(term).reduce((sum, t) => sum + idf(t), 0); + + const scored: Array<{ + match: ComponentMatch; + score: number; + coverage: number; + evidence: Evidence[]; + }> = []; + + for (const component of components) { + const matched: string[] = []; + const evidence: Evidence[] = []; + let weight = 0; + for (const term of queryTerms) { + const hit = matchingEntry(term, component); + if (hit === null) continue; + const w = termWeight(term); + matched.push(term); + weight += w; + const provenance = + hit.source === "i18n" + ? ` (i18n key ${hit.key ?? "?"}, locale ${hit.locale ?? "?"})` + : hit.branch !== undefined + ? ` (renders only when ${hit.branch})` + : ""; + evidence.push({ + kind: "text-match", + detail: `"${term}" matched rendered text "${hit.text}"${provenance} — rarity weight ${w.toFixed(2)}`, + loc: component.loc, }); } + if (matched.length === 0) continue; + const combination = 1 + 0.5 * (matched.length - 1); + scored.push({ + match: { + component, + instances: instancesByDefinition.get(component.id) ?? [], + matchedText: matched, + }, + score: weight * combination, + coverage: matched.length / queryTerms.length, + evidence, + }); } if (scored.length === 0) return declined("no-signal"); @@ -97,18 +150,19 @@ export function matchComponentsByText( ); const candidates: Candidate[] = scored.map((s) => ({ value: s.match, - confidence: confidenceFromScore(s.score), + confidence: confidenceFromScore(s.coverage), evidence: s.evidence, })); const top = scored[0]; - const tied = top === undefined ? [] : scored.filter((s) => s.score === top.score); + const tied = + top === undefined ? [] : scored.filter((s) => Math.abs(s.score - top.score) < 1e-9); if (tied.length > 1) { const names = tied.map((s) => s.match.component.name).join(", "); return ambiguous( candidates, - `Multiple components match equally well (${names}). ` + - `Which file or page is the screenshot from, or what other text is visible?`, + `Several components match equally well (${names}). ` + + `Which page is the screenshot from, or what other distinctive text is visible?`, ); } return ok(candidates); @@ -482,6 +536,24 @@ export function journeys( return ok([{ value: paths, confidence: confidenceFromScore(0.9), evidence }]); } +/** True when `phrase` tokens appear as a contiguous, in-order, fuzzy run in `haystack`. */ +function containsPhrase(haystack: string[], phrase: string[]): boolean { + if (phrase.length === 0 || phrase.length > haystack.length) return false; + for (let start = 0; start + phrase.length <= haystack.length; start += 1) { + let ok = true; + for (let k = 0; k < phrase.length; k += 1) { + const h = haystack[start + k]; + const p = phrase[k]; + if (h === undefined || p === undefined || !fuzzyTokenMatch(h, p)) { + ok = false; + break; + } + } + if (ok) return true; + } + return false; +} + function groupInstances(graph: LineageGraph): Map { const byDefinition = new Map(); for (const node of graph.nodes) { diff --git a/packages/core/src/text.ts b/packages/core/src/text.ts index 03ab32f..671eda4 100644 --- a/packages/core/src/text.ts +++ b/packages/core/src/text.ts @@ -42,3 +42,49 @@ function foldPlural(token: string): string { if (token.length > 3 && token.endsWith("s") && !token.endsWith("ss")) return token.slice(0, -1); return token; } + +/** Normalized significant tokens of a string (drops empties). */ +export function tokenize(input: string): string[] { + return normalizeText(input) + .split(" ") + .filter((t) => t.length > 0); +} + +/** + * Levenshtein distance between `a` and `b`, abandoned once it exceeds `max` + * (returns `max + 1`). Bounding keeps the matcher's fuzzy comparisons cheap. + */ +export function editDistance(a: string, b: string, max: number): number { + if (a === b) return 0; + if (Math.abs(a.length - b.length) > max) return max + 1; + const prev = new Array(b.length + 1); + const curr = new Array(b.length + 1); + for (let j = 0; j <= b.length; j += 1) prev[j] = j; + for (let i = 1; i <= a.length; i += 1) { + curr[0] = i; + let rowMin = curr[0]; + for (let j = 1; j <= b.length; j += 1) { + const cost = a[i - 1] === b[j - 1] ? 0 : 1; + const value = Math.min((prev[j] ?? 0) + 1, (curr[j - 1] ?? 0) + 1, (prev[j - 1] ?? 0) + cost); + curr[j] = value; + if (value < rowMin) rowMin = value; + } + if (rowMin > max) return max + 1; + for (let j = 0; j <= b.length; j += 1) prev[j] = curr[j] ?? 0; + } + return prev[b.length] ?? max + 1; +} + +/** + * OCR-tolerant token equality: exact after normalization, or within a small + * edit distance that scales with length — short tokens ("save") must match + * exactly so they don't collide ("safe"), longer distinctive words tolerate the + * 1–2 character slips OCR introduces ("reconcilliation" ≈ "reconciliation"). + */ +export function fuzzyTokenMatch(a: string, b: string): boolean { + if (a === b) return true; + const len = Math.max(a.length, b.length); + if (len < 5) return false; + const budget = len <= 7 ? 1 : 2; + return editDistance(a, b, budget) <= budget; +}