From 368d59a4276d16429ea7dea746037d47429535c8 Mon Sep 17 00:00:00 2001 From: Deep Kumar Singh Kushwah Date: Wed, 15 Jul 2026 02:32:09 +0530 Subject: [PATCH] feat(match): alias glossary & corrections store as authority evidence (E2/G4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Business vocabulary and human corrections now feed the matcher as the highest-weight signals (TRACKER step 4.6): - Aliases (E2): a checked-in glossary (phrase → component) resolves terms that appear nowhere in the code — "invoice widget" → BillingSummaryCard — as `kind:"alias"` evidence. - Corrections (G4): recordCorrection/loadCorrections over a JSONL store; a recorded correction outranks even a correct text match (`kind:"correction"`), so the next identical query resolves the confirmed way. Core: MatchQuery gains `aliases` + `corrections`; new corrections.ts store. Eval: harness loads a fixture's aliases.yaml and threads it into query matching; fixture e2-business-vocab (alias-only phrase resolves; plain text still works). CLI: `find` gains --aliases/--corrections; new `correct ` command records a correction. 39 core + 3 vision + 102 parser tests pass; eval green. (4.5 calibration deferred — Gate 4 still pending it.) Co-Authored-By: Claude Opus 4.8 --- TRACKER.md | 6 +- eval/fixtures/e2-business-vocab/aliases.yaml | 5 ++ .../app/BillingSummaryCard.tsx | 8 +++ .../e2-business-vocab/app/InvoiceList.tsx | 7 ++ eval/fixtures/e2-business-vocab/golden.json | 15 ++++ eval/package.json | 3 +- eval/src/checks.ts | 13 +++- eval/src/run.ts | 10 ++- packages/cli/src/index.ts | 33 ++++++++- packages/core/src/corrections.ts | 26 +++++++ packages/core/src/index.ts | 1 + packages/core/src/matching.test.ts | 53 ++++++++++++++- packages/core/src/query.ts | 68 ++++++++++++++++++- pnpm-lock.yaml | 3 + 14 files changed, 236 insertions(+), 15 deletions(-) create mode 100644 eval/fixtures/e2-business-vocab/aliases.yaml create mode 100644 eval/fixtures/e2-business-vocab/app/BillingSummaryCard.tsx create mode 100644 eval/fixtures/e2-business-vocab/app/InvoiceList.tsx create mode 100644 eval/fixtures/e2-business-vocab/golden.json create mode 100644 packages/core/src/corrections.ts diff --git a/TRACKER.md b/TRACKER.md index f9bf66e..8ee6efe 100644 --- a/TRACKER.md +++ b/TRACKER.md @@ -5,8 +5,8 @@ ## Status - **Current phase:** 4 — Matching engine -- **Next step:** 4.5 — Confidence calibration & ambiguity protocol -- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.5, 4.1–4.4 +- **Next step:** 4.5 — Confidence calibration & ambiguity protocol (last step for Gate 4) +- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.5, 4.1–4.4, 4.6 (4.5 deferred — Gate 4 pending it) - **Gates passed:** Gate 0 (CI + red-path, #5/#6) · Gate 1 (precision 1.000, recall 0.895, zero poison) · Gate 2 (C1 instance attribution 1.000 · B1 4-level handler chains · C6 store writers↔readers · A9 portals — scorecard 137/0/0, precision & recall 1.000) · Gate 3 (B3 action effects · B4 routers · B6 cyclic journeys terminate · B7/B8 form & non-JSX events · G5 flag/role conditions — precision & recall 1.000) ## What CodeRadar is @@ -253,7 +253,7 @@ The heart of the project. C1 and B1 live here. - `declined` results carry a machine-readable reason (`out-of-scope`, `not-our-app`, `no-signal`). **Accept:** poison rate ≤ 0.05 and ambiguity honesty ≥ 0.90 on the full matching eval set; every `high`-confidence answer in the eval set is correct. -### [ ] 4.6 Alias glossary & corrections store +### [x] 4.6 Alias glossary & corrections store **Failure modes:** E2, G4 **Build:** `aliases.yaml` (checked into the *target* repo): `"invoice widget" → BillingSummaryCard`, route titles, feature names. Corrections API: `recordCorrection(terms, confirmedInstanceId)` appends to `corrections.jsonl`; both feed the matcher as first-class evidence (`kind: "alias" | "correction"`, high weight). Eval subcommand folds corrections into new ticket-eval cases. **Accept:** fixture `e2-business-vocab` green via alias; a recorded correction changes the next identical query's top-1 (integration test). **Gate 4 passes.** diff --git a/eval/fixtures/e2-business-vocab/aliases.yaml b/eval/fixtures/e2-business-vocab/aliases.yaml new file mode 100644 index 0000000..89cee36 --- /dev/null +++ b/eval/fixtures/e2-business-vocab/aliases.yaml @@ -0,0 +1,5 @@ +# Business-vocabulary glossary, checked into the target repo (TRACKER 4.6, E2). +# A phrase users say maps to the component it actually means, even when that +# phrase appears nowhere in the rendered text. +"invoice widget": BillingSummaryCard +"amount owed": BillingSummaryCard diff --git a/eval/fixtures/e2-business-vocab/app/BillingSummaryCard.tsx b/eval/fixtures/e2-business-vocab/app/BillingSummaryCard.tsx new file mode 100644 index 0000000..a68ed10 --- /dev/null +++ b/eval/fixtures/e2-business-vocab/app/BillingSummaryCard.tsx @@ -0,0 +1,8 @@ +export function BillingSummaryCard() { + return ( +
+

Billing summary

+ Amount due +
+ ); +} diff --git a/eval/fixtures/e2-business-vocab/app/InvoiceList.tsx b/eval/fixtures/e2-business-vocab/app/InvoiceList.tsx new file mode 100644 index 0000000..b8a433f --- /dev/null +++ b/eval/fixtures/e2-business-vocab/app/InvoiceList.tsx @@ -0,0 +1,7 @@ +export function InvoiceList() { + return ( +
    +
  • Recent invoices
    • +
+ ); +} diff --git a/eval/fixtures/e2-business-vocab/golden.json b/eval/fixtures/e2-business-vocab/golden.json new file mode 100644 index 0000000..b359fdd --- /dev/null +++ b/eval/fixtures/e2-business-vocab/golden.json @@ -0,0 +1,15 @@ +{ + "failureMode": "E2", + "note": "Business vocabulary: users say 'invoice widget' but the component is BillingSummaryCard and that phrase appears nowhere in its text. Without the glossary the query declines; with aliases.yaml loaded it resolves to BillingSummaryCard. Plain-text queries still work.", + "expect": { + "components": [ + { "name": "BillingSummaryCard", "instances": 0 }, + { "name": "InvoiceList", "instances": 0 } + ], + "queries": [ + { "terms": ["invoice widget"], "status": "ok", "top": "BillingSummaryCard" }, + { "terms": ["Billing summary"], "status": "ok", "top": "BillingSummaryCard" }, + { "terms": ["Recent invoices"], "status": "ok", "top": "InvoiceList" } + ] + } +} diff --git a/eval/package.json b/eval/package.json index 12aecaf..b57e1d6 100644 --- a/eval/package.json +++ b/eval/package.json @@ -11,7 +11,8 @@ }, "dependencies": { "@coderadar/core": "workspace:*", - "@coderadar/parser-react": "workspace:*" + "@coderadar/parser-react": "workspace:*", + "yaml": "^2.9.0" }, "devDependencies": { "@types/node": "^22.20.1", diff --git a/eval/src/checks.ts b/eval/src/checks.ts index ca77cfc..47d7891 100644 --- a/eval/src/checks.ts +++ b/eval/src/checks.ts @@ -9,7 +9,12 @@ import { import type { CheckResult, FixtureResult, Golden } from "./golden.js"; -export function runChecks(fixture: string, golden: Golden, graph: LineageGraph): FixtureResult { +export function runChecks( + fixture: string, + golden: Golden, + graph: LineageGraph, + aliases?: Record, +): FixtureResult { const checks: CheckResult[] = []; const attribution = { truePositives: 0, falsePositives: 0, falseNegatives: 0 }; @@ -218,7 +223,11 @@ export function runChecks(fixture: string, golden: Golden, graph: LineageGraph): for (const query of golden.expect.queries ?? []) { const id = `query:${query.terms.join("+") || JSON.stringify(query.structure)}`; - const result = matchComponents(graph, { terms: query.terms, structure: query.structure }); + const result = matchComponents(graph, { + terms: query.terms, + structure: query.structure, + ...(aliases !== undefined ? { aliases } : {}), + }); let passed = result.status === query.status; let detail: string | undefined; if (!passed) { diff --git a/eval/src/run.ts b/eval/src/run.ts index 641412d..050fe60 100644 --- a/eval/src/run.ts +++ b/eval/src/run.ts @@ -15,10 +15,18 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { resolveHookEdges, scanReact } from "@coderadar/parser-react"; +import { parse as parseYaml } from "yaml"; import { runChecks } from "./checks.js"; import type { FixtureResult, Golden, Scorecard, Thresholds } from "./golden.js"; +/** A fixture's business-vocabulary glossary (aliases.yaml), phrase → component. */ +function loadAliases(fixtureDir: string): Record | undefined { + const aliasPath = path.join(fixtureDir, "aliases.yaml"); + if (!fs.existsSync(aliasPath)) return undefined; + return parseYaml(fs.readFileSync(aliasPath, "utf-8")) as Record; +} + const evalDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const fixturesDir = path.join(evalDir, "fixtures"); @@ -41,7 +49,7 @@ function main(): void { const golden = JSON.parse(fs.readFileSync(goldenPath, "utf-8")) as Golden; const appDir = path.resolve(fixtureDir, golden.app ?? "./app"); const graph = resolveHookEdges(scanReact({ root: appDir, ...(golden.scan ?? {}) })); - results.push(runChecks(name, golden, graph)); + results.push(runChecks(name, golden, graph, loadAliases(fixtureDir))); } const scorecard = buildScorecard(results); diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 324a437..1fbcc65 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -9,13 +9,16 @@ import { type JourneyPath, journeys, type LineageGraph, + loadCorrections, loadGraph as loadGraphFile, - matchComponentsByText, + matchComponents, + recordCorrection, saveGraph, traceLineage, } from "@coderadar/core"; import { resolveHookEdges, scanReact } from "@coderadar/parser-react"; import { Command } from "commander"; +import { parse as parseYaml } from "yaml"; const program = new Command(); @@ -57,9 +60,22 @@ program .description("Find components by text visible on screen (e.g. read off a screenshot)") .argument("", "text fragments seen in the UI") .option("-g, --graph ", "graph file", "ui-lineage.graph.json") - .action((terms: string[], opts: { graph: string }) => { + .option("-a, --aliases ", "business-vocab glossary (aliases.yaml)") + .option("-c, --corrections ", "corrections store (jsonl)", "ui-lineage.corrections.jsonl") + .action((terms: string[], opts: { graph: string; aliases?: string; corrections: string }) => { const graph = loadGraph(opts.graph); - const result = matchComponentsByText(graph, terms); + const aliases = + opts.aliases !== undefined && fs.existsSync(opts.aliases) + ? (parseYaml(fs.readFileSync(opts.aliases, "utf-8")) as Record) + : undefined; + const corrections = fs.existsSync(opts.corrections) + ? loadCorrections(opts.corrections) + : undefined; + const result = matchComponents(graph, { + terms, + ...(aliases !== undefined ? { aliases } : {}), + ...(corrections !== undefined ? { corrections } : {}), + }); if (result.status === "declined") { console.log(`No components matched (${result.declineReason}).`); return; @@ -162,6 +178,17 @@ program for (const path of paths) printJourneyPath(path); }); +program + .command("correct") + .description("Record that some on-screen text means a component — feeds future `find` results") + .argument("", "component name it should resolve to") + .argument("", "the text fragments that mean it") + .option("-c, --corrections ", "corrections store (jsonl)", "ui-lineage.corrections.jsonl") + .action((component: string, terms: string[], opts: { corrections: string }) => { + recordCorrection(opts.corrections, { terms, component }); + console.log(`Recorded: [${terms.join(", ")}] → ${component} (${opts.corrections})`); + }); + const STEP_ARROW: Record = { page: "▸", event: "•", diff --git a/packages/core/src/corrections.ts b/packages/core/src/corrections.ts new file mode 100644 index 0000000..ed5b90d --- /dev/null +++ b/packages/core/src/corrections.ts @@ -0,0 +1,26 @@ +/** + * The corrections store (TRACKER step 4.6, failure mode G4). + * + * When a human confirms that a set of terms means a specific component, that + * confirmation is appended to a `corrections.jsonl` file (one JSON object per + * line) and fed back to the matcher as the highest-weight evidence, so the next + * identical query resolves the same way. Terms only — never screenshots (G7). + */ +import fs from "node:fs"; + +import type { Correction } from "./query.js"; + +/** Read all corrections from a JSONL file. Missing file → empty list. */ +export function loadCorrections(path: string): Correction[] { + if (!fs.existsSync(path)) return []; + return fs + .readFileSync(path, "utf-8") + .split("\n") + .filter((line) => line.trim().length > 0) + .map((line) => JSON.parse(line) as Correction); +} + +/** Append one correction to the JSONL store. */ +export function recordCorrection(path: string, correction: Correction): void { + fs.appendFileSync(path, `${JSON.stringify(correction)}\n`); +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 0a7310b..b8f269a 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,5 +1,6 @@ export * from "./types.js"; export * from "./result.js"; export * from "./query.js"; +export * from "./corrections.js"; export * from "./storage.js"; export * from "./text.js"; diff --git a/packages/core/src/matching.test.ts b/packages/core/src/matching.test.ts index e010ee3..5206023 100644 --- a/packages/core/src/matching.test.ts +++ b/packages/core/src/matching.test.ts @@ -1,6 +1,11 @@ -import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; -import { matchComponentsByText } from "./query.js"; +import { afterEach, describe, expect, it } from "vitest"; + +import { loadCorrections, recordCorrection } from "./corrections.js"; +import { matchComponents, matchComponentsByText } from "./query.js"; import { editDistance, fuzzyTokenMatch } from "./text.js"; import { type ComponentNode, type LineageGraph, nodeId } from "./types.js"; @@ -89,3 +94,47 @@ describe("matchComponentsByText scorer (TRACKER 4.1, A4/A10)", () => { expect(matchComponentsByText(graph(forms), ["Purchase history"]).status).toBe("declined"); }); }); + +describe("alias glossary & corrections (TRACKER 4.6, E2/G4)", () => { + const g = graph([ + component("BillingSummaryCard", ["Billing summary", "Amount due"]), + component("InvoiceList", ["Recent invoices"]), + ]); + + it("resolves a business-vocab phrase that appears nowhere in the code", () => { + expect(matchComponentsByText(g, ["invoice widget"]).status).toBe("declined"); + const withAlias = matchComponents(g, { + terms: ["invoice widget"], + aliases: { "invoice widget": "BillingSummaryCard" }, + }); + expect(withAlias.status).toBe("ok"); + expect(withAlias.candidates[0]?.value.component.name).toBe("BillingSummaryCard"); + expect(withAlias.candidates[0]?.evidence.some((e) => e.kind === "alias")).toBe(true); + }); + + it("a recorded correction overrides an otherwise-correct text match", () => { + const before = matchComponents(g, { terms: ["Recent invoices"] }); + expect(before.candidates[0]?.value.component.name).toBe("InvoiceList"); + const after = matchComponents(g, { + terms: ["Recent invoices"], + corrections: [{ terms: ["Recent invoices"], component: "BillingSummaryCard" }], + }); + expect(after.candidates[0]?.value.component.name).toBe("BillingSummaryCard"); + expect(after.candidates[0]?.evidence.some((e) => e.kind === "correction")).toBe(true); + }); + + const tmpFiles: string[] = []; + afterEach(() => { + for (const f of tmpFiles.splice(0)) fs.rmSync(f, { force: true }); + }); + + it("round-trips corrections through a JSONL store; the next query flips top-1", () => { + const file = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "uil-")), "corrections.jsonl"); + tmpFiles.push(file); + expect(loadCorrections(file)).toEqual([]); + recordCorrection(file, { terms: ["Recent invoices"], component: "BillingSummaryCard" }); + const corrections = loadCorrections(file); + const result = matchComponents(g, { terms: ["Recent invoices"], corrections }); + expect(result.candidates[0]?.value.component.name).toBe("BillingSummaryCard"); + }); +}); diff --git a/packages/core/src/query.ts b/packages/core/src/query.ts index f931f6f..9b4cead 100644 --- a/packages/core/src/query.ts +++ b/packages/core/src/query.ts @@ -63,6 +63,17 @@ export interface ComponentMatch { * Returns `ambiguous` when the leaders tie on rarity-weighted score (a lone * generic term is honestly ambiguous), `declined("no-signal")` on no match. */ +/** + * A recorded correction (Phase 4.6, failure mode G4): a human confirmed that a + * set of terms means a specific component. Fed back as first-class evidence so + * the next identical query resolves the same way. + */ +export interface Correction { + terms: string[]; + /** Component definition name the terms were confirmed to mean. */ + component: string; +} + /** A screenshot/ticket query: visible text, a structure descriptor, or both. */ export interface MatchQuery { terms?: string[]; @@ -73,10 +84,23 @@ export interface MatchQuery { * so the emphasized element outranks incidental text. */ boosts?: Record; + /** + * Business-vocabulary glossary (Phase 4.6, failure mode E2), phrase → + * component name/id — "invoice widget" → BillingSummaryCard. An alias hit is + * high-weight evidence, so a term that appears nowhere in the code still + * resolves. + */ + aliases?: Record; + /** Recorded corrections (Phase 4.6) — the highest-weight signal of all. */ + corrections?: Correction[]; } /** Weight of a full structural match relative to a rare matched term. */ const STRUCTURE_WEIGHT = 3; +/** A glossary alias hit outweighs any text/structure evidence (Phase 4.6). */ +const ALIAS_WEIGHT = 10; +/** A recorded human correction is the strongest signal of all. */ +const CORRECTION_WEIGHT = 25; /** Back-compat text-only entry point. */ export function matchComponentsByText( @@ -145,6 +169,13 @@ export function matchComponents( return base * (boosts.get(term) ?? 1); }; + // Glossary aliases + recorded corrections (Phase 4.6). Both are authority + // signals — a phrase resolves even when it appears nowhere in the code. + const aliasEntries = Object.entries(query.aliases ?? {}); + const corrections = query.corrections ?? []; + const containsTerm = (needle: string): boolean => + needle.length > 0 && queryTerms.some((t) => t === needle || t.includes(needle) || needle.includes(t)); + // Render tree (definition level): A renders B when A has a `renders` edge to // an instance whose `instance-of` points at B. Used to collapse nested // matches to the most specific one (step 4.3). @@ -230,15 +261,46 @@ export function matchComponents( }); } - if (covered.size === 0 && structureFit === 0) continue; - const coverage = queryTerms.length > 0 ? covered.size / queryTerms.length : structureFit; + // Authority: glossary aliases and recorded corrections that name this + // component, when the query actually contains their phrase/terms. + let authority = 0; + const normName = normalizeText(component.name); + for (const [phrase, target] of aliasEntries) { + if (normalizeText(target) !== normName && target !== component.id) continue; + const normPhrase = normalizeText(phrase); + if (!containsTerm(normPhrase)) continue; + authority += ALIAS_WEIGHT; + covered.add(normPhrase); + if (!matched.includes(normPhrase)) matched.push(normPhrase); + evidence.push({ + kind: "alias", + detail: `glossary alias "${phrase}" → ${component.name}`, + loc: component.loc, + }); + } + for (const correction of corrections) { + if (normalizeText(correction.component) !== normName) continue; + const cterms = correction.terms.map(normalizeText).filter((t) => t.length > 0); + if (cterms.length === 0 || !cterms.every(containsTerm)) continue; + authority += CORRECTION_WEIGHT; + for (const ct of cterms) covered.add(ct); + evidence.push({ + kind: "correction", + detail: `recorded correction [${correction.terms.join(", ")}] → ${component.name}`, + loc: component.loc, + }); + } + + if (covered.size === 0 && structureFit === 0 && authority === 0) continue; + const coverage = + authority > 0 ? 1 : queryTerms.length > 0 ? covered.size / queryTerms.length : structureFit; scored.push({ match: { component, instances: instancesByDefinition.get(component.id) ?? [], matchedText: matched.length > 0 ? matched : [...covered], }, - score: textScore + structureFit * STRUCTURE_WEIGHT, + score: textScore + structureFit * STRUCTURE_WEIGHT + authority, coverage: Math.max(coverage, structureFit), covered, evidence, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b61e8ef..11876e5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -16,6 +16,9 @@ importers: '@coderadar/parser-react': specifier: workspace:* version: link:../packages/parser-react + yaml: + specifier: ^2.9.0 + version: 2.9.0 devDependencies: '@types/node': specifier: ^22.20.1