Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions TRACKER.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
## Status

- **Current phase:** 4 — Matching engine
- **Next step:** 4.5 — Confidence calibration & ambiguity protocol
- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.5, 4.1–4.4
- **Next step:** 4.5 — Confidence calibration & ambiguity protocol (last step for Gate 4)
- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.5, 4.1–4.4, 4.6 (4.5 deferred — Gate 4 pending it)
- **Gates passed:** Gate 0 (CI + red-path, #5/#6) · Gate 1 (precision 1.000, recall 0.895, zero poison) · Gate 2 (C1 instance attribution 1.000 · B1 4-level handler chains · C6 store writers↔readers · A9 portals — scorecard 137/0/0, precision & recall 1.000) · Gate 3 (B3 action effects · B4 routers · B6 cyclic journeys terminate · B7/B8 form & non-JSX events · G5 flag/role conditions — precision & recall 1.000)

## What CodeRadar is
Expand Down Expand Up @@ -253,7 +253,7 @@ The heart of the project. C1 and B1 live here.
- `declined` results carry a machine-readable reason (`out-of-scope`, `not-our-app`, `no-signal`).
**Accept:** poison rate ≤ 0.05 and ambiguity honesty ≥ 0.90 on the full matching eval set; every `high`-confidence answer in the eval set is correct.

### [ ] 4.6 Alias glossary & corrections store
### [x] 4.6 Alias glossary & corrections store
**Failure modes:** E2, G4
**Build:** `aliases.yaml` (checked into the *target* repo): `"invoice widget" → BillingSummaryCard`, route titles, feature names. Corrections API: `recordCorrection(terms, confirmedInstanceId)` appends to `corrections.jsonl`; both feed the matcher as first-class evidence (`kind: "alias" | "correction"`, high weight). Eval subcommand folds corrections into new ticket-eval cases.
**Accept:** fixture `e2-business-vocab` green via alias; a recorded correction changes the next identical query's top-1 (integration test). **Gate 4 passes.**
Expand Down
5 changes: 5 additions & 0 deletions eval/fixtures/e2-business-vocab/aliases.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Business-vocabulary glossary, checked into the target repo (TRACKER 4.6, E2).
# A phrase users say maps to the component it actually means, even when that
# phrase appears nowhere in the rendered text.
"invoice widget": BillingSummaryCard
"amount owed": BillingSummaryCard
8 changes: 8 additions & 0 deletions eval/fixtures/e2-business-vocab/app/BillingSummaryCard.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
export function BillingSummaryCard() {
return (
<div className="card">
<h3>Billing summary</h3>
<span>Amount due</span>
</div>
);
}
7 changes: 7 additions & 0 deletions eval/fixtures/e2-business-vocab/app/InvoiceList.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
export function InvoiceList() {
return (
<ul>
<li>Recent invoices</li>
</ul>
);
}
15 changes: 15 additions & 0 deletions eval/fixtures/e2-business-vocab/golden.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
{
"failureMode": "E2",
"note": "Business vocabulary: users say 'invoice widget' but the component is BillingSummaryCard and that phrase appears nowhere in its text. Without the glossary the query declines; with aliases.yaml loaded it resolves to BillingSummaryCard. Plain-text queries still work.",
"expect": {
"components": [
{ "name": "BillingSummaryCard", "instances": 0 },
{ "name": "InvoiceList", "instances": 0 }
],
"queries": [
{ "terms": ["invoice widget"], "status": "ok", "top": "BillingSummaryCard" },
{ "terms": ["Billing summary"], "status": "ok", "top": "BillingSummaryCard" },
{ "terms": ["Recent invoices"], "status": "ok", "top": "InvoiceList" }
]
}
}
3 changes: 2 additions & 1 deletion eval/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@
},
"dependencies": {
"@coderadar/core": "workspace:*",
"@coderadar/parser-react": "workspace:*"
"@coderadar/parser-react": "workspace:*",
"yaml": "^2.9.0"
},
"devDependencies": {
"@types/node": "^22.20.1",
Expand Down
13 changes: 11 additions & 2 deletions eval/src/checks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,12 @@ import {

import type { CheckResult, FixtureResult, Golden } from "./golden.js";

export function runChecks(fixture: string, golden: Golden, graph: LineageGraph): FixtureResult {
export function runChecks(
fixture: string,
golden: Golden,
graph: LineageGraph,
aliases?: Record<string, string>,
): FixtureResult {
const checks: CheckResult[] = [];
const attribution = { truePositives: 0, falsePositives: 0, falseNegatives: 0 };

Expand Down Expand Up @@ -218,7 +223,11 @@ export function runChecks(fixture: string, golden: Golden, graph: LineageGraph):

for (const query of golden.expect.queries ?? []) {
const id = `query:${query.terms.join("+") || JSON.stringify(query.structure)}`;
const result = matchComponents(graph, { terms: query.terms, structure: query.structure });
const result = matchComponents(graph, {
terms: query.terms,
structure: query.structure,
...(aliases !== undefined ? { aliases } : {}),
});
let passed = result.status === query.status;
let detail: string | undefined;
if (!passed) {
Expand Down
10 changes: 9 additions & 1 deletion eval/src/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,18 @@ import path from "node:path";
import { fileURLToPath } from "node:url";

import { resolveHookEdges, scanReact } from "@coderadar/parser-react";
import { parse as parseYaml } from "yaml";

import { runChecks } from "./checks.js";
import type { FixtureResult, Golden, Scorecard, Thresholds } from "./golden.js";

/** A fixture's business-vocabulary glossary (aliases.yaml), phrase → component. */
function loadAliases(fixtureDir: string): Record<string, string> | undefined {
const aliasPath = path.join(fixtureDir, "aliases.yaml");
if (!fs.existsSync(aliasPath)) return undefined;
return parseYaml(fs.readFileSync(aliasPath, "utf-8")) as Record<string, string>;
}

const evalDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const fixturesDir = path.join(evalDir, "fixtures");

Expand All @@ -41,7 +49,7 @@ function main(): void {
const golden = JSON.parse(fs.readFileSync(goldenPath, "utf-8")) as Golden;
const appDir = path.resolve(fixtureDir, golden.app ?? "./app");
const graph = resolveHookEdges(scanReact({ root: appDir, ...(golden.scan ?? {}) }));
results.push(runChecks(name, golden, graph));
results.push(runChecks(name, golden, graph, loadAliases(fixtureDir)));
}

const scorecard = buildScorecard(results);
Expand Down
33 changes: 30 additions & 3 deletions packages/cli/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,16 @@ import {
type JourneyPath,
journeys,
type LineageGraph,
loadCorrections,
loadGraph as loadGraphFile,
matchComponentsByText,
matchComponents,
recordCorrection,
saveGraph,
traceLineage,
} from "@coderadar/core";
import { resolveHookEdges, scanReact } from "@coderadar/parser-react";
import { Command } from "commander";
import { parse as parseYaml } from "yaml";

const program = new Command();

Expand Down Expand Up @@ -57,9 +60,22 @@ program
.description("Find components by text visible on screen (e.g. read off a screenshot)")
.argument("<terms...>", "text fragments seen in the UI")
.option("-g, --graph <file>", "graph file", "ui-lineage.graph.json")
.action((terms: string[], opts: { graph: string }) => {
.option("-a, --aliases <file>", "business-vocab glossary (aliases.yaml)")
.option("-c, --corrections <file>", "corrections store (jsonl)", "ui-lineage.corrections.jsonl")
.action((terms: string[], opts: { graph: string; aliases?: string; corrections: string }) => {
const graph = loadGraph(opts.graph);
const result = matchComponentsByText(graph, terms);
const aliases =
opts.aliases !== undefined && fs.existsSync(opts.aliases)
? (parseYaml(fs.readFileSync(opts.aliases, "utf-8")) as Record<string, string>)
: undefined;
const corrections = fs.existsSync(opts.corrections)
? loadCorrections(opts.corrections)
: undefined;
const result = matchComponents(graph, {
terms,
...(aliases !== undefined ? { aliases } : {}),
...(corrections !== undefined ? { corrections } : {}),
});
if (result.status === "declined") {
console.log(`No components matched (${result.declineReason}).`);
return;
Expand Down Expand Up @@ -162,6 +178,17 @@ program
for (const path of paths) printJourneyPath(path);
});

program
.command("correct")
.description("Record that some on-screen text means a component — feeds future `find` results")
.argument("<component>", "component name it should resolve to")
.argument("<terms...>", "the text fragments that mean it")
.option("-c, --corrections <file>", "corrections store (jsonl)", "ui-lineage.corrections.jsonl")
.action((component: string, terms: string[], opts: { corrections: string }) => {
recordCorrection(opts.corrections, { terms, component });
console.log(`Recorded: [${terms.join(", ")}] → ${component} (${opts.corrections})`);
});

const STEP_ARROW: Record<string, string> = {
page: "▸",
event: "•",
Expand Down
26 changes: 26 additions & 0 deletions packages/core/src/corrections.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
/**
* The corrections store (TRACKER step 4.6, failure mode G4).
*
* When a human confirms that a set of terms means a specific component, that
* confirmation is appended to a `corrections.jsonl` file (one JSON object per
* line) and fed back to the matcher as the highest-weight evidence, so the next
* identical query resolves the same way. Terms only — never screenshots (G7).
*/
import fs from "node:fs";

import type { Correction } from "./query.js";

/** Read all corrections from a JSONL file. Missing file → empty list. */
export function loadCorrections(path: string): Correction[] {
if (!fs.existsSync(path)) return [];
return fs
.readFileSync(path, "utf-8")
.split("\n")
.filter((line) => line.trim().length > 0)
.map((line) => JSON.parse(line) as Correction);
}

/** Append one correction to the JSONL store. */
export function recordCorrection(path: string, correction: Correction): void {
fs.appendFileSync(path, `${JSON.stringify(correction)}\n`);
}
1 change: 1 addition & 0 deletions packages/core/src/index.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
export * from "./types.js";
export * from "./result.js";
export * from "./query.js";
export * from "./corrections.js";
export * from "./storage.js";
export * from "./text.js";
53 changes: 51 additions & 2 deletions packages/core/src/matching.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
import { describe, expect, it } from "vitest";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";

import { matchComponentsByText } from "./query.js";
import { afterEach, describe, expect, it } from "vitest";

import { loadCorrections, recordCorrection } from "./corrections.js";
import { matchComponents, matchComponentsByText } from "./query.js";
import { editDistance, fuzzyTokenMatch } from "./text.js";
import { type ComponentNode, type LineageGraph, nodeId } from "./types.js";

Expand Down Expand Up @@ -89,3 +94,47 @@ describe("matchComponentsByText scorer (TRACKER 4.1, A4/A10)", () => {
expect(matchComponentsByText(graph(forms), ["Purchase history"]).status).toBe("declined");
});
});

describe("alias glossary & corrections (TRACKER 4.6, E2/G4)", () => {
const g = graph([
component("BillingSummaryCard", ["Billing summary", "Amount due"]),
component("InvoiceList", ["Recent invoices"]),
]);

it("resolves a business-vocab phrase that appears nowhere in the code", () => {
expect(matchComponentsByText(g, ["invoice widget"]).status).toBe("declined");
const withAlias = matchComponents(g, {
terms: ["invoice widget"],
aliases: { "invoice widget": "BillingSummaryCard" },
});
expect(withAlias.status).toBe("ok");
expect(withAlias.candidates[0]?.value.component.name).toBe("BillingSummaryCard");
expect(withAlias.candidates[0]?.evidence.some((e) => e.kind === "alias")).toBe(true);
});

it("a recorded correction overrides an otherwise-correct text match", () => {
const before = matchComponents(g, { terms: ["Recent invoices"] });
expect(before.candidates[0]?.value.component.name).toBe("InvoiceList");
const after = matchComponents(g, {
terms: ["Recent invoices"],
corrections: [{ terms: ["Recent invoices"], component: "BillingSummaryCard" }],
});
expect(after.candidates[0]?.value.component.name).toBe("BillingSummaryCard");
expect(after.candidates[0]?.evidence.some((e) => e.kind === "correction")).toBe(true);
});

const tmpFiles: string[] = [];
afterEach(() => {
for (const f of tmpFiles.splice(0)) fs.rmSync(f, { force: true });
});

it("round-trips corrections through a JSONL store; the next query flips top-1", () => {
const file = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "uil-")), "corrections.jsonl");
tmpFiles.push(file);
expect(loadCorrections(file)).toEqual([]);
recordCorrection(file, { terms: ["Recent invoices"], component: "BillingSummaryCard" });
const corrections = loadCorrections(file);
const result = matchComponents(g, { terms: ["Recent invoices"], corrections });
expect(result.candidates[0]?.value.component.name).toBe("BillingSummaryCard");
});
});
68 changes: 65 additions & 3 deletions packages/core/src/query.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,17 @@ export interface ComponentMatch {
* Returns `ambiguous` when the leaders tie on rarity-weighted score (a lone
* generic term is honestly ambiguous), `declined("no-signal")` on no match.
*/
/**
* A recorded correction (Phase 4.6, failure mode G4): a human confirmed that a
* set of terms means a specific component. Fed back as first-class evidence so
* the next identical query resolves the same way.
*/
export interface Correction {
terms: string[];
/** Component definition name the terms were confirmed to mean. */
component: string;
}

/** A screenshot/ticket query: visible text, a structure descriptor, or both. */
export interface MatchQuery {
terms?: string[];
Expand All @@ -73,10 +84,23 @@ export interface MatchQuery {
* so the emphasized element outranks incidental text.
*/
boosts?: Record<string, number>;
/**
* Business-vocabulary glossary (Phase 4.6, failure mode E2), phrase →
* component name/id — "invoice widget" → BillingSummaryCard. An alias hit is
* high-weight evidence, so a term that appears nowhere in the code still
* resolves.
*/
aliases?: Record<string, string>;
/** Recorded corrections (Phase 4.6) — the highest-weight signal of all. */
corrections?: Correction[];
}

/** Weight of a full structural match relative to a rare matched term. */
const STRUCTURE_WEIGHT = 3;
/** A glossary alias hit outweighs any text/structure evidence (Phase 4.6). */
const ALIAS_WEIGHT = 10;
/** A recorded human correction is the strongest signal of all. */
const CORRECTION_WEIGHT = 25;

/** Back-compat text-only entry point. */
export function matchComponentsByText(
Expand Down Expand Up @@ -145,6 +169,13 @@ export function matchComponents(
return base * (boosts.get(term) ?? 1);
};

// Glossary aliases + recorded corrections (Phase 4.6). Both are authority
// signals — a phrase resolves even when it appears nowhere in the code.
const aliasEntries = Object.entries(query.aliases ?? {});
const corrections = query.corrections ?? [];
const containsTerm = (needle: string): boolean =>
needle.length > 0 && queryTerms.some((t) => t === needle || t.includes(needle) || needle.includes(t));

// Render tree (definition level): A renders B when A has a `renders` edge to
// an instance whose `instance-of` points at B. Used to collapse nested
// matches to the most specific one (step 4.3).
Expand Down Expand Up @@ -230,15 +261,46 @@ export function matchComponents(
});
}

if (covered.size === 0 && structureFit === 0) continue;
const coverage = queryTerms.length > 0 ? covered.size / queryTerms.length : structureFit;
// Authority: glossary aliases and recorded corrections that name this
// component, when the query actually contains their phrase/terms.
let authority = 0;
const normName = normalizeText(component.name);
for (const [phrase, target] of aliasEntries) {
if (normalizeText(target) !== normName && target !== component.id) continue;
const normPhrase = normalizeText(phrase);
if (!containsTerm(normPhrase)) continue;
authority += ALIAS_WEIGHT;
covered.add(normPhrase);
if (!matched.includes(normPhrase)) matched.push(normPhrase);
evidence.push({
kind: "alias",
detail: `glossary alias "${phrase}" → ${component.name}`,
loc: component.loc,
});
}
for (const correction of corrections) {
if (normalizeText(correction.component) !== normName) continue;
const cterms = correction.terms.map(normalizeText).filter((t) => t.length > 0);
if (cterms.length === 0 || !cterms.every(containsTerm)) continue;
authority += CORRECTION_WEIGHT;
for (const ct of cterms) covered.add(ct);
evidence.push({
kind: "correction",
detail: `recorded correction [${correction.terms.join(", ")}] → ${component.name}`,
loc: component.loc,
});
}

if (covered.size === 0 && structureFit === 0 && authority === 0) continue;
const coverage =
authority > 0 ? 1 : queryTerms.length > 0 ? covered.size / queryTerms.length : structureFit;
scored.push({
match: {
component,
instances: instancesByDefinition.get(component.id) ?? [],
matchedText: matched.length > 0 ? matched : [...covered],
},
score: textScore + structureFit * STRUCTURE_WEIGHT,
score: textScore + structureFit * STRUCTURE_WEIGHT + authority,
coverage: Math.max(coverage, structureFit),
covered,
evidence,
Expand Down
3 changes: 3 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading