diff --git a/TRACKER.md b/TRACKER.md index eb0c418..869ab7e 100644 --- a/TRACKER.md +++ b/TRACKER.md @@ -5,8 +5,8 @@ ## Status - **Current phase:** 6F — Field hardening, feedback round 1 (runs before 6.1–6.5) -- **Next step:** ship **0.4.0** to the tester for a re-validation round; then 6F.6 test-coverage hardening (blocked: needs the field's actual failing variant — the fixture's custom-wrapper shape already passes; get a failing test file from the tester or ship the defensive `coverage-unmapped` half only) -- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.6, 4.1–4.6, 5.1–5.7, 6F.1–6F.5, 6F.7–6F.8 · **release 0.4.0 prepared** (all packages bumped, changelog + version strings, `npm pack` → ui-lineage-0.4.0.tgz verified, publish pending) +- **Next step:** 6F.10 HTML-entity rendered-text noise (self-found on Grafana — ` `→"nbsp", `"`→"34" create spurious matches; decode/strip entities in the parser's text extraction, failure mode A16); then 6F.6 test-coverage hardening (blocked: needs the field's actual failing variant — Grafana produced 1,009 covered-by edges, so detection works on real code; the fixture's custom-wrapper shape already passes) +- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.6, 4.1–4.6, 5.1–5.7, 6F.1–6F.5, 6F.7–6F.9 · **released 0.4.0** (tag v0.4.0, PR #50 to main; publish to npm pending). **Self-validated on Grafana frontend** (6,461 files, 15,334 nodes / 18,367 edges in 72 s): 55 RTK-query data sources, 32 routes, 1,009 coverage edges, gibberish declines — all previously 0/1 in the field run. - **Gates passed:** Gate 0 (CI + red-path, #5/#6) · Gate 1 (precision 1.000, recall 0.895, zero poison) · Gate 2 (C1 instance attribution 1.000 · B1 4-level handler chains · C6 store writers↔readers · A9 portals — scorecard 137/0/0, precision & recall 1.000) · Gate 3 (B3 action effects · B4 routers · B6 cyclic journeys terminate · B7/B8 form & non-JSX events · G5 flag/role conditions — precision & recall 1.000) · Gate 4 (A4 rarity · A10 fuzzy/OCR · A1 structural · A6 subtree · E3 vision annotations · E2 aliases · G4 corrections — high-conf correct 1.000, ambiguity honesty 1.000, poison rate 0.000) · Gate 5 (F1 context bundle · F2 blast radius · F3 test coverage · F4 response schema · F5 git history · MCP server over stdio — scorecard 265/0/0, all honesty metrics 1.000; **M5 reached** — ticket in → budgeted context bundle out, over MCP) ## What CodeRadar is @@ -537,6 +537,35 @@ round-trips, dangling-edge drop, `` breakout guard, self-contained asse README documents the command. eval unaffected (290/0/0/0). **Gate 6F extractor criteria met; only 6F.6 (coverage, data-blocked) remains before the gate fully closes.** +### [x] 6F.9 Stopword & rare-literal scoring noise +**Failure modes:** A15 (new), A4, D6 +**Build:** self-found while validating 0.4.0 on Grafana's frontend — `find "Find silences by +matcher"` ranked `OrderBySection` (renders a bare `BY`) top-1 over `SilencesFilter`, because a +lone stopword is a *rare literal* with high IDF; and `find "The"` returned confident matches on +"the". New `isLowSignal(normalized)` in core/text.ts (extends the A14 `hasMatchSignal` guard): +true when a string is empty, punctuation-only, or entirely stopwords (folded through +`foldPlural` so it compares to normalized text). `matchComponents` now drops stopword-only query +terms and skips stopword-only rendered-text targets, so the exact-phrase component wins and a +stopword-only query declines `no-signal`. +**Accept:** new fixture `a15-stopword-noise` (a `BY` component, an exact-phrase component, a +`the`-rendering component): the phrase query ranks the exact-phrase component top-1; a stopword +query declines; a stopword mixed with a real term ignores the stopword-rendering component. +**Done:** implemented + 5 unit tests (2 text, 3 matcher) + fixture. **Verified on the real +Grafana graph:** `Find silences by matcher` → `SilencesFilter` top-1 (`OrderBySection` gone), +`The` → `no-signal`, and `resolve` on a stopword-heavy ticket went from confidently-wrong to an +honest `no-signal`. eval 297/0/0/0, gate OK, all metrics 1.000. A16 (HTML-entity rendered text) +spun out to 6F.10 — out of scope for a scoring PR. + +### [ ] 6F.10 HTML-entity rendered-text noise +**Failure modes:** A16 (new) +**Build:** self-found on Grafana — rendered text that is an HTML entity (` `, `"`, +`>`) normalizes to a junk token (`nbsp`, `34`, `gt`); numeric entities make gibberish +containing those digits match. Decode/strip HTML entities in the parser's text-extraction pass +(parser-react), or treat entity-only rendered text as low-signal. Not a scoring fix — belongs in +extraction, hence a separate step from 6F.9. +**Accept:** fixture with entity-only rendered text: it produces no match target; gibberish that +shares digits with a numeric entity declines. eval green. + **Gate 6F:** field-patterns fixture fully green (skip list empty ✅) · instance resolution ≥ 95% (✅ 100%) · RTK data sources > 0 (✅) · route nodes > 0 (✅) · gibberish queries decline `no-signal` (✅) · `pnpm eval` green end-to-end (✅ 290/0/0/0). Remaining before the gate is diff --git a/docs/failure-modes.md b/docs/failure-modes.md index 8a7c009..323ca4b 100644 --- a/docs/failure-modes.md +++ b/docs/failure-modes.md @@ -31,6 +31,8 @@ every downstream agent. When in doubt, return ranked candidates with evidence, o | A12 | **Non-text UI** — charts, icons, canvases | Nothing to text-match | Graceful degradation: structural candidates + explicit low confidence | 4 | | A13 | **Third-party embeds** — Intercom, iframes | Content isn't ours | Detect and report "not in this codebase" | 4 | | A14 | **Empty-normalizing rendered text** — components rendering bare `\|` / `/` / `-` | Normalization strips punctuation → empty target treated as a substring of every query; matcher becomes a universal wildcard (field-found in v0.3.0) | Discard targets that normalize to empty; minimum alphanumeric token length; `no-signal` decline when only such targets match | 6F | +| A15 | **Stopword & rare-literal noise** — a component renders a bare `BY`; a ticket says "The …" | A stopword is a rare literal with high IDF, so it outranks the exact-phrase match; stopword-only queries return confident junk (field-found on Grafana 0.4.0) | Drop stopword-only query terms; reject stopword-only rendered-text targets (extends A14's low-signal guard) | 6F | +| A16 | **HTML-entity rendered text** — a component renders ` ` / `"` / `>` | Entities normalize to junk tokens (`nbsp`, `34`, `gt`); numeric entities let gibberish sharing those digits match (self-found on Grafana 0.4.0) | Decode/strip HTML entities in the text-extraction pass, or treat entity-only rendered text as low-signal | 6F | ## B. User-journey extraction diff --git a/eval/fixtures/a15-stopword-noise/app/AckPanel.tsx b/eval/fixtures/a15-stopword-noise/app/AckPanel.tsx new file mode 100644 index 0000000..f66dcbd --- /dev/null +++ b/eval/fixtures/a15-stopword-noise/app/AckPanel.tsx @@ -0,0 +1,3 @@ +export function AckPanel() { + return

Before you continue, acknowledge the following

; +} diff --git a/eval/fixtures/a15-stopword-noise/app/OrderBySection.tsx b/eval/fixtures/a15-stopword-noise/app/OrderBySection.tsx new file mode 100644 index 0000000..d3ac3d7 --- /dev/null +++ b/eval/fixtures/a15-stopword-noise/app/OrderBySection.tsx @@ -0,0 +1,8 @@ +export function OrderBySection() { + return ( +
+ + BY +
+ ); +} diff --git a/eval/fixtures/a15-stopword-noise/app/SilencesFilter.tsx b/eval/fixtures/a15-stopword-noise/app/SilencesFilter.tsx new file mode 100644 index 0000000..08daad0 --- /dev/null +++ b/eval/fixtures/a15-stopword-noise/app/SilencesFilter.tsx @@ -0,0 +1,7 @@ +export function SilencesFilter() { + return ( +
+ +
+ ); +} diff --git a/eval/fixtures/a15-stopword-noise/golden.json b/eval/fixtures/a15-stopword-noise/golden.json new file mode 100644 index 0000000..162f03d --- /dev/null +++ b/eval/fixtures/a15-stopword-noise/golden.json @@ -0,0 +1,17 @@ +{ + "failureMode": "A15", + "note": "Stopword & rare-literal noise (field-found on Grafana, 0.4.0): a component rendering a bare stopword ('BY') is a rare literal whose high IDF let it outrank the component that renders the whole phrase, and a stopword-only query ('The') returned confident matches. Fix (6F.9): drop stopword-only query terms and reject stopword-only rendered-text targets, so the exact-phrase component wins and a stopword query declines no-signal.", + "expect": { + "components": [ + { "name": "OrderBySection", "instances": 0 }, + { "name": "SilencesFilter", "instances": 0 }, + { "name": "AckPanel", "instances": 0 } + ], + "queries": [ + { "terms": ["Find silences by matcher"], "status": "ok", "top": "SilencesFilter" }, + { "terms": ["silences"], "status": "ok", "top": "SilencesFilter" }, + { "terms": ["The"], "status": "declined" }, + { "terms": ["the", "acknowledge"], "status": "ok", "top": "AckPanel" } + ] + } +} diff --git a/packages/core/src/matching.test.ts b/packages/core/src/matching.test.ts index 0551bca..605f77c 100644 --- a/packages/core/src/matching.test.ts +++ b/packages/core/src/matching.test.ts @@ -227,3 +227,36 @@ describe("identifier affinity & top-line score (TRACKER 6F.7)", () => { expect([...scores].sort((a, b) => b - a)).toEqual(scores); }); }); + +describe("stopword & rare-literal noise (TRACKER 6F.9, A15)", () => { + // Grafana field-found: `find "Find silences by matcher"` ranked OrderBySection + // (renders bare "BY") top-1 over SilencesFilter, because "BY" is a rare + // literal with huge IDF; and `find "The"` matched components on "the". + const g = graph([ + component("OrderBySection", ["BY"]), + component("SilencesFilter", ["Find silences by matcher"]), + component("AckPanel", ["acknowledge the following"]), + ]); + + it("the exact-phrase component outranks a rare stopword literal", () => { + const result = matchComponentsByText(g, ["Find silences by matcher"]); + expect(result.candidates[0]?.value.component.name).toBe("SilencesFilter"); + const names = result.candidates.map((c) => c.value.component.name); + expect(names).not.toContain("OrderBySection"); + }); + + it("a stopword-only query declines no-signal", () => { + const result = matchComponentsByText(g, ["The"]); + expect(result.status).toBe("declined"); + expect(result.declineReason).toBe("no-signal"); + }); + + it("stopwords in a real query don't pull in stopword-rendering components", () => { + // Word-level terms as a ticket extractor produces them: "the" is dropped, + // "silences" resolves — AckPanel (renders "the") must not surface. + const result = matchComponentsByText(g, ["the", "silences"]); + expect(result.status).toBe("ok"); + expect(result.candidates[0]?.value.component.name).toBe("SilencesFilter"); + expect(result.candidates.map((c) => c.value.component.name)).not.toContain("AckPanel"); + }); +}); diff --git a/packages/core/src/query.ts b/packages/core/src/query.ts index 38c339e..294682f 100644 --- a/packages/core/src/query.ts +++ b/packages/core/src/query.ts @@ -14,7 +14,7 @@ import { ok, type QueryResult, } from "./result.js"; -import { fuzzyTokenMatch, normalizeText, textMatches, tokenize } from "./text.js"; +import { fuzzyTokenMatch, isLowSignal, normalizeText, textMatches, tokenize } from "./text.js"; import type { ComponentNode, DataSourceNode, @@ -128,7 +128,9 @@ export function matchComponents( graph: LineageGraph, query: MatchQuery, ): QueryResult { - const queryTerms = (query.terms ?? []).map((t) => normalizeText(t)).filter((t) => t.length > 1); + const queryTerms = (query.terms ?? []) + .map((t) => normalizeText(t)) + .filter((t) => t.length > 1 && !isLowSignal(t)); const descriptor = query.structure; if (queryTerms.length === 0 && descriptor === undefined) return declined("no-signal"); @@ -162,7 +164,12 @@ export function matchComponents( const termTokens = tokenize(term); if (termTokens.length === 0) return null; for (const entry of component.renderedText) { - if (textMatches(normalizeText(entry.text), term)) return entry; + const normEntry = normalizeText(entry.text); + // A stopword-only / punctuation-only entry ("BY", "|") is not a real + // target: it matches too much and, being a rare literal, its high IDF + // would let it outrank an exact-phrase match (A14/A15). + if (isLowSignal(normEntry)) continue; + if (textMatches(normEntry, term)) return entry; if (containsPhrase(tokenize(entry.text), termTokens)) return entry; } return null; diff --git a/packages/core/src/text.test.ts b/packages/core/src/text.test.ts index 34eb160..2dc0348 100644 --- a/packages/core/src/text.test.ts +++ b/packages/core/src/text.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { hasMatchSignal, normalizeText, textMatches } from "./text.js"; +import { hasMatchSignal, isLowSignal, normalizeText, textMatches } from "./text.js"; describe("normalizeText", () => { it("lowercases, strips punctuation, collapses whitespace", () => { @@ -60,3 +60,20 @@ describe("hasMatchSignal", () => { expect(hasMatchSignal("ok")).toBe(true); }); }); + +describe("isLowSignal (A15 — stopword/punctuation targets)", () => { + it("flags empty, punctuation-only, and stopword-only strings", () => { + expect(isLowSignal("")).toBe(true); + expect(isLowSignal(normalizeText("|"))).toBe(true); + expect(isLowSignal(normalizeText("BY"))).toBe(true); // rare literal → no signal + expect(isLowSignal(normalizeText("The"))).toBe(true); + expect(isLowSignal(normalizeText("of the"))).toBe(true); + expect(isLowSignal(normalizeText("This"))).toBe(true); // folds to "thi" + }); + + it("keeps strings with at least one discriminating token", () => { + expect(isLowSignal(normalizeText("silences"))).toBe(false); + expect(isLowSignal(normalizeText("Find silences by matcher"))).toBe(false); + expect(isLowSignal(normalizeText("the invoice"))).toBe(false); + }); +}); diff --git a/packages/core/src/text.ts b/packages/core/src/text.ts index 66591bd..200aff7 100644 --- a/packages/core/src/text.ts +++ b/packages/core/src/text.ts @@ -61,6 +61,39 @@ export function textMatches(haystack: string, needle: string): boolean { return haystack.includes(needle) || needle.includes(haystack); } +/** + * Common English function words that carry no discriminating match signal. + * Folded through `foldPlural` at load so they compare equal to normalized text + * (which is also folded) — e.g. "this" → "thi", "does" → "doe". + */ +const STOPWORDS = new Set( + [ + "a", "an", "and", "are", "as", "at", "be", "been", "being", "but", "by", + "can", "could", "did", "do", "does", "for", "from", "had", "has", "have", + "he", "her", "his", "i", "if", "in", "into", "is", "it", "its", "just", + "may", "me", "might", "must", "my", "no", "not", "of", "on", "only", "or", + "our", "out", "over", "per", "she", "should", "so", "such", "than", "that", + "the", "their", "them", "then", "these", "they", "this", "those", "to", + "up", "us", "via", "was", "we", "were", "will", "with", "would", "you", + "your", + ].map(foldPlural), +); + +/** + * True when a normalized string carries no *discriminating* signal: it is + * empty, punctuation-only (A14), or nothing but stopwords ("the", "by"). + * Such a string is both a poor query term (matches everything) and a poor + * match target — a component rendering a bare "BY" is a rare literal whose + * high IDF would otherwise let it outrank the component that renders the whole + * phrase (failure mode A15, field-found on Grafana). `matchComponents` drops + * these from query terms and rejects them as rendered-text targets. + */ +export function isLowSignal(normalized: string): boolean { + if (!hasMatchSignal(normalized)) return true; + const tokens = normalized.split(" ").filter((t) => t.length > 0 && t !== "*"); + return tokens.length > 0 && tokens.every((t) => STOPWORDS.has(t)); +} + function foldPlural(token: string): string { if (token.length > 4 && token.endsWith("ies")) return `${token.slice(0, -3)}y`; if (token.length > 3 && token.endsWith("s") && !token.endsWith("ss")) return token.slice(0, -1);