diff --git a/TRACKER.md b/TRACKER.md index 869ab7e..184436a 100644 --- a/TRACKER.md +++ b/TRACKER.md @@ -5,8 +5,8 @@ ## Status - **Current phase:** 6F — Field hardening, feedback round 1 (runs before 6.1–6.5) -- **Next step:** 6F.10 HTML-entity rendered-text noise (self-found on Grafana — ` `→"nbsp", `"`→"34" create spurious matches; decode/strip entities in the parser's text extraction, failure mode A16); then 6F.6 test-coverage hardening (blocked: needs the field's actual failing variant — Grafana produced 1,009 covered-by edges, so detection works on real code; the fixture's custom-wrapper shape already passes) -- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.6, 4.1–4.6, 5.1–5.7, 6F.1–6F.5, 6F.7–6F.9 · **released 0.4.0** (tag v0.4.0, PR #50 to main; publish to npm pending). **Self-validated on Grafana frontend** (6,461 files, 15,334 nodes / 18,367 edges in 72 s): 55 RTK-query data sources, 32 routes, 1,009 coverage edges, gibberish declines — all previously 0/1 in the field run. +- **Next step:** 6F.6 test-coverage hardening (blocked: needs the field's actual failing variant — Grafana produced 1,009 covered-by edges, so detection works on real code; the fixture's custom-wrapper shape already passes) +- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.6, 4.1–4.6, 5.1–5.7, 6F.1–6F.5, 6F.7–6F.10 · **released 0.4.0** (tag v0.4.0, PR #50 to main; publish to npm pending). **Self-validated on Grafana frontend** (6,461 files, 15,334 nodes / 18,367 edges in 72 s): 55 RTK-query data sources, 32 routes, 1,009 coverage edges, gibberish declines — all previously 0/1 in the field run. - **Gates passed:** Gate 0 (CI + red-path, #5/#6) · Gate 1 (precision 1.000, recall 0.895, zero poison) · Gate 2 (C1 instance attribution 1.000 · B1 4-level handler chains · C6 store writers↔readers · A9 portals — scorecard 137/0/0, precision & recall 1.000) · Gate 3 (B3 action effects · B4 routers · B6 cyclic journeys terminate · B7/B8 form & non-JSX events · G5 flag/role conditions — precision & recall 1.000) · Gate 4 (A4 rarity · A10 fuzzy/OCR · A1 structural · A6 subtree · E3 vision annotations · E2 aliases · G4 corrections — high-conf correct 1.000, ambiguity honesty 1.000, poison rate 0.000) · Gate 5 (F1 context bundle · F2 blast radius · F3 test coverage · F4 response schema · F5 git history · MCP server over stdio — scorecard 265/0/0, all honesty metrics 1.000; **M5 reached** — ticket in → budgeted context bundle out, over MCP) ## What CodeRadar is @@ -556,7 +556,7 @@ Grafana graph:** `Find silences by matcher` → `SilencesFilter` top-1 (`OrderBy honest `no-signal`. eval 297/0/0/0, gate OK, all metrics 1.000. A16 (HTML-entity rendered text) spun out to 6F.10 — out of scope for a scoring PR. -### [ ] 6F.10 HTML-entity rendered-text noise +### [x] 6F.10 HTML-entity rendered-text noise **Failure modes:** A16 (new) **Build:** self-found on Grafana — rendered text that is an HTML entity (` `, `"`, `>`) normalizes to a junk token (`nbsp`, `34`, `gt`); numeric entities make gibberish @@ -565,10 +565,25 @@ containing those digits match. Decode/strip HTML entities in the parser's text-e extraction, hence a separate step from 6F.9. **Accept:** fixture with entity-only rendered text: it produces no match target; gibberish that shares digits with a numeric entity declines. eval green. +**Done:** new `entities.ts` in parser-react — `decodeEntities(text)` resolves the HTML entities +React decodes at render time (numeric decimal `"` / hex `"` generically, named entities +from a curated map: markup, whitespace, punctuation, symbols, currency, accented Latin-1; +unknown names left verbatim, matching React). `extractRenderedText` decodes JSX text and quoted +attribute values — the two surfaces React HTML-decodes — while JS string/template literals stay +untouched (React renders `{" "}` literally). Decoded entities become the character React +renders, which the normalizer strips: ` `→space→dropped, `>`/`"`/`·`→ +punctuation that normalizes to empty, so an entity-only component yields **no discriminating +target** (verified: `EntitySpacer.renderedText` = `["\"", ">", "·", "<", "›"]`, zero +alphanumeric tokens). New fixture `a16-html-entities` (entity-only `EntitySpacer` + real +`QuotaNotice`): the named-entity token `nbsp`, the numeric-entity token `34`, and a gibberish +query sharing those digits (`zzqwxnomatch12345`, which pre-fix matched via `"`→"34") all +decline `no-signal`, while the real query still lands on `QuotaNotice` and isn't poisoned by the +digit-sharing gibberish. 7 new tests (4 unit decode + 3 fixture integration), 136 parser-react +total; eval 304/0/0/0, gate OK, all metrics 1.000. **Gate 6F:** field-patterns fixture fully green (skip list empty ✅) · instance resolution ≥ 95% (✅ 100%) · RTK data sources > 0 (✅) · route nodes > 0 (✅) · gibberish queries decline -`no-signal` (✅) · `pnpm eval` green end-to-end (✅ 290/0/0/0). Remaining before the gate is +`no-signal` (✅) · `pnpm eval` green end-to-end (✅ 304/0/0/0). Remaining before the gate is formally stamped: 6F.6 test-coverage hardening (blocked on a real failing sample). --- diff --git a/eval/fixtures/a16-html-entities/app/EntitySpacer.tsx b/eval/fixtures/a16-html-entities/app/EntitySpacer.tsx new file mode 100644 index 0000000..9238de4 --- /dev/null +++ b/eval/fixtures/a16-html-entities/app/EntitySpacer.tsx @@ -0,0 +1,16 @@ +// Its only rendered text is HTML entities — the shape the Grafana frontend hit +// (26  , 4 ", plus "/>/</·/›). Each decodes to +// punctuation/whitespace, so after decoding there is no match target at all. +export function EntitySpacer() { + return ( + + ); +} diff --git a/eval/fixtures/a16-html-entities/app/QuotaNotice.tsx b/eval/fixtures/a16-html-entities/app/QuotaNotice.tsx new file mode 100644 index 0000000..c19e603 --- /dev/null +++ b/eval/fixtures/a16-html-entities/app/QuotaNotice.tsx @@ -0,0 +1,6 @@ +// A real component with distinctive text, so the fixture isn't trivially +// all-declining: a real query must still land here, and gibberish that shares +// digits with a numeric entity must not poison it. +export function QuotaNotice() { + return

Storage quota exceeded

; +} diff --git a/eval/fixtures/a16-html-entities/golden.json b/eval/fixtures/a16-html-entities/golden.json new file mode 100644 index 0000000..0ab8827 --- /dev/null +++ b/eval/fixtures/a16-html-entities/golden.json @@ -0,0 +1,17 @@ +{ + "failureMode": "A16", + "note": "HTML-entity rendered text (self-found on Grafana, 0.4.0): a component whose only rendered text is HTML entities ( , ", >, ", ·, <, ›) normalized to junk tokens ('nbsp', '34', 'gt'), creating spurious matches — and a numeric entity like " let a gibberish query sharing its digits ('...12345') match. Fix (6F.10): decode HTML entities in the parser's text-extraction pass so they become the character React renders, which normalizes away as punctuation. The entity-only component then produces no match target: a named-entity token ('nbsp'), a numeric-entity token ('34'), and a gibberish query that shares those digits all decline no-signal, while a real query still lands on the real component and isn't poisoned by the digit-sharing gibberish.", + "expect": { + "components": [ + { "name": "EntitySpacer", "instances": 0 }, + { "name": "QuotaNotice", "instances": 0 } + ], + "queries": [ + { "terms": ["34"], "status": "declined" }, + { "terms": ["nbsp"], "status": "declined" }, + { "terms": ["zzqwxnomatch12345"], "status": "declined" }, + { "terms": ["Storage quota exceeded"], "status": "ok", "top": "QuotaNotice" }, + { "terms": ["zzqwxnomatch12345", "Storage quota exceeded"], "status": "ok", "top": "QuotaNotice" } + ] + } +} diff --git a/packages/parser-react/src/entities.test.ts b/packages/parser-react/src/entities.test.ts new file mode 100644 index 0000000..cded1a0 --- /dev/null +++ b/packages/parser-react/src/entities.test.ts @@ -0,0 +1,73 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { matchComponentsByText } from "@coderadar/core"; +import { describe, expect, it } from "vitest"; + +import { decodeEntities } from "./entities.js"; +import { scanReact } from "./scan.js"; + +const fixtures = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../../eval/fixtures", +); + +describe("decodeEntities", () => { + it("decodes named entities React resolves", () => { + expect(decodeEntities(" ")).toBe(" "); + expect(decodeEntities("a & b")).toBe("a & b"); + expect(decodeEntities("<tag>")).toBe(""); + expect(decodeEntities(""q"")).toBe('"q"'); + expect(decodeEntities("·›")).toBe("·›"); + expect(decodeEntities("café")).toBe("café"); + }); + + it("decodes decimal and hex numeric entities", () => { + expect(decodeEntities(""")).toBe('"'); + expect(decodeEntities(""")).toBe('"'); + expect(decodeEntities(""")).toBe('"'); + expect(decodeEntities("é")).toBe("é"); + }); + + it("leaves unknown names and out-of-range code points untouched", () => { + expect(decodeEntities("&foobar;")).toBe("&foobar;"); + expect(decodeEntities("�")).toBe("�"); + // Double-encoded text stays literal — only one decoding pass, no `"`. + expect(decodeEntities("&#34;")).toBe("""); + }); + + it("preserves surrounding text and mixed content", () => { + expect(decodeEntities("Hello World")).toBe("Hello World"); + expect(decodeEntities("no entities here")).toBe("no entities here"); + }); +}); + +describe("html-entity rendered text (a16 fixture)", () => { + const graph = scanReact({ root: path.join(fixtures, "a16-html-entities", "app") }); + + it("leaves an entity-only component with no discriminating match target", () => { + const node = graph.nodes.find((n) => n.kind === "component" && n.name === "EntitySpacer"); + if (node?.kind !== "component") throw new Error("EntitySpacer not found"); + // Decoded entities are whitespace/punctuation only — nothing survives + // normalization as a token, so ` ` never leaks a "nbsp" target. + expect(node.renderedText.every((e) => !/[a-z0-9]/i.test(e.text))).toBe(true); + }); + + it("declines junk tokens the raw entities would have produced", () => { + // Named-entity junk ("nbsp"), numeric-entity junk ("34"), and gibberish + // that shares those digits all decline instead of matching EntitySpacer. + for (const term of ["nbsp", "34", "zzqwxnomatch12345"]) { + expect(matchComponentsByText(graph, [term]).status).toBe("declined"); + } + }); + + it("still matches a real component and isn't poisoned by digit-sharing gibberish", () => { + const real = matchComponentsByText(graph, ["Storage quota exceeded"]); + expect(real.status).toBe("ok"); + expect(real.candidates[0]?.value.component.name).toBe("QuotaNotice"); + + const mixed = matchComponentsByText(graph, ["zzqwxnomatch12345", "Storage quota exceeded"]); + expect(mixed.status).toBe("ok"); + expect(mixed.candidates[0]?.value.component.name).toBe("QuotaNotice"); + }); +}); diff --git a/packages/parser-react/src/entities.ts b/packages/parser-react/src/entities.ts new file mode 100644 index 0000000..79948d1 --- /dev/null +++ b/packages/parser-react/src/entities.ts @@ -0,0 +1,82 @@ +/** + * HTML-entity decoding for extracted rendered text (failure mode A16). + * + * JSX text and quoted attribute values are HTML-decoded by React at render + * time — ` ` renders a non-breaking space, `>` a + * `>`, `"` a double quote — but ts-morph hands us the raw source, so + * without decoding the entity survives into `renderedText` and normalizes to a + * junk token: ` `→"nbsp", `>`→"gt", `"`→"34". Those spurious tokens + * create false matches; a numeric entity like `"` even lets a gibberish + * query that happens to share its digits ("zzqwxnomatch12345") match. Decoding + * restores the character React actually renders, which then normalizes away as + * punctuation/whitespace (or becomes real letters for accented entities), so + * entity-only text stops acting as a match target (self-found on Grafana's + * frontend, 0.4.0 — 26 ` `, 4 `"`, plus `"`/`>`/`<`/ + * `·`/`›`). + * + * Only the entities React decodes are handled. Unknown names are left verbatim + * (React renders `&foobar;` literally), which is also the safe choice — we + * never guess. + */ + +const SPACE = " "; + +/** + * Named HTML entities React decodes, mapped to their character. Markup, + * whitespace, punctuation, symbols and currency all decode to characters the + * normalizer strips (so entity-only text yields no target); accented Latin-1 + * letters decode to real letters, which is faithful rendered signal. + */ +const NAMED: Record = { + // Markup + whitespace. Spaces collapse to a plain ASCII space (what OCR reads + // off a screenshot); the invisible soft hyphen decodes to nothing. + amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", + nbsp: SPACE, ensp: SPACE, emsp: SPACE, thinsp: SPACE, shy: "", + // Punctuation + hellip: "…", mdash: "—", ndash: "–", minus: "−", + lsquo: "‘", rsquo: "’", sbquo: "‚", + ldquo: "“", rdquo: "”", bdquo: "„", + laquo: "«", raquo: "»", lsaquo: "‹", rsaquo: "›", + middot: "·", bull: "•", dagger: "†", Dagger: "‡", + sect: "§", para: "¶", prime: "′", Prime: "″", + // Symbols + currency + fractions + copy: "©", reg: "®", trade: "™", deg: "°", + plusmn: "±", times: "×", divide: "÷", + frac12: "½", frac14: "¼", frac34: "¾", + euro: "€", pound: "£", cent: "¢", yen: "¥", + // Common accented Latin-1 letters (decode to real match signal) + agrave: "à", aacute: "á", acirc: "â", atilde: "ã", + auml: "ä", aring: "å", aelig: "æ", ccedil: "ç", + egrave: "è", eacute: "é", ecirc: "ê", euml: "ë", + igrave: "ì", iacute: "í", icirc: "î", iuml: "ï", + ntilde: "ñ", ograve: "ò", oacute: "ó", ocirc: "ô", + otilde: "õ", ouml: "ö", oslash: "ø", ugrave: "ù", + uacute: "ú", ucirc: "û", uuml: "ü", yuml: "ÿ", szlig: "ß", +}; + +const ENTITY = /&(#[Xx][0-9A-Fa-f]+|#[0-9]+|[A-Za-z][A-Za-z0-9]*);/g; + +/** A code point safe to materialize: in range and not a lone surrogate. */ +function fromCodePoint(code: number): string | null { + if (!Number.isInteger(code) || code < 0 || code > 0x10ffff) return null; + if (code >= 0xd800 && code <= 0xdfff) return null; + return String.fromCodePoint(code); +} + +/** + * Decode the HTML entities React resolves at render time. Numeric entities + * (decimal `"`, hex `"`) decode generically; named entities decode + * from {@link NAMED}. Unrecognized names and out-of-range code points are left + * untouched, matching React's literal rendering. + */ +export function decodeEntities(input: string): string { + if (!input.includes("&")) return input; + return input.replace(ENTITY, (whole, body: string) => { + if (body.charCodeAt(0) === 35 /* # */) { + const isHex = body.charCodeAt(1) === 88 || body.charCodeAt(1) === 120; /* X | x */ + const code = Number.parseInt(body.slice(isHex ? 2 : 1), isHex ? 16 : 10); + return fromCodePoint(code) ?? whole; + } + return NAMED[body] ?? whole; + }); +} diff --git a/packages/parser-react/src/scan.ts b/packages/parser-react/src/scan.ts index 8654f64..0b55435 100644 --- a/packages/parser-react/src/scan.ts +++ b/packages/parser-react/src/scan.ts @@ -30,6 +30,7 @@ import { } from "ts-morph"; import { fetchMethod, resolveEndpoint, type ResolvedEndpoint, resolveStringValue } from "./endpoint.js"; +import { decodeEntities } from "./entities.js"; import { i18nRenderedText, type I18nOptions, loadLocaleTable, type LocaleTable } from "./i18n.js"; import { linkOpenApiResponses, loadOpenApi, responseFromCall } from "./response.js"; import { detectRoutes } from "./routes.js"; @@ -368,8 +369,12 @@ function extractRenderedText(fn: Node): RenderedText[] { entries.set(`${entry.source}:${entry.text}:${entry.branch ?? ""}`, entry); }; + // JSX text and quoted attribute values are HTML-decoded by React at render + // time, so decode entities here too (failure mode A16) — otherwise ` `, + // `>`, `"` survive as junk tokens ("nbsp", "gt", "34"). JS string and + // template literals below are NOT JSX-decoded, so they stay untouched. for (const jsxText of fn.getDescendantsOfKind(SyntaxKind.JsxText)) { - const text = jsxText.getText().replace(/\s+/g, " ").trim(); + const text = decodeEntities(jsxText.getText()).replace(/\s+/g, " ").trim(); if (text.length === 0) continue; add({ text, source: "jsx", ...branchTag(jsxText, fn) }); } @@ -378,7 +383,7 @@ function extractRenderedText(fn: Node): RenderedText[] { if (!TEXT_ATTRIBUTES.has(attr.getNameNode().getText())) continue; const init = attr.getInitializer(); if (init !== undefined && Node.isStringLiteral(init)) { - const text = init.getLiteralValue().trim(); + const text = decodeEntities(init.getLiteralValue()).trim(); if (text.length > 0) add({ text, source: "attribute", ...branchTag(attr, fn) }); } }