diff --git a/TRACKER.md b/TRACKER.md index dabdcc2..f9bad4c 100644 --- a/TRACKER.md +++ b/TRACKER.md @@ -4,9 +4,9 @@ ## Status -- **Current phase:** 6F — Field hardening, feedback round 1 (runs before 6.1–6.5) -- **Next step:** publish 0.4.1 to the tester (all shippable fixes: 6F.6–6F.10 + visualizer). 6F.6 detection half remains blocked on a real failing test file — its defensive `coverage-unmapped` half shipped. -- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.6, 4.1–4.6, 5.1–5.7, 6F.1–6F.5, 6F.7–6F.10 + 6F.6 (defensive half) · **0.4.1 prepared** (all packages bumped, changelog + version strings, `npm pack` → ui-lineage-0.4.1.tgz verified: both bins, no @coderadar leak; publish to npm pending). 0.4.0 tagged/merged earlier. **Self-validated on Grafana frontend** (6,461 files, 15,334 nodes / 18,367 edges in 72 s): 55 RTK-query data sources, 32 routes, 1,009 coverage edges, gibberish declines — all previously 0/1 in the field run. +- **Current phase:** 6 — Lifecycle, scale, hardening (6F field-hardening done bar the data-blocked 6F.6 detection half) +- **Next step:** 6.5 generated/vendored classification & PII policy, then 6.4 version skew, 6.1 incremental re-scan, 6.2 scale (bench repo). 6.3 determinism landed first as it de-risks the rest (eval now hard-gates two-run byte-identity). Publish 0.4.1 to the tester still pending (needs npm creds). +- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.6, 4.1–4.6, 5.1–5.7, 6F.1–6F.5, 6F.7–6F.10 + 6F.6 (defensive half), **6.3** · **0.4.1 prepared** (all packages bumped, changelog + version strings, `npm pack` → ui-lineage-0.4.1.tgz verified: both bins, no @coderadar leak; publish to npm pending). 0.4.0 tagged/merged earlier. **Self-validated on Grafana frontend** (6,461 files, 15,334 nodes / 18,367 edges in 72 s): 55 RTK-query data sources, 32 routes, 1,009 coverage edges, gibberish declines — all previously 0/1 in the field run. - **Gates passed:** Gate 0 (CI + red-path, #5/#6) · Gate 1 (precision 1.000, recall 0.895, zero poison) · Gate 2 (C1 instance attribution 1.000 · B1 4-level handler chains · C6 store writers↔readers · A9 portals — scorecard 137/0/0, precision & recall 1.000) · Gate 3 (B3 action effects · B4 routers · B6 cyclic journeys terminate · B7/B8 form & non-JSX events · G5 flag/role conditions — precision & recall 1.000) · Gate 4 (A4 rarity · A10 fuzzy/OCR · A1 structural · A6 subtree · E3 vision annotations · E2 aliases · G4 corrections — high-conf correct 1.000, ambiguity honesty 1.000, poison rate 0.000) · Gate 5 (F1 context bundle · F2 blast radius · F3 test coverage · F4 response schema · F5 git history · MCP server over stdio — scorecard 265/0/0, all honesty metrics 1.000; **M5 reached** — ticket in → budgeted context bundle out, over MCP) ## What CodeRadar is @@ -323,10 +323,29 @@ The heart of the project. C1 and B1 live here. **Build:** `eval/bench/` generator (2,000+ file synthetic app with realistic import depth); profile; apply: lazy ts-morph project loading, file-batch parallelism (worker threads), tree-sitter fast path for the text-extraction pass if ts-morph remains the bottleneck. Perf budget asserted in nightly CI. **Accept:** full scan < 5 min, peak RSS < 4 GB on the bench repo. -### [ ] 6.3 Determinism +### [x] 6.3 Determinism **Failure modes:** G8 **Build:** stable ordering everywhere (nodes, edges, candidates — explicit sort keys, no map-iteration order leaks); vision/OCR results cached by image hash; `generatedAt` excluded from equality. Determinism check in the eval runner (two runs, byte-diff). **Accept:** double-run byte-identical on all fixtures + bench repo. +**Done:** `scanReact` now iterates source files in a stable path order (new +`sortedSourceFiles` — ts-morph returns glob-enumeration order, which varies by +platform) across all three file passes, and returns nodes sorted by `id` + edges +sorted by a total-order key over every identifying field (`edgeSortKey`). +`resolveHookEdges` re-sorts and dedups after rewriting `unresolved-hook:` +placeholders (a rewrite can collide two edges onto one hook id). Candidate +ranking gains a final `id` tiebreak so same-named components in different files +rank deterministically. The eval runner scans every fixture **twice** and +byte-compares (dropping only `generatedAt`): new `determinismStablePct` metric, +printed each run and a **hard gate** — any non-deterministic fixture fails the +run outright. 4 new parser-react unit tests (two-run byte-identical · nodes in id +order · edges in key order · no duplicate edges). Full suite green; eval +304/0/0/0, determinism 1.000, all metrics 1.000. One fragile unit assertion in +`queryfn.test.ts` (relied on node insertion order to find the react-query +`/api/stats` source ahead of the raw fetch it wraps) rewritten to assert the +react-query resolution *exists*, order-independently. Bench-repo half of the +accept criterion lands with 6.2. Vision/OCR image-hash caching deferred — the +eval path exercises no live OCR, so it isn't a determinism risk today; revisit if +6.4/vision work reintroduces it. ### [ ] 6.4 Version skew & rename tracking **Failure modes:** G3, A11 diff --git a/eval/src/golden.ts b/eval/src/golden.ts index 80b14f1..cc502ef 100644 --- a/eval/src/golden.ts +++ b/eval/src/golden.ts @@ -265,7 +265,11 @@ export interface Scorecard { entryPointAccuracy: number | null; /** Fraction of out-of-domain tickets correctly declined (step 5.1). */ oodRejection: number | null; + /** Fixtures whose two independent scans were byte-identical (6.3, G8). */ + determinismStablePct: number | null; }; + /** Fixtures whose second scan differed from the first (6.3) — empty when clean. */ + determinismFailures?: string[]; } export interface Thresholds { diff --git a/eval/src/run.ts b/eval/src/run.ts index 748043d..1cdc657 100644 --- a/eval/src/run.ts +++ b/eval/src/run.ts @@ -74,6 +74,8 @@ function main(): void { .sort(); const results: FixtureResult[] = []; + const determinismFailures: string[] = []; + let determinismChecked = 0; for (const name of fixtureNames) { const fixtureDir = path.join(fixturesDir, name); const goldenPath = path.join(fixtureDir, "golden.json"); @@ -83,11 +85,22 @@ function main(): void { } const golden = JSON.parse(fs.readFileSync(goldenPath, "utf-8")) as Golden; const appDir = path.resolve(fixtureDir, golden.app ?? "./app"); - const graph = resolveHookEdges(scanReact({ root: appDir, ...(golden.scan ?? {}) })); + const scanOpts = { root: appDir, ...(golden.scan ?? {}) }; + const graph = resolveHookEdges(scanReact(scanOpts)); results.push(runChecks(name, golden, graph, loadAliases(fixtureDir))); + + // Determinism (6.3, G8): a second independent scan of the same tree must + // serialize byte-for-byte identically, ignoring only the wall-clock + // `generatedAt`. Catches map-iteration/glob-order leaks before they ship. + const second = resolveHookEdges(scanReact(scanOpts)); + determinismChecked += 1; + if (canonicalGraph(graph) !== canonicalGraph(second)) determinismFailures.push(name); } - const scorecard = buildScorecard(results); + const scorecard = buildScorecard(results, { + failures: determinismFailures, + checked: determinismChecked, + }); fs.writeFileSync(path.join(evalDir, "scorecard.json"), JSON.stringify(scorecard, null, 2)); if (record) { fs.appendFileSync( @@ -140,7 +153,16 @@ function calibrate(results: FixtureResult[]): void { ); } -function buildScorecard(results: FixtureResult[]): Scorecard { +/** Serialize a graph for byte-comparison, dropping only the volatile timestamp (6.3). */ +function canonicalGraph(graph: { generatedAt?: string }): string { + const { generatedAt: _drop, ...rest } = graph; + return JSON.stringify(rest); +} + +function buildScorecard( + results: FixtureResult[], + determinism: { failures: string[]; checked: number }, +): Scorecard { let pass = 0; let fail = 0; let xfail = 0; @@ -196,8 +218,13 @@ function buildScorecard(results: FixtureResult[]): Scorecard { okAnswers.length > 0 ? round(okAnswers.filter((o) => !o.correct).length / okAnswers.length) : null, + determinismStablePct: + determinism.checked > 0 + ? round((determinism.checked - determinism.failures.length) / determinism.checked) + : null, ...runTickets(), }, + ...(determinism.failures.length > 0 ? { determinismFailures: determinism.failures } : {}), }; } @@ -223,6 +250,10 @@ function print(scorecard: Scorecard): void { console.log( `ticket entry-point accuracy=${fmt(s.entryPointAccuracy)} · OOD rejection=${fmt(s.oodRejection)}`, ); + console.log(`determinism (two-run byte-identical)=${fmt(s.determinismStablePct)}`); + if (scorecard.determinismFailures !== undefined && scorecard.determinismFailures.length > 0) { + console.log(` non-deterministic fixtures: ${scorecard.determinismFailures.join(", ")}`); + } } function gate(scorecard: Scorecard): void { @@ -254,6 +285,12 @@ function gate(scorecard: Scorecard): void { ) { violations.push(`poisonRate ${s.poisonRate} > max ${thresholds.maxPoisonRate}`); } + // Determinism is a hard gate (6.3, G8): any fixture that scans differently + // twice fails the run outright — a non-deterministic graph breaks the + // scan/query contract regardless of how the metrics score. + if (scorecard.determinismFailures !== undefined && scorecard.determinismFailures.length > 0) { + violations.push(`non-deterministic scans: ${scorecard.determinismFailures.join(", ")}`); + } if (violations.length > 0) { console.error(`\nEVAL GATE FAILED:\n ${violations.join("\n ")}`); diff --git a/packages/core/src/query.ts b/packages/core/src/query.ts index 294682f..c77eaeb 100644 --- a/packages/core/src/query.ts +++ b/packages/core/src/query.ts @@ -360,7 +360,10 @@ export function matchComponents( (a, b) => b.score - a.score || subtreeSize(a) - subtreeSize(b) || - a.match.component.name.localeCompare(b.match.component.name), + a.match.component.name.localeCompare(b.match.component.name) || + // Final tiebreak on the unique id so same-named components in different + // files rank deterministically (6.3, G8) rather than by scan order. + a.match.component.id.localeCompare(b.match.component.id), ); const collapsed = new Set(); for (const s of scored) { diff --git a/packages/parser-react/src/determinism.test.ts b/packages/parser-react/src/determinism.test.ts new file mode 100644 index 0000000..c6c18bb --- /dev/null +++ b/packages/parser-react/src/determinism.test.ts @@ -0,0 +1,47 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import type { LineageEdge } from "@coderadar/core"; +import { describe, expect, it } from "vitest"; + +import { resolveHookEdges, scanReact } from "./scan.js"; + +const demoApp = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../../examples/demo-app/src", +); + +/** Serialize a graph for byte-comparison, dropping only the volatile timestamp. */ +function canonical(graph: { generatedAt?: string }): string { + const { generatedAt: _drop, ...rest } = graph; + return JSON.stringify(rest); +} + +const edgeKey = (e: LineageEdge): string => + [e.kind, e.from, e.to, e.via ?? "", e.condition?.expression ?? ""].join(" "); + +describe("scan determinism (6.3, G8)", () => { + it("two independent scans of the same tree are byte-identical", () => { + const a = resolveHookEdges(scanReact({ root: demoApp })); + const b = resolveHookEdges(scanReact({ root: demoApp })); + expect(canonical(a)).toBe(canonical(b)); + }); + + it("emits nodes in canonical id order", () => { + const graph = resolveHookEdges(scanReact({ root: demoApp })); + const ids = graph.nodes.map((n) => n.id); + expect(ids).toStrictEqual([...ids].sort()); + }); + + it("emits edges in canonical key order", () => { + const graph = resolveHookEdges(scanReact({ root: demoApp })); + const keys = graph.edges.map(edgeKey); + expect(keys).toStrictEqual([...keys].sort()); + }); + + it("has no duplicate edges after hook resolution", () => { + const graph = resolveHookEdges(scanReact({ root: demoApp })); + const keys = graph.edges.map(edgeKey); + expect(new Set(keys).size).toBe(keys.length); + }); +}); diff --git a/packages/parser-react/src/queryfn.test.ts b/packages/parser-react/src/queryfn.test.ts index 377d010..124a661 100644 --- a/packages/parser-react/src/queryfn.test.ts +++ b/packages/parser-react/src/queryfn.test.ts @@ -32,8 +32,12 @@ describe("queryFn following (c5 fixture)", () => { }); it("resolves inline arrow queryFns", () => { - const stats = sources.find((s) => s.endpoint === "/api/stats"); - expect(stats?.sourceKind).toBe("react-query"); + // The inline arrow queryFn (`() => fetch("/api/stats")`) yields both a + // react-query source at the hook call site and the raw fetch it wraps, so + // assert the react-query resolution exists rather than depending on which + // node the canonical ordering (6.3) surfaces first. + const stats = sources.filter((s) => s.endpoint === "/api/stats"); + expect(stats.some((s) => s.sourceKind === "react-query")).toBe(true); }); it("never reports a query key as the endpoint", () => { diff --git a/packages/parser-react/src/scan.ts b/packages/parser-react/src/scan.ts index be5e6d5..8c2a714 100644 --- a/packages/parser-react/src/scan.ts +++ b/packages/parser-react/src/scan.ts @@ -188,7 +188,7 @@ export function scanReact(options: ScanOptions): LineageGraph { /** Event node id → the handler expressions it wires, mined for effects (3.2). */ const handlerExprs = new Map(); - for (const sourceFile of project.getSourceFiles()) { + for (const sourceFile of sortedSourceFiles(project)) { const file = toPosix(path.relative(root, sourceFile.getFilePath())); // Test files are swept separately (5.4) — they exercise components, they // don't define the app's UI, so they must not produce component/hook nodes. @@ -272,15 +272,45 @@ export function scanReact(options: ScanOptions): LineageGraph { root, generatedAt: new Date().toISOString(), generator: "ui-lineage@0.4.1", - nodes: [...nodes.values()], - edges, + // Canonical output order (6.3, G8): sort nodes and edges by stable keys so + // the serialized graph is byte-identical across runs and machines. Node ids + // are unique; edges are keyed by every identifying field. The query side + // addresses nodes by id, so array order carries no semantics — only + // reproducibility. + nodes: [...nodes.values()].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)), + edges: edges.sort((a, b) => { + const ka = edgeSortKey(a); + const kb = edgeSortKey(b); + return ka < kb ? -1 : ka > kb ? 1 : 0; + }), }; } +/** Total-order key over an edge's identifying fields (6.3, G8). */ +function edgeSortKey(e: LineageEdge): string { + return [e.kind, e.from, e.to, e.via ?? "", e.condition?.kind ?? "", e.condition?.expression ?? ""].join( + "", + ); +} + function toPosix(p: string): string { return p.split(path.sep).join("/"); } +/** + * Source files in a stable path order (6.3, G8). ts-morph returns files in glob + * enumeration order, which varies across platforms and filesystems; every pass + * that builds nodes/edges must iterate deterministically so two scans of the + * same tree agree byte-for-byte. + */ +function sortedSourceFiles(project: Project): SourceFile[] { + return [...project.getSourceFiles()].sort((a, b) => { + const pa = a.getFilePath(); + const pb = b.getFilePath(); + return pa < pb ? -1 : pa > pb ? 1 : 0; + }); +} + /** Spread helper: emit a `responseType` field only when one was recovered (5.5). */ function responseTypeProp(rt: ResponseType | null): { responseType?: ResponseType } { return rt !== null ? { responseType: rt } : {}; @@ -1093,7 +1123,7 @@ function detectStores( }; // Pass 1: slices, zustand stores, thunks. - for (const sourceFile of project.getSourceFiles()) { + for (const sourceFile of sortedSourceFiles(project)) { const file = toPosix(path.relative(root, sourceFile.getFilePath())); for (const variable of sourceFile.getVariableDeclarations()) { const init = variable.getInitializer(); @@ -1167,7 +1197,7 @@ function detectStores( } // Pass 2: thunk → slice associations via extraReducers addCase(thunk.fulfilled). - for (const sourceFile of project.getSourceFiles()) { + for (const sourceFile of sortedSourceFiles(project)) { for (const access of sourceFile.getDescendantsOfKind(SyntaxKind.PropertyAccessExpression)) { if (!["fulfilled", "rejected", "pending"].includes(access.getName())) continue; const thunkName = access.getExpression().getText(); @@ -2268,14 +2298,30 @@ export function resolveHookEdges(graph: LineageGraph): LineageGraph { if (node.kind === "hook") hooksByName.set(node.name, node.id); } const edges: LineageEdge[] = []; + const seen = new Set(); + const push = (edge: LineageEdge): void => { + // Rewriting placeholders can collide two edges onto the same hook id; + // dedup by full key so the result stays canonical (6.3, G8). + const key = edgeSortKey(edge); + if (seen.has(key)) return; + seen.add(key); + edges.push(edge); + }; for (const edge of graph.edges) { if (edge.to.startsWith("unresolved-hook:")) { const target = hooksByName.get(edge.to.slice("unresolved-hook:".length)); - if (target !== undefined) edges.push({ ...edge, to: target }); + if (target !== undefined) push({ ...edge, to: target }); // Unknown hooks (from libraries we don't model) are dropped. } else { - edges.push(edge); + push(edge); } } + // Rewritten `to` values may no longer be in the canonical order scanReact + // produced, so re-sort here — this is the graph the CLI and eval serialize. + edges.sort((a, b) => { + const ka = edgeSortKey(a); + const kb = edgeSortKey(b); + return ka < kb ? -1 : ka > kb ? 1 : 0; + }); return { ...graph, edges }; }