diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml new file mode 100644 index 0000000..c47823a --- /dev/null +++ b/.github/workflows/perf.yml @@ -0,0 +1,36 @@ +# CodeRadar scale benchmark (TRACKER step 6.2, failure mode D3). +# +# Generates a ~2,000-file synthetic React app, scans it, and asserts the perf +# budget: full scan under 5 minutes and peak RSS under 4 GB. Runs nightly so a +# scaling regression fails loudly without slowing the per-PR CI. Also runnable +# on demand via the Actions "Run workflow" button. + +name: Perf + +on: + schedule: + - cron: "0 6 * * *" # 06:00 UTC nightly + workflow_dispatch: + +jobs: + bench: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 # reads the packageManager field + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + + - name: Install + run: pnpm install --frozen-lockfile + + - name: Build + run: pnpm build + + - name: Scale benchmark (budget-gated) + run: node eval/dist/bench.js --files 2000 --budget-seconds 300 --budget-rss-mb 4096 diff --git a/.gitignore b/.gitignore index 5e5130a..d8d4104 100644 --- a/.gitignore +++ b/.gitignore @@ -150,3 +150,6 @@ eval/scorecard.json # ui-lineage SHA-keyed graph store (6.4) .coderadar/ + +# Scale benchmark synthetic app (6.2, generated/regenerable) +eval/bench/ diff --git a/TRACKER.md b/TRACKER.md index ca6277e..9bca7ef 100644 --- a/TRACKER.md +++ b/TRACKER.md @@ -5,8 +5,8 @@ ## Status - **Current phase:** 6 — Lifecycle, scale, hardening (6F field-hardening done bar the data-blocked 6F.6 detection half) -- **Next step:** 6.1 incremental re-scan, then 6.2 scale (bench repo) — the last two before Gate 6 is fully stamped. 6.3/6.4/6.5 landed (determinism gate, version-skew/rename tracking, generated-classification/PII). Publish 0.4.1 to the tester still pending (needs npm creds). -- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.6, 4.1–4.6, 5.1–5.7, 6F.1–6F.5, 6F.7–6F.10 + 6F.6 (defensive half), **6.3, 6.4, 6.5** · **0.4.1 prepared** (all packages bumped, changelog + version strings, `npm pack` → ui-lineage-0.4.1.tgz verified: both bins, no @coderadar leak; publish to npm pending). 0.4.0 tagged/merged earlier. **Self-validated on Grafana frontend** (6,461 files, 15,334 nodes / 18,367 edges in 72 s): 55 RTK-query data sources, 32 routes, 1,009 coverage edges, gibberish declines — all previously 0/1 in the field run. +- **Next step:** 6.1 incremental re-scan — the last Phase 6 step before Gate 6 is fully stamped; its property test uses the 6.2 bench repo. 6.2/6.3/6.4/6.5 landed (scale bench + budget, determinism gate, version-skew/rename tracking, generated-classification/PII). Publish 0.4.1 to the tester still pending (needs npm creds). +- **Done:** 0.1–0.4, 1.1–1.6, 2.1–2.5, 3.1–3.6, 4.1–4.6, 5.1–5.7, 6F.1–6F.5, 6F.7–6F.10 + 6F.6 (defensive half), **6.2, 6.3, 6.4, 6.5** · **0.4.1 prepared** (all packages bumped, changelog + version strings, `npm pack` → ui-lineage-0.4.1.tgz verified: both bins, no @coderadar leak; publish to npm pending). 0.4.0 tagged/merged earlier. **Self-validated on Grafana frontend** (6,461 files, 15,334 nodes / 18,367 edges in 72 s): 55 RTK-query data sources, 32 routes, 1,009 coverage edges, gibberish declines — all previously 0/1 in the field run. - **Gates passed:** Gate 0 (CI + red-path, #5/#6) · Gate 1 (precision 1.000, recall 0.895, zero poison) · Gate 2 (C1 instance attribution 1.000 · B1 4-level handler chains · C6 store writers↔readers · A9 portals — scorecard 137/0/0, precision & recall 1.000) · Gate 3 (B3 action effects · B4 routers · B6 cyclic journeys terminate · B7/B8 form & non-JSX events · G5 flag/role conditions — precision & recall 1.000) · Gate 4 (A4 rarity · A10 fuzzy/OCR · A1 structural · A6 subtree · E3 vision annotations · E2 aliases · G4 corrections — high-conf correct 1.000, ambiguity honesty 1.000, poison rate 0.000) · Gate 5 (F1 context bundle · F2 blast radius · F3 test coverage · F4 response schema · F5 git history · MCP server over stdio — scorecard 265/0/0, all honesty metrics 1.000; **M5 reached** — ticket in → budgeted context bundle out, over MCP) ## What CodeRadar is @@ -318,10 +318,28 @@ The heart of the project. C1 and B1 live here. **Build:** per-file content hashes in `GraphMeta`; `scan --update` re-parses only changed files + dependents (import graph), rebuilds affected cross-file passes (instances/prop-flow are the tricky part — dependents include all parents of changed components). `--watch` mode for dev. **Accept:** correctness: incremental result deep-equals full re-scan on 20 randomized single-file edits of the bench repo (property test); 10-file change < 15 s. -### [ ] 6.2 Scale & performance +### [x] 6.2 Scale & performance **Failure modes:** D3 **Build:** `eval/bench/` generator (2,000+ file synthetic app with realistic import depth); profile; apply: lazy ts-morph project loading, file-batch parallelism (worker threads), tree-sitter fast path for the text-extraction pass if ts-morph remains the bottleneck. Perf budget asserted in nightly CI. **Accept:** full scan < 5 min, peak RSS < 4 GB on the bench repo. +**Done:** new `eval/src/bench.ts` deterministically generates a synthetic app +(default ~2,016 files: N features × page → 8 sections → 8 atoms + a hook, with +real import depth and a direct-`fetch` per section so the endpoint pass is +exercised — 1,904 components / 896 data sources / 2,688 instances / 8,176 edges), +scans it, and gates a perf budget (`--files` / `--budget-seconds` / +`--budget-rss-mb`), measuring wall time and peak RSS via +`process.resourceUsage().maxRSS`. **Measured: ~2,016 files in ~2 s, ~400 MB RSS** +— vastly under the 300 s / 4 GB budget (the synthetic app has no tsconfig, so no +type-resolution cost; the field Grafana run was 6,461 files in 72 s / 1.5 GB). +Because ts-morph is nowhere near the bottleneck at target scale, the optional +worker-thread / tree-sitter fast paths were **deliberately not applied** — they'd +add complexity for no measured win; revisit if a future budget run regresses. +Root `pnpm bench` script; nightly `.github/workflows/perf.yml` (schedule + +manual dispatch) runs the budget-gated bench without slowing per-PR CI; the +generated tree (`eval/bench/`) is gitignored and self-cleaned after each run. +eval 317/0/0/0, determinism 1.000; typecheck + lint clean. **Gate 6 fully +passes** once 6.1 lands (incremental re-scan uses this bench for its property +test). ### [x] 6.3 Determinism **Failure modes:** G8 diff --git a/eval/src/bench.ts b/eval/src/bench.ts new file mode 100644 index 0000000..7ba17f6 --- /dev/null +++ b/eval/src/bench.ts @@ -0,0 +1,249 @@ +/** + * CodeRadar scale benchmark (TRACKER step 6.2, failure mode D3). + * + * Generates a large, deterministic synthetic React app with realistic import + * depth (pages -> sections -> atoms, plus shared hooks and api modules), scans + * it, and asserts a performance budget: full scan under a wall-clock limit and + * under a peak-RSS limit. Run in nightly CI so a scaling regression fails loudly. + * + * Usage: node eval/dist/bench.js [--files N] [--budget-seconds S] + * [--budget-rss-mb M] [--keep] [--regenerate] + * + * The generated tree lives at eval/bench/app (gitignored, regenerable). It is + * reused across runs unless --regenerate is passed or the file count differs. + */ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { resolveHookEdges, scanReact } from "@coderadar/parser-react"; + +const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "bench"); +const appDir = path.join(benchDir, "app"); + +/** Small deterministic PRNG (mulberry32) so a given seed always yields the same app. */ +function mulberry32(seed: number): () => number { + let a = seed; + return () => { + a |= 0; + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +const WORDS = [ + "revenue", "invoice", "billing", "user", "team", "project", "report", "dashboard", + "settings", "notification", "profile", "account", "payment", "subscription", "usage", + "metric", "alert", "audit", "session", "workspace", "member", "role", "permission", + "export", "import", "schedule", "webhook", "integration", "token", "log", +]; + +interface BenchShape { + /** Feature modules; each contributes a page + components + atoms + a hook. */ + features: number; + componentsPerFeature: number; + atomsPerFeature: number; +} + +/** Total files a shape produces: per feature = page + components + atoms + hook. */ +function fileCount(shape: BenchShape): number { + return shape.features * (1 + shape.componentsPerFeature + shape.atomsPerFeature + 1); +} + +/** Choose a shape whose file count is at least `minFiles` (>= 2000 for the D3 budget). */ +function shapeForFiles(minFiles: number): BenchShape { + const componentsPerFeature = 8; + const atomsPerFeature = 8; + const perFeature = 1 + componentsPerFeature + atomsPerFeature + 1; // 18 + const features = Math.max(1, Math.ceil(minFiles / perFeature)); + return { features, componentsPerFeature, atomsPerFeature }; +} + +const cap = (s: string): string => s.charAt(0).toUpperCase() + s.slice(1); + +/** Generate the synthetic app on disk. Deterministic for a given shape. */ +function generateBenchApp(shape: BenchShape): number { + fs.rmSync(appDir, { recursive: true, force: true }); + const rand = mulberry32(0x5eed); + const word = () => WORDS[Math.floor(rand() * WORDS.length)] ?? "widget"; + + let written = 0; + const write = (rel: string, contents: string): void => { + const full = path.join(appDir, rel); + fs.mkdirSync(path.dirname(full), { recursive: true }); + fs.writeFileSync(full, contents); + written += 1; + }; + + for (let f = 0; f < shape.features; f += 1) { + const feat = `${word()}${f}`; + const Feat = cap(feat); + + // A hook the components read. + write( + `features/${feat}/use${Feat}.ts`, + `import { useState } from "react";\n\n` + + `export function use${Feat}() {\n` + + ` const [ready, setReady] = useState(false);\n` + + ` return { ready, setReady };\n` + + `}\n`, + ); + + // Atoms: leaf components with distinctive rendered text. + for (let a = 0; a < shape.atomsPerFeature; a += 1) { + const Atom = `${Feat}Atom${a}`; + write( + `features/${feat}/atoms/${Atom}.tsx`, + `export function ${Atom}({ label }: { label: string }) {\n` + + ` return {label} ${word()} ${a};\n` + + `}\n`, + ); + } + + // Components: import a couple of atoms, the hook, and the api; fetch on mount. + for (let c = 0; c < shape.componentsPerFeature; c += 1) { + const Comp = `${Feat}Section${c}`; + const atomA = c % shape.atomsPerFeature; + const atomB = (c + 1) % shape.atomsPerFeature; + write( + `features/${feat}/components/${Comp}.tsx`, + `import { useEffect, useState } from "react";\n\n` + + `import { ${Feat}Atom${atomA} } from "../atoms/${Feat}Atom${atomA}";\n` + + `import { ${Feat}Atom${atomB} } from "../atoms/${Feat}Atom${atomB}";\n` + + `import { use${Feat} } from "../use${Feat}";\n\n` + + `export function ${Comp}() {\n` + + ` const { ready } = use${Feat}();\n` + + ` const [rows, setRows] = useState([]);\n` + + ` useEffect(() => {\n` + + ` fetch("/api/${feat}/${c}")\n` + + ` .then((r) => r.json())\n` + + ` .then(setRows);\n` + + ` }, []);\n` + + ` return (\n` + + `
\n` + + `

${Feat} ${word()} ${c}

\n` + + ` <${Feat}Atom${atomA} label="${word()}" />\n` + + ` <${Feat}Atom${atomB} label="${word()}" />\n` + + `

{ready ? rows.length : 0} ${word()}

\n` + + `
\n` + + ` );\n` + + `}\n`, + ); + } + + // Page: imports every section — the top of the import depth for the feature. + const imports = []; + const renders = []; + for (let c = 0; c < shape.componentsPerFeature; c += 1) { + const Comp = `${Feat}Section${c}`; + imports.push(`import { ${Comp} } from "./components/${Comp}";`); + renders.push(` <${Comp} />`); + } + write( + `features/${feat}/${Feat}Page.tsx`, + imports.join("\n") + + `\n\nexport function ${Feat}Page() {\n` + + ` return (\n` + + `
\n` + + `

${Feat} ${word()} overview

\n` + + renders.join("\n") + + `\n
\n` + + ` );\n` + + `}\n`, + ); + } + return written; +} + +interface Args { + minFiles: number; + budgetSeconds: number; + budgetRssMb: number; + keep: boolean; + regenerate: boolean; +} + +function parseArgs(argv: string[]): Args { + const get = (flag: string, fallback: number): number => { + const i = argv.indexOf(flag); + if (i === -1 || argv[i + 1] === undefined) return fallback; + const n = Number(argv[i + 1]); + return Number.isFinite(n) ? n : fallback; + }; + return { + minFiles: get("--files", 2000), + budgetSeconds: get("--budget-seconds", 300), + budgetRssMb: get("--budget-rss-mb", 4096), + keep: argv.includes("--keep"), + regenerate: argv.includes("--regenerate"), + }; +} + +/** Peak resident set size of this process, in MB. Node reports maxRSS in KB. */ +function peakRssMb(): number { + return process.resourceUsage().maxRSS / 1024; +} + +function countFiles(dir: string): number { + let n = 0; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) n += countFiles(path.join(dir, entry.name)); + else if (/\.(tsx?|jsx?)$/.test(entry.name)) n += 1; + } + return n; +} + +function main(): void { + const args = parseArgs(process.argv.slice(2)); + const shape = shapeForFiles(args.minFiles); + const expected = fileCount(shape); + + const needsGen = + args.regenerate || + !fs.existsSync(appDir) || + countFiles(appDir) !== expected; + if (needsGen) { + process.stdout.write(`Generating bench app (~${expected} files)... `); + const written = generateBenchApp(shape); + process.stdout.write(`${written} files.\n`); + } else { + console.log(`Reusing bench app at ${appDir} (${expected} files).`); + } + + const files = countFiles(appDir); + const start = performance.now(); + const graph = resolveHookEdges(scanReact({ root: appDir })); + const seconds = (performance.now() - start) / 1000; + const rssMb = peakRssMb(); + + console.log(`\nScanned ${files} files in ${seconds.toFixed(1)}s, peak RSS ${rssMb.toFixed(0)} MB.`); + console.log(` nodes: ${graph.nodes.length} edges: ${graph.edges.length}`); + console.log( + ` budget: <= ${args.budgetSeconds}s wall, <= ${args.budgetRssMb} MB RSS (files >= ${args.minFiles})`, + ); + + if (!args.keep && !args.regenerate) { + // Leave the tree in place for reuse only when explicitly kept; default is to + // remove it so a workspace/CI checkout is not left dirty (it is gitignored + // regardless). + fs.rmSync(benchDir, { recursive: true, force: true }); + } + + const violations: string[] = []; + if (files < args.minFiles) violations.push(`only ${files} files (< ${args.minFiles})`); + if (seconds > args.budgetSeconds) { + violations.push(`scan ${seconds.toFixed(1)}s > ${args.budgetSeconds}s`); + } + if (rssMb > args.budgetRssMb) violations.push(`peak RSS ${rssMb.toFixed(0)}MB > ${args.budgetRssMb}MB`); + + if (violations.length > 0) { + console.error(`\nPERF BUDGET FAILED:\n ${violations.join("\n ")}`); + process.exitCode = 1; + } else { + console.log("\nperf budget: OK"); + } +} + +main(); diff --git a/package.json b/package.json index c1ee8ed..bc4eef2 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "lint": "pnpm -r lint", "typecheck": "pnpm -r typecheck", "eval": "pnpm -r build && node eval/dist/run.js", + "bench": "pnpm -r build && node eval/dist/bench.js", "schemas": "pnpm --filter @coderadar/core schema" } }