From 49f7cd6295e96ba1763b0bf7824bcd8cd39289d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 28 Jul 2026 15:28:30 +0000 Subject: [PATCH 1/4] obs: add repo-health snapshot command aggregating existing analyzers into one JSON Adds `pnpm repo-health [--json]`, an offline command that aggregates the signals this repo already computes into one deterministic JSON snapshot by reusing each analyzer (depgraph, layering ratchets, coverage json-summary, size-report, fallow baselines, slow-test ratchet, bench/skillgym registries) rather than reimplementing any metric. Carries mandatory v1 provenance (schemaVersion, commit, per-analyzer content hashes, input provenance) so #1424 can persist history. Component metrics (instability/abstractness/main-sequence distance) are observatory-only. The only gating behaviour is the depgraph-vs-layering R6 consistency assertion, already wired into the Layering Guard job via scripts/depgraph/model.test.ts. Closes #1423 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- package.json | 1 + scripts/layering/check.ts | 4 +- scripts/lib/run-as-main.ts | 29 +++ scripts/repo-health/README.md | 126 +++++++++++ scripts/repo-health/components.ts | 132 ++++++++++++ scripts/repo-health/model.test.ts | 186 ++++++++++++++++ scripts/repo-health/model.ts | 308 +++++++++++++++++++++++++++ scripts/repo-health/run.test.ts | 70 ++++++ scripts/repo-health/run.ts | 236 ++++++++++++++++++++ scripts/vitest-slow-test-budgets.ts | 21 ++ scripts/vitest-slow-test-reporter.ts | 18 +- vitest.config.ts | 7 + 12 files changed, 1125 insertions(+), 13 deletions(-) create mode 100644 scripts/lib/run-as-main.ts create mode 100644 scripts/repo-health/README.md create mode 100644 scripts/repo-health/components.ts create mode 100644 scripts/repo-health/model.test.ts create mode 100644 scripts/repo-health/model.ts create mode 100644 scripts/repo-health/run.test.ts create mode 100644 scripts/repo-health/run.ts create mode 100644 scripts/vitest-slow-test-budgets.ts diff --git a/package.json b/package.json index 0da425b7d..63020f05d 100644 --- a/package.json +++ b/package.json @@ -128,6 +128,7 @@ "check:layering": "node --experimental-strip-types --test scripts/layering/model.test.ts && node --experimental-strip-types scripts/layering/check.ts", "depgraph": "node --experimental-strip-types scripts/depgraph/build.ts", "depgraph:test": "node --experimental-strip-types --test scripts/depgraph/model.test.ts", + "repo-health": "node --experimental-strip-types scripts/repo-health/run.ts", "check:production-exports": "fallow dead-code --config fallow-production-exports.json --production --unused-exports --fail-on-issues", "check:bundle-owner-files": "node --experimental-strip-types scripts/check-bundle-owner-files.ts", "check:command-docs": "vitest run --project unit-core src/__tests__/command-doc-coverage.test.ts", diff --git a/scripts/layering/check.ts b/scripts/layering/check.ts index 1d6733d9f..b30b49ca5 100644 --- a/scripts/layering/check.ts +++ b/scripts/layering/check.ts @@ -315,7 +315,9 @@ function checkTypeInversions(edges: readonly ResolvedImportEdge[]): Violation[] // moves here bring it to 102. Measured on the rebased tree — an earlier 87 was taken against an // older main and was stale rather than violated, which is exactly the kind of drift a ratchet // measured at merge time prevents. -const TYPE_CYCLE_BASELINE = 102; +// Exported so the repo-health snapshot (scripts/repo-health) can record the R9 ratchet value as +// observatory data without re-deriving it. The gate remains the authority; the snapshot follows. +export const TYPE_CYCLE_BASELINE = 102; function checkTypeCycleGrowth(actual: number): Violation[] { if (actual <= TYPE_CYCLE_BASELINE) return []; diff --git a/scripts/lib/run-as-main.ts b/scripts/lib/run-as-main.ts new file mode 100644 index 000000000..a5b4dd5e9 --- /dev/null +++ b/scripts/lib/run-as-main.ts @@ -0,0 +1,29 @@ +import { pathToFileURL } from 'node:url'; + +/** + * Run `main` as the process entrypoint, but only when `moduleUrl` (pass `import.meta.url`) is the + * script node was actually invoked with — so importing the module for its exports never runs it. + * Exits with `main`'s status and turns a throw into a `label`-prefixed non-zero exit. Keeps the + * `import.meta.url === pathToFileURL(process.argv[1])` guard from being copy-pasted per script. + */ +function isEntrypoint(moduleUrl: string): boolean { + return moduleUrl === pathToFileURL(process.argv[1] ?? '').href; +} + +function messageOf(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export function runAsMain( + moduleUrl: string, + label: string, + main: (argv: string[]) => number, +): void { + if (!isEntrypoint(moduleUrl)) return; + try { + process.exit(main(process.argv.slice(2))); + } catch (error: unknown) { + process.stderr.write(`${label}: ${messageOf(error)}\n`); + process.exit(1); + } +} diff --git a/scripts/repo-health/README.md b/scripts/repo-health/README.md new file mode 100644 index 000000000..a577df2fa --- /dev/null +++ b/scripts/repo-health/README.md @@ -0,0 +1,126 @@ +# Repo-health snapshot + +```sh +pnpm repo-health # human-readable summary +pnpm repo-health --json # the raw JSON snapshot on stdout +pnpm repo-health --out p # also write the JSON snapshot to a file +``` + +One JSON snapshot aggregating the signals this repo already computes across 24+ CI checks, so an +agent can query the repo's state in one read instead of scraping job logs. It is **data, not a +dashboard** (#1409's lesson): there is no rendering, and it draws no conclusions for you. + +It reuses each analyzer rather than reimplementing any metric, runs from a clean checkout in well +under two minutes, and needs no network — everything comes from the working tree, `git`, and the +artifacts other gates already write. + +## What it does NOT do + +- **No history / trends / PR comments.** That is the follow-up quality-delta issue (#1424); this + snapshot is the input it persists. +- **No gating on component metrics.** Instability / abstractness / main-sequence distance are + observatory data to locate concrete high-fan-in modules worth pinning harder (CONTEXT.md + "Principles and their gates"). They must **never** become CI thresholds. +- **The one thing that can fail the command** is the depgraph-vs-layering R6 consistency + assertion — see below. + +## The R6 consistency assertion (the only wired-into-CI part) + +The dependency graph the snapshot builds must reproduce the layering gate's +`TYPE_INVERSION_BASELINE` (`scripts/layering/check.ts`), pair for pair (#1410). `pnpm repo-health` +exits non-zero and prints the difference when they diverge. The **identical** assertion is already +wired into CI by the **Layering Guard** job, which runs `scripts/depgraph/model.test.ts` — so no +new CI job or cost is added here. The gate stays the authority: a mismatch means the tree or the +baseline changed, never the snapshot. + +## Schema (`schemaVersion: 1`) + +Field names are the stable **trend/delta contract** consumed by #1424. Any change bumps +`schemaVersion`; #1424 refuses to diff across schema versions without a migration note. + +```jsonc +{ + "schemaVersion": 1, + "provenance": { + "schemaVersion": 1, + "commit": "", + "ref": "", + "node": "v22.x", + "generatedAt": "", + // Content hash per analyzer, standing in for a version: a source change is a version bump. + "tool": { "depgraph": "…", "layering": "…", "fallow": "…", "slowTest": "…", + "bench": "…", "skillgym": "…" }, + "inputs": { + "sourceFiles": 932, // production files fed to the graph build + "lockfile": { "path": "pnpm-lock.yaml", "sha256": "…" }, + "coverageSummary": "coverage/coverage-summary.json" | null, // null when the gate hasn't run + "sizeReport": ".tmp/size-report.json" | null + } + }, + "metrics": { + "depgraph": { // from scripts/depgraph (reuses the layering edge model) + "files": 932, "edges": 4791, + "valueCycles": 0, "typeCycles": 7, "dynamicCycles": 1, + "redundantEdges": 1366, // value edges also reachable at distance >= 2 (NOT removable) + "backEdges": 0, // R5 ranked-spine value back-edges (gate keeps this at 0) + "typeInversionEdges": 7 // R6 type-only spine inversions, summed + }, + "layering": { // ratchet values from scripts/layering/check.ts + "typeInversionBaseline": { "commands -> client": 3, "…": 0 }, + "typeInversionTotal": 7, // R6 ratchet (may only shrink) + "typeCycleBaseline": 102 // R9 largest-type-cycle ceiling (growth-only) + }, + "coverage": { // from coverage/coverage-summary.json (json-summary reporter) + "available": true, + "lines": { "total": …, "covered": …, "pct": … }, + "statements": …, "functions": …, "branches": … + }, // { "available": false } when the artifact is absent + "size": { // from .tmp/size-report.json (scripts/size-report.mjs) + "available": true, + "jsRawBytes": …, "jsGzipBytes": …, "npmTarballBytes": …, "npmUnpackedBytes": … + }, // { "available": false } when the artifact is absent + "fallow": { // from fallow-baselines/*.json + .fallowrc.json + "deadCodeFindings": 0, "healthFindings": 161, + "suppressions": { "ignoredExports": …, "ignoredDependencies": …, "total": 19 } + }, + "slowTest": { // ratchet state from scripts/vitest-slow-test-reporter.ts + "unitBudgetMs": 2500, "integrationBudgetMs": 15000, "enforceFactor": 2 + }, + "bench": { "cases": 19, "topics": 10 }, // scripts/help-conformance-cases.mjs registry + "skillgym": { "cases": 5 }, // test/skillgym/suites/agent-device-smoke-suite.ts + "components": { // OBSERVATORY ONLY — never a CI threshold + "byZone": [ + { "zone": "kernel", "rank": 0, "classification": "ranked", + "files": …, "loc": …, + "afferent": 764, "efferent": 0, // cross-zone couplings (Ca / Ce) + "instability": 0.0, // I = Ce / (Ca + Ce) + "abstractness": 0.36, // A ≈ type-only share of afferent edges (approx.) + "distance": 0.64 } // |A + I - 1| + ] + }, + "mainSequence": { // OBSERVATORY ONLY + "concreteHighFanIn": [ // concrete (A < 0.5), fan-in >= 10, most-depended-on first + { "file": "utils/exec.ts", "zone": "utils", "fanIn": 81, "fanOut": 3, + "instability": 0.04, "abstractness": 0.17, "distance": 0.79, "concrete": true } + ] + } + }, + "consistency": { "r6": { "ok": true, "expected": { … }, "actual": { … } } } +} +``` + +### Coverage and size availability + +`coverage` and `size` are read from the artifacts the coverage and size gates already produce +(`coverage/coverage-summary.json` via the vitest `json-summary` reporter; `.tmp/size-report.json` +via `pnpm size:markdown`). Recomputing either would blow the two-minute / no-network budget, so a +clean checkout that has run neither gets `{ "available": false }` with the producing command +named in the summary. Run `pnpm test:coverage` and/or `pnpm size:markdown` first (or point #1424's +history job at a run that already did) to populate them. + +### Abstractness is a first approximation + +Per the issue, `abstractness` approximates a component's type-only export share by the share of +its **incoming edges that are type-only**. A module most of whose dependents import it for values +(e.g. `src/utils/exec.ts`, `src/daemon/ref-frame.ts`) reads as concrete; when its fan-in is also +high, it is a foundation worth pinning harder. This is a locator, not a verdict. diff --git a/scripts/repo-health/components.ts b/scripts/repo-health/components.ts new file mode 100644 index 000000000..e90e084ef --- /dev/null +++ b/scripts/repo-health/components.ts @@ -0,0 +1,132 @@ +// Component metrics for the repo-health snapshot (#1423): instability, abstractness, and +// main-sequence distance derived from the dependency graph's fan-in/fan-out. +// +// These are OBSERVATORY DATA ONLY. They locate concrete, high-fan-in modules worth pinning harder; +// they must NEVER become CI thresholds (CONTEXT.md "Principles and their gates"). Kept in their own +// module so that constraint — and the approximations these metrics make — stays visible instead of +// blending into the analyzer adapters that feed the gate-adjacent numbers. + +import { zoneRank } from '../layering/model.ts'; +import type { GraphData } from '../depgraph/model.ts'; + +export type ZoneComponent = { + zone: string; + rank: number | null; + classification: string; + files: number; + loc: number; + /** Afferent couplings: cross-zone edges INTO the zone (how much depends on it). */ + afferent: number; + /** Efferent couplings: cross-zone edges OUT of the zone (how much it depends on). */ + efferent: number; + /** Instability I = efferent / (afferent + efferent); 0 when the zone has no cross-zone edges. */ + instability: number; + /** + * Abstractness A, approximated by the type-only share of afferent edges (first approximation + * per the issue). 0 when nothing depends on the zone. + */ + abstractness: number; + /** Distance from the main sequence, |A + I - 1|. Lower is "more balanced". */ + distance: number; +}; + +function round(value: number): number { + return Math.round(value * 1000) / 1000; +} + +function zoneMapOf(graph: GraphData): Map { + return new Map(graph.nodes.map((node) => [node.id, node.zone])); +} + +/** + * Per-zone instability / abstractness / main-sequence distance from the graph's cross-zone edges. + * Observatory data — never a CI threshold. + */ +export function zoneComponents(graph: GraphData): ZoneComponent[] { + const zoneOf = zoneMapOf(graph); + const afferent = new Map(); + const efferent = new Map(); + const typeAfferent = new Map(); + const bump = (map: Map, key: string): void => + map.set(key, (map.get(key) ?? 0) + 1); + + for (const edge of graph.edges) { + const from = zoneOf.get(edge.from); + const to = zoneOf.get(edge.to); + if (from === undefined || to === undefined || from === to) continue; + bump(efferent, from); + bump(afferent, to); + if (edge.kind === 'type') bump(typeAfferent, to); + } + + return graph.zones + .map((zone) => { + const ca = afferent.get(zone.id) ?? 0; + const ce = efferent.get(zone.id) ?? 0; + const instability = ca + ce === 0 ? 0 : ce / (ca + ce); + const abstractness = ca === 0 ? 0 : (typeAfferent.get(zone.id) ?? 0) / ca; + return { + zone: zone.id, + rank: zoneRank(zone.id), + classification: zone.classification, + files: zone.files, + loc: zone.loc, + afferent: ca, + efferent: ce, + instability: round(instability), + abstractness: round(abstractness), + distance: round(Math.abs(abstractness + instability - 1)), + }; + }) + .sort((left, right) => right.afferent - left.afferent); +} + +export type MainSequenceModule = { + file: string; + zone: string; + fanIn: number; + fanOut: number; + instability: number; + abstractness: number; + distance: number; + concrete: boolean; +}; + +/** + * The main-sequence report: concrete, high-fan-in modules worth pinning harder. Per-file + * abstractness is approximated by the module's type-only dependent share; a module most of whose + * dependents import it for VALUES (e.g. src/utils/exec.ts, src/daemon/ref-frame.ts) is concrete + * and, when its fan-in is high, a foundation the tree leans on. Observatory data only. + */ +export function mainSequenceModules( + graph: GraphData, + options: { minFanIn?: number; limit?: number } = {}, +): MainSequenceModule[] { + const minFanIn = options.minFanIn ?? 10; + const limit = options.limit ?? 40; + const typeIn = new Map(); + for (const edge of graph.edges) { + if (edge.kind === 'type') typeIn.set(edge.to, (typeIn.get(edge.to) ?? 0) + 1); + } + + return graph.nodes + .filter((node) => node.fanIn >= minFanIn) + .map((node) => { + const abstractness = node.fanIn === 0 ? 0 : (typeIn.get(node.id) ?? 0) / node.fanIn; + const total = node.fanIn + node.fanOut; + const instability = total === 0 ? 0 : node.fanOut / total; + return { + file: node.id.replace(/^src\//, ''), + zone: node.zone, + fanIn: node.fanIn, + fanOut: node.fanOut, + instability: round(instability), + abstractness: round(abstractness), + distance: round(Math.abs(abstractness + instability - 1)), + concrete: abstractness < 0.5, + }; + }) + .filter((module) => module.concrete) + .sort((left, right) => right.fanIn - left.fanIn) + .slice(0, limit); +} diff --git a/scripts/repo-health/model.test.ts b/scripts/repo-health/model.test.ts new file mode 100644 index 000000000..f482897a2 --- /dev/null +++ b/scripts/repo-health/model.test.ts @@ -0,0 +1,186 @@ +import { readFileSync } from 'node:fs'; +import { describe, expect, test } from 'vitest'; +import { listSourceFiles, TYPE_INVERSION_BASELINE } from '../layering/check.ts'; +import { resolveImportEdges } from '../layering/model.ts'; +import { buildGraph, type GraphData } from '../depgraph/model.ts'; +import { mainSequenceModules, zoneComponents } from './components.ts'; +import { + benchMetrics, + buildSnapshot, + coverageMetrics, + depgraphMetrics, + fallowMetrics, + layeringRatchets, + r6Consistency, + ratchetTotal, + sizeMetrics, + SNAPSHOT_SCHEMA_VERSION, + type Provenance, +} from './model.ts'; + +/** Build the report's graph over the real production tree, the way the command does. */ +function realGraph(): GraphData { + const files = listSourceFiles(); + const sources = new Map(files.map((file) => [file, readFileSync(file, 'utf8')])); + return buildGraph(sources, resolveImportEdges(sources)); +} + +/** + * A tiny hand-made graph: two zones depend on kernel for VALUES and one for TYPES, so kernel is + * concrete (abstractness 1/3) with fan-in 3 — a miniature of the real exec.ts/ref-frame.ts shape. + */ +function fixtureGraph(): GraphData { + const sources = new Map([ + ['src/kernel/errors.ts', 'export const fail = 1;\nexport type Err = string;\n'], + ['src/core/run.ts', "import { fail } from '../kernel/errors.ts';\nexport const run = 1;\n"], + ['src/core/wrap.ts', "import { fail } from '../kernel/errors.ts';\nexport const wrap = 1;\n"], + [ + 'src/commands/tap.ts', + [ + "import { run } from '../core/run.ts';", + "import type { Err } from '../kernel/errors.ts';", + ].join('\n'), + ], + ]); + return buildGraph(sources, resolveImportEdges(sources)); +} + +test('depgraphMetrics reuses the graph rather than recomputing any number', () => { + const metrics = depgraphMetrics(fixtureGraph()); + expect(metrics.files).toBe(4); + expect(metrics.edges).toBe(4); + expect(metrics.valueCycles).toBe(0); + // commands -> kernel is a type-only import; the ranked spine outranks kernel so it is an R6 + // inversion, not a value/back edge. + expect(metrics.backEdges).toBe(0); +}); + +test('ratchetTotal sums a per-pair ratchet map', () => { + expect(ratchetTotal({ a: 3, b: 1, c: 2 })).toBe(6); +}); + +test('layeringRatchets reports the gate ratchets without re-deriving them', () => { + const ratchets = layeringRatchets(); + expect(ratchets.typeInversionBaseline).toBe(TYPE_INVERSION_BASELINE); + expect(ratchets.typeInversionTotal).toBe(ratchetTotal(TYPE_INVERSION_BASELINE)); + expect(ratchets.typeCycleBaseline).toBeGreaterThan(0); +}); + +describe('component metrics are derived from fan-in/fan-out', () => { + test('zoneComponents computes instability, abstractness and main-sequence distance', () => { + const byZone = zoneComponents(fixtureGraph()); + const kernel = byZone.find((zone) => zone.zone === 'kernel')!; + // Nothing in the fixture leaves kernel, and three files depend on it (one type-only). + expect(kernel.efferent).toBe(0); + expect(kernel.afferent).toBe(3); + expect(kernel.instability).toBe(0); + expect(kernel.abstractness).toBe(0.333); + }); + + test('mainSequenceModules surfaces concrete high-fan-in modules only', () => { + const modules = mainSequenceModules(fixtureGraph(), { minFanIn: 1 }); + const kernel = modules.find((module) => module.file === 'kernel/errors.ts')!; + expect(kernel.fanIn).toBe(3); + // Most dependents import it for values, so abstractness is 1/3 and it is concrete. + expect(kernel.abstractness).toBe(0.333); + expect(kernel.concrete).toBe(true); + }); +}); + +test('r6Consistency passes when the graph reproduces the baseline and fails on drift', () => { + const passing = r6Consistency(TYPE_INVERSION_BASELINE); + expect(passing.ok).toBe(true); + + const drifted = r6Consistency({ ...TYPE_INVERSION_BASELINE, 'commands -> client': 99 }); + expect(drifted.ok).toBe(false); + expect(drifted.expected).toEqual(passing.expected); +}); + +test('fallowMetrics counts findings from the baselines and suppressions from the config', () => { + const metrics = fallowMetrics( + { unused_exports: [{}, {}], circular_dependencies: [], unused_files: [{}] }, + { + finding_counts: { + 'src/a.ts': { complexity_critical: { count: 1 }, crap_moderate: { count: 2 } }, + 'src/b.ts': { crap_moderate: { count: 1 } }, + }, + }, + { + ignoreExports: [{ exports: ['x', 'y'] }, { exports: ['z'] }], + ignoreDependencies: ['@theme'], + }, + ); + expect(metrics.deadCodeFindings).toBe(3); + expect(metrics.healthFindings).toBe(4); + expect(metrics.suppressions).toEqual({ ignoredExports: 3, ignoredDependencies: 1, total: 4 }); +}); + +test('coverage and size degrade to unavailable when the artifact is absent', () => { + expect(coverageMetrics(null)).toEqual({ available: false }); + expect(sizeMetrics(null)).toEqual({ available: false }); + expect( + coverageMetrics({ total: { lines: { total: 10, covered: 8, pct: 80 } } as never }).available, + ).toBe(true); +}); + +test('benchMetrics counts cases and distinct topics from the case registry', () => { + expect(benchMetrics([{ docs: ['a', 'b'] }, { docs: ['b'] }, { docs: ['c'] }])).toEqual({ + cases: 3, + topics: 3, + }); +}); + +const PROVENANCE: Provenance = { + schemaVersion: SNAPSHOT_SCHEMA_VERSION, + commit: 'abc', + ref: 'main', + node: 'v22', + generatedAt: '2026-07-28T00:00:00.000Z', + tool: {}, + inputs: { sourceFiles: 3, lockfile: null, coverageSummary: null, sizeReport: null }, +}; + +test('buildSnapshot assembles every metric family and pins the schema version', () => { + const snapshot = buildSnapshot({ + provenance: PROVENANCE, + graph: fixtureGraph(), + fallow: { deadCode: {}, health: {}, config: {} }, + coverage: null, + size: null, + slowTest: { unitBudgetMs: 2500, integrationBudgetMs: 15000, enforceFactor: 2 }, + benchCases: [{ docs: ['a'] }], + skillgymCaseCount: 5, + }); + expect(snapshot.schemaVersion).toBe(SNAPSHOT_SCHEMA_VERSION); + expect(Object.keys(snapshot.metrics).sort()).toEqual([ + 'bench', + 'components', + 'coverage', + 'depgraph', + 'fallow', + 'layering', + 'mainSequence', + 'size', + 'skillgym', + 'slowTest', + ]); + expect(snapshot.metrics.skillgym.cases).toBe(5); +}); + +// The acceptance spot-check (issue #1423): the main-sequence report must locate the concrete, +// high-fan-in foundations. src/utils/exec.ts (the process-exec seam) and src/daemon/ref-frame.ts +// both have many value dependents, so both must appear as concrete. Guards the metric against a +// sign flip or an abstractness inversion that would make the report point at the wrong modules. +test('the real-tree main-sequence report surfaces exec.ts and ref-frame.ts as concrete', () => { + const graph = realGraph(); + const concrete = new Map(mainSequenceModules(graph).map((module) => [module.file, module])); + for (const file of ['utils/exec.ts', 'daemon/ref-frame.ts']) { + const module = concrete.get(file); + expect(module, `${file} should surface as a concrete high-fan-in module`).toBeDefined(); + expect(module!.concrete).toBe(true); + expect(module!.fanIn).toBeGreaterThanOrEqual(10); + } + + // And the graph the report builds must reproduce the gate's R6 baseline over the real tree. + expect(r6Consistency(graph.typeInversions).ok).toBe(true); +}); diff --git a/scripts/repo-health/model.ts b/scripts/repo-health/model.ts new file mode 100644 index 000000000..8e7a41e5a --- /dev/null +++ b/scripts/repo-health/model.ts @@ -0,0 +1,308 @@ +// Repo-health snapshot model — pure aggregation over the analyzers this repo already runs. +// +// The snapshot is DATA for agents to query (issue #1423, Track C of #1412), not a dashboard and +// not a new gate. Every metric here is REUSED from an existing analyzer rather than recomputed: +// the dependency graph comes from scripts/depgraph (which itself reuses the layering gate's edge +// model), the R6/R9 ratchet values come from scripts/layering/check.ts, fallow counts come from +// the checked-in baselines, coverage/size come from the artifacts their own gates already write, +// the slow-test ratchet comes from the vitest reporter, and the bench/skillgym counts come from +// their case registries. +// +// Field names are the trend/delta contract consumed by the follow-up quality-delta comment +// (#1424): they must stay stable, and any change bumps SNAPSHOT_SCHEMA_VERSION so a consumer +// refuses to diff across schema versions without a migration note. +// +// The component metrics (instability / abstractness / main-sequence distance) are OBSERVATORY +// DATA ONLY. They locate concrete, high-fan-in modules worth pinning harder; they must NEVER +// become CI thresholds (CONTEXT.md "Principles and their gates"). The single thing wired into CI +// is the depgraph-vs-layering R6 consistency assertion, which lives in the Layering Guard job. + +import { TYPE_CYCLE_BASELINE, TYPE_INVERSION_BASELINE } from '../layering/check.ts'; +import type { EdgeKind, GraphData } from '../depgraph/model.ts'; +import { + mainSequenceModules, + zoneComponents, + type MainSequenceModule, + type ZoneComponent, +} from './components.ts'; + +export const SNAPSHOT_SCHEMA_VERSION = 1; + +/** Total across a per-pair ratchet map, e.g. TYPE_INVERSION_BASELINE. */ +export function ratchetTotal(byPair: Readonly>): number { + return Object.values(byPair).reduce((sum, count) => sum + count, 0); +} + +export type DepgraphMetrics = { + files: number; + edges: number; + valueCycles: number; + typeCycles: number; + dynamicCycles: number; + /** Value edges whose target is ALSO reachable at distance >= 2. Reachability, not removability. */ + redundantEdges: number; + /** R5 ranked-spine back-edges over value edges (the gate keeps this at zero). */ + backEdges: number; + /** R6 type-only spine-inversion edges, summed across zone pairs. */ + typeInversionEdges: number; +}; + +export function depgraphMetrics(graph: GraphData): DepgraphMetrics { + const cyclesByKind = (kind: EdgeKind): number => + graph.cycles.filter((cycle) => cycle.kind === kind).length; + return { + files: graph.nodes.length, + edges: graph.edges.length, + valueCycles: cyclesByKind('value'), + typeCycles: cyclesByKind('type'), + dynamicCycles: cyclesByKind('dynamic'), + redundantEdges: graph.edges.filter((edge) => edge.transitivelyReachable).length, + backEdges: graph.edges.filter((edge) => edge.backEdge !== null).length, + typeInversionEdges: ratchetTotal(graph.typeInversions), + }; +} + +export type LayeringRatchets = { + /** R6 baseline, per zone pair, and its total. The counts may only shrink (gate authority). */ + typeInversionBaseline: Readonly>; + typeInversionTotal: number; + /** R9 largest-type-cycle ceiling (growth-only ratchet). */ + typeCycleBaseline: number; +}; + +export function layeringRatchets(): LayeringRatchets { + return { + typeInversionBaseline: TYPE_INVERSION_BASELINE, + typeInversionTotal: ratchetTotal(TYPE_INVERSION_BASELINE), + typeCycleBaseline: TYPE_CYCLE_BASELINE, + }; +} + +/** + * The depgraph-vs-layering R6 cross-check (#1410): the graph the report builds over the real tree + * must reproduce the gate's TYPE_INVERSION_BASELINE, pair for pair. `pnpm repo-health` fails when + * this is false, and the identical assertion is wired into the Layering Guard CI job via + * scripts/depgraph/model.test.ts. The gate stays the authority; a mismatch means the tree or the + * baseline changed, never this check. + */ +export type R6Consistency = { + ok: boolean; + expected: Readonly>; + actual: Readonly>; +}; + +function sortedEntries(byPair: Readonly>): [string, number][] { + return Object.entries(byPair).sort(([left], [right]) => left.localeCompare(right)); +} + +export function r6Consistency( + graphTypeInversions: Readonly>, +): R6Consistency { + const expected = Object.fromEntries(sortedEntries(TYPE_INVERSION_BASELINE)); + const actual = Object.fromEntries(sortedEntries(graphTypeInversions)); + return { + ok: JSON.stringify(actual) === JSON.stringify(expected), + expected, + actual, + }; +} + +// ---- Fallow ----------------------------------------------------------------------------------- + +/** The fallow dead-code baseline: arrays of findings keyed by category. */ +export type FallowDeadCodeBaseline = Record; +/** The fallow health baseline: per-file maps of finding kind -> { count }. */ +export type FallowHealthBaseline = { + finding_counts?: Record>; +}; +export type FallowConfig = { + ignoreExports?: { exports: string[] }[]; + ignoreDependencies?: string[]; +}; + +export type FallowMetrics = { + /** Total dead-code findings recorded in the dead-code baseline. */ + deadCodeFindings: number; + /** Total complexity/health findings recorded in the health baseline. */ + healthFindings: number; + suppressions: { + /** Distinct export entries suppressed via `ignoreExports` in .fallowrc.json. */ + ignoredExports: number; + ignoredDependencies: number; + total: number; + }; +}; + +export function fallowMetrics( + deadCode: FallowDeadCodeBaseline, + health: FallowHealthBaseline, + config: FallowConfig, +): FallowMetrics { + const deadCodeFindings = Object.values(deadCode).reduce( + (sum, list) => sum + (Array.isArray(list) ? list.length : 0), + 0, + ); + let healthFindings = 0; + for (const perKind of Object.values(health.finding_counts ?? {})) { + for (const finding of Object.values(perKind)) healthFindings += finding.count; + } + const ignoredExports = (config.ignoreExports ?? []).reduce( + (sum, entry) => sum + (entry.exports?.length ?? 0), + 0, + ); + const ignoredDependencies = (config.ignoreDependencies ?? []).length; + return { + deadCodeFindings, + healthFindings, + suppressions: { + ignoredExports, + ignoredDependencies, + total: ignoredExports + ignoredDependencies, + }, + }; +} + +// ---- Coverage & size (read from the artifacts their own gates already produce) ---------------- + +type CoverageTotalEntry = { total: number; covered: number; pct: number }; +export type CoverageSummary = { + total?: Record<'lines' | 'statements' | 'functions' | 'branches', CoverageTotalEntry>; +}; +export type CoverageMetrics = { + available: boolean; + lines?: CoverageTotalEntry; + statements?: CoverageTotalEntry; + functions?: CoverageTotalEntry; + branches?: CoverageTotalEntry; +}; + +export function coverageMetrics(summary: CoverageSummary | null): CoverageMetrics { + if (!summary?.total) return { available: false }; + const { lines, statements, functions, branches } = summary.total; + return { available: true, lines, statements, functions, branches }; +} + +export type SizeReport = { + js?: { rawBytes: number; gzipBytes: number }; + npmPack?: { tarballBytes: number; unpackedBytes: number }; +}; +export type SizeMetrics = { + available: boolean; + jsRawBytes?: number; + jsGzipBytes?: number; + npmTarballBytes?: number; + npmUnpackedBytes?: number; +}; + +export function sizeMetrics(report: SizeReport | null): SizeMetrics { + if (!report?.js || !report.npmPack) return { available: false }; + return { + available: true, + jsRawBytes: report.js.rawBytes, + jsGzipBytes: report.js.gzipBytes, + npmTarballBytes: report.npmPack.tarballBytes, + npmUnpackedBytes: report.npmPack.unpackedBytes, + }; +} + +// ---- Slow-test / bench / skillgym (registry-derived counts) ----------------------------------- + +export type SlowTestRatchet = { + unitBudgetMs: number; + integrationBudgetMs: number; + enforceFactor: number; +}; + +export type BenchMetrics = { + /** Help-conformance bench cases (scripts/help-conformance-cases.mjs). */ + cases: number; + /** Distinct help topics the cases exercise. */ + topics: number; +}; + +export function benchMetrics(cases: readonly { docs: readonly string[] }[]): BenchMetrics { + return { + cases: cases.length, + topics: new Set(cases.flatMap((testCase) => testCase.docs)).size, + }; +} + +// ---- Provenance ------------------------------------------------------------------------------- + +/** + * What produced the snapshot and from which inputs. Mandatory in v1 (issue #1423 amendment) so + * #1424 can persist history and refuse to diff across schema versions. `tool` holds a content + * hash per analyzer (its own source, standing in for a version); `inputs` records which + * lockfile/config each family of metrics was computed from. + */ +export type Provenance = { + schemaVersion: number; + commit: string; + ref: string | null; + node: string; + generatedAt: string; + tool: Record; + inputs: { + /** Production source files fed to the graph build. */ + sourceFiles: number; + lockfile: { path: string; sha256: string } | null; + /** Analyzer artifacts that were present and read (coverage/size degrade to null when absent). */ + coverageSummary: string | null; + sizeReport: string | null; + }; +}; + +// ---- Full snapshot ---------------------------------------------------------------------------- + +export type RepoHealthSnapshot = { + schemaVersion: number; + provenance: Provenance; + /** + * Component metrics are observatory data — they locate concrete high-fan-in modules and must + * NEVER become CI thresholds. Only `consistency.r6` gates anything. + */ + metrics: { + depgraph: DepgraphMetrics; + layering: LayeringRatchets; + coverage: CoverageMetrics; + size: SizeMetrics; + fallow: FallowMetrics; + slowTest: SlowTestRatchet; + bench: BenchMetrics; + skillgym: { cases: number }; + components: { byZone: ZoneComponent[] }; + mainSequence: { concreteHighFanIn: MainSequenceModule[] }; + }; + consistency: { r6: R6Consistency }; +}; + +export type SnapshotInputs = { + provenance: Provenance; + graph: GraphData; + fallow: { deadCode: FallowDeadCodeBaseline; health: FallowHealthBaseline; config: FallowConfig }; + coverage: CoverageSummary | null; + size: SizeReport | null; + slowTest: SlowTestRatchet; + benchCases: readonly { docs: readonly string[] }[]; + skillgymCaseCount: number; +}; + +export function buildSnapshot(inputs: SnapshotInputs): RepoHealthSnapshot { + return { + schemaVersion: SNAPSHOT_SCHEMA_VERSION, + provenance: inputs.provenance, + metrics: { + depgraph: depgraphMetrics(inputs.graph), + layering: layeringRatchets(), + coverage: coverageMetrics(inputs.coverage), + size: sizeMetrics(inputs.size), + fallow: fallowMetrics(inputs.fallow.deadCode, inputs.fallow.health, inputs.fallow.config), + slowTest: inputs.slowTest, + bench: benchMetrics(inputs.benchCases), + skillgym: { cases: inputs.skillgymCaseCount }, + components: { byZone: zoneComponents(inputs.graph) }, + mainSequence: { concreteHighFanIn: mainSequenceModules(inputs.graph) }, + }, + consistency: { r6: r6Consistency(inputs.graph.typeInversions) }, + }; +} diff --git a/scripts/repo-health/run.test.ts b/scripts/repo-health/run.test.ts new file mode 100644 index 000000000..bcaa4c3c6 --- /dev/null +++ b/scripts/repo-health/run.test.ts @@ -0,0 +1,70 @@ +// End-to-end coverage of the `pnpm repo-health` CLI. It spawns the command as a subprocess (so it +// exercises argument handling and the file it writes over the real tree), which is why this lives +// in the `subprocess-stub` vitest project rather than `unit-core` — see #1412 coordination rule 4. + +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { expect, test } from 'vitest'; +import type { RepoHealthSnapshot } from './model.ts'; + +function runRepoHealth(args: readonly string[]): { + status: number | null; + stdout: string; + stderr: string; +} { + const result = spawnSync( + process.execPath, + ['--experimental-strip-types', 'scripts/repo-health/run.ts', ...args], + { encoding: 'utf8', maxBuffer: 1024 * 1024 * 32 }, + ); + return { status: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' }; +} + +test('the default run prints a summary and exits zero over the real tree', () => { + const { status, stdout } = runRepoHealth([]); + expect(status, stdout).toBe(0); + expect(stdout).toContain('Repo health snapshot'); + expect(stdout).toContain('graph reproduces baseline: yes'); + expect(stdout).toContain('observatory only'); +}); + +test('--json emits a schema-versioned snapshot with every metric family', () => { + const { status, stdout } = runRepoHealth(['--json']); + expect(status, stdout).toBe(0); + + const snapshot = JSON.parse(stdout) as RepoHealthSnapshot; + expect(snapshot.schemaVersion).toBe(1); + + // Provenance is mandatory in v1 (issue amendment): schemaVersion, commit, per-analyzer tool + // hashes, and input provenance must all be present so #1424 can persist history. + expect(snapshot.provenance.commit).toMatch(/^[0-9a-f]{7,40}$/); + expect(Object.keys(snapshot.provenance.tool)).toContain('depgraph'); + expect(snapshot.provenance.inputs.sourceFiles).toBeGreaterThan(0); + + // Field names are the #1424 delta contract — pin the ones the comment consumes. + expect(snapshot.metrics.depgraph.redundantEdges).toBeGreaterThan(0); + expect(snapshot.metrics.layering.typeInversionTotal).toBeGreaterThanOrEqual(0); + expect(snapshot.metrics.slowTest.unitBudgetMs).toBeGreaterThan(0); + expect(snapshot.metrics.fallow.suppressions.total).toBeGreaterThanOrEqual(0); + expect(snapshot.metrics.bench.cases).toBeGreaterThan(0); + expect(snapshot.metrics.skillgym.cases).toBeGreaterThan(0); + + // The consistency assertion must pass on a clean tree. + expect(snapshot.consistency.r6.ok).toBe(true); + + // The main-sequence spot-check the acceptance names. + const concrete = snapshot.metrics.mainSequence.concreteHighFanIn.map((module) => module.file); + expect(concrete).toContain('utils/exec.ts'); + expect(concrete).toContain('daemon/ref-frame.ts'); +}); + +test('--out writes the same JSON snapshot to a file', () => { + const out = join(mkdtempSync(join(tmpdir(), 'repo-health-')), 'snapshot.json'); + const { status } = runRepoHealth(['--out', out]); + expect(status).toBe(0); + const written = JSON.parse(readFileSync(out, 'utf8')) as RepoHealthSnapshot; + expect(written.schemaVersion).toBe(1); + expect(written.consistency.r6.ok).toBe(true); +}); diff --git a/scripts/repo-health/run.ts b/scripts/repo-health/run.ts new file mode 100644 index 000000000..808614cf2 --- /dev/null +++ b/scripts/repo-health/run.ts @@ -0,0 +1,236 @@ +// Repo-health snapshot command (issue #1423, Track C of #1412): +// +// pnpm repo-health # human-readable summary +// pnpm repo-health --json # the raw JSON snapshot on stdout +// pnpm repo-health --out p # also write the JSON snapshot to a file +// +// One JSON snapshot aggregating the signals this repo already computes, by REUSING each analyzer +// rather than reimplementing any metric (see model.ts for which analyzer owns each number). It is +// DATA for agents to query, not a dashboard: no rendering, no history (that is #1424), and the +// component metrics are observatory-only and must never gate. +// +// It runs from a clean checkout in well under two minutes with no network: everything is read +// from the working tree, git, and the artifacts other gates already write. Coverage and package +// size are read from coverage/coverage-summary.json and .tmp/size-report.json when present and +// degrade to `available: false` (with the producing command) when they are not, so a fresh +// checkout still produces a complete, honest snapshot. +// +// The one thing that can FAIL the command is the depgraph-vs-layering R6 consistency assertion: +// the graph built here must reproduce scripts/layering/check.ts's TYPE_INVERSION_BASELINE. The +// identical assertion is wired into CI by the Layering Guard job (scripts/depgraph/model.test.ts). + +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { listSourceFiles } from '../layering/check.ts'; +import { resolveImportEdges } from '../layering/model.ts'; +import { buildGraph, type GraphData } from '../depgraph/model.ts'; +import { SLOW_TEST_RATCHET } from '../vitest-slow-test-budgets.ts'; +import { parseScriptArgs } from '../lib/cli-args.ts'; +import { runAsMain } from '../lib/run-as-main.ts'; +import { CASES } from '../help-conformance-cases.mjs'; +import skillgymSuite from '../../test/skillgym/suites/agent-device-smoke-suite.ts'; +import { + buildSnapshot, + SNAPSHOT_SCHEMA_VERSION, + type FallowConfig, + type FallowDeadCodeBaseline, + type FallowHealthBaseline, + type Provenance, + type RepoHealthSnapshot, +} from './model.ts'; + +const USAGE = 'Usage: pnpm repo-health [--json] [--out ]\n'; + +const COVERAGE_SUMMARY_PATH = 'coverage/coverage-summary.json'; +const SIZE_REPORT_PATH = '.tmp/size-report.json'; +const LOCKFILE_PATH = 'pnpm-lock.yaml'; + +const repoRoot = execFileSync('git', ['rev-parse', '--show-toplevel'], { + encoding: 'utf8', +}).trim(); + +function absolute(relativePath: string): string { + return path.join(repoRoot, relativePath); +} + +function readIfExists(relativePath: string): string | null { + const abs = absolute(relativePath); + return fs.existsSync(abs) ? fs.readFileSync(abs, 'utf8') : null; +} + +function readJsonIfExists(relativePath: string): T | null { + const raw = readIfExists(relativePath); + return raw === null ? null : (JSON.parse(raw) as T); +} + +/** Short content hash standing in for an analyzer's version — a source change is a version bump. */ +function contentHash(relativePath: string): string { + const raw = readIfExists(relativePath); + return raw === null ? 'absent' : createHash('sha256').update(raw).digest('hex').slice(0, 12); +} + +function lockfileProvenance(): { path: string; sha256: string } | null { + const raw = readIfExists(LOCKFILE_PATH); + if (raw === null) return null; + return { + path: LOCKFILE_PATH, + sha256: createHash('sha256').update(raw).digest('hex').slice(0, 12), + }; +} + +function gitOutput(args: string[]): string | null { + try { + return execFileSync('git', args, { cwd: repoRoot, encoding: 'utf8' }).trim(); + } catch { + return null; + } +} + +function buildGraphFromTree(): GraphData { + const files = listSourceFiles(); + const sources = new Map(files.map((file) => [file, fs.readFileSync(absolute(file), 'utf8')])); + return buildGraph(sources, resolveImportEdges(sources)); +} + +function currentCommit(): string { + return gitOutput(['rev-parse', 'HEAD']) ?? 'unknown'; +} + +function currentRef(): string | null { + return process.env.GITHUB_REF_NAME ?? gitOutput(['rev-parse', '--abbrev-ref', 'HEAD']); +} + +// A content hash per analyzer, standing in for a version: a change to the analyzer's own source +// is a version bump the snapshot records without a real semver to point at. +function toolHashes(): Record { + return { + depgraph: contentHash('scripts/depgraph/model.ts'), + layering: contentHash('scripts/layering/check.ts'), + fallow: contentHash('.fallowrc.json'), + slowTest: contentHash('scripts/vitest-slow-test-budgets.ts'), + bench: contentHash('scripts/help-conformance-cases.mjs'), + skillgym: contentHash('test/skillgym/suites/agent-device-smoke-suite.ts'), + }; +} + +function collectProvenance(graph: GraphData, hasCoverage: boolean, hasSize: boolean): Provenance { + return { + schemaVersion: SNAPSHOT_SCHEMA_VERSION, + commit: currentCommit(), + ref: currentRef(), + node: process.version, + generatedAt: new Date().toISOString(), + tool: toolHashes(), + inputs: { + sourceFiles: graph.nodes.length, + lockfile: lockfileProvenance(), + coverageSummary: hasCoverage ? COVERAGE_SUMMARY_PATH : null, + sizeReport: hasSize ? SIZE_REPORT_PATH : null, + }, + }; +} + +function collectSnapshot(): RepoHealthSnapshot { + const graph = buildGraphFromTree(); + const coverage = readJsonIfExists(COVERAGE_SUMMARY_PATH); + const size = readJsonIfExists(SIZE_REPORT_PATH); + return buildSnapshot({ + provenance: collectProvenance(graph, coverage !== null, size !== null), + graph, + fallow: { + deadCode: readJsonIfExists('fallow-baselines/dead-code.json') ?? {}, + health: readJsonIfExists('fallow-baselines/health.json') ?? {}, + config: readJsonIfExists('.fallowrc.json') ?? {}, + }, + coverage, + size, + slowTest: SLOW_TEST_RATCHET, + benchCases: CASES, + skillgymCaseCount: skillgymSuite.length, + }); +} + +function pct(entry: { covered: number; total: number; pct: number } | undefined): string { + return entry + ? `${entry.pct.toFixed(2)}% (${entry.covered}/${entry.total})` + : 'n/a (run pnpm test:coverage)'; +} + +function bytes(value: number | undefined): string { + if (value === undefined) return 'n/a (run pnpm size:markdown)'; + if (value < 1000) return `${value} B`; + if (value < 1_000_000) return `${(value / 1000).toFixed(1)} kB`; + return `${(value / 1_000_000).toFixed(2)} MB`; +} + +function renderSummary(snapshot: RepoHealthSnapshot): string { + const { metrics, provenance, consistency } = snapshot; + const lines: string[] = [ + `Repo health snapshot (schemaVersion ${snapshot.schemaVersion}) @ ${provenance.commit.slice(0, 12)}`, + ` depgraph: ${metrics.depgraph.files} files, ${metrics.depgraph.edges} edges, ` + + `${metrics.depgraph.redundantEdges} redundant; cycles value=${metrics.depgraph.valueCycles} ` + + `type=${metrics.depgraph.typeCycles} dynamic=${metrics.depgraph.dynamicCycles}`, + ` layering: R6 type-inversions=${metrics.layering.typeInversionTotal} ` + + `(graph reproduces baseline: ${consistency.r6.ok ? 'yes' : 'NO'}); R9 type-cycle baseline=${metrics.layering.typeCycleBaseline}`, + ` coverage: lines ${pct(metrics.coverage.lines)}; statements ${pct(metrics.coverage.statements)}`, + ` size: js raw ${bytes(metrics.size.jsRawBytes)}, gzip ${bytes(metrics.size.jsGzipBytes)}; ` + + `npm tarball ${bytes(metrics.size.npmTarballBytes)}`, + ` fallow: dead-code findings=${metrics.fallow.deadCodeFindings}, health findings=${metrics.fallow.healthFindings}, ` + + `suppressions=${metrics.fallow.suppressions.total}`, + ` slow-test: unit ${metrics.slowTest.unitBudgetMs}ms / integration ${metrics.slowTest.integrationBudgetMs}ms (enforce ${metrics.slowTest.enforceFactor}x)`, + ` bench: ${metrics.bench.cases} cases across ${metrics.bench.topics} help topics; skillgym ${metrics.skillgym.cases} cases`, + '', + ' Component metrics (observatory only — never a CI threshold):', + ...metrics.components.byZone + .slice(0, 8) + .map( + (zone) => + ` ${zone.zone.padEnd(16)} Ca=${String(zone.afferent).padStart(4)} Ce=${String(zone.efferent).padStart(4)} ` + + `I=${zone.instability.toFixed(2)} A=${zone.abstractness.toFixed(2)} D=${zone.distance.toFixed(2)}`, + ), + ' Concrete high-fan-in modules worth pinning harder:', + ...metrics.mainSequence.concreteHighFanIn + .slice(0, 8) + .map( + (module) => + ` ${module.file.padEnd(40)} fanIn=${String(module.fanIn).padStart(3)} A=${module.abstractness.toFixed(2)}`, + ), + ]; + return `${lines.join('\n')}\n`; +} + +function run(argv: readonly string[]): number { + const values = parseScriptArgs(argv, USAGE, { + json: { type: 'boolean', default: false }, + out: { type: 'string' }, + }); + + const snapshot = collectSnapshot(); + const serialized = `${JSON.stringify(snapshot, null, 2)}\n`; + + if (typeof values.out === 'string') { + const outPath = path.resolve(values.out); + fs.mkdirSync(path.dirname(outPath), { recursive: true }); + fs.writeFileSync(outPath, serialized); + } + + process.stdout.write(values.json ? serialized : renderSummary(snapshot)); + + const { r6 } = snapshot.consistency; + if (!r6.ok) { + process.stderr.write( + 'repo-health: the dependency graph disagrees with the layering gate about R6 type-only ' + + 'spine inversions.\n' + + ` gate baseline (TYPE_INVERSION_BASELINE): ${JSON.stringify(r6.expected)}\n` + + ` graph over the current tree: ${JSON.stringify(r6.actual)}\n` + + 'The gate is the authority — fix the inverting edge, or update TYPE_INVERSION_BASELINE in ' + + 'scripts/layering/check.ts. See scripts/depgraph/README.md.\n', + ); + return 1; + } + return 0; +} + +runAsMain(import.meta.url, 'repo-health', run); diff --git a/scripts/vitest-slow-test-budgets.ts b/scripts/vitest-slow-test-budgets.ts new file mode 100644 index 000000000..e1d42efdb --- /dev/null +++ b/scripts/vitest-slow-test-budgets.ts @@ -0,0 +1,21 @@ +// The slow-test ratchet's budgets, as a single source of truth (see docs/agents/testing.md +// "Speed rules"). Kept in a data-only module so consumers that only need the numbers — the +// repo-health snapshot (scripts/repo-health), which records them as observatory data — can import +// them without pulling the vitest reporter (a config-loaded module) into their dependency graph. + +// Unit tests must not wait real time; integration scenarios drive a real daemon request path and +// get more room. +export const UNIT_BUDGET_MS = 2_500; +export const INTEGRATION_BUDGET_MS = 15_000; + +// Enforcement fires at 2x budget: host load legitimately stretches a borderline test by tens of +// percent, and a wall-clock gate that flakes under contention trains people to ignore it. Between +// budget and 2x budget the gate reports without failing. +export const ENFORCE_FACTOR = 2; + +/** The ratchet as one record, for callers that report it rather than enforce it. */ +export const SLOW_TEST_RATCHET = { + unitBudgetMs: UNIT_BUDGET_MS, + integrationBudgetMs: INTEGRATION_BUDGET_MS, + enforceFactor: ENFORCE_FACTOR, +} as const; diff --git a/scripts/vitest-slow-test-reporter.ts b/scripts/vitest-slow-test-reporter.ts index c55b0ad62..6963d812b 100644 --- a/scripts/vitest-slow-test-reporter.ts +++ b/scripts/vitest-slow-test-reporter.ts @@ -1,4 +1,9 @@ import type { Reporter, TestCase, TestModule } from 'vitest/node'; +import { + ENFORCE_FACTOR, + INTEGRATION_BUDGET_MS, + UNIT_BUDGET_MS, +} from './vitest-slow-test-budgets.ts'; /** * Slow-test ratchet (see docs/agents/testing.md "Speed rules"). @@ -7,19 +12,8 @@ import type { Reporter, TestCase, TestModule } from 'vitest/node'; * wall clock was bounded by files whose tests slept through production * timeouts (a 10.8s test proving "times out" by waiting the full 10s budget). * This reporter fails the run when a unit test exceeds the enforced budget. - * - * Budgets are per suite family: integration scenarios drive a real daemon - * request path and get more room; unit tests get 2.5s, which is already - * generous for injected-time tests. + * The budgets themselves live in ./vitest-slow-test-budgets.ts. */ -const UNIT_BUDGET_MS = 2_500; -const INTEGRATION_BUDGET_MS = 15_000; -// Enforcement fires at 2x budget: host load legitimately stretches a -// borderline test by tens of percent, and a wall-clock gate that flakes under -// contention trains people to ignore it. Between budget and 2x budget the -// gate reports without failing. -const ENFORCE_FACTOR = 2; - type Offender = { key: string; durationMs: number; budgetMs: number; enforce: boolean }; function budgetForPath(relativePath: string): number { diff --git a/vitest.config.ts b/vitest.config.ts index 7b6a7d118..762349033 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -22,6 +22,9 @@ export const SUBPROCESS_STUB_TESTS = [ // Replays the fuzz regression corpus (#1414) through that same worker watchdog, so a promoted // hang case fails against its per-case budget instead of wedging the unit job. 'scripts/fuzz/corpus-replay.test.ts', + // Spawns the `pnpm repo-health` CLI as a subprocess to exercise its arg handling and the JSON + // it writes over the real tree (#1423). + 'scripts/repo-health/run.test.ts', ]; export default defineConfig({ @@ -47,6 +50,10 @@ export default defineConfig({ 'scripts/__tests__/help-conformance-error-recovery-coverage.test.ts', 'scripts/__tests__/help-conformance-sample-outputs.test.ts', 'scripts/__tests__/help-conformance-topic-coverage.test.ts', + // Repo-health snapshot model (#1423): pure aggregation over the analyzers, no + // subprocess or device work, so it runs in the fast lane. The CLI's subprocess test + // lives in the subprocess-stub project. + 'scripts/repo-health/model.test.ts', // Parses CI configuration only, so this action guard needs no device or subprocess lane. 'test/ci/upload-agent-device-artifacts.test.ts', 'test/skillgym/suites/local-cli-help-policy.test.ts', From 6e351df662ae7dee9e4c3e98d71c73b02285981b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 28 Jul 2026 15:42:57 +0000 Subject: [PATCH 2/4] repo-health: hash read artifacts and flag staleness in provenance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Coverage and size are artifacts repo-health reads but does not produce, so they carry no producing-commit stamp. Previously provenance recorded only their paths and the current HEAD, so a snapshot taken after a source edit without rerunning those producers would pair prior metrics with the new SHA — #1424 would persist a false commit-indexed history entry. Now each read artifact is bound via artifactProvenance() to its content hash (so history keys on the bytes the metrics came from, not a commit they may predate) and an explicit `stale` flag (true when a production source file is newer than the artifact). The coverage/size analyzer-config hashes are added to provenance.tool, and the human summary marks stale artifacts. Adds a pure stale-artifact regression test. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- scripts/repo-health/README.md | 14 ++++-- scripts/repo-health/model.test.ts | 14 ++++++ scripts/repo-health/model.ts | 45 ++++++++++++++++-- scripts/repo-health/run.ts | 79 ++++++++++++++++++++++++------- 4 files changed, 126 insertions(+), 26 deletions(-) diff --git a/scripts/repo-health/README.md b/scripts/repo-health/README.md index a577df2fa..205bcc1f1 100644 --- a/scripts/repo-health/README.md +++ b/scripts/repo-health/README.md @@ -47,14 +47,18 @@ Field names are the stable **trend/delta contract** consumed by #1424. Any chang "ref": "", "node": "v22.x", "generatedAt": "", - // Content hash per analyzer, standing in for a version: a source change is a version bump. - "tool": { "depgraph": "…", "layering": "…", "fallow": "…", "slowTest": "…", - "bench": "…", "skillgym": "…" }, + // Content hash per analyzer/config, standing in for a version: a source change is a version bump. + "tool": { "depgraph": "…", "layering": "…", "fallow": "…", "coverage": "…", "size": "…", + "slowTest": "…", "bench": "…", "skillgym": "…" }, "inputs": { "sourceFiles": 932, // production files fed to the graph build "lockfile": { "path": "pnpm-lock.yaml", "sha256": "…" }, - "coverageSummary": "coverage/coverage-summary.json" | null, // null when the gate hasn't run - "sizeReport": ".tmp/size-report.json" | null + // coverage and size are read artifacts this command does NOT produce, so they carry no + // producing-commit stamp. They are hashed (so history keys on the bytes, not the commit) and + // flagged `stale` when a source file is newer than the artifact — the tree moved on without + // the producer being rerun. A consumer (#1424) must treat stale metrics as lagging `commit`. + "coverageSummary": { "path": "coverage/coverage-summary.json", "sha256": "…", "stale": false } | null, + "sizeReport": { "path": ".tmp/size-report.json", "sha256": "…", "stale": false } | null } }, "metrics": { diff --git a/scripts/repo-health/model.test.ts b/scripts/repo-health/model.test.ts index f482897a2..0bf87da33 100644 --- a/scripts/repo-health/model.test.ts +++ b/scripts/repo-health/model.test.ts @@ -5,6 +5,7 @@ import { resolveImportEdges } from '../layering/model.ts'; import { buildGraph, type GraphData } from '../depgraph/model.ts'; import { mainSequenceModules, zoneComponents } from './components.ts'; import { + artifactProvenance, benchMetrics, buildSnapshot, coverageMetrics, @@ -123,6 +124,19 @@ test('coverage and size degrade to unavailable when the artifact is absent', () ).toBe(true); }); +test('artifactProvenance binds bytes and flags an artifact older than the tree as stale', () => { + const args = { path: 'coverage/coverage-summary.json', sha256: 'deadbeef' }; + // Artifact newer than every source file: the metrics match the current tree. + expect(artifactProvenance({ ...args, artifactMtimeMs: 2000, newestSourceMtimeMs: 1000 })).toEqual( + { path: args.path, sha256: 'deadbeef', stale: false }, + ); + // Source edited after the artifact was produced: the metrics predate the tree and must be flagged + // so #1424 cannot persist them against the current commit as if they were fresh. + expect(artifactProvenance({ ...args, artifactMtimeMs: 1000, newestSourceMtimeMs: 2000 })).toEqual( + { path: args.path, sha256: 'deadbeef', stale: true }, + ); +}); + test('benchMetrics counts cases and distinct topics from the case registry', () => { expect(benchMetrics([{ docs: ['a', 'b'] }, { docs: ['b'] }, { docs: ['c'] }])).toEqual({ cases: 3, diff --git a/scripts/repo-health/model.ts b/scripts/repo-health/model.ts index 8e7a41e5a..d02d0c2e4 100644 --- a/scripts/repo-health/model.ts +++ b/scripts/repo-health/model.ts @@ -229,11 +229,46 @@ export function benchMetrics(cases: readonly { docs: readonly string[] }[]): Ben // ---- Provenance ------------------------------------------------------------------------------- +/** + * Provenance for an analyzer artifact this snapshot READ but did not itself produce (coverage, + * size). These artifacts carry no producing-commit stamp, so the snapshot cannot claim they match + * `commit`. We instead record their content hash — so #1424 keys history on the bytes the metrics + * came from, not on a commit they may predate — and an explicit `stale` flag: true when a + * production source file is newer than the artifact, i.e. the tree moved on without the producer + * being rerun. A consumer must treat stale metrics as lagging `commit`, never as current. + */ +export type ArtifactProvenance = { + path: string; + sha256: string; + stale: boolean; +}; + +/** + * Bind an artifact to the bytes and freshness it was read with. `stale` compares mtimes: an + * artifact older than the newest production source predates the current working tree. mtime is a + * local, offline signal (these artifacts are git-ignored, so there is no blob to diff), evaluated + * against a single tree in one run, which is exactly the "edited code but did not rerun the + * producer" case this guards against. + */ +export function artifactProvenance(params: { + path: string; + sha256: string; + artifactMtimeMs: number; + newestSourceMtimeMs: number; +}): ArtifactProvenance { + return { + path: params.path, + sha256: params.sha256, + stale: params.newestSourceMtimeMs > params.artifactMtimeMs, + }; +} + /** * What produced the snapshot and from which inputs. Mandatory in v1 (issue #1423 amendment) so * #1424 can persist history and refuse to diff across schema versions. `tool` holds a content - * hash per analyzer (its own source, standing in for a version); `inputs` records which - * lockfile/config each family of metrics was computed from. + * hash per analyzer (its own source/config, standing in for a version); `inputs` records the + * lockfile and the read-only artifacts each family of metrics was computed from, each hashed and + * flagged for staleness so a false commit-indexed history entry cannot slip through. */ export type Provenance = { schemaVersion: number; @@ -246,9 +281,9 @@ export type Provenance = { /** Production source files fed to the graph build. */ sourceFiles: number; lockfile: { path: string; sha256: string } | null; - /** Analyzer artifacts that were present and read (coverage/size degrade to null when absent). */ - coverageSummary: string | null; - sizeReport: string | null; + /** Read artifacts, hashed and stale-flagged; null when absent (coverage/size degrade cleanly). */ + coverageSummary: ArtifactProvenance | null; + sizeReport: ArtifactProvenance | null; }; }; diff --git a/scripts/repo-health/run.ts b/scripts/repo-health/run.ts index 808614cf2..9c1f180cb 100644 --- a/scripts/repo-health/run.ts +++ b/scripts/repo-health/run.ts @@ -13,7 +13,10 @@ // from the working tree, git, and the artifacts other gates already write. Coverage and package // size are read from coverage/coverage-summary.json and .tmp/size-report.json when present and // degrade to `available: false` (with the producing command) when they are not, so a fresh -// checkout still produces a complete, honest snapshot. +// checkout still produces a complete, honest snapshot. Those two artifacts carry no producing-commit +// stamp, so provenance hashes them and flags `stale` when a source file is newer (see +// artifactProvenance in model.ts) — a consumer like #1424 must never pair stale metrics with the +// current commit. // // The one thing that can FAIL the command is the depgraph-vs-layering R6 consistency assertion: // the graph built here must reproduce scripts/layering/check.ts's TYPE_INVERSION_BASELINE. The @@ -32,8 +35,10 @@ import { runAsMain } from '../lib/run-as-main.ts'; import { CASES } from '../help-conformance-cases.mjs'; import skillgymSuite from '../../test/skillgym/suites/agent-device-smoke-suite.ts'; import { + artifactProvenance, buildSnapshot, SNAPSHOT_SCHEMA_VERSION, + type ArtifactProvenance, type FallowConfig, type FallowDeadCodeBaseline, type FallowHealthBaseline, @@ -65,19 +70,44 @@ function readJsonIfExists(relativePath: string): T | null { return raw === null ? null : (JSON.parse(raw) as T); } +function shortSha(raw: string): string { + return createHash('sha256').update(raw).digest('hex').slice(0, 12); +} + /** Short content hash standing in for an analyzer's version — a source change is a version bump. */ function contentHash(relativePath: string): string { const raw = readIfExists(relativePath); - return raw === null ? 'absent' : createHash('sha256').update(raw).digest('hex').slice(0, 12); + return raw === null ? 'absent' : shortSha(raw); } function lockfileProvenance(): { path: string; sha256: string } | null { const raw = readIfExists(LOCKFILE_PATH); - if (raw === null) return null; - return { - path: LOCKFILE_PATH, - sha256: createHash('sha256').update(raw).digest('hex').slice(0, 12), - }; + return raw === null ? null : { path: LOCKFILE_PATH, sha256: shortSha(raw) }; +} + +/** The newest mtime across the production source tree, the freshness bar for read artifacts. */ +function newestSourceMtimeMs(files: readonly string[]): number { + let newest = 0; + for (const file of files) { + const mtimeMs = fs.statSync(absolute(file)).mtimeMs; + if (mtimeMs > newest) newest = mtimeMs; + } + return newest; +} + +/** Hash a read artifact and flag it stale when a source file is newer; null when absent. */ +function artifactInput( + relativePath: string, + newestSourceMtimeMs: number, +): ArtifactProvenance | null { + const abs = absolute(relativePath); + if (!fs.existsSync(abs)) return null; + return artifactProvenance({ + path: relativePath, + sha256: shortSha(fs.readFileSync(abs, 'utf8')), + artifactMtimeMs: fs.statSync(abs).mtimeMs, + newestSourceMtimeMs, + }); } function gitOutput(args: string[]): string | null { @@ -88,8 +118,7 @@ function gitOutput(args: string[]): string | null { } } -function buildGraphFromTree(): GraphData { - const files = listSourceFiles(); +function buildGraphFromTree(files: readonly string[]): GraphData { const sources = new Map(files.map((file) => [file, fs.readFileSync(absolute(file), 'utf8')])); return buildGraph(sources, resolveImportEdges(sources)); } @@ -109,13 +138,19 @@ function toolHashes(): Record { depgraph: contentHash('scripts/depgraph/model.ts'), layering: contentHash('scripts/layering/check.ts'), fallow: contentHash('.fallowrc.json'), + coverage: contentHash('vitest.config.ts'), + size: contentHash('scripts/size-report.mjs'), slowTest: contentHash('scripts/vitest-slow-test-budgets.ts'), bench: contentHash('scripts/help-conformance-cases.mjs'), skillgym: contentHash('test/skillgym/suites/agent-device-smoke-suite.ts'), }; } -function collectProvenance(graph: GraphData, hasCoverage: boolean, hasSize: boolean): Provenance { +function collectProvenance( + graph: GraphData, + coverage: ArtifactProvenance | null, + size: ArtifactProvenance | null, +): Provenance { return { schemaVersion: SNAPSHOT_SCHEMA_VERSION, commit: currentCommit(), @@ -126,18 +161,24 @@ function collectProvenance(graph: GraphData, hasCoverage: boolean, hasSize: bool inputs: { sourceFiles: graph.nodes.length, lockfile: lockfileProvenance(), - coverageSummary: hasCoverage ? COVERAGE_SUMMARY_PATH : null, - sizeReport: hasSize ? SIZE_REPORT_PATH : null, + coverageSummary: coverage, + sizeReport: size, }, }; } function collectSnapshot(): RepoHealthSnapshot { - const graph = buildGraphFromTree(); + const files = listSourceFiles(); + const graph = buildGraphFromTree(files); + const newestSource = newestSourceMtimeMs(files); const coverage = readJsonIfExists(COVERAGE_SUMMARY_PATH); const size = readJsonIfExists(SIZE_REPORT_PATH); return buildSnapshot({ - provenance: collectProvenance(graph, coverage !== null, size !== null), + provenance: collectProvenance( + graph, + artifactInput(COVERAGE_SUMMARY_PATH, newestSource), + artifactInput(SIZE_REPORT_PATH, newestSource), + ), graph, fallow: { deadCode: readJsonIfExists('fallow-baselines/dead-code.json') ?? {}, @@ -165,6 +206,10 @@ function bytes(value: number | undefined): string { return `${(value / 1_000_000).toFixed(2)} MB`; } +function staleMark(artifact: ArtifactProvenance | null): string { + return artifact?.stale ? ' [STALE — source newer than artifact; rerun the producer]' : ''; +} + function renderSummary(snapshot: RepoHealthSnapshot): string { const { metrics, provenance, consistency } = snapshot; const lines: string[] = [ @@ -174,9 +219,11 @@ function renderSummary(snapshot: RepoHealthSnapshot): string { `type=${metrics.depgraph.typeCycles} dynamic=${metrics.depgraph.dynamicCycles}`, ` layering: R6 type-inversions=${metrics.layering.typeInversionTotal} ` + `(graph reproduces baseline: ${consistency.r6.ok ? 'yes' : 'NO'}); R9 type-cycle baseline=${metrics.layering.typeCycleBaseline}`, - ` coverage: lines ${pct(metrics.coverage.lines)}; statements ${pct(metrics.coverage.statements)}`, + ` coverage: lines ${pct(metrics.coverage.lines)}; statements ${pct(metrics.coverage.statements)}` + + staleMark(provenance.inputs.coverageSummary), ` size: js raw ${bytes(metrics.size.jsRawBytes)}, gzip ${bytes(metrics.size.jsGzipBytes)}; ` + - `npm tarball ${bytes(metrics.size.npmTarballBytes)}`, + `npm tarball ${bytes(metrics.size.npmTarballBytes)}` + + staleMark(provenance.inputs.sizeReport), ` fallow: dead-code findings=${metrics.fallow.deadCodeFindings}, health findings=${metrics.fallow.healthFindings}, ` + `suppressions=${metrics.fallow.suppressions.total}`, ` slow-test: unit ${metrics.slowTest.unitBudgetMs}ms / integration ${metrics.slowTest.integrationBudgetMs}ms (enforce ${metrics.slowTest.enforceFactor}x)`, From be01065d80726e407404b29bccdad8aa7bb0733d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 28 Jul 2026 16:24:48 +0000 Subject: [PATCH 3/4] repo-health: bind artifact freshness to the whole producer input set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stale check derived freshness only from listSourceFiles() (src/*.ts) and applied it to both coverage and size. Coverage also depends on tests and vitest config; size-report.mjs reads package.json and runs npm pack — so a change to a non-src producer input could leave an old artifact marked stale:false while provenance recorded the current HEAD, a commit the metrics predate. Each read artifact now declares its full producer input set (PRODUCER_INPUTS) and is flagged stale when ANY tracked input in that set is newer than the artifact (via git ls-files mtimes). Freshness that cannot be proven — an empty observable input set — is reported stale, never falsely fresh. The input set is recorded as provenance.inputs.*.producerInputs. Adds isArtifactStale() with a regression covering a non-src input newer than the artifact and the unprovable case. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- scripts/repo-health/README.md | 12 ++++-- scripts/repo-health/model.test.ts | 38 ++++++++++++----- scripts/repo-health/model.ts | 34 ++++++++++----- scripts/repo-health/run.ts | 71 +++++++++++++++++++++++-------- 4 files changed, 113 insertions(+), 42 deletions(-) diff --git a/scripts/repo-health/README.md b/scripts/repo-health/README.md index 205bcc1f1..d409cb9a4 100644 --- a/scripts/repo-health/README.md +++ b/scripts/repo-health/README.md @@ -55,10 +55,14 @@ Field names are the stable **trend/delta contract** consumed by #1424. Any chang "lockfile": { "path": "pnpm-lock.yaml", "sha256": "…" }, // coverage and size are read artifacts this command does NOT produce, so they carry no // producing-commit stamp. They are hashed (so history keys on the bytes, not the commit) and - // flagged `stale` when a source file is newer than the artifact — the tree moved on without - // the producer being rerun. A consumer (#1424) must treat stale metrics as lagging `commit`. - "coverageSummary": { "path": "coverage/coverage-summary.json", "sha256": "…", "stale": false } | null, - "sizeReport": { "path": ".tmp/size-report.json", "sha256": "…", "stale": false } | null + // flagged `stale` when ANY of their `producerInputs` is newer than the artifact — coverage + // tracks sources + tests + vitest config + package.json; size tracks package.json + lockfile + + // tsdown/tsconfig + size-report.mjs, not just `src`. Unprovable freshness (no observable input) + // is reported stale. A consumer (#1424) must treat stale metrics as lagging `commit`. + "coverageSummary": { "path": "coverage/coverage-summary.json", "sha256": "…", + "producerInputs": ["src/**/*.ts", "test", "…"], "stale": false } | null, + "sizeReport": { "path": ".tmp/size-report.json", "sha256": "…", + "producerInputs": ["src/**/*.ts", "package.json", "…"], "stale": false } | null } }, "metrics": { diff --git a/scripts/repo-health/model.test.ts b/scripts/repo-health/model.test.ts index 0bf87da33..761536e6d 100644 --- a/scripts/repo-health/model.test.ts +++ b/scripts/repo-health/model.test.ts @@ -6,6 +6,7 @@ import { buildGraph, type GraphData } from '../depgraph/model.ts'; import { mainSequenceModules, zoneComponents } from './components.ts'; import { artifactProvenance, + isArtifactStale, benchMetrics, buildSnapshot, coverageMetrics, @@ -124,17 +125,32 @@ test('coverage and size degrade to unavailable when the artifact is absent', () ).toBe(true); }); -test('artifactProvenance binds bytes and flags an artifact older than the tree as stale', () => { - const args = { path: 'coverage/coverage-summary.json', sha256: 'deadbeef' }; - // Artifact newer than every source file: the metrics match the current tree. - expect(artifactProvenance({ ...args, artifactMtimeMs: 2000, newestSourceMtimeMs: 1000 })).toEqual( - { path: args.path, sha256: 'deadbeef', stale: false }, - ); - // Source edited after the artifact was produced: the metrics predate the tree and must be flagged - // so #1424 cannot persist them against the current commit as if they were fresh. - expect(artifactProvenance({ ...args, artifactMtimeMs: 1000, newestSourceMtimeMs: 2000 })).toEqual( - { path: args.path, sha256: 'deadbeef', stale: true }, - ); +test('isArtifactStale flags an artifact predating ANY producer input, incl. non-src ones', () => { + const artifact = 1500; + const srcMtimes = [1000, 1200]; // all older than the artifact + // Freshness judged on src alone would (wrongly) call this fresh — the earlier false-negative. + expect(isArtifactStale(artifact, srcMtimes)).toBe(false); + // A non-src producer input (e.g. package.json / vitest config / tsdown config) edited after the + // artifact was produced must flip the verdict, so a coverage/size number is not paired with a + // commit it predates. + const packageJsonMtime = 1800; + expect(isArtifactStale(artifact, [...srcMtimes, packageJsonMtime])).toBe(true); + // No observable producer input => freshness cannot be proven => stale, never a false "fresh". + expect(isArtifactStale(artifact, [])).toBe(true); +}); + +test('artifactProvenance binds bytes, the producer input set, and the staleness verdict', () => { + const args = { + path: 'coverage/coverage-summary.json', + sha256: 'deadbeef', + producerInputs: ['src/**/*.ts', 'package.json'], + }; + expect( + artifactProvenance({ ...args, artifactMtimeMs: 2000, producerInputMtimesMs: [1000, 1500] }), + ).toEqual({ ...args, stale: false }); + expect( + artifactProvenance({ ...args, artifactMtimeMs: 1000, producerInputMtimesMs: [1500] }), + ).toEqual({ ...args, stale: true }); }); test('benchMetrics counts cases and distinct topics from the case registry', () => { diff --git a/scripts/repo-health/model.ts b/scripts/repo-health/model.ts index d02d0c2e4..3ec3db182 100644 --- a/scripts/repo-health/model.ts +++ b/scripts/repo-health/model.ts @@ -233,33 +233,47 @@ export function benchMetrics(cases: readonly { docs: readonly string[] }[]): Ben * Provenance for an analyzer artifact this snapshot READ but did not itself produce (coverage, * size). These artifacts carry no producing-commit stamp, so the snapshot cannot claim they match * `commit`. We instead record their content hash — so #1424 keys history on the bytes the metrics - * came from, not on a commit they may predate — and an explicit `stale` flag: true when a - * production source file is newer than the artifact, i.e. the tree moved on without the producer - * being rerun. A consumer must treat stale metrics as lagging `commit`, never as current. + * came from, not on a commit they may predate — the `producerInputs` freshness was judged against, + * and an explicit `stale` flag. Freshness is bound to the artifact's WHOLE producer input set, not + * just `src` (coverage also depends on tests/config; size on package.json and packaging config), so + * a change to any of them re-flags the artifact. A consumer must treat stale metrics as lagging + * `commit`, never as current. */ export type ArtifactProvenance = { path: string; sha256: string; + /** The producer input pathspecs whose freshness this verdict was computed against. */ + producerInputs: string[]; stale: boolean; }; /** - * Bind an artifact to the bytes and freshness it was read with. `stale` compares mtimes: an - * artifact older than the newest production source predates the current working tree. mtime is a - * local, offline signal (these artifacts are git-ignored, so there is no blob to diff), evaluated - * against a single tree in one run, which is exactly the "edited code but did not rerun the - * producer" case this guards against. + * Whether an artifact predates any of its producer inputs. mtime is a local, offline signal (these + * artifacts are git-ignored, so there is no blob to diff), evaluated against a single tree in one + * run. Freshness we cannot prove is reported as stale: an EMPTY input set (nothing observable) and + * any input newer than the artifact both mean the metrics may lag the tree — we never claim fresh + * on the strength of `src` alone, which was the earlier false-negative. */ +export function isArtifactStale( + artifactMtimeMs: number, + producerInputMtimesMs: readonly number[], +): boolean { + return producerInputMtimesMs.length === 0 || Math.max(...producerInputMtimesMs) > artifactMtimeMs; +} + +/** Bind an artifact to the bytes and the full producer input set its freshness was judged against. */ export function artifactProvenance(params: { path: string; sha256: string; + producerInputs: string[]; artifactMtimeMs: number; - newestSourceMtimeMs: number; + producerInputMtimesMs: readonly number[]; }): ArtifactProvenance { return { path: params.path, sha256: params.sha256, - stale: params.newestSourceMtimeMs > params.artifactMtimeMs, + producerInputs: params.producerInputs, + stale: isArtifactStale(params.artifactMtimeMs, params.producerInputMtimesMs), }; } diff --git a/scripts/repo-health/run.ts b/scripts/repo-health/run.ts index 9c1f180cb..ac1f4c59d 100644 --- a/scripts/repo-health/run.ts +++ b/scripts/repo-health/run.ts @@ -14,9 +14,10 @@ // size are read from coverage/coverage-summary.json and .tmp/size-report.json when present and // degrade to `available: false` (with the producing command) when they are not, so a fresh // checkout still produces a complete, honest snapshot. Those two artifacts carry no producing-commit -// stamp, so provenance hashes them and flags `stale` when a source file is newer (see -// artifactProvenance in model.ts) — a consumer like #1424 must never pair stale metrics with the -// current commit. +// stamp, so provenance hashes them and flags `stale` when any of their producer inputs is newer +// (PRODUCER_INPUTS below; see artifactProvenance in model.ts) — coverage tracks tests + config, size +// tracks package.json + packaging config, not just `src`. A consumer like #1424 must never pair +// stale metrics with the current commit. // // The one thing that can FAIL the command is the depgraph-vs-layering R6 consistency assertion: // the graph built here must reproduce scripts/layering/check.ts's TYPE_INVERSION_BASELINE. The @@ -85,28 +86,66 @@ function lockfileProvenance(): { path: string; sha256: string } | null { return raw === null ? null : { path: LOCKFILE_PATH, sha256: shortSha(raw) }; } -/** The newest mtime across the production source tree, the freshness bar for read artifacts. */ -function newestSourceMtimeMs(files: readonly string[]): number { - let newest = 0; +// The tracked inputs whose change can invalidate each READ artifact — NOT just `src`. Coverage is +// the whole test run (sources + tests + vitest config + the scripts package.json wires); size runs +// tsdown + `npm pack`, so packaging metadata and bundler/ts config are inputs. An old artifact is +// flagged stale when ANY of these is newer than it, closing the src-only false-negative. +const PRODUCER_INPUTS = { + coverage: [ + 'src/*.ts', + 'src/**/*.ts', + 'test', + 'vitest.config.ts', + 'vitest.*.config.ts', + 'package.json', + ], + size: [ + 'src/*.ts', + 'src/**/*.ts', + 'package.json', + 'pnpm-lock.yaml', + 'tsdown.config.ts', + 'tsconfig.json', + 'tsconfig.lib.json', + 'scripts/size-report.mjs', + ], +} as const; + +/** Tracked files matching the pathspecs (offline, ignores untracked/generated); [] on failure. */ +function trackedFiles(pathspecs: readonly string[]): string[] { + try { + const out = execFileSync('git', ['ls-files', '-z', '--', ...pathspecs], { + cwd: repoRoot, + encoding: 'utf8', + }); + return out.split('\0').filter(Boolean); + } catch { + return []; + } +} + +function mtimesMs(files: readonly string[]): number[] { + const out: number[] = []; for (const file of files) { - const mtimeMs = fs.statSync(absolute(file)).mtimeMs; - if (mtimeMs > newest) newest = mtimeMs; + const abs = absolute(file); + if (fs.existsSync(abs)) out.push(fs.statSync(abs).mtimeMs); } - return newest; + return out; } -/** Hash a read artifact and flag it stale when a source file is newer; null when absent. */ +/** Hash a read artifact and judge freshness against its whole producer input set; null when absent. */ function artifactInput( relativePath: string, - newestSourceMtimeMs: number, + producerInputs: readonly string[], ): ArtifactProvenance | null { const abs = absolute(relativePath); if (!fs.existsSync(abs)) return null; return artifactProvenance({ path: relativePath, sha256: shortSha(fs.readFileSync(abs, 'utf8')), + producerInputs: [...producerInputs], artifactMtimeMs: fs.statSync(abs).mtimeMs, - newestSourceMtimeMs, + producerInputMtimesMs: mtimesMs(trackedFiles(producerInputs)), }); } @@ -168,16 +207,14 @@ function collectProvenance( } function collectSnapshot(): RepoHealthSnapshot { - const files = listSourceFiles(); - const graph = buildGraphFromTree(files); - const newestSource = newestSourceMtimeMs(files); + const graph = buildGraphFromTree(listSourceFiles()); const coverage = readJsonIfExists(COVERAGE_SUMMARY_PATH); const size = readJsonIfExists(SIZE_REPORT_PATH); return buildSnapshot({ provenance: collectProvenance( graph, - artifactInput(COVERAGE_SUMMARY_PATH, newestSource), - artifactInput(SIZE_REPORT_PATH, newestSource), + artifactInput(COVERAGE_SUMMARY_PATH, PRODUCER_INPUTS.coverage), + artifactInput(SIZE_REPORT_PATH, PRODUCER_INPUTS.size), ), graph, fallow: { From 10a41951dc2a9881641b64c457bca72bc9dda179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 28 Jul 2026 16:46:55 +0000 Subject: [PATCH 4/4] repo-health: prove artifact freshness from a stamped producer commit; route git through runCmdSync Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- scripts/repo-health/README.md | 18 ++--- scripts/repo-health/model.test.ts | 51 +++++++------ scripts/repo-health/model.ts | 91 ++++++++++++++---------- scripts/repo-health/run.ts | 114 ++++++++---------------------- 4 files changed, 119 insertions(+), 155 deletions(-) diff --git a/scripts/repo-health/README.md b/scripts/repo-health/README.md index d409cb9a4..1f41a1bbf 100644 --- a/scripts/repo-health/README.md +++ b/scripts/repo-health/README.md @@ -53,16 +53,18 @@ Field names are the stable **trend/delta contract** consumed by #1424. Any chang "inputs": { "sourceFiles": 932, // production files fed to the graph build "lockfile": { "path": "pnpm-lock.yaml", "sha256": "…" }, - // coverage and size are read artifacts this command does NOT produce, so they carry no - // producing-commit stamp. They are hashed (so history keys on the bytes, not the commit) and - // flagged `stale` when ANY of their `producerInputs` is newer than the artifact — coverage - // tracks sources + tests + vitest config + package.json; size tracks package.json + lockfile + - // tsdown/tsconfig + size-report.mjs, not just `src`. Unprovable freshness (no observable input) - // is reported stale. A consumer (#1424) must treat stale metrics as lagging `commit`. + // coverage and size are read artifacts this command does NOT produce. Their freshness relative + // to `commit` is proven ONLY from a producing commit the artifact stamps in its own bytes — + // never mtime (a copy/restore/CI-cache download resets it) nor an enumerated input list (never + // provably complete). status: "fresh" = stamped commit == HEAD; "stale" = stamped commit != + // HEAD; "unknown" = no stamp. Today neither producer stamps a commit, so both report "unknown" + // (honest — cannot be proven current); if one starts emitting `commit`, it is verified. The + // bytes are still hashed so #1424 keys history on the exact metrics. A consumer must treat + // "unknown"/"stale" alike as not-current, never as `commit`. "coverageSummary": { "path": "coverage/coverage-summary.json", "sha256": "…", - "producerInputs": ["src/**/*.ts", "test", "…"], "stale": false } | null, + "producerCommit": null, "status": "unknown" } | null, "sizeReport": { "path": ".tmp/size-report.json", "sha256": "…", - "producerInputs": ["src/**/*.ts", "package.json", "…"], "stale": false } | null + "producerCommit": null, "status": "unknown" } | null } }, "metrics": { diff --git a/scripts/repo-health/model.test.ts b/scripts/repo-health/model.test.ts index 761536e6d..dea1a43ab 100644 --- a/scripts/repo-health/model.test.ts +++ b/scripts/repo-health/model.test.ts @@ -5,8 +5,7 @@ import { resolveImportEdges } from '../layering/model.ts'; import { buildGraph, type GraphData } from '../depgraph/model.ts'; import { mainSequenceModules, zoneComponents } from './components.ts'; import { - artifactProvenance, - isArtifactStale, + collectArtifactProvenance, benchMetrics, buildSnapshot, coverageMetrics, @@ -125,32 +124,32 @@ test('coverage and size degrade to unavailable when the artifact is absent', () ).toBe(true); }); -test('isArtifactStale flags an artifact predating ANY producer input, incl. non-src ones', () => { - const artifact = 1500; - const srcMtimes = [1000, 1200]; // all older than the artifact - // Freshness judged on src alone would (wrongly) call this fresh — the earlier false-negative. - expect(isArtifactStale(artifact, srcMtimes)).toBe(false); - // A non-src producer input (e.g. package.json / vitest config / tsdown config) edited after the - // artifact was produced must flip the verdict, so a coverage/size number is not paired with a - // commit it predates. - const packageJsonMtime = 1800; - expect(isArtifactStale(artifact, [...srcMtimes, packageJsonMtime])).toBe(true); - // No observable producer input => freshness cannot be proven => stale, never a false "fresh". - expect(isArtifactStale(artifact, [])).toBe(true); -}); +// Drives the real collector (parse → read stamped commit → hash → status), so removing the +// commit-verification breaks these — unlike a fabricated status. Freshness is proven ONLY from a +// commit the artifact stamps, never mtime or a producer-input list. +test('collectArtifactProvenance proves freshness only from a commit the artifact stamps', () => { + const path = 'coverage/coverage-summary.json'; -test('artifactProvenance binds bytes, the producer input set, and the staleness verdict', () => { - const args = { - path: 'coverage/coverage-summary.json', - sha256: 'deadbeef', - producerInputs: ['src/**/*.ts', 'package.json'], - }; - expect( - artifactProvenance({ ...args, artifactMtimeMs: 2000, producerInputMtimesMs: [1000, 1500] }), - ).toEqual({ ...args, stale: false }); + // A real coverage-summary shape stamps no commit — we cannot prove it matches HEAD, so `unknown`. + const coverageShaped = JSON.stringify({ total: { lines: { total: 10, covered: 8, pct: 80 } } }); + const unknown = collectArtifactProvenance(path, coverageShaped, 'abc123'); + expect(unknown?.status).toBe('unknown'); + expect(unknown?.producerCommit).toBeNull(); + expect(unknown?.sha256).toMatch(/^[0-9a-f]{12}$/); // bytes still hashed for #1424 + + // An artifact that stamps the current commit is proven current. expect( - artifactProvenance({ ...args, artifactMtimeMs: 1000, producerInputMtimesMs: [1500] }), - ).toEqual({ ...args, stale: true }); + collectArtifactProvenance(path, JSON.stringify({ commit: 'abc123', total: {} }), 'abc123') + ?.status, + ).toBe('fresh'); + + // A stamped commit that differs is stale — metrics predate HEAD. + const stale = collectArtifactProvenance(path, JSON.stringify({ commit: 'old999' }), 'abc123'); + expect(stale?.status).toBe('stale'); + expect(stale?.producerCommit).toBe('old999'); + + // Absent artifact degrades cleanly to null (coverage/size become { available: false }). + expect(collectArtifactProvenance(path, null, 'abc123')).toBeNull(); }); test('benchMetrics counts cases and distinct topics from the case registry', () => { diff --git a/scripts/repo-health/model.ts b/scripts/repo-health/model.ts index 3ec3db182..9246d2ecb 100644 --- a/scripts/repo-health/model.ts +++ b/scripts/repo-health/model.ts @@ -17,6 +17,7 @@ // become CI thresholds (CONTEXT.md "Principles and their gates"). The single thing wired into CI // is the depgraph-vs-layering R6 consistency assertion, which lives in the Layering Guard job. +import { createHash } from 'node:crypto'; import { TYPE_CYCLE_BASELINE, TYPE_INVERSION_BASELINE } from '../layering/check.ts'; import type { EdgeKind, GraphData } from '../depgraph/model.ts'; import { @@ -229,51 +230,69 @@ export function benchMetrics(cases: readonly { docs: readonly string[] }[]): Ben // ---- Provenance ------------------------------------------------------------------------------- +/** + * Freshness of a READ artifact relative to the snapshot's `commit`. `fresh` requires proof; anything + * unproven is `unknown`, never silently trusted. + */ +export type ArtifactStatus = 'fresh' | 'stale' | 'unknown'; + /** * Provenance for an analyzer artifact this snapshot READ but did not itself produce (coverage, - * size). These artifacts carry no producing-commit stamp, so the snapshot cannot claim they match - * `commit`. We instead record their content hash — so #1424 keys history on the bytes the metrics - * came from, not on a commit they may predate — the `producerInputs` freshness was judged against, - * and an explicit `stale` flag. Freshness is bound to the artifact's WHOLE producer input set, not - * just `src` (coverage also depends on tests/config; size on package.json and packaging config), so - * a change to any of them re-flags the artifact. A consumer must treat stale metrics as lagging - * `commit`, never as current. + * size). Its freshness relative to `commit` is proven ONLY from a producing commit the artifact + * stamps in its own bytes. We deliberately reject the two signals that cannot prove it: mtime (a + * copy, restore, or CI cache download resets it) and an enumerated producer-input list (never + * provably complete — the last review showed it silently omitted inputs). So a stamped commit equal + * to `commit` is `fresh`; a different stamped commit is `stale`; NO stamp is `unknown`. Today + * neither producer stamps a commit, so both report `unknown` — honest, since we cannot prove they + * match; if a producer starts emitting one, it is verified automatically. `sha256` still records the + * bytes so #1424 keys history on the exact metrics read. A consumer must treat `stale` and `unknown` + * alike as not-current, never as `commit`. */ export type ArtifactProvenance = { path: string; sha256: string; - /** The producer input pathspecs whose freshness this verdict was computed against. */ - producerInputs: string[]; - stale: boolean; + /** The producing commit read from the artifact's own bytes, or null when it stamps none. */ + producerCommit: string | null; + status: ArtifactStatus; }; -/** - * Whether an artifact predates any of its producer inputs. mtime is a local, offline signal (these - * artifacts are git-ignored, so there is no blob to diff), evaluated against a single tree in one - * run. Freshness we cannot prove is reported as stale: an EMPTY input set (nothing observable) and - * any input newer than the artifact both mean the metrics may lag the tree — we never claim fresh - * on the strength of `src` alone, which was the earlier false-negative. - */ -export function isArtifactStale( - artifactMtimeMs: number, - producerInputMtimesMs: readonly number[], -): boolean { - return producerInputMtimesMs.length === 0 || Math.max(...producerInputMtimesMs) > artifactMtimeMs; +/** The producing commit an artifact stamps in its own JSON, if any (`producerCommit` or `commit`). */ +function readStampedCommit(raw: string): string | null { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return null; + } + if (parsed === null || typeof parsed !== 'object') return null; + const record = parsed as Record; + for (const key of ['producerCommit', 'commit']) { + const value = record[key]; + if (typeof value === 'string' && value.length > 0) return value; + } + return null; } -/** Bind an artifact to the bytes and the full producer input set its freshness was judged against. */ -export function artifactProvenance(params: { - path: string; - sha256: string; - producerInputs: string[]; - artifactMtimeMs: number; - producerInputMtimesMs: readonly number[]; -}): ArtifactProvenance { +/** + * The whole read-artifact collector: hash the bytes, read any producing commit the artifact stamps, + * and derive freshness against `currentCommit`. Returns null when the artifact is absent. This is + * the function the runner wires to disk, so a test that drives it with real artifact bytes exercises + * the actual freshness path end-to-end (not a fabricated status). + */ +export function collectArtifactProvenance( + path: string, + raw: string | null, + currentCommit: string, +): ArtifactProvenance | null { + if (raw === null) return null; + const producerCommit = readStampedCommit(raw); + const status: ArtifactStatus = + producerCommit === null ? 'unknown' : producerCommit === currentCommit ? 'fresh' : 'stale'; return { - path: params.path, - sha256: params.sha256, - producerInputs: params.producerInputs, - stale: isArtifactStale(params.artifactMtimeMs, params.producerInputMtimesMs), + path, + sha256: createHash('sha256').update(raw).digest('hex').slice(0, 12), + producerCommit, + status, }; } @@ -282,7 +301,7 @@ export function artifactProvenance(params: { * #1424 can persist history and refuse to diff across schema versions. `tool` holds a content * hash per analyzer (its own source/config, standing in for a version); `inputs` records the * lockfile and the read-only artifacts each family of metrics was computed from, each hashed and - * flagged for staleness so a false commit-indexed history entry cannot slip through. + * given a proven freshness `status` so a false commit-indexed history entry cannot slip through. */ export type Provenance = { schemaVersion: number; @@ -295,7 +314,7 @@ export type Provenance = { /** Production source files fed to the graph build. */ sourceFiles: number; lockfile: { path: string; sha256: string } | null; - /** Read artifacts, hashed and stale-flagged; null when absent (coverage/size degrade cleanly). */ + /** Read artifacts, hashed with a proven freshness status; null when absent (degrade cleanly). */ coverageSummary: ArtifactProvenance | null; sizeReport: ArtifactProvenance | null; }; diff --git a/scripts/repo-health/run.ts b/scripts/repo-health/run.ts index ac1f4c59d..0a2ff9528 100644 --- a/scripts/repo-health/run.ts +++ b/scripts/repo-health/run.ts @@ -14,19 +14,19 @@ // size are read from coverage/coverage-summary.json and .tmp/size-report.json when present and // degrade to `available: false` (with the producing command) when they are not, so a fresh // checkout still produces a complete, honest snapshot. Those two artifacts carry no producing-commit -// stamp, so provenance hashes them and flags `stale` when any of their producer inputs is newer -// (PRODUCER_INPUTS below; see artifactProvenance in model.ts) — coverage tracks tests + config, size -// tracks package.json + packaging config, not just `src`. A consumer like #1424 must never pair -// stale metrics with the current commit. +// stamp today, so provenance hashes them and reports freshness as `unknown` unless the artifact +// itself stamps a commit equal to HEAD (see collectArtifactProvenance in model.ts) — we never infer +// freshness from mtime or a producer-input list, neither of which can prove it. A consumer like +// #1424 must treat `unknown`/`stale` as not-current, never as HEAD. // // The one thing that can FAIL the command is the depgraph-vs-layering R6 consistency assertion: // the graph built here must reproduce scripts/layering/check.ts's TYPE_INVERSION_BASELINE. The // identical assertion is wired into CI by the Layering Guard job (scripts/depgraph/model.test.ts). -import { execFileSync } from 'node:child_process'; import { createHash } from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; +import { runCmdSync } from '../../src/utils/exec.ts'; import { listSourceFiles } from '../layering/check.ts'; import { resolveImportEdges } from '../layering/model.ts'; import { buildGraph, type GraphData } from '../depgraph/model.ts'; @@ -36,8 +36,8 @@ import { runAsMain } from '../lib/run-as-main.ts'; import { CASES } from '../help-conformance-cases.mjs'; import skillgymSuite from '../../test/skillgym/suites/agent-device-smoke-suite.ts'; import { - artifactProvenance, buildSnapshot, + collectArtifactProvenance, SNAPSHOT_SCHEMA_VERSION, type ArtifactProvenance, type FallowConfig, @@ -53,9 +53,7 @@ const COVERAGE_SUMMARY_PATH = 'coverage/coverage-summary.json'; const SIZE_REPORT_PATH = '.tmp/size-report.json'; const LOCKFILE_PATH = 'pnpm-lock.yaml'; -const repoRoot = execFileSync('git', ['rev-parse', '--show-toplevel'], { - encoding: 'utf8', -}).trim(); +const repoRoot = runCmdSync('git', ['rev-parse', '--show-toplevel']).stdout.trim(); function absolute(relativePath: string): string { return path.join(repoRoot, relativePath); @@ -71,6 +69,12 @@ function readJsonIfExists(relativePath: string): T | null { return raw === null ? null : (JSON.parse(raw) as T); } +/** A read artifact's raw bytes (for content-hash provenance) and its parsed form (for metrics). */ +function readArtifact(relativePath: string): { raw: string | null; parsed: T | null } { + const raw = readIfExists(relativePath); + return { raw, parsed: raw === null ? null : (JSON.parse(raw) as T) }; +} + function shortSha(raw: string): string { return createHash('sha256').update(raw).digest('hex').slice(0, 12); } @@ -86,75 +90,9 @@ function lockfileProvenance(): { path: string; sha256: string } | null { return raw === null ? null : { path: LOCKFILE_PATH, sha256: shortSha(raw) }; } -// The tracked inputs whose change can invalidate each READ artifact — NOT just `src`. Coverage is -// the whole test run (sources + tests + vitest config + the scripts package.json wires); size runs -// tsdown + `npm pack`, so packaging metadata and bundler/ts config are inputs. An old artifact is -// flagged stale when ANY of these is newer than it, closing the src-only false-negative. -const PRODUCER_INPUTS = { - coverage: [ - 'src/*.ts', - 'src/**/*.ts', - 'test', - 'vitest.config.ts', - 'vitest.*.config.ts', - 'package.json', - ], - size: [ - 'src/*.ts', - 'src/**/*.ts', - 'package.json', - 'pnpm-lock.yaml', - 'tsdown.config.ts', - 'tsconfig.json', - 'tsconfig.lib.json', - 'scripts/size-report.mjs', - ], -} as const; - -/** Tracked files matching the pathspecs (offline, ignores untracked/generated); [] on failure. */ -function trackedFiles(pathspecs: readonly string[]): string[] { - try { - const out = execFileSync('git', ['ls-files', '-z', '--', ...pathspecs], { - cwd: repoRoot, - encoding: 'utf8', - }); - return out.split('\0').filter(Boolean); - } catch { - return []; - } -} - -function mtimesMs(files: readonly string[]): number[] { - const out: number[] = []; - for (const file of files) { - const abs = absolute(file); - if (fs.existsSync(abs)) out.push(fs.statSync(abs).mtimeMs); - } - return out; -} - -/** Hash a read artifact and judge freshness against its whole producer input set; null when absent. */ -function artifactInput( - relativePath: string, - producerInputs: readonly string[], -): ArtifactProvenance | null { - const abs = absolute(relativePath); - if (!fs.existsSync(abs)) return null; - return artifactProvenance({ - path: relativePath, - sha256: shortSha(fs.readFileSync(abs, 'utf8')), - producerInputs: [...producerInputs], - artifactMtimeMs: fs.statSync(abs).mtimeMs, - producerInputMtimesMs: mtimesMs(trackedFiles(producerInputs)), - }); -} - function gitOutput(args: string[]): string | null { - try { - return execFileSync('git', args, { cwd: repoRoot, encoding: 'utf8' }).trim(); - } catch { - return null; - } + const result = runCmdSync('git', args, { cwd: repoRoot, allowFailure: true }); + return result.exitCode === 0 ? result.stdout.trim() : null; } function buildGraphFromTree(files: readonly string[]): GraphData { @@ -187,12 +125,13 @@ function toolHashes(): Record { function collectProvenance( graph: GraphData, + commit: string, coverage: ArtifactProvenance | null, size: ArtifactProvenance | null, ): Provenance { return { schemaVersion: SNAPSHOT_SCHEMA_VERSION, - commit: currentCommit(), + commit, ref: currentRef(), node: process.version, generatedAt: new Date().toISOString(), @@ -208,13 +147,15 @@ function collectProvenance( function collectSnapshot(): RepoHealthSnapshot { const graph = buildGraphFromTree(listSourceFiles()); - const coverage = readJsonIfExists(COVERAGE_SUMMARY_PATH); - const size = readJsonIfExists(SIZE_REPORT_PATH); + const commit = currentCommit(); + const coverage = readArtifact(COVERAGE_SUMMARY_PATH); + const size = readArtifact(SIZE_REPORT_PATH); return buildSnapshot({ provenance: collectProvenance( graph, - artifactInput(COVERAGE_SUMMARY_PATH, PRODUCER_INPUTS.coverage), - artifactInput(SIZE_REPORT_PATH, PRODUCER_INPUTS.size), + commit, + collectArtifactProvenance(COVERAGE_SUMMARY_PATH, coverage.raw, commit), + collectArtifactProvenance(SIZE_REPORT_PATH, size.raw, commit), ), graph, fallow: { @@ -222,8 +163,8 @@ function collectSnapshot(): RepoHealthSnapshot { health: readJsonIfExists('fallow-baselines/health.json') ?? {}, config: readJsonIfExists('.fallowrc.json') ?? {}, }, - coverage, - size, + coverage: coverage.parsed, + size: size.parsed, slowTest: SLOW_TEST_RATCHET, benchCases: CASES, skillgymCaseCount: skillgymSuite.length, @@ -244,7 +185,10 @@ function bytes(value: number | undefined): string { } function staleMark(artifact: ArtifactProvenance | null): string { - return artifact?.stale ? ' [STALE — source newer than artifact; rerun the producer]' : ''; + if (artifact === null || artifact.status === 'fresh') return ''; + return artifact.status === 'stale' + ? ' [STALE — artifact stamps a different commit]' + : ' [UNKNOWN — artifact stamps no commit; may not reflect HEAD]'; } function renderSummary(snapshot: RepoHealthSnapshot): string {