From 99fdaaf24a8e790c40d35c6472734dd3ace2d385 Mon Sep 17 00:00:00 2001 From: Alexander Akait <4567934+alexander-akait@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:04:29 +0000 Subject: [PATCH 1/2] perf: sign typed-lint results once per build rather than once per batch While webpack builds, the ESLint check is handed files in batches, and every batch signed its files from scratch: it parsed each config file again and checked every file of the program again, so a build paid for the whole project once per batch. One signer now answers for a build, reading each file and config file once and keeping the signatures it worked out for the batches after. Over 3,000 modules checked by typed ESLint and the TypeScript check with threads: false, a warm build took 13.3-13.7 s and now takes 11.9-12.4 s; signing took 1.9 s of it and now takes 0.3 s. Refs #374. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013FhutQipYi25fJ4rs9WE1c --- .changeset/sign-once-per-build.md | 5 + src/checks/eslint.js | 19 ++- src/import-graph.js | 243 +++++++++++++++++++----------- test/import-graph.test.js | 38 +++++ 4 files changed, 212 insertions(+), 93 deletions(-) create mode 100644 .changeset/sign-once-per-build.md diff --git a/.changeset/sign-once-per-build.md b/.changeset/sign-once-per-build.md new file mode 100644 index 00000000..a23a4f3a --- /dev/null +++ b/.changeset/sign-once-per-build.md @@ -0,0 +1,5 @@ +--- +"diagnostics-webpack-plugin": patch +--- + +Read the files kept under typed linting once per build rather than once per batch of files webpack hands over while it builds: every batch read the `tsconfig.json` and checked every file of the program again. Over 3,000 modules checked by both typed ESLint and the TypeScript check with `threads: false`, a build answered from what it kept took 13.3–13.7 s and now takes 11.9–12.4 s, and signing them took 1.9 s and now takes 0.3 s. diff --git a/src/checks/eslint.js b/src/checks/eslint.js index 5cfac3b1..5942c782 100644 --- a/src/checks/eslint.js +++ b/src/checks/eslint.js @@ -5,7 +5,7 @@ import { createRequire } from "node:module"; import { isAbsolute, join, resolve } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; -import { getGraph, hashOf, signatures } from "../import-graph.js"; +import { createSigner, getGraph, hashOf } from "../import-graph.js"; import { openStore, saveStore } from "../result-store.js"; import { countThreads, createPool } from "../threads.js"; import { omitPluginOptions, toPosixPath } from "../utils.js"; @@ -387,8 +387,9 @@ async function create({ key, id, options, compilation }) { let across; /** * Under typed linting, where ESLint's own cache cannot answer: the import - * graph each file's results are kept against, and where they are kept. - * @type {{ graph: import("../import-graph.js").Graph, store: import("../result-store.js").Store } | undefined} + * graph each file's results are kept against, where they are kept, and what + * signs the files of this build's batches. + * @type {{ graph: import("../import-graph.js").Graph, store: import("../result-store.js").Store, sign: (files: string[]) => Map } | undefined} */ let remembered; @@ -424,7 +425,11 @@ async function create({ key, id, options, compilation }) { : undefined; if (graph) { - remembered = { graph, store: openStore(resultsPathFor(eslintOptions)) }; + remembered = { + graph, + store: openStore(resultsPathFor(eslintOptions)), + sign: createSigner(graph), + }; eslintOptions.cache = false; changed = true; } else if (cacheIsOurs) { @@ -502,12 +507,12 @@ async function create({ key, id, options, compilation }) { /** * Answers for what was linted before under the same everything from what * was found then, and lints the rest. - * @param {NonNullable} kept the graph and the store + * @param {NonNullable} kept the graph, the store and the signer * @param {string[]} files the files to lint * @returns {Promise<{ found: LintResult[], linted: LintResult[] }>} every result, and the ones linted now */ - const lintRemembered = async ({ graph, store }, files) => { - const signed = signatures(graph, files); + const lintRemembered = async ({ store, sign }, files) => { + const signed = sign(files); /** @type {Map} */ const keys = new Map(); /** @type {LintResult[]} */ diff --git a/src/import-graph.js b/src/import-graph.js index 58878b42..3dc798ec 100644 --- a/src/import-graph.js +++ b/src/import-graph.js @@ -624,11 +624,11 @@ function readConfig(graph, config) { * files its imports resolve through, and the declaration files and global * declarations of the programs they describe, which a file sees without * importing them. - * @param {Graph} graph the graph * @param {Map} configs the config files as they are now + * @param {(file: string) => Read | undefined} read reads a file as it is now * @returns {string} what they hold, hashed */ -function projectSignature(graph, configs) { +function projectSignature(configs, read) { /** @type {string[]} */ const parts = []; @@ -640,7 +640,7 @@ function projectSignature(graph, configs) { if (isPackage(file)) continue; - const known = readNow(graph, file); + const known = read(file); if (!known) continue; @@ -662,54 +662,56 @@ function projectSignature(graph, configs) { * imports however indirectly, and what the whole project sees — so that a * result kept for a file is good for as long as this is the same, and no * longer. A package is read for when it was installed rather than for what it - * holds. + * holds. One signer answers for one build, which hands it its files in batches: + * what it read for one batch is not read again for the next. * @param {Graph} graph the graph - * @param {string[]} files the files to answer for - * @returns {Map} the signature of each of them that can be read, by its posix path + * @returns {(files: string[]) => Map} the signature of each of the files given that can be read, by its posix path */ -function signatures(graph, files) { +function createSigner(graph) { graph.resolutions.clear(); - const wanted = files.map((file) => toPosixPath(resolve(file))); - - // Read first, so that the config files they resolve through are known. - for (const file of wanted) readNow(graph, file); - + /** + * The files read in this build, whether or not they were there. + * @type {Set} + */ + const checked = new Set(); /** @type {Map} */ - let configs = new Map(); + const configsRead = new Map(); + /** @type {{ configs: Map, signature: string } | undefined} */ + let project; - for (const config of graph.options.keys()) { - configs.set(config, readConfig(graph, config)); - } + /** + * @param {string} file the file, as a posix path + * @returns {Read | undefined} it as it is in this build + */ + const read = (file) => { + if (checked.has(file)) return graph.reads.get(file); - // A config file that changed decides how every import resolves, so what was - // read under it is read again, under it as it is now. - if ( - [...configs].some( - ([config, { hash }]) => graph.configHashes.get(config) !== hash, - ) && - graph.configHashes.size > 0 - ) { - graph.reads.clear(); - graph.options.clear(); - graph.configs.clear(); + checked.add(file); - for (const file of wanted) readNow(graph, file); + return readNow(graph, file); + }; - configs = new Map(); + /** + * @returns {Map} every config file read so far, as it is in this build + */ + const readConfigs = () => { + /** @type {Map} */ + const configs = new Map(); for (const config of graph.options.keys()) { - configs.set(config, readConfig(graph, config)); - } - } + let now = configsRead.get(config); - graph.configHashes.clear(); + if (!now) { + now = readConfig(graph, config); + configsRead.set(config, now); + } - for (const [config, { hash }] of configs) { - graph.configHashes.set(config, hash); - } + configs.set(config, now); + } - const project = projectSignature(graph, configs); + return configs; + }; /** * What each file says of itself, and what it imports, read once however * many of the closures reach it. @@ -739,17 +741,17 @@ function signatures(graph, files) { imports: [], }; } else { - const read = readNow(graph, file); + const now = read(file); - described = read + described = now ? { parts: [ - `${file} ${read.hash}`, - ...read.imports.missing.map( + `${file} ${now.hash}`, + ...now.imports.missing.map( (specifier) => `${file} misses ${specifier}`, ), ], - imports: read.imports.resolved, + imports: now.imports.resolved, } : { parts: [`${file} gone`], imports: [] }; } @@ -824,70 +826,139 @@ function signatures(graph, files) { for (const member of members) signed.set(member, signature); }; - for (const root of wanted) { - if (!graph.reads.has(root) || order.has(root)) continue; + return (files) => { + const wanted = files.map((file) => toPosixPath(resolve(file))); + + // Read first, so that the config files they resolve through are known. + for (const file of wanted) read(file); + + let configs = readConfigs(); + + // A config file that changed decides how every import resolves, so what was + // read under it is read again, under it as it is now. + if ( + [...configs].some( + ([config, { hash }]) => graph.configHashes.get(config) !== hash, + ) && + graph.configHashes.size > 0 + ) { + graph.reads.clear(); + graph.options.clear(); + graph.configs.clear(); + checked.clear(); + configsRead.clear(); + known.clear(); + signed.clear(); + order.clear(); + lowest.clear(); + project = undefined; + + for (const file of wanted) read(file); + + configs = readConfigs(); + } + + graph.configHashes.clear(); + + for (const [config, { hash }] of configs) { + graph.configHashes.set(config, hash); + } + + const kept = project; + const current = + !kept || + kept.configs.size !== configs.size || + [...configs].some(([config, now]) => kept.configs.get(config) !== now) + ? { configs, signature: projectSignature(configs, read) } + : kept; + + project = current; + + const projectSigned = current.signature; + + for (const root of wanted) { + if (!graph.reads.has(root) || order.has(root)) continue; + + enter(root); + + /** @type {[string, number][]} */ + const stack = [[root, 0]]; - enter(root); + while (stack.length > 0) { + const frame = stack[stack.length - 1]; + const [file, next] = frame; + const { imports } = describe(file); - /** @type {[string, number][]} */ - const stack = [[root, 0]]; + if (next < imports.length) { + const target = imports[next]; - while (stack.length > 0) { - const frame = stack[stack.length - 1]; - const [file, next] = frame; - const { imports } = describe(file); + frame[1] = next + 1; - if (next < imports.length) { - const target = imports[next]; + if (!order.has(target)) { + enter(target); + stack.push([target, 0]); + } else if (onOpen.has(target)) { + lowest.set( + file, + Math.min( + /** @type {number} */ (lowest.get(file)), + /** @type {number} */ (order.get(target)), + ), + ); + } + + continue; + } - frame[1] = next + 1; + stack.pop(); + + if (stack.length > 0) { + const [parent] = stack[stack.length - 1]; - if (!order.has(target)) { - enter(target); - stack.push([target, 0]); - } else if (onOpen.has(target)) { lowest.set( - file, + parent, Math.min( + /** @type {number} */ (lowest.get(parent)), /** @type {number} */ (lowest.get(file)), - /** @type {number} */ (order.get(target)), ), ); } - continue; + if (lowest.get(file) === order.get(file)) close(file); } + } - stack.pop(); + /** @type {Map} */ + const found = new Map(); - if (stack.length > 0) { - const [parent] = stack[stack.length - 1]; + for (const file of wanted) { + const signature = signed.get(file); - lowest.set( - parent, - Math.min( - /** @type {number} */ (lowest.get(parent)), - /** @type {number} */ (lowest.get(file)), - ), - ); + if (signature !== undefined) { + found.set(file, hashOf(`${projectSigned}\n${signature}`)); } - - if (lowest.get(file) === order.get(file)) close(file); } - } - - /** @type {Map} */ - const found = new Map(); - - for (const file of wanted) { - const signature = signed.get(file); - if (signature !== undefined) { - found.set(file, hashOf(`${project}\n${signature}`)); - } - } + return found; + }; +} - return found; +/** + * The signatures of files signed on their own, as one batch of a build. + * @param {Graph} graph the graph + * @param {string[]} files the files to answer for + * @returns {Map} the signature of each of them that can be read, by its posix path + */ +function signatures(graph, files) { + return createSigner(graph)(files); } -export { affectedFiles, createGraph, getGraph, hashOf, keepTexts, signatures }; +export { + affectedFiles, + createGraph, + createSigner, + getGraph, + hashOf, + keepTexts, + signatures, +}; diff --git a/test/import-graph.test.js b/test/import-graph.test.js index 720f5276..5bfdf017 100644 --- a/test/import-graph.test.js +++ b/test/import-graph.test.js @@ -14,6 +14,7 @@ import { afterEach, beforeEach, describe, it } from "node:test"; import { affectedFiles, createGraph, + createSigner, getGraph, keepTexts, signatures, @@ -715,6 +716,43 @@ describe("signatures", () => { assert.deepStrictEqual(changed(before, sign()), ["top.ts"]); }); + it("should read the config file once however many batches a build signs", () => { + let reads = 0; + const counted = createGraph({ + ...ts, + sys: { + ...ts.sys, + readFile: (file, encoding) => { + if (file.endsWith("tsconfig.json")) reads += 1; + + return ts.sys.readFile(file, encoding); + }, + }, + }); + const sign = createSigner(counted); + + sign([at(names[0])]); + + const once = reads; + + for (const name of names.slice(1)) sign([at(name)]); + + assert.ok(once > 0); + assert.strictEqual(reads, once); + }); + + it("should sign files in batches as it signs them at once", () => { + const whole = sign(); + const sign2 = createSigner(createGraph(ts)); + const batched = {}; + + for (const name of names) { + batched[name] = sign2([at(name)]).get(toPosixPath(at(name))); + } + + assert.deepStrictEqual(batched, whole); + }); + it("should change once an import that resolved to nothing does", () => { write( "alone.ts", From 31efd31d7cc2a2f29a87aca5717e989692501bf3 Mon Sep 17 00:00:00 2001 From: Alexander Akait <4567934+alexander-akait@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:13:39 +0000 Subject: [PATCH 2/2] perf: keep what a build read when a later batch reaches another config file A config file first reached in a later batch had no hash recorded and was taken for one that changed: the signer forgot everything the build had read and read the batch again. In a monorepo with a tsconfig.json per package that happened for every batch reaching a new package. Only a config file whose recorded hash differs now starts over. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013FhutQipYi25fJ4rs9WE1c --- src/import-graph.js | 10 ++++++---- test/import-graph.test.js | 30 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/import-graph.js b/src/import-graph.js index 3dc798ec..7b3aeb77 100644 --- a/src/import-graph.js +++ b/src/import-graph.js @@ -835,12 +835,14 @@ function createSigner(graph) { let configs = readConfigs(); // A config file that changed decides how every import resolves, so what was - // read under it is read again, under it as it is now. + // read under it is read again, under it as it is now. One first reached in + // this batch has nothing read under it yet. if ( [...configs].some( - ([config, { hash }]) => graph.configHashes.get(config) !== hash, - ) && - graph.configHashes.size > 0 + ([config, { hash }]) => + graph.configHashes.has(config) && + graph.configHashes.get(config) !== hash, + ) ) { graph.reads.clear(); graph.options.clear(); diff --git a/test/import-graph.test.js b/test/import-graph.test.js index 5bfdf017..16f280bb 100644 --- a/test/import-graph.test.js +++ b/test/import-graph.test.js @@ -741,6 +741,36 @@ describe("signatures", () => { assert.strictEqual(reads, once); }); + it("should read nothing again when a later batch reaches another config file", () => { + mkdirSync(at("a")); + mkdirSync(at("b")); + write("a/tsconfig.json", "{}"); + write("b/tsconfig.json", "{}"); + write("a/one.ts", "export const one = 1;\n"); + write("b/two.ts", "export const two = 2;\n"); + + /** @type {Map} */ + const reads = new Map(); + const counted = createGraph({ + ...ts, + sys: { + ...ts.sys, + readFile: (file, encoding) => { + reads.set(file, (reads.get(file) || 0) + 1); + + return ts.sys.readFile(file, encoding); + }, + }, + }); + const sign = createSigner(counted); + + sign([at("a/one.ts")]); + sign([at("b/two.ts")]); + + assert.strictEqual(reads.get(toPosixPath(at("a/one.ts"))), 1); + assert.strictEqual(reads.get(toPosixPath(at("b/two.ts"))), 1); + }); + it("should sign files in batches as it signs them at once", () => { const whole = sign(); const sign2 = createSigner(createGraph(ts));