diff --git a/README.md b/README.md index eabee30..00a687b 100644 --- a/README.md +++ b/README.md @@ -572,11 +572,12 @@ Compares two texts and returns a structured diff at a character, word, or senten previousText: string | null | undefined, currentText: string | null | undefined, options?: { - separation?: "character" | "word" | "sentence", // "word" by default + separation?: "character" | "word" | "sentence" // "word" by default accuracy?: "normal" | "high", // "normal" by default detectMoves?: boolean // false by default ignoreCase?: boolean, // false by default ignorePunctuation?: boolean, // false by default + preserveWhitespace?: boolean, // false by default locale?: Intl.Locale | string // undefined by default } ``` @@ -592,6 +593,7 @@ Compares two texts and returns a structured diff at a character, word, or senten - `true`: semantically precise, but noisier — a single insertion shifts all following tokens, breaking equality. - `ignoreCase`: if `true`, `hello` and `HELLO` are considered equal. - `ignorePunctuation`: if `true`, `hello!` and `hello` are considered equal. + - `preserveWhitespace`: if `true`, each token keeps the whitespace preceding it, so the original text can be rebuilt from the diff and whitespace-only edits are detected. Only available in normal accuracy mode. - `locale`: the locale of your text. Enables locale‑aware segmentation in high accuracy mode. **Output** diff --git a/benchmark/texts.ts b/benchmark/texts.ts index 5f5c28f..6223a93 100644 --- a/benchmark/texts.ts +++ b/benchmark/texts.ts @@ -50,7 +50,7 @@ export function runTextBench10KSentences() { const diff = bench("diff", 1, () => diffSentences(prev, curr, {})); const superdiff = bench("Superdiff", 1, () => { - getTextDiff(prev, curr, { separation: "sentences" }); + getTextDiff(prev, curr, { separation: "sentence" }); }); return { superdiff, diff }; } diff --git a/package.json b/package.json index bc56ef3..e416de6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@donedeal0/superdiff", - "version": "4.2.3", + "version": "4.2.4", "type": "module", "description": "Superdiff provides a rich and readable diff for arrays, objects, texts and coordinates. It supports stream and file inputs for handling large datasets efficiently, is battle-tested, has zero dependencies, and offers a top-tier performance.", "main": "dist/index.js", @@ -11,7 +11,16 @@ "dist" ], "exports": { - ".": "./dist/index.js", + ".": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + }, "./client": "./dist/client.js", "./server": "./dist/server.cjs" }, diff --git a/src/lib/text-diff/lcs/myers.ts b/src/lib/text-diff/lcs/myers.ts index 4697922..cdc07f6 100644 --- a/src/lib/text-diff/lcs/myers.ts +++ b/src/lib/text-diff/lcs/myers.ts @@ -5,11 +5,15 @@ type MyersEdit = | { status: TextStatus.ADDED; curr: number } | { status: TextStatus.DELETED; prev: number }; -function backtrack( - trace: Map[], - a: TextToken[], - b: TextToken[], -): MyersEdit[] { +type Trace = Int32Array[]; + +function readDiagonal(trace: Int32Array, k: number, d: number): number { + const index = k + (d + 1); + if (index < 0 || index >= trace.length) return 0; + return trace[index]; +} + +function backtrack(trace: Trace, a: TextToken[], b: TextToken[]): MyersEdit[] { let x = a.length; let y = b.length; const edits: MyersEdit[] = []; @@ -19,13 +23,16 @@ function backtrack( const k = x - y; let prevK: number; - if (k === -d || (k !== d && (v.get(k - 1) ?? 0) < (v.get(k + 1) ?? 0))) { + if ( + k === -d || + (k !== d && readDiagonal(v, k - 1, d) < readDiagonal(v, k + 1, d)) + ) { prevK = k + 1; } else { prevK = k - 1; } - const prevX = v.get(prevK) ?? 0; + const prevX = readDiagonal(v, prevK, d); const prevY = prevX - prevK; while (x > prevX && y > prevY) { @@ -63,20 +70,20 @@ export function myersDiff(a: TextToken[], b: TextToken[]): MyersEdit[] { const M = b.length; const max = N + M; - const trace: Map[] = []; - const v = new Map(); - v.set(1, 0); + const trace: Trace = []; + const offset = max + 1; + const v = new Int32Array(2 * max + 3); for (let d = 0; d <= max; d++) { - const vSnapshot = new Map(v); + trace.push(v.slice(offset - d - 1, offset + d + 2)); for (let k = -d; k <= d; k += 2) { let x: number; - if (k === -d || (k !== d && (v.get(k - 1) ?? 0) < (v.get(k + 1) ?? 0))) { - x = v.get(k + 1) ?? 0; + if (k === -d || (k !== d && v[offset + k - 1] < v[offset + k + 1])) { + x = v[offset + k + 1]; } else { - x = (v.get(k - 1) ?? 0) + 1; + x = v[offset + k - 1] + 1; } let y = x - k; @@ -86,15 +93,12 @@ export function myersDiff(a: TextToken[], b: TextToken[]): MyersEdit[] { y++; } - v.set(k, x); + v[offset + k] = x; if (x >= N && y >= M) { - trace.push(vSnapshot); return backtrack(trace, a, b); } } - - trace.push(vSnapshot); } return []; diff --git a/src/lib/text-diff/text-diff.test.ts b/src/lib/text-diff/text-diff.test.ts index a7ef169..10d56bb 100644 --- a/src/lib/text-diff/text-diff.test.ts +++ b/src/lib/text-diff/text-diff.test.ts @@ -5565,3 +5565,221 @@ describe("getTextDiff – with moves detection", () => { ).toStrictEqual(resultFrench); }); }); + +describe("getTextDiff - preserveWhitespace", () => { + const CODE_LINES: [string, string][] = [ + [" const foo = bar(1);", " const foo = baz(2);"], + ["var myVar = 2;", "var myVariable = 3;"], + ["a b\tc", "a b\tc d"], + ["\t\tif (a && b) { ", "\t\tif (a || b) { "], + [" return x;", " return x;"], + ["\tif (a) {", " if (a) {"], + [" ", " "], + ["", " hello "], + [" hello ", ""], + [" hello ", " hello "], + ]; + + const rebuildBothTexts = (diff: ReturnType) => { + let previous = ""; + let current = ""; + for (const part of diff.diff) { + if (part.status === "updated") { + previous += part.previousValue ?? ""; + current += part.value; + continue; + } + if (part.status !== "added") previous += part.value; + if (part.status !== "deleted") current += part.value; + } + return { previous, current }; + }; + + it("keeps the indentation on the surrounding tokens", () => { + expect( + getTextDiff(" const a = 1;", " const b = 1;", { + separation: "word", + preserveWhitespace: true, + }), + ).toStrictEqual({ + type: "text", + status: "updated", + diff: [ + { value: " const", index: 0, previousIndex: 0, status: "equal" }, + { value: " a", index: null, previousIndex: 1, status: "deleted" }, + { value: " b", index: 1, previousIndex: null, status: "added" }, + { value: " =", index: 2, previousIndex: 2, status: "equal" }, + { value: " 1;", index: 3, previousIndex: 3, status: "equal" }, + ], + }); + }); + + it("reports a re-indentation that is invisible by default", () => { + expect( + getTextDiff(" return x;", " return x;", { separation: "word" }), + ).toStrictEqual({ + type: "text", + status: "equal", + diff: [ + { value: "return", index: 0, previousIndex: 0, status: "equal" }, + { value: "x;", index: 1, previousIndex: 1, status: "equal" }, + ], + }); + + expect( + getTextDiff(" return x;", " return x;", { + separation: "word", + preserveWhitespace: true, + }), + ).toStrictEqual({ + type: "text", + status: "updated", + diff: [ + { value: " return", index: null, previousIndex: 0, status: "deleted" }, + { + value: " return", + index: 0, + previousIndex: null, + status: "added", + }, + { value: " x;", index: 1, previousIndex: 1, status: "equal" }, + ], + }); + }); + + it("reports tabs converted to spaces", () => { + expect( + getTextDiff("\tif (a) {", " if (a) {", { + separation: "word", + preserveWhitespace: true, + }).status, + ).toBe("updated"); + }); + + it("stays equal when the spacing is untouched", () => { + expect( + getTextDiff(" a b ", " a b ", { + separation: "word", + preserveWhitespace: true, + }).status, + ).toBe("equal"); + }); + + it.each(CODE_LINES)("rebuilds %j and %j - word", (previous, current) => { + expect( + rebuildBothTexts( + getTextDiff(previous, current, { + separation: "word", + preserveWhitespace: true, + }), + ), + ).toEqual({ previous, current }); + }); + + it.each(CODE_LINES)("rebuilds %j and %j - character", (previous, current) => { + expect( + rebuildBothTexts( + getTextDiff(previous, current, { + separation: "character", + preserveWhitespace: true, + }), + ), + ).toEqual({ previous, current }); + }); + + it.each(CODE_LINES)( + "rebuilds %j and %j - detectMoves", + (previous, current) => { + expect( + rebuildBothTexts( + getTextDiff(previous, current, { + separation: "word", + preserveWhitespace: true, + detectMoves: true, + }), + ), + ).toEqual({ previous, current }); + }, + ); + + it("rebuilds sentences", () => { + const previous = " Hello world. How are you? "; + const current = " Hello there. How are you? "; + expect( + rebuildBothTexts( + getTextDiff(previous, current, { + separation: "sentence", + preserveWhitespace: true, + }), + ), + ).toEqual({ previous, current }); + }); + + it("keeps surrogate pairs intact", () => { + const previous = " a😀b"; + const current = " a😀c"; + const diff = getTextDiff(previous, current, { + separation: "character", + preserveWhitespace: true, + }); + + expect(rebuildBothTexts(diff)).toEqual({ previous, current }); + expect(diff.diff.some((part) => part.value === "😀")).toBe(true); + }); + + it("reports the same statuses as the default when the spacing is unchanged", () => { + const previous = " const foo = bar(1);"; + const current = " const foo = baz(2);"; + const statuses = (diff: ReturnType) => + diff.diff.map((part) => part.status); + + expect( + statuses( + getTextDiff(previous, current, { + separation: "word", + preserveWhitespace: true, + }), + ), + ).toEqual(statuses(getTextDiff(previous, current, { separation: "word" }))); + }); + + it("still honours ignoreCase", () => { + expect( + getTextDiff(" Hello", " hello", { + separation: "word", + preserveWhitespace: true, + ignoreCase: true, + }).status, + ).toBe("equal"); + }); +}); + +describe("getTextDiff - large dissimilar inputs", () => { + it("diffs two large, mostly dissimilar texts without exhausting memory", () => { + const previous = Array.from( + { length: 900 }, + (_unused, i) => `alpha_${i}(x${i}).beta;`, + ).join(" "); + const current = Array.from( + { length: 900 }, + (_unused, i) => `gamma${i % 7}[y${i}] = delta_${i} + 1;`, + ).join(" "); + + const diff = getTextDiff(previous, current, { + separation: "word", + preserveWhitespace: true, + }); + + expect(diff.status).toBe("updated"); + + let rebuiltPrevious = ""; + let rebuiltCurrent = ""; + for (const part of diff.diff) { + if (part.status !== "added") rebuiltPrevious += part.value; + if (part.status !== "deleted") rebuiltCurrent += part.value; + } + + expect(rebuiltPrevious).toBe(previous); + expect(rebuiltCurrent).toBe(current); + }, 15_000); +}); diff --git a/src/lib/text-diff/tokenize/normal.ts b/src/lib/text-diff/tokenize/normal.ts index ff32d19..707a84e 100644 --- a/src/lib/text-diff/tokenize/normal.ts +++ b/src/lib/text-diff/tokenize/normal.ts @@ -2,6 +2,7 @@ import { DEFAULT_TEXT_DIFF_OPTIONS, PUNCTUATION_REGEX, TextDiffOptions, + TextSeparation, TextToken, } from "@models/text"; @@ -16,58 +17,115 @@ function normalizeToken(token: string, options: TextDiffOptions): string { return normalizedToken; } +const TOKEN = { + character: /\S/gu, + word: /\S+/g, + sentence: /[^.!?]+[.!?]+|\S+/g, +}; + +const TOKEN_WITH_LEADING_WHITESPACE = { + character: /\s*\S/gu, + word: /\s*\S+/g, + sentence: /\s*(?:[^.!?]+[.!?]+|\S+)/g, +}; + +function tokenizePreservingWhitespace( + text: string, + options: TextDiffOptions, + separation: TextSeparation, +): TextToken[] { + const result: TextToken[] = []; + const tokens = text.match(TOKEN_WITH_LEADING_WHITESPACE[separation]) || []; + + let matchedLength = 0; + for (let i = 0; i < tokens.length; i++) { + const value = tokens[i]; + matchedLength += value.length; + result.push({ + value, + normalizedValue: normalizeToken(value, options), + index: i, + }); + } + + const trailingWhitespace = text.slice(matchedLength); + if (!trailingWhitespace) return result; + + if (result.length === 0) { + result.push({ + value: trailingWhitespace, + normalizedValue: trailingWhitespace, + index: 0, + }); + return result; + } + + const lastToken = result[result.length - 1]; + lastToken.value += trailingWhitespace; + lastToken.normalizedValue += trailingWhitespace; + return result; +} + export const tokenizeNormalText = ( text: string | null | undefined, options: TextDiffOptions = DEFAULT_TEXT_DIFF_OPTIONS, ): TextToken[] => { const separation = options.separation || DEFAULT_TEXT_DIFF_OPTIONS.separation; const result: TextToken[] = []; - if (!text || !text.trim()) return result; + if (!text) return result; + + if (options.preserveWhitespace) { + return tokenizePreservingWhitespace(text, options, separation ?? "word"); + } - if (separation === "character") { - let index = 0; - for (const char of text) { - const trimmedChar = char.trim(); - if (trimmedChar) { - const normalizedValue = normalizeToken(trimmedChar, options); - if (normalizedValue) { + if (!text.trim()) return result; + + if (separation !== "word") { + if (separation === "sentence") { + const sentences = text.match(TOKEN.sentence) || []; + let index = 0; + for (const sentence of sentences) { + const trimmedSentence = sentence.trim(); + if (trimmedSentence) { result.push({ - value: trimmedChar, - normalizedValue, + value: trimmedSentence, + normalizedValue: normalizeToken(trimmedSentence, options), index: index, }); index++; } } + return result; } - return result; - } - if (separation === "word") { - const tokens = text.match(/\S+/g) || []; - for (let i = 0; i < tokens.length; i++) { - const value = tokens[i]; - result.push({ - value, - normalizedValue: normalizeToken(value, options), - index: i, - }); - } - return result; - } else { - const sentences = text.match(/[^.!?]+[.!?]+|\S+/g) || []; - let index = 0; - for (const data of sentences) { - const trimmedSentence = data.trim(); - if (trimmedSentence) { - result.push({ - value: trimmedSentence, - normalizedValue: normalizeToken(trimmedSentence, options), - index: index, - }); - index++; + if (separation === "character") { + let index = 0; + for (const char of text) { + const trimmedChar = char.trim(); + if (trimmedChar) { + const normalizedValue = normalizeToken(trimmedChar, options); + if (normalizedValue) { + result.push({ + value: trimmedChar, + normalizedValue, + index: index, + }); + index++; + } + } } + return result; } - return result; } + + const tokens = text.match(TOKEN.word) || []; + for (let i = 0; i < tokens.length; i++) { + const value = tokens[i]; + result.push({ + value, + normalizedValue: normalizeToken(value, options), + index: i, + }); + } + return result; }; diff --git a/src/lib/text-diff/tokenize/tokenize-normal.test.ts b/src/lib/text-diff/tokenize/tokenize-normal.test.ts index be954f2..02fe027 100644 --- a/src/lib/text-diff/tokenize/tokenize-normal.test.ts +++ b/src/lib/text-diff/tokenize/tokenize-normal.test.ts @@ -212,4 +212,105 @@ describe("tokenizeText", () => { }, ]); }); + + describe("preserveWhitespace", () => { + it("keeps the whitespace preceding each word", () => { + const tokens = tokenizeNormalText(" const foo = bar; ", { + separation: "word", + preserveWhitespace: true, + }); + + expect(tokens).toEqual([ + { value: " const", normalizedValue: " const", index: 0 }, + { value: " foo", normalizedValue: " foo", index: 1 }, + { value: " =", normalizedValue: " =", index: 2 }, + { value: " bar; ", normalizedValue: " bar; ", index: 3 }, + ]); + }); + + it("keeps the whitespace preceding each character", () => { + const tokens = tokenizeNormalText(" ab", { + separation: "character", + preserveWhitespace: true, + }); + + expect(tokens).toEqual([ + { value: " a", normalizedValue: " a", index: 0 }, + { value: "b", normalizedValue: "b", index: 1 }, + ]); + }); + + it("keeps the whitespace preceding each sentence", () => { + const tokens = tokenizeNormalText(" Hi there. Bye! ", { + separation: "sentence", + preserveWhitespace: true, + }); + + expect(tokens).toEqual([ + { value: " Hi there.", normalizedValue: " Hi there.", index: 0 }, + { value: " Bye! ", normalizedValue: " Bye! ", index: 1 }, + ]); + }); + + it("appends the trailing whitespace to the last token", () => { + const tokens = tokenizeNormalText("a b ", { + separation: "word", + preserveWhitespace: true, + }); + + expect(tokens[tokens.length - 1]).toEqual({ + value: " b ", + normalizedValue: " b ", + index: 1, + }); + }); + + it("returns a single token for a whitespace-only text", () => { + expect( + tokenizeNormalText(" ", { + separation: "word", + preserveWhitespace: true, + }), + ).toEqual([{ value: " ", normalizedValue: " ", index: 0 }]); + }); + + it("returns no token for an empty text", () => { + expect( + tokenizeNormalText("", { + separation: "word", + preserveWhitespace: true, + }), + ).toEqual([]); + }); + + it("rebuilds the original text by concatenating the values", () => { + const text = "\t\tif (a && b) { "; + for (const separation of ["word", "character", "sentence"] as const) { + const tokens = tokenizeNormalText(text, { + separation, + preserveWhitespace: true, + }); + expect(tokens.map((token) => token.value).join("")).toBe(text); + } + }); + + it("still normalizes the token", () => { + const tokens = tokenizeNormalText(" Hello", { + separation: "word", + preserveWhitespace: true, + ignoreCase: true, + }); + + expect(tokens).toEqual([ + { value: " Hello", normalizedValue: " hello", index: 0 }, + ]); + }); + + it("drops the whitespace when disabled", () => { + expect(tokenizeNormalText(" a b", { separation: "word" })).toEqual([ + { value: "a", normalizedValue: "a", index: 0 }, + { value: "b", normalizedValue: "b", index: 1 }, + ]); + }); + }); }); diff --git a/src/lib/text-diff/tokenize/tokenize-strict.test.ts b/src/lib/text-diff/tokenize/tokenize-strict.test.ts index 1720cb1..618b4a9 100644 --- a/src/lib/text-diff/tokenize/tokenize-strict.test.ts +++ b/src/lib/text-diff/tokenize/tokenize-strict.test.ts @@ -228,4 +228,26 @@ describe("tokenizeText", () => { }, ]); }); + + describe("preserveWhitespace", () => { + it("is rejected by the type system, and ignored if forced at runtime", () => { + const forced = { + separation: "word", + preserveWhitespace: true, + } as Parameters[1]; + + expect(tokenizeStrictText(" a b", forced)).toEqual( + tokenizeStrictText(" a b", { separation: "word" }), + ); + }); + + it("never warns", () => { + const warn = jest.spyOn(console, "warn").mockImplementation(() => {}); + + tokenizeStrictText(" a b", { separation: "word" }); + + expect(warn).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + }); }); diff --git a/src/models/text/index.ts b/src/models/text/index.ts index 4ec3815..b935ea6 100644 --- a/src/models/text/index.ts +++ b/src/models/text/index.ts @@ -9,6 +9,7 @@ export const DEFAULT_TEXT_DIFF_OPTIONS: TextDiffOptions = { separation: "word", ignoreCase: false, ignorePunctuation: false, + preserveWhitespace: false, locale: undefined, }; @@ -34,15 +35,29 @@ export enum TextStatus { MOVED = "moved", } -export type TextDiffOptions = { - separation?: "character" | "word" | "sentence"; - accuracy?: "normal" | "high"; +export type TextSeparation = "character" | "word" | "sentence"; + +type TextDiffCommonOptions = { detectMoves?: boolean; ignoreCase?: boolean; ignorePunctuation?: boolean; locale?: Intl.Locale | string; }; +export type TextDiffOptions = TextDiffCommonOptions & + ( + | { + accuracy?: "normal"; + separation?: TextSeparation; + preserveWhitespace?: boolean; + } + | { + accuracy: "high"; + separation?: TextSeparation; + preserveWhitespace?: never; + } + ); + export type TextDiff = { type: "text"; status: "added" | "equal" | "deleted" | "updated";