fix(extraction): a file over the size limit is never read (#1910)

CHANGELOG.md

@@ -172,6 +172,10 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 #### MCP / indexing
 
 - On Windows, the shared MCP daemon now waits longer for another program — an antivirus scan, an indexer, or another session reading its lock file — to let go of that file, so it starts instead of leaving the session to fall back to a slower in-process server. (#1773)
+- An MPEG transport stream video that happens to be named `.ts` (golden fixtures under `testdata/`, e2e clips) is now recognised from its first bytes and skipped as non-source instead of being fed to the TypeScript parser — a 900 KB clip used to cost about 28 seconds of CPU per file for no symbols, and a folder of them minutes. Real TypeScript is never affected. (#1910)
+- A file over the size limit is no longer read before it is skipped: committed video and blob fixtures used to be decoded in full — a 400 MB fixture cost 3.4 GB of memory — only to be discarded, and the same file was read again by every resolution pass. Its size stamp now stands in for its content, during indexing and when checking for changes. (#1910)
+- A file over the size limit is no longer read before it is skipped. Large committed video and blob fixtures used to be loaded into memory in full, once per indexing pass, only to be discarded; indexing, change detection and the viewer now go by the file's size instead. (#1910)
+- An MPEG transport stream video that happens to be named `.ts` (golden fixtures under `testdata/`, e2e clips) is now recognised from its first bytes and skipped instead of being fed to the TypeScript parser, which spent a long time on each clip for no symbols. Real TypeScript files are still indexed. (#1910)
 - File watching no longer drops the full re-scan a removed directory asks for when that sync fails, so the deleted files leave the index instead of lingering. (#1964)
 - Daemon startup and cleanup now preserve live legacy PID-only locks while still reclaiming dead or identity-disproved records, preventing two writers from serving the same project.
 - Incremental sync now keeps edge rebinding crash-safe: replacing a resolved edge with its recovery reference commits atomically, so an interruption cannot permanently remove the relationship.

__tests__/bounded-source.test.ts

@@ -0,0 +1,96 @@
+/**
+ * Source reads are bounded by the read itself, not only by a stat taken
+ * before it (#1910). A file can grow between the stat and the read; the
+ * reader must still never hold more than the limit plus one byte.
+ */
+import { describe, it, expect, afterEach, vi } from 'vitest';
+import * as fs from 'fs';
+import * as fsp from 'fs/promises';
+import * as os from 'os';
+import * as path from 'path';
+import { readBoundedSource, readBoundedSourceSync, MAX_SOURCE_FILE_SIZE_BYTES as LIMIT } from '../src/file-limits';
+
+// Pass-through wrappers, so a test can inject a file growing mid-read into the
+// exact calls file-limits.ts makes.
+vi.mock('fs', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('fs')>();
+  return { ...actual, readSync: vi.fn(actual.readSync) };
+});
+vi.mock('fs/promises', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('fs/promises')>();
+  return { ...actual, open: vi.fn(actual.open) };
+});
+
+const dirs: string[] = [];
+function sourceFile(bytes: Buffer): string {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-bounded-'));
+  dirs.push(dir);
+  const file = path.join(dir, 'source.ts');
+  fs.writeFileSync(file, bytes);
+  return file;
+}
+
+afterEach(() => {
+  for (const dir of dirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true });
+});
+
+describe('bounded source reads (#1910)', () => {
+  for (const size of [0, 100, LIMIT, LIMIT + 1]) {
+    it(`returns the bytes up to the limit and null past it (${size} bytes)`, async () => {
+      const file = sourceFile(Buffer.alloc(size, 0x61));
+      for (const result of [readBoundedSourceSync(file), await readBoundedSource(file)]) {
+        expect(result.stats.size).toBe(size);
+        if (size > LIMIT) expect(result.bytes).toBeNull();
+        else expect(result.bytes?.length).toBe(size);
+      }
+    });
+  }
+
+  it('stops at the limit when the file grows during a synchronous read', () => {
+    const file = sourceFile(Buffer.from('hello'));
+    const readSync = vi.mocked(fs.readSync);
+    const original = readSync.getMockImplementation()!;
+    let requested = 0;
+    let grown = false;
+    readSync.mockImplementation(((fd: number, buf: Buffer, off: number, len: number, pos: number) => {
+      requested += len;
+      if (!grown) {
+        grown = true;
+        fs.writeFileSync(file, Buffer.alloc(LIMIT + 10, 0x61));
+      }
+      return original(fd, buf, off, len, pos);
+    }) as typeof fs.readSync);
+    let result;
+    try { result = readBoundedSourceSync(file); } finally { readSync.mockImplementation(original); }
+    expect(grown).toBe(true);
+    expect(requested).toBeLessThanOrEqual(LIMIT + 1);
+    expect(result.bytes).toBeNull();
+  });
+
+  it('rechecks the open descriptor when the file grows between stat and open', async () => {
+    const file = sourceFile(Buffer.from('hello'));
+    const open = vi.mocked(fsp.open);
+    const original = open.getMockImplementation()!;
+    let grown = false;
+    open.mockImplementationOnce((async (...args: Parameters<typeof fsp.open>) => {
+      grown = true;
+      fs.writeFileSync(file, Buffer.alloc(LIMIT + 1));
+      return original(...args);
+    }) as typeof fsp.open);
+    expect((await readBoundedSource(file)).bytes).toBeNull();
+    expect(grown).toBe(true);
+  });
+
+  it('returns multibyte source byte-for-byte', async () => {
+    const text = 'export const greeting = "こんにちは 🌿";';
+    const file = sourceFile(Buffer.from(text));
+    expect((await readBoundedSource(file)).bytes?.toString('utf8')).toBe(text);
+    expect(readBoundedSourceSync(file).bytes?.toString('utf8')).toBe(text);
+  });
+
+  it('refuses a path that is not a regular file', () => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-bounded-'));
+    dirs.push(dir);
+    expect(() => readBoundedSourceSync(dir)).toThrow(/not a regular file/);
+  });
+});

__tests__/git-index-currency.test.ts

@@ -132,12 +132,18 @@ describe('git index currency across commits and restores (#1829)', () => {
 
   it('keeps a committed path pending when sync cannot read it', async () => {
     write('new.ts', 'newSymbol'); commit();
-    const real = fs.readFileSync;
+    // Sync reads a source file through a bounded reader that opens a
+    // descriptor (#1910), so the failure is injected at openSync as well as
+    // readFileSync: it must reach whichever one the read goes through.
+    const realRead = fs.readFileSync;
+    const realOpen = fs.openSync;
     let injected = 0;
-    vi.spyOn(fs, 'readFileSync').mockImplementation(((file: any, ...args: any[]) => {
+    const failNewTs = (real: (...args: any[]) => unknown) => (file: any, ...args: any[]) => {
       if (String(file) === path.join(root, 'new.ts')) { injected++; throw new Error('Injected transient read error'); }
-      return (real as any)(file, ...args);
-    }) as typeof fs.readFileSync);
+      return real(file, ...args);
+    };
+    vi.spyOn(fs, 'readFileSync').mockImplementation(failNewTs(realRead as any) as typeof fs.readFileSync);
+    vi.spyOn(fs, 'openSync').mockImplementation(failNewTs(realOpen as any) as typeof fs.openSync);
     await cg.sync();
     expect(injected).toBeGreaterThan(0);
     expect(symbols('newSymbol')).not.toContain('newSymbol');

__tests__/mpeg-ts-not-typescript.test.ts

@@ -0,0 +1,230 @@
+/**
+ * An MPEG transport stream named `.ts` is not TypeScript (#1910).
+ *
+ * Golden video fixtures (`testdata/*.ts`) share TypeScript's extension; fed to
+ * the tree-sitter TypeScript parser a 900 KB clip costs ~28 s of CPU for zero
+ * symbols. The fix recognises the stream from the head of the file (0x47 sync
+ * byte at each 188-byte packet boundary, plus a NUL byte no UTF-8 source has)
+ * and drops it at discovery — not indexed, not parsed, not counted, not
+ * reported as an unsupported language.
+ */
+import { describe, it, expect, afterEach } from 'vitest';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+import { execFileSync } from 'child_process';
+import { CodeGraph } from '../src';
+import { scanDirectoryAsync, type ScanSkipStats } from '../src/extraction';
+import { detectLanguage, isMpegTransportStream, MPEG_TS_SNIFF_BYTES } from '../src/extraction/grammars';
+
+const PACKET = 188;
+
+/** A synthetic transport stream: `packets` × 188 bytes, 0x47 then pseudo-random payload. */
+function makeMpegTs(packets: number, seed = 1): Buffer {
+  const buf = Buffer.alloc(packets * PACKET);
+  let x = seed >>> 0;
+  for (let i = 0; i < buf.length; i++) {
+    x = (x * 1664525 + 1013904223) >>> 0;
+    buf[i] = i % PACKET === 0 ? 0x47 : x >>> 24;
+  }
+  // Every real stream opens with PSI tables whose pointer field is 0x00.
+  buf[4] = 0;
+  return buf;
+}
+
+/**
+ * Real TypeScript engineered to put the letter `G` (0x47) at the start of each
+ * of its first `packets` 188-byte strides — the sync-byte pattern alone. With
+ * `nul`, one raw NUL sits inside a comment (the review's counterexample on
+ * #1915). It must stay TypeScript either way.
+ */
+function makeGammaSource(packets = 4, nul = false): string {
+  let text = '';
+  for (let i = 0; i < packets; i++) {
+    const line = `Gamma${i}();` + (nul && i === 1 ? ' // \u0000' : '');
+    const pad = PACKET - line.length - 1;
+    text += line + ' '.repeat(pad) + '\n';
+  }
+  for (let i = 0; i < packets; i++) text += `export function Gamma${i}() { return ${i}; }\n`;
+  text += 'export function realFn() { return 1; }\n';
+  for (let off = 0; off < packets * PACKET; off += PACKET) {
+    if (text.charCodeAt(off) !== 0x47) throw new Error(`fixture: expected G at ${off}`);
+  }
+  return text;
+}
+
+/** A stream that opens the usual way: PAT and PMT packets stuffed with 0xFF, then payload. */
+function makePsiLedMpegTs(packets: number): Buffer {
+  const buf = makeMpegTs(packets, 7);
+  for (const start of [0, PACKET]) {
+    buf.fill(0xff, start + 4, start + PACKET);
+    buf[start + 1] = 0x40;
+    buf[start + 2] = start === 0 ? 0x00 : 0x10;
+    buf[start + 3] = 0x10;
+  }
+  return buf;
+}
+
+const tempDirs: string[] = [];
+function createProject(): string {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-mpegts-'));
+  tempDirs.push(dir);
+  return dir;
+}
+
+afterEach(() => {
+  for (const d of tempDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true });
+});
+
+describe('isMpegTransportStream', () => {
+  it('recognises a transport stream from its head', () => {
+    const ts = makeMpegTs(40);
+    expect(isMpegTransportStream(ts.subarray(0, MPEG_TS_SNIFF_BYTES))).toBe(true);
+    expect(isMpegTransportStream(ts)).toBe(true);
+  });
+
+  it('needs sixteen aligned sync bytes — a head too short, or one packet off, is not video', () => {
+    const ts = makeMpegTs(40);
+    expect(isMpegTransportStream(ts.subarray(0, 15 * PACKET))).toBe(false);
+    for (const packet of [2, 15]) {
+      const broken = Buffer.from(ts);
+      broken[packet * PACKET] = 0x48;
+      expect(isMpegTransportStream(broken)).toBe(false);
+    }
+    expect(isMpegTransportStream(Buffer.alloc(0))).toBe(false);
+  });
+
+  it('recognises a stream that opens with 0xFF-stuffed PAT and PMT packets', () => {
+    expect(isMpegTransportStream(makePsiLedMpegTs(40).subarray(0, MPEG_TS_SNIFF_BYTES))).toBe(true);
+  });
+
+  it('does not take source text with G at every 188th byte for video', () => {
+    const bytes = Buffer.from(makeGammaSource(), 'utf-8');
+    expect(bytes[0]).toBe(0x47);
+    expect(bytes[3 * PACKET]).toBe(0x47);
+    expect(isMpegTransportStream(bytes)).toBe(false);
+    expect(detectLanguage('gamma.ts', makeGammaSource())).toBe('typescript');
+  });
+
+  it('does not take source with G at every stride and a NUL in a comment for video (#1915 review)', () => {
+    for (const packets of [4, 16, 20]) {
+      const bytes = Buffer.from(makeGammaSource(packets, true), 'utf-8');
+      expect(bytes.includes(0)).toBe(true);
+      expect(isMpegTransportStream(bytes.subarray(0, MPEG_TS_SNIFF_BYTES))).toBe(false);
+    }
+  });
+});
+
+describe('MPEG-TS video named .ts is skipped, real TypeScript is indexed (#1910)', () => {
+  it('drops the clip at discovery: no file record, no nodes, no unsupported-language report', async () => {
+    const dir = createProject();
+    fs.mkdirSync(path.join(dir, 'testdata'));
+    fs.writeFileSync(path.join(dir, 'testdata', 'clip.ts'), makeMpegTs(40));
+    fs.writeFileSync(path.join(dir, 'app.ts'), 'export function greet(n: string) { return `hi ${n}`; }\n');
+    fs.writeFileSync(path.join(dir, 'gamma.ts'), makeGammaSource());
+    fs.writeFileSync(path.join(dir, 'gamma-nul.ts'), makeGammaSource(16, true));
+
+    const stats: ScanSkipStats = { unsupportedByExtension: new Map() };
+    const scanned = await scanDirectoryAsync(dir, undefined, stats);
+    expect(scanned.sort()).toEqual(['app.ts', 'gamma-nul.ts', 'gamma.ts']);
+    expect(stats.unsupportedByExtension.size).toBe(0);
+
+    const cg = await CodeGraph.init(dir, { index: true });
+    try {
+      const files = cg.getFiles().map((f) => f.path).sort();
+      expect(files).toEqual(['app.ts', 'gamma-nul.ts', 'gamma.ts']);
+      expect(cg.searchNodes('greet').some((r) => r.node.name === 'greet')).toBe(true);
+      expect(cg.searchNodes('Gamma2').some((r) => r.node.name === 'Gamma2')).toBe(true);
+      // The review's counterexample: G at every stride and a NUL in a comment.
+      expect(cg.searchNodes('realFn').filter((r) => r.node.filePath === 'gamma-nul.ts')).toHaveLength(1);
+
+      // A named re-sync (the watcher / `sync` path hands files in by name) must
+      // not let the clip back in either.
+      await cg.sync({ paths: ['testdata/clip.ts', 'app.ts'] });
+      expect(cg.getFiles().map((f) => f.path).sort()).toEqual(['app.ts', 'gamma-nul.ts', 'gamma.ts']);
+    } finally {
+      await cg.close();
+    }
+  });
+
+  it('reports nothing skipped for a project of only source and video', async () => {
+    const dir = createProject();
+    fs.writeFileSync(path.join(dir, 'clip.ts'), makeMpegTs(40));
+    fs.writeFileSync(path.join(dir, 'app.ts'), 'export const a = 1;\n');
+    const cg = await CodeGraph.init(dir);
+    try {
+      const result = await cg.indexAll();
+      expect(result.filesSkippedUnsupported).toBeUndefined();
+      expect(result.topUnsupportedExtensions).toBeUndefined();
+      expect(result.errors.filter((e) => e.filePath === 'clip.ts')).toEqual([]);
+      expect(cg.getFiles().map((f) => f.path)).toEqual(['app.ts']);
+    } finally {
+      await cg.close();
+    }
+  });
+
+  it('indexes past a 900 KB clip in well under two seconds', async () => {
+    const dir = createProject();
+    fs.mkdirSync(path.join(dir, 'testdata'));
+    fs.writeFileSync(path.join(dir, 'testdata', 'golden.ts'), makeMpegTs(Math.ceil((900 * 1024) / PACKET)));
+    fs.writeFileSync(path.join(dir, 'app.ts'), 'export function greet(n: string) { return `hi ${n}`; }\n');
+    const cg = await CodeGraph.init(dir);
+    try {
+      const t0 = Date.now();
+      const result = await cg.indexAll();
+      const elapsed = Date.now() - t0;
+      expect(result.filesIndexed).toBe(1);
+      expect(cg.getFiles().map((f) => f.path)).toEqual(['app.ts']);
+      expect(elapsed).toBeLessThan(2000);
+    } finally {
+      await cg.close();
+    }
+  });
+});
+
+describe('a video .ts never stays pending (#1910)', () => {
+  const dirs: string[] = [];
+  afterEach(() => { for (const d of dirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }); });
+  const gitProject = (): string => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-mpegts-git-'));
+    dirs.push(dir);
+    const git = (...a: string[]) => execFileSync('git', a, { cwd: dir, stdio: 'pipe' });
+    git('init', '-q'); git('config', 'user.email', 't@t'); git('config', 'user.name', 't');
+    fs.writeFileSync(path.join(dir, 'app.ts'), 'export const a = 1;\n');
+    git('add', '.'); git('commit', '-qm', 'init');
+    return dir;
+  };
+
+  it('an untracked clip is not reported as added, before or after sync', async () => {
+    const dir = gitProject();
+    const cg = await CodeGraph.init(dir, { index: true });
+    try {
+      fs.writeFileSync(path.join(dir, 'clip.ts'), makeMpegTs(40));
+      expect(cg.getChangedFiles()).toEqual({ added: [], modified: [], removed: [] });
+      await cg.sync();
+      expect(cg.getChangedFiles()).toEqual({ added: [], modified: [], removed: [] });
+      expect(cg.getFiles().map((f) => f.path)).toEqual(['app.ts']);
+    } finally {
+      cg.close();
+    }
+  });
+
+  it('a tracked TypeScript file that becomes a clip is removed, on the git path and on a scoped sync', async () => {
+    for (const scoped of [false, true]) {
+      const dir = gitProject();
+      fs.writeFileSync(path.join(dir, 'clip.ts'), 'export const clip = 1;\n');
+      const cg = await CodeGraph.init(dir, { index: true });
+      try {
+        expect(cg.getNodesInFile('clip.ts').some((n) => n.name === 'clip')).toBe(true);
+        fs.writeFileSync(path.join(dir, 'clip.ts'), makeMpegTs(40));
+        if (!scoped) expect(cg.getChangedFiles()).toEqual({ added: [], modified: [], removed: ['clip.ts'] });
+        await (scoped ? cg.sync({ paths: ['clip.ts'] }) : cg.sync());
+        expect(cg.getFiles().map((f) => f.path)).toEqual(['app.ts']);
+        expect(cg.getNodesInFile('clip.ts')).toEqual([]);
+        expect(cg.getChangedFiles()).toEqual({ added: [], modified: [], removed: [] });
+      } finally {
+        cg.close();
+      }
+    }
+  });
+});