fix(extraction): an MPEG-TS video named .ts is not TypeScript (#1910)

CHANGELOG.md

@@ -172,6 +172,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 #### MCP / indexing
 
 - On Windows, the shared MCP daemon now waits longer for another program — an antivirus scan, an indexer, or another session reading its lock file — to let go of that file, so it starts instead of leaving the session to fall back to a slower in-process server. (#1773)
+- An MPEG transport stream video that happens to be named `.ts` (golden fixtures under `testdata/`, e2e clips) is now recognised from its first bytes and skipped as non-source instead of being fed to the TypeScript parser — a 900 KB clip used to cost about 28 seconds of CPU per file for no symbols, and a folder of them minutes. Real TypeScript is never affected. (#1910)
+- An MPEG transport stream video that happens to be named `.ts` (golden fixtures under `testdata/`, e2e clips) is now recognised from its first bytes and skipped instead of being fed to the TypeScript parser, which spent a long time on each clip for no symbols. Real TypeScript files are still indexed. (#1910)
 - File watching no longer drops the full re-scan a removed directory asks for when that sync fails, so the deleted files leave the index instead of lingering. (#1964)
 - Daemon startup and cleanup now preserve live legacy PID-only locks while still reclaiming dead or identity-disproved records, preventing two writers from serving the same project.
 - Incremental sync now keeps edge rebinding crash-safe: replacing a resolved edge with its recovery reference commits atomically, so an interruption cannot permanently remove the relationship.

__tests__/mpeg-ts-not-typescript.test.ts

@@ -0,0 +1,230 @@
+/**
+ * An MPEG transport stream named `.ts` is not TypeScript (#1910).
+ *
+ * Golden video fixtures (`testdata/*.ts`) share TypeScript's extension; fed to
+ * the tree-sitter TypeScript parser a 900 KB clip costs ~28 s of CPU for zero
+ * symbols. The fix recognises the stream from the head of the file (0x47 sync
+ * byte at each 188-byte packet boundary, plus a NUL byte no UTF-8 source has)
+ * and drops it at discovery — not indexed, not parsed, not counted, not
+ * reported as an unsupported language.
+ */
+import { describe, it, expect, afterEach } from 'vitest';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+import { execFileSync } from 'child_process';
+import { CodeGraph } from '../src';
+import { scanDirectoryAsync, type ScanSkipStats } from '../src/extraction';
+import { detectLanguage, isMpegTransportStream, MPEG_TS_SNIFF_BYTES } from '../src/extraction/grammars';
+
+const PACKET = 188;
+
+/** A synthetic transport stream: `packets` × 188 bytes, 0x47 then pseudo-random payload. */
+function makeMpegTs(packets: number, seed = 1): Buffer {
+  const buf = Buffer.alloc(packets * PACKET);
+  let x = seed >>> 0;
+  for (let i = 0; i < buf.length; i++) {
+    x = (x * 1664525 + 1013904223) >>> 0;
+    buf[i] = i % PACKET === 0 ? 0x47 : x >>> 24;
+  }
+  // Every real stream opens with PSI tables whose pointer field is 0x00.
+  buf[4] = 0;
+  return buf;
+}
+
+/**
+ * Real TypeScript engineered to put the letter `G` (0x47) at the start of each
+ * of its first `packets` 188-byte strides — the sync-byte pattern alone. With
+ * `nul`, one raw NUL sits inside a comment (the review's counterexample on
+ * #1915). It must stay TypeScript either way.
+ */
+function makeGammaSource(packets = 4, nul = false): string {
+  let text = '';
+  for (let i = 0; i < packets; i++) {
+    const line = `Gamma${i}();` + (nul && i === 1 ? ' // \u0000' : '');
+    const pad = PACKET - line.length - 1;
+    text += line + ' '.repeat(pad) + '\n';
+  }
+  for (let i = 0; i < packets; i++) text += `export function Gamma${i}() { return ${i}; }\n`;
+  text += 'export function realFn() { return 1; }\n';
+  for (let off = 0; off < packets * PACKET; off += PACKET) {
+    if (text.charCodeAt(off) !== 0x47) throw new Error(`fixture: expected G at ${off}`);
+  }
+  return text;
+}
+
+/** A stream that opens the usual way: PAT and PMT packets stuffed with 0xFF, then payload. */
+function makePsiLedMpegTs(packets: number): Buffer {
+  const buf = makeMpegTs(packets, 7);
+  for (const start of [0, PACKET]) {
+    buf.fill(0xff, start + 4, start + PACKET);
+    buf[start + 1] = 0x40;
+    buf[start + 2] = start === 0 ? 0x00 : 0x10;
+    buf[start + 3] = 0x10;
+  }
+  return buf;
+}
+
+const tempDirs: string[] = [];
+function createProject(): string {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-mpegts-'));
+  tempDirs.push(dir);
+  return dir;
+}
+
+afterEach(() => {
+  for (const d of tempDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true });
+});
+
+describe('isMpegTransportStream', () => {
+  it('recognises a transport stream from its head', () => {
+    const ts = makeMpegTs(40);
+    expect(isMpegTransportStream(ts.subarray(0, MPEG_TS_SNIFF_BYTES))).toBe(true);
+    expect(isMpegTransportStream(ts)).toBe(true);
+  });
+
+  it('needs sixteen aligned sync bytes — a head too short, or one packet off, is not video', () => {
+    const ts = makeMpegTs(40);
+    expect(isMpegTransportStream(ts.subarray(0, 15 * PACKET))).toBe(false);
+    for (const packet of [2, 15]) {
+      const broken = Buffer.from(ts);
+      broken[packet * PACKET] = 0x48;
+      expect(isMpegTransportStream(broken)).toBe(false);
+    }
+    expect(isMpegTransportStream(Buffer.alloc(0))).toBe(false);
+  });
+
+  it('recognises a stream that opens with 0xFF-stuffed PAT and PMT packets', () => {
+    expect(isMpegTransportStream(makePsiLedMpegTs(40).subarray(0, MPEG_TS_SNIFF_BYTES))).toBe(true);
+  });
+
+  it('does not take source text with G at every 188th byte for video', () => {
+    const bytes = Buffer.from(makeGammaSource(), 'utf-8');
+    expect(bytes[0]).toBe(0x47);
+    expect(bytes[3 * PACKET]).toBe(0x47);
+    expect(isMpegTransportStream(bytes)).toBe(false);
+    expect(detectLanguage('gamma.ts', makeGammaSource())).toBe('typescript');
+  });
+
+  it('does not take source with G at every stride and a NUL in a comment for video (#1915 review)', () => {
+    for (const packets of [4, 16, 20]) {
+      const bytes = Buffer.from(makeGammaSource(packets, true), 'utf-8');
+      expect(bytes.includes(0)).toBe(true);
+      expect(isMpegTransportStream(bytes.subarray(0, MPEG_TS_SNIFF_BYTES))).toBe(false);
+    }
+  });
+});
+
+describe('MPEG-TS video named .ts is skipped, real TypeScript is indexed (#1910)', () => {
+  it('drops the clip at discovery: no file record, no nodes, no unsupported-language report', async () => {
+    const dir = createProject();
+    fs.mkdirSync(path.join(dir, 'testdata'));
+    fs.writeFileSync(path.join(dir, 'testdata', 'clip.ts'), makeMpegTs(40));
+    fs.writeFileSync(path.join(dir, 'app.ts'), 'export function greet(n: string) { return `hi ${n}`; }\n');
+    fs.writeFileSync(path.join(dir, 'gamma.ts'), makeGammaSource());
+    fs.writeFileSync(path.join(dir, 'gamma-nul.ts'), makeGammaSource(16, true));
+
+    const stats: ScanSkipStats = { unsupportedByExtension: new Map() };
+    const scanned = await scanDirectoryAsync(dir, undefined, stats);
+    expect(scanned.sort()).toEqual(['app.ts', 'gamma-nul.ts', 'gamma.ts']);
+    expect(stats.unsupportedByExtension.size).toBe(0);
+
+    const cg = await CodeGraph.init(dir, { index: true });
+    try {
+      const files = cg.getFiles().map((f) => f.path).sort();
+      expect(files).toEqual(['app.ts', 'gamma-nul.ts', 'gamma.ts']);
+      expect(cg.searchNodes('greet').some((r) => r.node.name === 'greet')).toBe(true);
+      expect(cg.searchNodes('Gamma2').some((r) => r.node.name === 'Gamma2')).toBe(true);
+      // The review's counterexample: G at every stride and a NUL in a comment.
+      expect(cg.searchNodes('realFn').filter((r) => r.node.filePath === 'gamma-nul.ts')).toHaveLength(1);
+
+      // A named re-sync (the watcher / `sync` path hands files in by name) must
+      // not let the clip back in either.
+      await cg.sync({ paths: ['testdata/clip.ts', 'app.ts'] });
+      expect(cg.getFiles().map((f) => f.path).sort()).toEqual(['app.ts', 'gamma-nul.ts', 'gamma.ts']);
+    } finally {
+      await cg.close();
+    }
+  });
+
+  it('reports nothing skipped for a project of only source and video', async () => {
+    const dir = createProject();
+    fs.writeFileSync(path.join(dir, 'clip.ts'), makeMpegTs(40));
+    fs.writeFileSync(path.join(dir, 'app.ts'), 'export const a = 1;\n');
+    const cg = await CodeGraph.init(dir);
+    try {
+      const result = await cg.indexAll();
+      expect(result.filesSkippedUnsupported).toBeUndefined();
+      expect(result.topUnsupportedExtensions).toBeUndefined();
+      expect(result.errors.filter((e) => e.filePath === 'clip.ts')).toEqual([]);
+      expect(cg.getFiles().map((f) => f.path)).toEqual(['app.ts']);
+    } finally {
+      await cg.close();
+    }
+  });
+
+  it('indexes past a 900 KB clip in well under two seconds', async () => {
+    const dir = createProject();
+    fs.mkdirSync(path.join(dir, 'testdata'));
+    fs.writeFileSync(path.join(dir, 'testdata', 'golden.ts'), makeMpegTs(Math.ceil((900 * 1024) / PACKET)));
+    fs.writeFileSync(path.join(dir, 'app.ts'), 'export function greet(n: string) { return `hi ${n}`; }\n');
+    const cg = await CodeGraph.init(dir);
+    try {
+      const t0 = Date.now();
+      const result = await cg.indexAll();
+      const elapsed = Date.now() - t0;
+      expect(result.filesIndexed).toBe(1);
+      expect(cg.getFiles().map((f) => f.path)).toEqual(['app.ts']);
+      expect(elapsed).toBeLessThan(2000);
+    } finally {
+      await cg.close();
+    }
+  });
+});
+
+describe('a video .ts never stays pending (#1910)', () => {
+  const dirs: string[] = [];
+  afterEach(() => { for (const d of dirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }); });
+  const gitProject = (): string => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-mpegts-git-'));
+    dirs.push(dir);
+    const git = (...a: string[]) => execFileSync('git', a, { cwd: dir, stdio: 'pipe' });
+    git('init', '-q'); git('config', 'user.email', 't@t'); git('config', 'user.name', 't');
+    fs.writeFileSync(path.join(dir, 'app.ts'), 'export const a = 1;\n');
+    git('add', '.'); git('commit', '-qm', 'init');
+    return dir;
+  };
+
+  it('an untracked clip is not reported as added, before or after sync', async () => {
+    const dir = gitProject();
+    const cg = await CodeGraph.init(dir, { index: true });
+    try {
+      fs.writeFileSync(path.join(dir, 'clip.ts'), makeMpegTs(40));
+      expect(cg.getChangedFiles()).toEqual({ added: [], modified: [], removed: [] });
+      await cg.sync();
+      expect(cg.getChangedFiles()).toEqual({ added: [], modified: [], removed: [] });
+      expect(cg.getFiles().map((f) => f.path)).toEqual(['app.ts']);
+    } finally {
+      cg.close();
+    }
+  });
+
+  it('a tracked TypeScript file that becomes a clip is removed, on the git path and on a scoped sync', async () => {
+    for (const scoped of [false, true]) {
+      const dir = gitProject();
+      fs.writeFileSync(path.join(dir, 'clip.ts'), 'export const clip = 1;\n');
+      const cg = await CodeGraph.init(dir, { index: true });
+      try {
+        expect(cg.getNodesInFile('clip.ts').some((n) => n.name === 'clip')).toBe(true);
+        fs.writeFileSync(path.join(dir, 'clip.ts'), makeMpegTs(40));
+        if (!scoped) expect(cg.getChangedFiles()).toEqual({ added: [], modified: [], removed: ['clip.ts'] });
+        await (scoped ? cg.sync({ paths: ['clip.ts'] }) : cg.sync());
+        expect(cg.getFiles().map((f) => f.path)).toEqual(['app.ts']);
+        expect(cg.getNodesInFile('clip.ts')).toEqual([]);
+        expect(cg.getChangedFiles()).toEqual({ added: [], modified: [], removed: [] });
+      } finally {
+        cg.close();
+      }
+    }
+  });
+});

src/extraction/grammars.ts

@@ -172,6 +172,67 @@ export const EXTENSION_MAP: Record<string, Language> = {
   '.tofu': 'terraform',
 };
 
+/** MPEG transport stream: fixed 188-byte packets, each opening with 0x47. */
+const MPEG_TS_PACKET_SIZE = 188;
+const MPEG_TS_SYNC_BYTE = 0x47;
+/**
+ * Consecutive packets whose sync byte must line up before a file counts as
+ * video — 3 KB of head. A stream shorter than that is cheap to parse anyway;
+ * the cost #1910 is about comes from clips hundreds of KB long.
+ */
+const MPEG_TS_MIN_PACKETS = 16;
+/**
+ * Share of the head that must be control bytes (below 0x20, other than the
+ * whitespace ones) for it to count as binary. Compressed audio and video put
+ * about one byte in eight there; source text puts none.
+ */
+const MPEG_TS_MIN_CONTROL_SHARE = 1 / 64;
+/**
+ * How many bytes of a file's head `isMpegTransportStream` needs — enough to
+ * see `MPEG_TS_MIN_PACKETS` sync bytes plus the packets between them.
+ */
+export const MPEG_TS_SNIFF_BYTES = MPEG_TS_PACKET_SIZE * MPEG_TS_MIN_PACKETS;
+
+/**
+ * Whether these leading bytes are an MPEG transport stream — the OTHER thing a
+ * `.ts` file can be. Golden video fixtures (`testdata/*.ts`, e2e clips) share
+ * TypeScript's extension, and tree-sitter takes ~28 s to chew through a 900 KB
+ * clip for zero symbols (#1910), so the decision has to be made from the head
+ * of the file, before any parse.
+ *
+ * Two conditions, both required:
+ *   1. the sync byte 0x47 sits at every 188-byte packet boundary of the first
+ *      `MPEG_TS_MIN_PACKETS` packets — every packet of a transport stream
+ *      opens with it, and nothing else pads to 188;
+ *   2. the head is binary: at least `MPEG_TS_MIN_CONTROL_SHARE` of it is
+ *      control bytes, as any compressed payload is.
+ * 0x47 is the letter `G`, so (1) alone could match source whose lines happen
+ * to put a `G` at every 188-byte stride. Checking for a single NUL was not
+ * enough to close that: one NUL in a comment is still TypeScript. (2) asks for
+ * dozens of control bytes, which no source file carries.
+ *
+ * `head` is the first `MPEG_TS_SNIFF_BYTES` (or fewer) bytes of the file.
+ */
+export function isMpegTransportStream(head: Uint8Array): boolean {
+  const lastSync = MPEG_TS_PACKET_SIZE * (MPEG_TS_MIN_PACKETS - 1);
+  if (head.length <= lastSync) return false;
+  for (let off = 0; off <= lastSync; off += MPEG_TS_PACKET_SIZE) {
+    if (head[off] !== MPEG_TS_SYNC_BYTE) return false;
+  }
+  let control = 0;
+  for (let i = 0; i < head.length; i++) {
+    const b = head[i]!;
+    // Tab, newline, vertical tab, form feed and carriage return are text.
+    if (b < 0x20 && (b < 0x09 || b > 0x0d)) control++;
+  }
+  return control >= head.length * MPEG_TS_MIN_CONTROL_SHARE;
+}
+
+/** Whether `filePath` carries the one extension MPEG-TS shares with a language. */
+export function hasMpegTsExtension(filePath: string): boolean {
+  return filePath.length > 3 && filePath.slice(-3).toLowerCase() === '.ts';
+}
+
 /**
  * Whether a file is one CodeGraph can parse, based purely on its extension.
  * This is the single source of truth for "should we index this file" — derived