From 260d095f6ff2690c712ce67ff1e52194c07c116e Mon Sep 17 00:00:00 2001 From: danusha2345 Date: Tue, 22 Sep 2026 18:01:22 +0300 Subject: [PATCH 1/3] fix(extraction): an MPEG-TS video named .ts is not TypeScript (#1910) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `.ts` mapped to TypeScript by extension alone, so MPEG transport-stream fixtures (`testdata/*.ts`) were parsed by tree-sitter: ~28 s of CPU for a 900 KB clip, minutes for a directory of them, for zero symbols. Recognise the stream from the head of the file — the 0x47 sync byte at offsets 0, 188, 376 and 564 plus a NUL byte, which every stream carries in its first packets and UTF-8 source never does — and drop it at discovery: not indexed, not parsed, not counted, not tallied as an unsupported language. The check reads under 1 KB and only for `.ts` files; the batch reader sniffs the bytes it already read, and the single-file path (sync, watcher) does the same head read, so a clip handed in by name is skipped too. The skip happens before parse dispatch, so no kernel mirror is needed. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 1 + __tests__/mpeg-ts-not-typescript.test.ts | 156 +++++++++++++++++++++++ src/extraction/grammars.ts | 47 +++++++ src/extraction/index.ts | 58 ++++++++- 4 files changed, 259 insertions(+), 3 deletions(-) create mode 100644 __tests__/mpeg-ts-not-typescript.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index e7559656f7..d9880d7bcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -172,6 +172,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). #### MCP / indexing - On Windows, the shared MCP daemon now waits longer for another program — an antivirus scan, an indexer, or another session reading its lock file — to let go of that file, so it starts instead of leaving the session to fall back to a slower in-process server. (#1773) +- An MPEG transport stream video that happens to be named `.ts` (golden fixtures under `testdata/`, e2e clips) is now recognised from its first bytes and skipped as non-source instead of being fed to the TypeScript parser — a 900 KB clip used to cost about 28 seconds of CPU per file for no symbols, and a folder of them minutes. Real TypeScript is never affected. (#1910) - File watching no longer drops the full re-scan a removed directory asks for when that sync fails, so the deleted files leave the index instead of lingering. (#1964) - Daemon startup and cleanup now preserve live legacy PID-only locks while still reclaiming dead or identity-disproved records, preventing two writers from serving the same project. - Incremental sync now keeps edge rebinding crash-safe: replacing a resolved edge with its recovery reference commits atomically, so an interruption cannot permanently remove the relationship. diff --git a/__tests__/mpeg-ts-not-typescript.test.ts b/__tests__/mpeg-ts-not-typescript.test.ts new file mode 100644 index 0000000000..2a6e12870b --- /dev/null +++ b/__tests__/mpeg-ts-not-typescript.test.ts @@ -0,0 +1,156 @@ +/** + * An MPEG transport stream named `.ts` is not TypeScript (#1910). + * + * Golden video fixtures (`testdata/*.ts`) share TypeScript's extension; fed to + * the tree-sitter TypeScript parser a 900 KB clip costs ~28 s of CPU for zero + * symbols. The fix recognises the stream from the head of the file (0x47 sync + * byte at each 188-byte packet boundary, plus a NUL byte no UTF-8 source has) + * and drops it at discovery — not indexed, not parsed, not counted, not + * reported as an unsupported language. + */ +import { describe, it, expect, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { CodeGraph } from '../src'; +import { scanDirectoryAsync, type ScanSkipStats } from '../src/extraction'; +import { detectLanguage, isMpegTransportStream, MPEG_TS_SNIFF_BYTES } from '../src/extraction/grammars'; + +const PACKET = 188; + +/** A synthetic transport stream: `packets` × 188 bytes, 0x47 then pseudo-random payload. */ +function makeMpegTs(packets: number, seed = 1): Buffer { + const buf = Buffer.alloc(packets * PACKET); + let x = seed >>> 0; + for (let i = 0; i < buf.length; i++) { + x = (x * 1664525 + 1013904223) >>> 0; + buf[i] = i % PACKET === 0 ? 0x47 : x >>> 24; + } + // Every real stream opens with PSI tables whose pointer field is 0x00. + buf[4] = 0; + return buf; +} + +/** + * Real TypeScript engineered to put the letter `G` (0x47) at offsets 0, 188, + * 376 and 564 — the sync-byte pattern alone. It must stay TypeScript. + */ +function makeGammaSource(): string { + const lines: string[] = []; + let text = ''; + for (let i = 0; i < 4; i++) { + const line = `Gamma${i}();`; + const pad = PACKET - line.length - 1; + text += line + ' '.repeat(pad) + '\n'; + lines.push(line); + } + text += 'export function Gamma0() { return 0; }\n'; + text += 'export function Gamma1() { return 1; }\n'; + text += 'export function Gamma2() { return 2; }\n'; + text += 'export function Gamma3() { return 3; }\n'; + for (const off of [0, PACKET, 2 * PACKET, 3 * PACKET]) { + if (text.charCodeAt(off) !== 0x47) throw new Error(`fixture: expected G at ${off}`); + } + void lines; + return text; +} + +const tempDirs: string[] = []; +function createProject(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-mpegts-')); + tempDirs.push(dir); + return dir; +} + +afterEach(() => { + for (const d of tempDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }); +}); + +describe('isMpegTransportStream', () => { + it('recognises a transport stream from its head', () => { + const ts = makeMpegTs(40); + expect(isMpegTransportStream(ts.subarray(0, MPEG_TS_SNIFF_BYTES))).toBe(true); + expect(isMpegTransportStream(ts)).toBe(true); + }); + + it('needs four aligned sync bytes — a head too short, or one packet off, is not video', () => { + const ts = makeMpegTs(40); + expect(isMpegTransportStream(ts.subarray(0, 3 * PACKET))).toBe(false); + const broken = Buffer.from(ts); + broken[2 * PACKET] = 0x48; + expect(isMpegTransportStream(broken)).toBe(false); + expect(isMpegTransportStream(Buffer.alloc(0))).toBe(false); + }); + + it('does not take source text with G at every 188th byte for video', () => { + const bytes = Buffer.from(makeGammaSource(), 'utf-8'); + expect(bytes[0]).toBe(0x47); + expect(bytes[3 * PACKET]).toBe(0x47); + expect(isMpegTransportStream(bytes)).toBe(false); + expect(detectLanguage('gamma.ts', makeGammaSource())).toBe('typescript'); + }); +}); + +describe('MPEG-TS video named .ts is skipped, real TypeScript is indexed (#1910)', () => { + it('drops the clip at discovery: no file record, no nodes, no unsupported-language report', async () => { + const dir = createProject(); + fs.mkdirSync(path.join(dir, 'testdata')); + fs.writeFileSync(path.join(dir, 'testdata', 'clip.ts'), makeMpegTs(40)); + fs.writeFileSync(path.join(dir, 'app.ts'), 'export function greet(n: string) { return `hi ${n}`; }\n'); + fs.writeFileSync(path.join(dir, 'gamma.ts'), makeGammaSource()); + + const stats: ScanSkipStats = { unsupportedByExtension: new Map() }; + const scanned = await scanDirectoryAsync(dir, undefined, stats); + expect(scanned.sort()).toEqual(['app.ts', 'gamma.ts']); + expect(stats.unsupportedByExtension.size).toBe(0); + + const cg = await CodeGraph.init(dir, { index: true }); + try { + const files = cg.getFiles().map((f) => f.path).sort(); + expect(files).toEqual(['app.ts', 'gamma.ts']); + expect(cg.searchNodes('greet').some((r) => r.node.name === 'greet')).toBe(true); + expect(cg.searchNodes('Gamma2').some((r) => r.node.name === 'Gamma2')).toBe(true); + + // A named re-sync (the watcher / `sync` path hands files in by name) must + // not let the clip back in either. + await cg.sync({ paths: ['testdata/clip.ts', 'app.ts'] }); + expect(cg.getFiles().map((f) => f.path).sort()).toEqual(['app.ts', 'gamma.ts']); + } finally { + await cg.close(); + } + }); + + it('reports nothing skipped for a project of only source and video', async () => { + const dir = createProject(); + fs.writeFileSync(path.join(dir, 'clip.ts'), makeMpegTs(40)); + fs.writeFileSync(path.join(dir, 'app.ts'), 'export const a = 1;\n'); + const cg = await CodeGraph.init(dir); + try { + const result = await cg.indexAll(); + expect(result.filesSkippedUnsupported).toBeUndefined(); + expect(result.topUnsupportedExtensions).toBeUndefined(); + expect(result.errors.filter((e) => e.filePath === 'clip.ts')).toEqual([]); + expect(cg.getFiles().map((f) => f.path)).toEqual(['app.ts']); + } finally { + await cg.close(); + } + }); + + it('indexes past a 900 KB clip in well under two seconds', async () => { + const dir = createProject(); + fs.mkdirSync(path.join(dir, 'testdata')); + fs.writeFileSync(path.join(dir, 'testdata', 'golden.ts'), makeMpegTs(Math.ceil((900 * 1024) / PACKET))); + fs.writeFileSync(path.join(dir, 'app.ts'), 'export function greet(n: string) { return `hi ${n}`; }\n'); + const cg = await CodeGraph.init(dir); + try { + const t0 = Date.now(); + const result = await cg.indexAll(); + const elapsed = Date.now() - t0; + expect(result.filesIndexed).toBe(1); + expect(cg.getFiles().map((f) => f.path)).toEqual(['app.ts']); + expect(elapsed).toBeLessThan(2000); + } finally { + await cg.close(); + } + }); +}); diff --git a/src/extraction/grammars.ts b/src/extraction/grammars.ts index c7710f2007..d75180e4e9 100644 --- a/src/extraction/grammars.ts +++ b/src/extraction/grammars.ts @@ -172,6 +172,53 @@ export const EXTENSION_MAP: Record = { '.tofu': 'terraform', }; +/** MPEG transport stream: fixed 188-byte packets, each opening with 0x47. */ +const MPEG_TS_PACKET_SIZE = 188; +const MPEG_TS_SYNC_BYTE = 0x47; +/** Consecutive packets whose sync byte must line up before a file counts as video. */ +const MPEG_TS_MIN_PACKETS = 4; +/** + * How many bytes of a file's head `isMpegTransportStream` needs — enough to + * see `MPEG_TS_MIN_PACKETS` sync bytes plus the packets between them. + */ +export const MPEG_TS_SNIFF_BYTES = MPEG_TS_PACKET_SIZE * MPEG_TS_MIN_PACKETS; + +/** + * Whether these leading bytes are an MPEG transport stream — the OTHER thing a + * `.ts` file can be. Golden video fixtures (`testdata/*.ts`, e2e clips) share + * TypeScript's extension, and tree-sitter takes ~28 s to chew through a 900 KB + * clip for zero symbols (#1910), so the decision has to be made from the head + * of the file, before any parse. + * + * Two conditions, both required: + * 1. the sync byte 0x47 sits at offsets 0, 188, 376 and 564 — every packet + * of a transport stream opens with it, and nothing else pads to 188; + * 2. a NUL byte appears somewhere in the head — every stream carries one + * within its first packets (the PSI pointer field, table reserved bits, + * the `00 00 01` PES start codes), and UTF-8 source text never does. + * 0x47 is the letter `G`, so (1) alone could in principle match a source file + * whose lines happen to put a `G` at four 188-byte strides; (2) closes that + * door, because a text file with a NUL in it is not TypeScript either. + * + * `head` is the first `MPEG_TS_SNIFF_BYTES` (or fewer) bytes of the file. + */ +export function isMpegTransportStream(head: Uint8Array): boolean { + const lastSync = MPEG_TS_PACKET_SIZE * (MPEG_TS_MIN_PACKETS - 1); + if (head.length <= lastSync) return false; + for (let off = 0; off <= lastSync; off += MPEG_TS_PACKET_SIZE) { + if (head[off] !== MPEG_TS_SYNC_BYTE) return false; + } + for (let i = 0; i < head.length; i++) { + if (head[i] === 0) return true; + } + return false; +} + +/** Whether `filePath` carries the one extension MPEG-TS shares with a language. */ +export function hasMpegTsExtension(filePath: string): boolean { + return filePath.length > 3 && filePath.slice(-3).toLowerCase() === '.ts'; +} + /** * Whether a file is one CodeGraph can parse, based purely on its extension. * This is the single source of truth for "should we index this file" — derived diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 399f333acf..c456f971fe 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -26,7 +26,7 @@ import { ParseWorkerPool, resolveParsePoolSize, resolveParseTimeoutMs } from './ import { StoreWriter, StoreBundle, finalizeStoreBundle } from './store-writer'; import { materializeKernelResult } from './kernel'; import { detectGeneratedFile } from './generated-detection'; -import { detectLanguage, isSourceFile, isLanguageSupported, isFileLevelOnlyLanguage, initGrammars, loadGrammarsForLanguages, readGrammarWasmBytes } from './grammars'; +import { detectLanguage, isSourceFile, isLanguageSupported, isFileLevelOnlyLanguage, initGrammars, loadGrammarsForLanguages, readGrammarWasmBytes, isMpegTransportStream, hasMpegTsExtension, MPEG_TS_SNIFF_BYTES } from './grammars'; import { loadExtensionOverrides, loadIncludeIgnoredPatterns, loadExcludePatterns, loadIncludePatterns, PROJECT_CONFIG_FILENAME } from '../project-config'; import { isCodeGraphDataDir } from '../directory'; import { logDebug, logWarn } from '../errors'; @@ -566,6 +566,7 @@ function collectIncludedFiles( if (!include.ignores(rel)) return; if (exclude && exclude.ignores(rel)) return; if (!isSourceFile(rel, overrides)) return; + if (isMpegTsVideoFile(rootDir, rel)) return; out.add(rel); } }; @@ -1504,6 +1505,7 @@ export function scanDirectory( let count = 0; for (const filePath of gitFiles) { if (isSourceFile(filePath, overrides)) { + if (isMpegTsVideoFile(rootDir, filePath)) continue; files.push(filePath); count++; onProgress?.(count, filePath); @@ -1520,6 +1522,31 @@ export function scanDirectory( * Async variant of scanDirectory that yields to the event loop periodically, * allowing worker threads to receive and render progress messages. */ +/** + * Whether a `.ts` file on disk is an MPEG transport stream rather than + * TypeScript (#1910). Reads only the file's head — `MPEG_TS_SNIFF_BYTES`, under + * 1 KB — so the check costs one small read per `.ts` file at discovery, never a + * whole-file read; any file the extension does not make ambiguous is not + * touched at all. A file that cannot be read is left to the indexing path, + * which reports the read error itself. + */ +function isMpegTsVideoFile(rootDir: string, relativePath: string): boolean { + if (!hasMpegTsExtension(relativePath)) return false; + let fd: number | null = null; + try { + fd = fs.openSync(path.join(rootDir, relativePath), 'r'); + const head = Buffer.allocUnsafe(MPEG_TS_SNIFF_BYTES); + const n = fs.readSync(fd, head, 0, head.length, 0); + if (!isMpegTransportStream(head.subarray(0, n))) return false; + } catch { + return false; + } finally { + if (fd !== null) fs.closeSync(fd); + } + logDebug('Skipping MPEG transport stream named .ts — not TypeScript', { filePath: relativePath }); + return true; +} + /** * What a scan saw but could not index, tallied by extension. * @@ -1554,6 +1581,7 @@ export async function scanDirectoryAsync( let count = 0; for (const filePath of gitFiles) { if (isSourceFile(filePath, overrides)) { + if (isMpegTsVideoFile(rootDir, filePath)) continue; files.push(filePath); count++; onProgress?.(count, filePath); @@ -1662,6 +1690,7 @@ function scanDirectoryWalk( } else if (stat.isFile()) { if (!isIgnored(fullPath, false, active)) { if (isSourceFile(relativePath, overrides)) { + if (isMpegTsVideoFile(rootDir, relativePath)) continue; files.push(relativePath); count++; onProgress?.(count, relativePath); @@ -1683,6 +1712,7 @@ function scanDirectoryWalk( } else if (entry.isFile()) { if (!isIgnored(fullPath, false, active)) { if (isSourceFile(relativePath, overrides)) { + if (isMpegTsVideoFile(rootDir, relativePath)) continue; files.push(relativePath); count++; onProgress?.(count, relativePath); @@ -2263,7 +2293,16 @@ export class ExtractionOrchestrator { logWarn('Path traversal blocked in batch reader', { filePath: fp }); return { filePath: fp, content: null as string | null, stats: null as fs.Stats | null, error: new Error('Path traversal blocked') }; } - const content = await fsp.readFile(fullPath, 'utf-8'); + // Read bytes, not text: a `.ts` that is really an MPEG transport + // stream (#1910) is recognised from its head here, at no extra I/O, + // and never decoded or parsed. The scan already drops these; this + // guards the paths that hand files in by name (sync, watcher). + const bytes = await fsp.readFile(fullPath); + if (hasMpegTsExtension(fp) && isMpegTransportStream(bytes.subarray(0, MPEG_TS_SNIFF_BYTES))) { + logDebug('Skipping MPEG transport stream named .ts — not TypeScript', { filePath: fp }); + return { filePath: fp, content: null as string | null, stats: null as fs.Stats | null, error: null as Error | null, skipped: true }; + } + const content = bytes.toString('utf-8'); const stats = await fsp.stat(fullPath); return { filePath: fp, content, stats, error: null as Error | null }; } catch (err) { @@ -2274,9 +2313,16 @@ export class ExtractionOrchestrator { // Dispatch each readable file into the bounded parse window; the window // stores results on the main thread as they arrive. - for (const { filePath, content, stats, error } of fileContents) { + for (const { filePath, content, stats, error, skipped } of fileContents) { if (signal?.aborted) { aborted = true; break; } + if (skipped) { + // Not a source file after all — counted as done, stored as nothing. + processed++; + onProgress?.({ phase: 'parsing', current: processed, total }); + continue; + } + if (error || content === null || stats === null) { processed++; filesErrored++; @@ -2630,6 +2676,12 @@ export class ExtractionOrchestrator { }; } + // An MPEG transport stream named `.ts` is not TypeScript (#1910): one + // sub-KB head read decides it, before the language lookup and the parse. + if (isMpegTsVideoFile(this.rootDir, relativePath)) { + return { nodes: [], edges: [], unresolvedReferences: [], errors: [], durationMs: 0 }; + } + const language = detectLanguage(relativePath, content, loadExtensionOverrides(this.rootDir)); // Check file size From b4e28d16e786c356af8beac5d1de12d8f362a0e4 Mon Sep 17 00:00:00 2001 From: danusha2345 Date: Tue, 22 Sep 2026 20:23:56 +0300 Subject: [PATCH 2/3] fix(extraction): a video .ts is never pending on the git path or a scoped sync Review found two places that decide "is this a source file" without the MPEG-TS check the scan applies: the git fast path's candidate loop in getChangedFiles and the scoped-sync path filter (watcher events). An untracked clip in a git repo was reported as `added`, skipped by sync without a record, and reported as `added` again on every status; a tracked .ts that became a clip stayed `modified` with its stale nodes. Both now treat the clip as non-source: removed when tracked, ignored otherwise. Regression tests fail without this change and pass with it, on both arms. Co-Authored-By: Claude Opus 5.5 --- __tests__/mpeg-ts-not-typescript.test.ts | 48 ++++++++++++++++++++++++ src/extraction/index.ts | 11 +++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/__tests__/mpeg-ts-not-typescript.test.ts b/__tests__/mpeg-ts-not-typescript.test.ts index 2a6e12870b..3d45924bdd 100644 --- a/__tests__/mpeg-ts-not-typescript.test.ts +++ b/__tests__/mpeg-ts-not-typescript.test.ts @@ -12,6 +12,7 @@ import { describe, it, expect, afterEach } from 'vitest'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; +import { execFileSync } from 'child_process'; import { CodeGraph } from '../src'; import { scanDirectoryAsync, type ScanSkipStats } from '../src/extraction'; import { detectLanguage, isMpegTransportStream, MPEG_TS_SNIFF_BYTES } from '../src/extraction/grammars'; @@ -154,3 +155,50 @@ describe('MPEG-TS video named .ts is skipped, real TypeScript is indexed (#1910) } }); }); + +describe('a video .ts never stays pending (#1910)', () => { + const dirs: string[] = []; + afterEach(() => { for (const d of dirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }); }); + const gitProject = (): string => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-mpegts-git-')); + dirs.push(dir); + const git = (...a: string[]) => execFileSync('git', a, { cwd: dir, stdio: 'pipe' }); + git('init', '-q'); git('config', 'user.email', 't@t'); git('config', 'user.name', 't'); + fs.writeFileSync(path.join(dir, 'app.ts'), 'export const a = 1;\n'); + git('add', '.'); git('commit', '-qm', 'init'); + return dir; + }; + + it('an untracked clip is not reported as added, before or after sync', async () => { + const dir = gitProject(); + const cg = await CodeGraph.init(dir, { index: true }); + try { + fs.writeFileSync(path.join(dir, 'clip.ts'), makeMpegTs(40)); + expect(cg.getChangedFiles()).toEqual({ added: [], modified: [], removed: [] }); + await cg.sync(); + expect(cg.getChangedFiles()).toEqual({ added: [], modified: [], removed: [] }); + expect(cg.getFiles().map((f) => f.path)).toEqual(['app.ts']); + } finally { + cg.close(); + } + }); + + it('a tracked TypeScript file that becomes a clip is removed, on the git path and on a scoped sync', async () => { + for (const scoped of [false, true]) { + const dir = gitProject(); + fs.writeFileSync(path.join(dir, 'clip.ts'), 'export const clip = 1;\n'); + const cg = await CodeGraph.init(dir, { index: true }); + try { + expect(cg.getNodesInFile('clip.ts').some((n) => n.name === 'clip')).toBe(true); + fs.writeFileSync(path.join(dir, 'clip.ts'), makeMpegTs(40)); + if (!scoped) expect(cg.getChangedFiles()).toEqual({ added: [], modified: [], removed: ['clip.ts'] }); + await (scoped ? cg.sync({ paths: ['clip.ts'] }) : cg.sync()); + expect(cg.getFiles().map((f) => f.path)).toEqual(['app.ts']); + expect(cg.getNodesInFile('clip.ts')).toEqual([]); + expect(cg.getChangedFiles()).toEqual({ added: [], modified: [], removed: [] }); + } finally { + cg.close(); + } + } + }); +}); diff --git a/src/extraction/index.ts b/src/extraction/index.ts index c456f971fe..251245bfc6 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -3153,7 +3153,10 @@ export class ExtractionOrchestrator { (p) => isSourceFile(p, overrides) && !scope.ignores(p) && - fs.existsSync(path.join(this.rootDir, p)) + fs.existsSync(path.join(this.rootDir, p)) && + // Same rule as the scan (#1910): a reported video clip is not a + // source file, so a tracked one is removed and a new one ignored. + !isMpegTsVideoFile(this.rootDir, p) ); trackedFiles = []; for (const p of unique) { @@ -3411,7 +3414,11 @@ export class ExtractionOrchestrator { for (const filePath of candidates) { const tracked = this.queries.getFileByPath(filePath); const fullPath = path.join(this.rootDir, filePath); - if (!isSourceFile(filePath, overrides) || scope.ignores(filePath) || !fs.existsSync(fullPath)) { + // A `.ts` that is an MPEG transport stream is not source (#1910): the + // scan never lists it, so git must not report it as pending either — + // an untracked clip would otherwise stay "added" after every sync. + if (!isSourceFile(filePath, overrides) || scope.ignores(filePath) || !fs.existsSync(fullPath) + || isMpegTsVideoFile(this.rootDir, filePath)) { if (tracked) removed.push(filePath); continue; } From 538fda8cc613a641e9e5d270fd8cdf3ee96df43e Mon Sep 17 00:00:00 2001 From: danusha2345 Date: Sun, 27 Sep 2026 10:57:40 +0300 Subject: [PATCH 3/3] fix(extraction): require a binary head before calling a .ts file video (#1910) Review on #1915: the sniff took four aligned 0x47 sync bytes plus any NUL as proof of a transport stream, so TypeScript with a `G` at four 188-byte strides and one raw NUL in a comment was dropped unindexed. The sniff now needs the sync byte on sixteen consecutive packets (3 KB of head) and at least 1/64 of the head to be control bytes. Compressed payload carries 8-20% of them (checked on ffmpeg-made H.264/AAC, MPEG-2 and MP2 streams); source text carries none. A shorter clip is cheap to parse anyway. Also drops the benchmark figures from the changelog entry. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + __tests__/mpeg-ts-not-typescript.test.ts | 68 ++++++++++++++++-------- src/extraction/grammars.ts | 38 ++++++++----- 3 files changed, 74 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9880d7bcc..d634363afe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -173,6 +173,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - On Windows, the shared MCP daemon now waits longer for another program — an antivirus scan, an indexer, or another session reading its lock file — to let go of that file, so it starts instead of leaving the session to fall back to a slower in-process server. (#1773) - An MPEG transport stream video that happens to be named `.ts` (golden fixtures under `testdata/`, e2e clips) is now recognised from its first bytes and skipped as non-source instead of being fed to the TypeScript parser — a 900 KB clip used to cost about 28 seconds of CPU per file for no symbols, and a folder of them minutes. Real TypeScript is never affected. (#1910) +- An MPEG transport stream video that happens to be named `.ts` (golden fixtures under `testdata/`, e2e clips) is now recognised from its first bytes and skipped instead of being fed to the TypeScript parser, which spent a long time on each clip for no symbols. Real TypeScript files are still indexed. (#1910) - File watching no longer drops the full re-scan a removed directory asks for when that sync fails, so the deleted files leave the index instead of lingering. (#1964) - Daemon startup and cleanup now preserve live legacy PID-only locks while still reclaiming dead or identity-disproved records, preventing two writers from serving the same project. - Incremental sync now keeps edge rebinding crash-safe: replacing a resolved edge with its recovery reference commits atomically, so an interruption cannot permanently remove the relationship. diff --git a/__tests__/mpeg-ts-not-typescript.test.ts b/__tests__/mpeg-ts-not-typescript.test.ts index 3d45924bdd..19fd75718c 100644 --- a/__tests__/mpeg-ts-not-typescript.test.ts +++ b/__tests__/mpeg-ts-not-typescript.test.ts @@ -33,29 +33,38 @@ function makeMpegTs(packets: number, seed = 1): Buffer { } /** - * Real TypeScript engineered to put the letter `G` (0x47) at offsets 0, 188, - * 376 and 564 — the sync-byte pattern alone. It must stay TypeScript. + * Real TypeScript engineered to put the letter `G` (0x47) at the start of each + * of its first `packets` 188-byte strides — the sync-byte pattern alone. With + * `nul`, one raw NUL sits inside a comment (the review's counterexample on + * #1915). It must stay TypeScript either way. */ -function makeGammaSource(): string { - const lines: string[] = []; +function makeGammaSource(packets = 4, nul = false): string { let text = ''; - for (let i = 0; i < 4; i++) { - const line = `Gamma${i}();`; + for (let i = 0; i < packets; i++) { + const line = `Gamma${i}();` + (nul && i === 1 ? ' // \u0000' : ''); const pad = PACKET - line.length - 1; text += line + ' '.repeat(pad) + '\n'; - lines.push(line); } - text += 'export function Gamma0() { return 0; }\n'; - text += 'export function Gamma1() { return 1; }\n'; - text += 'export function Gamma2() { return 2; }\n'; - text += 'export function Gamma3() { return 3; }\n'; - for (const off of [0, PACKET, 2 * PACKET, 3 * PACKET]) { + for (let i = 0; i < packets; i++) text += `export function Gamma${i}() { return ${i}; }\n`; + text += 'export function realFn() { return 1; }\n'; + for (let off = 0; off < packets * PACKET; off += PACKET) { if (text.charCodeAt(off) !== 0x47) throw new Error(`fixture: expected G at ${off}`); } - void lines; return text; } +/** A stream that opens the usual way: PAT and PMT packets stuffed with 0xFF, then payload. */ +function makePsiLedMpegTs(packets: number): Buffer { + const buf = makeMpegTs(packets, 7); + for (const start of [0, PACKET]) { + buf.fill(0xff, start + 4, start + PACKET); + buf[start + 1] = 0x40; + buf[start + 2] = start === 0 ? 0x00 : 0x10; + buf[start + 3] = 0x10; + } + return buf; +} + const tempDirs: string[] = []; function createProject(): string { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-mpegts-')); @@ -74,15 +83,21 @@ describe('isMpegTransportStream', () => { expect(isMpegTransportStream(ts)).toBe(true); }); - it('needs four aligned sync bytes — a head too short, or one packet off, is not video', () => { + it('needs sixteen aligned sync bytes — a head too short, or one packet off, is not video', () => { const ts = makeMpegTs(40); - expect(isMpegTransportStream(ts.subarray(0, 3 * PACKET))).toBe(false); - const broken = Buffer.from(ts); - broken[2 * PACKET] = 0x48; - expect(isMpegTransportStream(broken)).toBe(false); + expect(isMpegTransportStream(ts.subarray(0, 15 * PACKET))).toBe(false); + for (const packet of [2, 15]) { + const broken = Buffer.from(ts); + broken[packet * PACKET] = 0x48; + expect(isMpegTransportStream(broken)).toBe(false); + } expect(isMpegTransportStream(Buffer.alloc(0))).toBe(false); }); + it('recognises a stream that opens with 0xFF-stuffed PAT and PMT packets', () => { + expect(isMpegTransportStream(makePsiLedMpegTs(40).subarray(0, MPEG_TS_SNIFF_BYTES))).toBe(true); + }); + it('does not take source text with G at every 188th byte for video', () => { const bytes = Buffer.from(makeGammaSource(), 'utf-8'); expect(bytes[0]).toBe(0x47); @@ -90,6 +105,14 @@ describe('isMpegTransportStream', () => { expect(isMpegTransportStream(bytes)).toBe(false); expect(detectLanguage('gamma.ts', makeGammaSource())).toBe('typescript'); }); + + it('does not take source with G at every stride and a NUL in a comment for video (#1915 review)', () => { + for (const packets of [4, 16, 20]) { + const bytes = Buffer.from(makeGammaSource(packets, true), 'utf-8'); + expect(bytes.includes(0)).toBe(true); + expect(isMpegTransportStream(bytes.subarray(0, MPEG_TS_SNIFF_BYTES))).toBe(false); + } + }); }); describe('MPEG-TS video named .ts is skipped, real TypeScript is indexed (#1910)', () => { @@ -99,23 +122,26 @@ describe('MPEG-TS video named .ts is skipped, real TypeScript is indexed (#1910) fs.writeFileSync(path.join(dir, 'testdata', 'clip.ts'), makeMpegTs(40)); fs.writeFileSync(path.join(dir, 'app.ts'), 'export function greet(n: string) { return `hi ${n}`; }\n'); fs.writeFileSync(path.join(dir, 'gamma.ts'), makeGammaSource()); + fs.writeFileSync(path.join(dir, 'gamma-nul.ts'), makeGammaSource(16, true)); const stats: ScanSkipStats = { unsupportedByExtension: new Map() }; const scanned = await scanDirectoryAsync(dir, undefined, stats); - expect(scanned.sort()).toEqual(['app.ts', 'gamma.ts']); + expect(scanned.sort()).toEqual(['app.ts', 'gamma-nul.ts', 'gamma.ts']); expect(stats.unsupportedByExtension.size).toBe(0); const cg = await CodeGraph.init(dir, { index: true }); try { const files = cg.getFiles().map((f) => f.path).sort(); - expect(files).toEqual(['app.ts', 'gamma.ts']); + expect(files).toEqual(['app.ts', 'gamma-nul.ts', 'gamma.ts']); expect(cg.searchNodes('greet').some((r) => r.node.name === 'greet')).toBe(true); expect(cg.searchNodes('Gamma2').some((r) => r.node.name === 'Gamma2')).toBe(true); + // The review's counterexample: G at every stride and a NUL in a comment. + expect(cg.searchNodes('realFn').filter((r) => r.node.filePath === 'gamma-nul.ts')).toHaveLength(1); // A named re-sync (the watcher / `sync` path hands files in by name) must // not let the clip back in either. await cg.sync({ paths: ['testdata/clip.ts', 'app.ts'] }); - expect(cg.getFiles().map((f) => f.path).sort()).toEqual(['app.ts', 'gamma.ts']); + expect(cg.getFiles().map((f) => f.path).sort()).toEqual(['app.ts', 'gamma-nul.ts', 'gamma.ts']); } finally { await cg.close(); } diff --git a/src/extraction/grammars.ts b/src/extraction/grammars.ts index d75180e4e9..84596d81c5 100644 --- a/src/extraction/grammars.ts +++ b/src/extraction/grammars.ts @@ -175,8 +175,18 @@ export const EXTENSION_MAP: Record = { /** MPEG transport stream: fixed 188-byte packets, each opening with 0x47. */ const MPEG_TS_PACKET_SIZE = 188; const MPEG_TS_SYNC_BYTE = 0x47; -/** Consecutive packets whose sync byte must line up before a file counts as video. */ -const MPEG_TS_MIN_PACKETS = 4; +/** + * Consecutive packets whose sync byte must line up before a file counts as + * video — 3 KB of head. A stream shorter than that is cheap to parse anyway; + * the cost #1910 is about comes from clips hundreds of KB long. + */ +const MPEG_TS_MIN_PACKETS = 16; +/** + * Share of the head that must be control bytes (below 0x20, other than the + * whitespace ones) for it to count as binary. Compressed audio and video put + * about one byte in eight there; source text puts none. + */ +const MPEG_TS_MIN_CONTROL_SHARE = 1 / 64; /** * How many bytes of a file's head `isMpegTransportStream` needs — enough to * see `MPEG_TS_MIN_PACKETS` sync bytes plus the packets between them. @@ -191,14 +201,15 @@ export const MPEG_TS_SNIFF_BYTES = MPEG_TS_PACKET_SIZE * MPEG_TS_MIN_PACKETS; * of the file, before any parse. * * Two conditions, both required: - * 1. the sync byte 0x47 sits at offsets 0, 188, 376 and 564 — every packet - * of a transport stream opens with it, and nothing else pads to 188; - * 2. a NUL byte appears somewhere in the head — every stream carries one - * within its first packets (the PSI pointer field, table reserved bits, - * the `00 00 01` PES start codes), and UTF-8 source text never does. - * 0x47 is the letter `G`, so (1) alone could in principle match a source file - * whose lines happen to put a `G` at four 188-byte strides; (2) closes that - * door, because a text file with a NUL in it is not TypeScript either. + * 1. the sync byte 0x47 sits at every 188-byte packet boundary of the first + * `MPEG_TS_MIN_PACKETS` packets — every packet of a transport stream + * opens with it, and nothing else pads to 188; + * 2. the head is binary: at least `MPEG_TS_MIN_CONTROL_SHARE` of it is + * control bytes, as any compressed payload is. + * 0x47 is the letter `G`, so (1) alone could match source whose lines happen + * to put a `G` at every 188-byte stride. Checking for a single NUL was not + * enough to close that: one NUL in a comment is still TypeScript. (2) asks for + * dozens of control bytes, which no source file carries. * * `head` is the first `MPEG_TS_SNIFF_BYTES` (or fewer) bytes of the file. */ @@ -208,10 +219,13 @@ export function isMpegTransportStream(head: Uint8Array): boolean { for (let off = 0; off <= lastSync; off += MPEG_TS_PACKET_SIZE) { if (head[off] !== MPEG_TS_SYNC_BYTE) return false; } + let control = 0; for (let i = 0; i < head.length; i++) { - if (head[i] === 0) return true; + const b = head[i]!; + // Tab, newline, vertical tab, form feed and carriage return are text. + if (b < 0x20 && (b < 0x09 || b > 0x0d)) control++; } - return false; + return control >= head.length * MPEG_TS_MIN_CONTROL_SHARE; } /** Whether `filePath` carries the one extension MPEG-TS shares with a language. */