From 409f32cdaa9e9f6e11ae6f5e1d7ddcddd75c6bbb Mon Sep 17 00:00:00 2001 From: Colby McHenry Date: Wed, 7 Oct 2026 01:32:54 -0500 Subject: [PATCH] fix(cpp): a class keeps its namespaces and enclosing class past code the parser misreads tree-sitter recovers from a construct it can't parse by inserting the token it expected, and inside a class body that is often a `}`: an unknown macro in front of a member (`PROTOBUF_FUTURE_ADD_EARLY_NODISCARD absl::string_view name() const`) or in a class head (`struct ALIGN_AS(64U) HandleImpl`) closed the class there, and every `}` after it closed the scope one level out. Recovery also ran the other way, keeping a scope open past its own `}` once an ERROR swallowed it. The walker scoped declarations by tree nesting, so qualified names lost their namespace and class segments, or gained ones they don't belong to: protobuf's `FieldDescriptor`, `MapFieldBase` and `MutableRepeatedFieldProxyImpl`, fmt's `detail::` buffers (and `fmt::format_int` put in `detail`), rocksdb's clock_cache classes and the `Opts` structs nested in them, gtest's `PrettyUnitTestResultPrinter`. Every such file parses with errors, and the kernel defers those to wasm, so the wasm walker produced all of them; the kernel's walker has the same tree-nested stack and reaches erroring trees only under the CODEGRAPH_KERNEL_CCPP_ERROR_EXTRACT sweep hatch. For a C++ file whose tree has errors, the walker now takes each declaration's namespaces, and at declaration level its enclosing classes, from the source's braces (languages/cpp-brace-scopes.ts: comments, literals, raw strings, digit separators and preprocessor lines skipped, each #if branch read from the braces open at the #if). A class-like node ends at its body's `}`, and a class the tree glued into a namespace-level declaration's type is walked as a class. A file whose braces don't balance, and every file that parses cleanly, keeps the tree's scopes. Ported to the kernel (ccpp/brace_scopes.rs) so the hatch stays parity-true: the same erroring files diverge before and after (UTF-8/UTF-16 recovery), every other one is byte-identical. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + __tests__/cpp-brace-scopes.test.ts | 382 +++++++++++++++++ codegraph-kernel/src/ccpp/brace_scopes.rs | 416 +++++++++++++++++++ codegraph-kernel/src/ccpp/mod.rs | 107 ++++- docs/design/ccpp-kernel-port-checklist.md | 1 + src/extraction/languages/cpp-brace-scopes.ts | 360 ++++++++++++++++ src/extraction/tree-sitter.ts | 85 ++++ 7 files changed, 1347 insertions(+), 5 deletions(-) create mode 100644 __tests__/cpp-brace-scopes.test.ts create mode 100644 codegraph-kernel/src/ccpp/brace_scopes.rs create mode 100644 src/extraction/languages/cpp-brace-scopes.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 61c468bda0..679435311e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - In C, C++ and Objective-C, a struct or class member declared as a pointer, reference, array or function, like `SharedState* shared;`, `jv elements[];` or `virtual Status Put(…) = 0;`, no longer makes the struct look like it inherits from the member's type. Before, `codegraph_explore` and impact listed those types as base classes, every struct holding a pointer to a type as one of its subclasses, and a method of that struct as an override of the type's method with the same name. Real base classes like `class Derived : public Base`, Objective-C superclasses and Go's embedded fields are unchanged. Re-index C, C++ and Objective-C projects after upgrading. - In C++, a call on a variable, parameter or member declared through a `typedef` or `using` alias now links to the method of the type the alias names, with the alias looked up the way C++ does it: in the calling function, its class and the classes that class inherits from, then the namespaces around it. In google/leveldb, `table_.Insert(buf)` on a `Table table_;` declared next to `typedef SkipList Table;` now reaches `SkipList::Insert` instead of the unrelated `HandleTable::Insert`, and `Table::Iterator iter(&table_);` reaches `SkipList::Iterator`'s constructor. A class that only shares the alias's name is no longer taken for it, and when the aliased type has no such method, as with an alias of `std::vector` or of a template parameter, no method is guessed from the receiver's name. Re-index C++ projects after upgrading. - In C++, a base class like `Message` in `class DynamicMessage final : public Message` now links to the class C++ itself would pick: the one declared in the namespaces around the class, or brought in by the file's `using` declarations and namespace aliases, and the class a `typedef` or `using` alias names. Before, a base class was matched by its name alone, so it could land on a class of the same name in another namespace or another language: in protocolbuffers/protobuf, `DynamicMessage` derived from the JSON parser's internal `Message` class, and in google/leveldb and RocksDB, iterators derived from an unrelated nested `Iterator` class such as the skip list's. The wrong base showed up in `codegraph_explore`'s type hierarchy and in impact, and tied the base's methods to the wrong overrides. A template parameter used as a base no longer links to a class that shares its name, and a base written from the global scope, like `public ::testing::Test`, now links to its class. Re-index C++ projects after upgrading. +- In C++, a class, struct or method now keeps the namespaces and the class it is declared in when the parser misreads something earlier in the file, such as an unknown macro in front of a member or in a class header. Before, everything after the misread code could lose them, or land inside a class it isn't in: in protocolbuffers/protobuf, `FieldDescriptor` was indexed without `google::protobuf::`, and in RocksDB, the `Opts` struct declared inside a cache table class was indexed outside it, so finding a class's base classes and the methods called on its objects fell back to guessing by name. Methods that had come loose from their class are its members again, and classes the misread code had hidden, like leveldb's POSIX file and environment classes, are now indexed. Re-index C++ projects after upgrading. - Framework support now switches on when an app's `package.json` sits three or more folders deep, as with an ASP.NET solution's Angular or React app in `src/Web/ClientApp/`, or a React app in a workspace folder like `web/ui/mantine-ui/`. Before, only the project root and the first two folder levels were read, so frameworks such as Angular, React Router, Vue Router, SvelteKit and TanStack Router could go undetected there: an Angular app's routes, its `routerLink` and `router.navigate(…)` navigation and its template's event bindings were missing, so a handler used only from a `(click)` binding had no callers, and a React app's `navigate('/login')` calls didn't reach their route. Re-index projects laid out this way after upgrading. - In Go, an interface that embeds another, like gin's `IRouter` embedding `IRoutes`, now links to it, so `codegraph_explore` and impact show the embedded interface as its base. Before, every embedded interface was dropped. An embedded field written through another package of your project, like `*refresh.Discovery` or `storage.Appender`, or with type arguments, like `List[int]`, now links too. One from the standard library or another module, like `io.Closer` or `sync.Mutex`, links to nothing rather than to a project type that shares its name, and a type-set constraint like `~int | ~float64` is not an embedding. An embedded type written without a package now always links to its own package's type: before, etcd's `Client`, which embeds its own `Lease` interface, was linked to a `Lease` struct in another package. Re-index Go projects after upgrading. - In Go, working out which structs implement an interface now counts the methods of the interfaces it embeds, and the methods a struct gets from the types it embeds. Before, only each type's own methods counted: etcd's `AuthReadTx`, which embeds `UnsafeAuthReader` and adds `RLock` and `RUnlock`, was listed as implemented by every read-write lock in the project, an interface that only embeds others had no implementations at all, and a struct that gets its methods from an embedded base, like prometheus's service discoveries embedding `refresh.Discovery` or gin's `Engine` embedding `RouterGroup`, was missing. `codegraph_explore`, impact and the type hierarchy now list the right implementations, and a call through the interface reaches a method the embedding struct overrides, like gin's `Engine.Use`. Re-index Go projects after upgrading. diff --git a/__tests__/cpp-brace-scopes.test.ts b/__tests__/cpp-brace-scopes.test.ts new file mode 100644 index 0000000000..2740f6007c --- /dev/null +++ b/__tests__/cpp-brace-scopes.test.ts @@ -0,0 +1,382 @@ +/** + * A C++ class keeps the namespaces and enclosing classes it is declared in + * when tree-sitter's error recovery closes one of them at the wrong `}`. + * + * tree-sitter recovers from a construct it can't parse by inserting the token + * it expected, and inside a class that is often a `}`. An unknown macro in + * front of a member (protobuf's `PROTOBUF_FUTURE_ADD_EARLY_NODISCARD + * absl::string_view name() const`) or in a class head (rocksdb's `struct + * ALIGN_AS(64U) HandleImpl`) closed the class there, and every `}` after it + * then closed the scope one level out. The rest of the class parsed as + * declarations of its namespace, and the rest of each namespace as + * declarations of the one around it: + * - protobuf's `FieldDescriptor` was indexed as `FieldDescriptor`, not + * `google::protobuf::FieldDescriptor`; + * - rocksdb's `struct Opts` lost the class it is declared in, and + * `AutoHyperClockTable` lost `clock_cache::`; + * - a scope could also stay open past its own `}`, putting the next class + * inside the previous one. + * Lookups that start from a declaration's scope (base classes, alias + * receivers, constructors) then had only a unique-name guess left. + * + * For a file whose tree has errors, the walker now takes each declaration's + * namespaces and enclosing classes from the source's braces, which the + * parser rarely misreads; a class the tree glued into a declaration's type is + * still walked as a class. Files that parse cleanly keep the tree's scopes. + * The native kernel defers erroring files to this walker; under its + * error-extract sweep hatch it must produce the same result. + */ +import { describe, it, expect, beforeAll, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import CodeGraph from '../src/index'; +import { extractFromSource } from '../src/extraction'; +import { initGrammars, loadGrammarsForLanguages, getParser } from '../src/extraction/grammars'; +import { cppExtractor } from '../src/extraction/languages/c-cpp'; +import { scanCppBraceScopes } from '../src/extraction/languages/cpp-brace-scopes'; +import { tryKernelExtract, resetKernelForTests } from '../src/extraction/kernel'; +import type { ExtractionResult, Node } from '../src/types'; + +const KERNEL_PATH = path.join( + __dirname, + '..', + 'codegraph-kernel', + 'prebuilds', + `${process.platform}-${process.arch}`, + 'codegraph-kernel.node' +); +const kernelAvailable = fs.existsSync(KERNEL_PATH) || process.env.CODEGRAPH_KERNEL_EXPECT === '1'; + +/** protocolbuffers/protobuf descriptor.h: an unknown attribute macro in front of a member. */ +const PROTOBUF_SHAPE = `namespace google { +namespace protobuf { + +class Names { + public: + PROTOBUF_FUTURE_ADD_EARLY_NODISCARD absl::string_view name() const { + return absl::string_view(payload_); + } + + private: + const char* payload_; +}; + +class FieldDescriptor : private internal::SymbolBase { + public: + int index() const; +}; + +} // namespace protobuf +} // namespace google +`; + +/** facebook/rocksdb cache/clock_cache.h: a macro call in a nested struct's head. */ +const ROCKSDB_SHAPE = `namespace rocksdb { +namespace clock_cache { + +class FixedTable : public BaseTable { + public: + struct ALIGN_AS(64U) HandleImpl : public ClockHandle { + RelaxedAtomic displacements{}; + bool standalone = false; + }; + + struct Opts : public BaseOpts { + explicit Opts(size_t size) : BaseOpts(0), size(size) {} + size_t size; + }; + + void Insert(const Opts& opts); +}; + +class AutoTable : public BaseTable { + public: + void Erase(); +}; + +} // namespace clock_cache +} // namespace rocksdb +`; + +/** A function head per #if branch: recovery keeps the class open past its own `}`. */ +const BRANCHES_SHAPE = `namespace net { + +class Socket { + public: +#ifdef _WIN32 + int Open(SOCKET handle) { +#else + int Open(int fd) { +#endif + return Bind(); + } + + int Bind(); +}; + +class Listener { + public: + void Accept() {} +}; + +} // namespace net +`; + +/** fmtlib/fmt core.h: \`unsigned(id) < unsigned(…)\` read as a template turns the namespaces into an ERROR. */ +const FMT_SHAPE = `namespace fmt { +namespace detail { + +template class basic_format_args { + public: + FMT_CONSTEXPR auto get(int id) const -> format_arg { + auto arg = format_arg(); + if (!is_packed()) { + if (unsigned(id) < unsigned(max_size())) arg = args_[id]; + return arg; + } + if (unsigned(id) >= detail::max_packed_args) return arg; + return arg; + } + + auto max_size() const -> int { return 0; } +}; + +class context { + public: + void advance_to(int it) {} +}; + +} // namespace detail +} // namespace fmt +`; + +const SHAPES: Record = { + 'descriptor.h': PROTOBUF_SHAPE, + 'clock_cache.h': ROCKSDB_SHAPE, + 'socket.h': BRANCHES_SHAPE, + 'core.h': FMT_SHAPE, +}; + +function wasmExtract(file: string, source: string): ExtractionResult { + const saved = process.env.CODEGRAPH_KERNEL; + process.env.CODEGRAPH_KERNEL = '0'; + try { + return extractFromSource(file, source, 'cpp'); + } finally { + if (saved === undefined) delete process.env.CODEGRAPH_KERNEL; + else process.env.CODEGRAPH_KERNEL = saved; + } +} + +const named = (result: ExtractionResult, kind: string): string[] => + result.nodes.filter((n) => n.kind === kind).map((n) => n.qualifiedName); + +const nodeAt = (result: ExtractionResult, qualifiedName: string): Node | undefined => + result.nodes.find((n) => n.qualifiedName === qualifiedName); + +describe('C++ scopes from the source braces of a file whose tree has errors', () => { + beforeAll(async () => { + await initGrammars(); + await loadGrammarsForLanguages(['cpp']); + }); + + it('every fixture still misparses (else the cases below prove nothing)', () => { + for (const [file, source] of Object.entries(SHAPES)) { + const tree = getParser('cpp')!.parse(cppExtractor.preParse!(source, file))!; + expect(tree.rootNode.hasError, file).toBe(true); + tree.delete(); + } + }); + + it('a class after one closed early keeps its namespaces (protobuf FieldDescriptor)', () => { + const result = wasmExtract('src/google/protobuf/descriptor.h', PROTOBUF_SHAPE); + expect(named(result, 'class')).toEqual(['google::protobuf::Names', 'google::protobuf::FieldDescriptor']); + // `Names` is the one recovery glued to the tokens after it: it ends at its own `};`. + expect(nodeAt(result, 'google::protobuf::Names')!.endLine).toBe(12); + }); + + it('a nested struct keeps its class and the next class its namespaces (rocksdb clock_cache)', () => { + const result = wasmExtract('cache/clock_cache.h', ROCKSDB_SHAPE); + expect(named(result, 'class')).toEqual(['rocksdb::clock_cache::FixedTable', 'rocksdb::clock_cache::AutoTable']); + expect(named(result, 'struct')).toEqual(['rocksdb::clock_cache::FixedTable::Opts']); + expect(named(result, 'method')).toEqual(['rocksdb::clock_cache::FixedTable::Opts::Opts']); + // The class spans its whole body, so the struct it holds lies inside it. + const fixed = nodeAt(result, 'rocksdb::clock_cache::FixedTable')!; + expect([fixed.startLine, fixed.endLine]).toEqual([4, 17]); + const contains = result.edges.find( + (e) => e.kind === 'contains' && e.target === nodeAt(result, 'rocksdb::clock_cache::FixedTable::Opts')!.id + ); + expect(contains?.source).toBe(fixed.id); + }); + + it('a class the tree kept open past its `}` lets go of the next class', () => { + const result = wasmExtract('net/socket.h', BRANCHES_SHAPE); + expect(named(result, 'class')).toEqual(['net::Socket', 'net::Listener']); + expect(named(result, 'method')).toContain('net::Listener::Accept'); + expect(nodeAt(result, 'net::Socket')!.endLine).toBe(14); + }); + + it('namespaces the tree swallowed into an ERROR still prefix what they hold (fmt core.h)', () => { + const result = wasmExtract('include/fmt/core.h', FMT_SHAPE); + expect(named(result, 'class')).toEqual(['fmt::detail::context']); + expect(named(result, 'method')).toEqual(['fmt::detail::context::advance_to']); + }); + + it('a file that parses cleanly keeps the tree scopes', () => { + const clean = PROTOBUF_SHAPE.replace('PROTOBUF_FUTURE_ADD_EARLY_NODISCARD ', ''); + const tree = getParser('cpp')!.parse(cppExtractor.preParse!(clean, 'descriptor.h'))!; + expect(tree.rootNode.hasError).toBe(false); + tree.delete(); + const result = wasmExtract('src/google/protobuf/descriptor.h', clean); + expect(named(result, 'class')).toEqual(['google::protobuf::Names', 'google::protobuf::FieldDescriptor']); + expect(named(result, 'method')).toEqual(['google::protobuf::Names::name']); + }); + + it('the base class of a class that lost its namespace resolves by scope lookup', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-cpp-brace-scopes-')); + try { + fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); + fs.writeFileSync(path.join(dir, 'src', 'descriptor.h'), PROTOBUF_SHAPE); + // The base, and a namesake in another namespace: with the namespace + // lost, neither a scope lookup nor the only-candidate guess finds it. + fs.writeFileSync( + path.join(dir, 'src', 'symbol_base.h'), + 'namespace google {\nnamespace protobuf {\nnamespace internal {\nclass SymbolBase {\n public:\n int kind() const;\n};\n} // namespace internal\n} // namespace protobuf\n} // namespace google\n' + ); + fs.writeFileSync( + path.join(dir, 'src', 'other.h'), + 'namespace other {\nnamespace internal {\nclass SymbolBase {\n public:\n int kind() const;\n};\n} // namespace internal\n} // namespace other\n' + ); + const cg = await CodeGraph.init(dir); + try { + await cg.indexAll(); + const derived = cg.searchNodes('FieldDescriptor').map((r) => r.node).find((n) => n.kind === 'class')!; + expect(derived.qualifiedName).toBe('google::protobuf::FieldDescriptor'); + const bases = cg + .getOutgoingEdges(derived.id) + .filter((e) => e.kind === 'extends') + .map((e) => cg.getNode(e.target)?.qualifiedName); + expect(bases).toEqual(['google::protobuf::internal::SymbolBase']); + } finally { + cg.close(); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +describe('scanCppBraceScopes', () => { + it('matches braces past comments, literals and preprocessor lines', () => { + const source = [ + 'namespace a {', + '// } a comment', + '/* { */ const char* s = "}{";', + "char c = '}'; int n = 1'000'000;", + 'auto r = R"x(})x";', + '#define OPEN {', + 'struct S { int x; };', + '}', + '', + ].join('\n'); + const scopes = scanCppBraceScopes(source)!; + expect(scopes).not.toBeNull(); + const open = source.indexOf('{'); + expect(scopes.closeOf(open)).toBe(source.lastIndexOf('}')); + const struct = source.indexOf('S {') + 2; + expect(source.slice(struct, scopes.closeOf(struct)! + 1)).toBe('{ int x; }'); + expect(scopes.namespacesAt(struct)).toEqual(['a']); + }); + + it('reads each #if branch from the braces open at the #if', () => { + const source = [ + 'namespace a {', + '#if defined(X)', + 'struct S : Base {', + '#else', + 'struct S {', + '#endif', + ' int x;', + '};', + 'struct T {};', + '}', + '', + ].join('\n'); + const scopes = scanCppBraceScopes(source)!; + expect(scopes).not.toBeNull(); + const first = source.indexOf('Base {') + 5; + expect(scopes.closeOf(first)).toBe(source.indexOf('};')); + expect(scopes.namespacesAt(source.indexOf('struct T'))).toEqual(['a']); + }); + + it('names namespaces as written and skips what declares none', () => { + const source = [ + 'namespace a::b {', + 'namespace [[deprecated]] c {', + 'inline namespace v1 {', + 'namespace {', + 'using namespace std;', + 'namespace fs = std::filesystem;', + 'int here;', + '}', + '}', + '}', + '}', + 'namespace std _GLIBCXX_VISIBILITY(default) {', + 'int there;', + '}', + '', + ].join('\n'); + const scopes = scanCppBraceScopes(source)!; + expect(scopes.namespacesAt(source.indexOf('int here'))).toEqual(['a::b', 'c', 'v1']); + expect(scopes.namespacesAt(source.indexOf('int there'))).toEqual(['std']); + }); + + it('gives up on braces that do not balance', () => { + expect(scanCppBraceScopes('namespace a {\nint x;\n')).toBeNull(); + expect(scanCppBraceScopes('int x;\n}\n')).toBeNull(); + }); + + it('places offsets in UTF-16 columns, as the parser does', () => { + const source = 'namespace n {\n/* é😀 */ class C { int x; };\n}\n'; + const scopes = scanCppBraceScopes(source)!; + const close = scopes.closeOf(source.indexOf('{ int'))!; + expect(scopes.positionOf(close + 1)).toEqual({ line: 2, column: 28 }); + }); +}); + +describe.skipIf(!kernelAvailable)('C++ brace scopes: native kernel parity (error-extract hatch)', () => { + beforeAll(async () => { + await initGrammars(); + await loadGrammarsForLanguages(['cpp']); + }); + + afterEach(() => { + delete process.env.CODEGRAPH_KERNEL_CCPP_ERROR_EXTRACT; + delete process.env.CODEGRAPH_KERNEL_LANGS; + resetKernelForTests(); + }); + + const canon = (result: ExtractionResult) => ({ + nodes: result.nodes.map(({ updatedAt: _u, ...n }) => JSON.stringify(n, Object.keys(n).sort())).sort(), + edges: result.edges.map((e) => JSON.stringify(e, Object.keys(e).sort())).sort(), + refs: result.unresolvedReferences.map((r) => JSON.stringify(r, Object.keys(r).sort())).sort(), + }); + + it.each(Object.keys(SHAPES))('%s: the kernel walks the same scopes as the wasm walker', (file) => { + for (const eol of ['\n', '\r\n']) { + const source = SHAPES[file]!.replace(/\n/g, eol); + process.env.CODEGRAPH_KERNEL_LANGS = 'cpp'; + process.env.CODEGRAPH_KERNEL_CCPP_ERROR_EXTRACT = '1'; + const viaKernel = tryKernelExtract(file, source, 'cpp'); + delete process.env.CODEGRAPH_KERNEL_CCPP_ERROR_EXTRACT; + expect(viaKernel).not.toBeNull(); + // Without the hatch a file with parse errors defers to wasm. (Asked + // second: a deferral is memoized for the same file and source.) + expect(tryKernelExtract(file, source, 'cpp')).toBeNull(); + const viaWasm = wasmExtract(file, source); + expect(canon(viaKernel!)).toEqual(canon(viaWasm)); + } + }); +}); diff --git a/codegraph-kernel/src/ccpp/brace_scopes.rs b/codegraph-kernel/src/ccpp/brace_scopes.rs new file mode 100644 index 0000000000..072c14113e --- /dev/null +++ b/codegraph-kernel/src/ccpp/brace_scopes.rs @@ -0,0 +1,416 @@ +//! C++ scopes read from the source's braces, for a file whose parse tree has +//! errors — a port of src/extraction/languages/cpp-brace-scopes.ts (read its +//! header for why). Byte offsets here, UTF-16 code units there; columns go +//! through `col16` so positions match the wasm walker's. +//! +//! Only reachable under the `CODEGRAPH_KERNEL_CCPP_ERROR_EXTRACT=1` sweep +//! hatch: every other erroring file defers to wasm before the walk starts. + +use crate::textutil as util; +use std::collections::HashMap; + +/// Intervals that nest properly, added in order of their opening offset. +pub struct NestedIntervals { + opens: Vec, + closes: Vec, + parents: Vec>, + values: Vec, +} + +impl NestedIntervals { + pub fn new() -> Self { + NestedIntervals { opens: Vec::new(), closes: Vec::new(), parents: Vec::new(), values: Vec::new() } + } + + /// Adds the interval `(open, close)`. One that opens before the last one + /// added, or crosses one that holds it, is refused (false). + pub fn add(&mut self, open: usize, close: usize, value: T) -> bool { + if close <= open || self.opens.last().is_some_and(|&last| open <= last) { + return false; + } + let parent = self.innermost(open); + if let Some(p) = parent { + if close >= self.closes[p] { + return false; + } + } + self.opens.push(open); + self.closes.push(close); + self.parents.push(parent); + self.values.push(value); + true + } + + /// The values of the intervals holding `offset` (open < offset < close), outermost first. + pub fn at(&self, offset: usize) -> Vec { + let mut out = Vec::new(); + let mut k = self.innermost(offset); + while let Some(i) = k { + out.push(self.values[i].clone()); + k = self.parents[i]; + } + out.reverse(); + out + } + + /// The innermost interval holding `offset`: the last interval opening + /// before `offset` or one of the intervals around that one. + fn innermost(&self, offset: usize) -> Option { + let count = self.opens.partition_point(|&o| o < offset); + let mut k = if count == 0 { None } else { Some(count - 1) }; + while let Some(i) = k { + if self.closes[i] > offset { + break; + } + k = self.parents[i]; + } + k + } +} + +pub struct BraceScopes { + closes: HashMap, + namespaces: NestedIntervals, +} + +impl BraceScopes { + /// The offset of the `}` that closes the `{` at `open`. + pub fn close_of(&self, open: usize) -> Option { + self.closes.get(&open).copied() + } + + /// The named namespaces whose braces hold `offset`, outermost first, as written. + pub fn namespaces_at(&self, offset: usize) -> Vec { + self.namespaces.at(offset) + } + + /// The 1-based line and the UTF-16 column of `offset`. + pub fn position_of(src: &str, line_starts: &[usize], offset: usize) -> (u32, u32) { + let row = line_starts.partition_point(|&s| s <= offset).saturating_sub(1); + (row as u32 + 1, util::col16(src, line_starts, row, offset)) + } +} + +fn is_blank(c: u8) -> bool { + matches!(c, b' ' | b'\t' | b'\r' | 0x0c | 0x0b) +} +fn is_word_start(c: u8) -> bool { + c.is_ascii_alphabetic() || c == b'_' || c == b'$' || c >= 0x80 +} +fn is_word_char(c: u8) -> bool { + is_word_start(c) || c.is_ascii_digit() +} +fn at(s: &[u8], i: usize) -> u8 { + s.get(i).copied().unwrap_or(0) +} +fn find(src: &str, from: usize, pat: &str) -> Option { + src.get(from..).and_then(|rest| rest.find(pat)).map(|k| from + k) +} + +/// The end of a `//` comment starting at `i`: its newline (a `\` before it continues the comment). +fn line_comment_end(src: &str, mut i: usize) -> usize { + let s = src.as_bytes(); + loop { + let Some(nl) = find(src, i, "\n") else { return s.len() }; + let mut k = nl as isize - 1; + if k >= 0 && s[k as usize] == b'\r' { + k -= 1; + } + if k < 0 || s[k as usize] != b'\\' { + return nl; + } + i = nl + 1; + } +} + +/// Past whitespace and comments from `i`. +fn skip_blank(src: &str, mut i: usize) -> usize { + let s = src.as_bytes(); + while i < s.len() { + let c = s[i]; + if is_blank(c) || c == b'\n' { + i += 1; + } else if c == b'/' && at(s, i + 1) == b'/' { + i = line_comment_end(src, i + 2); + } else if c == b'/' && at(s, i + 1) == b'*' { + i = find(src, i + 2, "*/").map_or(s.len(), |e| e + 2); + } else { + break; + } + } + i +} + +/// The end of the identifier at `i`, if one starts there. +fn word_end(s: &[u8], i: usize) -> Option { + if i >= s.len() || !is_word_start(s[i]) { + return None; + } + let mut j = i + 1; + while j < s.len() && is_word_char(s[j]) { + j += 1; + } + Some(j) +} + +/// Past the string or character literal whose quote is at `i` (it stops at a newline). +fn quoted_end(s: &[u8], i: usize) -> usize { + let quote = s[i]; + let mut j = i + 1; + while j < s.len() { + let c = s[j]; + if c == b'\\' { + j += if at(s, j + 1) == b'\r' && at(s, j + 2) == b'\n' { 3 } else { 2 }; + } else if c == quote { + return j + 1; + } else if c == b'\n' { + return j; + } else { + j += 1; + } + } + s.len() +} + +/// Past the `(…)` group at `i` (strings inside skipped). +fn paren_group_end(s: &[u8], i: usize) -> Option { + let mut depth = 0usize; + let mut j = i; + while j < s.len() { + let c = s[j]; + if c == b'"' || c == b'\'' { + j = quoted_end(s, j); + continue; + } + if c == b'(' { + depth += 1; + } else if c == b')' { + depth = depth.saturating_sub(1); + if depth == 0 { + return Some(j + 1); + } + } + j += 1; + } + None +} + +/// namespaceHead: after the `namespace` keyword ending at `from`, the name as +/// written and the offset of the body's `{`. +fn namespace_head(src: &str, from: usize) -> Option<(String, usize)> { + let s = src.as_bytes(); + let mut i = skip_blank(src, from); + while src.get(i..).is_some_and(|r| r.starts_with("[[")) { + let end = find(src, i + 2, "]]")?; + i = skip_blank(src, end + 2); + } + let mut name: Option<(usize, usize)> = None; + while let Some(end) = word_end(s, i) { + name = Some((name.map_or(i, |(start, _)| start), end)); + i = skip_blank(src, end); + if !src.get(i..).is_some_and(|r| r.starts_with("::")) { + break; + } + i = skip_blank(src, i + 2); + if let Some(inline) = word_end(s, i) { + if &src[i..inline] == "inline" { + i = skip_blank(src, inline); + } + } + } + if name.is_some() { + while let Some(end) = word_end(s, i) { + let paren = skip_blank(src, end); + if at(s, paren) != b'(' { + return None; + } + i = skip_blank(src, paren_group_end(s, paren)?); + } + } + if at(s, i) != b'{' { + return None; + } + Some((name.map_or(String::new(), |(start, end)| src[start..end].to_string()), i)) +} + +fn is_raw_delimiter(d: &[u8]) -> bool { + d.len() <= 16 && d.iter().all(|&b| (0x21..=0x7e).contains(&b) && !matches!(b, b'"' | b'(' | b')' | b'\\')) +} + +fn is_raw_prefix(w: &[u8]) -> bool { + matches!(w, b"R" | b"uR" | b"UR" | b"LR" | b"u8R") +} + +struct Branch { + at_if: Vec, + after_first: Option>, +} + +/// scanCppBraceScopes: `src` as the parser saw it (after preParse). None when +/// the braces don't balance. +pub fn scan(src: &str) -> Option { + let s = src.as_bytes(); + let n = s.len(); + let mut closes: HashMap = HashMap::new(); + let mut open: Vec = Vec::new(); + let mut branches: Vec = Vec::new(); + let mut namespace_at: Vec<(usize, String)> = Vec::new(); + let mut pending_namespace: Option<(String, usize)> = None; + let mut balanced = true; + let mut line_start = true; + let mut word_start: usize = 0; + let mut word_end_at: Option = None; + // Past a byte-order mark, so line 1 can be a directive. + let mut i = if src.starts_with('\u{feff}') { 3 } else { 0 }; + while i < n { + let c = s[i]; + if c == b'\n' { + line_start = true; + i += 1; + continue; + } + if is_blank(c) { + i += 1; + continue; + } + if c == b'/' && at(s, i + 1) == b'/' { + i = line_comment_end(src, i + 2); + continue; + } + if c == b'/' && at(s, i + 1) == b'*' { + i = find(src, i + 2, "*/").map_or(n, |e| e + 2); + continue; + } + if c == b'#' && line_start { + let mut j = i + 1; + while j < n && is_blank(s[j]) { + j += 1; + } + let directive: &[u8] = match word_end(s, j) { + Some(e) => &s[j..e], + None => b"", + }; + while j < n { + let d = s[j]; + if d == b'\n' { + let mut k = j as isize - 1; + if k >= 0 && s[k as usize] == b'\r' { + k -= 1; + } + if k < 0 || s[k as usize] != b'\\' { + break; + } + j += 1; + } else if d == b'/' && at(s, j + 1) == b'/' { + j = line_comment_end(src, j + 2); + } else if d == b'/' && at(s, j + 1) == b'*' { + j = find(src, j + 2, "*/").map_or(n, |e| e + 2); + } else if d == b'"' { + j = quoted_end(s, j); + } else { + j += 1; + } + } + match directive { + b"if" | b"ifdef" | b"ifndef" => branches.push(Branch { at_if: open.clone(), after_first: None }), + b"else" | b"elif" | b"elifdef" | b"elifndef" => { + if let Some(group) = branches.last_mut() { + let at_if = group.at_if.clone(); + let current = std::mem::replace(&mut open, at_if); + if group.after_first.is_none() { + group.after_first = Some(current); + } + } + } + b"endif" => { + if let Some(group) = branches.pop() { + if let Some(after) = group.after_first { + open = after; + } + } + } + _ => {} + } + i = j; + continue; + } + line_start = false; + if c == b'"' { + if word_end_at == Some(i) && is_raw_prefix(&s[word_start..i]) { + if let Some(paren) = find(src, i + 1, "(") { + let delimiter = &src[i + 1..paren]; + if is_raw_delimiter(delimiter.as_bytes()) { + let close = format!("){delimiter}\""); + i = find(src, paren + 1, &close).map_or(n, |e| e + delimiter.len() + 2); + continue; + } + } + } + i = quoted_end(s, i); + continue; + } + if c == b'\'' { + i = quoted_end(s, i); + continue; + } + if c.is_ascii_digit() || (c == b'.' && at(s, i + 1).is_ascii_digit()) { + let mut j = i + 1; + while j < n { + let d = s[j]; + if is_word_char(d) || d == b'.' { + j += 1; + } else if d == b'\'' && is_word_char(at(s, j + 1)) { + j += 2; + } else if (d == b'+' || d == b'-') && matches!(s[j - 1], b'e' | b'E' | b'p' | b'P') { + j += 1; + } else { + break; + } + } + word_start = i; + word_end_at = Some(j); + i = j; + continue; + } + if is_word_start(c) { + let end = word_end(s, i).unwrap_or(i + 1); + word_start = i; + word_end_at = Some(end); + if &s[i..end] == b"namespace" { + if let Some(head) = namespace_head(src, end) { + pending_namespace = Some(head); + } + } + i = end; + continue; + } + if c == b'{' { + open.push(i); + if pending_namespace.as_ref().is_some_and(|(_, brace)| *brace == i) { + let (name, _) = pending_namespace.take().unwrap(); + if !name.is_empty() { + namespace_at.push((i, name)); + } + } + } else if c == b'}' { + match open.pop() { + None => balanced = false, + Some(from) => { + closes.entry(from).or_insert(i); + } + } + } + i += 1; + } + if !balanced || !open.is_empty() { + return None; + } + namespace_at.sort_by_key(|(brace, _)| *brace); + let mut namespaces = NestedIntervals::new(); + for (brace, name) in namespace_at { + if let Some(&close) = closes.get(&brace) { + namespaces.add(brace, close, name); + } + } + Some(BraceScopes { closes, namespaces }) +} diff --git a/codegraph-kernel/src/ccpp/mod.rs b/codegraph-kernel/src/ccpp/mod.rs index d92bc33141..0772ac09f2 100644 --- a/codegraph-kernel/src/ccpp/mod.rs +++ b/codegraph-kernel/src/ccpp/mod.rs @@ -14,6 +14,10 @@ //! - cpp namespace prefix stack (#1291): named `namespace a::b {` pushes the //! name AS WRITTEN onto the qualifiedName prefix; anonymous falls through. //! No namespace NODE is minted (#1093 crowd-out). +//! - cpp brace scopes (brace_scopes.rs): when the tree has errors, each node +//! is walked in the namespaces, and at declaration level the classes, the +//! source's braces put it in, not the tree's (error recovery closes scopes +//! at the wrong `}`). Only the error-extract hatch reaches it here. //! - out-of-line `Cls::method` defs: name = LAST `::` segment of the //! declarator's qualified_identifier (BFS that skips parameter_list + //! trailing_return_type), receiver = the template-stripped qualifier, @@ -75,6 +79,9 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::OnceLock; use tree_sitter::{Node, Parser}; +mod brace_scopes; +use brace_scopes::{BraceScopes, NestedIntervals}; + const MAX_VALUE_REF_NODES: usize = 20_000; // --- compiled regexes (JS \w/\s spelled as ASCII classes for parity) --------- @@ -282,6 +289,7 @@ pub enum Variant { Cpp, } +#[derive(Clone)] struct Scope { row: u32, kind: &'static str, @@ -297,6 +305,8 @@ struct Extra { is_abstract: Option, return_type: Option, qualified_name: Option, + /// (endLine, endColumn) in place of the node's own (brace scopes). + end: Option<(u32, u32)>, } struct ValueScope<'t> { @@ -353,6 +363,12 @@ pub struct Walker<'t> { fs_values: HashMap, fs_value_counts: HashMap, value_scopes: Vec>, + /// A cpp file whose tree has errors is walked in the scopes its braces + /// open (visit_in_brace_scope; only under the error-extract hatch): the + /// scan, and the class-like nodes extracted so far by their bodies' braces. + brace_scopes: Option, + class_scopes: NestedIntervals, + class_scope_rows: HashSet, } pub fn extract(file_path: &str, source: &str, language: &str) -> Result { @@ -397,6 +413,13 @@ pub fn extract(file_path: &str, source: &str, language: &str) -> Result Walker<'t> { let column = self.col_of(node); let id = self.node_id_allocator.generate(self.file_path, kind, name, start_line, column); // (c/cpp define no resolveBody hook, so createNode's endLine extension - // for sibling-body grammars never fires — endLine is the node's own.) - let end_line = node.end_position().row as u32 + 1; + // for sibling-body grammars never fires — endLine is the node's own, + // or the `}` of its body in brace scopes.) + let (end_line, end_column) = + extra.end.unwrap_or_else(|| (node.end_position().row as u32 + 1, self.end_col_of(node))); let qualified = extra.qualified_name.unwrap_or_else(|| { let mut parts: Vec<&str> = self.namespace_prefix.iter().map(|s| s.as_str()).collect(); @@ -533,7 +558,7 @@ impl<'t> Walker<'t> { start_line, end_line, start_column: self.col_of(node), - end_column: self.end_col_of(node), + end_column, name: name_ref, qualified_name: qn_ref, id: id_ref, @@ -899,6 +924,59 @@ impl<'t> Walker<'t> { // --- visitNode ----------------------------------------------------------- fn visit_node(&mut self, node: Node<'t>) { + if self.brace_scopes.is_some() { + self.visit_in_brace_scope(node); + } else { + self.dispatch_node(node); + } + } + + /// visitInCppBraceScope: walk `node` in the namespaces and classes its + /// source braces put it in (see brace_scopes.rs). The namespaces apply + /// everywhere; the enclosing classes only at declaration level, where the + /// stack above the file node holds nothing but class scopes. + fn visit_in_brace_scope(&mut self, node: Node<'t>) { + let at = node.start_byte(); + let namespaces = self.brace_scopes.as_ref().map(|s| s.namespaces_at(at)).unwrap_or_default(); + let saved = std::mem::replace(&mut self.namespace_prefix, namespaces); + let mut base = self.stack.len(); + while base > 1 && self.class_scope_rows.contains(&self.stack[base - 1].row) { + base -= 1; + } + let mut walked: Option> = None; + if base == 1 { + let classes = self.class_scopes.at(at); + let same = classes.len() == self.stack.len() - base + && classes.iter().zip(&self.stack[base..]).all(|(c, s)| c.row == s.row); + if !same { + walked = Some(self.stack.split_off(base)); + self.stack.extend(classes); + } + } + self.dispatch_node(node); + if let Some(walked) = walked { + self.stack.truncate(base); + self.stack.extend(walked); + } + self.namespace_prefix = saved; + } + + /// cppBodyEnd: where a class-like node ends in brace scopes — the `}` + /// that closes its body — or None to keep the tree's. + fn brace_body_end(&self, body: Node<'t>) -> Option<(u32, u32)> { + let close = self.brace_scopes.as_ref()?.close_of(body.start_byte())?; + Some(BraceScopes::position_of(self.src, &self.line_starts, close + 1)) + } + + /// openCppClassScope: a class-like node's body as a scope for visit_in_brace_scope. + fn open_class_scope(&mut self, scope: &Scope, body: Node<'t>) { + let Some(close) = self.brace_scopes.as_ref().and_then(|s| s.close_of(body.start_byte())) else { return }; + if self.class_scopes.add(body.start_byte(), close, scope.clone()) { + self.class_scope_rows.insert(scope.row); + } + } + + fn dispatch_node(&mut self, node: Node<'t>) { stack_guard!(); let kind = node.kind(); let mut skip_children = false; @@ -965,6 +1043,19 @@ impl<'t> Walker<'t> { { skip_children = self.extract_type_alias(node); } else if kind == "declaration" && !self.inside_class_like() { + // In brace scopes, a class the tree reads as a declaration's type + // (glued to the tokens after it by error recovery) is still a class. + if self.brace_scopes.is_some() { + if let Some(t) = node.child_by_field_name("type") { + let class_like = matches!( + t.kind(), + "class_specifier" | "struct_specifier" | "union_specifier" | "enum_specifier" + ); + if class_like && t.child_by_field_name("body").is_some() { + self.visit_node(t); + } + } + } self.extract_variable(node); self.scan_fn_ref_subtree(node, 0); skip_children = true; @@ -1121,11 +1212,14 @@ impl<'t> Walker<'t> { let extra = Extra { docstring: preceding_docstring(node, self.src), visibility: self.visibility_of(node), + end: self.brace_body_end(body), ..Extra::default() }; let Some(row) = self.create_node("class", &name, node, extra) else { return }; + let scope = Scope { row, kind: "class", name }; + self.open_class_scope(&scope, body); self.extract_inheritance(node, row); - self.stack.push(Scope { row, kind: "class", name }); + self.stack.push(scope); for i in 0..body.named_child_count() { if let Some(c) = body.named_child(i) { self.visit_node(c); @@ -1142,11 +1236,14 @@ impl<'t> Walker<'t> { let extra = Extra { docstring: preceding_docstring(node, self.src), visibility: if self.variant == Variant::Cpp { self.visibility_of(node) } else { None }, + end: self.brace_body_end(body), ..Extra::default() }; let Some(row) = self.create_node(kind, &name, node, extra) else { return }; + let scope = Scope { row, kind, name }; + self.open_class_scope(&scope, body); self.extract_inheritance(node, row); - self.stack.push(Scope { row, kind, name }); + self.stack.push(scope); for i in 0..body.named_child_count() { if let Some(c) = body.named_child(i) { self.visit_node(c); diff --git a/docs/design/ccpp-kernel-port-checklist.md b/docs/design/ccpp-kernel-port-checklist.md index 148bad5252..839f0abf75 100644 --- a/docs/design/ccpp-kernel-port-checklist.md +++ b/docs/design/ccpp-kernel-port-checklist.md @@ -186,6 +186,7 @@ function_declarator → null, 12-hop guard). | Line | Mechanism | Must-mirror details | |---|---|---| | 962 | cpp namespace prefix stack (#1291) | named `namespace_definition` pushes its name (C++17 `a::b` as written) onto the QN prefix while walking children; anonymous falls through bare | +| — | cpp brace scopes (erroring trees only) | when the tree has errors, every visitNode takes its namespace prefix from the source's braces, and at declaration level (only class scopes above the file node) its enclosing classes: `languages/cpp-brace-scopes.ts` ↔ `ccpp/brace_scopes.rs`. A class-like node's end is its body's `}`; a class-like `type` of a namespace-level `declaration` is walked as a class. Reachable in the kernel only under `CODEGRAPH_KERNEL_CCPP_ERROR_EXTRACT=1` (every other erroring file defers) — kept parity-true there | | 2795 | C file-scope variables | only when NO function ancestor; iterate declarators; accept ONLY init_declarator / pointer_declarator / array_declarator — a BARE identifier declarator is a macro-prototype misparse, skip (loses uninit scalars by design); name via cDeclaratorIdentifier; signature `= `; kind constant/variable via isConst | | 4313 | explicit operator calls (#1247) | callee = `function` field + ERROR-wrapped `operator_name` sibling; compact symbolic spacing (`operator *`→`operator*`, word forms keep space); receiver `->`→`.`; DROP unless receiver is `this` (bare name) or simple identifier/member chain (silent miss over wrong edge) | | ~4340 | field_expression method calls | `recv.method`/`ptr->method` → `recv.method`; SKIP_RECEIVERS {self,this,cls,super} → bare name; LITERAL receiver → emit nothing (#1230) | diff --git a/src/extraction/languages/cpp-brace-scopes.ts b/src/extraction/languages/cpp-brace-scopes.ts new file mode 100644 index 0000000000..f83968117e --- /dev/null +++ b/src/extraction/languages/cpp-brace-scopes.ts @@ -0,0 +1,360 @@ +/** + * C++ scopes read from the source's braces, for a file whose parse tree has + * errors. + * + * tree-sitter recovers from a construct it can't parse by inserting the token + * it expected, and inside a class body that token is often a `}`: an unknown + * macro in front of a member (`PROTOBUF_FUTURE_ADD_EARLY_NODISCARD + * absl::string_view name() const`, `struct ALIGN_AS(64U) HandleImpl`) closes + * the class there. Every `}` after it then closes the scope one level out, so + * the rest of the class, and of each namespace around it, parses as + * declarations of an outer scope: protobuf's `FieldDescriptor` indexed as + * `FieldDescriptor` instead of `google::protobuf::FieldDescriptor`, rocksdb's + * `struct Opts` outside the class it is declared in. Recovery also runs the + * other way and keeps a scope open past its own `}`. + * + * The braces are rarely what the parser misread, so the walker takes each + * declaration's namespaces and enclosing classes from them instead + * (TreeSitterExtractor.visitInCppBraceScope; mirrored in the kernel, + * ccpp/mod.rs). The scan skips comments, string and character literals (raw + * strings and digit separators too) and preprocessor lines. Each branch of an + * `#if` is read from the braces open at the `#if`, and the scan continues + * after `#endif` with the first branch's — so `#if A` `struct X : B {` + * `#else` `struct X {` `#endif` opens one scope, not two. A file whose braces + * don't balance gets no scopes and keeps the tree's. + */ + +/** Intervals that nest properly, added in order of their opening offset. */ +export class NestedIntervals { + private readonly opens: number[] = []; + private readonly closes: number[] = []; + private readonly parents: number[] = []; + private readonly values: T[] = []; + + /** + * Adds the interval `(open, close)`. One that opens before the last one + * added, or crosses one that holds it, is refused (false). + */ + add(open: number, close: number, value: T): boolean { + const count = this.opens.length; + if (close <= open || (count > 0 && open <= this.opens[count - 1]!)) return false; + const parent = this.innermost(open); + if (parent >= 0 && close >= this.closes[parent]!) return false; + this.opens.push(open); + this.closes.push(close); + this.parents.push(parent); + this.values.push(value); + return true; + } + + /** The values of the intervals holding `offset` (open < offset < close), outermost first. */ + at(offset: number): T[] { + const out: T[] = []; + for (let k = this.innermost(offset); k >= 0; k = this.parents[k]!) out.push(this.values[k]!); + return out.reverse(); + } + + /** + * The innermost interval holding `offset`, or -1. It is the last interval + * opening before `offset` or one of the intervals around that one. + */ + private innermost(offset: number): number { + let lo = 0; + let hi = this.opens.length - 1; + let k = -1; + while (lo <= hi) { + const mid = (lo + hi) >> 1; + if (this.opens[mid]! < offset) { + k = mid; + lo = mid + 1; + } else { + hi = mid - 1; + } + } + while (k >= 0 && this.closes[k]! <= offset) k = this.parents[k]!; + return k; + } +} + +export interface CppBraceScopes { + /** The offset of the `}` that closes the `{` at `open`. */ + closeOf(open: number): number | undefined; + /** The named namespaces whose braces hold `offset`, outermost first, as written (`a::b`). */ + namespacesAt(offset: number): string[]; + /** The 1-based line and the column of `offset`. */ + positionOf(offset: number): { line: number; column: number }; +} + +const NEWLINE = 10; +const isBlank = (c: number): boolean => c === 32 || c === 9 || c === 13 || c === 12 || c === 11; +const isWordStart = (c: number): boolean => + (c >= 97 && c <= 122) || (c >= 65 && c <= 90) || c === 95 || c === 36 || c >= 128; +const isDigit = (c: number): boolean => c >= 48 && c <= 57; +const isWordChar = (c: number): boolean => isWordStart(c) || isDigit(c); +const RAW_STRING_PREFIX = /^(?:u8|[uUL])?R$/; +// A raw string's delimiter: up to 16 printable ASCII characters but `"`, `(`, `)` and `\`. +const RAW_STRING_DELIMITER = /^[!#-'*-[\]-~]{0,16}$/; + +/** The end of a `//` comment starting at `i`: its newline (a `\` before it continues the comment). */ +function lineCommentEnd(source: string, i: number): number { + for (;;) { + const nl = source.indexOf('\n', i); + if (nl < 0) return source.length; + let k = nl - 1; + if (source.charCodeAt(k) === 13) k--; + if (source.charCodeAt(k) !== 92) return nl; + i = nl + 1; + } +} + +/** Past whitespace and comments from `i`. */ +function skipBlank(source: string, i: number): number { + const n = source.length; + while (i < n) { + const c = source.charCodeAt(i); + if (isBlank(c) || c === NEWLINE) i++; + else if (c === 47 && source.charCodeAt(i + 1) === 47) i = lineCommentEnd(source, i + 2); + else if (c === 47 && source.charCodeAt(i + 1) === 42) { + const end = source.indexOf('*/', i + 2); + i = end < 0 ? n : end + 2; + } else break; + } + return i; +} + +/** The end of the identifier at `i`, or -1 when none starts there. */ +function wordEnd(source: string, i: number): number { + if (!isWordStart(source.charCodeAt(i))) return -1; + let j = i + 1; + while (j < source.length && isWordChar(source.charCodeAt(j))) j++; + return j; +} + +/** Past the `(…)` group at `i` (strings inside skipped), or -1. */ +function parenGroupEnd(source: string, i: number): number { + let depth = 0; + for (let j = i; j < source.length; j++) { + const c = source.charCodeAt(j); + if (c === 34 || c === 39) j = quotedEnd(source, j) - 1; + else if (c === 40) depth++; + else if (c === 41 && --depth === 0) return j + 1; + } + return -1; +} + +/** Past the string or character literal whose quote is at `i` (it stops at a newline). */ +function quotedEnd(source: string, i: number): number { + const quote = source.charCodeAt(i); + let j = i + 1; + while (j < source.length) { + const c = source.charCodeAt(j); + if (c === 92) j += source.charCodeAt(j + 1) === 13 && source.charCodeAt(j + 2) === NEWLINE ? 3 : 2; + else if (c === quote) return j + 1; + else if (c === NEWLINE) return j; + else j++; + } + return source.length; +} + +/** + * After the `namespace` keyword ending at `from`: the name as written and the + * offset of the `{` that opens the body. Null for anything else, such as + * `using namespace std;` or `namespace fs = std::filesystem;`. Takes C++17 + * attributes (`namespace [[deprecated]] old {`), nested and inline names + * (`namespace a::inline b {`), and an attribute macro after the name + * (`namespace std _GLIBCXX_VISIBILITY(default) {`). + */ +function namespaceHead(source: string, from: number): { name: string; brace: number } | null { + let i = skipBlank(source, from); + while (source.startsWith('[[', i)) { + const end = source.indexOf(']]', i + 2); + if (end < 0) return null; + i = skipBlank(source, end + 2); + } + let nameStart = -1; + let nameEnd = -1; + for (let end = wordEnd(source, i); end >= 0; end = wordEnd(source, i)) { + if (nameStart < 0) nameStart = i; + nameEnd = end; + i = skipBlank(source, end); + if (!source.startsWith('::', i)) break; + i = skipBlank(source, i + 2); + const inline = wordEnd(source, i); + if (inline >= 0 && source.slice(i, inline) === 'inline') i = skipBlank(source, inline); + } + for (let end = wordEnd(source, i); end >= 0 && nameStart >= 0; end = wordEnd(source, i)) { + const paren = skipBlank(source, end); + const after = source.charCodeAt(paren) === 40 ? parenGroupEnd(source, paren) : -1; + if (after < 0) return null; + i = skipBlank(source, after); + } + if (source.charCodeAt(i) !== 123) return null; + return { name: nameStart < 0 ? '' : source.slice(nameStart, nameEnd), brace: i }; +} + +/** + * Scan `source` (a C++ file as the parser saw it, after preParse) for its + * braces and namespaces. Null when the braces don't balance. + */ +export function scanCppBraceScopes(source: string): CppBraceScopes | null { + const n = source.length; + const closes = new Map(); + let open: number[] = []; + // Per `#if` group: the braces open at the `#if`, and those open after its + // first branch (null until a second branch starts). + const branches: Array<{ atIf: number[]; afterFirst: number[] | null }> = []; + const namespaceAt = new Map(); // `{` offset → name, for named namespaces + let pendingNamespace: { name: string; brace: number } | null = null; + let balanced = true; + let lineStart = true; + let wordStart = -1; // the last identifier or number read + let wordEndAt = -1; + let i = source.charCodeAt(0) === 0xfeff ? 1 : 0; // past a byte-order mark, so line 1 can be a directive + while (i < n) { + const c = source.charCodeAt(i); + if (c === NEWLINE) { + lineStart = true; + i++; + continue; + } + if (isBlank(c)) { + i++; + continue; + } + if (c === 47 && source.charCodeAt(i + 1) === 47) { + i = lineCommentEnd(source, i + 2); + continue; + } + if (c === 47 && source.charCodeAt(i + 1) === 42) { + const end = source.indexOf('*/', i + 2); + i = end < 0 ? n : end + 2; + continue; + } + if (c === 35 && lineStart) { + // A directive runs to the end of its line, through `\` continuations + // and comments. Only `"` strings are skipped in it: an apostrophe in + // `#error Don't …` is no character literal. + let j = i + 1; + while (j < n && isBlank(source.charCodeAt(j))) j++; + const nameEnd = wordEnd(source, j); + const directive = nameEnd < 0 ? '' : source.slice(j, nameEnd); + while (j < n) { + const d = source.charCodeAt(j); + if (d === NEWLINE) { + let k = j - 1; + if (source.charCodeAt(k) === 13) k--; + if (source.charCodeAt(k) !== 92) break; + j++; + } else if (d === 47 && source.charCodeAt(j + 1) === 47) { + j = lineCommentEnd(source, j + 2); + } else if (d === 47 && source.charCodeAt(j + 1) === 42) { + const end = source.indexOf('*/', j + 2); + j = end < 0 ? n : end + 2; + } else if (d === 34) { + j = quotedEnd(source, j); + } else { + j++; + } + } + if (directive === 'if' || directive === 'ifdef' || directive === 'ifndef') { + branches.push({ atIf: open.slice(), afterFirst: null }); + } else if ( + (directive === 'else' || directive === 'elif' || directive === 'elifdef' || directive === 'elifndef') && + branches.length > 0 + ) { + const group = branches[branches.length - 1]!; + if (group.afterFirst === null) group.afterFirst = open; + open = group.atIf.slice(); + } else if (directive === 'endif' && branches.length > 0) { + const group = branches.pop()!; + if (group.afterFirst !== null) open = group.afterFirst; + } + i = j; + continue; + } + lineStart = false; + if (c === 34) { + // A raw string (`R"x(…)x"`) can hold any brace and quote; its prefix is + // the word that ends right at the quote. + if (wordEndAt === i && i - wordStart <= 3 && RAW_STRING_PREFIX.test(source.slice(wordStart, i))) { + const paren = source.indexOf('(', i + 1); + const delimiter = paren < 0 ? '' : source.slice(i + 1, paren); + if (paren >= 0 && RAW_STRING_DELIMITER.test(delimiter)) { + const end = source.indexOf(`)${delimiter}"`, paren + 1); + i = end < 0 ? n : end + delimiter.length + 2; + continue; + } + } + i = quotedEnd(source, i); + continue; + } + if (c === 39) { + i = quotedEnd(source, i); + continue; + } + if (isDigit(c) || (c === 46 && isDigit(source.charCodeAt(i + 1)))) { + // A number, digit separators (`1'000'000`) and exponent signs included. + let j = i + 1; + while (j < n) { + const d = source.charCodeAt(j); + if (isWordChar(d) || d === 46) j++; + else if (d === 39 && isWordChar(source.charCodeAt(j + 1))) j += 2; + else if ((d === 43 || d === 45) && /[eEpP]/.test(source[j - 1]!)) j++; + else break; + } + wordStart = i; + wordEndAt = j; + i = j; + continue; + } + if (isWordStart(c)) { + const end = wordEnd(source, i); + wordStart = i; + wordEndAt = end; + if (end - i === 9 && source.startsWith('namespace', i)) { + const head = namespaceHead(source, end); + if (head) pendingNamespace = head; + } + i = end; + continue; + } + if (c === 123) { + open.push(i); + if (pendingNamespace && pendingNamespace.brace === i) { + if (pendingNamespace.name) namespaceAt.set(i, pendingNamespace.name); + pendingNamespace = null; + } + } else if (c === 125) { + const from = open.pop(); + if (from === undefined) balanced = false; + else if (!closes.has(from)) closes.set(from, i); + } + i++; + } + if (!balanced || open.length > 0) return null; + + const namespaces = new NestedIntervals(); + for (const [brace, name] of [...namespaceAt].sort((a, b) => a[0] - b[0])) { + const close = closes.get(brace); + if (close !== undefined) namespaces.add(brace, close, name); + } + let lineStarts: number[] | null = null; + return { + closeOf: (at) => closes.get(at), + namespacesAt: (offset) => namespaces.at(offset), + positionOf: (offset) => { + if (!lineStarts) { + lineStarts = [0]; + for (let k = source.indexOf('\n'); k >= 0; k = source.indexOf('\n', k + 1)) lineStarts.push(k + 1); + } + let lo = 0; + let hi = lineStarts.length - 1; + while (lo < hi) { + const mid = (lo + hi + 1) >> 1; + if (lineStarts[mid]! <= offset) lo = mid; + else hi = mid - 1; + } + return { line: lo + 1, column: offset - lineStarts[lo]! }; + }, + }; +} diff --git a/src/extraction/tree-sitter.ts b/src/extraction/tree-sitter.ts index 997d735170..580cccce55 100644 --- a/src/extraction/tree-sitter.ts +++ b/src/extraction/tree-sitter.ts @@ -22,6 +22,7 @@ import { isGeneratedFile, isMinifiedContent } from './generated-detection'; import type { LanguageExtractor, ExtractorContext } from './tree-sitter-types'; import { EXTRACTORS } from './languages'; import { stripCppTemplateArgs, isCppConstructorDeclaration } from './languages/c-cpp'; +import { NestedIntervals, scanCppBraceScopes, type CppBraceScopes } from './languages/cpp-brace-scopes'; import { rustImplTypeName } from './languages/rust'; import { goEmbeddedTypeName } from './languages/go'; import { dartMisparsedGenericCall, dartReceiverOf, isDartTypeName, pushDartTypeRefs } from './languages/dart'; @@ -69,6 +70,9 @@ const REACT_COMPONENT_HOCS = new Set(['forwardRef', 'memo', 'React.forwardRef', */ const SIGNATURE_METHOD_NODE_TYPES = new Set(['method_signature']); +/** C++ specifiers that define a class-like type (or an enum) when they carry a body. */ +const CPP_CLASS_LIKE_SPECIFIERS = new Set(['class_specifier', 'struct_specifier', 'union_specifier', 'enum_specifier']); + /** Vue store collections whose object-literal members are the symbols an agent * looks for. Extracted as function nodes so `actions`/`mutations`/`getters` are * findable + readable (the foundation under any later dispatch-bridge synth). */ @@ -600,6 +604,13 @@ export class TreeSitterExtractor { // files, and a node per block would flood search with same-named symbols // (the #1093 crowd-out failure mode). Always empty outside C/C++. private namespacePrefix: string[] = []; + // A C++ file whose tree has errors is walked in the scopes its braces + // open (see visitInCppBraceScope): the scan of them, null for every other + // file, and the class-like nodes extracted so far by the braces of their + // bodies. + private cppBraceScopes: CppBraceScopes | null = null; + private cppClassScopes = new NestedIntervals(); + private cppClassScopeIds = new Set(); // C++ local function-pointer bindings, per enclosing symbol: // `auto kernel = &flash_fwd_kernel<…>;` recorded as callerId → kernel → // {flash_fwd_kernel}, so a later `kernel<<>>(params)` (or plain @@ -696,6 +707,9 @@ export class TreeSitterExtractor { if (!this.tree) { throw new Error('Parser returned null tree'); } + if (this.language === 'cpp' && this.tree.rootNode.hasError) { + this.cppBraceScopes = scanCppBraceScopes(this.source); + } // Create file node representing the source file const fileNode: Node = { @@ -1154,6 +1168,65 @@ export class TreeSitterExtractor { * Visit a node and extract information */ private visitNode(node: SyntaxNode): void { + if (this.cppBraceScopes) this.visitInCppBraceScope(node, this.cppBraceScopes); + else this.dispatchNode(node); + } + + /** + * Walk a node of a C++ file whose tree has errors in the namespaces and + * classes its source braces put it in. That is where the tree put it unless + * error recovery closed a scope at the wrong `}` or ran one past its own — + * the rest of a class then parses as declarations of the namespace around + * it (rocksdb's `struct Opts`), and the rest of each namespace one level out + * (protobuf's `FieldDescriptor` at the top level). See cpp-brace-scopes.ts. + * + * The namespaces apply everywhere. The enclosing classes are set only at + * declaration level, where the stack above the file node holds nothing but + * class scopes; inside a function (a local class's members) the walk's own + * stack stands. + */ + private visitInCppBraceScope(node: SyntaxNode, scopes: CppBraceScopes): void { + const at = node.startIndex; + const namespaces = this.namespacePrefix; + this.namespacePrefix = scopes.namespacesAt(at); + const stack = this.nodeStack; + let base = stack.length; + while (base > 1 && this.cppClassScopeIds.has(stack[base - 1]!)) base--; + let walked: string[] | null = null; + if (base === 1) { + const classes = this.cppClassScopes.at(at); + if (classes.length !== stack.length - base || classes.some((id, i) => stack[base + i] !== id)) { + walked = stack.splice(base); + stack.push(...classes); + } + } + this.dispatchNode(node); + if (walked) stack.splice(base, stack.length - base, ...walked); + this.namespacePrefix = namespaces; + } + + /** + * The textual extent of a C++ class-like body while the file is walked in + * brace scopes: where the node ends (the `}` that closes its body) to pass + * to createNode, or null to keep the tree's. + */ + private cppBodyEnd(body: SyntaxNode | null | undefined): Partial | null { + const close = body ? this.cppBraceScopes?.closeOf(body.startIndex) : undefined; + if (close === undefined) return null; + const end = this.cppBraceScopes!.positionOf(close + 1); + return { endLine: end.line, endColumn: end.column }; + } + + /** Open a class-like node's body as a scope for visitInCppBraceScope. */ + private openCppClassScope(node: Node, body: SyntaxNode | null | undefined): void { + if (!body) return; + const close = this.cppBraceScopes?.closeOf(body.startIndex); + if (close !== undefined && this.cppClassScopes.add(body.startIndex, close, node.id)) { + this.cppClassScopeIds.add(node.id); + } + } + + private dispatchNode(node: SyntaxNode): void { if (!this.extractor) return; const nodeType = node.type; @@ -1403,6 +1476,14 @@ export class TreeSitterExtractor { this.extractor.variableTypes.includes(nodeType) && (!this.isInsideClassLikeNode() || this.isClassScopeConstantAssignment(node)) ) { + // In a file walked in brace scopes, a class the tree reads as the type + // of a declaration (`class X {…}` glued to the tokens after it by + // error recovery) is still a class. In a class body the children walk + // reached it; out of one, this branch skips that walk. + const declaredType = this.cppBraceScopes ? getChildByField(node, 'type') : null; + if (declaredType && CPP_CLASS_LIKE_SPECIFIERS.has(declaredType.type) && getChildByField(declaredType, 'body')) { + this.visitNode(declaredType); + } this.extractVariable(node); // extractVariable doesn't walk every initializer shape (object literals // are deliberately skipped; Python/Ruby don't walk at all), so scan the @@ -2033,8 +2114,10 @@ export class TreeSitterExtractor { docstring, visibility, isExported, + ...this.cppBodyEnd(resolvedBody), }); if (!classNode) return; + this.openCppClassScope(classNode, resolvedBody); // Extract extends/implements this.extractInheritance(node, classNode.id); @@ -2247,8 +2330,10 @@ export class TreeSitterExtractor { docstring, visibility, isExported, + ...this.cppBodyEnd(body), }); if (!aggregateNode) return; + this.openCppClassScope(aggregateNode, body); // Extract inheritance (e.g. Swift: struct HTTPMethod: RawRepresentable) this.extractInheritance(node, aggregateNode.id);