merge: reconcile upstream through 1d3619d6 with native unittests directory rule

CHANGELOG.md

@@ -448,6 +448,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 - In a repository that holds more than one Shopify theme, a theme's `{% render %}`, `{% include %}` and `{% section %}` tags and the sections its JSON templates list now link only to that theme's own snippets and sections, which is where Shopify looks for them. Before, a section or snippet the theme doesn't have was linked to another theme's file of the same name, and a theme kept in a subfolder could link to the root theme's copy instead of its own. A reference the theme can't satisfy now links to nothing. Projects with a single theme are indexed as before. Re-index projects with several Shopify themes after upgrading.
 
+- A file in a `unittests/` folder, where llvm/llvm-project and google/breakpad keep their unit tests, or in a folder named like a unittest, such as `dcheck_unittest/` in google/glog, now counts as a test whatever its own name says, like LLVM's `CountCopyAndMove.h` helper. Before, such a file was taken for production code unless its name said test, and none of them were kept apart from the program the way a `tests/` folder is: search and `codegraph_explore` ranked them alongside production code, `codegraph affected` missed them, and production code linked to a unit test's own declarations, as LLVM's `Ctx` and `OpInfo` variables did to ones its unit tests declare. A folder named just `unittest/`, like CPython's `Lib/unittest/`, is still production code. Re-index projects that have such folders after upgrading.
 - In C, C++ and Objective-C, a struct or class member declared as a pointer, reference, array or function, like `SharedState* shared;`, `jv elements[];` or `virtual Status Put(…) = 0;`, no longer makes the struct look like it inherits from the member's type. Before, `codegraph_explore` and impact listed those types as base classes, every struct holding a pointer to a type as one of its subclasses, and a method of that struct as an override of the type's method with the same name. Real base classes like `class Derived : public Base`, Objective-C superclasses and Go's embedded fields are unchanged. Re-index C, C++ and Objective-C projects after upgrading.
 
 - In C++, a call on a variable, parameter or member declared through a `typedef` or `using` alias now links to the method of the type the alias names, with the alias looked up the way C++ does it: in the calling function, its class and the classes that class inherits from, then the namespaces around it. In google/leveldb, `table_.Insert(buf)` on a `Table table_;` declared next to `typedef SkipList<const char*, KeyComparator> Table;` now reaches `SkipList::Insert` instead of the unrelated `HandleTable::Insert`, and `Table::Iterator iter(&table_);` reaches `SkipList::Iterator`'s constructor. A class that only shares the alias's name is no longer taken for it, and when the aliased type has no such method, as with an alias of `std::vector` or of a template parameter, no method is guessed from the receiver's name. Re-index C++ projects after upgrading.

README.md

@@ -52,7 +52,7 @@ Follow [@getcodegraph](https://x.com/getcodegraph) on X for updates.
 
 ## About this fork
 
-This is **bompus/codegraph**, a fork of [colbymchenry/codegraph](https://github.com/colbymchenry/codegraph). Its default branch, `fork/consolidated`, contains upstream `main` through [`1ce1167e`](https://github.com/colbymchenry/codegraph/commit/1ce1167e) (after v1.6.2) plus the fork's own work, and it takes upstream changes as they land. Changes that suit upstream are also offered there as pull requests.
+This is **bompus/codegraph**, a fork of [colbymchenry/codegraph](https://github.com/colbymchenry/codegraph). Its default branch, `fork/consolidated`, contains upstream `main` through [`1d3619d6`](https://github.com/colbymchenry/codegraph/commit/1d3619d6) (after v1.6.2) plus the fork's own work, and it takes upstream changes as they land. Changes that suit upstream are also offered there as pull requests.
 
 The fork publishes no releases. The install scripts, npm package, badges and `codegraph upgrade` further down this page install **upstream's** releases. To run the fork, build it from source (below).
 
@@ -80,7 +80,7 @@ Session search uses `.codegraph/sessions-v2.db`. The first session search import
 
 ### What the fork adds
 
-Compared with upstream `main` at `1ce1167e` (after v1.6.2). Each item here and in the dispatch and framework lists below was checked against upstream's tree at that commit.
+Compared with upstream `main` at `1d3619d6` (after v1.6.2). Each item here and in the dispatch and framework lists below was checked against upstream's tree at that commit.
 
 | Feature | Upstream | Fork | What it does |
 |---|:-:|:-:|---|

__tests__/cli-affected-test-conventions.test.ts

@@ -44,6 +44,9 @@ describe('codegraph affected — test-file conventions (#1507)', () => {
     w('src/wire_format.h', '#pragma once\n\nint ByteSize(int value);\n');
     w('src/wire_format.cc', '#include "wire_format.h"\n\nint ByteSize(int value) { return value; }\n');
     w('src/wire_format_unittest.cc', '#include "wire_format.h"\n\nvoid ComputesByteSize() { ByteSize(1); }\n');
+    w('include/llvm/ADT/FoldingSet.h', '#pragma once\n\nint FoldingSetSize();\n');
+    w('lib/Support/FoldingSet.cpp', '#include "llvm/ADT/FoldingSet.h"\n\nint FoldingSetSize() { return 0; }\n');
+    w('unittests/ADT/FoldingSet.cpp', '#include "llvm/ADT/FoldingSet.h"\n\nvoid SizesTheSet() { FoldingSetSize(); }\n');
     const cg = CodeGraph.initSync(dir);
     await cg.indexAll();
     cg.close();
@@ -66,6 +69,10 @@ describe('codegraph affected — test-file conventions (#1507)', () => {
     expect(affected(dir, ['src/wire_format.h', 'src/wire_format.cc'])).toEqual(['src/wire_format_unittest.cc']);
   });
 
+  it("reports a file in LLVM's unittests/ tree, whatever its name", () => {
+    expect(affected(dir, ['include/llvm/ADT/FoldingSet.h', 'lib/Support/FoldingSet.cpp'])).toEqual(['unittests/ADT/FoldingSet.cpp']);
+  });
+
   it('still honours an explicit --filter glob', () => {
     expect(affected(dir, ['math.go', '--filter', '*_test.go'])).toEqual(['math_test.go']);
     expect(affected(dir, ['math.go', '--filter', '*.spec.ts'])).toEqual([]);

__tests__/is-test-file.test.ts

@@ -80,6 +80,37 @@ describe('isTestFile', () => {
     expect(isTestFile('cmd/promtool/unittest.go')).toBe(false);
   });
 
+  it('flags everything in a unittests/ or foo_unittest/ directory', () => {
+    // LLVM's and Breakpad's unit-test trees hold helpers with no test-like
+    // name, and glog builds each `*_unittest/` directory as a test program.
+    for (const file of [
+      'llvm/unittests/ADT/CountCopyAndMove.h',
+      'unittests/ADT/FoldingSet.cpp',
+      'clang-tools-extra/clangd/unittests/Annotations.cpp',
+      'src/client/windows/unittests/dump_analysis.cc',
+      'src/dcheck_unittest/glog_dcheck.cc',
+      'src/includes_unittest/glog_includes_logging.cc',
+      'tools/net-unittests/fake_socket.py',
+    ]) {
+      expect(isTestPath(file), file).toBe(true);
+      expect(isTestFile(file), file).toBe(true);
+    }
+  });
+
+  it('does NOT flag a bare unittest/ directory', () => {
+    // CPython's `Lib/unittest/` is the framework, and so is the googletest LLVM
+    // vendors under `third-party/unittest/`; `unittest2` is the framework's backport.
+    for (const file of [
+      'Lib/unittest/mock.py',
+      'Lib/unittest/__init__.py',
+      'third-party/unittest/googletest/src/gtest.cc',
+      'unittest2/case.py',
+    ]) {
+      expect(isTestPath(file), file).toBe(false);
+      expect(isTestFile(file), file).toBe(false);
+    }
+  });
+
   it('does NOT flag production files that merely contain "test" lowercase', () => {
     // The fix is capital-led so camelCase boundaries distinguish these.
     expect(isTestFile('src/latest/loader.kt')).toBe(false);

__tests__/unittests-directories.test.ts

@@ -0,0 +1,153 @@
+/**
+ * LLVM keeps its unit tests in `unittests/` (`llvm/unittests/`,
+ * `clang/unittests/`, clangd's `unittests/`), Breakpad in
+ * `src/client/windows/unittests/`, and glog builds each `foo_unittest/`
+ * directory as a test program of its own. The test-file check knew `tests/`
+ * but not these, so a helper there with no test-like name read as production
+ * code, and none of the tree counted as a test suite: production locals named
+ * `Ctx` or `OpInfo` linked to a test's own `LLVMContext Ctx;`, and a test
+ * fixture's method kept a production call from resolving at all.
+ */
+import { describe, it, expect, afterAll, beforeAll } from 'vitest';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { CodeGraph } from '../src';
+
+let root = '';
+let cg: CodeGraph;
+
+beforeAll(async () => {
+  root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-unittests-dirs-'));
+  const files: Record<string, string> = {
+    // A call the receiver's type doesn't settle, so its method is guessed by
+    // name — and LLVM's SelectionDAGTestBase.h, a test fixture, has a method of
+    // that name too.
+    'include/llvm/CodeGen/TargetLowering.h': `#pragma once
+
+namespace llvm {
+
+class TargetLoweringBase {
+ public:
+  int getTypeToTransformTo(int VT) const { return VT * 2; }
+};
+
+class TargetLowering : public TargetLoweringBase {};
+
+}  // namespace llvm
+`,
+    'lib/CodeGen/SelectionDAG/LegalizeTypes.h': `#pragma once
+
+#include "llvm/CodeGen/TargetLowering.h"
+
+namespace llvm {
+
+class DAGTypeLegalizer {
+  const TargetLowering &TLI;
+
+ public:
+  explicit DAGTypeLegalizer(const TargetLowering &T) : TLI(T) {}
+  int softenFloat(int VT);
+};
+
+}  // namespace llvm
+`,
+    'lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp': `#include "LegalizeTypes.h"
+
+namespace llvm {
+
+int DAGTypeLegalizer::softenFloat(int VT) {
+  return TLI.getTypeToTransformTo(VT);
+}
+
+}  // namespace llvm
+`,
+    'unittests/CodeGen/SelectionDAGTestBase.h': `#pragma once
+
+#include "llvm/CodeGen/TargetLowering.h"
+
+namespace llvm {
+
+class SelectionDAGTestBase {
+ protected:
+  const TargetLowering *Lowering = nullptr;
+  int getTypeToTransformTo(int VT) { return Lowering->getTypeToTransformTo(VT); }
+};
+
+}  // namespace llvm
+`,
+    // A production parameter named like a variable a unittest declares.
+    'lib/Object/WasmObjectFile.cpp': `namespace llvm {
+
+struct ReadContext {
+  const unsigned char *Ptr;
+  const unsigned char *End;
+};
+
+unsigned readByte(ReadContext &Ctx) {
+  if (Ctx.Ptr == Ctx.End) return 0;
+  return *Ctx.Ptr++;
+}
+
+}  // namespace llvm
+`,
+    'unittests/FuzzMutate/RandomIRBuilderTest.cpp': `namespace {
+
+struct LLVMContext {
+  int Kind = 0;
+};
+
+LLVMContext Ctx;
+
+}  // namespace
+
+int RandomIRBuilderTestKind() { return Ctx.Kind; }
+`,
+    // The same through one of glog's test programs.
+    'src/logging.cc': `struct LogSeverityNames {
+  const char *Name;
+};
+
+const char *SeverityName(const LogSeverityNames &Severity) {
+  return Severity.Name;
+}
+`,
+    'src/log_severity_unittest/glog_log_severity_constants.cc': `struct Severity {
+  int Value;
+};
+
+static Severity Fatal{3};
+
+int main() { return Fatal.Value == 3 ? 0 : 1; }
+`,
+  };
+  for (const [rel, content] of Object.entries(files)) {
+    fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true });
+    fs.writeFileSync(path.join(root, rel), content);
+  }
+  cg = await CodeGraph.init(root, { index: true });
+}, 60_000);
+
+afterAll(() => {
+  cg?.close();
+  if (root) fs.rmSync(root, { recursive: true, force: true });
+});
+
+const edgesFrom = (file: string) => {
+  const ids = cg.getNodesInFile(file).map((n) => n.id);
+  return cg.getOutgoingEdgesFrom(ids).filter((e) => e.kind !== 'contains').map((e) => cg.getNode(e.target)!);
+};
+
+describe('unittests/ and foo_unittest/ directories are test suites', () => {
+  it("production code never resolves into a unit test's own declarations", () => {
+    for (const file of ['lib/Object/WasmObjectFile.cpp', 'src/logging.cc']) {
+      const intoTests = edgesFrom(file).map((n) => n.filePath).filter((f) => /unittests?\//.test(f));
+      expect(intoTests, file).toEqual([]);
+    }
+  });
+
+  it("a test fixture's method no longer keeps a production call from resolving", () => {
+    expect(edgesFrom('lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp').map((n) => n.qualifiedName))
+      .toContain('llvm::TargetLoweringBase::getTypeToTransformTo');
+  });
+});

codegraph-kernel/src/resolve/resolver_upstream.rs

@@ -9,7 +9,7 @@ pub(super) fn test_suite_path(path: &str) -> bool {
         || re!(r"[._-](?:tests?|specs?|unittest)\.[a-z0-9]+$").is_match(name)
         || re!(r"(?:Test|Tests|TestCase)\.(?:java|kt|kts|swift|cs|scala|groovy|m|mm|vb|fs)$")
             .is_match(path)
-        || re!(r"(?:^|/)(?:tests?|__tests__|specs?|e2e)/").is_match(&lower)
+        || re!(r"(?:^|/)(?:tests?|__tests__|specs?|e2e|unittests|[\w.]+[-_]unittests?)/").is_match(&lower)
         || re!(r"(?:^|/)[A-Za-z0-9]*(?:Test|Tests|Spec)/").is_match(path)
 }
 

codegraph-kernel/src/resolve/tables.rs

@@ -1139,6 +1139,9 @@ pub(super) fn is_test_path(path: &str) -> bool {
         || re!(r"(?:^|/)(?:tests?|__tests__|specs?|testlib|testing|e2e)/").is_match(&lower)
         || re!(r"(?:^|/)[A-Za-z0-9]*(?:Test|Tests|Spec)/").is_match(path)
         || re!(r"(?:^|/)(?:[\w.]+[-_]test(?:s|ing)?|testdata|testutils?|test[-_]utils?|fakes?|mocks?|__mocks__|stubs)/").is_match(&lower)
+        // Unit-test trees: LLVM's and Breakpad's `unittests/`, glog's `dcheck_unittest/`. Not a
+        // bare `unittest/`: CPython's `Lib/unittest/` is the framework itself.
+        || re!(r"(?:^|/)(?:unittests|[\w.]+[-_]unittests?)/").is_match(&lower)
 }
 
 #[cfg(test)]
@@ -1155,6 +1158,17 @@ mod tests {
         }
     }
 
+    #[test]
+    fn unittest_directories_are_test_paths() {
+        for path in ["llvm/unittests/ADT/Foo.cpp", "src/dcheck_unittest/main.cc", "src/foo-unittests/main.cc"] {
+            assert!(is_test_path(path), "{path}");
+            assert!(super::super::resolver_upstream::test_suite_path(path), "{path}");
+        }
+        for path in ["Lib/unittest/case.py", "src/unittesting/a.cc"] {
+            assert!(!is_test_path(path), "{path}");
+        }
+    }
+
     #[test]
     fn rust_use_groups_flatten_in_order() {
         let uses = collect_rust_use_bindings("use a::{b::{C, D as Dee}, E};\n");

src/search/query-utils.ts

@@ -345,7 +345,11 @@ export function isTestPath(filePath: string): boolean {
     // Test-support modules and doubles by directory name: Gradle's
     // `core/data-test/`, `core/datastore-test/`, `:testing`; Go's `testdata/`;
     // `testutil(s)/`, `test-utils/`, `fakes/`, `mocks/`, `__mocks__/`.
-    /(?:^|\/)(?:[\w.]+[-_]test(?:s|ing)?|testdata|testutils?|test[-_]utils?|fakes?|mocks?|__mocks__|stubs)\//.test(lower)
+    /(?:^|\/)(?:[\w.]+[-_]test(?:s|ing)?|testdata|testutils?|test[-_]utils?|fakes?|mocks?|__mocks__|stubs)\//.test(lower) ||
+    // Unit-test trees: LLVM's and Breakpad's `unittests/`, glog's
+    // `dcheck_unittest/`. Not a bare `unittest/`: CPython's `Lib/unittest/` is
+    // the framework itself.
+    /(?:^|\/)(?:unittests|[\w.]+[-_]unittests?)\//.test(lower)
   ) {
     return true;
   }