diff --git a/CHANGELOG.md b/CHANGELOG.md index 31ab29da8..c750ed86e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,6 +80,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - `codegraph sync` and the file watcher now link a file to a module it imports under a name of its own when the module is added after the file was indexed, comes back after being deleted, or gets the export in a later edit: a default import like `import tagsController from './tag/tag.controller'`, a namespace import like `import * as api from './api'`, or an aliased one like `import { Component as GroupsPageWrapper } from './groups_page'`, in JavaScript, TypeScript, Vue, Svelte and Astro. Before, the import and every use of the name, such as a call, `new`, a base class, a JSX or template tag or an Express `app.use` mount, stayed unlinked until the importing file changed or the project was re-indexed, so the module looked unused and `codegraph affected` missed the files that use it. Re-index after upgrading to pick up links an earlier version missed. - `codegraph sync` and the file watcher now do the same for a Python import under a name of its own, like `from .models import User as UserModel`, `from . import helpers as h` or `from app.pkg import sub as subpkg`: when the module or package is added after the importing file was indexed, comes back after being deleted, or gets the imported name in a later edit, the import and every use of the alias, such as a call, a base class or the class passed as a value, now link to it. Before, they stayed unlinked until the importing file changed or the project was re-indexed. Re-index Python projects after upgrading to pick up links an earlier version missed. - `codegraph sync` and the file watcher now also link a Python import of a name a package re-exports, like `from fastapi import FastAPI` through `fastapi/__init__.py`, when that `__init__.py` is added after the importing files were indexed or gains the re-export in a later edit. Before, calls through the name stayed unlinked until each importing file changed or the project was re-indexed. Large projects gain the most: sync also used to skip, in any language, a module or package that more than 500 references were waiting for, and now restoring Django's `django/urls/__init__.py` relinks every file that imports `path` from it. Re-index Python projects after upgrading to pick up links an earlier version missed. +- `codegraph sync` and the file watcher now update the files that import through a module that re-exports what it imports, like a package's `__init__.py` or an `index.ts` barrel, when that module is deleted or added, and in Python also the files that reach it through another package re-exporting it. Before, deleting `shop/__init__.py` left every `from shop import Order` linked to the class it re-exported, and deleting `components/index.ts` left every `import { Button } from './components'` linked, until each importing file changed or the project was re-indexed. When such a module came back, the imports stayed linked the way they were while it was missing, and in a large Python project an import of a module whose name is as common as `models` or `test`, like `from django.test import TestCase`, stayed unlinked. Python, JavaScript, TypeScript, Vue, Svelte and Astro. - In C, C++ and Objective-C, a struct or class member declared as a pointer, reference, array or function, like `SharedState* shared;`, `jv elements[];` or `virtual Status Put(…) = 0;`, no longer makes the struct look like it inherits from the member's type. Before, `codegraph_explore` and impact listed those types as base classes, every struct holding a pointer to a type as one of its subclasses, and a method of that struct as an override of the type's method with the same name. Real base classes like `class Derived : public Base`, Objective-C superclasses and Go's embedded fields are unchanged. Re-index C, C++ and Objective-C projects after upgrading. - In C++, a call on a variable, parameter or member declared through a `typedef` or `using` alias now links to the method of the type the alias names, with the alias looked up the way C++ does it: in the calling function, its class and the classes that class inherits from, then the namespaces around it. In google/leveldb, `table_.Insert(buf)` on a `Table table_;` declared next to `typedef SkipList Table;` now reaches `SkipList::Insert` instead of the unrelated `HandleTable::Insert`, and `Table::Iterator iter(&table_);` reaches `SkipList::Iterator`'s constructor. A class that only shares the alias's name is no longer taken for it, and when the aliased type has no such method, as with an alias of `std::vector` or of a template parameter, no method is guessed from the receiver's name. Re-index C++ projects after upgrading. - In C++, a call on a variable, parameter or member declared with template arguments nested inside its template arguments, like `autovector>` or `Striped>`, now links to the method of the class it is declared as. Before, the nested arguments hid that class: in RocksDB, `files_marked_for_compaction_.clear()` on such an `autovector` linked to an unrelated `CompactionInputFiles::clear` that only shares words with the variable's name, and `mutex_.Get(key)` on a `Striped` linked to nothing. Re-index C++ projects after upgrading. diff --git a/__tests__/sync-module-importers.test.ts b/__tests__/sync-module-importers.test.ts new file mode 100644 index 000000000..2a8c27236 --- /dev/null +++ b/__tests__/sync-module-importers.test.ts @@ -0,0 +1,154 @@ +/** + * A sync re-resolves what an import statement binds when the module it names + * appears or goes away. + * + * `from shop import Order` reaches `shop/models.py` through `shop/__init__.py`, + * which re-exports `Order`; `import { Button } from './components'` reaches + * `components/Button.ts` through `components/index.ts`. Deleting the + * re-exporting file left every importer linked through it: those edges point + * into files that stay, so nothing removed them, while a fresh index of the + * tree without the file links them by name or not at all. And a module that + * appeared after its importers were bound by name in its absence left those + * edges as the name lookup made them, while a fresh index binds them through + * the module. + */ +import { describe, it, expect, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import CodeGraph from '../src/index'; + +type Files = Record; + +/** + * Python: an absolute import, imports from inside and beside the package, a + * renamed re-export, a plain module, a subpackage imported from the package + * around it (`from app import orm`, then `orm.Model`), and an import inside a + * function, which has no import node. Two packages re-export the shop package + * in turn, one with a star import and one by name, and are imported in its + * stead. `vendored.py` makes `shop` as common a name among failed imports as + * `models` or `test` is in a large project: more than sync's failed-import + * retry looks up by name. + */ +const PYTHON: Files = { + 'vendored.py': Array.from({ length: 501 }, (_, i) => `import vendor${i}.shop\n`).join(''), + 'app/store/__init__.py': 'from app.shop import *\n', + 'report.py': 'from app.store import Order\n\n\ndef report():\n return Order()\n', + 'app/front/__init__.py': 'from ..shop import Cart as Basket\n', + 'kiosk.py': 'from app.front import Basket\n\n\ndef sell():\n return Basket()\n', + 'app/__init__.py': '', + 'app/shop/models.py': 'class _Order:\n pass\n\n\nclass Cart:\n pass\n', + 'app/shop/__init__.py': 'from .models import _Order as Order\nfrom .models import Cart\n', + 'app/shop/api/views.py': 'from .. import Order, Cart\n\n\ndef view():\n return Order(), Cart()\n', + 'app/checkout.py': 'from .shop import Cart\n\n\ndef run():\n return Cart()\n', + 'main.py': 'from app.shop import Order\n\n\ndef go():\n return Order()\n', + 'lib/__init__.py': '', + 'lib/impl.py': 'def helper():\n return 1\n', + 'lib/api.py': 'from .impl import helper as public_helper\n', + 'tool.py': 'from lib.api import public_helper\n\n\ndef use():\n return public_helper()\n', + 'app/orm/base.py': 'class _Model:\n pass\n', + 'app/orm/__init__.py': 'from .base import _Model as Model\n', + 'app/blog.py': 'from app import orm\n\n\nclass Post(orm.Model):\n pass\n', + 'test_orders.py': 'def test_order():\n from app.shop import Order\n return Order()\n', +}; +const PYTHON_BARRELS = ['app/shop/__init__.py', 'lib/api.py', 'app/orm/__init__.py']; + +/** ES modules: an `index.ts` re-exporting under the same name and under another. */ +const TS: Files = { + 'package.json': '{"name":"app"}', + 'src/components/Button.ts': 'export function Button() { return 1; }\n', + 'src/components/Icon.ts': 'export function BaseIcon() { return 2; }\n', + 'src/components/index.ts': "export { Button } from './Button';\nexport { BaseIcon as Icon } from './Icon';\n", + 'src/app.ts': "import { Button, Icon } from './components';\nexport function render() { return [Button(), Icon()]; }\n", +}; +const TS_BARRELS = ['src/components/index.ts']; + +let roots: string[] = []; +let graphs: CodeGraph[] = []; + +afterEach(() => { + for (const graph of graphs) graph.close(); + graphs = []; + for (const root of roots) fs.rmSync(root, { recursive: true, force: true }); + roots = []; +}); + +function tempRoot(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cg-sync-importers-')); + roots.push(root); + return root; +} + +function write(root: string, files: Files): void { + for (const [file, text] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); + fs.writeFileSync(path.join(root, file), text); + } +} + +async function index(root: string): Promise { + const graph = await CodeGraph.init(root, { index: true }); + graphs.push(graph); + return graph; +} + +/** Every edge, by its ends' natural keys, with where it is written and how it resolved. */ +function edgesOf(graph: CodeGraph): string[] { + const keys = new Map( + graph.getFiles() + .flatMap((file) => graph.getNodesInFile(file.path)) + .map((n) => [n.id, `${n.kind} ${n.filePath}:${n.qualifiedName}:${n.startLine}`] as const) + ); + return graph.getOutgoingEdgesFrom([...keys.keys()]) + .map((e) => `${e.kind} ${keys.get(e.source)} -> ${keys.get(e.target) ?? e.target} @${e.line}:${e.column} ${JSON.stringify(e.metadata ?? {})}`) + .sort(); +} + +/** The tree, indexed from scratch in a folder of its own: what the sync has to match. */ +async function fullIndexOf(files: Files): Promise { + const root = tempRoot(); + write(root, files); + return edgesOf(await index(root)); +} + +const without = (files: Files, gone: string[]): Files => + Object.fromEntries(Object.entries(files).filter(([file]) => !gone.includes(file))); +const only = (files: Files, kept: string[]): Files => + Object.fromEntries(Object.entries(files).filter(([file]) => kept.includes(file))); + +describe.each([ + ['Python package and module', PYTHON, PYTHON_BARRELS], + ['ES module barrel', TS, TS_BARRELS], +])('sync re-resolves the imports through a re-exporting file: %s', (_label, files, barrels) => { + it('when it is deleted', async () => { + const root = tempRoot(); + write(root, files); + const graph = await index(root); + + for (const file of barrels) fs.rmSync(path.join(root, file)); + expect((await graph.sync()).filesRemoved).toBe(barrels.length); + expect(edgesOf(graph)).toEqual(await fullIndexOf(without(files, barrels))); + }, 120_000); + + it('when it comes back after it was deleted', async () => { + const root = tempRoot(); + write(root, files); + const graph = await index(root); + + for (const file of barrels) fs.rmSync(path.join(root, file)); + await graph.sync(); + write(root, only(files, barrels)); + expect((await graph.sync()).filesAdded).toBe(barrels.length); + expect(edgesOf(graph)).toEqual(await fullIndexOf(files)); + }, 120_000); + + it('when it appears after its importers were bound without it', async () => { + const root = tempRoot(); + write(root, without(files, barrels)); + const graph = await index(root); + + write(root, only(files, barrels)); + expect((await graph.sync()).filesAdded).toBe(barrels.length); + expect(edgesOf(graph)).toEqual(await fullIndexOf(files)); + }, 120_000); +}); diff --git a/src/db/queries.ts b/src/db/queries.ts index e5a2cccf0..9700e3d63 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -74,6 +74,16 @@ const IS_INTERFACE_MEMBER = (alias: string): string => `EXISTS ( WHERE ce.target = ${alias}.id AND ce.kind = 'contains' AND owner.kind = 'interface' )`; +/** + * A SQL predicate: is the reference name in `column` written through one of + * the names in the JSON array bound to the next two parameters, as the name + * itself or as a member read through it (`Order.objects` through `Order`)? + */ +const THROUGH_NAMES = (column: string): string => `( + ${column} IN (SELECT value FROM json_each(?)) + OR (instr(${column}, '.') > 1 AND substr(${column}, 1, instr(${column}, '.') - 1) IN (SELECT value FROM json_each(?))) +)`; + /** * How much of the exact-name bonus a `deprioritize`d path keeps (#982). Damped * rather than zeroed: a query that genuinely targets that tree must still rank @@ -3086,6 +3096,134 @@ export class QueryBuilder { })); } + /** + * Resolution edges out of the nodes of `filePath` whose reference was + * written through one of `names` (see {@link THROUGH_NAMES}), with the source + * file and language a resurrection needs and the file of each target. + * Synthesized (`provenance='heuristic'`) and `contains` edges are left out. + */ + getResolutionEdgesThrough( + filePath: string, + names: readonly string[] + ): Array { + const rows = this.db + .prepare( + `SELECT * FROM ( + SELECT e.*, src.file_path AS source_file_path, src.language AS source_language, tgt.file_path AS target_file_path, + CASE WHEN json_valid(e.metadata) THEN json_extract(e.metadata, '$.refName') END AS ref_name + FROM nodes src + JOIN edges e ON e.source = src.id + JOIN nodes tgt ON tgt.id = e.target + WHERE src.file_path = ? + AND e.kind != 'contains' + AND (e.provenance IS NULL OR e.provenance != 'heuristic') + ) WHERE ${THROUGH_NAMES('ref_name')}` + ) + .all(filePath, JSON.stringify(names), JSON.stringify(names)) as Array< + EdgeRow & { source_file_path: string; source_language: Language; target_file_path: string } + >; + return rows.map((row) => ({ + ...rowToEdge(row), + edgeId: row.id, + sourceFilePath: row.source_file_path, + sourceLanguage: row.source_language, + targetFilePath: row.target_file_path, + })); + } + + /** + * The row ids of the failed references of `filePath` that import one of + * `modules` or are written through one of `names` (see {@link THROUGH_NAMES}). + */ + getFailedReferencesThrough(filePath: string, modules: readonly string[], names: readonly string[]): number[] { + return (this.db + .prepare( + `SELECT id FROM unresolved_refs + WHERE file_path = ? AND status = 'failed' + AND ((reference_kind = 'imports' AND reference_name IN (SELECT value FROM json_each(?))) + OR ${THROUGH_NAMES('reference_name')})` + ) + .all(filePath, JSON.stringify(modules), JSON.stringify(names), JSON.stringify(names)) as Array<{ id: number }>) + .map((row) => row.id); + } + + /** The targets, in other files, of the non-synthesized `imports` edges out of the nodes of `filePath`. */ + getImportTargetsFromFile(filePath: string): string[] { + return (this.db + .prepare( + `SELECT DISTINCT e.target AS target + FROM nodes src + JOIN edges e ON e.source = src.id + JOIN nodes tgt ON tgt.id = e.target + WHERE src.file_path = ? + AND e.kind = 'imports' + AND (e.provenance IS NULL OR e.provenance != 'heuristic') + AND tgt.file_path != ?` + ) + .all(filePath, filePath) as Array<{ target: string }>) + .map((row) => row.target); + } + + /** + * The `imports` edges into the node `targetId` from files other than + * `exceptFile`: each importing file, its language, and the module specifier + * it wrote (the edge's `refName`; null when it carries none). + */ + getImportsInto( + targetId: string, + exceptFile: string + ): Array<{ filePath: string; language: Language; refName: string | null }> { + return (this.db + .prepare( + `SELECT src.file_path AS file_path, src.language AS language, + CASE WHEN json_valid(e.metadata) THEN json_extract(e.metadata, '$.refName') END AS ref_name + FROM edges e + JOIN nodes src ON src.id = e.source + WHERE e.target = ? AND e.kind = 'imports' AND src.file_path != ?` + ) + .all(targetId, exceptFile) as Array<{ file_path: string; language: Language; ref_name: unknown }>) + .map((row) => ({ + filePath: row.file_path, + language: row.language, + refName: typeof row.ref_name === 'string' ? row.ref_name : null, + })); + } + + /** The files with a resolution edge into one of `targetIds`. */ + getFilesWithEdgesInto(targetIds: string[]): string[] { + const files = new Set(); + for (let i = 0; i < targetIds.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = targetIds.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const placeholders = chunk.map(() => '?').join(','); + const rows = this.db + .prepare( + `SELECT DISTINCT src.file_path AS file_path + FROM edges e + JOIN nodes src ON src.id = e.source + WHERE e.target IN (${placeholders}) + AND e.kind != 'contains' + AND (e.provenance IS NULL OR e.provenance != 'heuristic')` + ) + .all(...chunk) as Array<{ file_path: string }>; + for (const row of rows) files.add(row.file_path); + } + return [...files]; + } + + /** The import nodes named one of `names` (an import node is named by the module path it imports). */ + getImportNodesByNames(names: string[]): Array<{ filePath: string; name: string; language: Language }> { + const out: Array<{ filePath: string; name: string; language: Language }> = []; + for (let i = 0; i < names.length; i += SQLITE_PARAM_CHUNK_SIZE) { + const chunk = names.slice(i, i + SQLITE_PARAM_CHUNK_SIZE); + const placeholders = chunk.map(() => '?').join(','); + const rows = this.db + .prepare(`SELECT file_path, name, language FROM nodes WHERE name IN (${placeholders}) AND kind = 'import'`) + .all(...chunk) as Array<{ file_path: string; name: string; language: Language }>; + for (const row of rows) out.push({ filePath: row.file_path, name: row.name, language: row.language }); + } + return out; + } + // =========================================================================== // File Operations // =========================================================================== diff --git a/src/extraction/index.ts b/src/extraction/index.ts index 42ff58f04..8623d7203 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -35,6 +35,7 @@ import { declaredDependencies } from '../resolution/frameworks/package-deps'; import type { ResolutionContext } from '../resolution/types'; import { createYielder, type MaybeYield } from '../resolution/cooperative-yield'; import { resurrectRefFromDroppedEdge } from '../resolution/resurrect-ref'; +import { createImportReader, reopenReferencesThroughRemovedFiles } from '../resolution/module-importers'; import { MAX_SOURCE_FILE_SIZE_BYTES, oversizeStamp, readBoundedSource, readBoundedSourceSync } from '../file-limits'; export { oversizeStamp }; @@ -3280,6 +3281,8 @@ export class ExtractionOrchestrator { // BEFORE their nodes are replaced/deleted. Compared against the post-store // pairs below to derive `definitionDelta` (CG-33). const pairsBefore = new Set(); + // Importers' import statements, read once per sync for the files it removes. + const importReader = createImportReader(this.rootDir); onProgress?.({ phase: 'scanning', @@ -3385,14 +3388,31 @@ export class ExtractionOrchestrator { this.queries.deleteFile(tracked.path); filesRemoved++; }; + const removed: FileRecord[] = []; for (const tracked of trackedFiles) { if (!currentSet.has(tracked.path) || !fs.existsSync(path.join(this.rootDir, tracked.path))) { - removeTracked(tracked); + removed.push(tracked); } if (++reconcileChecks % SYNC_RECONCILE_YIELD_INTERVAL === 0) { await new Promise((resolve) => setImmediate(resolve)); } } + // The references other files make THROUGH the removed ones to files that + // stay (a package `__init__.py` or an `index.ts` re-exporting what it + // imports) go back to pending too: the cascade keeps their edges, and a + // fresh index of the tree without these files binds them differently or + // not at all. Before any of them is deleted, and for all of them at once. + if (removed.length > 0) { + const tImporters = Date.now(); + const reopened = reopenReferencesThroughRemovedFiles(this.queries, importReader, removed); + if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-removed-module-importers: ${Date.now() - tImporters}ms (${reopened} refs re-opened)`); + } + for (const tracked of removed) { + removeTracked(tracked); + if (++reconcileChecks % SYNC_RECONCILE_YIELD_INTERVAL === 0) { + await new Promise((resolve) => setImmediate(resolve)); + } + } // Adds / modifications. for (const filePath of currentFiles) { @@ -3441,7 +3461,10 @@ export class ExtractionOrchestrator { // Not source after all — an MPEG transport stream named `.ts` (#1910): // a new one is ignored, a tracked file that became one is removed. if (content === null) { - if (tracked) removeTracked(tracked); + if (tracked) { + reopenReferencesThroughRemovedFiles(this.queries, importReader, [tracked]); + removeTracked(tracked); + } continue; } const contentHash = hashContent(content); diff --git a/src/index.ts b/src/index.ts index 75e32ad28..2e777dbf5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -53,6 +53,7 @@ import { ResolutionResult, } from './resolution'; import { hasSynthesisPattern, isJsxChildCandidate } from './resolution/callback-synthesizer'; +import { createImportReader, reopenReferencesThroughAddedFiles } from './resolution/module-importers'; import { GraphTraverser, GraphQueryManager } from './graph'; import { findNamedCopybooks, type NamedCopybook } from './graph/cobol-copybooks'; import { ContextBuilder, createContextBuilder } from './context'; @@ -1002,6 +1003,9 @@ export class CodeGraph { // per-name ceiling counts only edges up to it (see // resurrectStaleResolutionEdges). let edgesBeforeRetry: number | undefined; + // And the last edge before this sync resolved anything: an importer's + // edge up to it was bound before a module this sync adds was there. + const edgesBeforeResolution = this.queries.getMaxEdgeId(); if (filesChanged) { if (result.changedFilePaths) { // Scope resolution to changed files (git fast path — bounded set) @@ -1079,6 +1083,21 @@ export class CodeGraph { }); } if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-failed-ref-retry: ${Date.now() - tRetry}ms (${retryable.length} refs)`); + + // A module that appeared, such as a package `__init__.py` or an + // `index.ts` re-exporting what it imports: what the files importing + // it bound by name in its absence, or failed, goes back to pending + // for the sweep below, which binds it through the module as a fresh + // index does. (The removal half runs before removed files are deleted.) + if (result.addedFilePaths && result.addedFilePaths.length > 0) { + const tAdded = Date.now(); + const added = result.addedFilePaths.flatMap((filePath) => { + const file = this.queries.getFileByPath(filePath); + return file ? [{ path: filePath, language: file.language }] : []; + }); + const reopened = reopenReferencesThroughAddedFiles(this.queries, createImportReader(this.projectRoot), added, edgesBeforeResolution); + if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] sync-added-module-importers: ${Date.now() - tAdded}ms (${reopened} refs re-opened)`); + } } else { // No git info — use batched resolution to avoid OOM const unresolvedCount = this.queries.getUnresolvedReferencesCount(); diff --git a/src/resolution/import-resolver.ts b/src/resolution/import-resolver.ts index c3b66202b..f6fa14041 100644 --- a/src/resolution/import-resolver.ts +++ b/src/resolution/import-resolver.ts @@ -3227,7 +3227,7 @@ function projectFolderSuffixes(context: ResolutionContext): Set { * from `app/sub/views.py`, `.models` → 'app/sub/models', `..` → 'app' and * `..forms` → 'app/forms'. Null when its dots climb above the project. */ -function pythonModulePath(source: string, fromFile: string): string | null { +export function pythonModulePath(source: string, fromFile: string): string | null { const name = source.replace(/^\.+/, ''); const dots = source.length - name.length; if (dots === 0) return name.replace(/\./g, '/'); diff --git a/src/resolution/module-importers.ts b/src/resolution/module-importers.ts new file mode 100644 index 000000000..2f8a1c575 --- /dev/null +++ b/src/resolution/module-importers.ts @@ -0,0 +1,362 @@ +/** + * The references an import statement makes through a module that appears or + * goes away. + * + * `from shop import Order` reaches `shop/models.py` through `shop/__init__.py`, + * which re-exports `Order`; `import { Button } from './components'` reaches + * `components/Button.ts` through `components/index.ts`. When the re-exporting + * file goes away, their edges point into files that stay, so the cascade keeps + * them; when it comes back, the edges a name lookup made in its absence stay. + * Nothing else revisits either: the importers did not change. A fresh index + * binds them through the module when it is there, and another way (by name) + * or not at all when it is not. + * + * A Python module that re-exports another one (`from .shop import *`, or a + * package's `__init__.py` importing names from it) is the way in for its own + * importers, so theirs are followed too, as deep as the resolver chases a + * name through such modules. An ES module's `export … from` leaves no import + * edge or node to find it by, so an `index.ts` re-exporting another is not + * followed. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import type { QueryBuilder } from '../db/queries'; +import type { Language, UnresolvedReference } from '../types'; +import type { ImportMapping } from './types'; +import { extractImportMappings, pythonModulePath } from './import-resolver'; +import { resurrectRefFromDroppedEdge } from './resurrect-ref'; + +/** + * Languages whose import bindings name what their references are written as: + * Python's `from x import Y` and the ES-module family's `import { Y } from`. + * An `#include` or a Go package import binds no reference name of its own. + */ +const BINDING_LANGUAGES = new Set([ + 'python', 'typescript', 'tsx', 'javascript', 'jsx', 'arkts', 'vue', 'svelte', 'astro', +]); + +/** + * How many re-exporting Python modules deep the importers of a file are + * followed: as deep as the resolver chases a name through star imports and + * re-exports (`pythonModuleSymbol`). + */ +const PYTHON_REEXPORT_DEPTH = 3; + +/** A Python star import, read the way the resolver reads one. */ +const PYTHON_STAR_IMPORT = /^\s*from\s+([\w.]+)\s+import\s+\*/gm; + +/** + * A file this sync adds or removes, or a module re-exporting from one, with + * the names it passes on from them (null: whatever it has). + */ +interface Module { + path: string; + language: Language; + passes: Set | null; +} + +/** + * Each importer's language and, per module specifier of a statement naming + * one of the modules, which of its bindings go through it: all of them + * (null), or those importing one of these names — what a re-exporting module + * passes on, or the module the package around a file imports as a name + * (`from django.db import models`, whose `models` is the file's module). + */ +type Importers = Map | null> }>; + +/** A statement naming a module: whole, or as `onlyName` of the package around it. */ +type AddImporter = (importer: string, language: Language, source: string, module: Module, onlyName?: string) => void; + +/** What a file's import statements say, each file read once per sync. */ +export interface ImportReader { + /** Its import mappings. */ + mappings(file: string, language: Language): ImportMapping[]; + /** The modules a Python file star-imports, which have no import mapping. */ + starImports(file: string, language: Language): string[]; +} + +export function createImportReader(rootDir: string): ImportReader { + const memo = new Map(); + const read = (file: string, language: Language) => { + let entry = memo.get(file); + if (!entry) { + let content = ''; + try { + content = fs.readFileSync(path.join(rootDir, file), 'utf8'); + } catch { + // Gone or unreadable: it imports nothing. + } + entry = { + mappings: content ? extractImportMappings(file, content, language) : [], + starImports: language === 'python' ? [...content.matchAll(PYTHON_STAR_IMPORT)].map((m) => m[1]!) : [], + }; + memo.set(file, entry); + } + return entry; + }; + return { + mappings: (file, language) => read(file, language).mappings, + starImports: (file, language) => read(file, language).starImports, + }; +} + +/** + * Re-open, as pending references for the sync's sweep, the references the + * files importing any of `removed` make through those imports. The sweep binds + * them against the post-sync graph, or parks them where a sync that brings a + * file back finds them. Called before the files' rows are deleted, for all of + * them at once, so an importer of several is visited once; an edge into a + * removed file is left to the resurrection its removal does. Returns the + * number of references re-opened. + */ +export function reopenReferencesThroughRemovedFiles( + queries: QueryBuilder, + reader: ImportReader, + removed: ReadonlyArray<{ path: string; language: Language }>, +): number { + const gone = new Set(removed.map((file) => file.path)); + const bindings = bindingsOf(importersOf(queries, removed, reader), reader); + return reopenEdges(queries, bindings, (e) => !gone.has(e.targetFilePath)); +} + +/** + * The other half: after a sync's retries, re-open the references the files + * importing any of `added` make through those imports — the edges a name + * lookup made while a module was missing (only those up to + * `resolvedUpToEdgeId`: later ones were just resolved with the modules there) + * and the references that failed, the import of the module itself among + * them. (The failed-import retry looks those up by the module's name, and + * skips a name as common as `models` or `test`.) Returns the number re-opened. + */ +export function reopenReferencesThroughAddedFiles( + queries: QueryBuilder, + reader: ImportReader, + added: ReadonlyArray<{ path: string; language: Language }>, + resolvedUpToEdgeId: number, +): number { + const bindings = bindingsOf(importersOf(queries, added, reader), reader); + const failed = [...bindings].flatMap(([file, { modules, names }]) => queries.getFailedReferencesThrough(file, modules, names)); + return reopenEdges(queries, bindings, (e) => e.edgeId <= resolvedUpToEdgeId) + queries.reopenFailedReferences(failed); +} + +/** + * The files with an import statement naming one of `files`, or one of the + * modules re-exporting from them, other than `files` themselves (going away, + * or just resolved with the others in place). + */ +function importersOf( + queries: QueryBuilder, + files: ReadonlyArray<{ path: string; language: Language }>, + reader: ImportReader, +): Importers { + const importers: Importers = new Map(); + const changed = new Set(files.map((file) => file.path)); + const seen = new Set(changed); + let level: Module[] = files.filter((file) => BINDING_LANGUAGES.has(file.language)).map((file) => ({ ...file, passes: null })); + for (let depth = 1; level.length > 0; depth++) { + // Per importer found here, its statements naming each module of the level. + const naming = new Map }>(); + const add: AddImporter = (importer, language, source, module, onlyName) => { + if (!BINDING_LANGUAGES.has(language) || changed.has(importer)) return; + const only = onlyName !== undefined ? new Set([onlyName]) : module.passes; + const entry = importers.get(importer) ?? { language, sources: new Map | null>() }; + const names = entry.sources.get(source); + entry.sources.set(source, names === null || only === null ? null : new Set([...(names ?? []), ...only])); + importers.set(importer, entry); + const found = naming.get(importer) ?? { language, statements: [] }; + found.statements.push({ source, module, onlyName }); + naming.set(importer, found); + }; + for (const module of level) { + // A module-path import that resolved to the module: an ES module + // specifier, a dotted Python import. + for (const i of queries.getImportsInto(`file:${module.path}`, module.path)) { + if (i.refName !== null) add(i.filePath, i.language, i.refName, module); + } + } + const python = level.filter((module) => module.language === 'python'); + if (python.length > 0) addPythonImporters(queries, python, reader, add); + // The next level: the Python importers that pass on what they import from these. + if (depth > PYTHON_REEXPORT_DEPTH) break; + const next: Module[] = []; + for (const [importer, { language, statements }] of naming) { + if (language !== 'python' || seen.has(importer)) continue; + const passes = pythonPassedOn(importer, statements, reader); + if (passes === undefined) continue; + seen.add(importer); + next.push({ path: importer, language, passes }); + } + level = next; + } + return importers; +} + +/** + * What the Python module `importer` passes on of what its `statements` + * import, or undefined when the resolver reads no name through it: what it + * star-imports, and for a package's `__init__.py`, the names it imports too. + */ +function pythonPassedOn( + importer: string, + statements: ReadonlyArray<{ source: string; module: Module; onlyName?: string }>, + reader: ImportReader, +): Set | null | undefined { + let passes: Set | null | undefined; + const pass = (names: Set | null): void => { + passes = passes === null || names === null ? null : new Set([...(passes ?? []), ...names]); + }; + const stars = new Set(reader.starImports(importer, 'python')); + const isPackage = importer === '__init__.py' || importer.endsWith('/__init__.py'); + for (const { source, module, onlyName } of statements) { + if (onlyName === undefined && stars.has(source)) pass(module.passes); + if (!isPackage) continue; + const names = reader.mappings(importer, 'python') + .filter((m) => m.source === source && (onlyName !== undefined + ? m.exportedName === onlyName + : module.passes === null || module.passes.has(m.exportedName))) + .map((m) => m.localName); + if (names.length > 0) pass(new Set(names)); + } + return passes; +} + +/** + * Python's other ways to name the modules: an import node whose module path + * denotes one, relative to the importer's own package or absolutely (`from . + * import Order`, `from .. import Order`, `from shop import Order`), which + * never resolve to a file, or the package around one with its module among + * the names imported (`from django.db import models`). An import inside a + * function has no import node, so a file that reaches what the modules pass + * on — the targets of their own import edges — counts too when its imports + * name one. + */ +function addPythonImporters( + queries: QueryBuilder, + modules: readonly Module[], + reader: ImportReader, + add: AddImporter, +): void { + // What a module path a statement names stands for: a module whole, or a + // package with the module of `onlyName` among the names imported. + type Naming = { module: Module; onlyName?: string }; + const byPath = new Map(); + const bySuffix = new Map(); + const importNames = new Set(); + const push = (map: Map, key: string, naming: Naming): void => { + const list = map.get(key); + if (list) list.push(naming); + else map.set(key, [naming]); + }; + const index = (modulePath: string, naming: Naming): void => { + push(byPath, modulePath, naming); + const segments = modulePath.split('/'); + for (let i = 0; i < segments.length; i++) push(bySuffix, segments.slice(i).join('/'), naming); + for (const name of pythonImportNamesFor(modulePath)) importNames.add(name); + }; + for (const module of modules) { + const modulePath = pythonModuleOfFile(module.path); + if (!modulePath) continue; + index(modulePath, { module }); + const slash = modulePath.lastIndexOf('/'); + if (slash > 0) index(modulePath.slice(0, slash), { module, onlyName: modulePath.slice(slash + 1) }); + } + if (importNames.size === 0) return; + // A relative import names a module by where the importer sits, an absolute + // one by the tail of its path (the project's import root is not known). + const named = (source: string, importer: string): Naming[] => + source.startsWith('.') + ? byPath.get(pythonModulePath(source, importer)?.replace(/\/+$/, '') ?? '') ?? [] + : bySuffix.get(source.replace(/\./g, '/')) ?? []; + + for (const node of queries.getImportNodesByNames([...importNames])) { + if (node.language !== 'python') continue; + for (const { module, onlyName } of named(node.name, node.filePath)) add(node.filePath, 'python', node.name, module, onlyName); + } + const paths = new Set(modules.map((module) => module.path)); + const passedOnTargets = new Set([...paths].flatMap((filePath) => queries.getImportTargetsFromFile(filePath))); + if (passedOnTargets.size === 0) return; + for (const candidate of queries.getFilesWithEdgesInto([...passedOnTargets])) { + if (!candidate.endsWith('.py') || paths.has(candidate)) continue; + for (const m of reader.mappings(candidate, 'python')) { + for (const { module, onlyName } of named(m.source, candidate)) { + if (onlyName === undefined || onlyName === m.exportedName) add(candidate, 'python', m.source, module, onlyName); + } + } + } +} + +/** + * Per importer, what its statements naming the modules import: the modules + * named whole (not the package around one), and the local names bound. + */ +function bindingsOf(importers: Importers, reader: ImportReader): Map { + const bindings = new Map(); + for (const [file, { language, sources }] of importers) { + const names = new Set( + reader.mappings(file, language) + .filter((m) => { + const only = sources.get(m.source); + return only === null || (only !== undefined && only.has(m.exportedName)); + }) + .map((m) => m.localName) + ); + const modules = [...sources].filter(([, only]) => only === null).map(([source]) => source); + if (modules.length > 0 || names.size > 0) bindings.set(file, { modules, names: [...names] }); + } + return bindings; +} + +/** + * Swap for their references the edges of each importer's references through + * its bound names — the binding itself or a member read through it + * (`Order.objects`, `ns.fn`) — that pass `keep`. Returns how many. + */ +function reopenEdges( + queries: QueryBuilder, + bindings: Map, + keep: (e: { edgeId: number; targetFilePath: string }) => boolean, +): number { + const edgeIds: number[] = []; + const refs: UnresolvedReference[] = []; + for (const [file, { names }] of bindings) { + if (names.length === 0) continue; + for (const e of queries.getResolutionEdgesThrough(file, names)) { + if (!keep(e)) continue; + const ref = resurrectRefFromDroppedEdge(e); + if (!ref) continue; + edgeIds.push(e.edgeId); + refs.push(ref); + } + } + if (refs.length > 0) queries.replaceResolutionEdgesWithUnresolvedRefs(edgeIds, refs); + return refs.length; +} + +/** A Python file's module as a path: `shop/models.py` → 'shop/models', `shop/__init__.py` → 'shop'. Null for a top-level `__init__.py`. */ +function pythonModuleOfFile(filePath: string): string | null { + if (!filePath.endsWith('.py') || filePath === '__init__.py') return null; + if (filePath.endsWith('/__init__.py')) return filePath.slice(0, -'/__init__.py'.length); + return filePath.slice(0, -'.py'.length); +} + +/** Deepest relative import a lookup spells out: `from ........ import x`. */ +const MAX_RELATIVE_DOTS = 8; + +/** + * The import names that can denote the module at `modulePath`: its trailing + * dotted paths (`models`, `shop.models`), each also behind one dot or more + * (`.models`, `..shop.models`), and the dots alone (`.`, `..`), which name a + * package by where the importer sits. + */ +function pythonImportNamesFor(modulePath: string): string[] { + const segments = modulePath.split('/'); + const absolute = segments.map((_, i) => segments.slice(i).join('.')); + const names = new Set(absolute); + for (let dots = 1; dots <= MAX_RELATIVE_DOTS; dots++) { + const prefix = '.'.repeat(dots); + names.add(prefix); + for (const name of absolute) names.add(prefix + name); + } + return [...names]; +}