From 313edb98d0fc7b39fad005fa6c85b5cfbb6ddd03 Mon Sep 17 00:00:00 2001 From: fridaybyjarvis Date: Fri, 11 Sep 2026 15:27:40 -0500 Subject: [PATCH 1/5] feat(gs): extract Google Apps Script as JavaScript MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apps Script sources are plain JavaScript saved as `.gs` — what `clasp` pulls down and what every Apps Script project in a repo looks like. `.gs` was absent from `CODE_EXTENSIONS`, the extractor `_DISPATCH`, and the JS language-family maps, so an entire Apps Script project was classified as non-code and contributed nothing: a build over a repo whose logic lives in `.gs` returned only its markdown and `appsscript.json` manifests. On a 29-file Apps Script project this was 83 nodes / 84 edges before and 285 / 835 after, with every god node before the change coming from a doc. `.gs` is not exclusive to Apps Script — GLSL geometry shaders and Gosu use it too — so routing goes through `_is_apps_script`, which withholds the extractor when a GLSL or Gosu marker is present rather than force-parsing the file into garbage. The sniff runs the opposite way from the `.m` Objective-C/MATLAB split: `.m` has a genuine rival and is guilty until proven innocent, while `.gs` in a repo is nearly always Apps Script and its own markers are unusable as a positive test (a pure-logic `.gs` helper calls no `SpreadsheetApp` or `DriveApp` service at all). `_JS_TS_CALL_SUFFIXES` deliberately does not gain `.gs`: that gate drops cross-file calls lacking import evidence, which is right for ES modules and wrong for Apps Script, where every file shares one global scope and imports do not exist. --- graphify/analyze.py | 2 +- graphify/build.py | 2 +- graphify/cli.py | 2 +- graphify/detect.py | 2 +- graphify/extract.py | 42 +++++++++- graphify/extractors/models.py | 3 +- tests/test_gs_apps_script.py | 150 ++++++++++++++++++++++++++++++++++ 7 files changed, 196 insertions(+), 7 deletions(-) create mode 100644 tests/test_gs_apps_script.py diff --git a/graphify/analyze.py b/graphify/analyze.py index 5186538e16..4940dd8b81 100644 --- a/graphify/analyze.py +++ b/graphify/analyze.py @@ -31,7 +31,7 @@ # Language families — extensions sharing a runtime can legitimately call each other _LANG_FAMILY: dict[str, str] = { **{e: "python" for e in (".py", ".pyw")}, - **{e: "js" for e in (".js", ".jsx", ".mjs", ".cjs", ".ejs", ".ts", ".tsx", ".mts", ".cts", ".vue", ".svelte")}, + **{e: "js" for e in (".js", ".jsx", ".mjs", ".cjs", ".ejs", ".gs", ".ts", ".tsx", ".mts", ".cts", ".vue", ".svelte")}, **{e: "go" for e in (".go",)}, **{e: "rust" for e in (".rs",)}, **{e: "jvm" for e in (".java", ".kt", ".kts", ".scala")}, diff --git a/graphify/build.py b/graphify/build.py index 1ddab9d22c..140bcd1e3c 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -334,7 +334,7 @@ def finalize_raw_graph_endpoints(data: dict) -> None: # which imports build.py — a cycle) and deliberately mirrors extract._LANG_FAMILY_BY_EXT. _EDGE_LANG_FAMILY: dict[str, str] = { ".py": "py", ".pyi": "py", - ".js": "js", ".mjs": "js", ".cjs": "js", ".jsx": "js", + ".js": "js", ".mjs": "js", ".cjs": "js", ".jsx": "js", ".gs": "js", ".ts": "js", ".tsx": "js", ".mts": "js", ".cts": "js", ".go": "go", ".rs": "rs", ".java": "jvm", ".kt": "jvm", ".scala": "jvm", ".groovy": "jvm", diff --git a/graphify/cli.py b/graphify/cli.py index 2f88318f26..1740ca5222 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -70,7 +70,7 @@ } }, ensure_ascii=False, separators=(",", ":")) + "\n" _HOOK_SOURCE_EXTS = ( - '.py', '.js', '.cjs', '.ts', '.tsx', '.jsx', '.astro', '.vue', '.svelte', '.ejs', '.go', + '.py', '.js', '.cjs', '.gs', '.ts', '.tsx', '.jsx', '.astro', '.vue', '.svelte', '.ejs', '.go', '.rs', '.java', '.rb', '.c', '.h', '.cpp', '.hpp', '.cc', '.cs', '.kt', '.swift', '.php', '.scala', '.lua', '.ps1', '.psm1', '.psd1', # #4271: PowerShell sources are graphable too '.sh', '.md', '.rst', '.txt', '.mdx', diff --git a/graphify/detect.py b/graphify/detect.py index 7566a553e0..e90f293884 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -45,7 +45,7 @@ class FileType(str, Enum): _MTIME_COARSE_S = 2.0 _MTIME_SUBSECOND_S = 0.05 -CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.vb', '.cbl', '.cob', '.cobol', '.cpy', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.ml', '.mli', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.vh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger', '.lisp', '.cl', '.lsp', '.asd', '.robot', '.resource', '.sol', '.erl', '.hrl', '.escript'} +CODE_EXTENSIONS = {'.gs', '.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.vb', '.cbl', '.cob', '.cobol', '.cpy', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.ml', '.mli', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.vh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger', '.lisp', '.cl', '.lsp', '.asd', '.robot', '.resource', '.sol', '.erl', '.hrl', '.escript'} DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml', '.j2', '.jinja', '.jinja2'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/extract.py b/graphify/extract.py index 6f08bb7edb..38be9b480f 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -3773,6 +3773,7 @@ def _lang_is_case_insensitive(source_file: object) -> bool: _LANG_FAMILY_BY_EXT: dict[str, str] = { # JS/TS module graph (SFCs embed JS/TS) ".js": "jsts", ".jsx": "jsts", ".mjs": "jsts", ".cjs": "jsts", + ".gs": "jsts", ".ts": "jsts", ".tsx": "jsts", ".mts": "jsts", ".cts": "jsts", ".vue": "jsts", ".svelte": "jsts", ".astro": "jsts", ".ejs": "jsts", # JVM interop @@ -4980,7 +4981,7 @@ def _inherited_method(caller: str, callee: str, skip_own: bool) -> "str | None": # Every suffix the TS/JS extractors claim, so a pure-ESM (`.mjs`) repo still activates # the member-call resolver below. -_JS_TS_SUFFIXES = (".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs") +_JS_TS_SUFFIXES = (".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs", ".gs") def _resolve_typescript_member_calls( @@ -8384,6 +8385,7 @@ def add_existing_edge(edge: dict) -> None: _DISPATCH: dict[str, Any] = { ".py": extract_python, ".js": extract_js, + ".gs": extract_js, ".jsx": extract_js, ".mjs": extract_js, ".cjs": extract_js, @@ -8632,6 +8634,40 @@ def _is_cpp_header(path: Path) -> bool: return any(marker in head for marker in _CPP_HEADER_MARKERS) +# `.gs` is Google Apps Script (plain JavaScript) but is also used by GLSL geometry +# shaders and by Gosu. Apps Script is the overwhelmingly common case in a repo +# graphify is pointed at, so `.gs` routes to the JS extractor unless the file +# carries a marker only the other two produce. +_NON_APPS_SCRIPT_GS_MARKERS = ( + b"#version ", # GLSL preprocessor directive — never valid JavaScript + b"gl_Position", + b"gl_in[", + b"EmitVertex", + b"EndPrimitive", + b"uses java.", # Gosu import + b"uses gw.", +) + + +def _is_apps_script(path: Path) -> bool: + """Whether a `.gs` file is Google Apps Script rather than GLSL/Gosu. + + Inverse of the `.m` sniff (`_is_objc_source`): there the ambiguous suffix is + guilty until proven innocent because MATLAB and Objective-C are both common. + Apps Script has no such rival — a `.gs` in a repo is almost always Apps + Script, and its own marker set is unusable as a positive test (a pure-logic + `.gs` helper calls no `SpreadsheetApp`/`DriveApp` service at all). So the + test runs the other way: JS unless a GLSL or Gosu token shows up, in which + case the file is left without an extractor (surfaced by the + no-AST-extractor warning) rather than force-parsed into garbage. + """ + try: + head = path.read_bytes()[:256 * 1024] + except OSError: + return False + return not any(marker in head for marker in _NON_APPS_SCRIPT_GS_MARKERS) + + def _get_extractor(path: Path) -> Any | None: """Return the correct extractor function for a file, or None if unsupported.""" if path.name.lower().endswith(".blade.php"): @@ -8673,6 +8709,10 @@ def _get_extractor(path: Path) -> Any | None: # mis-parsed. `.mm` is unambiguously Objective-C++ and stays on extract_objc. if suffix == ".m" and not _is_objc_source(path): return None + # `.gs` is Apps Script (JS) OR a GLSL geometry shader OR Gosu. Route to + # extract_js unless the file looks like one of the other two. + if suffix == ".gs" and not _is_apps_script(path): + return None # Extensionless files: resolve by shebang, mirroring detect.classify_file. # Without this, detect labels e.g. `#!/usr/bin/env bash` CLIs as code but # extraction returns no extractor and the file silently contributes nothing. diff --git a/graphify/extractors/models.py b/graphify/extractors/models.py index 02e1d38b63..877ae42b9a 100644 --- a/graphify/extractors/models.py +++ b/graphify/extractors/models.py @@ -8,8 +8,7 @@ _WORKSPACE_PACKAGE_CACHE: dict[str, dict[str, Path]] = {} -_JS_CACHE_BYPASS_SUFFIXES = {".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".mts", ".cts", ".vue", ".svelte", ".ejs"} - +_JS_CACHE_BYPASS_SUFFIXES = {".js", ".gs", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".mts", ".cts", ".vue", ".svelte", ".ejs"} @dataclass class LanguageConfig: ts_module: str # e.g. "tree_sitter_python" diff --git a/tests/test_gs_apps_script.py b/tests/test_gs_apps_script.py new file mode 100644 index 0000000000..12a025eee3 --- /dev/null +++ b/tests/test_gs_apps_script.py @@ -0,0 +1,150 @@ +"""Google Apps Script (`.gs`) is treated as JavaScript code. + +Apps Script source files are plain JavaScript saved with a `.gs` extension — +that is what `clasp` pulls down and what every Apps Script project in a repo +looks like. The extension was absent from `CODE_EXTENSIONS`, the extractor +`_DISPATCH`, and the JS language-family maps, so an entire Apps Script project +was detected as non-code and contributed nothing to the graph: a build over a +repo whose logic lives in `.gs` returned only its markdown and `appsscript.json` +manifests. + +`.gs` is not exclusive to Apps Script (GLSL geometry shaders and Gosu also use +it), so routing is guarded by `_is_apps_script`, which withholds the extractor +when a GLSL or Gosu marker is present rather than force-parsing the file through +the JS grammar. Same shape of gap (and fix) as `.cjs` in +tests/test_cjs_module_extension.py; the sniff mirrors the `.m` Objective-C / +MATLAB split in `_is_objc_source`. +""" +from __future__ import annotations + +from pathlib import Path + + +def _labels(r): + return [n["label"] for n in r["nodes"]] + + +def test_gs_registered_as_code(): + from graphify.detect import CODE_EXTENSIONS + assert ".gs" in CODE_EXTENSIONS + + +def test_gs_in_extractor_dispatch(): + from graphify.extract import _DISPATCH, extract_js + assert _DISPATCH.get(".gs") is extract_js + + +def test_gs_in_js_language_family(): + from graphify.analyze import _LANG_FAMILY + from graphify.build import _EDGE_LANG_FAMILY + from graphify.extract import _LANG_FAMILY_BY_EXT + assert _LANG_FAMILY.get(".gs") == "js" + assert _EDGE_LANG_FAMILY.get(".gs") == "js" + assert _LANG_FAMILY_BY_EXT.get(".gs") == "jsts" + + +def test_gs_in_js_cache_bypass(): + from graphify.extract import _JS_CACHE_BYPASS_SUFFIXES + assert ".gs" in _JS_CACHE_BYPASS_SUFFIXES + + +def test_gs_in_hook_source_exts(): + from graphify.cli import _HOOK_SOURCE_EXTS + assert ".gs" in _HOOK_SOURCE_EXTS + + +# A representative Apps Script source: global helper functions, a class, a +# container-bound trigger, and calls into the Apps Script service globals. +_GS_SOURCE = ( + "function onOpen() {\n" + " SpreadsheetApp.getUi().createMenu('Tools').addToUi();\n" + "}\n" + "class VolunteerSheet {\n" + " readAll() { return SpreadsheetApp.getActive().getDataRange().getValues(); }\n" + "}\n" + "function syncVolunteers() {\n" + " const sheet = new VolunteerSheet();\n" + " return sheet.readAll();\n" + "}\n" +) + + +def _extract(tmp_path: Path, ext: str, source: str = _GS_SOURCE): + from graphify.extract import extract_js + f = tmp_path / f"Code{ext}" + f.write_text(source, encoding="utf-8") + return extract_js(f) + + +def test_gs_extracts_like_js(tmp_path): + # `.gs` must parse identically to the same source saved as `.js` — same + # node set (class, functions) modulo the file-node label. + gs = _extract(tmp_path, ".gs") + js = _extract(tmp_path, ".js") + assert "error" not in gs + gs_labels = set(_labels(gs)) + js_labels = set(_labels(js)) + assert any("VolunteerSheet" in label for label in gs_labels), ( + ".gs class declaration missing — file was not parsed as JS" + ) + assert any("syncVolunteers" in label for label in gs_labels), ( + ".gs function declaration missing — file was not parsed as JS" + ) + assert {label for label in gs_labels if not label.endswith(".gs")} == { + label for label in js_labels if not label.endswith(".js") + } + + +def test_gs_routed_to_js_extractor(tmp_path): + from graphify.extract import _get_extractor, extract_js + f = tmp_path / "Code.gs" + f.write_text(_GS_SOURCE, encoding="utf-8") + assert _get_extractor(f) is extract_js + + +def test_gs_without_service_globals_still_routed(tmp_path): + # A pure-logic Apps Script helper touches no SpreadsheetApp/DriveApp service + # at all. Routing must not depend on spotting one. + from graphify.extract import _get_extractor, extract_js + f = tmp_path / "Schema.gs" + f.write_text( + "function normalizeRow_(row) {\n" + " return row.map(function (cell) { return String(cell).trim(); });\n" + "}\n", + encoding="utf-8", + ) + assert _get_extractor(f) is extract_js + + +def test_glsl_geometry_shader_gs_is_not_extracted(tmp_path): + # A GLSL geometry shader forced through the JS grammar yields garbage, so it + # gets no extractor at all (same outcome as MATLAB `.m`). + from graphify.extract import _get_extractor + f = tmp_path / "outline.gs" + f.write_text( + "#version 330 core\n" + "layout(triangles) in;\n" + "layout(line_strip, max_vertices = 6) out;\n" + "void main() {\n" + " gl_Position = gl_in[0].gl_Position;\n" + " EmitVertex();\n" + " EndPrimitive();\n" + "}\n", + encoding="utf-8", + ) + assert _get_extractor(f) is None + + +def test_gosu_gs_is_not_extracted(tmp_path): + from graphify.extract import _get_extractor + f = tmp_path / "Policy.gs" + f.write_text( + "package example.policy\n" + "uses java.util.ArrayList\n" + "uses gw.api.database.Query\n" + "class Policy {\n" + " function renew() : String { return \"ok\" }\n" + "}\n", + encoding="utf-8", + ) + assert _get_extractor(f) is None From 5a48413e3dccb919647a8835202b45f3a80de3d2 Mon Sep 17 00:00:00 2001 From: himanshupatro-334 Date: Thu, 10 Sep 2026 22:26:24 +0530 Subject: [PATCH 2/5] Add CSS and SCSS design token extraction --- graphify/cache.py | 4 +- graphify/detect.py | 2 +- graphify/extract.py | 125 +++++++++ graphify/extractors/__init__.py | 3 + graphify/extractors/css.py | 322 +++++++++++++++++++++++ tests/test_css.py | 450 ++++++++++++++++++++++++++++++++ tests/test_detect.py | 18 ++ 7 files changed, 921 insertions(+), 3 deletions(-) create mode 100644 graphify/extractors/css.py create mode 100644 tests/test_css.py diff --git a/graphify/cache.py b/graphify/cache.py index 83d3ecf3c0..ba7ac216a1 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -692,7 +692,7 @@ def _relativize_source_files_in(payload: dict, root: Path) -> None: # definition_file (#2990) is a path into the scanned tree exactly like # source_file; a cache entry keeping it absolute replayed the build host's # layout on every warm hit (#3223). - for bucket in ("nodes", "edges", "hyperedges", "raw_calls"): + for bucket in ("nodes", "edges", "hyperedges", "raw_calls", "raw_token_uses"): for item in payload.get(bucket, []): if not isinstance(item, dict): continue @@ -994,7 +994,7 @@ def _absolutize_source_files_in(payload: dict, root: Path) -> None: root_resolved = Path(root).resolve() except OSError: return - for bucket in ("nodes", "edges", "hyperedges", "raw_calls"): + for bucket in ("nodes", "edges", "hyperedges", "raw_calls", "raw_token_uses"): for item in payload.get(bucket, []): if not isinstance(item, dict): continue diff --git a/graphify/detect.py b/graphify/detect.py index e90f293884..b365a3b1f0 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -45,7 +45,7 @@ class FileType(str, Enum): _MTIME_COARSE_S = 2.0 _MTIME_SUBSECOND_S = 0.05 -CODE_EXTENSIONS = {'.gs', '.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.vb', '.cbl', '.cob', '.cobol', '.cpy', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.ml', '.mli', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.vh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger', '.lisp', '.cl', '.lsp', '.asd', '.robot', '.resource', '.sol', '.erl', '.hrl', '.escript'} +CODE_EXTENSIONS = {'.gs', '.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.vb', '.cbl', '.cob', '.cobol', '.cpy', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.ml', '.mli', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.vh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger', '.lisp', '.cl', '.lsp', '.asd', '.robot', '.resource', '.sol', '.erl', '.hrl', '.escript', '.css', '.scss'} DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml', '.j2', '.jinja', '.jinja2'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/extract.py b/graphify/extract.py index 38be9b480f..204eaf8d77 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -55,6 +55,7 @@ _resolve_cross_file_csharp_imports, _resolve_csharp_type_references, ) +from graphify.extractors.css import extract_css # noqa: F401 from graphify.extractors.dart import extract_dart # noqa: F401 from graphify.extractors.dm import extract_dm, extract_dmf, extract_dmi, extract_dmm # noqa: F401 from graphify.extractors.elixir import extract_elixir # noqa: F401 @@ -8450,6 +8451,8 @@ def add_existing_edge(edge: dict) -> None: ".svelte": extract_svelte, ".astro": extract_astro, ".ejs": extract_ejs, + ".css": extract_css, + ".scss": extract_css, ".dart": extract_dart, ".ml": extract_ocaml, ".mli": extract_ocaml, @@ -9012,6 +9015,110 @@ def _extract_sequential( print(f" AST extraction: {_done}/{_done} uncached files (100%)", flush=True) +def _resolve_css_tokens( + raw_token_uses: list[dict], + resolution_nodes: list[dict], + all_edges: list[dict], + *, + resolution_context_edges: list[dict] | None = None, +) -> None: + """Resolve CSS custom property references (var(--name)) to design-token definitions. + + Emits: + consumer stylesheet -> uses_token -> token node + with confidence 'EXTRACTED'. + + Resolution rules: + - same-file matching definition -> resolve; + - exactly one matching definition across available graph/resolution nodes -> resolve; + - multiple matching definitions -> emit nothing (conservative, no fan-out); + - no matching definition -> emit nothing. + """ + if not raw_token_uses or not resolution_nodes: + return + + def _norm_sf(sf: str | Path | None) -> str: + if not sf: + return "" + try: + return str(Path(sf).resolve()) + except Exception: + return str(sf).replace("\\", "/") + + # Map source_file -> file node id + sf_to_file_nid: dict[str, str] = {} + for n in resolution_nodes: + sf = n.get("source_file") + if sf and n.get("label") == Path(str(sf)).name: + norm = _norm_sf(sf) + sf_to_file_nid.setdefault(str(sf), n["id"]) + sf_to_file_nid.setdefault(norm, n["id"]) + + # Index token definition nodes + tokens_by_label: dict[str, list[dict]] = {} + tokens_by_file_and_label: dict[tuple[str, str], list[dict]] = {} + for n in resolution_nodes: + if n.get("node_kind") == "token": + lbl = n.get("label", "") + if lbl.startswith("--"): + tokens_by_label.setdefault(lbl, []).append(n) + sf = n.get("source_file") + if sf: + tokens_by_file_and_label.setdefault((_norm_sf(sf), lbl), []).append(n) + + seen_edges = { + (e["source"], e["target"], e.get("relation")) + for e in (all_edges + list(resolution_context_edges or [])) + } + + for tu in raw_token_uses: + token_name = tu.get("token_name", "") + if not token_name: + continue + + source_file = tu.get("source_file", "") + norm_sf = _norm_sf(source_file) + + # 1. Check same-file match first + target_node: dict | None = None + same_file_matches = tokens_by_file_and_label.get((norm_sf, token_name), []) + if same_file_matches: + target_node = same_file_matches[0] + else: + # 2. Exactly one matching definition across available graph/resolution nodes + global_matches = tokens_by_label.get(token_name, []) + if len(global_matches) == 1: + target_node = global_matches[0] + else: + # 0 matches or multiple ambiguous matches -> emit nothing + continue + + target_nid = target_node.get("id") + if not target_nid: + continue + + consumer_nid = sf_to_file_nid.get(source_file) or sf_to_file_nid.get(norm_sf) or tu.get("source_nid") + if not consumer_nid: + continue + + edge_key = (consumer_nid, target_nid, "uses_token") + if edge_key in seen_edges: + continue + seen_edges.add(edge_key) + + line = tu.get("line") + all_edges.append({ + "source": consumer_nid, + "target": target_nid, + "relation": "uses_token", + "confidence": "EXTRACTED", + "confidence_score": 1.0, + "source_file": source_file, + "source_location": f"L{line}" if line else None, + "weight": 1.0, + }) + + _PARALLEL_THRESHOLD = 20 @@ -9444,10 +9551,12 @@ def _describe_syntax_error(rel: str, line: "int | None", kept: int) -> str: all_nodes: list[dict] = [] all_edges: list[dict] = [] all_raw_calls: list[dict] = [] + all_raw_token_uses: list[dict] = [] for result in per_file: all_nodes.extend(result.get("nodes", [])) all_edges.extend(result.get("edges", [])) all_raw_calls.extend(result.get("raw_calls", [])) + all_raw_token_uses.extend(result.get("raw_token_uses", [])) # Function / method / class def ids for the cross-file indirect_call callable # guard. Built from the `_callable` node marker AFTER the id-remap / disambiguation # passes below (which rewrite node ids), so it can never go stale — see the @@ -9695,6 +9804,10 @@ def _resolved(p: "str | Path") -> Path: cn = rc.get("caller_nid") if cn in id_remap: rc["caller_nid"] = id_remap[cn] + for tu in all_raw_token_uses: + sn = tu.get("source_nid") + if sn in id_remap: + tu["source_nid"] = id_remap[sn] # swift_extensions[].nid is the same kind of id carrier as caller_nid # above (cache.py remaps both), consumed by _merge_swift_extensions far # below. Left stale it matches no node, so whether the extension merge @@ -9770,6 +9883,10 @@ def _resolved(p: "str | Path") -> Path: cn = rc.get("caller_nid") if cn in sym_remap: rc["caller_nid"] = sym_remap[cn] + for tu in all_raw_token_uses: + sn = tu.get("source_nid") + if sn in sym_remap: + tu["source_nid"] = sym_remap[sn] # Same for swift_extensions[].nid (see the id_remap pass above). for result in per_file: for ext in result.get("swift_extensions", []) or []: @@ -10602,6 +10719,14 @@ def _has_import_evidence(candidate_id: str) -> bool: for e in all_edges: e.pop("_rust_use_binding", None) + if all_raw_token_uses: + _resolve_css_tokens( + all_raw_token_uses, + resolution_nodes, + all_edges, + resolution_context_edges=resolution_context_edges, + ) + # Relativize source_file fields so paths are portable across machines (#555). # When the node's id was itself minted from the absolute path, remap it to a # portable id and rewrite the edge endpoints that reference it. diff --git a/graphify/extractors/__init__.py b/graphify/extractors/__init__.py index e2c8afc627..9d0f0b185a 100644 --- a/graphify/extractors/__init__.py +++ b/graphify/extractors/__init__.py @@ -15,6 +15,7 @@ from graphify.extractors.blade import extract_blade from graphify.extractors.cobol import extract_cobol from graphify.extractors.commonlisp import extract_commonlisp +from graphify.extractors.css import extract_css from graphify.extractors.dart import extract_dart from graphify.extractors.dm import extract_dm, extract_dmf, extract_dmi, extract_dmm from graphify.extractors.elixir import extract_elixir @@ -45,6 +46,7 @@ "blade": extract_blade, "cobol": extract_cobol, "commonlisp": extract_commonlisp, + "css": extract_css, "dart": extract_dart, "delphi_form": extract_delphi_form, "dm": extract_dm, @@ -66,6 +68,7 @@ "r": extract_r, "razor": extract_razor, "rust": extract_rust, + "scss": extract_css, "sln": extract_sln, "solidity": extract_solidity, "sql": extract_sql, diff --git a/graphify/extractors/css.py b/graphify/extractors/css.py new file mode 100644 index 0000000000..3a8775d6f5 --- /dev/null +++ b/graphify/extractors/css.py @@ -0,0 +1,322 @@ +"""CSS and SCSS design-token extractor. + +Extracts design-token definitions (--name: value;) declared within root or +theme contexts (:root, :host, html, body, [data-theme...], [data-mode...], +.dark, .theme-*, @theme, including when enclosed by @media/@layer/@supports), +and collects var(--name) references for post-extraction cross-file resolution. +""" +from __future__ import annotations + +import re +from pathlib import Path + +from graphify.extractors.base import _file_stem, _make_id + +_ROOT_START_RE = re.compile( + r'^(?::(?:root|host)(?![a-zA-Z0-9_-])|html(?![a-zA-Z0-9_-])|body(?![a-zA-Z0-9_-])|@theme(?![a-zA-Z0-9_-])|\[data-(?:theme|mode)[^\]]*\]|\.dark(?![a-zA-Z0-9_-])|\.theme-[a-zA-Z0-9_-]+)', + re.IGNORECASE, +) +_SCSS_NESTED_ROOT_RE = re.compile( + r'^&(?:(?::(?:root|host)(?![a-zA-Z0-9_-]))|(?:\[data-(?:theme|mode)[^\]]*\])|\.dark(?![a-zA-Z0-9_-])|\.theme-[a-zA-Z0-9_-]+)', + re.IGNORECASE, +) +_TRANSPARENT_AT_RULES = ("@media", "@layer", "@supports") +_URL_START_RE = re.compile(r"^url\s*\(", re.IGNORECASE) +_VAR_RE = re.compile(r"var\(\s*(--[a-zA-Z0-9_-]+)") +_DECL_RE = re.compile(r"^\s*(--[a-zA-Z0-9_-]+)\s*:\s*([^;]+?)\s*(?:;|\Z)") + + +def _strip_comments_preserving_lines(text: str) -> str: + """Replace comment characters with spaces, preserving newlines so line numbers remain exact.""" + out: list[str] = [] + i = 0 + n = len(text) + in_str: str | None = None + in_url: bool = False + while i < n: + c = text[i] + nxt = text[i + 1] if i + 1 < n else "" + if in_str: + out.append(c) + if c == "\\" and i + 1 < n: + out.append(text[i + 1]) + i += 2 + continue + elif c == in_str: + in_str = None + i += 1 + continue + + if c in ('"', "'"): + in_str = c + out.append(c) + i += 1 + continue + + # Track unquoted url(...) so protocol slashes (http://, https://) don't trigger SCSS comments + if not in_url and c in ("u", "U") and _URL_START_RE.match(text[i:]): + in_url = True + out.append(c) + i += 1 + continue + + if in_url and (c == ")" or c == "\n"): + in_url = False + out.append(c) + i += 1 + continue + + # Block comment /* ... */ + if c == "/" and nxt == "*": + out.append(" ") + out.append(" ") + i += 2 + while i < n: + if text[i] == "*" and i + 1 < n and text[i + 1] == "/": + out.append(" ") + out.append(" ") + i += 2 + break + else: + out.append("\n" if text[i] == "\n" else " ") + i += 1 + continue + + # SCSS single-line comment // ... (only outside unquoted url(...)) + if not in_url and c == "/" and nxt == "/": + out.append(" ") + out.append(" ") + i += 2 + while i < n and text[i] != "\n": + out.append(" ") + i += 1 + continue + + out.append(c) + i += 1 + + return "".join(out) + + +def _blank_strings_preserving_lines(text: str) -> str: + """Replace characters inside quoted string literals with spaces, preserving newlines.""" + out: list[str] = [] + i = 0 + n = len(text) + in_str: str | None = None + while i < n: + c = text[i] + if in_str: + if c == "\\" and i + 1 < n: + out.append(" ") + out.append("\n" if text[i + 1] == "\n" else " ") + i += 2 + continue + elif c == in_str: + in_str = None + out.append(c) + i += 1 + continue + else: + out.append("\n" if c == "\n" else " ") + i += 1 + continue + + if c in ('"', "'"): + in_str = c + out.append(c) + i += 1 + continue + + out.append(c) + i += 1 + + return "".join(out) + + +def _is_root_branch(branch: str) -> bool: + """Check whether a single selector branch is rooted and contains only root/theme selectors.""" + segments = [seg.strip() for seg in re.split(r'[\s>]+', branch) if seg.strip()] + if not segments: + return False + return all(_ROOT_START_RE.match(seg) for seg in segments) + + +def _is_root_context(stack: list[str]) -> bool: + """Return True if the current selector stack represents a root or theme context.""" + non_at = [s.strip() for s in stack if not s.strip().startswith(_TRANSPARENT_AT_RULES)] + if not non_at: + return False + base_branches = [b.strip() for b in non_at[0].split(",") if b.strip()] + if not any(_is_root_branch(b) for b in base_branches): + return False + for s in non_at[1:]: + branches = [b.strip() for b in s.split(",") if b.strip()] + for b in branches: + if not (_SCSS_NESTED_ROOT_RE.match(b) or _is_root_branch(b)): + return False + return True + + +def extract_css(path: Path) -> dict: + """Extract design-token definitions and references from a .css or .scss file.""" + str_path = str(path) + try: + src = path.read_text(encoding="utf-8", errors="replace") + except OSError as exc: + return {"nodes": [], "edges": [], "error": str(exc)} + + clean = _strip_comments_preserving_lines(src) + code_only = _blank_strings_preserving_lines(clean) + + file_nid = _make_id(str_path) + stem = _file_stem(path) + + nodes: list[dict] = [ + { + "id": file_nid, + "label": path.name, + "file_type": "code", + "source_file": str_path, + "source_location": "L1", + } + ] + edges: list[dict] = [] + raw_token_uses: list[dict] = [] + + # 1. Collect var(--token) references outside string literals + seen_uses: set[tuple[str, int]] = set() + for m in _VAR_RE.finditer(code_only): + token_name = m.group(1) + line = code_only[: m.start()].count("\n") + 1 + key = (token_name, line) + if key in seen_uses: + continue + seen_uses.add(key) + raw_token_uses.append( + { + "source_nid": file_nid, + "token_name": token_name, + "source_file": str_path, + "line": line, + } + ) + + # 2. Collect token definitions inside root/theme contexts + seen_tokens: set[str] = set() + stack: list[str] = [] + buf: list[str] = [] + i = 0 + n = len(clean) + in_str: str | None = None + + while i < n: + c = clean[i] + if in_str: + buf.append(c) + if c == "\\" and i + 1 < n: + buf.append(clean[i + 1]) + i += 2 + continue + elif c == in_str: + in_str = None + i += 1 + continue + + if c in ('"', "'"): + in_str = c + buf.append(c) + i += 1 + continue + + if c == "{": + selector = "".join(buf).strip() + stack.append(selector) + buf = [] + i += 1 + continue + + if c == "}": + stmt = "".join(buf).strip() + if stmt and stack and _is_root_context(stack): + m = _DECL_RE.match(stmt) + if m: + token_name = m.group(1) + if token_name not in seen_tokens: + seen_tokens.add(token_name) + pos = i - len(stmt) + line = clean[:pos].count("\n") + 1 + token_nid = _make_id(stem, token_name) + nodes.append( + { + "id": token_nid, + "label": token_name, + "file_type": "code", + "node_kind": "token", + "source_file": str_path, + "source_location": f"L{line}", + } + ) + edges.append( + { + "source": file_nid, + "target": token_nid, + "relation": "defines_token", + "confidence": "EXTRACTED", + "confidence_score": 1.0, + "source_file": str_path, + "source_location": f"L{line}", + "weight": 1.0, + } + ) + if stack: + stack.pop() + buf = [] + i += 1 + continue + + if c == ";": + stmt = "".join(buf).strip() + if stmt and stack and _is_root_context(stack): + m = _DECL_RE.match(stmt) + if m: + token_name = m.group(1) + if token_name not in seen_tokens: + seen_tokens.add(token_name) + pos = i - len(stmt) + line = clean[:pos].count("\n") + 1 + token_nid = _make_id(stem, token_name) + nodes.append( + { + "id": token_nid, + "label": token_name, + "file_type": "code", + "node_kind": "token", + "source_file": str_path, + "source_location": f"L{line}", + } + ) + edges.append( + { + "source": file_nid, + "target": token_nid, + "relation": "defines_token", + "confidence": "EXTRACTED", + "confidence_score": 1.0, + "source_file": str_path, + "source_location": f"L{line}", + "weight": 1.0, + } + ) + buf = [] + i += 1 + continue + + buf.append(c) + i += 1 + + return { + "nodes": nodes, + "edges": edges, + "raw_token_uses": raw_token_uses, + } diff --git a/tests/test_css.py b/tests/test_css.py new file mode 100644 index 0000000000..61a784cfe6 --- /dev/null +++ b/tests/test_css.py @@ -0,0 +1,450 @@ +"""Focused tests for CSS and SCSS design-token extraction and resolution (#3473).""" +from pathlib import Path +import pytest + +from graphify.detect import FileType, classify_file, _is_sensitive +from graphify.extract import extract, _file_node_id, _make_id +from graphify.extractors.css import extract_css + + +# ── 1. Detection ───────────────────────────────────────────────────────────── + +def test_detect_css_classified_as_code(): + assert classify_file(Path("styles.css")) == FileType.CODE + assert classify_file(Path("src/components/button.css")) == FileType.CODE + + +def test_detect_scss_classified_as_code(): + assert classify_file(Path("styles.scss")) == FileType.CODE + assert classify_file(Path("src/styles/_tokens.scss")) == FileType.CODE + + +def test_tokens_stylesheet_not_flagged_as_sensitive(): + assert not _is_sensitive(Path("tokens.css")) + assert not _is_sensitive(Path("tokens.scss")) + assert not _is_sensitive(Path("design-tokens.css")) + assert not _is_sensitive(Path("styles/tokens.scss")) + + +# ── 2. Extraction: Root & Theme Contexts ────────────────────────────────────── + +def test_extract_root_tokens(tmp_path): + f = tmp_path / "root.css" + f.write_text( + """:root { + --color-primary: #0070f3; + --spacing-sm: 8px; +} +""", + encoding="utf-8", + ) + res = extract_css(f) + token_nodes = [n for n in res["nodes"] if n.get("node_kind") == "token"] + assert len(token_nodes) == 2 + labels = {n["label"] for n in token_nodes} + assert labels == {"--color-primary", "--spacing-sm"} + + edges = [e for e in res["edges"] if e.get("relation") == "defines_token"] + assert len(edges) == 2 + for e in edges: + assert e["confidence"] == "EXTRACTED" + assert e["confidence_score"] == 1.0 + + +def test_extract_host_tokens(tmp_path): + f = tmp_path / "shadow.css" + f.write_text( + """:host { + --widget-bg: #fff; +} +""", + encoding="utf-8", + ) + res = extract_css(f) + tokens = [n["label"] for n in res["nodes"] if n.get("node_kind") == "token"] + assert tokens == ["--widget-bg"] + + +def test_extract_html_and_body_tokens(tmp_path): + f = tmp_path / "base.css" + f.write_text( + """html { + --font-base: sans-serif; +} +body { + --text-color: #333; +} +""", + encoding="utf-8", + ) + res = extract_css(f) + tokens = [n["label"] for n in res["nodes"] if n.get("node_kind") == "token"] + assert set(tokens) == {"--font-base", "--text-color"} + + +def test_extract_theme_selectors(tmp_path): + f = tmp_path / "themes.css" + f.write_text( + """[data-theme="dark"] { + --bg-dark: #121212; +} +[data-mode="dim"] { + --bg-dim: #222; +} +.dark { + --dark-accent: #f00; +} +.theme-dracula { + --dracula-pink: #ff79c6; +} +""", + encoding="utf-8", + ) + res = extract_css(f) + tokens = {n["label"] for n in res["nodes"] if n.get("node_kind") == "token"} + assert tokens == {"--bg-dark", "--bg-dim", "--dark-accent", "--dracula-pink"} + + +def test_extract_at_theme(tmp_path): + f = tmp_path / "tailwind.css" + f.write_text( + """@theme { + --font-display: 'Inter', sans-serif; +} +""", + encoding="utf-8", + ) + res = extract_css(f) + tokens = [n["label"] for n in res["nodes"] if n.get("node_kind") == "token"] + assert tokens == ["--font-display"] + + +def test_extract_nested_under_transparent_at_rules(tmp_path): + f = tmp_path / "media.css" + f.write_text( + """@media (prefers-color-scheme: dark) { + :root { + --media-dark: #000; + } +} +@layer base { + :root { + --layer-base-token: #fff; + } +} +@supports (display: grid) { + :root { + --grid-gap: 16px; + } +} +""", + encoding="utf-8", + ) + res = extract_css(f) + tokens = {n["label"] for n in res["nodes"] if n.get("node_kind") == "token"} + assert tokens == {"--media-dark", "--layer-base-token", "--grid-gap"} + + +def test_ordinary_component_selectors_rejected(tmp_path): + f = tmp_path / "components.css" + f.write_text( + """.card { + --card-bg: #fff; +} +.button { + --btn-padding: 4px; +} +#header { + --header-h: 60px; +} +table tr:hover { + --hover-bg: #f5f5f5; +} +.p-4 { + --spacing: 16px; +} +.card .dark { + --scoped-dark: #000; +} +:root .card { + --nested-card-override: #111; +} +""", + encoding="utf-8", + ) + res = extract_css(f) + tokens = [n["label"] for n in res["nodes"] if n.get("node_kind") == "token"] + assert tokens == [], f"Expected no token nodes from component selectors, got {tokens}" + + +# ── 3. SCSS Handling ───────────────────────────────────────────────────────── + +def test_scss_comments_and_nested_root(tmp_path): + f = tmp_path / "theme.scss" + f.write_text( + """// Single-line SCSS comment with --fake-token: 1; +/* Block comment with :root { --commented-token: 2; } */ +$sass-var: #ff0000; +$primary-color: #fff; + +:root { + // line comment inside root + /* block comment inside root: --inside-comment: 3; */ + --real-token: #123456; + + &.dark { + --nested-dark-token: #000; + } +} +""", + encoding="utf-8", + ) + res = extract_css(f) + tokens = {n["label"] for n in res["nodes"] if n.get("node_kind") == "token"} + assert tokens == {"--real-token", "--nested-dark-token"} + assert "--fake-token" not in tokens + assert "--commented-token" not in tokens + assert "--inside-comment" not in tokens + assert "$sass-var" not in tokens + assert "$primary-color" not in tokens + + +def test_scss_partials_extracted_normally(tmp_path): + f = tmp_path / "_tokens.scss" + f.write_text( + """:root { + --brand: #abc; +} +""", + encoding="utf-8", + ) + res = extract_css(f) + tokens = [n["label"] for n in res["nodes"] if n.get("node_kind") == "token"] + assert tokens == ["--brand"] + + +# ── 4. Identity & Schema ───────────────────────────────────────────────────── + +def test_token_identity_and_schema(tmp_path): + f_a = tmp_path / "theme-a.css" + f_b = tmp_path / "theme-b.css" + f_a.write_text(":root { --color-card: #aaa; }", encoding="utf-8") + f_b.write_text(":root { --color-card: #bbb; }", encoding="utf-8") + + res_a = extract_css(f_a) + res_b = extract_css(f_b) + + token_a = next(n for n in res_a["nodes"] if n.get("node_kind") == "token") + token_b = next(n for n in res_b["nodes"] if n.get("node_kind") == "token") + + # Distinct file-scoped IDs + assert token_a["id"] != token_b["id"] + + # Schema verification + assert token_a["file_type"] == "code" + assert token_a["node_kind"] == "token" + assert token_a["label"] == "--color-card" + assert token_a["source_file"] == str(f_a) + assert token_a["source_location"] == "L1" + + edge_a = next(e for e in res_a["edges"] if e.get("relation") == "defines_token") + assert edge_a["source"] == res_a["nodes"][0]["id"] + assert edge_a["target"] == token_a["id"] + assert edge_a["confidence"] == "EXTRACTED" + assert edge_a["confidence_score"] == 1.0 + + +# ── 5. Post-Extraction Resolution ──────────────────────────────────────────── + +def test_same_file_unique_token_resolves(tmp_path): + f = tmp_path / "styles.css" + f.write_text( + """:root { + --brand: #123; +} +.btn { + color: var(--brand); +} +""", + encoding="utf-8", + ) + g = extract([f], root=tmp_path) + edges = [e for e in g["edges"] if e.get("relation") == "uses_token"] + assert len(edges) == 1 + edge = edges[0] + token_node = next(n for n in g["nodes"] if n.get("label") == "--brand") + file_node = next(n for n in g["nodes"] if n.get("label") == "styles.css") + assert edge["source"] == file_node["id"] + assert edge["target"] == token_node["id"] + assert edge["confidence"] == "EXTRACTED" + assert edge["confidence_score"] == 1.0 + + +def test_unique_cross_file_token_resolves(tmp_path): + tokens_file = tmp_path / "tokens.css" + app_file = tmp_path / "app.css" + tokens_file.write_text(":root { --accent: #ff0; }", encoding="utf-8") + app_file.write_text(".sidebar { background: var(--accent); }", encoding="utf-8") + + g = extract([tokens_file, app_file], root=tmp_path) + uses_edges = [e for e in g["edges"] if e.get("relation") == "uses_token"] + assert len(uses_edges) == 1 + edge = uses_edges[0] + token_node = next(n for n in g["nodes"] if n.get("label") == "--accent") + app_file_node = next(n for n in g["nodes"] if n.get("label") == "app.css") + assert edge["source"] == app_file_node["id"] + assert edge["target"] == token_node["id"] + assert edge["confidence"] == "EXTRACTED" + + +def test_multiple_same_name_definitions_produce_no_uses_edge(tmp_path): + theme_a = tmp_path / "theme_a.css" + theme_b = tmp_path / "theme_b.css" + consumer = tmp_path / "consumer.css" + + theme_a.write_text(":root { --ambiguous-token: #111; }", encoding="utf-8") + theme_b.write_text(":root { --ambiguous-token: #222; }", encoding="utf-8") + consumer.write_text(".box { color: var(--ambiguous-token); }", encoding="utf-8") + + g = extract([theme_a, theme_b, consumer], root=tmp_path) + consumer_node = next(n for n in g["nodes"] if n.get("label") == "consumer.css") + consumer_uses = [ + e for e in g["edges"] + if e.get("relation") == "uses_token" and e.get("source") == consumer_node["id"] + ] + # Ambiguous cross-file references must NOT resolve (conservative, no fan-out) + assert consumer_uses == [] + + +def test_same_file_precedence_over_cross_file_ambiguity(tmp_path): + theme_a = tmp_path / "theme_a.css" + theme_b = tmp_path / "theme_b.css" + + # Both define --local-token, but theme_a also uses it + theme_a.write_text( + """:root { --local-token: #111; } +.my-el { color: var(--local-token); } +""", + encoding="utf-8", + ) + theme_b.write_text(":root { --local-token: #222; }", encoding="utf-8") + + g = extract([theme_a, theme_b], root=tmp_path) + theme_a_node = next(n for n in g["nodes"] if n.get("label") == "theme_a.css") + token_a = next( + n for n in g["nodes"] + if n.get("label") == "--local-token" and "theme_a" in n["id"] + ) + uses = [ + e for e in g["edges"] + if e.get("relation") == "uses_token" and e.get("source") == theme_a_node["id"] + ] + # Same-file match must resolve to theme_a's own token definition + assert len(uses) == 1 + assert uses[0]["target"] == token_a["id"] + + +def test_nonexistent_token_produces_no_uses_edge(tmp_path): + f = tmp_path / "orphan.css" + f.write_text(".box { color: var(--nonexistent-var); }", encoding="utf-8") + g = extract([f], root=tmp_path) + uses = [e for e in g["edges"] if e.get("relation") == "uses_token"] + assert uses == [] + + +# ── 6. Incremental / Context Resolution ────────────────────────────────────── + +def test_incremental_resolution_with_context_nodes(tmp_path): + # Simulate an incremental build where consumer.css is freshly extracted + # while the token definition in tokens.css is supplied via resolution_context_nodes + consumer = tmp_path / "consumer.css" + consumer.write_text(".panel { background: var(--theme-color); }", encoding="utf-8") + + # Mock token node from unchanged tokens.css + token_id = "tokens_theme_color" + context_node = { + "id": token_id, + "label": "--theme-color", + "file_type": "code", + "node_kind": "token", + "source_file": str(tmp_path / "tokens.css"), + "source_location": "L1", + } + + g = extract( + [consumer], + root=tmp_path, + resolution_context_nodes=[context_node], + ) + consumer_node = next(n for n in g["nodes"] if n.get("label") == "consumer.css") + uses = [ + e for e in g["edges"] + if e.get("relation") == "uses_token" and e.get("source") == consumer_node["id"] + ] + assert len(uses) == 1 + assert uses[0]["target"] == token_id + assert uses[0]["confidence"] == "EXTRACTED" + + +# ── 7. Regression Tests for Edge Cases ──────────────────────────────────────── + +def test_unquoted_url_and_scss_comments(tmp_path): + f = tmp_path / "urls.scss" + f.write_text( + """:root { + --font-url: url(https://example.com/font.woff); + --valid: 123; + --http-url: url(http://example.com/image.png); +} +// --ignored: value; +""", + encoding="utf-8", + ) + res = extract_css(f) + tokens = {n["label"] for n in res["nodes"] if n.get("node_kind") == "token"} + assert "--font-url" in tokens + assert "--valid" in tokens + assert "--http-url" in tokens + assert "--ignored" not in tokens + + +def test_var_inside_quoted_strings_ignored(tmp_path): + f = tmp_path / "content.css" + f.write_text( + """:root { + --brand: blue; +} + +.example::before { + content: "Use var(--brand) here"; + color: var(--brand); +} +""", + encoding="utf-8", + ) + res = extract_css(f) + # Only the real outside-string color: var(--brand) should be captured + assert len(res["raw_token_uses"]) == 1 + use = res["raw_token_uses"][0] + assert use["token_name"] == "--brand" + assert use["line"] == 7 # line of `color: var(--brand);` + + +def test_warm_cache_roundtrip(tmp_path): + t = tmp_path / "tokens.css" + t.write_text(":root { --cached-token: #123; }", encoding="utf-8") + c = tmp_path / "consumer.css" + c.write_text(".btn { color: var(--cached-token); }", encoding="utf-8") + + # Cold extraction + g1 = extract([t, c], root=tmp_path) + uses1 = [e for e in g1["edges"] if e.get("relation") == "uses_token"] + assert len(uses1) == 1 + + # Warm extraction from cache + g2 = extract([t, c], root=tmp_path) + uses2 = [e for e in g2["edges"] if e.get("relation") == "uses_token"] + assert len(uses2) == 1 + assert uses1[0]["source"] == uses2[0]["source"] + assert uses1[0]["target"] == uses2[0]["target"] + diff --git a/tests/test_detect.py b/tests/test_detect.py index 6b47ea265e..afc3449d86 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -69,6 +69,15 @@ def test_classify_powershell_manifest(): # #1331: .psd1 manifests must be classified as CODE so the manifest extractor runs. assert classify_file(Path("MyModule.psd1")) == FileType.CODE +def test_classify_css(): + # #3473: .css files must be classified as CODE for token extraction + assert classify_file(Path("styles.css")) == FileType.CODE + +def test_classify_scss(): + # #3473: .scss files must be classified as CODE for token extraction + assert classify_file(Path("styles.scss")) == FileType.CODE + + def test_classify_markdown(): assert classify_file(Path("README.md")) == FileType.DOCUMENT @@ -2073,6 +2082,15 @@ def test_sensitive_does_not_flag_ruby_code_modules(): assert not _is_sensitive(Path("app/controllers/api/v1/passwords_controller.rb")) +def test_sensitive_does_not_flag_stylesheet_tokens(): + # #3473: stylesheet token files are code, not sensitive secret stores + assert not _is_sensitive(Path("tokens.css")) + assert not _is_sensitive(Path("tokens.scss")) + assert not _is_sensitive(Path("design-tokens.css")) + assert not _is_sensitive(Path("styles/tokens.scss")) + + + def test_sensitive_still_flags_data_secret_stores(): # #1666 guard: the exemption is ONLY for real source code, not data/config # formats — credentials.json / oauth_token.json / secrets.yaml are the secret From 4dc2a1e40320c55f150665aed60689d623c0a397 Mon Sep 17 00:00:00 2001 From: nothariharan Date: Fri, 9 Oct 2026 21:54:44 +0530 Subject: [PATCH 3/5] fix(detect,extract): explicit filter override, design-token files, .gs/.css/.scss/.less and dotfile configs (#2498, #2698, #3473) An explicit wildcard-free '!path' entry in .graphifyignore now overrides only the Stage-3 secrets-keyword heuristic, never the .env/key-file or credential-store stages; a rescued file is kept and the skip report names the escape hatch (#2498). Well-known extensionless config/build files (.htaccess, Dockerfile, Makefile, ...) are classified as documents instead of dropped; .less routes to the CSS extractor (#2698). Combines and supersedes #2747/#3505 (.gs Apps Script), #3480/#3067/#2240 (CSS/SCSS design tokens) and #1921 (allowlist) against current v8. --- graphify/cli.py | 4 +- graphify/detect.py | 93 ++++++++++++++++++++++++++++++--- graphify/extract.py | 1 + graphify/extractors/__init__.py | 1 + graphify/report.py | 14 +++++ tests/test_css.py | 7 +++ tests/test_detect.py | 71 +++++++++++++++++++++++-- 7 files changed, 178 insertions(+), 13 deletions(-) diff --git a/graphify/cli.py b/graphify/cli.py index 1740ca5222..6b4d790d99 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -4087,7 +4087,9 @@ def _parse_float(name: str, raw: str) -> float: _smore = f" (+{len(_sec) - 6} more)" if len(_sec) > 6 else "" print( f"[graphify extract] {len(_sec)} file(s) skipped as potentially sensitive " - f"(rename or move if wrongly flagged): {_snames}{_smore}" + f"(rename or move if wrongly flagged): {_snames}{_smore}. " + f"To keep one anyway, add a '!' line to .graphifyignore " + f"(exact path, no wildcards)." ) stages.mark("detect") diff --git a/graphify/detect.py b/graphify/detect.py index b365a3b1f0..4a528342a9 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -45,7 +45,7 @@ class FileType(str, Enum): _MTIME_COARSE_S = 2.0 _MTIME_SUBSECOND_S = 0.05 -CODE_EXTENSIONS = {'.gs', '.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.vb', '.cbl', '.cob', '.cobol', '.cpy', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.ml', '.mli', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.vh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger', '.lisp', '.cl', '.lsp', '.asd', '.robot', '.resource', '.sol', '.erl', '.hrl', '.escript', '.css', '.scss'} +CODE_EXTENSIONS = {'.gs', '.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.vb', '.cbl', '.cob', '.cobol', '.cpy', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.ml', '.mli', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.vh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger', '.lisp', '.cl', '.lsp', '.asd', '.robot', '.resource', '.sol', '.erl', '.hrl', '.escript', '.css', '.scss', '.less'} DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml', '.j2', '.jinja', '.jinja2'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} @@ -284,8 +284,22 @@ def _is_graphable_source(path: Path) -> bool: return classify_file(path) == FileType.CODE and path.suffix.lower() not in _SECRET_PRONE_DATA_EXTS -def _is_sensitive(path: Path) -> bool: - """Return True if this file likely contains secrets and should be skipped.""" +# Stages of the sensitive-file screen, returned by _sensitive_reason. Stages 1 +# and 2 match specific, near-certain secret locations; Stage 3 is a name-only +# heuristic guess, so it is the only stage an explicit allowlist entry may +# override (#2498). +_SENSITIVE_DIR = "dir" +_SENSITIVE_PATTERN = "pattern" +_SENSITIVE_KEYWORD = "keyword" + + +def _sensitive_reason(path: Path) -> str | None: + """Return which stage flags this file as sensitive, or None if it is kept. + + Kept separate from ``_is_sensitive`` so ``detect()`` can tell the generic + keyword heuristic (Stage 3) — the only stage a user may explicitly override — + apart from the specific dir/key-file stages, which are never overridable. + """ # Stage 1: any PARENT directory is a known secrets dir (parts[:-1] excludes # the filename itself so a root-level file named "credentials" is not falsely # skipped — the name patterns in Stage 2 handle the filename). Dedicated @@ -296,15 +310,15 @@ def _is_sensitive(path: Path) -> bool: # Lowercase the segment comparison so `Secrets/`/`SECRETS/` (real on # case-insensitive macOS/Windows filesystems) are still caught (#2106). if any(part.lower() in _CREDENTIAL_STORE_DIRS for part in parents): - return True + return _SENSITIVE_DIR if any(part.lower() in _AMBIGUOUS_SENSITIVE_DIRS for part in parents) and not _is_graphable_source(path): - return True + return _SENSITIVE_DIR # Stage 2: filename pattern match. Template suffixes (.example/.sample/…) # on .env / .envrc are the usual "safe to commit" convention — keep them # in the graph without opening a broad Stage 2 allowlist (#2184 / #1921). name = path.name if any(p.search(name) for p in _SENSITIVE_PATTERNS) and not _is_env_template(name): - return True + return _SENSITIVE_PATTERN # Stage 3: generic keywords, only when load-bearing in the name. Do NOT let a # bare name keyword silently drop a genuine programming-language source file: # a .rb/.py named device_token or passwords_controller is a module, not a secret @@ -316,7 +330,50 @@ def _is_sensitive(path: Path) -> bool: if _generic_keyword_hit(name): # Genuine source AND multi-word prose notes are exempt; a bare-keyword # name (secrets.md, token.txt) still drops (#1666, #2106). - return not (_is_graphable_source(path) or _is_prose_note(path)) + if _is_graphable_source(path) or _is_prose_note(path): + return None + return _SENSITIVE_KEYWORD + return None + + +def _is_sensitive(path: Path) -> bool: + """Return True if this file likely contains secrets and should be skipped.""" + return _sensitive_reason(path) is not None + + +def _explicit_include_match( + path: Path, + root: Path, + patterns: list[tuple[Path, str]], +) -> bool: + """True if a wildcard-free ``!`` ignore entry names this exact path. + + A ``.graphifyignore`` negation states explicit user intent to graph a path. + Only the Stage-3 name heuristic may be overridden by it (#2498): a specific + Stage-2 pattern (``.env``, ``id_rsa``, ``.pem``) or a credential-store + directory is never rescued, and entries containing glob characters are + ignored so a broad ``!*.json`` opt-in cannot silently start ingesting + credential stores. This mirrors the repo's own guidance that ignored paths + are re-included with ``!`` negation in ``.graphifyignore``. + """ + name = _nfc(path.name) + for anchor, pattern in patterns: + if not pattern.startswith("!"): + continue + raw = pattern[1:] + if not raw or any(ch in raw for ch in "*?["): + continue + if raw.endswith("/"): + continue # a directory-only entry cannot name a file + pat = raw.strip("/") + if not pat: + continue + if "/" in pat: + rel = _lexical_relative(path, path.parts, anchor) + if rel is not None and rel != "." and _match_anchored_ignore_pattern(rel, pat): + return True + elif fnmatch.fnmatch(name, pat): + return True return False @@ -515,6 +572,21 @@ def _shebang_file_type(path: Path) -> FileType | None: return None +# Well-known extensionless config/build files: no suffix, no shebang, but real +# graph-worthy content (a `.htaccess` is the access-control boundary for a +# shared-host deployment; Dockerfile/Makefile/Procfile are the build-and-deploy +# layer). No grammar exists, so they are classified as DOCUMENT for the semantic +# path rather than dropped as "not classified" (#2698). Matched on the exact +# basename, case-insensitively, so a source file that merely shares a stem in a +# different extension is unaffected. +_EXTENSIONLESS_CONFIG_NAMES = frozenset({ + ".htaccess", "dockerfile", "containerfile", "makefile", "gnumakefile", + "procfile", "vagrantfile", "jenkinsfile", "rakefile", "gemfile", + "brewfile", "justfile", "taskfile", "caddyfile", "fastfile", "appfile", + "berksfile", "podfile", "cartfile", "snakefile", "sconstruct", "sconscript", +}) + + def classify_file(path: Path) -> FileType | None: # Package manifests (apm.yml, pyproject.toml, Cargo.toml, go.mod, pom.xml) are parsed # deterministically, so route them to the AST path (CODE) rather than the LLM @@ -530,6 +602,8 @@ def classify_file(path: Path) -> FileType | None: # (.graphifyrc `language.inc=php`, #2961); classify by the declared one. ext = effective_suffix(path).lower() if not ext: + if path.name.lower() in _EXTENSIONLESS_CONFIG_NAMES: + return FileType.DOCUMENT return _shebang_file_type(path) if ext in CODE_EXTENSIONS: return FileType.CODE @@ -2107,7 +2181,10 @@ def _on_walk_error(err: OSError) -> None: # try/except cannot help and the whole run hangs with no output. skipped_sensitive.append(str(p) + " [not a regular file]") continue - if _is_sensitive(p): + sensitive = _sensitive_reason(p) + if sensitive == _SENSITIVE_KEYWORD and _explicit_include_match(p, root, ignore_patterns): + sensitive = None # explicit wildcard-free ! entry beats the name heuristic (#2498) + if sensitive is not None: skipped_sensitive.append(str(p)) continue ftype = classify_file(p) diff --git a/graphify/extract.py b/graphify/extract.py index 204eaf8d77..208439357f 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -8453,6 +8453,7 @@ def add_existing_edge(edge: dict) -> None: ".ejs": extract_ejs, ".css": extract_css, ".scss": extract_css, + ".less": extract_css, ".dart": extract_dart, ".ml": extract_ocaml, ".mli": extract_ocaml, diff --git a/graphify/extractors/__init__.py b/graphify/extractors/__init__.py index 9d0f0b185a..0b029d635f 100644 --- a/graphify/extractors/__init__.py +++ b/graphify/extractors/__init__.py @@ -60,6 +60,7 @@ "json": extract_json, "julia": extract_julia, "lazarus_form": extract_lazarus_form, + "less": extract_css, "markdown": extract_markdown, "objc": extract_objc, "pascal": extract_pascal, diff --git a/graphify/report.py b/graphify/report.py index 0af731ff3d..6f12c324ef 100644 --- a/graphify/report.py +++ b/graphify/report.py @@ -153,6 +153,20 @@ def generate( f"the graph (top: {top})" ) + # Every skipped file by name, not a bare count: one benign file swallowed + # by the secrets filter must stay findable by grepping the report, and the + # report is where the exclusion is durable rather than a transient stderr + # line (#2498). Files rescued by a `!` entry in .graphifyignore never + # reach here, so the list is exactly what is absent from the graph. + skipped = detection_result.get("skipped_sensitive") or [] + if skipped: + lines += [ + "", + "## Skipped Files", + f"{len(skipped)} file(s) were skipped during the scan and are absent from the graph:", + ] + lines += [f"- `{entry}`" for entry in skipped] + from .analyze import _is_file_node, _is_concept_node, _is_contains_only_ast_node def _real_node(n) -> bool: diff --git a/tests/test_css.py b/tests/test_css.py index 61a784cfe6..399e069beb 100644 --- a/tests/test_css.py +++ b/tests/test_css.py @@ -19,6 +19,13 @@ def test_detect_scss_classified_as_code(): assert classify_file(Path("src/styles/_tokens.scss")) == FileType.CODE +def test_detect_less_classified_as_code(): + # #2698 names .less alongside .scss; Less shares the --custom-property / + # var() syntax the CSS extractor handles, so it routes to the same extractor. + assert classify_file(Path("styles.less")) == FileType.CODE + assert not _is_sensitive(Path("tokens.less")) + + def test_tokens_stylesheet_not_flagged_as_sensitive(): assert not _is_sensitive(Path("tokens.css")) assert not _is_sensitive(Path("tokens.scss")) diff --git a/tests/test_detect.py b/tests/test_detect.py index afc3449d86..8b9feeb86f 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -3616,16 +3616,21 @@ def test_nested_graphify_out_prunes_only_configured_path( def test_detect_records_unclassified_extensionless_files(tmp_path): - # #1692: extensionless, non-shebang project files (Dockerfile, Makefile, ...) - # were considered but left no trace. detect() now lists them under - # "unclassified" so they can be surfaced instead of silently vanishing. + # #1692: extensionless, non-shebang project files (LICENSE, ...) were + # considered but left no trace. detect() now lists them under "unclassified" + # so they can be surfaced instead of silently vanishing. + # #2698: well-known build/config dotfiles (Dockerfile, Makefile, ...) are now + # classified as documents instead, so they leave the list — a stronger trace, + # not a silent drop. (tmp_path / "app.py").write_text("def f():\n return 1\n") (tmp_path / "Dockerfile").write_text("FROM python:3.12\nRUN pip install x\n") (tmp_path / "Makefile").write_text("build:\n\techo hi\n") (tmp_path / "LICENSE").write_text("MIT License\n") res = detect(tmp_path) unclassified = sorted(Path(p).name for p in res.get("unclassified", [])) - assert unclassified == ["Dockerfile", "LICENSE", "Makefile"] + assert unclassified == ["LICENSE"] + docs = sorted(Path(p).name for p in res["files"].get("document", [])) + assert docs == ["Dockerfile", "Makefile"] # real code is still classified, not swept into unclassified assert any("app.py" in f for f in res["files"].get("code", [])) @@ -4472,3 +4477,61 @@ def test_globstar_matcher_leaves_no_reference_cycle(): finally: gc.enable() assert collected == 0, f"globstar matcher leaked {collected} cyclic objects per run" + + +# --- #2498: an explicit `!` entry in .graphifyignore beats the name heuristic --- +def test_sensitive_keyword_skip_rescued_by_graphifyignore_negation(tmp_path): + """A wildcard-free `!path` entry in .graphifyignore is explicit user intent + and rescues a file from the Stage-3 secrets-keyword heuristic, while the + default behaviour (no entry) still drops it (#2498).""" + (tmp_path / "tokens.json").write_text('{"color": {"brand": "#fff"}}') + (tmp_path / "tokens.contract.json").write_text('{"color.brand": "#fff"}') + (tmp_path / ".graphifyignore").write_text("!tokens.json\n!tokens.contract.json\n") + res = detect(tmp_path) + kept = sorted(Path(p).name for p in res["files"].get("code", [])) + assert kept == ["tokens.contract.json", "tokens.json"] + assert not res.get("skipped_sensitive") + + +def test_sensitive_keyword_skip_still_dropped_without_negation(tmp_path): + (tmp_path / "tokens.json").write_text('{"color": {"brand": "#fff"}}') + res = detect(tmp_path) + assert not any("tokens.json" in f for f in res["files"].get("code", [])) + assert any("tokens.json" in s for s in res.get("skipped_sensitive", [])) + + +def test_sensitive_dir_and_pattern_skips_not_rescued_by_negation(tmp_path): + """Only the Stage-3 keyword heuristic is overridable: a Stage-2 key file and + a Stage-1 credential-store path stay dropped even when explicitly negated.""" + (tmp_path / ".env").write_text("API_KEY=secret\n") + (tmp_path / "server.pem").write_text("-----BEGIN PRIVATE KEY-----\n") + creds = tmp_path / "credentials" + creds.mkdir() + (creds / "id_rsa").write_text("PRIVATE") + (tmp_path / ".graphifyignore").write_text("!.env\n!server.pem\n!credentials/id_rsa\n") + res = detect(tmp_path) + skipped = " ".join(res.get("skipped_sensitive", [])) + assert ".env" in skipped + assert "server.pem" in skipped + assert "id_rsa" in skipped + assert not res["files"].get("code") + + +def test_sensitive_negation_with_wildcard_does_not_rescue(tmp_path): + """A broad `!*.json` opt-in must not silently start ingesting keyword-named + credential stores (#2498).""" + (tmp_path / "tokens.json").write_text('{"a": 1}') + (tmp_path / ".graphifyignore").write_text("!*.json\n") + res = detect(tmp_path) + assert not any("tokens.json" in f for f in res["files"].get("code", [])) + assert any("tokens.json" in s for s in res.get("skipped_sensitive", [])) + + +def test_extensionless_config_dotfiles_classified_as_documents(tmp_path): + """#2698: well-known build/config dotfiles are classified by filename as + documents instead of being dropped as unclassified.""" + from graphify.detect import FileType, classify_file + for name in (".htaccess", "Dockerfile", "Makefile", "Procfile"): + assert classify_file(tmp_path / name) == FileType.DOCUMENT, name + # an unrelated extensionless file is still unclassified + assert classify_file(tmp_path / "LICENSE") is None From fbede6f5aab3642c864f40bd0b7e570a5d76d868 Mon Sep 17 00:00:00 2001 From: nothariharan Date: Fri, 9 Oct 2026 22:07:54 +0530 Subject: [PATCH 4/5] fix(detect,extract): scope the #2498 override to explicit ignores; stop false CSS token edges (#2498, #3473) Verification follow-up. The rescue now consults only the explicit .graphifyignore/--exclude pattern set (never .gitignore), is anchored to the entry's own directory so a nested ignore cannot rescue a same-named file elsewhere, and matches case-sensitively. A custom property declared outside a root/theme context now marks its stylesheet's var() uses as locally shadowed, so cross-file resolution no longer invents a false EXTRACTED uses_token edge. --- graphify/detect.py | 22 +++++++++++++++++----- graphify/extract.py | 5 +++++ graphify/extractors/css.py | 16 ++++++++++++++++ tests/test_css.py | 14 ++++++++++++++ tests/test_detect.py | 37 +++++++++++++++++++++++++++++++++++++ tests/test_report.py | 17 +++++++++++++++++ 6 files changed, 106 insertions(+), 5 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index 4a528342a9..1c59cf6ab0 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -346,7 +346,7 @@ def _explicit_include_match( root: Path, patterns: list[tuple[Path, str]], ) -> bool: - """True if a wildcard-free ``!`` ignore entry names this exact path. + """True if a wildcard-free ``!`` entry from an explicit ignore file names this path. A ``.graphifyignore`` negation states explicit user intent to graph a path. Only the Stage-3 name heuristic may be overridden by it (#2498): a specific @@ -355,6 +355,13 @@ def _explicit_include_match( ignored so a broad ``!*.json`` opt-in cannot silently start ingesting credential stores. This mirrors the repo's own guidance that ignored paths are re-included with ``!`` negation in ``.graphifyignore``. + + ``patterns`` must be the EXPLICIT set (``.graphifyignore`` + ``--exclude``), + not the merged ``.gitignore`` set: un-ignoring a secret in ``.gitignore`` is + a Git operation, not a statement that the file is safe to summarise into a + knowledge graph. A bare entry is scoped to its own anchor directory so a + nested ``.graphifyignore`` cannot rescue a same-named file elsewhere, and + matching is case-sensitive so the "exact path" the hint promises is exact. """ name = _nfc(path.name) for anchor, pattern in patterns: @@ -368,11 +375,14 @@ def _explicit_include_match( pat = raw.strip("/") if not pat: continue + # The entry only governs paths beneath its own anchor directory. + rel = _lexical_relative(path, path.parts, anchor) + if rel is None or rel == ".": + continue if "/" in pat: - rel = _lexical_relative(path, path.parts, anchor) - if rel is not None and rel != "." and _match_anchored_ignore_pattern(rel, pat): + if _match_anchored_ignore_pattern(rel, pat): return True - elif fnmatch.fnmatch(name, pat): + elif fnmatch.fnmatchcase(name, pat): return True return False @@ -2182,7 +2192,9 @@ def _on_walk_error(err: OSError) -> None: skipped_sensitive.append(str(p) + " [not a regular file]") continue sensitive = _sensitive_reason(p) - if sensitive == _SENSITIVE_KEYWORD and _explicit_include_match(p, root, ignore_patterns): + if sensitive == _SENSITIVE_KEYWORD and _explicit_include_match( + p, root, explicit_ignore_patterns + ): sensitive = None # explicit wildcard-free ! entry beats the name heuristic (#2498) if sensitive is not None: skipped_sensitive.append(str(p)) diff --git a/graphify/extract.py b/graphify/extract.py index 208439357f..4bad2fc2f6 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -9076,6 +9076,11 @@ def _norm_sf(sf: str | Path | None) -> str: token_name = tu.get("token_name", "") if not token_name: continue + if tu.get("local_decl"): + # A local `--x:` declaration in the same stylesheet shadows any + # same-named global token, so binding var(--x) across files would be + # a false EXTRACTED edge (e.g. a component overriding a theme token). + continue source_file = tu.get("source_file", "") norm_sf = _norm_sf(source_file) diff --git a/graphify/extractors/css.py b/graphify/extractors/css.py index 3a8775d6f5..de95892314 100644 --- a/graphify/extractors/css.py +++ b/graphify/extractors/css.py @@ -183,6 +183,11 @@ def extract_css(path: Path) -> dict: ] edges: list[dict] = [] raw_token_uses: list[dict] = [] + # Custom properties declared OUTSIDE a root/theme context (e.g. + # `.card { --color: red; }`). They shadow any same-named global token within + # this stylesheet, so a later var(--color) must not bind to a cross-file + # definition; recorded here and stamped onto each use below. + local_nonroot_names: set[str] = set() # 1. Collect var(--token) references outside string literals seen_uses: set[tuple[str, int]] = set() @@ -269,6 +274,10 @@ def extract_css(path: Path) -> dict: "weight": 1.0, } ) + elif stmt and stack: + m = _DECL_RE.match(stmt) + if m: + local_nonroot_names.add(m.group(1)) if stack: stack.pop() buf = [] @@ -308,6 +317,10 @@ def extract_css(path: Path) -> dict: "weight": 1.0, } ) + elif stmt and stack: + m = _DECL_RE.match(stmt) + if m: + local_nonroot_names.add(m.group(1)) buf = [] i += 1 continue @@ -315,6 +328,9 @@ def extract_css(path: Path) -> dict: buf.append(c) i += 1 + for tu in raw_token_uses: + tu["local_decl"] = tu["token_name"] in local_nonroot_names + return { "nodes": nodes, "edges": edges, diff --git a/tests/test_css.py b/tests/test_css.py index 399e069beb..5219539f58 100644 --- a/tests/test_css.py +++ b/tests/test_css.py @@ -26,6 +26,20 @@ def test_detect_less_classified_as_code(): assert not _is_sensitive(Path("tokens.less")) +def test_local_declaration_shadows_global_token(tmp_path): + """A custom property declared OUTSIDE a root/theme context (a component + overriding a theme token) must stop that stylesheet's var() from binding to + a cross-file global definition — otherwise the graph invents a false + EXTRACTED uses_token edge (#3473).""" + theme = tmp_path / "theme.css" + theme.write_text(":root { --color: blue; }\n") + card = tmp_path / "card.css" + card.write_text(".card { --color: red; color: var(--color); }\n") + graph = extract([theme, card], root=tmp_path) + uses = [e for e in graph["edges"] if e.get("relation") == "uses_token"] + assert uses == [] + + def test_tokens_stylesheet_not_flagged_as_sensitive(): assert not _is_sensitive(Path("tokens.css")) assert not _is_sensitive(Path("tokens.scss")) diff --git a/tests/test_detect.py b/tests/test_detect.py index 8b9feeb86f..2ec931f001 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -4535,3 +4535,40 @@ def test_extensionless_config_dotfiles_classified_as_documents(tmp_path): assert classify_file(tmp_path / name) == FileType.DOCUMENT, name # an unrelated extensionless file is still unclassified assert classify_file(tmp_path / "LICENSE") is None + + +def test_sensitive_rescue_ignores_gitignore_only_negation(tmp_path): + """Only an EXPLICIT .graphifyignore/--exclude entry may rescue a keyword + skip: un-ignoring a secret in .gitignore is a Git operation, not a safety + statement (#2498).""" + (tmp_path / "tokens.json").write_text('{"a": 1}') + (tmp_path / ".gitignore").write_text("!tokens.json\n") + res = detect(tmp_path) + assert not any("tokens.json" in f for f in res["files"].get("code", [])) + assert any("tokens.json" in s for s in res.get("skipped_sensitive", [])) + + +def test_sensitive_rescue_bare_entry_scoped_to_its_anchor(tmp_path): + """A bare `!name` entry only governs its own directory: a nested + .graphifyignore cannot rescue a same-named file elsewhere (#2498).""" + (tmp_path / "tokens.json").write_text('{"a": 1}') + sub = tmp_path / "sub" + sub.mkdir() + (sub / "tokens.json").write_text('{"b": 2}') + (sub / ".graphifyignore").write_text("!tokens.json\n") + res = detect(tmp_path) + kept = {Path(p).resolve() for p in res["files"].get("code", [])} + assert (sub / "tokens.json").resolve() in kept + assert (tmp_path / "tokens.json").resolve() not in kept + + +def test_sensitive_rescue_path_entry_matches_only_that_path(tmp_path): + (tmp_path / "tokens.json").write_text('{"a": 1}') + sub = tmp_path / "sub" + sub.mkdir() + (sub / "tokens.json").write_text('{"b": 2}') + (tmp_path / ".graphifyignore").write_text("!sub/tokens.json\n") + res = detect(tmp_path) + kept = {Path(p).resolve() for p in res["files"].get("code", [])} + assert (sub / "tokens.json").resolve() in kept + assert (tmp_path / "tokens.json").resolve() not in kept diff --git a/tests/test_report.py b/tests/test_report.py index 98f256907d..e677c389c0 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -52,6 +52,23 @@ def test_report_omits_unclassified_line_when_none(): report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") assert "Unclassified:" not in report + +def test_report_lists_skipped_sensitive_files(): + """#2498: a file dropped by the secrets filter is named in the report, so an + exclusion is durable and greppable rather than a transient stderr line.""" + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + detection = {**detection, "skipped_sensitive": ["tokens.json", "server.pem"]} + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") + assert "## Skipped Files" in report + assert "tokens.json" in report + assert "server.pem" in report + + +def test_report_omits_skipped_section_when_none(): + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") + assert "## Skipped Files" not in report + def test_report_contains_god_nodes(): G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") From 1f97311710a3bacd82a14b442be903e1e73510d1 Mon Sep 17 00:00:00 2001 From: nothariharan Date: Fri, 9 Oct 2026 22:17:42 +0530 Subject: [PATCH 5/5] fix(extract,detect): keep same-file token edges; anchor leading-slash override entries (#2498, #3473) A stylesheet that defines a token in :root and overrides it locally still gets its same-file uses_token edge; the local shadow now only suppresses cross-file binding. A leading-slash .graphifyignore entry (!/name) is anchored to its own directory again, matching gitignore semantics. --- graphify/detect.py | 5 ++++- graphify/extract.py | 12 +++++++----- tests/test_css.py | 13 +++++++++++++ tests/test_detect.py | 14 ++++++++++++++ 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/graphify/detect.py b/graphify/detect.py index 1c59cf6ab0..31df05cfa5 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -372,6 +372,9 @@ def _explicit_include_match( continue if raw.endswith("/"): continue # a directory-only entry cannot name a file + # A leading `/` anchors the entry to its own directory (gitignore + # semantics), so `!/tokens.json` must not match `sub/tokens.json`. + anchored = raw.startswith("/") pat = raw.strip("/") if not pat: continue @@ -379,7 +382,7 @@ def _explicit_include_match( rel = _lexical_relative(path, path.parts, anchor) if rel is None or rel == ".": continue - if "/" in pat: + if anchored or "/" in pat: if _match_anchored_ignore_pattern(rel, pat): return True elif fnmatch.fnmatchcase(name, pat): diff --git a/graphify/extract.py b/graphify/extract.py index 4bad2fc2f6..20d81575af 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -9076,11 +9076,6 @@ def _norm_sf(sf: str | Path | None) -> str: token_name = tu.get("token_name", "") if not token_name: continue - if tu.get("local_decl"): - # A local `--x:` declaration in the same stylesheet shadows any - # same-named global token, so binding var(--x) across files would be - # a false EXTRACTED edge (e.g. a component overriding a theme token). - continue source_file = tu.get("source_file", "") norm_sf = _norm_sf(source_file) @@ -9090,6 +9085,13 @@ def _norm_sf(sf: str | Path | None) -> str: same_file_matches = tokens_by_file_and_label.get((norm_sf, token_name), []) if same_file_matches: target_node = same_file_matches[0] + elif tu.get("local_decl"): + # No same-file token NODE for this name, but the stylesheet declares + # `--x:` locally outside a root/theme context (e.g. a component + # overriding a theme token). That local declaration shadows any + # cross-file global, so binding var(--x) to another file would be a + # false EXTRACTED edge. Same-file resolution above is unaffected. + continue else: # 2. Exactly one matching definition across available graph/resolution nodes global_matches = tokens_by_label.get(token_name, []) diff --git a/tests/test_css.py b/tests/test_css.py index 5219539f58..222211ab4c 100644 --- a/tests/test_css.py +++ b/tests/test_css.py @@ -40,6 +40,19 @@ def test_local_declaration_shadows_global_token(tmp_path): assert uses == [] +def test_same_file_root_token_links_despite_local_override(tmp_path): + """A stylesheet that defines --x in :root and also overrides it locally still + gets its own same-file uses_token edge; the local shadow only suppresses + cross-file binding (#3473).""" + css = tmp_path / "t.css" + css.write_text(":root { --x: 1; }\n.card { --x: 2; }\n.a { color: var(--x); }\n") + graph = extract([css], root=tmp_path) + uses = [e for e in graph["edges"] if e.get("relation") == "uses_token"] + token_ids = {n["id"] for n in graph["nodes"] if n.get("node_kind") == "token"} + assert len(uses) == 1 + assert uses[0]["target"] in token_ids + + def test_tokens_stylesheet_not_flagged_as_sensitive(): assert not _is_sensitive(Path("tokens.css")) assert not _is_sensitive(Path("tokens.scss")) diff --git a/tests/test_detect.py b/tests/test_detect.py index 2ec931f001..be3e1dc04b 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -4572,3 +4572,17 @@ def test_sensitive_rescue_path_entry_matches_only_that_path(tmp_path): kept = {Path(p).resolve() for p in res["files"].get("code", [])} assert (sub / "tokens.json").resolve() in kept assert (tmp_path / "tokens.json").resolve() not in kept + + +def test_sensitive_rescue_leading_slash_entry_is_anchor_exact(tmp_path): + """`!/tokens.json` anchors to the entry's own directory (gitignore + semantics), so it must not rescue `sub/tokens.json` (#2498).""" + (tmp_path / "tokens.json").write_text('{"a": 1}') + sub = tmp_path / "sub" + sub.mkdir() + (sub / "tokens.json").write_text('{"b": 2}') + (tmp_path / ".graphifyignore").write_text("!/tokens.json\n") + res = detect(tmp_path) + kept = {Path(p).resolve() for p in res["files"].get("code", [])} + assert (tmp_path / "tokens.json").resolve() in kept + assert (sub / "tokens.json").resolve() not in kept