feat(extract,detect): Apps Script, CSS/SCSS design tokens and an explicit secrets-filter override (#2498, #2698, #3473)

graphify/analyze.py

@@ -31,7 +31,7 @@
 # Language families — extensions sharing a runtime can legitimately call each other
 _LANG_FAMILY: dict[str, str] = {
     **{e: "python" for e in (".py", ".pyw")},
-    **{e: "js" for e in (".js", ".jsx", ".mjs", ".cjs", ".ejs", ".ts", ".tsx", ".mts", ".cts", ".vue", ".svelte")},
+    **{e: "js" for e in (".js", ".jsx", ".mjs", ".cjs", ".ejs", ".gs", ".ts", ".tsx", ".mts", ".cts", ".vue", ".svelte")},
     **{e: "go" for e in (".go",)},
     **{e: "rust" for e in (".rs",)},
     **{e: "jvm" for e in (".java", ".kt", ".kts", ".scala")},

graphify/build.py

@@ -334,7 +334,7 @@ def finalize_raw_graph_endpoints(data: dict) -> None:
 # which imports build.py — a cycle) and deliberately mirrors extract._LANG_FAMILY_BY_EXT.
 _EDGE_LANG_FAMILY: dict[str, str] = {
     ".py": "py", ".pyi": "py",
-    ".js": "js", ".mjs": "js", ".cjs": "js", ".jsx": "js",
+    ".js": "js", ".mjs": "js", ".cjs": "js", ".jsx": "js", ".gs": "js",
     ".ts": "js", ".tsx": "js", ".mts": "js", ".cts": "js",
     ".go": "go", ".rs": "rs",
     ".java": "jvm", ".kt": "jvm", ".scala": "jvm", ".groovy": "jvm",

graphify/cache.py

@@ -692,7 +692,7 @@ def _relativize_source_files_in(payload: dict, root: Path) -> None:
     # definition_file (#2990) is a path into the scanned tree exactly like
     # source_file; a cache entry keeping it absolute replayed the build host's
     # layout on every warm hit (#3223).
-    for bucket in ("nodes", "edges", "hyperedges", "raw_calls"):
+    for bucket in ("nodes", "edges", "hyperedges", "raw_calls", "raw_token_uses"):
         for item in payload.get(bucket, []):
             if not isinstance(item, dict):
                 continue
@@ -994,7 +994,7 @@ def _absolutize_source_files_in(payload: dict, root: Path) -> None:
         root_resolved = Path(root).resolve()
     except OSError:
         return
-    for bucket in ("nodes", "edges", "hyperedges", "raw_calls"):
+    for bucket in ("nodes", "edges", "hyperedges", "raw_calls", "raw_token_uses"):
         for item in payload.get(bucket, []):
             if not isinstance(item, dict):
                 continue

graphify/cli.py

@@ -70,7 +70,7 @@
     }
 }, ensure_ascii=False, separators=(",", ":")) + "\n"
 _HOOK_SOURCE_EXTS = (
-    '.py', '.js', '.cjs', '.ts', '.tsx', '.jsx', '.astro', '.vue', '.svelte', '.ejs', '.go',
+    '.py', '.js', '.cjs', '.gs', '.ts', '.tsx', '.jsx', '.astro', '.vue', '.svelte', '.ejs', '.go',
     '.rs', '.java', '.rb', '.c', '.h', '.cpp', '.hpp', '.cc', '.cs', '.kt',
     '.swift', '.php', '.scala', '.lua', '.ps1', '.psm1', '.psd1',  # #4271: PowerShell sources are graphable too
     '.sh', '.md', '.rst', '.txt', '.mdx',
@@ -4087,7 +4087,9 @@ def _parse_float(name: str, raw: str) -> float:
             _smore = f" (+{len(_sec) - 6} more)" if len(_sec) > 6 else ""
             print(
                 f"[graphify extract] {len(_sec)} file(s) skipped as potentially sensitive "
-                f"(rename or move if wrongly flagged): {_snames}{_smore}"
+                f"(rename or move if wrongly flagged): {_snames}{_smore}. "
+                f"To keep one anyway, add a '!<relative-path>' line to .graphifyignore "
+                f"(exact path, no wildcards)."
             )
         stages.mark("detect")
 

graphify/detect.py

@@ -45,7 +45,7 @@ class FileType(str, Enum):
 _MTIME_COARSE_S = 2.0
 _MTIME_SUBSECOND_S = 0.05
 
-CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.vb', '.cbl', '.cob', '.cobol', '.cpy', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.ml', '.mli', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.vh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger', '.lisp', '.cl', '.lsp', '.asd', '.robot', '.resource', '.sol', '.erl', '.hrl', '.escript'}
+CODE_EXTENSIONS = {'.gs', '.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.vb', '.cbl', '.cob', '.cobol', '.cpy', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.ml', '.mli', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.vh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger', '.lisp', '.cl', '.lsp', '.asd', '.robot', '.resource', '.sol', '.erl', '.hrl', '.escript', '.css', '.scss', '.less'}
 DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml', '.j2', '.jinja', '.jinja2'}
 PAPER_EXTENSIONS = {'.pdf'}
 IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'}
@@ -284,8 +284,22 @@ def _is_graphable_source(path: Path) -> bool:
     return classify_file(path) == FileType.CODE and path.suffix.lower() not in _SECRET_PRONE_DATA_EXTS
 
 
-def _is_sensitive(path: Path) -> bool:
-    """Return True if this file likely contains secrets and should be skipped."""
+# Stages of the sensitive-file screen, returned by _sensitive_reason. Stages 1
+# and 2 match specific, near-certain secret locations; Stage 3 is a name-only
+# heuristic guess, so it is the only stage an explicit allowlist entry may
+# override (#2498).
+_SENSITIVE_DIR = "dir"
+_SENSITIVE_PATTERN = "pattern"
+_SENSITIVE_KEYWORD = "keyword"
+
+
+def _sensitive_reason(path: Path) -> str | None:
+    """Return which stage flags this file as sensitive, or None if it is kept.
+
+    Kept separate from ``_is_sensitive`` so ``detect()`` can tell the generic
+    keyword heuristic (Stage 3) — the only stage a user may explicitly override —
+    apart from the specific dir/key-file stages, which are never overridable.
+    """
     # Stage 1: any PARENT directory is a known secrets dir (parts[:-1] excludes
     # the filename itself so a root-level file named "credentials" is not falsely
     # skipped — the name patterns in Stage 2 handle the filename). Dedicated
@@ -296,15 +310,15 @@ def _is_sensitive(path: Path) -> bool:
     # Lowercase the segment comparison so `Secrets/`/`SECRETS/` (real on
     # case-insensitive macOS/Windows filesystems) are still caught (#2106).
     if any(part.lower() in _CREDENTIAL_STORE_DIRS for part in parents):
-        return True
+        return _SENSITIVE_DIR
     if any(part.lower() in _AMBIGUOUS_SENSITIVE_DIRS for part in parents) and not _is_graphable_source(path):
-        return True
+        return _SENSITIVE_DIR
     # Stage 2: filename pattern match. Template suffixes (.example/.sample/…)
     # on .env / .envrc are the usual "safe to commit" convention — keep them
     # in the graph without opening a broad Stage 2 allowlist (#2184 / #1921).
     name = path.name
     if any(p.search(name) for p in _SENSITIVE_PATTERNS) and not _is_env_template(name):
-        return True
+        return _SENSITIVE_PATTERN
     # Stage 3: generic keywords, only when load-bearing in the name. Do NOT let a
     # bare name keyword silently drop a genuine programming-language source file:
     # a .rb/.py named device_token or passwords_controller is a module, not a secret
@@ -316,7 +330,63 @@ def _is_sensitive(path: Path) -> bool:
     if _generic_keyword_hit(name):
         # Genuine source AND multi-word prose notes are exempt; a bare-keyword
         # name (secrets.md, token.txt) still drops (#1666, #2106).
-        return not (_is_graphable_source(path) or _is_prose_note(path))
+        if _is_graphable_source(path) or _is_prose_note(path):
+            return None
+        return _SENSITIVE_KEYWORD
+    return None
+
+
+def _is_sensitive(path: Path) -> bool:
+    """Return True if this file likely contains secrets and should be skipped."""
+    return _sensitive_reason(path) is not None
+
+
+def _explicit_include_match(
+    path: Path,
+    root: Path,
+    patterns: list[tuple[Path, str]],
+) -> bool:
+    """True if a wildcard-free ``!`` entry from an explicit ignore file names this path.
+
+    A ``.graphifyignore`` negation states explicit user intent to graph a path.
+    Only the Stage-3 name heuristic may be overridden by it (#2498): a specific
+    Stage-2 pattern (``.env``, ``id_rsa``, ``.pem``) or a credential-store
+    directory is never rescued, and entries containing glob characters are
+    ignored so a broad ``!*.json`` opt-in cannot silently start ingesting
+    credential stores. This mirrors the repo's own guidance that ignored paths
+    are re-included with ``!`` negation in ``.graphifyignore``.
+
+    ``patterns`` must be the EXPLICIT set (``.graphifyignore`` + ``--exclude``),
+    not the merged ``.gitignore`` set: un-ignoring a secret in ``.gitignore`` is
+    a Git operation, not a statement that the file is safe to summarise into a
+    knowledge graph. A bare entry is scoped to its own anchor directory so a
+    nested ``.graphifyignore`` cannot rescue a same-named file elsewhere, and
+    matching is case-sensitive so the "exact path" the hint promises is exact.
+    """
+    name = _nfc(path.name)
+    for anchor, pattern in patterns:
+        if not pattern.startswith("!"):
+            continue
+        raw = pattern[1:]
+        if not raw or any(ch in raw for ch in "*?["):
+            continue
+        if raw.endswith("/"):
+            continue  # a directory-only entry cannot name a file
+        # A leading `/` anchors the entry to its own directory (gitignore
+        # semantics), so `!/tokens.json` must not match `sub/tokens.json`.
+        anchored = raw.startswith("/")
+        pat = raw.strip("/")
+        if not pat:
+            continue
+        # The entry only governs paths beneath its own anchor directory.
+        rel = _lexical_relative(path, path.parts, anchor)
+        if rel is None or rel == ".":
+            continue
+        if anchored or "/" in pat:
+            if _match_anchored_ignore_pattern(rel, pat):
+                return True
+        elif fnmatch.fnmatchcase(name, pat):
+            return True
     return False
 
 
@@ -515,6 +585,21 @@ def _shebang_file_type(path: Path) -> FileType | None:
     return None
 
 
+# Well-known extensionless config/build files: no suffix, no shebang, but real
+# graph-worthy content (a `.htaccess` is the access-control boundary for a
+# shared-host deployment; Dockerfile/Makefile/Procfile are the build-and-deploy
+# layer). No grammar exists, so they are classified as DOCUMENT for the semantic
+# path rather than dropped as "not classified" (#2698). Matched on the exact
+# basename, case-insensitively, so a source file that merely shares a stem in a
+# different extension is unaffected.
+_EXTENSIONLESS_CONFIG_NAMES = frozenset({
+    ".htaccess", "dockerfile", "containerfile", "makefile", "gnumakefile",
+    "procfile", "vagrantfile", "jenkinsfile", "rakefile", "gemfile",
+    "brewfile", "justfile", "taskfile", "caddyfile", "fastfile", "appfile",
+    "berksfile", "podfile", "cartfile", "snakefile", "sconstruct", "sconscript",
+})
+
+
 def classify_file(path: Path) -> FileType | None:
     # Package manifests (apm.yml, pyproject.toml, Cargo.toml, go.mod, pom.xml) are parsed
     # deterministically, so route them to the AST path (CODE) rather than the LLM
@@ -530,6 +615,8 @@ def classify_file(path: Path) -> FileType | None:
     # (.graphifyrc `language.inc=php`, #2961); classify by the declared one.
     ext = effective_suffix(path).lower()
     if not ext:
+        if path.name.lower() in _EXTENSIONLESS_CONFIG_NAMES:
+            return FileType.DOCUMENT
         return _shebang_file_type(path)
     if ext in CODE_EXTENSIONS:
         return FileType.CODE
@@ -2107,7 +2194,12 @@ def _on_walk_error(err: OSError) -> None:
             # try/except cannot help and the whole run hangs with no output.
             skipped_sensitive.append(str(p) + " [not a regular file]")
             continue
-        if _is_sensitive(p):
+        sensitive = _sensitive_reason(p)
+        if sensitive == _SENSITIVE_KEYWORD and _explicit_include_match(
+            p, root, explicit_ignore_patterns
+        ):
+            sensitive = None  # explicit wildcard-free ! entry beats the name heuristic (#2498)
+        if sensitive is not None:
             skipped_sensitive.append(str(p))
             continue
         ftype = classify_file(p)