feat: Twig templates and controller-to-template links

CHANGELOG.md

@@ -4,6 +4,13 @@ Full release notes with details on each version: [GitHub Releases](https://githu
 
 ## Unreleased
 
+**Twig templates and controller-to-template links (`alexwrite/graphify#19`):**
+
+- Feature: `.twig` files are extracted (Twig extractor from Graphify-Labs/graphify#2240, Twig part only). `{% extends %}`, `{% include %}`, `{% embed %}`, `{% import %}`, `{% from %}`, `{% use %}` and `include()`/`source()` link to the real template file node; blocks and Drupal `attach_library()` are recorded too.
+- Fix: a bare Twig name resolves against the loader root (the enclosing `templates/` dir) only, never against the referring file's own directory, and a Symfony `@Namespace/` name stays a stub instead of being matched against the app's templates. Drupal `@extension/` names resolve under that extension, as core does.
+- Feature: a PHP method that renders a literal template name (`render()`, `renderView()`, `renderBlock()`, `renderBlockView()`, `#[Template(...)]`) gets a `renders` edge to the template file. A name built at runtime, or one with no file on disk, gets no edge. AST cache schema 9.
+- Not covered: Twig components (`<twig:Name>`, `component()`), and Twig paths configured in `twig.yaml`.
+
 Consolidated PHP-support and resolution work from the [`lawnstarter/graphify`](https://github.com/lawnstarter/graphify) fork — #1682 and everything found while fixing it. The full defect report is #2615. The groups below are ordered as the work landed on the fork, so a later group can lift a refusal an earlier group states; those spots are marked.
 
 **PHP member-call resolution, cross-language isolation, and import-edge repair:**

graphify/cache.py

@@ -37,7 +37,7 @@
     _EXTRACTOR_VERSION = "unknown"
 
 # Bump when AST cache-key semantics change independently of the package version.
-_AST_CACHE_SCHEMA = 8  # PHP typed-receiver, use-import and declared-FQN facts in persisted raw calls.
+_AST_CACHE_SCHEMA = 9  # PHP `renders` edges to Twig templates.
 
 
 # Per-process memo: distributions() scans site-packages metadata, which is

graphify/detect.py

@@ -44,7 +44,7 @@ class FileType(str, Enum):
 _MTIME_COARSE_S = 2.0
 _MTIME_SUBSECOND_S = 0.05
 
-CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.vb', '.cbl', '.cob', '.cobol', '.cpy', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.ml', '.mli', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.vh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger', '.lisp', '.cl', '.lsp', '.asd', '.robot', '.resource', '.sol', '.erl', '.hrl', '.escript'}
+CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.vb', '.cbl', '.cob', '.cobol', '.cpy', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.ml', '.mli', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.vh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger', '.twig', '.lisp', '.cl', '.lsp', '.asd', '.robot', '.resource', '.sol', '.erl', '.hrl', '.escript'}
 DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml'}
 PAPER_EXTENSIONS = {'.pdf'}
 IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'}

graphify/extract.py

@@ -73,6 +73,7 @@
 )
 from graphify.extractors.sql import extract_sql  # noqa: F401
 from graphify.extractors.terraform import extract_terraform, prepare_terraform, resolve_terraform_modules  # noqa: F401
+from graphify.extractors.twig import extract_twig  # noqa: F401
 from graphify.extractors.verilog import extract_verilog  # noqa: F401
 from graphify.extractors.vbnet import extract_vbnet, resolve_vbnet_partial_calls  # noqa: F401
 from graphify.extractors.zig import extract_zig  # noqa: F401
@@ -3554,6 +3555,10 @@ def _lang_is_case_insensitive(source_file: object) -> bool:
     ".dart": "dart",
     ".sh": "shell", ".bash": "shell",
     ".ps1": "powershell", ".psm1": "powershell", ".psd1": "powershell",
+    # Naming the template family keeps a Twig include stub from being remapped
+    # onto a same-labeled function in an unrelated language, which is what an
+    # unknown (None) family permits.
+    ".twig": "twig",
 }
 
 
@@ -7938,6 +7943,7 @@ def add_existing_edge(edge: dict) -> None:
     ".resource": extract_robot,
     ".cls": extract_apex,
     ".trigger": extract_apex,
+    ".twig": extract_twig,
 }
 
 

graphify/extractors/__init__.py

@@ -35,6 +35,7 @@
 from graphify.extractors.solidity import extract_solidity
 from graphify.extractors.sql import extract_sql
 from graphify.extractors.terraform import extract_terraform
+from graphify.extractors.twig import extract_twig
 from graphify.extractors.verilog import extract_verilog
 from graphify.extractors.vbnet import extract_vbnet
 from graphify.extractors.zig import extract_zig
@@ -70,6 +71,7 @@
     "solidity": extract_solidity,
     "sql": extract_sql,
     "terraform": extract_terraform,
+    "twig": extract_twig,
     "verilog": extract_verilog,
     "vbnet": extract_vbnet,
     "zig": extract_zig,

graphify/extractors/base.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 import sys
+import re
 from pathlib import Path
 
 from graphify.ids import make_id
@@ -151,3 +152,34 @@ def _read_source_text(path: Path, *, warn: bool = True) -> str:
     """
     text = _read_source_bytes(path, warn=warn).decode("utf-8")
     return text.replace("\r\n", "\n").replace("\r", "\n")
+def _blank_spans(src: str, pattern: "re.Pattern[str]") -> str:
+    """Replace every match of *pattern* with newlines and spaces of equal length.
+
+    Used to strip comments before a regex-based extractor scans a file, without
+    disturbing byte offsets — so reported ``L<n>`` locations still refer to the
+    real line in the original source. Matters because component docblocks
+    routinely contain example usage (``{# Usage: {% include 'theme:card' %} #}``)
+    that would otherwise be extracted as a real dependency, giving the component
+    a spurious edge to itself.
+    """
+    return pattern.sub(
+        lambda m: "".join("\n" if ch == "\n" else " " for ch in m.group(0)), src
+    )
+
+
+def _corpus_relative_path(resolved: Path, referrer: Path) -> str:
+    """Express *resolved* the same way *referrer* is, for node-ID parity.
+
+    A file node's ID is ``_make_id(str(path))`` for whatever path form the
+    extractor was handed. An extractor that resolves a cross-file reference to an
+    absolute path while the corpus was scanned relatively would mint a second,
+    disconnected node for a file that already has one, so match the referrer's
+    form. Mirrors the inline normalization in ``extract_bash``.
+    """
+    target = resolved.resolve()
+    if not referrer.is_absolute():
+        try:
+            target = target.relative_to(Path.cwd().resolve())
+        except ValueError:
+            pass
+    return str(target)

graphify/extractors/engine.py

@@ -13,6 +13,7 @@
 )
 from graphify.ids import normalize_id
 from graphify.extractors.models import LanguageConfig
+from graphify.extractors.php_views import render_call_template, render_edge, template_attribute_templates
 from graphify.extractors.resolution import _resolve_js_import_target
 from graphify.security import sanitize_metadata
 from pathlib import Path
@@ -5408,6 +5409,14 @@ def add_edge(src: str, tgt: str, relation: str, line: int,
             edge["metadata"] = sanitize_metadata(metadata)
         edges.append(edge)
 
+    seen_render_pairs: set[tuple[str, str]] = set()
+
+    def add_render_edge(ref: tuple[str, str], src: str, line: int) -> None:
+        edge = render_edge(ref, src, path, line)
+        if edge is not None and (src, edge["target"]) not in seen_render_pairs:
+            seen_render_pairs.add((src, edge["target"]))
+            edges.append(edge)
+
     def ensure_named_node(name: str, line: int) -> str:
         nid = _make_id(stem, ".".join(namespace_stack), name)
         if nid in seen_ids:
@@ -7120,6 +7129,8 @@ def scala_base_name(type_node) -> str | None:
                         annotation_targets.add(target_nid)
 
             if config.ts_module == "tree_sitter_php":
+                for engine_name, template, attr_line in template_attribute_templates(node, source):
+                    add_render_edge((engine_name, template), func_nid, attr_line)
                 params_container = None
                 for c in node.children:
                     if c.type == "formal_parameters":
@@ -9135,6 +9146,11 @@ def walk_calls(
                         if arg.type == "identifier":
                             _emit_indirect_ref(arg, caller_nid, enclosing_locals, "argument")
 
+            if config.ts_module == "tree_sitter_php":
+                rendered = render_call_template(node, source)
+                if rendered is not None:
+                    add_render_edge(rendered, caller_nid, node.start_point[0] + 1)
+
             # Helper function calls: config('foo.bar') → uses_config edge to "foo"
             if (callee_name and callee_name in config.helper_fn_names):
                 args_node = node.child_by_field_name("arguments")
@@ -9498,7 +9514,7 @@ def _scan_js_module_dispatch(n) -> None:
     _seen_edge_payloads: set[str] = set()
     for edge in edges:
         src, tgt = edge["source"], edge["target"]
-        if src in valid_ids and (tgt in valid_ids or edge["relation"] in ("imports", "imports_from", "re_exports")):
+        if src in valid_ids and (tgt in valid_ids or edge["relation"] in ("imports", "imports_from", "re_exports", "renders")):
             payload = json.dumps(edge, sort_keys=True, default=str)
             if payload in _seen_edge_payloads:
                 continue

graphify/extractors/php_views.py

@@ -0,0 +1,99 @@
+"""PHP code that renders a template: the call or attribute, resolved to the template file.
+
+Only a literal template name is followed, and only when it resolves to a file on
+disk; a name built at runtime (variable, concatenation, interpolation) yields no
+edge.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+from graphify.extractors.base import _corpus_relative_path, _make_id, _read_text
+from graphify.extractors.twig import _resolve_path_ref as _resolve_twig_name
+
+# Symfony AbstractController and Twig\Environment render methods (template first).
+_TWIG_RENDER_METHODS = frozenset({"render", "renderView", "renderBlock", "renderBlockView"})
+
+_RESOLVERS = {
+    "twig": _resolve_twig_name,
+}
+
+
+def _string_literal(node, source: bytes) -> str | None:
+    """Text of a PHP string with no interpolation, else None."""
+    if node is None or node.type not in ("string", "encapsed_string"):
+        return None
+    parts = [c for c in node.children if c.is_named]
+    if len(parts) != 1 or parts[0].type != "string_content":
+        return None
+    return _read_text(parts[0], source)
+
+
+def _first_argument_literal(args_node, source: bytes, named: str) -> str | None:
+    """Literal value of the first positional argument, or of the argument named *named*."""
+    if args_node is None:
+        return None
+    for arg in args_node.children:
+        if arg.type != "argument":
+            continue
+        name_node = arg.child_by_field_name("name")
+        if name_node is not None and _read_text(name_node, source) != named:
+            continue
+        values = [c for c in arg.children if c.is_named and c != name_node]
+        return _string_literal(values[0], source) if len(values) == 1 else None
+    return None
+
+
+def _short_name(node, source: bytes) -> str:
+    return _read_text(node, source).rsplit("\\", 1)[-1]
+
+
+def render_call_template(node, source: bytes) -> tuple[str, str] | None:
+    """(engine, name) for a call that renders a literal template name, else None."""
+    if node.type in ("member_call_expression", "nullsafe_member_call_expression"):
+        method = node.child_by_field_name("name")
+        if method is None or _read_text(method, source) not in _TWIG_RENDER_METHODS:
+            return None
+        name = _first_argument_literal(node.child_by_field_name("arguments"), source, "view")
+        if name and name.endswith(".twig"):
+            return ("twig", name)
+    return None
+
+
+def template_attribute_templates(method_node, source: bytes) -> list[tuple[str, str, int]]:
+    """(engine, name, line) for each `#[Template('...')]` on a method."""
+    found: list[tuple[str, str, int]] = []
+    attributes = method_node.child_by_field_name("attributes")
+    if attributes is None:
+        return found
+    for group in attributes.children:
+        for attr in group.children:
+            if attr.type != "attribute":
+                continue
+            name_node = next((c for c in attr.children if c.type in ("name", "qualified_name")), None)
+            if name_node is None or _short_name(name_node, source) != "Template":
+                continue
+            name = _first_argument_literal(attr.child_by_field_name("parameters"), source, "template")
+            if name and name.endswith(".twig"):
+                found.append(("twig", name, attr.start_point[0] + 1))
+    return found
+
+
+def render_edge(ref: tuple[str, str], source_nid: str, php_path: Path, line: int) -> dict | None:
+    """`renders` edge from *source_nid* to the template file *ref* names, if it exists."""
+    engine, name = ref
+    resolved = _RESOLVERS[engine](name, php_path)
+    if resolved is None:
+        return None
+    target_file = _corpus_relative_path(resolved, php_path)
+    return {
+        "source": source_nid,
+        "target": _make_id(target_file),
+        "relation": "renders",
+        "confidence": "EXTRACTED",
+        "confidence_score": 1.0,
+        "source_file": str(php_path),
+        "source_location": f"L{line}",
+        "weight": 1.0,
+        "target_file": target_file,
+    }

graphify/extractors/resolution.py

@@ -997,6 +997,22 @@ def _node_disambiguation_source_key(node: dict, root: Path) -> str:
         return _source_key(source_file, root)
     return _source_key(str(node.get("origin_file", "")), root)
 
+# Relations whose target is a FILE node, so a `target_file` stamp on the edge
+# must key the target's salt. Template composition belongs here for the same
+# reason imports do, and the need is sharper: a Single Directory Component keeps
+# `card.twig` and `card.scss` side by side, so every component's file id
+# collides across extensions and is salted apart. An `includes` edge that fell
+# back to the importer's own source_key would then look up an id that
+# disambiguation has already retired, and the edge would dangle.
+_TEMPLATE_TARGET_RELATIONS = frozenset({
+    "includes", "embeds", "extends", "imports_macro", "uses_template", "renders",
+})
+
+_FILE_TARGET_RELATIONS = frozenset({
+    "imports", "imports_from", "re_exports",
+}) | _TEMPLATE_TARGET_RELATIONS
+
+
 def _disambiguate_colliding_node_ids(
     nodes: list[dict],
     edges: list[dict],
@@ -1096,6 +1112,24 @@ def _disambiguate_colliding_node_ids(
                     header_remaps[old_id] = new_id
                     break
 
+    # The same shape, for template composition. A Single Directory Component
+    # keeps `card.twig` beside `card.scss`, so the two collapse to one `card`
+    # file id and disambiguation salts them apart. An `{% include %}` from a
+    # THIRD template carries neither salt's source_key, so the
+    # (target, edge_source_key) lookup misses and the edge dangles on the now
+    # dead `card` id — the #1475 failure exactly. An include always targeted the
+    # template, so repoint to the template variant.
+    _TEMPLATE_SUFFIXES = (".twig",)
+    template_remaps: dict[str, str] = {}
+    for old_id in ambiguous_ids:
+        for node in by_id.get(old_id, []):
+            sk = _node_disambiguation_source_key(node, root)
+            if sk and Path(sk).suffix.lower() in _TEMPLATE_SUFFIXES:
+                new_id = remap.get((old_id, sk))
+                if new_id:
+                    template_remaps[old_id] = new_id
+                    break
+
     for edge in edges:
         edge_source_key = _source_key(str(edge.get("source_file", "")), root)
         source_key = (edge.get("source", ""), edge_source_key)
@@ -1108,7 +1142,7 @@ def _disambiguate_colliding_node_ids(
         # every language and to re_exports. `pop` it as we consume it: this is the
         # hint's only reader, and its absolute path must not persist into graph.json.
         target_file = edge.pop("target_file", None)
-        if target_file and edge.get("relation") in ("imports", "imports_from", "re_exports"):
+        if target_file and edge.get("relation") in _FILE_TARGET_RELATIONS:
             target_edge_key = _source_key(str(target_file), root)
         else:
             target_edge_key = edge_source_key
@@ -1125,6 +1159,9 @@ def _disambiguate_colliding_node_ids(
         if (edge.get("relation") in ("imports", "imports_from")
                 and edge.get("target") in header_remaps):
             edge["target"] = header_remaps[str(edge["target"])]
+        elif (edge.get("relation") in _TEMPLATE_TARGET_RELATIONS
+                and edge.get("target") in template_remaps):
+            edge["target"] = template_remaps[str(edge["target"])]
         elif target_key in remap:
             edge["target"] = remap[target_key]
         elif edge.get("target") in unambiguous_remaps: