updater: ownership closure for untracked changed components, keep update history

codewiki/cli/adapters/doc_generator.py

@@ -6,6 +6,7 @@
 """
 
 import asyncio
+import json
 import logging
 import os
 import sys
@@ -442,8 +443,11 @@ async def _run_incremental_update(
             )
         working_dir = str(self.output_dir.absolute())
         if record.outcome in ("incremental", "no_change"):
+            # create_documentation_metadata rewrites metadata.json from scratch;
+            # keep the history of earlier updates so a chain of updates accumulates.
+            prior_history = self._read_update_history(working_dir)
             doc_generator.create_documentation_metadata(working_dir, components, len(leaf_nodes))
-            self._merge_update_summary(working_dir)
+            self._merge_update_summary(working_dir, prior_history)
             for file_path in os.listdir(working_dir):
                 if file_path.endswith((".md", ".json")):
                     self.job.files_generated.append(file_path)
@@ -480,7 +484,17 @@ def _move_docs_aside(self) -> None:
         if self.verbose:
             self.progress_tracker.update_stage(0.1, f"Previous docs preserved at {target}")
 
-    def _merge_update_summary(self, working_dir: str) -> None:
+    @staticmethod
+    def _read_update_history(working_dir: str) -> list:
+        path = os.path.join(working_dir, "metadata.json")
+        try:
+            with open(path, encoding="utf-8") as f:
+                history = (json.load(f) or {}).get("update_history")
+        except (OSError, json.JSONDecodeError):
+            return []
+        return history if isinstance(history, list) else []
+
+    def _merge_update_summary(self, working_dir: str, prior_history: list | None = None) -> None:
         record = getattr(self, "_last_update_record", None)
         if record is None:
             return
@@ -490,7 +504,7 @@ def _merge_update_summary(self, working_dir: str) -> None:
         preserved = getattr(self, "_preserved_docs_dir", None)
         if preserved:
             summary["previous_docs"] = preserved
-        merge_into_metadata(working_dir, summary)
+        merge_into_metadata(working_dir, summary, prior_history=prior_history)
         try:
             record.save(working_dir)
         except OSError:

codewiki/cli/commands/generate.py

@@ -386,6 +386,15 @@ def _find_affected(tree, parent_names=None):
     default=None,
     help="Cap on one component diff in a report (default 8000).",
 )
+@click.option(
+    "--no-ownership-closure",
+    is_flag=True,
+    default=False,
+    help=(
+        "Do not give changed components that no module tracks an effective owner by the "
+        "placement rules; such changes then reach no page (default: give one)."
+    ),
+)
 @click.option(
     "--compare-to",
     type=str,
@@ -427,6 +436,7 @@ def generate_command(
     tau_tree: float | None = None,
     k_hop: int | None = None,
     max_diff_tokens: int | None = None,
+    no_ownership_closure: bool = False,
 ):
     """
     Generate comprehensive documentation for a code repository.
@@ -761,6 +771,7 @@ def generate_command(
                     "tau_tree": tau_tree,
                     "k_hop": k_hop,
                     "max_diff_tokens": max_diff_tokens,
+                    "use_ownership_closure": False if no_ownership_closure else None,
                 },
             },
             verbose=verbose,

codewiki/src/be/updater/change_report.py

@@ -26,6 +26,9 @@ class LeafReport:
     leaf_path: tuple[str, ...]
     mode: str = MODE_EDIT
     own: list[str] = field(default_factory=list)
+    # Own entries this leaf does not list: changed components no leaf tracks,
+    # given to this leaf as effective owner (Step 3a). id -> placement rule.
+    adopted: dict[str, str] = field(default_factory=dict)
     up: list[str] = field(default_factory=list)
     context: list[str] = field(default_factory=list)  # untracked changes next to this leaf
     refch: list[str] = field(default_factory=list)
@@ -88,11 +91,24 @@ def build_reports(
     repair: RepairResult,
     opts: UpdateOptions,
     reclustered: set[tuple[str, ...]] | None = None,
+    adopted: dict[str, Any] | None = None,
 ) -> dict[tuple[str, ...], LeafReport]:
-    """Build ``Report(l)`` for every unit of the new tree plus deleted leaves."""
+    """Build ``Report(l)`` for every unit of the new tree plus deleted leaves.
+
+    ``adopted`` (Step 3a, ``ownership.close_ownership``) maps a changed
+    component that no leaf tracks to its effective owner; it enters that
+    leaf's Own exactly like a tracked component."""
     reclustered = reclustered or set()
+    adopted = adopted or {}
     old_owner = T.owner_map(old_tree)
     new_owner = T.owner_map(new_tree)
+
+    def owner_of(cid: str) -> tuple[str, ...] | None:
+        o = T.resolve_owner(new_owner, cid) or T.resolve_owner(old_owner, cid)
+        if o is None and cid in adopted:
+            o = tuple(adopted[cid].leaf_path)
+        return o
+
     old_units = set(T.unit_paths(old_tree))
     new_units = T.unit_paths(new_tree)
     reports: dict[tuple[str, ...], LeafReport] = {p: LeafReport(leaf_path=p) for p in new_units}
@@ -109,9 +125,13 @@ def build_reports(
             o = T.resolve_owner(old_owner, rec.old_id)
             if o is not None:
                 owners.add(o)
+        if not owners and cid in adopted:
+            owners.add(tuple(adopted[cid].leaf_path))
         for p in owners:
             if p in reports and cid not in reports[p].own:
                 reports[p].own.append(cid)
+                if cid in adopted:
+                    reports[p].adopted[cid] = adopted[cid].rule
 
     # Up: interface / deleted / renamed components used by this leaf's code.
     contract_moved = set(diff.interface) | set(diff.deleted) | set(diff.renamed.values())
@@ -135,20 +155,18 @@ def build_reports(
                 p = T.resolve_owner(new_owner, user) or T.resolve_owner(old_owner, user)
                 if p is None or p not in reports:
                     continue
-                own_leaf = (
-                    T.resolve_owner(new_owner, cid)
-                    or T.resolve_owner(old_owner, cid)
-                    or T.resolve_owner(old_owner, old_ids_of_renames.get(cid, ""))
+                own_leaf = owner_of(cid) or T.resolve_owner(
+                    old_owner, old_ids_of_renames.get(cid, "")
                 )
                 if p == own_leaf:
                     continue
                 if cid not in reports[p].up:
                     reports[p].up.append(cid)
 
-    # Untracked changed components (no class to attach to): context for the
-    # leaves owning their neighbours.
+    # Untracked changed components that the closure could not place either:
+    # context for the leaves owning their neighbours.
     for cid in diff.changed_ids:
-        if T.resolve_owner(new_owner, cid) or T.resolve_owner(old_owner, cid):
+        if owner_of(cid) is not None:
             continue
         neighbours = _out_edges(cid, old_graph, new_graph)
         for other, node in new_graph.items():

codewiki/src/be/updater/options.py

@@ -29,6 +29,11 @@ class UpdateOptions:
 
     # Step 3: change report
     k_hop: int = 1  # dependency hops followed for Up
+    # Step 3a: ownership closure. A changed component that no leaf tracks gets
+    # an effective owner by the placement rules of Step 2 (same file, same
+    # directory, neighbour majority, routing agent) and enters that leaf's Own.
+    # Activation only: the tree is not changed.
+    use_ownership_closure: bool = True
 
     # Step 4: fallback
     tau_full: float = 0.5  # active leaves / all leaves
@@ -57,6 +62,7 @@ def from_rung(cls, rung: str | int, **overrides: Any) -> "UpdateOptions":
             opts.agent_may_patch_leaf = False
             opts.agent_patches_related = False
             opts.use_stale_scan = False
+            opts.use_ownership_closure = False
         elif rung == "2":
             opts.agent_may_patch_leaf = False
         elif rung == "3b":

codewiki/src/be/updater/orchestrator.py

@@ -25,6 +25,7 @@
 from codewiki.src.be.updater.graph_store import load_graph
 from codewiki.src.be.updater.leaf_agent import LeafAgentRunner
 from codewiki.src.be.updater.options import UpdateOptions
+from codewiki.src.be.updater.ownership import close_ownership
 from codewiki.src.be.updater.record import (
     OUTCOME_DETECTOR_FAILURE,
     OUTCOME_FULL_FALLBACK,
@@ -288,6 +289,14 @@ async def _run(
         rec.repair = repair.to_dict()
         self._deleted_nodes = list(repair.deleted_nodes)
 
+        # ---- Step 3a: effective owners for changed components no leaf tracks
+        adopted = {}
+        if not self.whole_repo:
+            adopted = close_ownership(
+                diff, old_tree, new_tree, old_graph, new_graph, self.opts, router, repair
+            )
+            rec.ownership = [a.to_dict() for a in adopted.values()]
+
         # ---- Step 3: reports
         reports = build_reports(
             diff,
@@ -299,6 +308,7 @@ async def _run(
             repair,
             self.opts,
             reclustered,
+            adopted,
         )
         active = active_set(reports)
         rec.reports = {"/".join(p): r.to_dict() for p, r in reports.items() if p in active}

codewiki/src/be/updater/ownership.py

@@ -0,0 +1,149 @@
+"""Step 3a: ownership closure.
+
+The module tree tracks a subset of the code graph (on svelte 764 of 2239
+components). A changed component outside that subset has no owner, so it
+can never enter a leaf's Own and, when its neighbours' leaves are otherwise
+quiet, is dropped. Here the owner map is extended to a total function: an
+untracked changed component gets an *effective owner* by the very rules that
+place a new component in Step 2 (same file, same directory with one leaf,
+neighbour majority, routing agent). The component then enters Own of that
+leaf. Activation only: the tree on disk is not changed, and the closure is
+recomputed at every step.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import Any
+
+from codewiki.src.be.dependency_analyzer.models.core import Node
+from codewiki.src.be.updater import tree as T
+from codewiki.src.be.updater.graph_diff import GraphDiff
+from codewiki.src.be.updater.options import UpdateOptions
+from codewiki.src.be.updater.tree_repair import (
+    RULE_AGENT,
+    OrphanRouter,
+    RepairResult,
+    RoutingDecision,
+    route_by_rules,
+)
+
+logger = logging.getLogger(__name__)
+
+PURPOSE_OWNERSHIP = "ownership"
+
+
+@dataclass
+class Adoption:
+    """Effective owner of one changed component that no leaf tracks."""
+
+    decision: RoutingDecision
+    deleted: bool = False  # the component exists only in the old graph
+
+    @property
+    def component_id(self) -> str:
+        return self.decision.component_id
+
+    @property
+    def leaf_path(self) -> tuple[str, ...]:
+        assert self.decision.leaf_path is not None
+        return self.decision.leaf_path
+
+    @property
+    def rule(self) -> str:
+        return self.decision.rule
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "component_id": self.component_id,
+            "rule": self.rule,
+            "leaf_path": list(self.leaf_path),
+            "detail": self.decision.detail,
+            "deleted": self.deleted,
+        }
+
+
+def untracked_changed(
+    diff: GraphDiff,
+    old_owner: dict[str, tuple[str, ...]],
+    new_owner: dict[str, tuple[str, ...]],
+    repair: RepairResult | None,
+) -> list[str]:
+    """Changed or deleted ids with no (class-resolved) owner in either tree,
+    minus added components the repair step deliberately left untracked."""
+    left_out = set()
+    if repair is not None:
+        left_out = {d.component_id for d in repair.routing if d.leaf_path is None}
+    out = []
+    for cid in sorted(diff.changed_ids | set(diff.deleted)):
+        if cid in left_out:
+            continue
+        if T.resolve_owner(new_owner, cid) or T.resolve_owner(old_owner, cid):
+            continue
+        out.append(cid)
+    return out
+
+
+def close_ownership(
+    diff: GraphDiff,
+    old_tree: dict[str, Any],
+    new_tree: dict[str, Any],
+    old_graph: dict[str, Node],
+    new_graph: dict[str, Node],
+    opts: UpdateOptions,
+    route_orphans: OrphanRouter | None = None,
+    repair: RepairResult | None = None,
+) -> dict[str, Adoption]:
+    """Return ``{component_id: Adoption}`` for every changed untracked component
+    that rules 1-3, or the routing agent as rule 4, can give an owner."""
+    if not opts.use_ownership_closure:
+        return {}
+    old_owner = T.owner_map(old_tree)
+    new_owner = T.owner_map(new_tree)
+    candidates = untracked_changed(diff, old_owner, new_owner, repair)
+    if not candidates:
+        return {}
+
+    adopted: dict[str, Adoption] = {}
+    unplaced: list[str] = []
+    for cid in candidates:
+        deleted = cid not in new_graph and cid in old_graph
+        graph, owner = (old_graph, old_owner) if deleted else (new_graph, new_owner)
+        node = graph.get(cid)
+        if node is None:
+            continue
+        decision = route_by_rules(cid, node, owner, graph, opts)
+        if decision is None:
+            if deleted:
+                continue  # no page can answer for a vanished untracked component
+            unplaced.append(cid)
+            continue
+        adopted[cid] = Adoption(decision, deleted)
+
+    if unplaced and route_orphans is not None and opts.use_routing_agent:
+        context = {
+            "tree": new_tree,
+            "owner": new_owner,
+            "graph": new_graph,
+            "purpose": PURPOSE_OWNERSHIP,
+        }
+        for d in route_orphans(unplaced, context):
+            if d.component_id not in unplaced or d.component_id in adopted:
+                continue
+            if d.leaf_path is None or d.new_leaf or T.node_at(new_tree, d.leaf_path) is None:
+                continue  # left untracked, or a new leaf: the tree is not changed here
+            d.rule = RULE_AGENT
+            adopted[d.component_id] = Adoption(d)
+
+    by_rule: dict[str, int] = {}
+    for a in adopted.values():
+        by_rule[a.rule] = by_rule.get(a.rule, 0) + 1
+    logger.info(
+        "Ownership closure: %d untracked changed, %d adopted %s, %d left as context",
+        len(candidates),
+        len(adopted),
+        by_rule,
+        len(candidates) - len(adopted),
+    )
+    return adopted