feat: add concepts plan and update prompt templates

Add _CONCEPTS_PLAN_USER (create/update/related JSON structure) and
_CONCEPT_UPDATE_USER templates; add TestParseConceptsPlan tests.

Author: KylinMountain

openkb/agent/compiler.py

@@ -69,6 +69,33 @@
 Return ONLY valid JSON array, no fences, no explanation.
 """
 
+_CONCEPTS_PLAN_USER = """\
+Based on the summary above, decide how to update the wiki's concept pages.
+
+Existing concept pages:
+{concept_briefs}
+
+Return a JSON object with three keys:
+
+1. "create" — new concepts not covered by any existing page. Array of objects:
+   {{"name": "concept-slug", "title": "Human-Readable Title"}}
+
+2. "update" — existing concepts that have significant new information from \
+this document worth integrating. Array of objects:
+   {{"name": "existing-slug", "title": "Existing Title"}}
+
+3. "related" — existing concepts tangentially related to this document but \
+not needing content changes, just a cross-reference link. Array of slug strings.
+
+Rules:
+- For the first few documents, create 2-3 foundational concepts at most.
+- Do NOT create a concept that overlaps with an existing one — use "update".
+- Do NOT create concepts that are just the document topic itself.
+- "related" is for lightweight cross-linking only, no content rewrite needed.
+
+Return ONLY valid JSON, no fences, no explanation.
+"""
+
 _CONCEPT_PAGE_USER = """\
 Write the concept page for: {title}
 
@@ -81,6 +108,20 @@
 - [[wikilinks]] to related concepts and [[summaries/{doc_name}]]
 """
 
+_CONCEPT_UPDATE_USER = """\
+Update the concept page for: {title}
+
+Current content of this page:
+{existing_content}
+
+New information from document "{doc_name}" (summarized above) should be \
+integrated into this page. Rewrite the full page incorporating the new \
+information naturally — do not just append. Maintain existing \
+[[wikilinks]] and add new ones where appropriate.
+
+Return ONLY the Markdown content (no frontmatter, no code fences).
+"""
+
 _LONG_DOC_SUMMARY_USER = """\
 This is a PageIndex summary for long document "{doc_name}" (doc_id: {doc_id}):
 
@@ -296,6 +337,36 @@ def _write_concept(wiki_dir: Path, name: str, content: str, source_file: str, is
         path.write_text(frontmatter + content, encoding="utf-8")
 
 
+def _add_related_link(wiki_dir: Path, concept_slug: str, doc_name: str, source_file: str) -> None:
+    """Add a cross-reference link to an existing concept page (no LLM call)."""
+    concepts_dir = wiki_dir / "concepts"
+    path = concepts_dir / f"{concept_slug}.md"
+    if not path.exists():
+        return
+
+    text = path.read_text(encoding="utf-8")
+    link = f"[[summaries/{doc_name}]]"
+    if link in text:
+        return
+
+    # Update sources in frontmatter
+    if source_file not in text:
+        if text.startswith("---"):
+            end = text.index("---", 3)
+            fm = text[:end + 3]
+            body = text[end + 3:]
+            if "sources:" in fm:
+                fm = fm.replace("sources: [", f"sources: [{source_file}, ")
+            else:
+                fm = fm.replace("---\n", f"---\nsources: [{source_file}]\n", 1)
+            text = fm + body
+        else:
+            text = f"---\nsources: [{source_file}]\n---\n\n" + text
+
+    text += f"\n\nSee also: {link}"
+    path.write_text(text, encoding="utf-8")
+
+
 def _update_index(wiki_dir: Path, doc_name: str, concept_names: list[str]) -> None:
     """Append document and concept entries to index.md."""
     index_path = wiki_dir / "index.md"

tests/test_compiler.py

@@ -16,6 +16,7 @@
     _update_index,
     _read_wiki_context,
     _read_concept_briefs,
+    _add_related_link,
 )
 
 
@@ -32,6 +33,31 @@ def test_invalid_json(self):
             _parse_json("not json")
 
 
+class TestParseConceptsPlan:
+    def test_dict_format(self):
+        text = json.dumps({
+            "create": [{"name": "foo", "title": "Foo"}],
+            "update": [{"name": "bar", "title": "Bar"}],
+            "related": ["baz"],
+        })
+        parsed = _parse_json(text)
+        assert isinstance(parsed, dict)
+        assert len(parsed["create"]) == 1
+        assert len(parsed["update"]) == 1
+        assert parsed["related"] == ["baz"]
+
+    def test_fallback_list_format(self):
+        text = json.dumps([{"name": "foo", "title": "Foo"}])
+        parsed = _parse_json(text)
+        assert isinstance(parsed, list)
+
+    def test_fenced_dict(self):
+        text = '```json\n{"create": [], "update": [], "related": []}\n```'
+        parsed = _parse_json(text)
+        assert isinstance(parsed, dict)
+        assert parsed["create"] == []
+
+
 class TestWriteSummary:
     def test_writes_with_frontmatter(self, tmp_path):
         wiki = tmp_path / "wiki"
@@ -180,6 +206,39 @@ def test_sorted_alphabetically(self, tmp_path):
         assert slugs == ["apple", "mango", "zebra"]
 
 
+class TestAddRelatedLink:
+    def test_adds_see_also_link(self, tmp_path):
+        wiki = tmp_path / "wiki"
+        concepts = wiki / "concepts"
+        concepts.mkdir(parents=True)
+        (concepts / "attention.md").write_text(
+            "---\nsources: [paper1.pdf]\n---\n\n# Attention\n\nSome content.",
+            encoding="utf-8",
+        )
+        _add_related_link(wiki, "attention", "new-doc", "paper2.pdf")
+        text = (concepts / "attention.md").read_text()
+        assert "[[summaries/new-doc]]" in text
+        assert "paper2.pdf" in text
+
+    def test_skips_if_already_linked(self, tmp_path):
+        wiki = tmp_path / "wiki"
+        concepts = wiki / "concepts"
+        concepts.mkdir(parents=True)
+        (concepts / "attention.md").write_text(
+            "---\nsources: [paper1.pdf]\n---\n\n# Attention\n\nSee also: [[summaries/new-doc]]",
+            encoding="utf-8",
+        )
+        _add_related_link(wiki, "attention", "new-doc", "paper1.pdf")
+        text = (concepts / "attention.md").read_text()
+        assert text.count("[[summaries/new-doc]]") == 1
+
+    def test_skips_if_file_missing(self, tmp_path):
+        wiki = tmp_path / "wiki"
+        wiki.mkdir()
+        # Should not raise
+        _add_related_link(wiki, "nonexistent", "doc", "file.pdf")
+
+
 def _mock_completion(responses: list[str]):
     """Create a mock for litellm.completion that returns responses in order."""
     call_count = {"n": 0}