fix: preserve non-ASCII characters in concept name slugs

Author: rejojer

openkb/agent/compiler.py

@@ -16,6 +16,7 @@
 import sys
 import threading
 import time
+import unicodedata
 from pathlib import Path
 
 import litellm
@@ -302,11 +303,12 @@ def _write_summary(wiki_dir: Path, doc_name: str, summary: str,
     (summaries_dir / f"{doc_name}.md").write_text(frontmatter + summary, encoding="utf-8")
 
 
-_SAFE_NAME_RE = re.compile(r'[^a-zA-Z0-9_\-]')
+_SAFE_NAME_RE = re.compile(r'[^\w\-]')
 
 
 def _sanitize_concept_name(name: str) -> str:
     """Sanitize a concept name for safe use as a filename."""
+    name = unicodedata.normalize("NFKC", name)
     sanitized = _SAFE_NAME_RE.sub("-", name).strip("-")
     return sanitized or "unnamed-concept"
 

tests/test_compiler.py

@@ -12,6 +12,7 @@
     compile_short_doc,
     _compile_concepts,
     _parse_json,
+    _sanitize_concept_name,
     _write_summary,
     _write_concept,
     _update_index,
@@ -74,6 +75,39 @@ def test_plain_text_fallback(self):
             _parse_json("Just plain markdown text without JSON")
 
 
+class TestSanitizeConceptName:
+    def test_ascii_passthrough(self):
+        assert _sanitize_concept_name("hello-world") == "hello-world"
+
+    def test_spaces_replaced(self):
+        assert _sanitize_concept_name("hello world") == "hello-world"
+
+    def test_chinese(self):
+        result = _sanitize_concept_name("注意力机制")
+        assert result == "注意力机制"
+
+    def test_japanese(self):
+        result = _sanitize_concept_name("トランスフォーマー")
+        assert result == "トランスフォーマー"
+
+    def test_french_accents(self):
+        result = _sanitize_concept_name("réseau neuronal")
+        assert "r" in result
+        assert result != "r-seau-neuronal"  # accented chars preserved, not stripped
+
+    def test_distinct_chinese_names_no_collision(self):
+        a = _sanitize_concept_name("注意力机制")
+        b = _sanitize_concept_name("变压器模型")
+        assert a != b
+
+    def test_empty_fallback(self):
+        assert _sanitize_concept_name("!!!") == "unnamed-concept"
+
+    def test_nfkc_normalization(self):
+        # U+FF21 (fullwidth A) should normalize to regular A
+        assert _sanitize_concept_name("\uff21\uff22") == "AB"
+
+
 class TestWriteSummary:
     def test_writes_with_frontmatter(self, tmp_path):
         wiki = tmp_path / "wiki"