feat: add SQLite backend and migration tests

Author: kdush

tests/test_cli.py

@@ -2,6 +2,7 @@
 from unittest.mock import patch
 
 import pytest
+import yaml
 from click.testing import CliRunner
 
 from openkb.cli import cli
@@ -30,11 +31,11 @@ def test_init_creates_structure(tmp_path):
         assert (cwd / "wiki" / "log.md").is_file()
         assert (cwd / "wiki" / "index.md").is_file()
         assert (cwd / ".openkb" / "config.yaml").is_file()
-        assert (cwd / ".openkb" / "hashes.json").is_file()
+        # SQLite DB 在首次访问时由 get_registry() 惰性创建
+        assert not (cwd / ".openkb" / "hashes.json").exists()
 
-        # hashes.json is empty object
-        hashes = json.loads((cwd / ".openkb" / "hashes.json").read_text())
-        assert hashes == {}
+        config = yaml.safe_load((cwd / ".openkb" / "config.yaml").read_text())
+        assert config["storage_backend"] == "sqlite"
 
         # index.md header
         index_content = (cwd / "wiki" / "index.md").read_text()

tests/test_config_storage_backend.py

@@ -0,0 +1,37 @@
+"""Tests for storage_backend config option."""
+from __future__ import annotations
+
+from pathlib import Path
+
+from openkb.config import DEFAULT_CONFIG, load_config, save_config
+
+
+def test_default_config_has_storage_backend():
+    """DEFAULT_CONFIG should include storage_backend key."""
+    assert "storage_backend" in DEFAULT_CONFIG
+
+
+def test_default_storage_backend_is_sqlite():
+    """Default storage_backend should be 'sqlite'."""
+    assert DEFAULT_CONFIG["storage_backend"] == "sqlite"
+
+
+def test_load_config_includes_storage_backend(tmp_path):
+    """load_config should return storage_backend from config file."""
+    config_path = tmp_path / "config.yaml"
+    save_config(config_path, {"storage_backend": "json"})
+    loaded = load_config(config_path)
+    assert loaded["storage_backend"] == "json"
+
+
+def test_storage_backend_valid_values(tmp_path):
+    """storage_backend should accept 'sqlite' or 'json'."""
+    config_path = tmp_path / "config.yaml"
+    
+    save_config(config_path, {"storage_backend": "sqlite"})
+    loaded = load_config(config_path)
+    assert loaded["storage_backend"] == "sqlite"
+    
+    save_config(config_path, {"storage_backend": "json"})
+    loaded = load_config(config_path)
+    assert loaded["storage_backend"] == "json"

tests/test_converter.py

@@ -48,14 +48,15 @@ def test_md_file_copied_to_wiki_sources(self, kb_dir):
 
     def test_md_duplicate_skipped(self, kb_dir):
         """Second call with same file returns skipped=True when hash is registered."""
-        from openkb.state import HashRegistry
+        from openkb.state import get_registry
 
         src = kb_dir / "raw" / "notes.md"
         src.write_text("# Notes\n\nSome content here.", encoding="utf-8")
 
         result1 = convert_document(src, kb_dir)  # first call
         # Simulate CLI registering the hash after successful compilation
-        registry = HashRegistry(kb_dir / ".openkb" / "hashes.json")
+        openkb_dir = kb_dir / ".openkb"
+        registry = get_registry(openkb_dir, backend="sqlite")
         registry.add(result1.file_hash, {"name": src.name, "type": "md"})
 
         result2 = convert_document(src, kb_dir)  # second call

tests/test_db_registry.py

@@ -0,0 +1,172 @@
+"""Tests for DbRegistry SQLite-backed storage."""
+from __future__ import annotations
+
+import json
+import sqlite3
+from pathlib import Path
+
+import pytest
+
+from openkb.state import DbRegistry
+
+
+def test_db_registry_creates_database_file(tmp_path):
+    """DbRegistry should create a .db file on init."""
+    db_path = tmp_path / "hashes.db"
+    registry = DbRegistry(db_path)
+    assert db_path.exists()
+
+
+def test_db_registry_creates_table(tmp_path):
+    """DbRegistry should create the registry table."""
+    db_path = tmp_path / "hashes.db"
+    registry = DbRegistry(db_path)
+    
+    conn = sqlite3.connect(str(db_path))
+    cursor = conn.execute(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name='registry'"
+    )
+    result = cursor.fetchone()
+    conn.close()
+    assert result is not None
+
+
+def test_db_empty_registry_is_known_false(tmp_path):
+    """Empty DbRegistry should return False for is_known."""
+    registry = DbRegistry(tmp_path / "hashes.db")
+    assert registry.is_known("abc123") is False
+
+
+def test_db_empty_registry_get_returns_none(tmp_path):
+    """Empty DbRegistry should return None for get."""
+    registry = DbRegistry(tmp_path / "hashes.db")
+    assert registry.get("abc123") is None
+
+
+def test_db_add_and_is_known(tmp_path):
+    """After add, is_known should return True."""
+    registry = DbRegistry(tmp_path / "hashes.db")
+    registry.add("deadbeef", {"filename": "test.pdf"})
+    assert registry.is_known("deadbeef") is True
+
+
+def test_db_add_and_get(tmp_path):
+    """After add, get should return the metadata."""
+    registry = DbRegistry(tmp_path / "hashes.db")
+    metadata = {"filename": "doc.pdf", "pages": 10}
+    registry.add("cafebabe", metadata)
+    assert registry.get("cafebabe") == metadata
+
+
+def test_db_persistence_across_instances(tmp_path):
+    """Data should persist across DbRegistry instances."""
+    db_path = tmp_path / "hashes.db"
+    r1 = DbRegistry(db_path)
+    r1.add("hash1", {"file": "a.pdf"})
+    
+    r2 = DbRegistry(db_path)
+    assert r2.is_known("hash1") is True
+    assert r2.get("hash1") == {"file": "a.pdf"}
+
+
+def test_db_all_entries_returns_all(tmp_path):
+    """all_entries should return all hash -> metadata mappings."""
+    registry = DbRegistry(tmp_path / "hashes.db")
+    registry.add("h1", {"name": "one"})
+    registry.add("h2", {"name": "two"})
+    entries = registry.all_entries()
+    assert "h1" in entries
+    assert "h2" in entries
+    assert entries["h1"] == {"name": "one"}
+    assert entries["h2"] == {"name": "two"}
+
+
+def test_db_all_entries_empty(tmp_path):
+    """all_entries on empty registry should return empty dict."""
+    registry = DbRegistry(tmp_path / "hashes.db")
+    assert registry.all_entries() == {}
+
+
+def test_db_hash_file_unchanged(tmp_path):
+    """DbRegistry.hash_file should work same as HashRegistry."""
+    f = tmp_path / "sample.txt"
+    f.write_text("hello world")
+    digest = DbRegistry.hash_file(f)
+    assert len(digest) == 64
+    assert all(c in "0123456789abcdef" for c in digest)
+
+
+def test_db_update_existing_hash(tmp_path):
+    """Adding same hash twice should update metadata."""
+    registry = DbRegistry(tmp_path / "hashes.db")
+    registry.add("hash1", {"version": 1})
+    registry.add("hash1", {"version": 2})
+    assert registry.get("hash1") == {"version": 2}
+
+
+def test_db_metadata_with_nested_dict(tmp_path):
+    """Metadata can contain nested dictionaries."""
+    registry = DbRegistry(tmp_path / "hashes.db")
+    metadata = {
+        "name": "doc.pdf",
+        "stats": {"pages": 10, "words": 5000},
+    }
+    registry.add("hash1", metadata)
+    assert registry.get("hash1") == metadata
+
+
+def test_db_wal_mode_enabled(tmp_path):
+    """Database should use WAL mode for concurrency."""
+    db_path = tmp_path / "hashes.db"
+    DbRegistry(db_path)
+    
+    conn = sqlite3.connect(str(db_path))
+    cursor = conn.execute("PRAGMA journal_mode")
+    result = cursor.fetchone()
+    conn.close()
+    assert result[0].lower() == "wal"
+
+
+def test_migrate_from_json(tmp_path):
+    """DbRegistry should migrate existing JSON data on first access."""
+    json_path = tmp_path / "hashes.json"
+    existing_data = {
+        "hash1": {"name": "doc1.pdf", "pages": 10},
+        "hash2": {"name": "doc2.pdf", "pages": 20},
+    }
+    json_path.write_text(json.dumps(existing_data), encoding="utf-8")
+    
+    db_path = tmp_path / "hashes.db"
+    registry = DbRegistry(db_path, migrate_from=json_path)
+    
+    assert registry.is_known("hash1")
+    assert registry.is_known("hash2")
+    assert registry.get("hash1") == {"name": "doc1.pdf", "pages": 10}
+    assert registry.get("hash2") == {"name": "doc2.pdf", "pages": 20}
+
+
+def test_migrate_only_once(tmp_path):
+    """Migration should only happen once, not on subsequent loads."""
+    json_path = tmp_path / "hashes.json"
+    existing_data = {"hash1": {"name": "doc1.pdf"}}
+    json_path.write_text(json.dumps(existing_data), encoding="utf-8")
+    
+    db_path = tmp_path / "hashes.db"
+    
+    r1 = DbRegistry(db_path, migrate_from=json_path)
+    assert r1.is_known("hash1")
+    
+    existing_data["hash2"] = {"name": "doc2.pdf"}
+    json_path.write_text(json.dumps(existing_data), encoding="utf-8")
+    
+    r2 = DbRegistry(db_path, migrate_from=json_path)
+    assert r2.is_known("hash1")
+    assert not r2.is_known("hash2")
+
+
+def test_migrate_optional(tmp_path):
+    """DbRegistry should work without migration."""
+    db_path = tmp_path / "hashes.db"
+    registry = DbRegistry(db_path)
+    registry.add("hash1", {"name": "doc.pdf"})
+    assert registry.is_known("hash1")

tests/test_migration.py

@@ -0,0 +1,74 @@
+"""Integration tests for JSON to SQLite migration."""
+from __future__ import annotations
+
+import json
+import threading
+from pathlib import Path
+
+import pytest
+
+from openkb.state import get_registry, DbRegistry
+
+
+def test_full_migration_workflow(tmp_path):
+    """Test complete migration from JSON to SQLite."""
+    openkb_dir = tmp_path / ".openkb"
+    openkb_dir.mkdir()
+    
+    # Step 1: Start with JSON backend
+    json_registry = get_registry(openkb_dir, backend="json")
+    json_registry.add("hash1", {"name": "doc1.pdf", "pages": 10})
+    json_registry.add("hash2", {"name": "doc2.pdf", "pages": 20})
+    
+    # Verify JSON file exists
+    json_path = openkb_dir / "hashes.json"
+    assert json_path.exists()
+    
+    # Step 2: Switch to SQLite backend (triggers migration)
+    sqlite_registry = get_registry(openkb_dir, backend="sqlite")
+    
+    # Verify data was migrated
+    assert sqlite_registry.is_known("hash1")
+    assert sqlite_registry.is_known("hash2")
+    assert sqlite_registry.get("hash1") == {"name": "doc1.pdf", "pages": 10}
+    assert sqlite_registry.get("hash2") == {"name": "doc2.pdf", "pages": 20}
+    
+    # Step 3: Add new data via SQLite
+    sqlite_registry.add("hash3", {"name": "doc3.pdf", "pages": 30})
+    
+    # Step 4: Create new SQLite instance - should have all data
+    sqlite_registry2 = get_registry(openkb_dir, backend="sqlite")
+    assert sqlite_registry2.is_known("hash1")
+    assert sqlite_registry2.is_known("hash2")
+    assert sqlite_registry2.is_known("hash3")
+
+
+def test_concurrent_sqlite_access(tmp_path):
+    """Test that SQLite handles concurrent access correctly."""
+    openkb_dir = tmp_path / ".openkb"
+    openkb_dir.mkdir()
+    
+    registry = get_registry(openkb_dir, backend="sqlite")
+    errors = []
+    
+    def add_entries(start: int, count: int) -> None:
+        try:
+            for i in range(start, start + count):
+                registry.add(f"hash{i}", {"index": i})
+        except Exception as e:
+            errors.append(e)
+    
+    threads = [
+        threading.Thread(target=add_entries, args=(0, 50)),
+        threading.Thread(target=add_entries, args=(50, 50)),
+        threading.Thread(target=add_entries, args=(100, 50)),
+    ]
+    
+    for t in threads:
+        t.start()
+    for t in threads:
+        t.join()
+    
+    assert not errors
+    entries = registry.all_entries()
+    assert len(entries) == 150

tests/test_state.py

@@ -82,3 +82,50 @@ def test_load_existing_json(tmp_path):
     registry = HashRegistry(path)
     assert registry.is_known("existinghash") is True
     assert registry.get("existinghash") == {"file": "pre.pdf"}
+
+
+# ---------------------------------------------------------------------------
+# Factory function tests
+# ---------------------------------------------------------------------------
+
+from openkb.state import get_registry
+
+
+def test_get_registry_returns_db_registry_by_default(tmp_path):
+    """get_registry should return DbRegistry by default."""
+    openkb_dir = tmp_path / ".openkb"
+    openkb_dir.mkdir()
+    
+    registry = get_registry(openkb_dir)
+    assert type(registry).__name__ == "DbRegistry"
+
+
+def test_get_registry_returns_hash_registry_for_json_backend(tmp_path):
+    """get_registry should return HashRegistry when backend is 'json'."""
+    openkb_dir = tmp_path / ".openkb"
+    openkb_dir.mkdir()
+    
+    registry = get_registry(openkb_dir, backend="json")
+    assert type(registry).__name__ == "HashRegistry"
+
+
+def test_get_registry_returns_db_registry_for_sqlite_backend(tmp_path):
+    """get_registry should return DbRegistry when backend is 'sqlite'."""
+    openkb_dir = tmp_path / ".openkb"
+    openkb_dir.mkdir()
+    
+    registry = get_registry(openkb_dir, backend="sqlite")
+    assert type(registry).__name__ == "DbRegistry"
+
+
+def test_get_registry_migrates_json_to_sqlite(tmp_path):
+    """get_registry should migrate existing JSON when switching to sqlite."""
+    openkb_dir = tmp_path / ".openkb"
+    openkb_dir.mkdir()
+    
+    json_path = openkb_dir / "hashes.json"
+    json_path.write_text('{"hash1": {"name": "doc.pdf"}}', encoding="utf-8")
+    
+    registry = get_registry(openkb_dir, backend="sqlite")
+    assert registry.is_known("hash1")
+    assert registry.get("hash1") == {"name": "doc.pdf"}