feat: Add TurboVec

README.md

@@ -135,8 +135,9 @@ The following backends are supported:
   - `ivfpq`: Inverted file search with product quantizer.
   - `ivfpqr`: Inverted file search with product quantizer and refinement.
 - [VOYAGER](https://github.com/spotify/voyager): Voyager is a library for performing fast approximate nearest-neighbor searches on an in-memory collection of vectors.
+- [TURBOVEC](https://github.com/RyanCodrai/turbovec): Quantized flat index using Google's TurboQuant, with 2-4 bits per dimension.
 
-NOTE: the ANN backends do not support dynamic deletion. To delete items, you need to recreate the index. Insertion is supported in the following backends: `FAISS`, `HNSW`, and `Usearch`. The `BASIC` backend supports both insertion and deletion.
+NOTE: most ANN backends do not support dynamic deletion. To delete items, you need to recreate the index. Insertion is supported in the following backends: `FAISS`, `HNSW`, `Usearch`, and `TurboVec`. The `BASIC` and `TurboVec` backends support both insertion and deletion.
 
 ### Backend Parameters
 
@@ -165,6 +166,9 @@ NOTE: the ANN backends do not support dynamic deletion. To delete items, you nee
 | **VOYAGER**        | `metric`            | Similarity space to use (`cosine`, `l2`).                                                     | `"cosine"`          |
 |                 | `ef_construction`   | The number of vectors that this index searches through when inserting a new vector into the index.                                           | `200`               |
 |                 | `m`                 | The number of connections between nodes in the tree’s internal data structure.                                                              | `16`                |
+| **TURBOVEC**    | `metric`            | Similarity metric to use (`cosine`).                                                          | `"cosine"`          |
+|                 | `bit_width`         | Bits per dimension (`2`, `3`, or `4`).                                                        | `4`                 |
+|                 | `calibrate`         | Fit TQ+ calibration on a random sample of the vectors before indexing, which improves recall. | `True`              |
 
 ## Installation
 
@@ -193,6 +197,7 @@ pip install vicinity[hnsw]
 pip install vicinity[pynndescent]
 pip install vicinity[usearch]
 pip install vicinity[voyager]
+pip install vicinity[turbovec]
 ```
 
 ## License

pyproject.toml

@@ -60,6 +60,7 @@ annoy = ["annoy"]
 faiss = ["faiss-cpu"]
 usearch = ["usearch"]
 voyager = ["voyager"]
+turbovec = ["turbovec"]
 backends = [
     "hnswlib",
     "pynndescent>=0.5.10",
@@ -69,7 +70,8 @@ backends = [
     "annoy",
     "faiss-cpu",
     "usearch",
-    "voyager"
+    "voyager",
+    "turbovec"
 ]
 
 all = [
@@ -82,7 +84,8 @@ all = [
     "annoy",
     "faiss-cpu",
     "usearch",
-    "voyager"
+    "voyager",
+    "turbovec"
 ]
 
 [project.urls]

tests/conftest.py

@@ -22,13 +22,13 @@
 
 
 @pytest.fixture(scope="session")
-def items() -> list[str]:
+def items() -> list[str | dict[str, object]]:
     """Fixture providing a list of item names."""
     return [f"item{i}" if i % 2 == 0 else {"name": f"item{i}", "id": i} for i in range(1, 10001)]
 
 
 @pytest.fixture(scope="session")
-def non_serializable_items() -> list[str]:
+def non_serializable_items() -> list[object]:
     """Fixture providing a list of non-serializable items."""
 
     class NonSerializable:
@@ -58,6 +58,7 @@ def query_vector() -> np.ndarray:
     (Backend.PYNNDESCENT, None),
     (Backend.USEARCH, None),
     (Backend.VOYAGER, None),
+    (Backend.TURBOVEC, None),
 ]
 
 

tests/test_vicinity.py

@@ -80,7 +80,10 @@ def test_vicinity_query_threshold(vicinity_instance: Vicinity, query_vector: np.
 
     results = vicinity_instance.query_threshold(np.stack([query_vector, query_vector]), threshold=0.7)
 
-    assert results[0] == results[1]
+    # PyNNDescent seeds each query in a batch from a shared random state, so identical queries can find different
+    # approximate neighbours.
+    if vicinity_instance.backend.backend_type != Backend.PYNNDESCENT:
+        assert results[0] == results[1]
 
 
 def test_vicinity_insert(vicinity_instance: Vicinity, query_vector: np.ndarray) -> None:
@@ -112,8 +115,8 @@ def test_vicinity_delete(vicinity_instance: Vicinity, items: list[str], vectors:
     :param items: List of item names.
     :param vectors: Array of vectors corresponding to items.
     """
-    if vicinity_instance.backend.backend_type != Backend.BASIC:
-        # Skip delete for non-basic backends
+    if vicinity_instance.backend.backend_type not in {Backend.BASIC, Backend.TURBOVEC}:
+        # Only the basic and TurboVec backends support deletion.
         return
 
     # Get the vector corresponding to "item2"
@@ -223,8 +226,8 @@ def test_vicinity_delete_nonexistent(vicinity_instance: Vicinity) -> None:
     :param vicinity_instance: A Vicinity instance.
     :raises ValueError: If deleting items that do not exist.
     """
-    if vicinity_instance.backend.backend_type != Backend.BASIC:
-        # Skip delete for non-basic backends
+    if vicinity_instance.backend.backend_type not in {Backend.BASIC, Backend.TURBOVEC}:
+        # Only the basic and TurboVec backends support deletion.
         return
     with pytest.raises(ValueError):
         vicinity_instance.delete(["item10002"])
@@ -295,8 +298,8 @@ def test_vicinity_delete_and_query(vicinity_instance: Vicinity, items: list[str]
     :param items: List of item names.
     :param vectors: Array of vectors corresponding to items.
     """
-    if vicinity_instance.backend.backend_type != Backend.BASIC:
-        # Skip delete for non-basic backends
+    if vicinity_instance.backend.backend_type not in {Backend.BASIC, Backend.TURBOVEC}:
+        # Only the basic and TurboVec backends support deletion.
         return
 
     # Delete some items from the Vicinity instance
@@ -385,12 +388,16 @@ def test_vicinity_usearch_binary_metrics(tmp_path: Path, metric: str) -> None:
         # Scalar quantization makes distances approximate.
         (Backend.FAISS, {"index_type": "scalar"}, 0.05),
         (Backend.FAISS, {"index_type": "ivf_scalar", "nlist": 50}, 0.05),
+        # 4-bit quantization of 8-dimensional vectors is coarse, with errors up to 0.055 depending on the platform.
+        (Backend.TURBOVEC, {}, 0.1),
     ],
 )
 def test_backend_distances_match_metric(
     backend_type: Backend, kwargs: dict, atol: float, metric: str, vectors: np.ndarray, query_vector: np.ndarray
 ) -> None:
     """Backends return true cosine or Euclidean distances without padding; exact backends return every close item."""
+    if backend_type == Backend.TURBOVEC and metric == "euclidean":
+        pytest.skip("TurboVec only supports cosine.")
     # Centred and scaled, so distances go beyond 0.5 (cosine) and 1 (Euclidean), where FAISS range radii differ.
     vectors, query = 2 * (vectors - 0.5), 2 * (query_vector - 0.5)
     vicinity = Vicinity.from_vectors_and_items(
@@ -428,6 +435,7 @@ def distance(a: np.ndarray, b: np.ndarray) -> np.ndarray:
         (Backend.FAISS, {"index_type": "pq", "m": 1, "nbits": 3}),
         (Backend.FAISS, {"index_type": "ivfpq", "nlist": 1, "m": 1, "nbits": 3}),
         (Backend.FAISS, {"index_type": "ivfpqr", "nlist": 1, "m": 1, "nbits": 3, "refine_nbits": 3}),
+        (Backend.TURBOVEC, {}),
     ],
 )
 def test_cosine_distance_to_zero_vector(tmp_path: Path, backend_type: Backend, kwargs: dict) -> None:
@@ -454,3 +462,18 @@ def test_faiss_lsh_returns_hamming_distances(vectors: np.ndarray, query_vector:
     assert isinstance(vicinity.backend, FaissBackend)
     hamming, _ = vicinity.backend.index.search(normalize(query_vector)[None], 10)
     assert [distance for _, distance in vicinity.query(query_vector, k=10)[0]] == hamming[0].tolist()
+
+
+def test_turbovec_delete_keeps_items_aligned(tmp_path: Path) -> None:
+    """Deleting from TurboVec moves vectors between slots, but every item still maps to its own vector."""
+    vectors = np.eye(16, dtype=np.float32)[:10]
+    vicinity = Vicinity.from_vectors_and_items(vectors, list(range(10)), backend_type=Backend.TURBOVEC)
+    vicinity.delete([1, 4, 9])
+    vicinity.insert([10], np.eye(16, dtype=np.float32)[10:11])
+    vicinity.save(tmp_path / "turbovec")
+    vicinity = Vicinity.load(tmp_path / "turbovec")
+
+    remaining = [0, 2, 3, 5, 6, 7, 8, 10]
+    assert vicinity.items == remaining
+    results = vicinity.query(np.eye(16, dtype=np.float32)[remaining], k=1)
+    assert [result[0][0] for result in results] == remaining

vicinity/backends/__init__.py

@@ -62,5 +62,11 @@ def get_backend_class(backend: Backend | str) -> type[AbstractBackend]:
 
         return VoyagerBackend
 
+    elif backend == Backend.TURBOVEC:
+        _require("turbovec", backend, "turbovec")
+        from vicinity.backends.turbovec import TurboVecBackend
+
+        return TurboVecBackend
+
 
 __all__ = ["get_backend_class", "AbstractBackend", "BasicVectorStore"]