add lib embedd models

models.py

@@ -32,4 +32,14 @@ REGISTRY: dict[str, ModelConfig] = {
         model_id="sentence-transformers/all-MiniLM-L6-v2",
         backend="fastembed",
     ),
+    "bge-small-le": ModelConfig(
+        name="bge-small-en-v1.5 (libembedding)",
+        model_id="BAAI/bge-small-en-v1.5",
+        backend="libembedding",
+    ),
+    "all-minilm-le": ModelConfig(
+        name="all-MiniLM-L6-v2 (libembedding)",
+        model_id="sentence-transformers/all-MiniLM-L6-v2",
+        backend="libembedding",
+    ),
 }

requirements.txt

@@ -4,5 +4,6 @@ datasets
 psutil
 tabulate
 fastembed
+libembedding
 numpy
 scipy

wrapper.py

@@ -51,10 +51,24 @@ class FastEmbedWrapper:
         return np.array(embeddings, dtype=np.float32)
 
 
-def load_model(cfg: ModelConfig) -> SBertWrapper | GGUFWrapper | FastEmbedWrapper:
+class LibEmbedWrapper:
+    """Wraps libembedding.TextEmbedding."""
+
+    def __init__(self, cfg: ModelConfig):
+        from libembedding import TextEmbedding
+        self._model = TextEmbedding(cfg.model_id)
+
+    def encode(self, sentences: list[str], batch_size: int = 64, **kwargs) -> np.ndarray:
+        embeddings = list(self._model.embed(sentences, batch_size=batch_size))
+        return np.array(embeddings, dtype=np.float32)
+
+
+def load_model(cfg: ModelConfig) -> SBertWrapper | GGUFWrapper | FastEmbedWrapper | LibEmbedWrapper:
     """Factory: returns the right wrapper for the model's backend."""
     if cfg.backend == "gguf":
         return GGUFWrapper(cfg)
     if cfg.backend == "fastembed":
         return FastEmbedWrapper(cfg)
+    if cfg.backend == "libembedding":
+        return LibEmbedWrapper(cfg)
     return SBertWrapper(cfg)