backend/vectorstore/text_store.py

from sentence_transformers import SentenceTransformer
import faiss
import numpy as np

class TextVectorStore:
    def __init__(self):
        self.model = SentenceTransformer('all-MiniLM-L6-v2')
        self.index = faiss.IndexFlatIP(384)  # 384 is the dimension of the embeddings from the model
        self.metadata = []

    def add_texts(self, texts, metadatas=None):
        embeddings = self.model.encode(texts)
        embeddings = np.array(embeddings).astype("float32")
        faiss.normalize_L2(embeddings)
        self.index.add(embeddings)

        for text, meta in zip(texts, metadatas):
            enriched_meta = meta.copy()
            enriched_meta["content"] = text
            self.metadata.append(enriched_meta)

    def search(self, query, k=5):
        q_emb = self.model.encode([query])
        q_emb = np.array(q_emb).astype("float32")
        faiss.normalize_L2(q_emb)
        _, idxs = self.index.search(q_emb.astype("float32"), k)
        return [self.metadata[i] for i in idxs[0]]