backend/vectorstore/document_index.py

import faiss
import numpy as np
import os
import pickle

class DocumentIndex:
    def __init__(self):
        self._model = None
        self.index = faiss.IndexFlatIP(384)
        self.metadata = []

    @property
    def model(self):
        if self._model is None:
            from sentence_transformers import SentenceTransformer
            self._model = SentenceTransformer("all-MiniLM-L6-v2")
        return self._model

    def add_document(self, full_text, source):
        embedding = self.model.encode([full_text])
        embedding = np.array(embedding).astype("float32")
        faiss.normalize_L2(embedding)

        self.index.add(embedding)

        self.metadata.append({
            "source": source,
            "full_text": full_text
        })

    def search(self, query, k=2):
        q_emb = self.model.encode([query])
        q_emb = np.array(q_emb).astype("float32")
        faiss.normalize_L2(q_emb)

        scores, idxs = self.index.search(q_emb, k)

        return [self.metadata[i] for i in idxs[0]]

    def save_local(self, folder_path):
        os.makedirs(folder_path, exist_ok=True)

        faiss.write_index(self.index, os.path.join(folder_path, "index.faiss"))

        with open(os.path.join(folder_path, "metadata.pkl"), "wb") as f:
            pickle.dump(self.metadata, f)
    
    def load_local(self, folder_path):
        self.index = faiss.read_index(os.path.join(folder_path, "index.faiss"))
            
        with open(os.path.join(folder_path, "metadata.pkl"), "rb") as f:
            self.metadata = pickle.load(f)