Spaces:
Sleeping
Sleeping
File size: 1,569 Bytes
2db8ee1 eaa16d9 2db8ee1 eaa16d9 2db8ee1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | import faiss
import numpy as np
import os
import pickle
class DocumentIndex:
def __init__(self):
self._model = None
self.index = faiss.IndexFlatIP(384)
self.metadata = []
@property
def model(self):
if self._model is None:
from sentence_transformers import SentenceTransformer
self._model = SentenceTransformer("all-MiniLM-L6-v2")
return self._model
def add_document(self, full_text, source):
embedding = self.model.encode([full_text])
embedding = np.array(embedding).astype("float32")
faiss.normalize_L2(embedding)
self.index.add(embedding)
self.metadata.append({
"source": source,
"full_text": full_text
})
def search(self, query, k=2):
q_emb = self.model.encode([query])
q_emb = np.array(q_emb).astype("float32")
faiss.normalize_L2(q_emb)
scores, idxs = self.index.search(q_emb, k)
return [self.metadata[i] for i in idxs[0]]
def save_local(self, folder_path):
os.makedirs(folder_path, exist_ok=True)
faiss.write_index(self.index, os.path.join(folder_path, "index.faiss"))
with open(os.path.join(folder_path, "metadata.pkl"), "wb") as f:
pickle.dump(self.metadata, f)
def load_local(self, folder_path):
self.index = faiss.read_index(os.path.join(folder_path, "index.faiss"))
with open(os.path.join(folder_path, "metadata.pkl"), "rb") as f:
self.metadata = pickle.load(f)
|