src/E_Faiss_utils.py

import os
import faiss
import numpy as np  
from pathlib import Path
import sys

# Add the project directory to the path
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from import_Path import BASE_DIR


def save_faiss_embeddings_index(embeddings, file_name):
    # Ensure embeddings are in float32 format
    if not isinstance(embeddings, np.ndarray):
        embeddings = embeddings.numpy()
    embeddings = embeddings.astype('float32')
    
    # Create the directory if it doesn't exist
    #os.makedirs(os.path.dirname(file_name), exist_ok=True)
    
    # Create a FAISS index
    index = faiss.IndexFlatL2(embeddings.shape[1])  # L2 distance
    index.add(embeddings)
    
    # Save the FAISS index
    # old faiss.write_index(index, file_name)    
    index_path = BASE_DIR / "embeddings" / file_name
    faiss.write_index(index, str(index_path))

def load_faiss_index(index_path):
    index = faiss.read_index(index_path)
    return index

def normalize_embeddings(embeddings):
    # Normalize embeddings
    embeddings = embeddings / np.linalg.norm(embeddings, axis=1)[:, None]
    return embeddings