src/Z_C_Train_and_save_embeddings.py

from A_Preprocess import load_pdf_data, preprocess_data
from E_Model_utils import load_model, get_embeddings
from E_Faiss_utils import save_embeddings

# Load and preprocess data
data_file_path = r'C:\Users\serban.tica\Documents\tobi_llm_intent_recognition\data\Pager_Intents_Cleaned.csv'
data = load_pdf_data(data_file_path)
#data = preprocess_data(data)

# Models to evaluate
models = {"multilingual-e5-large":"intfloat/multilingual-e5-large"}

    #"multilingual-e5-small":"intfloat/multilingual-e5-small", "all-MiniLM-L6-v2": "sentence-transformers/all-MiniLM-L6-v2", "all-mpnet-base-v2":"sentence-transformers/all-mpnet-base-v2"
 #"bert-base-nli-mean-tokens":"sentence-transformers/bert-base-nli-mean-tokens", #"all-MiniLM-L6-v2": "sentence-transformers/all-MiniLM-L6-v2", "all-distilroberta-v1":"sentence-transformers/all-distilroberta-v1"}
# 'sentence-transformers/paraphrase-multilingual-mpnet-base-v2'
# "all-mpnet-base-v2":"sentence-transformers/all-mpnet-base-v2",
# "bert-base-nli":"sentence-transformers/bert-base-nli-mean-tokens",
# "all-MiniLM-L6-v2": "sentence-transformers/all-MiniLM-L6-v2",
# "all-distilroberta-v1":"sentence-transformers/all-distilroberta-v1"
# "bert-base-romanian-cased-v1": "sentence-transformers/bert-base-romanian-cased-v1",
# "bert-base-romanian-uncased-v1": "sentence-transformers/dumitrescustefan/bert-base-romanian-uncased-v1",
#"mBERT": "bert-base-multilingual-cased", "XLM-R": "xlm-roberta-base", "Romanian BERT": "dumitrescustefan/bert-base-romanian-cased-v1", "dumitrescustefan/bert-base-romanian-uncased-v1": "dumitrescustefan/bert-base-romanian-uncased-v1"
# Generate and save embeddings for each model, "xlm-r-distilroberta-base-paraphrase-v1"

for model_name, model_path in models.items():
    print(f"Processing model: {model_name}")
    model = load_model(model_path)
    texts = data['utterance'].tolist()
    embeddings = get_embeddings(model, texts)
    save_embeddings(embeddings, file_name=f"embeddings/{model_name}_vector_db.index")