create_vector_db.py

#import libraries
from langchain.vectorstores import FAISS
from langchain.embeddings import HuggingFaceEmbeddings
from data_processing import *
import pickle


# to store the embedding of skillset and interest 
def created_vector_database():
    # Initialize HuggingFace embedding model
    embedding_model = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")

    # embed skill-set and interests
    documents = return_clean_df()

    # Generate embeddings for documents
    doc_embeddings = [embedding_model.embed_query(doc) for doc in documents]

    # Create FAISS vector store
    vectorstore = FAISS.from_texts(texts=documents, embedding=embedding_model)
    with open("vector_db.pkl", "wb") as f:
        pickle.dump(vectorstore, f)

created_vector_database()