train.py

from langchain.document_loaders import  PyPDFLoader
from langchain_together.embeddings import TogetherEmbeddings
import faiss
import os
import time
import numpy as np
import pickle

os.environ["TOGETHER_API_KEY"] = st.secrets["together_api_key"]
embeddings = TogetherEmbeddings(model="togethercomputer/m2-bert-80M-8k-retrieval")

loader = PyPDFLoader("ship.pdf")
data = loader.load()
print (f'You have {len(data)} document(s) in your data')
print (f'There are {len(data[0].page_content)} characters in your sample document')
print (f'Here is a sample: {data[0].page_content}')

list_of_texts = []
list_of_embeddings = []
for val in data:
    text_content = val.page_content
    list_of_texts.append(text_content)
    embedding_vector = embeddings.embed_query(text_content)
    list_of_embeddings.append(embedding_vector)


embeddings_array = np.array(list_of_embeddings).astype('float32')
d = len(list_of_embeddings[0])
index = faiss.IndexFlatL2(d)
index.add(embeddings_array)

# Save the index
faiss.write_index(index, "faiss.index")
# Save the list of texts
with open("list_of_texts.pkl", 'wb') as f:
    pickle.dump(list_of_texts, f)