Update app.py

app.py

@@ -5,6 +5,14 @@ import ctypes #to run on C api directly
 import llama_cpp
 from llama_cpp import Llama
 from huggingface_hub import hf_hub_download #load from huggingfaces 
+from dotenv import load_dotenv
+from PyPDF2 import PdfReader
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.vectorstores import FAISS
+from langchain.chat_models import ChatOpenAI
+from langchain.embeddings import OpenAIEmbeddings, HuggingFaceInstructEmbeddings
+from langchain.memory import ConversationBufferMemory
+from langchain.chains import ConversationalRetrievalChain
 
 
 llm = Llama(model_path= hf_hub_download(repo_id="TheBloke/Dolphin-Llama2-7B-GGML", filename="dolphin-llama2-7b.ggmlv3.q4_1.bin"), n_ctx=2048) #download model from hf/ n_ctx=2048 for high ccontext length
@@ -13,6 +21,26 @@ history = []
 
 pre_prompt = " The user and the AI are having a conversation : <|endoftext|> \n "
 
+def get_pdf_text(pdfs):
+  text=""
+  for pdf in pdfs:
+    pdf_reader = PdfReader(pdf)
+    for page in pdf_reader.pages:
+      text+= page.extract_text()
+  return text
+
+def get_text_chunks(text):
+  text_splitter = CharacterTextSplitter(separator="\n",
+  chunk_size=1000, chunk_overlap = 200, length_function=len)
+  chunks = text_splitter.split_text(text)
+  return chunks
+
+def get_vectorstore(text_chunks):
+    embeddings = OpenAIEmbeddings()
+#     embeddings = HuggingFaceInstructEmbeddings(model_name="hkunlp/instructor-xl")
+    vectorstore = FAISS.from_texts(texts=text_chunks, embedding=embeddings)
+    return vectorstore
+
 def generate_text(input_text, history):
     print("history ",history)
     print("input ", input_text)