test

app.py

@@ -1,52 +1,56 @@
+from dotenv import load_dotenv
 import streamlit as st
-import os 
-import nltk
-import json
-
-os.system('chmod 777 /tmp')
-os.system('apt-get update -y')
-os.system('apt-get install tesseract-ocr -y')
-os.system('pip install -q pytesseract')
-#os.system('apt-get install poppler-utils')
+from PyPDF2 import PdfReader
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.chains.question_answering import load_qa_chain
+from langchain.llms import OpenAI
+from langchain.callbacks import get_openai_callback
 
 os.environ["OPENAI_API_KEY"] = os.getenv("SECRET_KEY")
 
-from langchain.document_loaders import PyPDFLoader
-from langchain.vectorstores import Chroma
-from langchain.embeddings.openai import OpenAIEmbeddings
-from langchain.document_loaders import UnstructuredPDFLoader
-from langchain.chat_models import ChatOpenAI
-from langchain.chains.question_answering import load_qa_chain   
-from PIL import Image
-from datetime import datetime
-from tempfile import NamedTemporaryFile
+def main():
+    load_dotenv()
+    st.set_page_config(page_title="Ask your PDF")
+    st.header("Ask your PDF 💬")
     
-st.subheader("Upload CV in PDF or image format")
-uploaded_file = st.file_uploader("Upload PDF or Images", type=["pdf","png","jpg","jpeg"])
-
-nltk.download('punkt')
-nltk.download('averaged_perceptron_tagger')
-
-if uploaded_file:
-    file_name, file_extension = os.path.splitext(uploaded_file.name)
-    if file_extension != '.pdf':
-        uploaded_image = Image.open(uploaded_file)
-        img = uploaded_image.convert('RGB')
-        loader = UnstructuredPDFLoader(img)
-        img.save(file_name+'.pdf')
-        PDFFileName = file_name+'.pdf'
-    else:
-        with NamedTemporaryFile(dir='.', suffix='.pdf') as f:
-            f.write(uploaded_file.getbuffer())
-            PDFFileName = f.name
-    loader = UnstructuredPDFLoader(PDFFileName)
-    pages = loader.load_and_split()
-    embeddings = OpenAIEmbeddings()
-    docsearch = Chroma.from_documents(pages, embeddings).as_retriever()    
+    # upload file
+    pdf = st.file_uploader("Upload your PDF", type="pdf")
+    
+    # extract the text
+    if pdf is not None:
+      pdf_reader = PdfReader(pdf)
+      text = ""
+      for page in pdf_reader.pages:
+        text += page.extract_text()
+        
+      # split into chunks
+      text_splitter = CharacterTextSplitter(
+        separator="\n",
+        chunk_size=1000,
+        chunk_overlap=200,
+        length_function=len
+      )
+      chunks = text_splitter.split_text(text)
+      
+      # create embeddings
+      embeddings = OpenAIEmbeddings()
+      knowledge_base = FAISS.from_texts(chunks, embeddings)
+      
+      # show user input
+      user_question = st.text_input("Ask a question about your PDF:")
+      if user_question:
+        docs = knowledge_base.similarity_search(user_question)
+        
+        llm = OpenAI()
+        chain = load_qa_chain(llm, chain_type="stuff")
+        with get_openai_callback() as cb:
+          response = chain.run(input_documents=docs, question=user_question)
+          print(cb)
+           
+        st.write(response)
     
-    current_date = datetime.now()
-    query = "Output informatio, (all in English), from the document in JSON format: full name, age, languages, education, school, places of work, skills.If the date of birth is not indicated, then please calculate the approximate age of the candidate based on the information provided in the document, for calculations, take into account that graduation from the university is usually at 22 years old. Current date = "+ current_date.date().strftime('%Y-%m-%d')
-    docs = docsearch.get_relevant_documents(query)
-    chain = load_qa_chain(ChatOpenAI(temperature=0), chain_type="stuff")
-    output = chain.run(input_documents=docs, question=query)
-    st.write(output)
+
+if __name__ == '__main__':
+    main()

requirements.txt

@@ -1,14 +1,6 @@
-numpy
-pandas
-streamlit
-langchain 
-openai 
-Pillow
-unstructured 
-chromadb 
-tiktoken 
-pypdfium2 
-unstructured_inference 
-tesseract
-poppler-utils
-pdf2image
+langchain==0.0.154
+PyPDF2==3.0.1
+python-dotenv==1.0.0
+streamlit==1.18.1
+faiss-cpu==1.7.4
+altair<5