first commit

app.py

@@ -0,0 +1,53 @@
+import gradio as gr 
+import os
+from pypdf import PdfReader
+from typing import Optional
+import json
+
+from load_db import load_vectorestore_from_pdf
+
+
+TEMP_PDF_PATH = "temp.pdf"
+retriever = None
+db = None
+documents = None
+
+def pdf_to_text(file_path:str, page_num:Optional[int]=None):
+    reader = PdfReader(file_path)
+    if page_num:
+        return reader.pages[page_num-1].extract_text()
+    text = ""
+    for page in reader.pages:
+        page_text = page.extract_text()
+        text += page_text
+    return text
+
+def load_vectore_store():
+    global retriever, db
+    db = load_vectorestore_from_pdf(TEMP_PDF_PATH,persist=False)
+    retriever = db.as_retriever(search_kwargs={"k": 4})
+
+def load_pdf(inp):
+    # Convert bytes back to a PDF file
+    with open(TEMP_PDF_PATH, "wb") as f:
+        f.write(inp)
+    # Extract text from the PDF file
+    text = pdf_to_text(TEMP_PDF_PATH)
+    load_vectore_store()
+    #print(text)
+    return text
+
+
+with gr.Blocks() as app:
+    file = gr.File(type="binary") 
+    load_file_button = gr.Button("Load")
+    with gr.Accordion("Modulhandbuch anzeigen",open=False):
+        handbook = gr.TextArea(label="Modulhandbuch")
+
+    load_file_button.click(load_pdf,inputs=file,outputs=handbook)
+
+
+
+
+if __name__ == "__main__":
+    app.launch(debug=True)

load_db.py

@@ -0,0 +1,51 @@
+from dotenv import load_dotenv
+#from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings
+
+from langchain.vectorstores import Chroma
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.llms import OpenAI
+from langchain.chains import ConversationalRetrievalChain, RetrievalQA
+from langchain.chat_models import ChatOpenAI
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+from langchain.document_loaders import TextLoader, PyPDFLoader
+from typing import Optional
+import os 
+
+
+load_dotenv()
+
+embeddings_model_name ="multi-qa-MiniLM-L6-cos-v1"
+persist_directory = "db"
+target_source_chunks = 4
+openai_api_key = os.environ.get('OPENAI_API_KEY')
+
+
+#embeddings = HuggingFaceEmbeddings(model_name=embeddings_model_name)
+embeddings = SentenceTransformerEmbeddings(model_name=embeddings_model_name)
+
+
+def load_vectorestore_from_pdf(path:str, embeddings=embeddings, persist:Optional[bool]=True):
+    
+    loader = PyPDFLoader(path)
+    documents = loader.load()
+    #print(len(documents))
+
+    text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
+    documents = text_splitter.split_documents(documents)
+
+    #print(len(documents))
+
+    
+    
+    if not persist:
+        vectorstore =     Chroma.from_documents(documents, embeddings, persist_directory=None)
+        return vectorstore
+    vectorstore =     Chroma.from_documents(documents, embeddings, persist_directory=persist_directory)
+    vectorstore.persist()
+    vectorstore = None
+    return None
+    
+
+if __name__ == "__main__":
+    load_vectorestore_from_pdf()

requirements.txt

@@ -0,0 +1,5 @@
+pypdf
+sentence-transformers
+openai
+gradio
+langchain