Update app.py

hello commit 1

app.py

@@ -1,46 +1,61 @@
 import gradio as gr
-import os
-
 from langchain_community.document_loaders import PyPDFLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.vectorstores import Chroma
-from langchain.chains import ConversationalRetrievalChain
-from langchain_community.embeddings import HuggingFaceEmbeddings 
-from langchain_community.llms import HuggingFacePipeline
-from langchain.chains import ConversationChain
-from langchain.memory import ConversationBufferMemory
-from langchain_community.llms import HuggingFaceEndpoint
-
+from langchain_community.embeddings import HuggingFaceEmbeddings
 from pathlib import Path
-import chromadb
 from unidecode import unidecode
 
-from transformers import AutoTokenizer
-import transformers
-import torch
-import tqdm 
-import accelerate
-import re
-
-list_llm = ["HuggingFaceH4/zephyr-7b-beta", "mistralai/Mistral-7B-Instruct-v0.2"]
-list_llm_simple = [os.path.basename(llm) for llm in list_llm]
-
 def summarize_document(document_text):
     # Your summarization code here
     summary = "The document covers various topics such as X, Y, and Z, providing detailed insights into each aspect."
     return summary
 
+def initialize_database(list_file_obj, chunk_size, chunk_overlap, progress=gr.Progress()):
+    list_file_path = [x.name for x in list_file_obj if x is not None]
+    collection_name = create_collection_name(list_file_path[0])
+    doc_splits = load_doc(list_file_path, chunk_size, chunk_overlap)
+    vector_db = create_db(doc_splits, collection_name)
+    return vector_db, collection_name, "Complete!"
+
+def load_doc(list_file_path, chunk_size, chunk_overlap):
+    loaders = [PyPDFLoader(x) for x in list_file_path]
+    pages = []
+    for loader in loaders:
+        pages.extend(loader.load())
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size = chunk_size, 
+        chunk_overlap = chunk_overlap)
+    doc_splits = text_splitter.split_documents(pages)
+    return doc_splits
+
+def create_db(splits, collection_name):
+    embedding = HuggingFaceEmbeddings()
+    new_client = chromadb.EphemeralClient()
+    vectordb = Chroma.from_documents(
+        documents=splits,
+        embedding=embedding,
+        client=new_client,
+        collection_name=collection_name,
+    )
+    return vectordb
+
+def create_collection_name(filepath):
+    collection_name = Path(filepath).stem
+    collection_name = unidecode(collection_name)
+    collection_name = re.sub('[^A-Za-z0-9]+', '-', collection_name)
+    collection_name = collection_name[:50]
+    if len(collection_name) < 3:
+        collection_name = collection_name + 'xyz'
+    if not collection_name[0].isalnum():
+        collection_name = 'A' + collection_name[1:]
+    if not collection_name[-1].isalnum():
+        collection_name = collection_name[:-1] + 'Z'
+    return collection_name
+
 def demo():
-    with gr.Blocks(theme="base") as demo:
-        gr.Markdown("<center><h2>PDF Summarizer</center></h2>")
-        
-        text_input = gr.Textbox(placeholder="Paste your document text here", label="Document Text")
-        summarize_btn = gr.Button("Summarize")
-        summary_output = gr.Textbox(readonly=True, label="Summary")
-        
-        summarize_btn.click(summarize_document, inputs=[text_input], outputs=[summary_output])
-        
-    demo.launch()
+    with gr.Interface(summarize_document, inputs="text", outputs="text", title="PDF Summarizer") as iface:
+        iface.launch()
 
 if __name__ == "__main__":
     demo()