Upload folder using huggingface_hub

app.py

@@ -18,12 +18,22 @@ from llama_index.core.query_engine import SimpleMultiModalQueryEngine
 from llama_index.llms.openai import OpenAI
 from llama_index.core import load_index_from_storage, get_response_synthesizer
 import tempfile
+from qdrant_client import QdrantClient, models
+import getpass
+
+curr_user = getpass.getuser()
+# from langchain.vectorstores import Chroma
+# To connect to the same event-loop,
+# allows async events to run on notebook
+
+# import nest_asyncio
+
+# nest_asyncio.apply()
+
+from dotenv import load_dotenv
+load_dotenv()
 
-# from dotenv import load_dotenv
-# load_dotenv()
 
-OPENAI_API_KEY = "sk-proj-beorroDjV4FeoL6OAzbET3BlbkFJT4WcMiP0x30GxzmbpIEC"
-os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY
 
 
 def extract_text_from_pdf(pdf_path):
@@ -104,20 +114,27 @@ def remove_duplicate_images(data_path) :
             except Exception as e:
                 print(e)
                 pass
-
-def initialize_qdrant(temp_dir):
+# from langchain_chroma import Chroma
+# import chromadb
+def initialize_qdrant(temp_dir , file_name , user):
     
-    # client = qdrant_client.QdrantClient(path="qdrant_mm_db_pipeline")
-    # client = qdrant_client.QdrantClient(host = "192.168.0.1" , port = 2401  , https = True)
+    client = qdrant_client.QdrantClient(path=f"qdrant_mm_db_pipeline_{user}_{file_name}")
     # client = qdrant_client.QdrantClient(url = "http://localhost:2452")    
-    client = qdrant_client.QdrantClient(url="4b0af7be-d5b3-47ac-b215-128ebd6aa495.europe-west3-0.gcp.cloud.qdrant.io:6333", api_key="CO1sNGLmC6R_Q45qSIUxBSX8sxwHud4MCm4as_GTI-vzQqdUs-bXqw",)
+    # client = qdrant_client.QdrantClient(url="4b0af7be-d5b3-47ac-b215-128ebd6aa495.europe-west3-0.gcp.cloud.qdrant.io:6333", api_key="CO1sNGLmC6R_Q45qSIUxBSX8sxwHud4MCm4as_GTI-vzQqdUs-bXqw",)
+    # client = qdrant_client.AsyncQdrantClient(location = ":memory:")
         
     if "vectordatabase" not in st.session_state or not st.session_state.vectordatabase:
-        text_store = QdrantVectorStore(client=client, collection_name="text_collection_pipeline")
-        image_store = QdrantVectorStore(client=client, collection_name="image_collection_pipeline")
+            
+        # text_store = client.create_collection(f"text_collection_pipeline_{user}_{file_name}"  ) 
+        # image_store = client.create_collection(f"image_collection_pipeline_{user}_{file_name}"  ) 
+
+
+        text_store = QdrantVectorStore( client = client , collection_name=f"text_collection_pipeline_{user}_{file_name}" )
+        image_store = QdrantVectorStore(client = client , collection_name=f"image_collection_pipeline_{user}_{file_name}")
         storage_context = StorageContext.from_defaults(vector_store=text_store, image_store=image_store)
-        documents = SimpleDirectoryReader(os.path.join(temp_dir, "my_own_data")).load_data()
+        documents = SimpleDirectoryReader(os.path.join(temp_dir, f"my_own_data_{user}_{file_name}")).load_data()
         index = MultiModalVectorStoreIndex.from_documents(documents, storage_context=storage_context)
+
         st.session_state.vectordatabase = index
     else :
         index = st.session_state.vectordatabase
@@ -153,7 +170,7 @@ def retrieve_and_query(query, retriever_engine):
     )
     qa_tmpl = PromptTemplate(qa_tmpl_str)
 
-    llm = OpenAI(model="gpt-4o-mini", temperature=0)
+    llm = OpenAI(model="gpt-4o", temperature=0)
     response_synthesizer = get_response_synthesizer(response_mode="refine", text_qa_template=qa_tmpl, llm=llm)
 
     response = response_synthesizer.synthesize(query, nodes=retrieval_results)
@@ -166,15 +183,16 @@ def retrieve_and_query(query, retriever_engine):
     
     return response, retrieved_image_path_list
 
-def process_pdf(pdf_file):
+def process_pdf(pdf_file , user):
+    import pdb; pdb.set_trace()
     temp_dir = tempfile.TemporaryDirectory()
     temp_pdf_path = os.path.join(temp_dir.name, pdf_file.name)
     with open(temp_pdf_path, "wb") as f:
         f.write(pdf_file.getvalue())
 
-    data_path = os.path.join(temp_dir.name, "my_own_data")
+    data_path = os.path.join(temp_dir.name, f"my_own_data_{user}_{os.path.splitext(pdf_file.name)[0]}")
     os.makedirs(data_path , exist_ok=True) 
-    img_save_path = os.path.join(temp_dir.name, "extracted_images")
+    img_save_path = os.path.join(temp_dir.name, f"extracted_images_{user}_{os.path.splitext(pdf_file.name)[0]}")
     os.makedirs(img_save_path , exist_ok=True) 
 
     extracted_text = extract_text_from_pdf(temp_pdf_path)
@@ -185,7 +203,8 @@ def process_pdf(pdf_file):
     moved_count = move_images(img_save_path, data_path)
     remove_low_size_images(data_path)
     remove_duplicate_images(data_path)
-    retriever_engine = initialize_qdrant(temp_dir.name)
+    import pdb; pdb.set_trace()
+    retriever_engine = initialize_qdrant(temp_dir.name , os.path.splitext(pdf_file.name)[0] , curr_user)
 
     return temp_dir, retriever_engine
 
@@ -199,13 +218,15 @@ def main():
         st.session_state.vectordatabase = None
 
     uploaded_file = st.file_uploader("Choose a PDF file", type="pdf")
+    # import pdb; pdb.set_trace()
     if uploaded_file is None:
         st.info("Please upload a PDF file.")
     else:
+        # import pdb; pdb.set_trace()
         st.info(f"Uploaded PDF: {uploaded_file.name}")
         if st.button("Process PDF"):
             with st.spinner("Processing PDF..."):
-                temp_dir, st.session_state.retriever_engine = process_pdf(uploaded_file)
+                temp_dir, st.session_state.retriever_engine = process_pdf(uploaded_file , curr_user)
         
                 st.success("PDF processed successfully!")