updated code

app.py

@@ -1,218 +1,13 @@
 import streamlit as st
-import os
-from PyPDF2 import PdfReader
-import pymupdf
-import numpy as np
-import cv2
-import shutil
-import imageio
-from PIL import Image
-import imagehash
-import matplotlib.pyplot as plt
-from llama_index.core.indices import MultiModalVectorStoreIndex
-from llama_index.vector_stores.qdrant import QdrantVectorStore
-from llama_index.core import SimpleDirectoryReader, StorageContext
-import qdrant_client
-from llama_index.core import PromptTemplate
-from llama_index.core.query_engine import SimpleMultiModalQueryEngine
-from llama_index.llms.openai import OpenAI
-from llama_index.core import load_index_from_storage, get_response_synthesizer
-import tempfile
-from qdrant_client import QdrantClient, models
-import getpass
 
-curr_user = getpass.getuser()
-# from langchain.vectorstores import Chroma
-# To connect to the same event-loop,
-# allows async events to run on notebook
-
-# import nest_asyncio
-
-# nest_asyncio.apply()
+from pdf_processing import process_pdf
+from retrieve_and_display import retrieve_and_query, plot_images
 
 from dotenv import load_dotenv
 load_dotenv()
 
-
-def extract_text_from_pdf(pdf_path):
-    reader = PdfReader(pdf_path)
-    full_text = ''
-    for page in reader.pages:
-        text = page.extract_text()
-        full_text += text
-    return full_text
-
-def extract_images_from_pdf(pdf_path, img_save_path):
-    doc = pymupdf.open(pdf_path)
-    for page in doc:
-        img_number = 0
-        for block in page.get_text("dict")["blocks"]:
-            if block['type'] == 1:
-                name = os.path.join(img_save_path, f"img{page.number}-{img_number}.{block['ext']}")
-                out = open(name, "wb")
-                out.write(block["image"])
-                out.close()
-                img_number += 1
-
-def is_empty(img_path):
-    image = cv2.imread(img_path, 0)
-    std_dev = np.std(image)
-    return std_dev < 1
-
-def move_images(source_folder, dest_folder):
-    image_files = [f for f in os.listdir(source_folder) 
-                   if f.lower().endswith(('.jpg', '.jpeg', '.png', '.gif'))]
-    os.makedirs(dest_folder, exist_ok=True)
-    moved_count = 0
-    for file in image_files:
-        src_path = os.path.join(source_folder, file)
-        if not is_empty(src_path):
-            shutil.move(src_path, os.path.join(dest_folder, file))
-            moved_count += 1
-    return moved_count
-
-def remove_low_size_images(data_path):
-    images_list = os.listdir(data_path)
-    low_size_photo_list = []
-    for one_image in images_list:
-        image_path = os.path.join(data_path, one_image)
-        try:
-            pic = imageio.imread(image_path)
-            size = pic.size
-            if size < 100:
-                low_size_photo_list.append(one_image)
-        except:
-            pass
-    for one_image in low_size_photo_list[1:]:
-        os.remove(os.path.join(data_path, one_image))
-
-def calc_diff(img1 , img2) :
-    i1 = Image.open(img1)
-    i2 = Image.open(img2)
-    h1 = imagehash.phash(i1)
-    h2 = imagehash.phash(i2)
-    return h1 - h2
-
-def remove_duplicate_images(data_path) :
-    image_files = os.listdir(data_path)
-    only_images = []
-    for one_image in image_files : 
-        if one_image.endswith('jpeg') or one_image.endswith('png') or one_image.endswith('jpg') :
-            only_images.append(one_image)
-    only_images1 = sorted(only_images) 
-    for one_image in only_images1 :
-        for another_image in only_images1 :
-            try :
-                if one_image == another_image :
-                    continue
-                else :
-                    diff = calc_diff(os.path.join(data_path ,one_image) , os.path.join(data_path ,another_image))
-                    if diff ==0  :
-                        os.remove(os.path.join(data_path , another_image))
-            except Exception as e:
-                print(e)
-                pass
-# from langchain_chroma import Chroma
-# import chromadb
-def initialize_qdrant(temp_dir , file_name , user):
-    client = qdrant_client.QdrantClient(path=f"qdrant_mm_db_pipeline_{user}_{file_name}")
-    # client = qdrant_client.QdrantClient(url = "http://localhost:2452")    
-    # client = qdrant_client.QdrantClient(url="4b0af7be-d5b3-47ac-b215-128ebd6aa495.europe-west3-0.gcp.cloud.qdrant.io:6333", api_key="CO1sNGLmC6R_Q45qSIUxBSX8sxwHud4MCm4as_GTI-vzQqdUs-bXqw",)
-    # client = qdrant_client.AsyncQdrantClient(location = ":memory:")
-        
-    if "vectordatabase" not in st.session_state or not st.session_state.vectordatabase:
-            
-        # text_store = client.create_collection(f"text_collection_pipeline_{user}_{file_name}"  ) 
-        # image_store = client.create_collection(f"image_collection_pipeline_{user}_{file_name}"  ) 
-
-
-        text_store = QdrantVectorStore( client = client , collection_name=f"text_collection_pipeline_{user}_{file_name}" )
-        image_store = QdrantVectorStore(client = client , collection_name=f"image_collection_pipeline_{user}_{file_name}")
-        storage_context = StorageContext.from_defaults(vector_store=text_store, image_store=image_store)
-        documents = SimpleDirectoryReader(os.path.join(temp_dir, f"my_own_data_{user}_{file_name}")).load_data()
-        index = MultiModalVectorStoreIndex.from_documents(documents, storage_context=storage_context)
-
-        st.session_state.vectordatabase = index
-    else :
-        index = st.session_state.vectordatabase
-    retriever_engine = index.as_retriever(similarity_top_k=1, image_similarity_top_k=1)
-    return retriever_engine
-
-def plot_images(image_paths):
-    images_shown = 0
-    plt.figure(figsize=(16, 9))
-    for img_path in image_paths:
-        if os.path.isfile(img_path):
-            image = Image.open(img_path)
-            plt.subplot(2, 3, images_shown + 1)
-            plt.imshow(image)
-            plt.xticks([])
-            plt.yticks([])
-            images_shown += 1
-            if images_shown >= 6:
-                break
-
-def retrieve_and_query(query, retriever_engine):
-    retrieval_results = retriever_engine.retrieve(query)
-    
-    qa_tmpl_str = (
-        "Context information is below.\n"
-        "---------------------\n"
-        "{context_str}\n"
-        "---------------------\n"
-        "Given the context information , "
-        "answer the query in detail.\n"
-        "Query: {query_str}\n"
-        "Answer: "
-    )
-    qa_tmpl = PromptTemplate(qa_tmpl_str)
-
-    llm = OpenAI(model="gpt-4o", temperature=0)
-    response_synthesizer = get_response_synthesizer(response_mode="refine", text_qa_template=qa_tmpl, llm=llm)
-
-    response = response_synthesizer.synthesize(query, nodes=retrieval_results)
-    
-    retrieved_image_path_list = []
-    for node in retrieval_results:
-        if (node.metadata['file_type'] == 'image/jpeg') or (node.metadata['file_type'] == 'image/png'):
-            if node.score > 0.25:
-                retrieved_image_path_list.append(node.metadata['file_path'])
-    
-    return response, retrieved_image_path_list
-#tmpnimvp35m , tmpnimvp35m , tmpydpissmv
-def process_pdf(pdf_file):
-    temp_dir = tempfile.TemporaryDirectory()
-    unique_folder_name = temp_dir.name.split('/')[-1]
-    temp_pdf_path = os.path.join(temp_dir.name, pdf_file.name)
-    with open(temp_pdf_path, "wb") as f:
-        f.write(pdf_file.getvalue())
-
-    data_path = os.path.join(temp_dir.name, f"my_own_data_{unique_folder_name}_{os.path.splitext(pdf_file.name)[0]}")
-    os.makedirs(data_path , exist_ok=True) 
-    img_save_path = os.path.join(temp_dir.name, f"extracted_images_{unique_folder_name}_{os.path.splitext(pdf_file.name)[0]}")
-    os.makedirs(img_save_path , exist_ok=True) 
-
-    extracted_text = extract_text_from_pdf(temp_pdf_path)
-    with open(os.path.join(data_path, "content.txt"), "w") as file:
-        file.write(extracted_text)
-
-    extract_images_from_pdf(temp_pdf_path, img_save_path)
-    moved_count = move_images(img_save_path, data_path)
-    remove_low_size_images(data_path)
-    remove_duplicate_images(data_path)
-    retriever_engine = initialize_qdrant(temp_dir.name , os.path.splitext(pdf_file.name)[0] , unique_folder_name)
-
-    return temp_dir, retriever_engine
-
-def main():
-    st.title("PDF Vector Database Query Tool")
-    st.markdown("This tool creates a vector database from a PDF and allows you to query it.")
-    
-    if "retriever_engine" not in st.session_state:
-        st.session_state.retriever_engine = None
-    if "vectordatabase" not in st.session_state:
-        st.session_state.vectordatabase = None
-
+def upload_file():
+    st.title("Upload File to chat with file")
     uploaded_file = st.file_uploader("Choose a PDF file", type="pdf")
     if uploaded_file is None:
         st.info("Please upload a PDF file.")
@@ -220,34 +15,42 @@ def main():
         st.info(f"Uploaded PDF: {uploaded_file.name}")
         if st.button("Process PDF"):
             with st.spinner("Processing PDF..."):
-                temp_dir, st.session_state.retriever_engine = process_pdf(uploaded_file)
-        
+                st.session_state.retriever_engine = process_pdf(uploaded_file)
                 st.success("PDF processed successfully!")
 
+def ask_question():
     if st.session_state.retriever_engine :
-        query = st.text_input("Enter your question:")
-        
-        
-        if st.button("Ask Question"):
-            print("running")
-            try:
+        if user_question := st.chat_input("Ask a question"):
+            with st.spinner("Retrieving information..."):
+                response, retrieved_image_path_list = retrieve_and_query(user_question, st.session_state.retriever_engine)
+                print(retrieved_image_path_list)
+            st.write("Retrieved Context:")
+            for node in response.source_nodes:
+                st.code(node.node.get_text())
+            
+            st.write("\nRetrieved Images:")
+            plot_images(retrieved_image_path_list)
+            st.pyplot()
 
-                with st.spinner("Retrieving information..."):
-                    response, retrieved_image_path_list = retrieve_and_query(query, st.session_state.retriever_engine)
-                    print(retrieved_image_path_list)
-                st.write("Retrieved Context:")
-                for node in response.source_nodes:
-                    st.code(node.node.get_text())
-                
-                st.write("\nRetrieved Images:")
-                plot_images(retrieved_image_path_list)
-                st.pyplot()
+            st.write("\nFinal Answer:")
+            st.code(response.response)
+    else:
+        st.title("Upload File to chat with file")
+        
+    
+    
+def main():
+    if "retriever_engine" not in st.session_state:
+        st.session_state.retriever_engine = None
+    page_names_to_funcs = {
+        "Upload File": upload_file,
+        "Chat": ask_question
+    }
 
-                st.write("\nFinal Answer:")
-                st.code(response.response)
-            
-            except Exception as e:
-                st.error(f"An error occurred: {e}")
+    demo_name = st.sidebar.selectbox("PDF Query Tool", page_names_to_funcs.keys())
+    page_names_to_funcs[demo_name]()
 
 if __name__ == "__main__":
+    # login_page()
     main()
+

awsfunctions.py

@@ -0,0 +1,164 @@
+
+import boto3
+from botocore.exceptions import ClientError, NoCredentialsError
+import os
+
+def upload_folder_to_s3(local_dir, prefix=''):
+    s3_bucket = os.getenv("AWS_BUCKET_NAME")
+    s3_client = boto3.client('s3')
+
+    for root, dirs, files in os.walk(local_dir):
+        for dir in dirs:
+            dir_path = os.path.join(root, dir)
+            relative_path = os.path.relpath(dir_path, local_dir)
+            
+            # Create the directory in S3 if it doesn't exist
+            try:
+                s3_client.put_object(Bucket=s3_bucket, Key=f"{prefix}{relative_path}/")
+            except ClientError as e:
+                if e.response['Error']['Code'] == '404':
+                    continue  # Directory already exists
+                else:
+                    raise
+            
+        for file in files:
+            file_path = os.path.join(root, file)
+            relative_path = os.path.relpath(file_path, local_dir)
+            
+            try:
+                s3_client.upload_file(file_path, s3_bucket, f"{prefix}{relative_path}")
+                print(f"Uploaded: {file_path} -> s3://{s3_bucket}/{prefix}{relative_path}")
+            except Exception as e:
+                raise e
+                # print(f"Error uploading {file_path}: {e}")
+
+def check_file_exists_in_s3(file_path):
+    bucket_name = os.getenv("AWS_BUCKET_NAME")
+    s3_client = boto3.client('s3')
+    
+    try:
+        s3_client.head_object(Bucket=bucket_name, Key=file_path)
+        return True
+    except ClientError as e:
+        if e.response['Error']['Code'] == '404':
+            return False
+        else:
+            raise e
+
+def download_files_from_s3(local_folder, file_path_list):
+    s3 = boto3.client('s3')
+    bucket_name = os.getenv("AWS_BUCKET_NAME")
+    folder_prefix = ''
+    
+    try:
+        # List objects in the S3 bucket
+        paginator = s3.get_paginator('list_objects_v2')
+        page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=folder_prefix)
+        
+        # Download filtered files
+        for page in page_iterator:
+            for obj in page.get('Contents', []):
+                key = obj['Key']
+                
+                # Apply file filter if specified
+                if key not in file_path_list:
+                    continue
+                
+                # Construct local file path
+                local_path = os.path.join(local_folder, key)
+                os.makedirs(os.path.dirname(local_path), exist_ok=True)
+                try:
+                    print(f"Downloading: {key} -> {local_path}")
+                    s3.download_file(bucket_name, key, local_path)
+                    print(f"Downloaded: {local_path}")
+                except Exception as e:
+                    print(f"Error downloading {key}: {e}")
+    
+    except NoCredentialsError:
+        print("No AWS credentials found.")
+    except Exception as e:
+        print(f"An error occurred: {e}")
+        
+def download_folder_from_s3(local_folder, aws_folder_prefix):
+    s3 = boto3.client('s3')
+    bucket_name = os.getenv("AWS_BUCKET_NAME")
+    folder_prefix = aws_folder_prefix
+    
+    try:
+        # List objects in the S3 bucket
+        paginator = s3.get_paginator('list_objects_v2')
+        page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=folder_prefix)
+        
+        # Download filtered files
+        for page in page_iterator:
+            for obj in page.get('Contents', []):
+                key = obj['Key']
+                                
+                # Construct local file path
+                local_path = os.path.join(local_folder, key)
+                os.makedirs(os.path.dirname(local_path), exist_ok=True)
+                try:
+                    print(f"Downloading: {key} -> {local_path}")
+                    s3.download_file(bucket_name, key, local_path)
+                    print(f"Downloaded: {local_path}")
+                except Exception as e:
+                    print(f"Error downloading {key}: {e}")
+                    raise e
+    
+    except NoCredentialsError:
+        print("No AWS credentials found.")
+    except Exception as e:
+        print(f"An error occurred: {e}")
+
+def delete_s3_folder(folder_path):
+    bucket_name = os.getenv("AWS_BUCKET_NAME")
+    s3_client = boto3.client('s3')
+    
+    try:
+        # List objects in the S3 bucket
+        paginator = s3_client.get_paginator('list_objects_v2')
+        page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=folder_path)
+        
+        # Delete objects within the folder_path
+        delete_keys = {'Objects': []}
+        for page in page_iterator:
+            for obj in page.get('Contents', []):
+                key = obj['Key']
+                
+                # Construct the full key for deletion
+                delete_key = {'Key': key}
+                delete_keys['Objects'].append(delete_key)
+                
+                print(f"Deleting: {key}")
+        
+        # Perform batch delete operation
+        if len(delete_keys['Objects']) > 0:
+            s3_client.delete_objects(Bucket=bucket_name, Delete=delete_keys)
+            print(f"Deleted {len(delete_keys['Objects'])} objects in folder '{folder_path}'")
+        else:
+            print(f"No objects found in folder '{folder_path}'")
+
+    except ClientError as e:
+        print(f"An error occurred: {e}")
+
+def list_s3_objects(prefix=''):
+    bucket_name = os.getenv("AWS_BUCKET_NAME")
+    s3_client = boto3.client('s3')
+    
+    try:
+        paginator = s3_client.get_paginator('list_objects_v2')
+        page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=prefix)
+        for page in page_iterator:
+            for obj in page.get('Contents', []):
+                print(f"Key: {obj['Key']}")
+                print(f"Size: {obj['Size']} bytes")
+                print(f"Last Modified: {obj['LastModified']}")
+                print(f"ETag: {obj['ETag']}")
+                print(f"File Extension: {os.path.splitext(obj['Key'])[-1]}")
+                print("---")
+    
+    except ClientError as e:
+        print(f"An error occurred: {e}")
+        
+        
+        

pdf_processing.py

@@ -0,0 +1,148 @@
+from PyPDF2 import PdfReader
+import pymupdf
+import numpy as np
+import cv2
+import shutil
+import imageio
+from PIL import Image
+import imagehash
+import tempfile
+import os
+from llama_index.core.indices import MultiModalVectorStoreIndex
+from llama_index.vector_stores.qdrant import QdrantVectorStore
+from llama_index.core import SimpleDirectoryReader, StorageContext
+from awsfunctions import upload_folder_to_s3, check_file_exists_in_s3, download_folder_from_s3
+import qdrant_client
+import streamlit as st
+
+def extract_text_from_pdf(pdf_path):
+    reader = PdfReader(pdf_path)
+    full_text = ''
+    for page in reader.pages:
+        text = page.extract_text()
+        full_text += text
+    return full_text
+
+def extract_images_from_pdf(pdf_path, img_save_path):
+    doc = pymupdf.open(pdf_path)
+    for page in doc:
+        img_number = 0
+        for block in page.get_text("dict")["blocks"]:
+            if block['type'] == 1:
+                name = os.path.join(img_save_path, f"img{page.number}-{img_number}.{block['ext']}")
+                out = open(name, "wb")
+                out.write(block["image"])
+                out.close()
+                img_number += 1
+
+def is_empty(img_path):
+    image = cv2.imread(img_path, 0)
+    std_dev = np.std(image)
+    return std_dev < 1
+
+def move_images(source_folder, dest_folder):
+    image_files = [f for f in os.listdir(source_folder) 
+                   if f.lower().endswith(('.jpg', '.jpeg', '.png', '.gif'))]
+    os.makedirs(dest_folder, exist_ok=True)
+    moved_count = 0
+    for file in image_files:
+        src_path = os.path.join(source_folder, file)
+        if not is_empty(src_path):
+            shutil.move(src_path, os.path.join(dest_folder, file))
+            moved_count += 1
+    return moved_count
+
+def remove_low_size_images(data_path):
+    images_list = os.listdir(data_path)
+    low_size_photo_list = []
+    for one_image in images_list:
+        image_path = os.path.join(data_path, one_image)
+        try:
+            pic = imageio.imread(image_path)
+            size = pic.size
+            if size < 100:
+                low_size_photo_list.append(one_image)
+        except:
+            pass
+    for one_image in low_size_photo_list[1:]:
+        os.remove(os.path.join(data_path, one_image))
+
+def calc_diff(img1 , img2) :
+    i1 = Image.open(img1)
+    i2 = Image.open(img2)
+    h1 = imagehash.phash(i1)
+    h2 = imagehash.phash(i2)
+    return h1 - h2
+
+def remove_duplicate_images(data_path) :
+    image_files = os.listdir(data_path)
+    only_images = []
+    for one_image in image_files : 
+        if one_image.endswith('jpeg') or one_image.endswith('png') or one_image.endswith('jpg') :
+            only_images.append(one_image)
+    only_images1 = sorted(only_images) 
+    for one_image in only_images1 :
+        for another_image in only_images1 :
+            try :
+                if one_image == another_image :
+                    continue
+                else :
+                    diff = calc_diff(os.path.join(data_path ,one_image) , os.path.join(data_path ,another_image))
+                    if diff ==0  :
+                        os.remove(os.path.join(data_path , another_image))
+            except Exception as e:
+                print(e)
+                pass
+# from langchain_chroma import Chroma
+# import chromadb
+def initialize_qdrant(temp_dir , aws_prefix):
+    client = qdrant_client.QdrantClient(path=os.path.join(temp_dir, "qdrant"))
+    text_store = QdrantVectorStore( client = client , collection_name=f"text_collection" )
+    image_store = QdrantVectorStore(client = client , collection_name=f"image_collection")
+    storage_context = StorageContext.from_defaults(vector_store=text_store, image_store=image_store)
+    documents = SimpleDirectoryReader(os.path.join(temp_dir, f"data")).load_data()
+    for doc in documents:
+        doc.metadata["file_path"] = os.path.join(aws_prefix, os.path.relpath(doc.metadata["file_path"], temp_dir))
+    index = MultiModalVectorStoreIndex.from_documents(documents, storage_context=storage_context)
+    retriever_engine = index.as_retriever(similarity_top_k=1, image_similarity_top_k=1)
+    return retriever_engine
+
+def process_pdf(pdf_file):
+    username = "ptchecker"
+    aws_prefix_path = os.path.join(os.getenv("FOLDER_PREFIX"), username, "FILES", os.path.splitext(pdf_file.name)[0])
+    if check_file_exists_in_s3(os.path.join(aws_prefix_path, pdf_file.name)):
+        temp_dir = tempfile.mkdtemp()
+        download_folder_from_s3(local_folder=temp_dir, aws_folder_prefix=os.path.join(aws_prefix_path, "qdrant"))
+        client = qdrant_client.QdrantClient(path=os.path.join(temp_dir))
+        image_store = QdrantVectorStore(client = client , collection_name=f"image_collection")
+        text_store = QdrantVectorStore(client = client , collection_name=f"text_collection")
+        index = MultiModalVectorStoreIndex.from_vector_store(vector_store=text_store, image_store=image_store)
+        retriever_engine = index.as_retriever(similarity_top_k=1, image_similarity_top_k=1)
+        shutil.rmtree(temp_dir)
+        return retriever_engine
+    else:
+        temp_dir = tempfile.mkdtemp()
+        temp_pdf_path = os.path.join(temp_dir, pdf_file.name)
+        with open(temp_pdf_path, "wb") as f:
+            f.write(pdf_file.getvalue())
+
+        data_path = os.path.join(temp_dir, "data")
+        os.makedirs(data_path , exist_ok=True) 
+        img_save_path = os.path.join(temp_dir, "images")
+        os.makedirs(img_save_path , exist_ok=True) 
+
+        extracted_text = extract_text_from_pdf(temp_pdf_path)
+        with open(os.path.join(data_path, "content.txt"), "w") as file:
+            file.write(extracted_text)
+
+        extract_images_from_pdf(temp_pdf_path, img_save_path)
+        moved_count = move_images(img_save_path, data_path)
+        print("Images moved count : ", moved_count)
+        remove_low_size_images(data_path)
+        remove_duplicate_images(data_path)
+        shutil.rmtree(img_save_path)
+        retriever_engine = initialize_qdrant(temp_dir=temp_dir, aws_prefix=aws_prefix_path) # os.path.join("folder" , os.path.splitext(pdf_file.name)[0] , unique_folder_name)
+        upload_folder_to_s3(temp_dir, aws_prefix_path)
+        shutil.rmtree(temp_dir)
+
+        return retriever_engine

qdrant_mm_db_pipeline/.lock

@@ -1 +0,0 @@
-tmp lock file

qdrant_mm_db_pipeline/collection/text_collection_pipeline/storage.sqlite

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:abc3d5e877a46fab7aef3c35b28f465d363af0f37e6a7097a4f25b578d941ff7
-size 3084288

qdrant_mm_db_pipeline/meta.json

@@ -1 +0,0 @@
-{"collections": {"text_collection_pipeline": {"vectors": {"size": 1536, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "image_collection_pipeline": {"vectors": {"size": 512, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}}, "aliases": {}}

qdrant_mm_db_pipeline_tmp0uzyg0nb_construction_pdf/.lock

@@ -1 +0,0 @@
-tmp lock file

qdrant_mm_db_pipeline_tmp0uzyg0nb_construction_pdf/collection/image_collection_pipeline_tmp0uzyg0nb_construction_pdf/storage.sqlite

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:716020e47d41d39ea7e3e302ac0687b1a62e2b94991bc700fb5bcbaf6722380f
-size 1585152

qdrant_mm_db_pipeline_tmp0uzyg0nb_construction_pdf/collection/text_collection_pipeline_tmp0uzyg0nb_construction_pdf/storage.sqlite

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c077e65278dfde8d472135f12860bf39fedcf018d44a00d54fc04b2f4eb6afb0
-size 2998272

qdrant_mm_db_pipeline_tmp0uzyg0nb_construction_pdf/meta.json

@@ -1 +0,0 @@
-{"collections": {"text_collection_pipeline_tmp0uzyg0nb_construction_pdf": {"vectors": {"size": 1536, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}, "image_collection_pipeline_tmp0uzyg0nb_construction_pdf": {"vectors": {"size": 512, "distance": "Cosine", "hnsw_config": null, "quantization_config": null, "on_disk": null, "datatype": null, "multivector_config": null}, "shard_number": null, "sharding_method": null, "replication_factor": null, "write_consistency_factor": null, "on_disk_payload": null, "hnsw_config": null, "wal_config": null, "optimizers_config": null, "init_from": null, "quantization_config": null, "sparse_vectors": null}}, "aliases": {}}

requirements.txt

@@ -12,4 +12,5 @@ pillow==10.4.0
 imagehash
 llama-index-embeddings-clip
 git+https://github.com/openai/CLIP.git
-python-dotenv
+python-dotenv
+boto3

retrieve_and_display.py

@@ -0,0 +1,59 @@
+from llama_index.llms.openai import OpenAI
+from llama_index.core import load_index_from_storage, get_response_synthesizer
+import matplotlib.pyplot as plt
+import os
+from PIL import Image
+from llama_index.core import PromptTemplate
+from awsfunctions import download_files_from_s3, check_file_exists_in_s3
+import tempfile, shutil
+import streamlit as st
+
+st.cache_resource()
+def get_image_from_s3(image_path):
+    temp_dir = tempfile.mkdtemp()
+    download_files_from_s3(temp_dir, [image_path])
+    image =  Image.open(os.path.join(temp_dir, image_path))
+    shutil.rmtree(temp_dir)
+    return image
+
+def plot_images(image_paths):
+    images_shown = 0
+    plt.figure(figsize=(16, 9))
+    for img_path in image_paths:
+        if check_file_exists_in_s3(img_path):
+            image = get_image_from_s3(img_path)
+            plt.subplot(2, 3, images_shown + 1)
+            plt.imshow(image)
+            plt.xticks([])
+            plt.yticks([])
+            images_shown += 1
+            if images_shown >= 6:
+                break
+
+def retrieve_and_query(query, retriever_engine):
+    retrieval_results = retriever_engine.retrieve(query)
+    
+    qa_tmpl_str = (
+        "Context information is below.\n"
+        "---------------------\n"
+        "{context_str}\n"
+        "---------------------\n"
+        "Given the context information , "
+        "answer the query in detail.\n"
+        "Query: {query_str}\n"
+        "Answer: "
+    )
+    qa_tmpl = PromptTemplate(qa_tmpl_str)
+
+    llm = OpenAI(model="gpt-4o", temperature=0)
+    response_synthesizer = get_response_synthesizer(response_mode="refine", text_qa_template=qa_tmpl, llm=llm)
+
+    response = response_synthesizer.synthesize(query, nodes=retrieval_results)
+    
+    retrieved_image_path_list = []
+    for node in retrieval_results:
+        if (node.metadata['file_type'] == 'image/jpeg') or (node.metadata['file_type'] == 'image/png'):
+            if node.score > 0.25:
+                retrieved_image_path_list.append(node.metadata['file_path'])
+    
+    return response, retrieved_image_path_list