modified file

README.md

@@ -10,3 +10,42 @@ pinned: false
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+# clip-search
+📸Image Search with Pinecone and CLIP
+
+🎈Overview
+
+♦This project implements an image search engine using OpenAI's CLIP model and Pinecone for vector search. The application allows users to search for images either by text or by uploading an image.
+
+Features
+
+🎈Image-to-Image Search: Upload an image to find visually similar images.
+
+🎈Text-to-Image Search: Enter a text query to find matching images.
+
+🎈Pinecone Integration: Uses Pinecone as a vector database for fast similarity searches.
+
+🎈Streamlit UI: A user-friendly interface for searching images.
+
+🎈Dataset Management: Reads image metadata from a dataset and indexes it into Pinecone.
+
+ ♦Users can upload an image, and the system finds visually similar images from the dataset.
+
+♦The uploaded image is converted into an embedding using CLIP, and similar images are retrieved from Pinecone.
+
+♦Users can enter a text query to find images matching the description.
+
+♦The text is converted into an embedding, and Pinecone retrieves the closest image embeddings.
+
+♦Uses Pinecone as a high-speed vector database.
+
+♦Efficiently stores and retrieves embeddings for similarity searches.
+
+♦Simple web-based interface for performing searches.
+
+♦Supports both text and image-based queries.
+
+♦Reads image metadata from a dataset (TSV file).
+
+♦Extracts image embeddings using CLIP and stores them in Pinecone.

src/frontend/app.py

@@ -6,18 +6,12 @@ from transformers import AutoProcessor, CLIPModel
 import streamlit as st
 from utils import logger
 from database import pinecone_index
-# from model.clip_model import ClipModel
-# from data import data_set
+from PIL import Image
 
 logger = logger.get_logger()
 model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
 processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")
 
-PAGE_TITLE = "Clip Search"
-PAGE_LAYOUT = "wide"
-SIDEBAR_TITLE = "Find Similar Images"
-PHOTO_ID_KEY = "photo_id" 
-IMAGE_URL_KEY = "photo_image_url"
 PINECONE_INDEX =  pinecone_index.create_index()
 
 def search_by_text(query_text, index):
@@ -27,18 +21,51 @@ def search_by_text(query_text, index):
     results = index.query(vector=query_vector, top_k=10, include_metadata=True, namespace="image-search-dataset")
     return results
 
+def search_by_image(image, index):
+    inputs = processor(images=image, return_tensors="pt")
+    image_features = model.get_image_features(**inputs)
+    query_vector = image_features.detach().cpu().numpy().flatten().tolist()
+    results = index.query(vector=query_vector, top_k=5, include_metadata=True, namespace="image-search-dataset")
+    return results
+
 def main():
-    st.set_page_config(page_title=PAGE_TITLE, layout=PAGE_LAYOUT)
+    st.set_page_config(page_title="Clip Search", layout="wide")
     st.title("📸Image Search with Pinecone and CLIP")
     option = st.selectbox("Choose Input Type", ["Text", "Image Upload"])
+    
     if option == "Text":
         user_text = st.text_input("Enter your search text", placeholder = "for eg: dogs or cat etc..")
+        
         if st.button("Search"):
             results = search_by_text(user_text, PINECONE_INDEX)
-            for match in results['matches']:
-                st.image(match['metadata']['url'], caption=f"Match: {match['metadata']['photo_id']}")
+            
+            columns = st.columns(2)
+
+            for idx, match in enumerate(results['matches']):
+                with columns[idx % 2]:
+                    st.image(
+                        match['metadata']['url'], 
+                        caption=f"Match: {match['metadata']['photo_id']}", 
+                        width=500
+                            )
+
+    elif option == "Image Upload":
+        uploaded_file = st.file_uploader("Upload an image", type=["jpg", "png", "jpeg"])
+        if uploaded_file is not None:
+            image = Image.open(uploaded_file)
+            st.image(image, caption="Uploaded Image")
+            if st.button("Search by Image"):
+                results = search_by_image(image, PINECONE_INDEX)
 
+                columns = st.columns(2)
 
+                for idx, match in enumerate(results['matches']):
+                    with columns[idx % 2]:
+                        st.image(
+                            match['metadata']['url'], 
+                            caption=f"Match: {match['metadata']['photo_id']}", 
+                            width=500
+                                )
  
 if __name__ == "__main__":
     main()