Upload 6 files

Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.8.15
+
+WORKDIR /Users/me_teor21/Workspace/item-search
+
+COPY requirements.txt ./
+
+RUN pip install -r requirements.txt 
+
+COPY search.py ./
+
+COPY app.py ./
+
+COPY constants.py ./
+
+# COPY . .
+
+ENTRYPOINT ["python", "app.py"]

README.md

@@ -1,13 +1,53 @@
----
-title: Item Search
-emoji: 🐢
-colorFrom: blue
-colorTo: green
-sdk: gradio
-sdk_version: 4.16.0
-app_file: app.py
-pinned: false
-license: mit
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Shopping Search Engine 
+
+## Description
+
+Look for the ideal clothing items 😎
+
+## Instructions
+
+1. Install libraries
+
+```
+pip install -r requirements.txt
+```
+
+2. Run
+
+```
+python app.py
+```
+
+## Build and run container
+
+1. Build container (uncomment launch call in app.py)
+
+```
+docker build --tag item-search .
+```
+
+2. Run container
+
+```
+docker run -it -d --name item-search-engine -p 7000:7000  item-search:latest
+```
+
+## Structure
+
+```
+.
+├── app.py
+├── Dockerfile
+├── LICENSE
+├── README.md
+├── search.py
+└── requirements.txt
+```
+
+## Author
+
+[Ismael C.](https://ismaelmekene.com)
+
+## License
+
+Licensed under the MIT License, Version 2.0. 

app.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*- 
+
+import os
+from pinecone import Pinecone, ServerlessSpec
+from pinecone_text.sparse import BM25Encoder
+from datasets import load_dataset
+from sentence_transformers import SentenceTransformer
+import torch
+from io import BytesIO
+from base64 import b64encode
+from tqdm.auto import tqdm
+from PIL import Image
+import gradio as gr
+from constants import *
+
+from search import SearchItem
+
+from fastapi import FastAPI
+
+
+
+
+
+
+# initialize connection to pinecone (get API key at app.pinecone.io)
+api_key = PINECONE_API_KEY or os.getenv(PINECONE_API_KEY) # or "PINECONE_API_KEY"
+# find your environment next to the api key in pinecone console
+env = PINECONE_ENVIRONMENT or os.getenv(PINECONE_ENVIRONMENT) # or "PINECONE_ENVIRONMENT"
+
+fashion_processor = SearchItem(api_key, env)
+    
+
+def retrieve_images(query, image=None):
+    if image:
+        # If image is provided, use retrieve_image_from_image function
+        return retrieve_image_from_image(image, query)
+    else:
+        # If image is not provided, use retrieve_image_from_query function
+        return retrieve_image_from_query(query)
+    
+
+
+def retrieve_image_from_query(query):
+
+    # create sparse and dense vectors
+    sparse = fashion_processor.bm25.encode_queries(query)
+    dense = fashion_processor.clip_model.encode(query).tolist()
+    hdense, hsparse = fashion_processor.hybrid_scale(dense, sparse)
+
+    result = fashion_processor.index.query(
+        top_k=10,
+        vector=hdense,
+        sparse_vector=hsparse,
+        include_metadata=True
+    )
+
+    imgs = [fashion_processor.images[int(r["id"])] for r in result["matches"]]
+
+    return imgs
+
+
+def retrieve_image_from_image(image, query):
+
+    try:
+        # create sparse and dense vectors
+        sparse = fashion_processor.bm25.encode_queries(query)
+        w, h = 60, 80
+        image = Image.open(image.name).resize((w, h))
+        dense = fashion_processor.clip_model.encode(image).tolist()
+        hdense, hsparse = fashion_processor.hybrid_scale(dense, sparse)
+
+
+        result = fashion_processor.index.query(
+            top_k=10,
+            vector=hdense,
+            sparse_vector=hsparse,
+            include_metadata=True
+        )
+
+        imgs = [fashion_processor.images[int(r["id"])] for r in result["matches"]]
+
+        return imgs
+    
+    except Exception as e:
+        # print(f"Error processing image: {e}")
+        print(e)
+        return None
+
+
+
+def show_img(image):
+    return image.name if image else "No image provided"
+
+
+with gr.Blocks() as demo:
+    gr.Markdown(
+    """
+    # Shopping Search Engine
+    
+    Look for the ideal clothing items 😎
+    """)
+    
+    with gr.Row():
+        with gr.Column():
+
+            query = gr.Textbox(placeholder="Search Items")
+            gr.HTML("OR")
+            photo = gr.Image()
+            with gr.Row():
+                button = gr.UploadButton(label="Upload Image", file_types=["image"])
+                textbox = gr.Textbox(placeholder="Additional Details ?")
+                submit_button = gr.Button(text="Submit")
+
+        with gr.Column():
+            gallery = gr.Gallery().style(
+                object_fit='contain',
+                height='auto',
+                preview=True
+            )
+
+    query.submit(fn=lambda query: retrieve_images(query), inputs=[query], outputs=[gallery])
+    submit_button.click(fn=lambda image, query: show_img(image), inputs=[button, textbox], outputs=[photo]) \
+        .then(fn=lambda image, query: retrieve_images(query, image), inputs=[button, textbox], outputs=[gallery])
+
+if __name__ == "__main__":
+    demo.launch(server_name="0.0.0.0", server_port=8000)

constants.py

@@ -0,0 +1,2 @@
+PINECONE_API_KEY = '810e1b45-1489-41a8-998e-1ed0fb2d21a5'
+PINECONE_ENVIRONMENT = 'gcp-starter'

requirements.txt

@@ -0,0 +1,10 @@
+datasets
+transformers
+sentence-transformers
+huggingface-hub
+pinecone-client
+pinecone-text
+protobuf==3.20.3
+gradio==3.41.2
+fastapi
+uvicorn==0.23.1

search.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*- 
+
+import os
+from pinecone import Pinecone, ServerlessSpec
+from pinecone_text.sparse import BM25Encoder
+from datasets import load_dataset
+from sentence_transformers import SentenceTransformer
+import torch
+from io import BytesIO
+from base64 import b64encode
+from tqdm.auto import tqdm
+from constants import *
+
+
+
+# initialize connection to pinecone (get API key at app.pinecone.io)
+api_key = PINECONE_API_KEY or os.getenv(PINECONE_API_KEY) # or "PINECONE_API_KEY"
+# find your environment next to the api key in pinecone console
+env = PINECONE_ENVIRONMENT or os.getenv(PINECONE_ENVIRONMENT) # or "PINECONE_ENVIRONMENT"
+
+
+
+class SearchItem():
+    def __init__(self, api_key=None, env=None, device='cuda' if torch.cuda.is_available() else 'cpu'):
+        self.api_key = api_key 
+        self.environment = env 
+        self.pinecone_instance = self.connect_to_pinecone(self.api_key,self.environment)
+        self.index = self.pinecone_instance.Index('clip')
+        self.images, self.metadata = self.load_fashion_dataset()
+        self.clip_model = self.initialize_clip_model(device=device)
+        self.bm25 = self.initialize_bm25_encoder(self.metadata)
+
+
+
+    def connect_to_pinecone(self, api_key, env):
+        api_key = api_key or os.getenv('PINECONE_API_KEY')
+        env = env or os.getenv('PINECONE_ENVIRONMENT')
+        
+        if not api_key or not env:
+            raise ValueError("Pinecone API key and environment are required.")
+        
+        pinecone_instance = Pinecone(api_key=api_key, environment=env)
+        return pinecone_instance
+    
+    def load_fashion_dataset(self):
+        fashion = load_dataset("ashraq/fashion-product-images-small", split="train")
+        images = fashion["image"]
+        metadata = fashion.remove_columns("image").to_pandas()
+        return images, metadata
+    
+    def initialize_clip_model(self, device='cuda' if torch.cuda.is_available() else 'cpu'):
+        model = SentenceTransformer('sentence-transformers/clip-ViT-B-32', device=device)
+        return model
+    
+    def initialize_bm25_encoder(self, metadata):
+        bm25 = BM25Encoder()
+        bm25.fit(metadata['productDisplayName'])
+        return bm25
+    
+    @staticmethod
+    def hybrid_scale(dense, sparse, alpha=0.05):
+        """Hybrid vector scaling using a convex combination
+
+        alpha * dense + (1 - alpha) * sparse
+
+        Args:
+            dense: Array of floats representing
+            sparse: a dict of `indices` and `values`
+            alpha: float between 0 and 1 where 0 == sparse only
+                and 1 == dense only
+        """
+        if alpha < 0 or alpha > 1:
+            raise ValueError("Alpha must be between 0 and 1")
+        
+        # Scale sparse and dense vectors to create hybrid search vectors
+        hsparse = {
+            'indices': sparse['indices'],
+            'values': [v * (1 - alpha) for v in sparse['values']]
+        }
+        hdense = [v * alpha for v in dense]
+        
+        return hdense, hsparse
+    
+
+if __name__ == "__main__":
+
+
+    fashion_processor = SearchItem(api_key, env)
+    
+    query = "blue shoes"
+    # create sparse and dense vectors
+    sparse = fashion_processor.bm25.encode_queries(query)
+    dense = fashion_processor.clip_model.encode(query).tolist()
+
+    hdense, hsparse = fashion_processor.hybrid_scale(dense, sparse)
+
+    result = fashion_processor.index.query(
+        top_k=5,
+        vector=hdense,
+        sparse_vector=hsparse,
+        include_metadata=True
+    )
+
+    imgs = [fashion_processor.images[int(r["id"])] for r in result["matches"]]
+
+    print('Ok')
+    # breakpoint()