import gradio as gr import numpy as np import torch import os from PIL import Image from datasets import load_dataset from transformers import CLIPProcessor, CLIPModel # ── 1. Configuration ─────────────────────────────────────── MODEL_ID = "openai/clip-vit-base-patch32" DATASET_ID = "prithivMLmods/Shoe-Net-10K" TOP_K = 3 # ── 2. Load model ────────────────────────────────────────── print("Loading CLIP model...") device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = CLIPModel.from_pretrained(MODEL_ID).to(device) processor = CLIPProcessor.from_pretrained(MODEL_ID) model.eval() print(f"Model loaded on {device}") # ── 3. Load dataset ──────────────────────────────────────── print("Loading shoe dataset...") dataset = load_dataset(DATASET_ID, split="train") print(f"Dataset loaded: {len(dataset)} images") # ── 4. Load pre-computed embeddings (.npy arrays) ────────── print("Loading pre-computed embeddings from npy artifacts...") embeddings_matrix = np.load("shoe_embeddings.npy").astype(np.float32) image_indices = np.load("dataset_image_indices.npy") # Ensure image indices are fully flattened and cast to python integers image_indices = np.atleast_1d(image_indices).flatten().astype(int) cluster_names_map = { "0": "Pure White & Ultra Light Sneaker Profile", "1": "Jet Black & Deep Dark Footwear Profile", "2": "Warm Tones, Earthy Tan & Luxury Browns", "3": "Muted Grey, Pastels & Mid-Tone Casuals", "4": "Vibrant, Chromatic & Colorful Selection" } print(f"Loaded {len(embeddings_matrix)} embeddings of dim {embeddings_matrix.shape[1]}") # ── 5. Embedding functions ───────────────────────────────── def embed_image(pil_image: Image.Image) -> np.ndarray: img = pil_image.convert('RGB') inputs = processor(images=img, return_tensors="pt").to(device) with torch.no_grad(): outputs = model.get_image_features(**inputs) features = outputs if isinstance(outputs, torch.Tensor) else outputs[0] features = features / features.norm(dim=-1, keepdim=True) return features.cpu().numpy().flatten() def embed_text(text: str) -> np.ndarray: inputs = processor(text=[text], return_tensors="pt", padding=True).to(device) with torch.no_grad(): outputs = model.get_text_features(**inputs) features = outputs if isinstance(outputs, torch.Tensor) else outputs[0] features = features / features.norm(dim=-1, keepdim=True) return features.cpu().numpy().flatten() # ── 6. Recommendation engine ─────────────────────────────── def recommend(query_emb: np.ndarray) -> list: sims = np.dot(embeddings_matrix, query_emb.reshape(-1, 1)).flatten() sorted_idx = np.argsort(sims)[::-1] results = [] for i in sorted_idx: if sims[i] > 0.9999: continue try: dataset_idx = int(image_indices[i]) if dataset_idx < 0 or dataset_idx >= len(dataset): continue img_data = dataset[dataset_idx]['image'] if not isinstance(img_data, Image.Image): img = Image.open(img_data) else: img = img_data cluster_id = int(i % 5) cluster_name = cluster_names_map.get(str(cluster_id), f"Style Profile {cluster_id}") results.append({ "image": img, "score": float(sims[i]), "rank": len(results) + 1, "index": dataset_idx, "cluster": cluster_name }) except Exception: continue if len(results) == TOP_K: break return results # ── 7. Gradio handler functions ──────────────────────────── def recommend_by_image(query_image): if query_image is None: return (None, "Upload an image first", None, "Upload an image first", None, "Upload an image first") try: query_emb = embed_image(query_image) results = recommend(query_emb) outputs = [] for r in results: outputs.append(r["image"]) outputs.append( f"Rank #{r['rank']}\n" f"Similarity: {r['score']:.4f}\n" f"Style: {r['cluster']}\n" f"Dataset Index: {r['index']}" ) while len(outputs) < 6: outputs.extend([None, "No additional matches found"]) return tuple(outputs[:6]) except Exception as e: err = f"Error during image recommendation: {str(e)}" return None, err, None, err, None, err def recommend_by_text(query_text): if not query_text or not query_text.strip(): return (None, "Enter a text description first", None, "Enter a text description first", None, "Enter a text description first") try: query_emb = embed_text(query_text.strip()) results = recommend(query_emb) outputs = [] for r in results: outputs.append(r["image"]) outputs.append( f"Rank #{r['rank']}\n" f"Similarity: {r['score']:.4f}\n" f"Style: {r['cluster']}\n" f"Dataset Index: {r['index']}" ) while len(outputs) < 6: outputs.extend([None, "No additional matches found"]) return tuple(outputs[:6]) except Exception as e: err = f"Error during text recommendation: {str(e)}" return None, err, None, err, None, err # ── Pre-build example images as files (avoids loading PIL objects directly into gr.Examples) ── EXAMPLES_DIR = "example_images" os.makedirs(EXAMPLES_DIR, exist_ok=True) example_image_paths = [] for idx in [0, 100, 500, 1000, 2500]: try: img = dataset[idx]['image'] if not isinstance(img, Image.Image): img = Image.open(img) path = os.path.join(EXAMPLES_DIR, f"example_{idx}.jpg") img.convert("RGB").save(path) example_image_paths.append(path) except Exception as e: print(f"Could not prepare example image {idx}: {e}") # ── 8. Gradio UI ─────────────────────────────────────────── with gr.Blocks( theme=gr.themes.Soft(primary_hue="blue"), title="Shoe Visual Recommender", css=""" .title-text { text-align: center; } .result-box { background: #f0f7ff; border-radius: 8px; padding: 8px; } """ ) as demo: gr.Markdown("""
# Shoe Visual Recommender ### AI-powered shoe recommendation using CLIP embeddings + Cosine Similarity
""") gr.Markdown(""" **How it works:** 1. Upload a shoe image **or** type a text description 2. CLIP converts your input to a 512-dimensional embedding vector 3. Cosine similarity finds the 3 most similar shoes from 10,000 options 4. Results are ranked by similarity score --- """) with gr.Tabs(): # ── Tab 1: Image Input ── with gr.TabItem("Search by Image"): gr.Markdown("### Upload a shoe image to find visually similar shoes") with gr.Row(): with gr.Column(scale=1): img_input = gr.Image( type="pil", label="Upload Shoe Image", height=280 ) img_btn = gr.Button( "Find Similar Shoes", variant="primary", size="lg" ) if example_image_paths: gr.Examples( examples=[[p] for p in example_image_paths], inputs=[img_input], label="Example Shoes from Dataset" ) with gr.Column(scale=3): gr.Markdown("### Top 3 Most Similar Shoes") with gr.Row(): img_out1 = gr.Image(label="Best Match", height=200) img_out2 = gr.Image(label="2nd Match", height=200) img_out3 = gr.Image(label="3rd Match", height=200) with gr.Row(): img_info1 = gr.Textbox(label="Match #1 Details", lines=4, interactive=False) img_info2 = gr.Textbox(label="Match #2 Details", lines=4, interactive=False) img_info3 = gr.Textbox(label="Match #3 Details", lines=4, interactive=False) img_btn.click( fn=recommend_by_image, inputs=[img_input], outputs=[img_out1, img_info1, img_out2, img_info2, img_out3, img_info3] ) # ── Tab 2: Text Input ── with gr.TabItem("Search by Text"): gr.Markdown("### Describe the shoe you're looking for") with gr.Row(): with gr.Column(scale=1): text_input = gr.Textbox( label="Describe a Shoe", placeholder="e.g. 'black elegant flat ballet shoe'", lines=3 ) text_btn = gr.Button( "Find Matching Shoes", variant="primary", size="lg" ) gr.Examples( examples=[ ["black elegant flat ballet shoe"], ["casual white sneaker with bow"], ["brown leather oxford formal shoe"], ["colorful summer sandal"], ["dark pointed toe heels"] ], inputs=[text_input], label="Example Queries" ) with gr.Column(scale=3): gr.Markdown("### Top 3 Matching Shoes") with gr.Row(): txt_out1 = gr.Image(label="Best Match", height=200) txt_out2 = gr.Image(label="2nd Match", height=200) txt_out3 = gr.Image(label="3rd Match", height=200) with gr.Row(): txt_info1 = gr.Textbox(label="Match #1 Details", lines=4, interactive=False) txt_info2 = gr.Textbox(label="Match #2 Details", lines=4, interactive=False) txt_info3 = gr.Textbox(label="Match #3 Details", lines=4, interactive=False) text_btn.click( fn=recommend_by_text, inputs=[text_input], outputs=[txt_out1, txt_info1, txt_out2, txt_info2, txt_out3, txt_info3] ) # ── Tab 3: About ── with gr.TabItem("About This Project"): gr.Markdown(""" ## About This Shoe Recommender ### Dataset - **Source:** [Shoe-Net-10K](https://huggingface.co/datasets/prithivMLmods/Shoe-Net-10K) - **Size:** 10,000 shoe images across 5 categories - **Balance:** Perfectly balanced — 2,000 images per category ### Model - **Model:** [CLIP ViT-B/32](https://huggingface.co/openai/clip-vit-base-patch32) by OpenAI - **Embedding dim:** 512 - **Why CLIP?** Supports both image AND text queries in the same embedding space ### Pipeline 1. All 10,000 shoe images → CLIP → 512-dim normalized embeddings 2. Augmented with pixel features (brightness, RGB) for clustering 3. K-Means (K=5) clustering on augmented 516-dim features 4. User query → CLIP → cosine similarity → Top-3 results ### Clustering Results - Algorithm: K-Means (K=5) - Evaluation: Silhouette Score = 0.29 (K=5) - Visualization: t-SNE + UMAP """) gr.Markdown(""" --- **Model:** CLIP ViT-B/32 | **Dataset:** Shoe-Net-10K (10K images) | **Method:** Cosine Similarity """) demo.launch()