Upload evaluation/evaluate_color_embeddings.py with huggingface_hub

evaluation/evaluate_color_embeddings.py

@@ -0,0 +1,1124 @@
+"""
+Comprehensive evaluation of color embeddings with Fashion-CLIP comparison.
+This file evaluates the quality of color embeddings generated by the ColorCLIP model
+by calculating intra-class and inter-class similarity metrics, classification accuracies,
+and generating confusion matrices. It also compares results with Fashion-CLIP as a baseline
+to measure relative performance.
+"""
+
+import torch
+import torch.nn as nn
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from sklearn.metrics.pairwise import cosine_similarity
+from sklearn.metrics import confusion_matrix, classification_report, accuracy_score
+from collections import defaultdict
+import os
+import json
+from tqdm import tqdm
+from torch.utils.data import Dataset, DataLoader
+from torchvision import transforms
+import requests
+from io import BytesIO
+from PIL import Image
+import warnings
+warnings.filterwarnings('ignore')
+from color_model import ColorCLIP, Tokenizer, ImageEncoder, TextEncoder, collate_batch
+from torch.utils.data import DataLoader
+from transformers import CLIPProcessor, CLIPModel as TransformersCLIPModel
+import config
+
+class ColorDataset(Dataset):
+    """
+    Dataset class for color embedding evaluation.
+    
+    Handles loading images from various sources (local paths, URLs, bytes) and
+    applying appropriate transformations for evaluation.
+    """
+    def __init__(self, dataframe):
+        """
+        Initialize the color dataset.
+        
+        Args:
+            dataframe: DataFrame containing image paths/URLs, text, and color labels
+        """
+        self.dataframe = dataframe
+        self.transform = transforms.Compose([
+            transforms.Resize((224, 224)),
+            transforms.ToTensor(),
+            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+        ])
+
+    def __len__(self):
+        return len(self.dataframe)
+
+    def __getitem__(self, idx):
+        row = self.dataframe.iloc[idx]
+        
+        # Handle image - it should be in row[config.column_url_image] and contain the image data
+        image_data = row[config.column_url_image]
+        
+        try:
+            # Check if image_data has 'bytes' key or is already PIL Image
+            if isinstance(image_data, dict) and 'bytes' in image_data:
+                image = Image.open(BytesIO(image_data['bytes'])).convert("RGB")
+            elif hasattr(image_data, 'convert'):  # Already a PIL Image
+                image = image_data.convert("RGB")
+            elif isinstance(image_data, str):
+                # It's a file path (local or URL)
+                if image_data.startswith('http'):
+                    # It's a URL - download the image
+                    response = requests.get(image_data, timeout=10)
+                    response.raise_for_status()
+                    image = Image.open(BytesIO(response.content)).convert("RGB")
+                else:
+                    # It's a local file path
+                    image = Image.open(image_data).convert("RGB")
+            else:
+                # Assume it's bytes data
+                image = Image.open(BytesIO(image_data)).convert("RGB")
+            
+            # Apply transform
+            image = self.transform(image)
+
+        except Exception as e:
+            print(f"⚠️ Failed to load image {idx}: {e}")
+            # Return a placeholder image
+            image = torch.zeros(3, 224, 224)
+
+        # Get text and color
+        description = row[config.text_column]  
+        color = row[config.color_column]
+
+        return image, description, color
+
+class EmbeddingEvaluator:
+    """
+    Evaluator for color embeddings generated by the ColorCLIP model.
+    
+    This class provides methods to evaluate the quality of color embeddings by computing
+    similarity metrics, classification accuracies, and generating visualizations.
+    """
+    
+    def __init__(self, model_path, embed_dim):
+        """
+        Initialize the embedding evaluator.
+        
+        Args:
+            model_path: Path to the trained ColorCLIP model checkpoint
+            embed_dim: Embedding dimension for the model
+        """
+        self.device = config.device
+        
+        # Initialize tokenizer first to get vocab size
+        self.tokenizer = Tokenizer()
+        vocab_size = None
+        
+        # Load vocabulary if available to determine vocab_size
+        if os.path.exists(config.tokeniser_path):
+            with open(config.tokeniser_path, 'r') as f:
+                vocab_dict = json.load(f)
+                # Manually load vocabulary
+                self.tokenizer.word2idx = defaultdict(lambda: 0, {k: int(v) for k, v in vocab_dict.items()})
+                self.tokenizer.idx2word = {int(v): k for k, v in vocab_dict.items() if int(v) > 0}
+                self.tokenizer.counter = max(self.tokenizer.word2idx.values(), default=0) + 1
+                vocab_size = self.tokenizer.counter
+                print(f"Tokenizer vocabulary loaded from {config.tokeniser_path}")
+        else:
+            print(f"Warning: {config.tokeniser_path} not found. Using default tokenizer.")
+        
+        # Load checkpoint to get vocab_size and state_dict
+        checkpoint = None
+        if os.path.exists(model_path):
+            checkpoint = torch.load(model_path, map_location=self.device)
+            
+            # Try to get vocab_size from model checkpoint if not already determined
+            if vocab_size is None:
+                # Try to get vocab_size from metadata
+                if isinstance(checkpoint, dict) and 'vocab_size' in checkpoint:
+                    vocab_size = checkpoint['vocab_size']
+                # Otherwise, try to infer from model state dict
+                elif isinstance(checkpoint, dict) and 'model_state_dict' in checkpoint:
+                    state_dict = checkpoint['model_state_dict']
+                    if 'text_encoder.embedding.weight' in state_dict:
+                        vocab_size = state_dict['text_encoder.embedding.weight'].shape[0]
+                elif isinstance(checkpoint, dict) and 'text_encoder.embedding.weight' in checkpoint:
+                    vocab_size = checkpoint['text_encoder.embedding.weight'].shape[0]
+        
+        # Fallback to default if still not determined
+        if vocab_size is None:
+            vocab_size = 39  # Default fallback
+            print(f"Warning: Could not determine vocab_size, using default: {vocab_size}")
+        
+        # Initialize model with determined vocab_size
+        self.model = ColorCLIP(vocab_size=vocab_size, embedding_dim=embed_dim).to(self.device)
+        
+        # Load trained model state dict
+        if checkpoint is not None:
+            state_dict = checkpoint.get('model_state_dict', checkpoint)
+            self.model.load_state_dict(state_dict)
+            print(f"Model loaded from {model_path}")
+        else:
+            print(f"Warning: Model file {model_path} not found. Using untrained model.")
+        
+        self.model.eval()
+    
+    def extract_embeddings(self, dataloader, embedding_type='text'):
+        """
+        Extract embeddings from the model for a given dataloader.
+        
+        Args:
+            dataloader: DataLoader containing images, texts, and colors
+            embedding_type: Type of embeddings to extract ('text', 'image', or 'color')
+            
+        Returns:
+            Tuple of (embeddings array, labels list, texts list)
+        """
+        all_embeddings = []
+        all_labels = []
+        all_texts = []
+        
+        with torch.no_grad():
+            for images, texts, colors in tqdm(dataloader, desc=f"Extracting {embedding_type} embeddings"):
+                if embedding_type == 'text':
+                    # Tokenize texts using the tokenizer
+                    tokenized_texts = [self.tokenizer(text) for text in texts]
+                    # Convert to tensors and pad sequences
+                    text_tensors = [torch.tensor(t, dtype=torch.long) for t in tokenized_texts]
+                    text_tokens = nn.utils.rnn.pad_sequence(text_tensors, batch_first=True, padding_value=0).to(self.device)
+                    lengths = torch.tensor([len(t) for t in tokenized_texts], dtype=torch.long).to(self.device)
+                    embeddings = self.model.text_encoder(text_tokens, lengths)
+                    labels = colors
+                elif embedding_type == 'image':
+                    images = images.to(self.device)
+                    embeddings = self.model.image_encoder(images)
+                    labels = colors
+                elif embedding_type == 'color':
+                    # Tokenize color names using the tokenizer
+                    tokenized_colors = [self.tokenizer(color) for color in colors]
+                    # Convert to tensors and pad sequences
+                    color_tensors = [torch.tensor(t, dtype=torch.long) for t in tokenized_colors]
+                    color_tokens = nn.utils.rnn.pad_sequence(color_tensors, batch_first=True, padding_value=0).to(self.device)
+                    lengths = torch.tensor([len(t) for t in tokenized_colors], dtype=torch.long).to(self.device)
+                    embeddings = self.model.text_encoder(color_tokens, lengths)
+                    labels = colors
+                
+                all_embeddings.append(embeddings.cpu().numpy())
+                all_labels.extend(labels)
+                all_texts.extend(texts)
+        
+        return np.vstack(all_embeddings), all_labels, all_texts
+    
+    def compute_similarity_metrics(self, embeddings, labels):
+        """Compute intra-class and inter-class similarities"""
+        similarities = cosine_similarity(embeddings)
+        
+        # Group embeddings by color
+        color_groups = defaultdict(list)
+        for i, color in enumerate(labels):
+            color_groups[color].append(i)
+        
+        # Calculate intra-class similarities (same color)
+        intra_class_similarities = []
+        for color, indices in color_groups.items():
+            if len(indices) > 1:
+                for i in range(len(indices)):
+                    for j in range(i+1, len(indices)):
+                        sim = similarities[indices[i], indices[j]]
+                        intra_class_similarities.append(sim)
+        
+        # Calculate inter-class similarities (different colors)
+        inter_class_similarities = []
+        colors = list(color_groups.keys())
+        for i in range(len(colors)):
+            for j in range(i+1, len(colors)):
+                color1_indices = color_groups[colors[i]]
+                color2_indices = color_groups[colors[j]]
+                
+                for idx1 in color1_indices:
+                    for idx2 in color2_indices:
+                        sim = similarities[idx1, idx2]
+                        inter_class_similarities.append(sim)
+        
+        # Calculate classification accuracy using nearest neighbor in embedding space
+        nn_accuracy = self.compute_embedding_accuracy(embeddings, labels, similarities)
+        
+        # Calculate classification accuracy using centroids
+        centroid_accuracy = self.compute_centroid_accuracy(embeddings, labels)
+        
+        return {
+            'intra_class_similarities': intra_class_similarities,
+            'inter_class_similarities': inter_class_similarities,
+            'intra_class_mean': np.mean(intra_class_similarities) if intra_class_similarities else 0,
+            'inter_class_mean': np.mean(inter_class_similarities) if inter_class_similarities else 0,
+            'separation_score': np.mean(intra_class_similarities) - np.mean(inter_class_similarities) if intra_class_similarities and inter_class_similarities else 0,
+            'accuracy': nn_accuracy,
+            'centroid_accuracy': centroid_accuracy
+        }
+    
+    def compute_embedding_accuracy(self, embeddings, labels, similarities):
+        """Compute classification accuracy using nearest neighbor in embedding space"""
+        correct_predictions = 0
+        total_predictions = len(labels)
+        
+        for i in range(len(embeddings)):
+            true_label = labels[i]
+            
+            # Find the most similar embedding (excluding itself)
+            similarities_row = similarities[i].copy()
+            similarities_row[i] = -1  # Exclude self-similarity
+            nearest_neighbor_idx = np.argmax(similarities_row)
+            predicted_label = labels[nearest_neighbor_idx]
+            
+            if predicted_label == true_label:
+                correct_predictions += 1
+        
+        return correct_predictions / total_predictions if total_predictions > 0 else 0
+    
+    def compute_centroid_accuracy(self, embeddings, labels):
+        """Compute classification accuracy using color centroids"""
+        # Create centroids for each color
+        unique_colors = list(set(labels))
+        centroids = {}
+        
+        for color in unique_colors:
+            color_indices = [i for i, label in enumerate(labels) if label == color]
+            color_embeddings = embeddings[color_indices]
+            centroids[color] = np.mean(color_embeddings, axis=0)
+        
+        # Classify each embedding to nearest centroid
+        correct_predictions = 0
+        total_predictions = len(labels)
+        
+        for i, embedding in enumerate(embeddings):
+            true_label = labels[i]
+            
+            # Find closest centroid
+            best_similarity = -1
+            predicted_label = None
+            
+            for color, centroid in centroids.items():
+                similarity = cosine_similarity([embedding], [centroid])[0][0]
+                if similarity > best_similarity:
+                    best_similarity = similarity
+                    predicted_label = color
+            
+            if predicted_label == true_label:
+                correct_predictions += 1
+        
+        return correct_predictions / total_predictions if total_predictions > 0 else 0
+    
+    def predict_colors_from_embeddings(self, embeddings, labels):
+        """Predict colors from embeddings using centroid-based classification"""
+        # Create color centroids from training data
+        unique_colors = list(set(labels))
+        centroids = {}
+        
+        for color in unique_colors:
+            color_indices = [i for i, label in enumerate(labels) if label == color]
+            color_embeddings = embeddings[color_indices]
+            centroids[color] = np.mean(color_embeddings, axis=0)
+        
+        # Predict colors for all embeddings
+        predictions = []
+        
+        for i, embedding in enumerate(embeddings):
+            # Find closest centroid
+            best_similarity = -1
+            predicted_color = None
+            
+            for color, centroid in centroids.items():
+                similarity = cosine_similarity([embedding], [centroid])[0][0]
+                if similarity > best_similarity:
+                    best_similarity = similarity
+                    predicted_color = color
+            
+            predictions.append(predicted_color)
+        
+        return predictions
+    
+    def create_confusion_matrix(self, true_labels, predicted_labels, title="Confusion Matrix"):
+        """Create and plot confusion matrix"""
+        # Get unique labels
+        unique_labels = sorted(list(set(true_labels + predicted_labels)))
+        
+        # Create confusion matrix
+        cm = confusion_matrix(true_labels, predicted_labels, labels=unique_labels)
+        
+        # Calculate accuracy
+        accuracy = accuracy_score(true_labels, predicted_labels)
+        
+        # Plot confusion matrix
+        plt.figure(figsize=(12, 10))
+        sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', 
+                   xticklabels=unique_labels, yticklabels=unique_labels)
+        plt.title(f'{title}\nAccuracy: {accuracy:.3f} ({accuracy*100:.1f}%)')
+        plt.ylabel('True Color')
+        plt.xlabel('Predicted Color')
+        plt.xticks(rotation=45)
+        plt.yticks(rotation=0)
+        plt.tight_layout()
+        
+        return plt.gcf(), accuracy, cm
+    
+    def evaluate_classification_performance(self, embeddings, labels, embedding_type="Embeddings"):
+        """Evaluate classification performance and create confusion matrix"""
+        # Predict colors
+        predictions = self.predict_colors_from_embeddings(embeddings, labels)
+        
+        # Calculate accuracy
+        accuracy = accuracy_score(labels, predictions)
+        
+        # Create confusion matrix
+        fig, acc, cm = self.create_confusion_matrix(labels, predictions, 
+                                                   f"{embedding_type} - Color Classification")
+        
+        # Generate classification report
+        unique_labels = sorted(list(set(labels)))
+        report = classification_report(labels, predictions, labels=unique_labels, 
+                                     target_names=unique_labels, output_dict=True)
+        
+        return {
+            'accuracy': accuracy,
+            'predictions': predictions,
+            'confusion_matrix': cm,
+            'classification_report': report,
+            'figure': fig
+        }
+    
+    def evaluate_dataset(self, dataframe, dataset_name="Dataset"):
+        """
+        Evaluate embeddings on a given dataset.
+        
+        This method extracts embeddings for text, image, and color, computes similarity metrics,
+        evaluates classification performance, and saves confusion matrices.
+        
+        Args:
+            dataframe: DataFrame containing the dataset
+            dataset_name: Name of the dataset for display purposes
+            
+        Returns:
+            Dictionary containing evaluation results for text, image, and color embeddings
+        """
+        print(f"\n{'='*60}")
+        print(f"Evaluating {dataset_name}")
+        print(f"{'='*60}")
+        
+        # Create dataset and dataloader - use KaglDataset for kagl data
+        if "kagl" in dataset_name.lower():
+            dataset = KaglDataset(dataframe)
+        else:
+            dataset = ColorDataset(dataframe)
+        # Optimize batch size and workers for faster processing
+        dataloader = DataLoader(dataset, batch_size=64, shuffle=False, num_workers=4, pin_memory=True)
+        
+        results = {}
+        
+        # Evaluate text embeddings
+        text_embeddings, text_labels, texts = self.extract_embeddings(dataloader, 'text')
+        text_metrics = self.compute_similarity_metrics(text_embeddings, text_labels)
+        text_classification = self.evaluate_classification_performance(text_embeddings, text_labels, "Text Embeddings")
+        text_metrics.update(text_classification)
+        results['text'] = text_metrics
+        
+        # Evaluate image embeddings
+        image_embeddings, image_labels, _ = self.extract_embeddings(dataloader, 'image')
+        image_metrics = self.compute_similarity_metrics(image_embeddings, image_labels)
+        image_classification = self.evaluate_classification_performance(image_embeddings, image_labels, "Image Embeddings")
+        image_metrics.update(image_classification)
+        results['image'] = image_metrics
+        
+        # Evaluate color embeddings
+        color_embeddings, color_labels, _ = self.extract_embeddings(dataloader, 'color')
+        color_metrics = self.compute_similarity_metrics(color_embeddings, color_labels)
+        color_classification = self.evaluate_classification_performance(color_embeddings, color_labels, "Color Embeddings")
+        color_metrics.update(color_classification)
+        results['color'] = color_metrics
+        
+        # Print results
+        print(f"\n{dataset_name} Results:")
+        print("-" * 40)
+        for emb_type, metrics in results.items():
+            print(f"{emb_type.capitalize()} Embeddings:")
+            print(f"  Intra-class similarity (same color): {metrics['intra_class_mean']:.4f}")
+            print(f"  Inter-class similarity (diff colors): {metrics['inter_class_mean']:.4f}")
+            print(f"  Separation score: {metrics['separation_score']:.4f}")
+            print(f"  Nearest Neighbor Accuracy: {metrics['accuracy']:.4f} ({metrics['accuracy']*100:.1f}%)")
+            print(f"  Centroid Accuracy: {metrics['centroid_accuracy']:.4f} ({metrics['centroid_accuracy']*100:.1f}%)")
+            
+            # Classification report summary
+            report = metrics['classification_report']
+            print(f"  📊 Classification Performance:")
+            print(f"    • Macro Avg F1-Score: {report['macro avg']['f1-score']:.4f}")
+            print(f"    • Weighted Avg F1-Score: {report['weighted avg']['f1-score']:.4f}")
+            print(f"    • Support: {report['macro avg']['support']:.0f} samples")
+            print()
+        
+        # Create visualizations
+        os.makedirs('embedding_evaluation', exist_ok=True)
+        
+        # Confusion matrices
+        results['text']['figure'].savefig(f'embedding_evaluation/{dataset_name.lower()}_text_confusion_matrix.png', dpi=300, bbox_inches='tight')
+        plt.close(results['text']['figure'])
+        
+        results['image']['figure'].savefig(f'embedding_evaluation/{dataset_name.lower()}_image_confusion_matrix.png', dpi=300, bbox_inches='tight')
+        plt.close(results['image']['figure'])
+        
+        results['color']['figure'].savefig(f'embedding_evaluation/{dataset_name.lower()}_color_confusion_matrix.png', dpi=300, bbox_inches='tight')
+        plt.close(results['color']['figure'])
+        
+        return results
+
+class FashionCLIPDataset(Dataset):
+    """
+    Special dataset for Fashion-CLIP that doesn't normalize images.
+    
+    This dataset is used when evaluating with Fashion-CLIP baseline model,
+    which requires different image preprocessing (no normalization).
+    """
+    def __init__(self, dataframe):
+        """
+        Initialize the Fashion-CLIP dataset.
+        
+        Args:
+            dataframe: DataFrame containing image paths/URLs, text, and color labels
+        """
+        self.dataframe = dataframe
+        # Only resize and convert to tensor, no normalization
+        self.transform = transforms.Compose([
+            transforms.Resize((224, 224)),
+            transforms.ToTensor()
+        ])
+
+    def __len__(self):
+        return len(self.dataframe)
+
+    def __getitem__(self, idx):
+        row = self.dataframe.iloc[idx]
+        
+        # Handle image - it should be in row[config.column_url_image] and contain the image data
+        image_data = row[config.column_url_image]
+        
+        try:
+            # Check if image_data has 'bytes' key or is already PIL Image
+            if isinstance(image_data, dict) and 'bytes' in image_data:
+                image = Image.open(BytesIO(image_data['bytes'])).convert("RGB")
+            elif hasattr(image_data, 'convert'):  # Already a PIL Image
+                image = image_data.convert("RGB")
+            elif isinstance(image_data, str):
+                # It's a file path (local or URL)
+                if image_data.startswith('http'):
+                    # It's a URL - download the image
+                    import requests
+                    response = requests.get(image_data, timeout=10)
+                    response.raise_for_status()
+                    image = Image.open(BytesIO(response.content)).convert("RGB")
+                else:
+                    # It's a local file path
+                    image = Image.open(image_data).convert("RGB")
+            else:
+                # Assume it's bytes data
+                image = Image.open(BytesIO(image_data)).convert("RGB")
+            
+            # Apply minimal transform (no normalization)
+            image = self.transform(image)
+
+        except Exception as e:
+            print(f"⚠️ Failed to load image {idx}: {e}")
+            # Return a placeholder image instead of undefined variable
+            image = torch.zeros(3, 224, 224)
+
+        # Get text and color
+        description = row[config.text_column]  
+        color = row[config.color_column]
+
+        return image, description, color
+
+class FashionCLIPEvaluator:
+    """
+    Evaluator for Fashion-CLIP baseline model.
+    
+    This class provides methods to evaluate embeddings from the Fashion-CLIP model
+    and compare them with the custom ColorCLIP model.
+    """
+    
+    def __init__(self):
+        """
+        Initialize the Fashion-CLIP evaluator.
+        
+        Loads the Fashion-CLIP model from Hugging Face and prepares it for evaluation.
+        """
+        # Load Fashion-CLIP model
+        patrick_model_name = "patrickjohncyh/fashion-clip"
+        print(f"🔄 Loading Fashion-CLIP model: {patrick_model_name}")
+        self.processor = CLIPProcessor.from_pretrained(patrick_model_name)
+        self.device = config.device
+        self.model = TransformersCLIPModel.from_pretrained(patrick_model_name).to(self.device)
+        self.model.eval()
+        print(f"✅ Fashion-CLIP model loaded successfully")
+    
+    def extract_embeddings(self, dataloader, embedding_type='text'):
+        """
+        Extract embeddings from the Fashion-CLIP model.
+        
+        Args:
+            dataloader: DataLoader containing images, texts, and colors
+            embedding_type: Type of embeddings to extract ('text', 'image', or 'color')
+            
+        Returns:
+            Tuple of (embeddings array, labels list, texts list)
+        """
+        all_embeddings = []
+        all_labels = []
+        all_texts = []
+        
+        with torch.no_grad():
+            for images, texts, colors in tqdm(dataloader, desc=f"Extracting {embedding_type} embeddings (Fashion-CLIP)"):
+                if embedding_type == 'text':
+                    # Process text through Fashion-CLIP
+                    inputs = self.processor(text=texts, return_tensors="pt", padding=True, truncation=True, max_length=77)
+                    inputs = {k: v.to(self.device) for k, v in inputs.items()}
+                    text_features = self.model.get_text_features(**inputs)
+                    text_features = text_features / text_features.norm(dim=-1, keepdim=True)
+                    embeddings = text_features.cpu().numpy()
+                    labels = colors
+                elif embedding_type == 'image':
+                    # Convert tensors back to PIL images for CLIP processor
+                    pil_images = []
+                    for i in range(images.shape[0]):
+                        # Convert tensor back to PIL Image
+                        img_tensor = images[i]
+                        # Denormalize if needed (images should be in [0,1] range)
+                        if img_tensor.min() < 0 or img_tensor.max() > 1:
+                            # If normalized, denormalize
+                            img_tensor = (img_tensor + 1) / 2  # Assuming [-1,1] to [0,1]
+                        img_tensor = torch.clamp(img_tensor, 0, 1)
+                        img_pil = transforms.ToPILImage()(img_tensor)
+                        pil_images.append(img_pil)
+                    
+                    # Process images through Fashion-CLIP
+                    inputs = self.processor(images=pil_images, return_tensors="pt", padding=True)
+                    inputs = {k: v.to(self.device) for k, v in inputs.items()}
+                    image_features = self.model.get_image_features(**inputs)
+                    image_features = image_features / image_features.norm(dim=-1, keepdim=True)
+                    embeddings = image_features.cpu().numpy()
+                    labels = colors
+                elif embedding_type == 'color':
+                    # Process color names as text through Fashion-CLIP
+                    inputs = self.processor(text=colors, return_tensors="pt", padding=True, truncation=True, max_length=77)
+                    inputs = {k: v.to(self.device) for k, v in inputs.items()}
+                    text_features = self.model.get_text_features(**inputs)
+                    text_features = text_features / text_features.norm(dim=-1, keepdim=True)
+                    embeddings = text_features.cpu().numpy()
+                    labels = colors
+                
+                all_embeddings.append(embeddings)
+                all_labels.extend(labels)
+                all_texts.extend(texts)
+        
+        return np.vstack(all_embeddings), all_labels, all_texts
+    
+    def compute_similarity_metrics(self, embeddings, labels):
+        """Compute intra-class and inter-class similarities"""
+        similarities = cosine_similarity(embeddings)
+        
+        # Group embeddings by color
+        color_groups = defaultdict(list)
+        for i, color in enumerate(labels):
+            color_groups[color].append(i)
+        
+        # Calculate intra-class similarities (same color)
+        intra_class_similarities = []
+        for color, indices in color_groups.items():
+            if len(indices) > 1:
+                for i in range(len(indices)):
+                    for j in range(i+1, len(indices)):
+                        sim = similarities[indices[i], indices[j]]
+                        intra_class_similarities.append(sim)
+        
+        # Calculate inter-class similarities (different colors)
+        inter_class_similarities = []
+        colors = list(color_groups.keys())
+        for i in range(len(colors)):
+            for j in range(i+1, len(colors)):
+                color1_indices = color_groups[colors[i]]
+                color2_indices = color_groups[colors[j]]
+                
+                for idx1 in color1_indices:
+                    for idx2 in color2_indices:
+                        sim = similarities[idx1, idx2]
+                        inter_class_similarities.append(sim)
+        
+        # Calculate classification accuracy using nearest neighbor in embedding space
+        nn_accuracy = self.compute_embedding_accuracy(embeddings, labels, similarities)
+        
+        # Calculate classification accuracy using centroids
+        centroid_accuracy = self.compute_centroid_accuracy(embeddings, labels)
+        
+        return {
+            'intra_class_similarities': intra_class_similarities,
+            'inter_class_similarities': inter_class_similarities,
+            'intra_class_mean': np.mean(intra_class_similarities) if intra_class_similarities else 0,
+            'inter_class_mean': np.mean(inter_class_similarities) if inter_class_similarities else 0,
+            'separation_score': np.mean(intra_class_similarities) - np.mean(inter_class_similarities) if intra_class_similarities and inter_class_similarities else 0,
+            'accuracy': nn_accuracy,
+            'centroid_accuracy': centroid_accuracy
+        }
+    
+    def compute_embedding_accuracy(self, embeddings, labels, similarities):
+        """Compute classification accuracy using nearest neighbor in embedding space"""
+        correct_predictions = 0
+        total_predictions = len(labels)
+        
+        for i in range(len(embeddings)):
+            true_label = labels[i]
+            
+            # Find the most similar embedding (excluding itself)
+            similarities_row = similarities[i].copy()
+            similarities_row[i] = -1  # Exclude self-similarity
+            nearest_neighbor_idx = np.argmax(similarities_row)
+            predicted_label = labels[nearest_neighbor_idx]
+            
+            if predicted_label == true_label:
+                correct_predictions += 1
+        
+        return correct_predictions / total_predictions if total_predictions > 0 else 0
+    
+    def compute_centroid_accuracy(self, embeddings, labels):
+        """Compute classification accuracy using color centroids"""
+        # Create centroids for each color
+        unique_colors = list(set(labels))
+        centroids = {}
+        
+        for color in unique_colors:
+            color_indices = [i for i, label in enumerate(labels) if label == color]
+            color_embeddings = embeddings[color_indices]
+            centroids[color] = np.mean(color_embeddings, axis=0)
+        
+        # Classify each embedding to nearest centroid
+        correct_predictions = 0
+        total_predictions = len(labels)
+        
+        for i, embedding in enumerate(embeddings):
+            true_label = labels[i]
+            
+            # Find closest centroid
+            best_similarity = -1
+            predicted_label = None
+            
+            for color, centroid in centroids.items():
+                similarity = cosine_similarity([embedding], [centroid])[0][0]
+                if similarity > best_similarity:
+                    best_similarity = similarity
+                    predicted_label = color
+            
+            if predicted_label == true_label:
+                correct_predictions += 1
+        
+        return correct_predictions / total_predictions if total_predictions > 0 else 0
+    
+    def predict_colors_from_embeddings(self, embeddings, labels):
+        """Predict colors from embeddings using centroid-based classification"""
+        # Create color centroids from training data
+        unique_colors = list(set(labels))
+        centroids = {}
+        
+        for color in unique_colors:
+            color_indices = [i for i, label in enumerate(labels) if label == color]
+            color_embeddings = embeddings[color_indices]
+            centroids[color] = np.mean(color_embeddings, axis=0)
+        
+        # Predict colors for all embeddings
+        predictions = []
+        
+        for i, embedding in enumerate(embeddings):
+            # Find closest centroid
+            best_similarity = -1
+            predicted_color = None
+            
+            for color, centroid in centroids.items():
+                similarity = cosine_similarity([embedding], [centroid])[0][0]
+                if similarity > best_similarity:
+                    best_similarity = similarity
+                    predicted_color = color
+            
+            predictions.append(predicted_color)
+        
+        return predictions
+    
+    def create_confusion_matrix(self, true_labels, predicted_labels, title="Confusion Matrix"):
+        """Create and plot confusion matrix"""
+        # Get unique labels
+        unique_labels = sorted(list(set(true_labels + predicted_labels)))
+        
+        # Create confusion matrix
+        cm = confusion_matrix(true_labels, predicted_labels, labels=unique_labels)
+        
+        # Calculate accuracy
+        accuracy = accuracy_score(true_labels, predicted_labels)
+        
+        # Plot confusion matrix
+        plt.figure(figsize=(12, 10))
+        sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', 
+                   xticklabels=unique_labels, yticklabels=unique_labels)
+        plt.title(f'{title}\nAccuracy: {accuracy:.3f} ({accuracy*100:.1f}%)')
+        plt.ylabel('True Color')
+        plt.xlabel('Predicted Color')
+        plt.xticks(rotation=45)
+        plt.yticks(rotation=0)
+        plt.tight_layout()
+        
+        return plt.gcf(), accuracy, cm
+    
+    def evaluate_classification_performance(self, embeddings, labels, embedding_type="Embeddings"):
+        """Evaluate classification performance and create confusion matrix"""
+        # Predict colors
+        predictions = self.predict_colors_from_embeddings(embeddings, labels)
+        
+        # Calculate accuracy
+        accuracy = accuracy_score(labels, predictions)
+        
+        # Create confusion matrix
+        fig, acc, cm = self.create_confusion_matrix(labels, predictions, 
+                                                   f"{embedding_type} - Color Classification (Fashion-CLIP)")
+        
+        # Generate classification report
+        unique_labels = sorted(list(set(labels)))
+        report = classification_report(labels, predictions, labels=unique_labels, 
+                                     target_names=unique_labels, output_dict=True)
+        
+        return {
+            'accuracy': accuracy,
+            'predictions': predictions,
+            'confusion_matrix': cm,
+            'classification_report': report,
+            'figure': fig
+        }
+    
+    def evaluate_dataset(self, dataframe, dataset_name="Dataset"):
+        """
+        Evaluate Fashion-CLIP embeddings on a given dataset.
+        
+        This method extracts embeddings for text, image, and color, computes similarity metrics,
+        evaluates classification performance, and saves confusion matrices.
+        
+        Args:
+            dataframe: DataFrame containing the dataset
+            dataset_name: Name of the dataset for display purposes
+            
+        Returns:
+            Dictionary containing evaluation results for text, image, and color embeddings
+        """
+        print(f"\n{'='*60}")
+        print(f"Evaluating {dataset_name} with Fashion-CLIP")
+        print(f"{'='*60}")
+        
+        # Create dataset and dataloader - use FashionCLIPDataset for Fashion-CLIP
+        if "kagl" in dataset_name.lower():
+            dataset = KaglDataset(dataframe)
+        else:
+            dataset = FashionCLIPDataset(dataframe)  # Use special dataset for Fashion-CLIP
+        # Optimize batch size for Fashion-CLIP
+        dataloader = DataLoader(dataset, batch_size=32, shuffle=False, num_workers=4, pin_memory=True)
+        
+        results = {}
+        
+        # Evaluate text embeddings
+        text_embeddings, text_labels, texts = self.extract_embeddings(dataloader, 'text')
+        text_metrics = self.compute_similarity_metrics(text_embeddings, text_labels)
+        text_classification = self.evaluate_classification_performance(text_embeddings, text_labels, "Text Embeddings")
+        text_metrics.update(text_classification)
+        results['text'] = text_metrics
+        
+        # Evaluate image embeddings
+        image_embeddings, image_labels, _ = self.extract_embeddings(dataloader, 'image')
+        image_metrics = self.compute_similarity_metrics(image_embeddings, image_labels)
+        image_classification = self.evaluate_classification_performance(image_embeddings, image_labels, "Image Embeddings")
+        image_metrics.update(image_classification)
+        results['image'] = image_metrics
+        
+        # Evaluate color embeddings
+        color_embeddings, color_labels, _ = self.extract_embeddings(dataloader, 'color')
+        color_metrics = self.compute_similarity_metrics(color_embeddings, color_labels)
+        color_classification = self.evaluate_classification_performance(color_embeddings, color_labels, "Color Embeddings")
+        color_metrics.update(color_classification)
+        results['color'] = color_metrics
+        
+        # Print results
+        print(f"\n{dataset_name} Results (Fashion-CLIP):")
+        print("-" * 40)
+        for emb_type, metrics in results.items():
+            print(f"{emb_type.capitalize()} Embeddings:")
+            print(f"  Intra-class similarity (same color): {metrics['intra_class_mean']:.4f}")
+            print(f"  Inter-class similarity (diff colors): {metrics['inter_class_mean']:.4f}")
+            print(f"  Separation score: {metrics['separation_score']:.4f}")
+            print(f"  Nearest Neighbor Accuracy: {metrics['accuracy']:.4f} ({metrics['accuracy']*100:.1f}%)")
+            print(f"  Centroid Accuracy: {metrics['centroid_accuracy']:.4f} ({metrics['centroid_accuracy']*100:.1f}%)")
+            
+            # Classification report summary
+            report = metrics['classification_report']
+            print(f"  📊 Classification Performance:")
+            print(f"    • Macro Avg F1-Score: {report['macro avg']['f1-score']:.4f}")
+            print(f"    • Weighted Avg F1-Score: {report['weighted avg']['f1-score']:.4f}")
+            print(f"    • Support: {report['macro avg']['support']:.0f} samples")
+            print()
+        
+        # Create visualizations
+        os.makedirs('embedding_evaluation', exist_ok=True)
+        
+        # Confusion matrices
+        results['text']['figure'].savefig(f'embedding_evaluation/{dataset_name.lower()}_text_confusion_matrix_fashion_clip.png', dpi=300, bbox_inches='tight')
+        plt.close(results['text']['figure'])
+        
+        results['image']['figure'].savefig(f'embedding_evaluation/{dataset_name.lower()}_image_confusion_matrix_fashion_clip.png', dpi=300, bbox_inches='tight')
+        plt.close(results['image']['figure'])
+        
+        results['color']['figure'].savefig(f'embedding_evaluation/{dataset_name.lower()}_color_confusion_matrix_fashion_clip.png', dpi=300, bbox_inches='tight')
+        plt.close(results['color']['figure'])
+        
+        return results
+
+class KaglDataset(Dataset):
+    """
+    Dataset class for KAGL Marqo dataset evaluation.
+    
+    Handles loading images from the KAGL dataset format (with 'bytes' in image_url).
+    """
+    def __init__(self, dataframe):
+        """
+        Initialize the KAGL dataset.
+        
+        Args:
+            dataframe: DataFrame containing image_url (with bytes), text, and color labels
+        """
+        self.dataframe = dataframe
+        self.transform = transforms.Compose([
+            transforms.Resize((224, 224)),
+            transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2),
+            transforms.ToTensor(),
+            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+        ])
+
+    def __len__(self):
+        return len(self.dataframe)
+
+    def __getitem__(self, idx):
+        row = self.dataframe.iloc[idx]
+        
+        # Handle image - it should be in row['image_url'] and contain the image data
+        image_data = row["image_url"]
+        
+        # Check if image_data has 'bytes' key or is already PIL Image
+        if isinstance(image_data, dict) and 'bytes' in image_data:
+            image = Image.open(BytesIO(image_data['bytes'])).convert("RGB")
+        elif hasattr(image_data, 'convert'):  # Already a PIL Image
+            image = image_data.convert("RGB")
+        else:
+            image = Image.open(BytesIO(image_data)).convert("RGB")
+        
+        image = self.transform(image)
+
+        # Get text and color from kagl
+        description = row['text']  
+        color = row['color']
+
+        return image, description, color
+
+def load_kagl_marqo_dataset():
+    """
+    Load and prepare KAGL Marqo dataset from Hugging Face.
+    
+    This function loads the Marqo/KAGL dataset, filters for valid colors,
+    and formats it for evaluation.
+    
+    Returns:
+        DataFrame with columns: image_url, text, color
+    """
+    from datasets import load_dataset
+    print("Loading kagl KAGL dataset...")
+
+    # Load the dataset
+    dataset = load_dataset("Marqo/KAGL")
+    df = dataset["data"].to_pandas()
+    print(f"✅ Dataset kagl loaded")
+
+    # Prepare data - Replace baseColour
+    df['baseColour'] = df['baseColour'].str.lower().str.replace("grey", "gray")
+    df_test = df[df['baseColour'].notna()].copy()
+
+    print(f"📊 Before filtering: {len(df_test)} samples")
+    
+    # Filter for common colors
+    valid_colors = ['red', 'blue', 'green', 'yellow', 'purple', 'pink', 'orange', 
+                    'brown', 'black', 'white', 'gray', 'navy', 'maroon', 'beige']
+    df_test = df_test[df_test['baseColour'].isin(valid_colors)]
+    
+    print(f"📊 After filtering invalid colors: {len(df_test)} samples")
+    print(f"🎨 Valid colors found: {sorted(df_test['baseColour'].unique())}")
+    
+    if len(df_test) == 0:
+        print("❌ No samples left after color filtering. Using mock dataset.")
+
+    # Map to our expected column names
+    kagl_formatted = pd.DataFrame({
+        'image_url': df_test['image_url'],
+        'text': df_test['text'],
+        'color': df_test['baseColour'].str.lower().str.replace("grey", "gray")
+    })
+    
+    # Additional validation - remove rows with missing data
+    print(f"📊 Before final validation: {len(kagl_formatted)} samples")
+    kagl_formatted = kagl_formatted.dropna(subset=[config.column_url_image, config.text_column, config.color_column])
+    print(f"📊 After removing missing data: {len(kagl_formatted)} samples")
+    
+    # Check for empty strings
+    kagl_formatted = kagl_formatted[
+        (kagl_formatted['text'].str.strip() != '') & 
+        (kagl_formatted['color'].str.strip() != '')
+    ]
+    print(f"📊 After removing empty strings: {len(kagl_formatted)} samples")
+    
+    print(f"📊 Final dataset size: {len(kagl_formatted)} samples")
+    
+    return kagl_formatted
+
+def create_comparison_table(val_results, kagl_results, val_results_fashion_clip, kagl_results_fashion_clip):
+    """
+    Create a structured comparison table between custom model and Fashion-CLIP baseline.
+    
+    Args:
+        val_results: Evaluation results for custom model on validation dataset
+        kagl_results: Evaluation results for custom model on KAGL dataset
+        val_results_fashion_clip: Evaluation results for Fashion-CLIP on validation dataset
+        kagl_results_fashion_clip: Evaluation results for Fashion-CLIP on KAGL dataset
+        
+    Returns:
+        DataFrame containing the comparison table
+    """
+    
+    # Create DataFrame for comparison
+    data = []
+    
+    # Define embedding types and their display names
+    embedding_types = [
+        ('text', 'Text Embeddings'),
+        ('image', 'Image Embeddings'), 
+        ('color', 'Color Embeddings')
+    ]
+    
+    # Define datasets
+    datasets = [
+        ('Validation Dataset', val_results, val_results_fashion_clip),
+        ('kagl Marqo Dataset', kagl_results, kagl_results_fashion_clip)
+    ]
+    
+    for dataset_name, custom_results, baseline_results in datasets:
+        for emb_type, emb_display in embedding_types:
+            # Your custom model results
+            custom_metrics = custom_results[emb_type]
+            # Baseline model results  
+            baseline_metrics = baseline_results[emb_type]
+            
+            data.append({
+                'Dataset': dataset_name,
+                'Embedding Type': emb_display,
+                'Model': 'Your Model',
+                'Separation Score': f"{custom_metrics['separation_score']:.4f}",
+                'NN Accuracy (%)': f"{custom_metrics['accuracy']*100:.1f}%",
+                'Centroid Accuracy (%)': f"{custom_metrics['centroid_accuracy']*100:.1f}%",
+                'Intra-class Similarity': f"{custom_metrics['intra_class_mean']:.4f}",
+                'Inter-class Similarity': f"{custom_metrics['inter_class_mean']:.4f}",
+                'Macro F1-Score': f"{custom_metrics['classification_report']['macro avg']['f1-score']:.4f}",
+                'Weighted F1-Score': f"{custom_metrics['classification_report']['weighted avg']['f1-score']:.4f}"
+            })
+            
+            data.append({
+                'Dataset': dataset_name,
+                'Embedding Type': emb_display,
+                'Model': 'Fashion-CLIP (Baseline)',
+                'Separation Score': f"{baseline_metrics['separation_score']:.4f}",
+                'NN Accuracy (%)': f"{baseline_metrics['accuracy']*100:.1f}%",
+                'Centroid Accuracy (%)': f"{baseline_metrics['centroid_accuracy']*100:.1f}%",
+                'Intra-class Similarity': f"{baseline_metrics['intra_class_mean']:.4f}",
+                'Inter-class Similarity': f"{baseline_metrics['inter_class_mean']:.4f}",
+                'Macro F1-Score': f"{baseline_metrics['classification_report']['macro avg']['f1-score']:.4f}",
+                'Weighted F1-Score': f"{baseline_metrics['classification_report']['weighted avg']['f1-score']:.4f}"
+            })
+    
+    # Create DataFrame
+    df_comparison = pd.DataFrame(data)
+    
+    # Save to CSV
+    df_comparison.to_csv('embedding_evaluation/model_comparison_table.csv', index=False)
+    
+    # Print formatted table
+    print(f"\n{'='*120}")
+    print("📊 COMPREHENSIVE MODEL COMPARISON TABLE")
+    print(f"{'='*120}")
+    
+    # Print table by dataset
+    for dataset_name in df_comparison['Dataset'].unique():
+        print(f"\n🔍 {dataset_name.upper()}")
+        print("-" * 120)
+        
+        dataset_df = df_comparison[df_comparison['Dataset'] == dataset_name]
+        
+        for emb_type in dataset_df['Embedding Type'].unique():
+            print(f"\n📈 {emb_type}:")
+            emb_df = dataset_df[dataset_df['Embedding Type'] == emb_type]
+            
+            # Print header
+            print(f"{'Model':<20} {'Separation':<12} {'NN Acc':<10} {'Centroid Acc':<13} {'Intra-class':<12} {'Inter-class':<12} {'Macro F1':<10} {'Weighted F1':<12}")
+            print("-" * 120)
+            
+            # Print data
+            for _, row in emb_df.iterrows():
+                print(f"{row['Model']:<20} {row['Separation Score']:<12} {row['NN Accuracy (%)']:<10} {row['Centroid Accuracy (%)']:<13} {row['Intra-class Similarity']:<12} {row['Inter-class Similarity']:<12} {row['Macro F1-Score']:<10} {row['Weighted F1-Score']:<12}")
+    
+    return df_comparison
+
+if __name__ == "__main__":
+    
+    # Initialize evaluator for your custom model
+    evaluator = EmbeddingEvaluator(model_path=config.color_model_path, embed_dim=config.color_emb_dim)
+    
+    # Initialize Fashion-CLIP evaluator
+    fashion_clip_evaluator = FashionCLIPEvaluator()
+    
+    # Load datasets
+    print("Loading datasets...")
+    
+    # Load validation dataset
+    df_val = pd.read_csv(config.local_dataset_path)
+    
+    # Filter for better quality data
+    print(f"📊 Original dataset size: {len(df_val)}")
+    samples_to_evaluate = 10000
+    
+
+    # Load kagl Marqo dataset
+    kagl_df = load_kagl_marqo_dataset()
+    
+    # Evaluate your custom model on validation dataset
+    val_results = evaluator.evaluate_dataset(df_val, "Validation Dataset")
+    
+    # Evaluate your custom model on kagl Marqo dataset (reduced sample for speed)
+    kagl_results = evaluator.evaluate_dataset(kagl_df.sample(min(samples_to_evaluate, len(kagl_df)), random_state=42), "kagl Marqo Dataset")
+    
+    # Evaluate Fashion-CLIP on validation dataset
+    val_results_fashion_clip = fashion_clip_evaluator.evaluate_dataset(df_val, "Validation Dataset")
+    
+    # Create comprehensive comparison table
+    comparison_df = create_comparison_table(
+        val_results, kagl_results, 
+        val_results_fashion_clip
+    )
+    
+    print(f"\n{'='*120}")
+    print("✅ Evaluation complete!")
+    print("📁 Confusion matrices saved in 'embedding_evaluation/' folder")
+    print("📁 Comparison table saved as 'model_comparison_table.csv'")
+    print("📁 Fashion-CLIP results are saved with '_fashion_clip' suffix.")
+    print(f"{'='*120}")