Upload evaluation/hierarchy_evaluation.py with huggingface_hub

evaluation/hierarchy_evaluation.py

@@ -1,9 +1,9 @@
 """
-Hierarchy embedding evaluation for clothing category classification.
-This file evaluates the quality of hierarchy embeddings generated by the hierarchy model
-by calculating intra-class and inter-class similarity metrics, nearest neighbor and centroid-based
-classification accuracies, and generating confusion matrices. It can be used on different datasets
-(local validation, Kagl Marqo) to measure model generalization.
+Hierarchy embedding evaluation with CLIP baseline comparison.
+This file evaluates the quality of hierarchy embeddings from the custom model and compares them
+with a CLIP baseline model (Fashion-CLIP by patrickjohncyh). It calculates similarity metrics, 
+classification accuracies, and generates confusion matrices for both models to measure relative 
+performance. It also supports evaluation on Fashion-MNIST and kagl Marqo datasets.
 """
 
 import torch
@@ -12,59 +12,165 @@ import numpy as np
 import matplotlib.pyplot as plt
 import seaborn as sns
 from sklearn.metrics.pairwise import cosine_similarity
-from sklearn.metrics import confusion_matrix, classification_report, accuracy_score
+from sklearn.metrics import confusion_matrix, classification_report, accuracy_score, f1_score
 from collections import defaultdict
 import os
+import requests
 from tqdm import tqdm
 from torch.utils.data import Dataset, DataLoader
 from torchvision import transforms
 from sklearn.model_selection import train_test_split
 from io import BytesIO
 from PIL import Image
-import config
+from config import device, hierarchy_model_path, hierarchy_column, local_dataset_path
 import warnings
 warnings.filterwarnings('ignore')
+
+# Import transformers CLIP
+from transformers import CLIPProcessor, CLIPModel as TransformersCLIPModel
+
+# Import your custom model
 from hierarchy_model import Model, HierarchyExtractor, HierarchyDataset, collate_fn
+import config
+
+def convert_fashion_mnist_to_image(pixel_values):
+    """Convert Fashion-MNIST pixel values to PIL image"""
+    # Reshape to 28x28 and convert to PIL Image
+    image_array = np.array(pixel_values).reshape(28, 28).astype(np.uint8)
+    # Convert to RGB by duplicating the grayscale channel
+    image_array = np.stack([image_array] * 3, axis=-1)
+    image = Image.fromarray(image_array)
+    return image
+
+def get_fashion_mnist_labels():
+    """Get Fashion-MNIST class labels"""
+    return {
+        0: "T-shirt/top", 
+        1: "Trouser", 
+        2: "Pullover", 
+        3: "Dress", 
+        4: "Coat", 
+        5: "Sandal", 
+        6: "Shirt", 
+        7: "Sneaker", 
+        8: "Bag", 
+        9: "Ankle boot"
+    }
+
+class FashionMNISTDataset(Dataset):
+    def __init__(self, dataframe, image_size=224):
+        self.dataframe = dataframe
+        self.image_size = image_size
+        self.labels_map = get_fashion_mnist_labels()
+        
+        # Simple transforms for validation/inference
+        self.transform = transforms.Compose([
+            transforms.Resize((image_size, image_size)),
+            transforms.ToTensor(),
+            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+        ])
+        
+    def __len__(self):
+        return len(self.dataframe)
+
+    def __getitem__(self, idx):
+        row = self.dataframe.iloc[idx]
+        
+        # Get pixel values (columns 1-784)
+        pixel_cols = [f'pixel{i}' for i in range(1, 785)]
+        pixel_values = row[pixel_cols].values
+        
+        # Convert to image
+        image = convert_fashion_mnist_to_image(pixel_values)
+        image = self.transform(image)
+
+        # Get text description
+        text = row['text']
+        
+        # Get hierarchy label
+        hierarchy = row['hierarchy']
+
+        return image, text, hierarchy
+
+class CLIPBaselineEvaluator:
+    def __init__(self, device='mps'):
+        self.device = torch.device(device)
+        
+        # Load Fashion-CLIP model and processor
+        print("🤗 Loading Fashion-CLIP baseline model from transformers...")
+        patrick_model_name = "patrickjohncyh/fashion-clip"
+        self.clip_model = TransformersCLIPModel.from_pretrained(patrick_model_name).to(self.device)
+        self.clip_processor = CLIPProcessor.from_pretrained(patrick_model_name)
+        
+        self.clip_model.eval()
+        print("✅ Fashion-CLIP model loaded successfully")
+        
+    def extract_clip_embeddings(self, images, texts):
+        """Extract Fashion-CLIP embeddings for images and texts"""
+        all_image_embeddings = []
+        all_text_embeddings = []
+        
+        with torch.no_grad():
+            for i in tqdm(range(len(images)), desc="Extracting CLIP embeddings"):
+                # Process image
+                if isinstance(images[i], torch.Tensor):
+                    # Convert tensor back to PIL Image
+                    image_tensor = images[i]
+                    if image_tensor.dim() == 3:
+                        # Denormalize
+                        mean = torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1)
+                        std = torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1)
+                        image_tensor = image_tensor * std + mean
+                        image_tensor = torch.clamp(image_tensor, 0, 1)
+                        
+                        # Convert to PIL
+                        image_pil = transforms.ToPILImage()(image_tensor)
+                elif isinstance(images[i], Image.Image):
+                    image_pil = images[i]
+                else:
+                    raise ValueError(f"Unsupported image type: {type(images[i])}")
+                
+                # Process with Fashion-CLIP
+                inputs = self.clip_processor(
+                    text=texts[i], 
+                    images=image_pil, 
+                    return_tensors="pt", 
+                    padding=True
+                ).to(self.device)
+                
+                outputs = self.clip_model(**inputs)
+                
+                # Get normalized embeddings
+                image_emb = outputs.image_embeds / outputs.image_embeds.norm(p=2, dim=-1, keepdim=True)
+                text_emb = outputs.text_embeds / outputs.text_embeds.norm(p=2, dim=-1, keepdim=True)
+                
+                all_image_embeddings.append(image_emb.cpu().numpy())
+                all_text_embeddings.append(text_emb.cpu().numpy())
+        
+        return np.vstack(all_image_embeddings), np.vstack(all_text_embeddings)
 
 
 class EmbeddingEvaluator:
-    """
-    Evaluator for hierarchy embeddings generated by the hierarchy model.
-    
-    This class provides methods to evaluate the quality of hierarchy embeddings by computing
-    similarity metrics, classification accuracies, and generating visualizations.
-    """
-    
     def __init__(self, model_path, directory):
-        """
-        Initialize the embedding evaluator.
-        
-        Args:
-            model_path: Path to the trained hierarchy model checkpoint
-            directory: Directory to save evaluation results and visualizations
-        """
-        self.device = config.device
         self.directory = directory
 
         # 1. Load the dataset
-        CSV = config.local_dataset_path
-        print(f"📁 Using dataset with local images: {CSV}")
-        df = pd.read_csv(CSV)
+        print(f"📁 Using dataset with local images: {local_dataset_path}")
+        df = pd.read_csv(local_dataset_path)
         
         print(f"📁 Loaded {len(df)} samples")
         
         # 2. Get unique hierarchy classes from the dataset
-        hierarchy_classes = sorted(df[config.hierarchy_column].unique().tolist())
+        hierarchy_classes = sorted(df[hierarchy_column].unique().tolist())
         print(f"📋 Found {len(hierarchy_classes)} hierarchy classes")
 
-        _, self.val_df = train_test_split(df, test_size=0.2, random_state=42, stratify=df[config.hierarchy_column])
+        _, self.val_df = train_test_split(df, test_size=0.2, random_state=42, stratify=df['hierarchy'])
         
         # 3. Load the model
         if os.path.exists(model_path):
             checkpoint = torch.load(model_path, map_location=self.device)
             
-            # Use model_config to avoid shadowing the imported config module
-            model_config = checkpoint.get('config', {})
+            config = checkpoint.get('config', {})
             saved_hierarchy_classes = checkpoint['hierarchy_classes']
             
             # Use the saved hierarchy classes
@@ -76,35 +182,34 @@ class EmbeddingEvaluator:
             # Create the model with the saved configuration
             self.model = Model(
                 num_hierarchy_classes=len(saved_hierarchy_classes),
-                embed_dim=model_config['embed_dim'],
-                dropout=model_config['dropout']
+                embed_dim=config['embed_dim'],
+                dropout=config['dropout']
             ).to(self.device)   
                 
             self.model.load_state_dict(checkpoint['model_state'])
             
-            print(f"✅ Model loaded with:")
+            print(f"✅ Custom model loaded with:")
             print(f"📋 Hierarchy classes: {len(saved_hierarchy_classes)}")
-            print(f"🎯 Embed dim: {model_config['embed_dim']}")
-            print(f"💧 Dropout: {model_config['dropout']}")
+            print(f"🎯 Embed dim: {config['embed_dim']}")
+            print(f"💧 Dropout: {config['dropout']}")
             print(f"📅 Epoch: {checkpoint.get('epoch', 'unknown')}")
                 
         else:
             raise FileNotFoundError(f"Model file {model_path} not found")
         
         self.model.eval()
+        
+        # Initialize Fashion-CLIP baseline
+        self.clip_evaluator = CLIPBaselineEvaluator(device)
     
     def create_dataloader(self, dataframe, batch_size=16):
-        """
-        Create a DataLoader for the hierarchy dataset.
-        
-        Args:
-            dataframe: DataFrame containing the dataset
-            batch_size: Batch size for the DataLoader
-            
-        Returns:
-            DataLoader instance
-        """
-        dataset = HierarchyDataset(dataframe, image_size=224)
+        """Create a dataloader for custom model"""
+        # Check if this is Fashion-MNIST data (has pixel1 column)
+        if 'pixel1' in dataframe.columns:
+            print("🔍 Detected Fashion-MNIST data, using FashionMNISTDataset")
+            dataset = FashionMNISTDataset(dataframe, image_size=224)
+        else:
+            dataset = HierarchyDataset(dataframe, image_size=224)
         
         dataloader = DataLoader(
             dataset, 
@@ -116,23 +221,32 @@ class EmbeddingEvaluator:
         
         return dataloader
     
-    def extract_embeddings(self, dataloader, embedding_type='text'):
-        """
-        Extract embeddings from the model for a given dataloader.
+    def create_clip_dataloader(self, dataframe, batch_size=16):
+        """Create a dataloader for Fashion-CLIP baseline"""
+        # Check if this is Fashion-MNIST data (has pixel1 column)
+        if 'pixel1' in dataframe.columns:
+            print("🔍 Detected Fashion-MNIST data for Fashion-CLIP, using FashionMNISTDataset")
+            dataset = FashionMNISTDataset(dataframe, image_size=224)
+        else:
+            dataset = CLIPDataset(dataframe)
         
-        Args:
-            dataloader: DataLoader containing images, texts, and hierarchy labels
-            embedding_type: Type of embeddings to extract ('text' or 'image')
-            
-        Returns:
-            Tuple of (embeddings array, labels list, texts list)
-        """
+        dataloader = DataLoader(
+            dataset, 
+            batch_size=batch_size, 
+            shuffle=False, 
+            num_workers=0
+        )
+        
+        return dataloader
+    
+    def extract_custom_embeddings(self, dataloader, embedding_type='text'):
+        """Extract embeddings from custom model"""
         all_embeddings = []
         all_labels = []
         all_texts = []
         
         with torch.no_grad():
-            for batch in tqdm(dataloader, desc=f"Extracting {embedding_type} embeddings"):
+            for batch in tqdm(dataloader, desc=f"Extracting custom {embedding_type} embeddings"):
                 images = batch['image'].to(self.device)
                 hierarchy_indices = batch['hierarchy_indices'].to(self.device)
                 hierarchy_labels = batch['hierarchy']
@@ -146,18 +260,9 @@ class EmbeddingEvaluator:
                 all_texts.extend(hierarchy_labels)
         
         return np.vstack(all_embeddings), all_labels, all_texts
-    
+
     def compute_similarity_metrics(self, embeddings, labels):
-        """
-        Compute intra-class and inter-class similarity metrics.
-        
-        Args:
-            embeddings: Array of embeddings [N, embed_dim]
-            labels: List of labels for each embedding
-            
-        Returns:
-            Dictionary containing similarity metrics, accuracies, and separation scores
-        """
+        """Compute intra-class and inter-class similarities"""
         similarities = cosine_similarity(embeddings)
         
         # Group embeddings by hierarchy
@@ -174,7 +279,6 @@ class EmbeddingEvaluator:
                         sim = similarities[indices[i], indices[j]]
                         intra_class_similarities.append(sim)
         
-                
         # Calculate inter-class similarities (different hierarchies)
         inter_class_similarities = []
         hierarchies = list(hierarchy_groups.keys())
@@ -205,17 +309,7 @@ class EmbeddingEvaluator:
         }
     
     def compute_embedding_accuracy(self, embeddings, labels, similarities):
-        """
-        Compute classification accuracy using nearest neighbor in embedding space.
-        
-        Args:
-            embeddings: Array of embeddings [N, embed_dim]
-            labels: List of true labels
-            similarities: Pre-computed similarity matrix [N, N]
-            
-        Returns:
-            Accuracy score (float between 0 and 1)
-        """
+        """Compute classification accuracy using nearest neighbor in embedding space"""
         correct_predictions = 0
         total_predictions = len(labels)
         
@@ -234,19 +328,7 @@ class EmbeddingEvaluator:
         return correct_predictions / total_predictions if total_predictions > 0 else 0
     
     def compute_centroid_accuracy(self, embeddings, labels):
-        """
-        Compute classification accuracy using hierarchy centroids.
-        
-        Each hierarchy class is represented by its centroid (mean embedding), and each
-        embedding is classified to the nearest centroid.
-        
-        Args:
-            embeddings: Array of embeddings [N, embed_dim]
-            labels: List of true labels
-            
-        Returns:
-            Accuracy score (float between 0 and 1)
-        """
+        """Compute classification accuracy using hierarchy centroids"""
         # Create centroids for each hierarchy
         unique_hierarchies = list(set(labels))
         centroids = {}
@@ -277,18 +359,33 @@ class EmbeddingEvaluator:
                 correct_predictions += 1
         
         return correct_predictions / total_predictions if total_predictions > 0 else 0
-    
-    def predict_hierarchy_from_embeddings(self, embeddings, labels):
-        """
-        Predict hierarchy from embeddings using centroid-based classification.
+
+    def create_confusion_matrix(self, true_labels, predicted_labels, title="Confusion Matrix"):
+        """Create and plot confusion matrix"""
+        # Get unique labels
+        unique_labels = sorted(list(set(true_labels + predicted_labels)))
         
-        Args:
-            embeddings: Array of embeddings [N, embed_dim]
-            labels: List of labels used to compute centroids
-            
-        Returns:
-            List of predicted hierarchy labels
-        """
+        # Create confusion matrix
+        cm = confusion_matrix(true_labels, predicted_labels, labels=unique_labels)
+        
+        # Calculate accuracy
+        accuracy = accuracy_score(true_labels, predicted_labels)
+        
+        # Plot confusion matrix
+        plt.figure(figsize=(12, 10))
+        sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', 
+                   xticklabels=unique_labels, yticklabels=unique_labels)
+        plt.title(f'{title}\nAccuracy: {accuracy:.3f} ({accuracy*100:.1f}%)')
+        plt.ylabel('True Hierarchy')
+        plt.xlabel('Predicted Hierarchy')
+        plt.xticks(rotation=45)
+        plt.yticks(rotation=0)
+        plt.tight_layout()
+        
+        return plt.gcf(), accuracy, cm
+
+    def predict_hierarchy_from_embeddings(self, embeddings, labels):
+        """Predict hierarchy from embeddings using centroid-based classification"""
         # Create hierarchy centroids from training data
         unique_hierarchies = list(set(labels))
         centroids = {}
@@ -315,155 +412,130 @@ class EmbeddingEvaluator:
             predictions.append(predicted_hierarchy)
         
         return predictions
-    
-    def create_confusion_matrix(self, true_labels, predicted_labels, title="Confusion Matrix"):
-        """
-        Create and plot a confusion matrix.
-        
-        Args:
-            true_labels: List of true labels
-            predicted_labels: List of predicted labels
-            title: Title for the confusion matrix plot
-            
-        Returns:
-            Tuple of (figure, accuracy, confusion_matrix)
-        """
-        # Get unique labels
-        unique_labels = sorted(list(set(true_labels + predicted_labels)))
-        
-        # Create confusion matrix
-        cm = confusion_matrix(true_labels, predicted_labels, labels=unique_labels)
-        
-        # Calculate accuracy
-        accuracy = accuracy_score(true_labels, predicted_labels)
-        
-        # Plot confusion matrix
-        plt.figure(figsize=(12, 10))
-        sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', 
-                   xticklabels=unique_labels, yticklabels=unique_labels)
-        plt.title(f'{title}\nAccuracy: {accuracy:.3f} ({accuracy*100:.1f}%)')
-        plt.ylabel('True Hierarchy')
-        plt.xlabel('Predicted Hierarchy')
-        plt.xticks(rotation=45)
-        plt.yticks(rotation=0)
-        plt.tight_layout()
-        
-        return plt.gcf(), accuracy, cm
-    
+
     def evaluate_classification_performance(self, embeddings, labels, embedding_type="Embeddings"):
-        """
-        Evaluate classification performance and create confusion matrix.
-        
-        Args:
-            embeddings: Array of embeddings [N, embed_dim]
-            labels: List of true labels
-            embedding_type: Type of embeddings for display purposes
-            
-        Returns:
-            Dictionary containing accuracy, predictions, confusion matrix, and classification report
-        """
+        """Evaluate classification performance and create confusion matrix"""
         # Predict hierarchy
         predictions = self.predict_hierarchy_from_embeddings(embeddings, labels)
         
         # Calculate accuracy
         accuracy = accuracy_score(labels, predictions)
         
+        # Calculate F1 scores
+        unique_labels = sorted(list(set(labels)))
+        f1_macro = f1_score(labels, predictions, labels=unique_labels, average='macro', zero_division=0)
+        f1_weighted = f1_score(labels, predictions, labels=unique_labels, average='weighted', zero_division=0)
+        f1_per_class = f1_score(labels, predictions, labels=unique_labels, average=None, zero_division=0)
+        
         # Create confusion matrix
         fig, acc, cm = self.create_confusion_matrix(labels, predictions, 
                                                    f"{embedding_type} - Hierarchy Classification")
         
         # Generate classification report
-        unique_labels = sorted(list(set(labels)))
         report = classification_report(labels, predictions, labels=unique_labels, 
                                      target_names=unique_labels, output_dict=True)
         
         return {
             'accuracy': accuracy,
+            'f1_macro': f1_macro,
+            'f1_weighted': f1_weighted,
+            'f1_per_class': f1_per_class,
             'predictions': predictions,
             'confusion_matrix': cm,
             'classification_report': report,
             'figure': fig
         }
-    
-    def evaluate_dataset(self, dataframe, dataset_name="Dataset"):
-        """
-        Evaluate embeddings on a given dataset.
-        
-        This method extracts embeddings for text and image, computes similarity metrics,
-        evaluates classification performance, and saves confusion matrices.
-        
-        Args:
-            dataframe: DataFrame containing the dataset
-            dataset_name: Name of the dataset for display purposes
-            
-        Returns:
-            Dictionary containing evaluation results for text and image embeddings
-        """
+
+    def evaluate_dataset_with_baselines(self, dataframe, dataset_name="Dataset"):
+        """Evaluate embeddings on a given dataset with both custom model and CLIP baseline"""
         print(f"\n{'='*60}")
         print(f"Evaluating {dataset_name}")
         print(f"{'='*60}")
         
-        # Create dataloader exactly as during training
-        dataloader = self.create_dataloader(dataframe, batch_size=16)
-        
         results = {}
         
+        # ===== CUSTOM MODEL EVALUATION =====
+        print(f"\n🔧 Evaluating Custom Model on {dataset_name}")
+        print("-" * 40)
+        
+        # Create dataloader for custom model
+        custom_dataloader = self.create_dataloader(dataframe, batch_size=16)
+        
         # Evaluate text embeddings
-        text_embeddings, text_labels, texts = self.extract_embeddings(dataloader, 'text')
+        text_embeddings, text_labels, texts = self.extract_custom_embeddings(custom_dataloader, 'text')
         text_metrics = self.compute_similarity_metrics(text_embeddings, text_labels)
-        text_classification = self.evaluate_classification_performance(text_embeddings, text_labels, "Text Embeddings")
+        text_classification = self.evaluate_classification_performance(text_embeddings, text_labels, "Custom Text Embeddings")
         text_metrics.update(text_classification)
-        results['text'] = text_metrics
+        results['custom_text'] = text_metrics
         
         # Evaluate image embeddings
-        image_embeddings, image_labels, _ = self.extract_embeddings(dataloader, 'image')
+        image_embeddings, image_labels, _ = self.extract_custom_embeddings(custom_dataloader, 'image')
         image_metrics = self.compute_similarity_metrics(image_embeddings, image_labels)
-        image_classification = self.evaluate_classification_performance(image_embeddings, image_labels, "Image Embeddings")
+        image_classification = self.evaluate_classification_performance(image_embeddings, image_labels, "Custom Image Embeddings")
         image_metrics.update(image_classification)
-        results['image'] = image_metrics
-        
-        # Evaluate hierarchy embeddings
-        hierarchy_embeddings, hierarchy_labels, _ = self.extract_embeddings(dataloader, 'category2')
-        hierarchy_metrics = self.compute_similarity_metrics(hierarchy_embeddings, hierarchy_labels)
-        hierarchy_classification = self.evaluate_classification_performance(hierarchy_embeddings, hierarchy_labels, "hierarchy Embeddings")
-        hierarchy_metrics.update(hierarchy_classification)
-        results['hierarchy'] = hierarchy_metrics
+        results['custom_image'] = image_metrics
         
-        # Print results
-        print(f"\n{dataset_name} Results:")
+        # ===== FASHION-CLIP BASELINE EVALUATION =====
+        print(f"\n🤗 Evaluating Fashion-CLIP Baseline on {dataset_name}")
         print("-" * 40)
-        for emb_type, metrics in results.items():
-            print(f"{emb_type.capitalize()} Embeddings:")
-            print(f"  Intra-class similarity (same hierarchy): {metrics['intra_class_mean']:.4f}")
-            print(f"  Inter-class similarity (diff hierarchy): {metrics['inter_class_mean']:.4f}")
-            print(f"  Separation score: {metrics['separation_score']:.4f}")
-            print(f"  Nearest Neighbor Accuracy: {metrics['accuracy']:.4f} ({metrics['accuracy']*100:.1f}%)")
-            print(f"  Centroid Accuracy: {metrics['centroid_accuracy']:.4f} ({metrics['centroid_accuracy']*100:.1f}%)")
-            
-            # Classification report summary
-            report = metrics['classification_report']
-            print(f"  📊 Classification Performance:")
-            print(f"    • Macro Avg F1-Score: {report['macro avg']['f1-score']:.4f}")
-            print(f"    • Weighted Avg F1-Score: {report['weighted avg']['f1-score']:.4f}")
-            print(f"    • Support: {report['macro avg']['support']:.0f} samples")
-            print()
-        
-        # Create visualizations
-        os.makedirs(f'{self.directory}', exist_ok=True)
         
-        # Confusion matrices
-        results['text']['figure'].savefig(f'{self.directory}/{dataset_name.lower()}_text_confusion_matrix.png', dpi=300, bbox_inches='tight')
-        plt.close(results['text']['figure'])
+        # Create dataloader for Fashion-CLIP
+        clip_dataloader = self.create_clip_dataloader(dataframe, batch_size=8)  # Smaller batch for Fashion-CLIP
         
-        results['image']['figure'].savefig(f'{self.directory}/{dataset_name.lower()}_image_confusion_matrix.png', dpi=300, bbox_inches='tight')
-        plt.close(results['image']['figure'])
+        # Extract data for Fashion-CLIP
+        all_images = []
+        all_texts = []
+        all_labels = []
+        
+        for batch in tqdm(clip_dataloader, desc="Preparing data for Fashion-CLIP"):
+            images, texts, labels = batch
+            all_images.extend(images)
+            all_texts.extend(texts)
+            all_labels.extend(labels)
+        
+        # Get Fashion-CLIP embeddings
+        clip_image_embeddings, clip_text_embeddings = self.clip_evaluator.extract_clip_embeddings(all_images, all_texts)
+        
+        # Evaluate Fashion-CLIP text embeddings
+        clip_text_metrics = self.compute_similarity_metrics(clip_text_embeddings, all_labels)
+        clip_text_classification = self.evaluate_classification_performance(clip_text_embeddings, all_labels, "Fashion-CLIP Text Embeddings")
+        clip_text_metrics.update(clip_text_classification)
+        results['clip_text'] = clip_text_metrics
+        
+        # Evaluate Fashion-CLIP image embeddings
+        clip_image_metrics = self.compute_similarity_metrics(clip_image_embeddings, all_labels)
+        clip_image_classification = self.evaluate_classification_performance(clip_image_embeddings, all_labels, "Fashion-CLIP Image Embeddings")
+        clip_image_metrics.update(clip_image_classification)
+        results['clip_image'] = clip_image_metrics
+        
+        # ===== PRINT COMPARISON RESULTS =====
+        print(f"\n{dataset_name} Results Comparison:")
+        print(f"Dataset size: {len(dataframe)} samples")
+        print("=" * 80)
+        print(f"{'Model':<20} {'Embedding':<10} {'Sep Score':<10} {'NN Acc':<8} {'Centroid Acc':<12} {'F1 Macro':<10}")
+        print("-" * 80)
+        
+        for model_type in ['custom', 'clip']:
+            for emb_type in ['text', 'image']:
+                key = f"{model_type}_{emb_type}"
+                if key in results:
+                    metrics = results[key]
+                    model_name = "Custom Model" if model_type == 'custom' else "Fashion-CLIP Baseline"
+                    print(f"{model_name:<20} {emb_type.capitalize():<10} {metrics['separation_score']:<10.4f} {metrics['accuracy']*100:<8.1f}% {metrics['centroid_accuracy']*100:<12.1f}% {metrics['f1_macro']*100:<10.1f}%")
+        
+        # ===== SAVE VISUALIZATIONS =====
+        os.makedirs(f'{self.directory}', exist_ok=True)
         
-        results['hierarchy']['figure'].savefig(f'{self.directory}/{dataset_name.lower()}_hierarchy_confusion_matrix.png', dpi=300, bbox_inches='tight')
-        plt.close(results['hierarchy']['figure'])
+        # Save confusion matrices
+        for key, metrics in results.items():
+            if 'figure' in metrics:
+                metrics['figure'].savefig(f'{self.directory}/{dataset_name.lower()}_{key}_confusion_matrix.png', dpi=300, bbox_inches='tight')
+                plt.close(metrics['figure'])
         
         return results
 
-class KaglDataset(Dataset):
+
+class CLIPDataset(Dataset):
     def __init__(self, dataframe):
         self.dataframe = dataframe
         # Use VALIDATION transforms (no augmentation)
@@ -479,26 +551,130 @@ class KaglDataset(Dataset):
     def __getitem__(self, idx):
         row = self.dataframe.iloc[idx]
         
-        # Handle image
-        image_data = row['image_url']
-        image = Image.open(BytesIO(image_data['bytes'])).convert("RGB")
-        image = self.transform(image)
+        # Handle image loading (same as HierarchyDataset)
+        if config.column_local_image_path in row.index and pd.notna(row[config.column_local_image_path]):
+            local_path = row[config.column_local_image_path]
+            try:
+                if os.path.exists(local_path):
+                    image = Image.open(local_path).convert("RGB")
+                else:
+                    print(f"⚠️ Local image not found: {local_path}")
+                    image = Image.new('RGB', (224, 224), color='gray')
+            except Exception as e:
+                print(f"⚠️ Failed to load local image {idx}: {e}")
+                image = Image.new('RGB', (224, 224), color='gray')
+        elif isinstance(row[config.column_url_image], dict):
+            image = Image.open(BytesIO(row[config.column_url_image]['bytes'])).convert('RGB')
+        elif isinstance(row['image_url'], (list, np.ndarray)):
+            pixels = np.array(row[config.column_url_image]).reshape(28, 28)
+            image = Image.fromarray(pixels.astype(np.uint8)).convert("RGB")
+        elif isinstance(row[config.column_url_image], Image.Image):
+            # Handle PIL Image objects directly (for Fashion-MNIST)
+            image = row[config.column_url_image].convert("RGB")
+        else:
+            try:
+                response = requests.get(row[config.column_url_image], timeout=10)
+                response.raise_for_status()
+                image = Image.open(BytesIO(response.content)).convert("RGB")
+            except Exception as e:
+                print(f"⚠️ Failed to load image {idx}: {e}")
+                image = Image.new('RGB', (224, 224), color='gray')
+        
+        # Apply transforms
+        image_tensor = self.transform(image)
 
-        # Get text and hierarchy
-        description = row['text']  
-        hierarchy = row['hierarchy']
+        description = row[config.text_column]  
+        hierarchy = row[config.hierarchy_column]
+
+        return image_tensor, description, hierarchy
 
-        return image, description, hierarchy
 
-def load_Kagl_marqo_dataset(evaluator):
-    """Load and prepare Kagl KAGL dataset"""
+def load_fashion_mnist_dataset(evaluator):
+    """Load and prepare Fashion-MNIST test dataset"""
+    print("Loading Fashion-MNIST test dataset...")
+    
+    # Load the dataset
+    df = pd.read_csv(config.fashion_mnist_test_path)
+    print(f"✅ Fashion-MNIST dataset loaded")
+    print(f"📊 Total samples: {len(df)}")
+    
+    # Fashion-MNIST class labels mapping
+    fashion_mnist_labels = get_fashion_mnist_labels()
+    
+    # Map labels to hierarchy classes
+    hierarchy_mapping = {
+        'T-shirt/top': 'top',
+        'Trouser': 'bottom', 
+        'Pullover': 'top',
+        'Dress': 'dress',
+        'Coat': 'top',
+        'Sandal': 'shoes',
+        'Shirt': 'top',
+        'Sneaker': 'shoes',
+        'Bag': 'bag',
+        'Ankle boot': 'shoes'
+    }
+    
+    # Apply label mapping
+    df['hierarchy'] = df['label'].map(fashion_mnist_labels).map(hierarchy_mapping)
+    
+    # Filter to only include hierarchies that exist in our model
+    valid_hierarchies = df['hierarchy'].dropna().unique()
+    print(f"🎯 Valid hierarchies found: {sorted(valid_hierarchies)}")
+    print(f"🎯 Model hierarchies: {sorted(evaluator.hierarchy_classes)}")
+    
+    # Filter to only include hierarchies that exist in our model
+    df = df[df['hierarchy'].isin(evaluator.hierarchy_classes)]
+    print(f"📊 After filtering to model hierarchies: {len(df)} samples")
+    
+    if len(df) == 0:
+        print("❌ No samples left after hierarchy filtering.")
+        return pd.DataFrame()
+    
+    # Keep pixel columns as they are (FashionMNISTDataset will handle them)
+    
+    # Create text descriptions based on hierarchy
+    text_descriptions = {
+        'top': 'A top clothing item',
+        'bottom': 'A bottom clothing item', 
+        'dress': 'A dress',
+        'shoes': 'A pair of shoes',
+        'bag': 'A bag'
+    }
+    
+    df['text'] = df['hierarchy'].map(text_descriptions)
+    
+    # Show sample of data
+    print(f"📝 Sample data:")
+    for i, (hierarchy, text) in enumerate(zip(df['hierarchy'].head(3), df['text'].head(3))):
+        print(f"  {i+1}. [{hierarchy}] {text}")
+    
+    df_test = df.copy()
+    
+    print(f"📊 After sampling: {len(df_test)} samples")
+    print(f"📊 Samples per hierarchy:")
+    for hierarchy in sorted(df_test['hierarchy'].unique()):
+        count = len(df_test[df_test['hierarchy'] == hierarchy])
+        print(f"  {hierarchy}: {count} samples")
+    
+    # Create formatted dataset with proper column names
+    # Keep all pixel columns for FashionMNISTDataset
+    pixel_cols = [f'pixel{i}' for i in range(1, 785)]
+    fashion_mnist_formatted = df_test[['label'] + pixel_cols + ['text', 'hierarchy']].copy()
+    
+    print(f"📊 Final dataset size: {len(fashion_mnist_formatted)} samples")
+    return fashion_mnist_formatted
+
+
+def load_kagl_marqo_dataset(evaluator):
+    """Load and prepare kagl dataset"""
     from datasets import load_dataset
-    print("Loading Kagl KAGL dataset...")
+    print("Loading kagl dataset...")
 
     # Load the dataset
     dataset = load_dataset("Marqo/KAGL")
     df = dataset["data"].to_pandas()
-    print(f"✅ Dataset Kagl loaded")
+    print(f"✅ Dataset kagl loaded")
     print(f"📊 Before filtering: {len(df)} samples")
     print(f"📋 Available columns: {list(df.columns)}")
 
@@ -530,60 +706,103 @@ def load_Kagl_marqo_dataset(evaluator):
     for i, (text, hierarchy) in enumerate(zip(df['text'].head(3), df['hierarchy'].head(3))):
         print(f"  {i+1}. [{hierarchy}] {text[:100]}...")
     
-    print(f"📊 After sampling: {len(df)} samples")
+    print(f"📊 After sampling: {len(df_test)} samples")
     print(f"📊 Samples per hierarchy:")
-    for hierarchy in sorted(df['hierarchy'].unique()):
-        count = len(df[df['hierarchy'] == hierarchy])
+    for hierarchy in sorted(df_test['hierarchy'].unique()):
+        count = len(df_test[df_test['hierarchy'] == hierarchy])
         print(f"  {hierarchy}: {count} samples")
     
     # Create formatted dataset with proper column names
-    Kagl_formatted = pd.DataFrame({
-        'image_url': df['image'],
-        'text': df['text'],
-        'hierarchy': df['hierarchy']
+    kagl_formatted = pd.DataFrame({
+        'image_url': df_test['image'],
+        'text': df_test['text'],
+        'hierarchy': df_test['hierarchy']
     })
     
-    print(f"📊 Final dataset size: {len(Kagl_formatted)} samples")
-    return Kagl_formatted
+    print(f"📊 Final dataset size: {len(kagl_formatted)} samples")
+    return kagl_formatted
+
 
 if __name__ == "__main__":
-    device = config.device
-    model_path = config.hierarchy_model_path
-    directory = config.evaluation_directory
+    directory = "hierarchy_model_analysis"
 
-    print(f"🚀 Starting evaluation with {model_path}")
+    print(f"🚀 Starting evaluation with custom model: {hierarchy_model_path}")
+    print(f"🤗 Including Fashion-CLIP baseline comparison")
     
-    evaluator = EmbeddingEvaluator(model_path, directory)
+    evaluator = EmbeddingEvaluator(hierarchy_model_path, directory)
     
     print(f"📊 Final hierarchy classes after initialization: {len(evaluator.vocab.hierarchy_classes)} classes")
     
     # Evaluate on validation dataset (same subset as during training)
     print("\n" + "="*60)
-    print("EVALUATING VALIDATION DATASET")
+    print("EVALUATING VALIDATION DATASET - CUSTOM MODEL vs FASHION-CLIP BASELINE")
     print("="*60)
-    val_results = evaluator.evaluate_dataset(evaluator.val_df, "Validation Dataset")
+    val_results = evaluator.evaluate_dataset_with_baselines(evaluator.val_df, "Validation Dataset")
     
     print("\n" + "="*60)
-    print("EVALUATING Kagl MARQO DATASET") 
+    print("EVALUATING FASHION-MNIST TEST DATASET - CUSTOM MODEL vs FASHION-CLIP BASELINE") 
     print("="*60)
-    df_Kagl_marqo = load_Kagl_marqo_dataset(evaluator)
-    Kagl_results = evaluator.evaluate_dataset(df_Kagl_marqo, "Kagl Marqo Dataset")
+    df_fashion_mnist = load_fashion_mnist_dataset(evaluator)
+    if len(df_fashion_mnist) > 0:
+        fashion_mnist_results = evaluator.evaluate_dataset_with_baselines(df_fashion_mnist, "Fashion-MNIST Test Dataset")
+    else:
+        fashion_mnist_results = {}
+    
+    print("\n" + "="*60)
+    print("EVALUATING kagl MARQO DATASET - CUSTOM MODEL vs FASHION-CLIP BASELINE") 
+    print("="*60)
+    df_kagl_marqo = load_kagl_marqo_dataset(evaluator)
+    if len(df_kagl_marqo) > 0:
+        kagl_results = evaluator.evaluate_dataset_with_baselines(df_kagl_marqo, "kagl Marqo Dataset")
+    else:
+        kagl_results = {}
     
     # Compare results
-    print(f"\n{'='*60}")
-    print("FINAL EVALUATION SUMMARY")
-    print(f"{'='*60}")
+    print(f"\n{'='*80}")
+    print("FINAL EVALUATION SUMMARY - CUSTOM MODEL vs FASHION-CLIP BASELINE")
+    print(f"{'='*80}")
     
     print("\n🔍 VALIDATION DATASET RESULTS:")
-    print(f"Text      - Separation: {val_results['text']['separation_score']:.4f} | NN Acc: {val_results['text']['accuracy']*100:.1f}% | Centroid Acc: {val_results['text']['centroid_accuracy']*100:.1f}%")
-    print(f"Image     - Separation: {val_results['image']['separation_score']:.4f} | NN Acc: {val_results['image']['accuracy']*100:.1f}% | Centroid Acc: {val_results['image']['centroid_accuracy']*100:.1f}%")
-    print(f"hierarchy     - Separation: {val_results['hierarchy']['separation_score']:.4f} | NN Acc: {val_results['hierarchy']['accuracy']*100:.1f}% | Centroid Acc: {val_results['hierarchy']['centroid_accuracy']*100:.1f}%")
+    print(f"Dataset size: {len(evaluator.val_df)} samples")
+    print(f"{'Model':<20} {'Embedding':<10} {'Sep Score':<12} {'NN Acc':<10} {'Centroid Acc':<12} {'F1 Macro':<10}")
+    print("-" * 80)
     
-    print("\n🌐 Kagl MARQO DATASET RESULTS:")
-    print(f"Text      - Separation: {Kagl_results['text']['separation_score']:.4f} | NN Acc: {Kagl_results['text']['accuracy']*100:.1f}% | Centroid Acc: {Kagl_results['text']['centroid_accuracy']*100:.1f}%")
-    print(f"Image     - Separation: {Kagl_results['image']['separation_score']:.4f} | NN Acc: {Kagl_results['image']['accuracy']*100:.1f}% | Centroid Acc: {Kagl_results['image']['centroid_accuracy']*100:.1f}%")
-    print(f"Hierarchy     - Separation: {Kagl_results['hierarchy']['separation_score']:.4f} | NN Acc: {Kagl_results['hierarchy']['accuracy']*100:.1f}% | Centroid Acc: {Kagl_results['hierarchy']['centroid_accuracy']*100:.1f}%")
+    for model_type in ['custom', 'clip']:
+        for emb_type in ['text', 'image']:
+            key = f"{model_type}_{emb_type}"
+            if key in val_results:
+                metrics = val_results[key]
+                model_name = "Custom Model" if model_type == 'custom' else "CLIP Baseline"
+                print(f"{model_name:<20} {emb_type.capitalize():<10} {metrics['separation_score']:<12.4f} {metrics['accuracy']*100:<10.1f}% {metrics['centroid_accuracy']*100:<12.1f}% {metrics['f1_macro']*100:<10.1f}%")
+
+    if fashion_mnist_results:
+        print("\n👗 FASHION-MNIST TEST DATASET RESULTS:")
+        print(f"Dataset size: {len(df_fashion_mnist)} samples")
+        print(f"{'Model':<20} {'Embedding':<10} {'Sep Score':<12} {'NN Acc':<10} {'Centroid Acc':<12} {'F1 Macro':<10}")
+        print("-" * 80)
+        
+        for model_type in ['custom', 'clip']:
+            for emb_type in ['text', 'image']:
+                key = f"{model_type}_{emb_type}"
+                if key in fashion_mnist_results:
+                    metrics = fashion_mnist_results[key]
+                    model_name = "Custom Model" if model_type == 'custom' else "Fashion-CLIP Baseline"
+                    print(f"{model_name:<20} {emb_type.capitalize():<10} {metrics['separation_score']:<12.4f} {metrics['accuracy']*100:<10.1f}% {metrics['centroid_accuracy']*100:<12.1f}% {metrics['f1_macro']*100:<10.1f}%")
     
+    if kagl_results:
+        print("\n🌐 kagl MARQO DATASET RESULTS:")
+        print(f"Dataset size: {len(df_kagl_marqo)} samples")
+        print(f"{'Model':<20} {'Embedding':<10} {'Sep Score':<12} {'NN Acc':<10} {'Centroid Acc':<12} {'F1 Macro':<10}")
+        print("-" * 80)
+        
+        for model_type in ['custom', 'clip']:
+            for emb_type in ['text', 'image']:
+                key = f"{model_type}_{emb_type}"
+                if key in kagl_results:
+                    metrics = kagl_results[key]
+                    model_name = "Custom Model" if model_type == 'custom' else "Fashion-CLIP Baseline"
+                    print(f"{model_name:<20} {emb_type.capitalize():<10} {metrics['separation_score']:<12.4f} {metrics['accuracy']*100:<10.1f}% {metrics['centroid_accuracy']*100:<12.1f}% {metrics['f1_macro']*100:<10.1f}%")
     
-    print(f"\n✅ Evaluation completed! Check 'improved_model_evaluation/' for visualization files.")
-    print(f"📊 Final hierarchy classes used: {len(evaluator.vocab.hierarchy_classes)} classes")
+    print(f"\n✅ Evaluation completed! Check '{directory}/' for visualization files.")
+    print(f"📊 Custom model hierarchy classes: {len(evaluator.vocab.hierarchy_classes)} classes")
+    print(f"🤗 Fashion-CLIP baseline comparison included")