Delete Evaluation

Evaluation/0_shot_classification.py

@@ -1,512 +0,0 @@
-"""
-Zero-shot classification evaluation on a new dataset.
-This file evaluates the main model's performance on unseen data by performing
-zero-shot classification. It compares three methods: color-to-color classification,
-text-to-text, and image-to-text. It generates confusion matrices and classification reports
-for each method to analyze the model's generalization capability.
-"""
-
-import os
-# Set environment variable to disable tokenizers parallelism warnings
-os.environ["TOKENIZERS_PARALLELISM"] = "false"
-
-import torch
-import torch.nn.functional as F
-import numpy as np
-import pandas as pd
-from torch.utils.data import Dataset
-import matplotlib.pyplot as plt
-from PIL import Image
-from torchvision import transforms
-from transformers import CLIPProcessor, CLIPModel as CLIPModel_transformers
-import warnings
-import config
-from tqdm import tqdm
-from sklearn.metrics import accuracy_score, confusion_matrix, classification_report
-import seaborn as sns
-from color_model import CLIPModel as ColorModel
-from hierarchy_model import Model, HierarchyExtractor
-
-# Suppress warnings
-warnings.filterwarnings("ignore", category=FutureWarning)
-warnings.filterwarnings("ignore", category=UserWarning)
-
-def load_trained_model(model_path, device):
-    """
-    Load the trained CLIP model from checkpoint
-    """
-    print(f"Loading trained model from: {model_path}")
-    
-    # Load checkpoint
-    checkpoint = torch.load(model_path, map_location=device)
-    
-    # Create the base CLIP model
-    model = CLIPModel_transformers.from_pretrained('laion/CLIP-ViT-B-32-laion2B-s34B-b79K')
-    
-    # Load the trained weights
-    model.load_state_dict(checkpoint['model_state_dict'])
-    model = model.to(device)
-    model.eval()
-    
-    print(f"✅ Model loaded successfully!")
-    print(f"📊 Training epoch: {checkpoint['epoch']}")
-    print(f"📉 Best validation loss: {checkpoint['best_val_loss']:.4f}")
-    
-    return model, checkpoint
-
-def load_feature_models(device):
-    """Load feature models (color and hierarchy)"""
-
-    # Load color model (embed_dim=16)
-    color_checkpoint = torch.load(config.color_model_path, map_location=device, weights_only=True)
-    color_model = ColorModel(embed_dim=config.color_emb_dim).to(device)
-    color_model.load_state_dict(color_checkpoint)
-    color_model.eval()
-    color_model.name = 'color'
-
-    # Load hierarchy model (embed_dim=64)
-    hierarchy_checkpoint = torch.load(config.hierarchy_model_path, map_location=device)
-    hierarchy_classes = hierarchy_checkpoint.get('hierarchy_classes', [])
-    hierarchy_model = Model(
-        num_hierarchy_classes=len(hierarchy_classes),
-        embed_dim=config.hierarchy_emb_dim
-    ).to(device)
-    hierarchy_model.load_state_dict(hierarchy_checkpoint['model_state'])
-
-    # Set up hierarchy extractor
-    hierarchy_extractor = HierarchyExtractor(hierarchy_classes, verbose=False)
-    hierarchy_model.set_hierarchy_extractor(hierarchy_extractor)
-    hierarchy_model.eval()
-    hierarchy_model.name = 'hierarchy'
-
-    feature_models = {model.name: model for model in [color_model, hierarchy_model]}
-    return feature_models
-
-def get_image_embedding(model, image, device):
-    """Get image embedding from the trained model"""
-    model.eval()
-    with torch.no_grad():
-        # Ensure image has 3 channels
-        if image.dim() == 3 and image.size(0) == 1:
-            image = image.expand(3, -1, -1)
-        elif image.dim() == 4 and image.size(1) == 1:
-            image = image.expand(-1, 3, -1, -1)
-        
-        # Add batch dimension if missing
-        if image.dim() == 3:
-            image = image.unsqueeze(0)  # Add batch dimension: (C, H, W) -> (1, C, H, W)
-        
-        image = image.to(device)
-        
-        # Use vision model directly to get image embeddings
-        vision_outputs = model.vision_model(pixel_values=image)
-        image_features = model.visual_projection(vision_outputs.pooler_output)
-        
-        return F.normalize(image_features, dim=-1)
-
-def get_text_embedding(model, text, processor, device):
-    """Get text embedding from the trained model"""
-    model.eval()
-    with torch.no_grad():
-        text_inputs = processor(text=text, padding=True, return_tensors="pt")
-        text_inputs = {k: v.to(device) for k, v in text_inputs.items()}
-        
-        # Use text model directly to get text embeddings
-        text_outputs = model.text_model(**text_inputs)
-        text_features = model.text_projection(text_outputs.pooler_output)
-        
-        return F.normalize(text_features, dim=-1)
-
-def evaluate_custom_csv_accuracy(model, dataset, processor, method='similarity'):
-    """
-    Evaluate the accuracy of the model on your custom CSV using text-to-text similarity
-    
-    Args:
-        model: The trained CLIP model
-        dataset: CustomCSVDataset
-        processor: CLIPProcessor
-        method: 'similarity' or 'classification'
-    """
-    print(f"\n📊 === Evaluation of the accuracy on custom CSV (TEXT-TO-TEXT method) ===")
-    
-    model.eval()
-    
-    # Get all unique colors for classification
-    all_colors = set()
-    for i in range(len(dataset)):
-        _, _, color = dataset[i]
-        all_colors.add(color)
-    
-    color_list = sorted(list(all_colors))
-    print(f"🎨 Colors found: {color_list}")
-    
-    true_labels = []
-    predicted_labels = []
-    
-    # Pre-calculate the embeddings of the color descriptions
-    print("🔄 Pre-calculating the embeddings of the colors...")
-    color_embeddings = {}
-    for color in color_list:
-        color_emb = get_text_embedding(model, color, processor)
-        color_embeddings[color] = color_emb
-    
-    print("🔄 Evaluation in progress...")
-    correct_predictions = 0
-    
-    for idx in tqdm(range(len(dataset)), desc="Evaluation"):
-        image, text, true_color = dataset[idx]
-        
-        # Get text embedding instead of image embedding
-        text_emb = get_text_embedding(model, text, processor)
-        
-        # Calculate the similarity with each possible color
-        best_similarity = -1
-        predicted_color = color_list[0]
-        
-        for color, color_emb in color_embeddings.items():
-            similarity = F.cosine_similarity(text_emb, color_emb, dim=1).item()
-            if similarity > best_similarity:
-                best_similarity = similarity
-                predicted_color = color
-        
-        true_labels.append(true_color)
-        predicted_labels.append(predicted_color)
-        
-        if true_color == predicted_color:
-            correct_predictions += 1
-    
-    # Calculate the accuracy
-    accuracy = accuracy_score(true_labels, predicted_labels)
-    
-    print(f"\n✅ Results of evaluation:")
-    print(f"🎯 Global accuracy: {accuracy:.4f} ({accuracy*100:.2f}%)")
-    print(f"📊 Correct predictions: {correct_predictions}/{len(true_labels)}")
-    
-    return true_labels, predicted_labels, accuracy
-
-def evaluate_custom_csv_accuracy_image(model, dataset, processor, method='similarity'):
-    """
-    Evaluate the accuracy of the model on your custom CSV using image-to-text similarity
-    
-    Args:
-        model: The trained CLIP model
-        dataset: CustomCSVDataset with images loaded
-        processor: CLIPProcessor
-        method: 'similarity' or 'classification'
-    """
-    print(f"\n📊 === Evaluation of the accuracy on custom CSV (IMAGE-TO-TEXT method) ===")
-    
-    model.eval()
-    
-    # Get all unique colors for classification
-    all_colors = set()
-    for i in range(len(dataset)):
-        _, _, color = dataset[i]
-        all_colors.add(color)
-    
-    color_list = sorted(list(all_colors))
-    print(f"🎨 Colors found: {color_list}")
-    
-    true_labels = []
-    predicted_labels = []
-    
-    # Pre-calculate the embeddings of the color descriptions
-    print("🔄 Pre-calculating the embeddings of the colors...")
-    color_embeddings = {}
-    for color in color_list:
-        color_emb = get_text_embedding(model, color, processor)
-        color_embeddings[color] = color_emb
-    
-    print("🔄 Evaluation in progress...")
-    correct_predictions = 0
-    
-    for idx in tqdm(range(len(dataset)), desc="Evaluation"):
-        image, text, true_color = dataset[idx]
-        
-        # Get image embedding (this is the key difference from text-to-text)
-        image_emb = get_image_embedding(model, image, processor)
-        
-        # Calculate the similarity with each possible color
-        best_similarity = -1
-        predicted_color = color_list[0]
-        
-        for color, color_emb in color_embeddings.items():
-            similarity = F.cosine_similarity(image_emb, color_emb, dim=1).item()
-            if similarity > best_similarity:
-                best_similarity = similarity
-                predicted_color = color
-        
-        true_labels.append(true_color)
-        predicted_labels.append(predicted_color)
-        
-        if true_color == predicted_color:
-            correct_predictions += 1
-    
-    # Calculate the accuracy
-    accuracy = accuracy_score(true_labels, predicted_labels)
-    
-    print(f"\n✅ Results of evaluation:")
-    print(f"🎯 Global accuracy: {accuracy:.4f} ({accuracy*100:.2f}%)")
-    print(f"📊 Correct predictions: {correct_predictions}/{len(true_labels)}")
-    
-    return true_labels, predicted_labels, accuracy
-
-def evaluate_custom_csv_accuracy_color_only(model, dataset, processor):
-    """
-    Evaluate the accuracy by encoding ONLY the color (not the full text)
-    This tests if the embedding space is consistent for colors
-    
-    Args:
-        model: The trained CLIP model
-        dataset: CustomCSVDataset
-        processor: CLIPProcessor
-    """
-    print(f"\n📊 === Evaluation of the accuracy on custom CSV (COLOR-TO-COLOR method) ===")
-    print("🔬 This test encodes ONLY the color name, not the full text")
-    
-    model.eval()
-    
-    # Get all unique colors for classification
-    all_colors = set()
-    for i in range(len(dataset)):
-        _, _, color = dataset[i]
-        all_colors.add(color)
-    
-    color_list = sorted(list(all_colors))
-    print(f"🎨 Colors found: {color_list}")
-    
-    true_labels = []
-    predicted_labels = []
-    
-    # Pre-calculate the embeddings of the color descriptions
-    print("🔄 Pre-calculating the embeddings of the colors...")
-    color_embeddings = {}
-    for color in color_list:
-        color_emb = get_text_embedding(model, color, processor)
-        color_embeddings[color] = color_emb
-    
-    print("🔄 Evaluation in progress...")
-    correct_predictions = 0
-    
-    for idx in tqdm(range(len(dataset)), desc="Evaluation"):
-        image, text, true_color = dataset[idx]
-        
-        # KEY DIFFERENCE: Get embedding of the TRUE COLOR only (not the full text)
-        true_color_emb = get_text_embedding(model, true_color, processor)
-        
-        # Calculate the similarity with each possible color
-        best_similarity = -1
-        predicted_color = color_list[0]
-        
-        for color, color_emb in color_embeddings.items():
-            similarity = F.cosine_similarity(true_color_emb, color_emb, dim=1).item()
-            if similarity > best_similarity:
-                best_similarity = similarity
-                predicted_color = color
-        
-        true_labels.append(true_color)
-        predicted_labels.append(predicted_color)
-        
-        if true_color == predicted_color:
-            correct_predictions += 1
-    
-    # Calculate the accuracy
-    accuracy = accuracy_score(true_labels, predicted_labels)
-    
-    print(f"\n✅ Results of evaluation:")
-    print(f"🎯 Global accuracy: {accuracy:.4f} ({accuracy*100:.2f}%)")
-    print(f"📊 Correct predictions: {correct_predictions}/{len(true_labels)}")
-    
-    return true_labels, predicted_labels, accuracy
-
-def search_custom_csv_by_text(model, dataset, query, processor, top_k=5):
-    """Search in your CSV by text query"""
-    print(f"\n🔍 Search in custom CSV: '{query}'")
-    
-    # Get the embedding of the query
-    query_emb = get_text_embedding(model, query, processor)
-    
-    similarities = []
-    
-    print("🔄 Calculating similarities...")
-    for idx in tqdm(range(len(dataset)), desc="Processing"):
-        image, text, color, _, image_path = dataset[idx]
-        
-        # Get the embedding of the image
-        image_emb = get_image_embedding(model, image, processor)
-        
-        # Calculer la similarité
-        similarity = F.cosine_similarity(query_emb, image_emb, dim=1).item()
-        
-        similarities.append((idx, similarity, text, color, color, image_path))
-    
-    # Trier par similarité
-    similarities.sort(key=lambda x: x[1], reverse=True)
-    
-    return similarities[:top_k]
-
-def plot_confusion_matrix(true_labels, predicted_labels, save_path=None, title_suffix="text"):
-    """
-    Display and save the confusion matrix
-    """
-    print("\n📈 === Generation of the confusion matrix ===")
-
-    # Calculate the confusion matrix
-    cm = confusion_matrix(true_labels, predicted_labels)
-    
-    # Get unique labels in sorted order
-    unique_labels = sorted(set(true_labels + predicted_labels))
-    
-    # Calculate accuracy
-    accuracy = accuracy_score(true_labels, predicted_labels)
-    
-    # Calculate the percentages and round to integers
-    cm_percent = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] * 100
-    cm_percent = np.around(cm_percent).astype(int)
-    
-    # Create the figure
-    plt.figure(figsize=(12, 10))
-    
-    # Confusion matrix with percentages and labels (no decimal points)
-    sns.heatmap(cm_percent, 
-                annot=True, 
-                fmt='d', 
-                cmap='Blues',
-                cbar_kws={'label': 'Percentage (%)'},
-                xticklabels=unique_labels,
-                yticklabels=unique_labels)
-    
-    plt.title(f"Confusion Matrix for {title_suffix} - new data - accuracy: {accuracy:.4f} ({accuracy*100:.2f}%)", fontsize=16)
-    plt.xlabel('Predictions', fontsize=12)
-    plt.ylabel('True colors', fontsize=12)
-    plt.xticks(rotation=45, ha='right')
-    plt.yticks(rotation=0)
-    plt.tight_layout()
-    
-    if save_path:
-        plt.savefig(save_path, dpi=300, bbox_inches='tight')
-        print(f"💾 Confusion matrix saved: {save_path}")
-    
-    plt.show()
-    
-    return cm
-
-class CustomCSVDataset(Dataset):
-    def __init__(self, dataframe, image_size=224, load_images=True):
-        self.dataframe = dataframe
-        self.image_size = image_size
-        self.load_images = load_images
-        
-        # Define image transformations
-        self.transform = transforms.Compose([
-            transforms.Resize((image_size, image_size)),
-            transforms.ToTensor(),
-            transforms.Normalize(mean=[0.48145466, 0.4578275, 0.40821073],
-                               std=[0.26862954, 0.26130258, 0.27577711])
-        ])
-        
-    def __len__(self):
-        return len(self.dataframe)
-
-    def __getitem__(self, idx):
-        row = self.dataframe.iloc[idx]
-        text = row[config.text_column] 
-        colors = row[config.color_column]
-        
-        if self.load_images and config.column_local_image_path in row:
-            # Load the actual image
-            try:
-                image = Image.open(row[config.column_local_image_path]).convert('RGB')
-                image = self.transform(image)
-            except Exception as e:
-                print(f"Warning: Could not load image {row.get(config.column_local_image_path, 'unknown')}: {e}")
-                image = torch.zeros(3, self.image_size, self.image_size)
-        else:
-            # Return dummy image if not loading images
-            image = torch.zeros(3, self.image_size, self.image_size)
-            
-        return image, text, colors
-
-if __name__ == "__main__":
-    """Main function with evaluation"""
-    print("🚀 === Test and Evaluation of the model on new dataset ===")
-    
-    # Load model
-    print("🔧 Loading the model...")
-    model, checkpoint = load_trained_model(config.main_model_path, config.device)
-    
-    # Create processor
-    processor = CLIPProcessor.from_pretrained('laion/CLIP-ViT-B-32-laion2B-s34B-b79K')
-    
-    # Load new dataset
-    print("📊 Loading the new dataset...")
-    df = pd.read_csv(config.local_dataset_path) # replace local_dataset_path with a new df
-
-    print("\n" + "="*80)
-    print("🎨 COLOR-TO-COLOR CLASSIFICATION (Control Test)")
-    print("="*80)
-    
-    # Create dataset without loading images
-    dataset_color = CustomCSVDataset(df, load_images=False)
-    
-    # 0. Evaluation encoding ONLY the color (control test)
-    true_labels_color, predicted_labels_color, accuracy_color = evaluate_custom_csv_accuracy_color_only(
-        model, dataset_color, processor
-    )
-    
-    # Confusion matrix for color-only
-    confusion_matrix_color = plot_confusion_matrix(
-        true_labels_color, predicted_labels_color, 
-        save_path="confusion_matrix_color_only.png",
-        title_suffix="color-only"
-    )
-    
-    print("\n" + "="*80)
-    print("📝 TEXT-TO-TEXT CLASSIFICATION")
-    print("="*80)
-    
-    # Create dataset without loading images for text-to-text
-    dataset_text = CustomCSVDataset(df, load_images=False)
-    
-    # 1. Evaluation of the accuracy (text-to-text)
-    true_labels_text, predicted_labels_text, accuracy_text = evaluate_custom_csv_accuracy(
-        model, dataset_text, processor, method='similarity'
-    )
-    
-    # 2. Confusion matrix for text
-    confusion_matrix_text = plot_confusion_matrix(
-        true_labels_text, predicted_labels_text, 
-        save_path="confusion_matrix_text.png",
-        title_suffix="text"
-    )
-    
-    print("\n" + "="*80)
-    print("🖼️  IMAGE-TO-TEXT CLASSIFICATION")
-    print("="*80)
-    
-    # Create dataset with images loaded for image-to-text
-    dataset_image = CustomCSVDataset(df, load_images=True)
-    
-    # 3. Evaluation of the accuracy (image-to-text)
-    true_labels_image, predicted_labels_image, accuracy_image = evaluate_custom_csv_accuracy_image(
-        model, dataset_image, processor, method='similarity'
-    )
-    
-    # 4. Confusion matrix for images
-    confusion_matrix_image = plot_confusion_matrix(
-        true_labels_image, predicted_labels_image, 
-        save_path="confusion_matrix_image.png",
-        title_suffix="image"
-    )
-    
-    # 5. Summary comparison
-    print("\n" + "="*80)
-    print("📊 SUMMARY")
-    print("="*80)
-    print(f"🎨 Color-to-Color Accuracy (Control): {accuracy_color:.4f} ({accuracy_color*100:.2f}%)")
-    print(f"📝 Text-to-Text Accuracy: {accuracy_text:.4f} ({accuracy_text*100:.2f}%)")
-    print(f"🖼️  Image-to-Text Accuracy: {accuracy_image:.4f} ({accuracy_image*100:.2f}%)")
-    print(f"\n📊 Analysis:")
-    print(f"   • Loss from full text vs color-only: {abs(accuracy_color - accuracy_text):.4f} ({abs(accuracy_color - accuracy_text)*100:.2f}%)")
-    print(f"   • Difference text vs image: {abs(accuracy_text - accuracy_image):.4f} ({abs(accuracy_text - accuracy_image)*100:.2f}%)")
-    

Evaluation/basic_test_generalized.py

@@ -1,425 +0,0 @@
-"""
-Generalized evaluation of the main model with sub-module comparison.
-This file evaluates the main model's performance by comparing specialized parts
-(color and hierarchy) with corresponding specialized models. It calculates similarity
-matrices, linear projections between embedding spaces, and generates detailed statistics
-on alignment between different representations.
-"""
-
-import os
-import json
-import argparse
-import config
-import torch
-import torch.nn.functional as F
-import pandas as pd
-from PIL import Image
-from torchvision import transforms
-from transformers import CLIPProcessor, CLIPModel as CLIPModelTransformers
-from tqdm.auto import tqdm
-
-# Local imports
-from color_model import ColorCLIP as ColorModel, ColorDataset, Tokenizer
-from config import color_model_path, color_emb_dim, device, hierarchy_model_path, hierarchy_emb_dim
-from hierarchy_model import Model as HierarchyModel, HierarchyExtractor
-
-
-def load_color_model(color_model_path, color_emb_dim, device):
-    # Load color model
-    color_checkpoint = torch.load(color_model_path, map_location=device, weights_only=True)
-    color_model = ColorModel(vocab_size=39, embedding_dim=color_emb_dim).to(device)
-    color_model.load_state_dict(color_checkpoint)
-
-    # Load and set the tokenizer
-    tokenizer = Tokenizer()
-    with open(config.tokeniser_path, 'r') as f:
-        vocab_dict = json.load(f)
-    color_model.tokenizer = tokenizer 
-
-    color_model.eval()
-    return color_model
-
-
-def get_emb_color_model(color_model, image_path_to_encode, text_to_encode):
-    # Load and preprocess image
-    image = Image.open(image_path_to_encode).convert('RGB')
-
-    transform = transforms.Compose([
-        transforms.Resize((224, 224)),
-        transforms.ToTensor(),
-        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
-    ])
-
-    processed_image = transform(image)
-
-    # Get embeddings
-    processed_image_batch = processed_image.unsqueeze(0).to(device)  # Shape: [1, 3, 224, 224]
-    with torch.no_grad():
-        image_emb = color_model.image_encoder(processed_image_batch)
-
-    # Text embedding via tokenizer + text_encoder
-    token_ids = torch.tensor([color_model.tokenizer(text_to_encode)], dtype=torch.long, device=device)
-    lengths = torch.tensor([token_ids.size(1) if token_ids.dim() > 1 else token_ids.size(0)], dtype=torch.long, device=device)
-    with torch.no_grad():
-        txt_emb = color_model.text_encoder(token_ids, lengths)
-
-    return image_emb, txt_emb
-
-def load_main_model(main_model_path, device):
-    checkpoint = torch.load(main_model_path, map_location=device)
-    main_model = CLIPModel_transformers.from_pretrained('laion/CLIP-ViT-B-32-laion2B-s34B-b79K')
-    state = checkpoint['model_state_dict'] if isinstance(checkpoint, dict) and 'model_state_dict' in checkpoint else checkpoint
-    try:
-        main_model.load_state_dict(state, strict=False)
-    except Exception:
-        # Fallback: filter matching keys
-        model_state = main_model.state_dict()
-        filtered = {k: v for k, v in state.items() if k in model_state and model_state[k].shape == v.shape}
-        main_model.load_state_dict(filtered, strict=False)
-    main_model.to(device)
-    main_model.eval()
-    processor = CLIPProcessor.from_pretrained('laion/CLIP-ViT-B-32-laion2B-s34B-b79K')
-    return main_model, processor
-
-
-def load_hierarchy_model(hierarchy_model_path, device):
-    checkpoint = torch.load(hierarchy_model_path, map_location=device)
-    hierarchy_classes = checkpoint.get('hierarchy_classes', [])
-    model = HierarchyModel(num_hierarchy_classes=len(hierarchy_classes), embed_dim=config.hierarchy_emb_dim).to(device)
-    model.load_state_dict(checkpoint['model_state'])
-    extractor = HierarchyExtractor(hierarchy_classes, verbose=False)
-    model.set_hierarchy_extractor(extractor)
-    model.eval()
-    return model
-
-
-def get_emb_hierarchy_model(hierarchy_model, image_path_to_encode, text_to_encode):
-    image = Image.open(image_path_to_encode).convert('RGB')
-    transform = transforms.Compose([
-        transforms.Resize((224, 224)),
-        transforms.ToTensor(),
-    ])
-    image_tensor = transform(image).unsqueeze(0).to(device)
-
-    with torch.no_grad():
-        img_emb = hierarchy_model.get_image_embeddings(image_tensor)
-        txt_emb = hierarchy_model.get_text_embeddings(text_to_encode)
-
-    return img_emb, txt_emb
-
-def get_emb_main_model(main_model, processor, image_path_to_encode, text_to_encode):
-    image = Image.open(image_path_to_encode).convert('RGB')
-    transform = transforms.Compose([
-        transforms.Resize((224, 224)),
-        transforms.ToTensor(),
-        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
-    ])
-    image = transform(image)
-    image = image.unsqueeze(0).to(device)
-    # Prepare text inputs via processor
-    text_inputs = processor(text=[text_to_encode], return_tensors="pt", padding=True)
-    text_inputs = {k: v.to(device) for k, v in text_inputs.items()}
-    outputs = main_model(**text_inputs, pixel_values=image)
-    text_emb = outputs.text_embeds
-    image_emb = outputs.image_embeds
-
-    return text_emb, image_emb
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Evaluate main model parts vs small models and build similarity matrices')
-    parser.add_argument('--main-checkpoint', type=str, default='models/laion_explicable_model.pth')
-    parser.add_argument('--color-checkpoint', type=str, default='models/color_model.pt')
-    parser.add_argument('--csv', type=str, default='data/data_with_local_paths.csv')
-    parser.add_argument('--color-emb-dim', type=int, default=16)
-    parser.add_argument('--num-samples', type=int, default=200)
-    parser.add_argument('--seed', type=int, default=42)
-    parser.add_argument('--primary-metric', type=str, default='sim_color_txt_img',
-                        choices=['sim_txt_color_part', 'sim_img_color_part', 'sim_color_txt_img', 'sim_small_txt_img',
-                                 'sim_txt_hierarchy_part', 'sim_img_hierarchy_part'])
-    parser.add_argument('--top-k', type=int, default=30)
-    parser.add_argument('--heatmap', action='store_true')
-    parser.add_argument('--l2-grid', type=str, default='1e-5,1e-4,1e-3,1e-2,1e-1')
-    args = parser.parse_args()
-
-    main_checkpoint = args.main_checkpoint
-    color_checkpoint = args.color_checkpoint
-    csv = args.csv
-    color_emb_dim = args.color_emb_dim
-    num_samples = args.num_samples
-    seed = args.seed
-    primary_metric = args.primary_metric
-    top_k = args.top_k
-    l2_grid = [float(x) for x in args.l2_grid.split(',') if x]
-    device = torch.device("mps")
-
-    df = pd.read_csv(csv)
-
-    # Normalize colors (reduce aliasing and sparsity)
-    def normalize_color(c):
-        if pd.isna(c):
-            return c
-        s = str(c).strip().lower()
-        aliases = {
-            'grey': 'gray',
-            'navy blue': 'navy',
-            'light blue': 'blue',
-            'dark blue': 'blue',
-            'light grey': 'gray',
-            'dark grey': 'gray',
-            'light gray': 'gray',
-            'dark gray': 'gray',
-        }
-        return aliases.get(s, s)
-
-    if config.color_column in df.columns:
-        df[config.color_column] = df[config.color_column].apply(normalize_color)
-
-    color_model = load_color_model(color_checkpoint, color_emb_dim, device)
-    main_model, processor = load_main_model(main_checkpoint, device)
-    hierarchy_model = load_hierarchy_model(hierarchy_model_path, device)
-
-    # Results container
-    results = []
-
-    # Accumulators for projection (A: main part, B: small model)
-    color_txt_As, color_txt_Bs = [], []
-    color_img_As, color_img_Bs = [], []
-    hier_txt_As, hier_txt_Bs = [], []
-    hier_img_As, hier_img_Bs = [], []
-
-    # Ensure determinism for sampling
-    pd.options.mode.copy_on_write = True
-    rng = pd.Series(range(len(df)), dtype=int)
-    _ = rng  # silence lint
-    torch.manual_seed(seed)
-
-    unique_hiers = sorted(df[config.hierarchy_column].dropna().unique())
-    unique_colors = sorted(df[config.color_column].dropna().unique())
-
-    # Progress bar across all (hierarchy, color) pairs
-    total_pairs = len(unique_hiers) * len(unique_colors)
-    pair_pbar = tqdm(total=total_pairs, desc="Evaluating pairs", leave=False)
-    for hierarchy in unique_hiers:
-        for color in unique_colors:
-            group = df[(df[config.hierarchy_column] == hierarchy) & (df[config.color_column] == color)]
-
-            # Sample up to num_samples per (hierarchy, color)
-            k = min(num_samples, len(group))
-            group_iter = group.sample(n=k, random_state=seed) if len(group) > k else group.iloc[:k]
-
-            # Progress bar for samples within the pair
-            inner_pbar = tqdm(total=len(group_iter), desc=f"{hierarchy}/{color}", leave=False)
-            for row_idx, (_, example) in enumerate(group_iter.iterrows()):
-                try:
-                    image_emb, txt_emb = get_emb_color_model(color_model, example['local_image_path'], example['text'])
-                    image_emb_hier, txt_emb_hier = get_emb_hierarchy_model(hierarchy_model, example['local_image_path'], example['text'])
-                    text_emb_main_model, image_emb_main_model = get_emb_main_model(
-                        main_model, processor, example['local_image_path'], example['text']
-                    )
-
-                    color_part_txt = text_emb_main_model[:, :color_emb_dim]
-                    color_part_img = image_emb_main_model[:, :color_emb_dim]
-                    hier_part_txt = text_emb_main_model[:, color_emb_dim:color_emb_dim + hierarchy_emb_dim]
-                    hier_part_img = image_emb_main_model[:, color_emb_dim:color_emb_dim + hierarchy_emb_dim]
-
-                    # L2-normalize parts and small-model embeddings for stable cosine
-                    color_part_txt = F.normalize(color_part_txt, dim=1)
-                    color_part_img = F.normalize(color_part_img, dim=1)
-                    hier_part_txt = F.normalize(hier_part_txt, dim=1)
-                    hier_part_img = F.normalize(hier_part_img, dim=1)
-                    txt_emb = F.normalize(txt_emb, dim=1)
-                    image_emb = F.normalize(image_emb, dim=1)
-                    txt_emb_hier = F.normalize(txt_emb_hier, dim=1)
-                    image_emb_hier = F.normalize(image_emb_hier, dim=1)
-
-                    sim_txt_color_part = F.cosine_similarity(txt_emb, color_part_txt).item()
-                    sim_img_color_part = F.cosine_similarity(image_emb, color_part_img).item()
-                    sim_color_txt_img = F.cosine_similarity(color_part_txt, color_part_img).item()
-                    sim_small_txt_img = F.cosine_similarity(txt_emb, image_emb).item()
-
-                    sim_txt_hierarchy_part = F.cosine_similarity(txt_emb_hier, hier_part_txt).item()
-                    sim_img_hierarchy_part = F.cosine_similarity(image_emb_hier, hier_part_img).item()
-
-                    # Accumulate for projection fitting later
-                    color_txt_As.append(color_part_txt.squeeze(0).detach().cpu())
-                    color_txt_Bs.append(txt_emb.squeeze(0).detach().cpu())
-                    color_img_As.append(color_part_img.squeeze(0).detach().cpu())
-                    color_img_Bs.append(image_emb.squeeze(0).detach().cpu())
-
-                    hier_txt_As.append(hier_part_txt.squeeze(0).detach().cpu())
-                    hier_txt_Bs.append(txt_emb_hier.squeeze(0).detach().cpu())
-                    hier_img_As.append(hier_part_img.squeeze(0).detach().cpu())
-                    hier_img_Bs.append(image_emb_hier.squeeze(0).detach().cpu())
-
-                    results.append({
-                        'hierarchy' "hierarchy",
-                        'color': color,
-                        'row_index': int(row_idx),
-                        'sim_txt_color_part': float(sim_txt_color_part),
-                        'sim_img_color_part': float(sim_img_color_part),
-                        'sim_color_txt_img': float(sim_color_txt_img),
-                        'sim_small_txt_img': float(sim_small_txt_img),
-                        'sim_txt_hierarchy_part': float(sim_txt_hierarchy_part),
-                        'sim_img_hierarchy_part': float(sim_img_hierarchy_part),
-                    })
-                except Exception as e:
-                    print(f"Skipping example due to error: {e}")
-                finally:
-                    inner_pbar.update(1)
-            inner_pbar.close()
-            pair_pbar.update(1)
-    pair_pbar.close()
-
-    results_df = pd.DataFrame(results)
-
-    # Save raw results
-    os.makedirs('evaluation_outputs', exist_ok=True)
-    raw_path = os.path.join('evaluation_outputs', 'similarities_raw.csv')
-    results_df.to_csv(raw_path, index=False)
-    print(f"Saved raw similarities to {raw_path}")
-
-    # Intelligent averages
-    metrics = ['sim_txt_color_part', 'sim_img_color_part', 'sim_color_txt_img', 'sim_small_txt_img',
-               'sim_txt_hierarchy_part', 'sim_img_hierarchy_part']
-
-    # Overall means
-    overall_means = results_df[metrics].mean().to_frame(name='mean').T
-    overall_means.insert(0, 'level', 'overall')
-
-    # By hierarchy
-    by_hierarchy = results_df.groupby(config.hierarchy_column)[metrics].mean().reset_index()
-    by_hierarchy.insert(0, 'level', config.hierarchy_column)
-
-    # By color
-    by_color = results_df.groupby(config.color_column)[metrics].mean().reset_index()
-    by_color.insert(0, 'level', config.color_column)
-
-    # By hierarchy+color
-    by_pair = results_df.groupby([config.hierarchy_column, config.color_column])[metrics].mean().reset_index()
-    by_pair.insert(0, 'level', 'hierarchy_color')
-
-    summary_df = pd.concat([overall_means, by_hierarchy, by_color, by_pair], ignore_index=True)
-    summary_path = os.path.join('evaluation_outputs', 'similarities_summary.csv')
-    summary_df.to_csv(summary_path, index=False)
-    print(f"Saved summary statistics to {summary_path}")
-
-    # =====================
-    # Similarity matrices for best hierarchy-color combinations
-    # =====================
-    try:
-        by_pair_core = results_df.groupby([config.hierarchy_column, config.color_column])[metrics].mean().reset_index()
-        top_pairs = by_pair_core.nlargest(top_k, primary_metric)
-        matrix = top_pairs.pivot(index=config.hierarchy_column, columns=config.color_column, values=primary_metric)
-        os.makedirs('evaluation_outputs', exist_ok=True)
-        matrix_csv_path = os.path.join('evaluation_outputs', f'similarity_matrix_{primary_metric}_top{top_k}.csv')
-        matrix.to_csv(matrix_csv_path)
-        print(f"Saved similarity matrix to {matrix_csv_path}")
-
-        if args.heatmap:
-            try:
-                import seaborn as sns
-                import matplotlib.pyplot as plt
-                plt.figure(figsize=(max(6, 0.5 * len(matrix.columns)), max(4, 0.5 * len(matrix.index))))
-                sns.heatmap(matrix, annot=False, cmap='viridis')
-                plt.title(f'Similarity matrix (top {top_k}) - {primary_metric}')
-                heatmap_path = os.path.join('evaluation_outputs', f'similarity_matrix_{primary_metric}_top{top_k}.png')
-                plt.tight_layout()
-                plt.savefig(heatmap_path, dpi=200)
-                plt.close()
-                print(f"Saved similarity heatmap to {heatmap_path}")
-            except Exception as e:
-                print(f"Skipping heatmap generation: {e}")
-    except Exception as e:
-        print(f"Skipping matrix generation: {e}")
-
-    # =====================
-    # Learn projections A->B and report projected cosine means
-    # =====================
-    def fit_ridge_projection(A, B, l2_reg=1e-3):
-        # A: [N, D_in], B: [N, D_out]
-        A = torch.stack(A)  # [N, D_in]
-        B = torch.stack(B)  # [N, D_out]
-        # Closed-form ridge: W = (A^T A + λI)^-1 A^T B
-        AtA = A.T @ A
-        D_in = AtA.shape[0]
-        AtA_reg = AtA + l2_reg * torch.eye(D_in)
-        W = torch.linalg.solve(AtA_reg, A.T @ B)
-        return W  # [D_in, D_out]
-
-    def fit_ridge_with_cv(A, B, l2_values):
-        # Simple holdout CV: 80/20 split
-        if len(A) < 10:
-            # Not enough data for split; fallback to middle lambda
-            best_l2 = l2_values[min(len(l2_values) // 2, len(l2_values)-1)]
-            W = fit_ridge_projection(A, B, best_l2)
-            return W, best_l2, None
-
-        N = len(A)
-        idx = torch.randperm(N)
-        split = int(0.8 * N)
-        train_idx = idx[:split]
-        val_idx = idx[split:]
-
-        A_tensor = torch.stack(A)
-        B_tensor = torch.stack(B)
-
-        A_train, B_train = A_tensor[train_idx], B_tensor[train_idx]
-        A_val, B_val = A_tensor[val_idx], B_tensor[val_idx]
-
-        def to_list(t):
-            return [row for row in t]
-
-        best_l2 = None
-        best_score = -1.0
-        for l2 in l2_values:
-            W = fit_ridge_projection(to_list(A_train), to_list(B_train), l2)
-            score = mean_projected_cosine(to_list(A_val), to_list(B_val), W)
-            if score > best_score:
-                best_score = score
-                best_l2 = l2
-
-        # Refit on all with best_l2
-        W_best = fit_ridge_projection(A, B, best_l2)
-        return W_best, best_l2, best_score
-
-    def mean_projected_cosine(A, B, W):
-        A = torch.stack(A)
-        B = torch.stack(B)
-        A_proj = A @ W
-        A_proj = F.normalize(A_proj, dim=1)
-        B = F.normalize(B, dim=1)
-        return torch.mean(torch.sum(A_proj * B, dim=1)).item()
-
-    projection_report = {}
-
-    if len(color_txt_As) >= 8:
-        W_ct, best_l2_ct, cv_ct = fit_ridge_with_cv(color_txt_As, color_txt_Bs, l2_grid)
-        projection_report['proj_sim_txt_color_part_mean'] = mean_projected_cosine(color_txt_As, color_txt_Bs, W_ct)
-        projection_report['proj_txt_color_part_best_l2'] = best_l2_ct
-        if cv_ct is not None:
-            projection_report['proj_txt_color_part_cv_val'] = cv_ct
-    if len(color_img_As) >= 8:
-        W_ci, best_l2_ci, cv_ci = fit_ridge_with_cv(color_img_As, color_img_Bs, l2_grid)
-        projection_report['proj_sim_img_color_part_mean'] = mean_projected_cosine(color_img_As, color_img_Bs, W_ci)
-        projection_report['proj_img_color_part_best_l2'] = best_l2_ci
-        if cv_ci is not None:
-            projection_report['proj_img_color_part_cv_val'] = cv_ci
-    if len(hier_txt_As) >= 8:
-        W_ht, best_l2_ht, cv_ht = fit_ridge_with_cv(hier_txt_As, hier_txt_Bs, l2_grid)
-        projection_report['proj_sim_txt_hierarchy_part_mean'] = mean_projected_cosine(hier_txt_As, hier_txt_Bs, W_ht)
-        projection_report['proj_txt_hierarchy_part_best_l2'] = best_l2_ht
-        if cv_ht is not None:
-            projection_report['proj_txt_hierarchy_part_cv_val'] = cv_ht
-    if len(hier_img_As) >= 8:
-        W_hi, best_l2_hi, cv_hi = fit_ridge_with_cv(hier_img_As, hier_img_Bs, l2_grid)
-        projection_report['proj_sim_img_hierarchy_part_mean'] = mean_projected_cosine(hier_img_As, hier_img_Bs, W_hi)
-        projection_report['proj_img_hierarchy_part_best_l2'] = best_l2_hi
-        if cv_hi is not None:
-            projection_report['proj_img_hierarchy_part_cv_val'] = cv_hi
-
-    proj_summary_path = os.path.join('evaluation_outputs', 'projection_summary.json')
-    with open(proj_summary_path, 'w') as f:
-        json.dump(projection_report, f, indent=2)
-    print(f"Saved projection summary to {proj_summary_path}")
-

Evaluation/evaluate_color_embeddings.py

@@ -1,1124 +0,0 @@
-"""
-Comprehensive evaluation of color embeddings with Fashion-CLIP comparison.
-This file evaluates the quality of color embeddings generated by the ColorCLIP model
-by calculating intra-class and inter-class similarity metrics, classification accuracies,
-and generating confusion matrices. It also compares results with Fashion-CLIP as a baseline
-to measure relative performance.
-"""
-
-import torch
-import torch.nn as nn
-import pandas as pd
-import numpy as np
-import matplotlib.pyplot as plt
-import seaborn as sns
-from sklearn.metrics.pairwise import cosine_similarity
-from sklearn.metrics import confusion_matrix, classification_report, accuracy_score
-from collections import defaultdict
-import os
-import json
-from tqdm import tqdm
-from torch.utils.data import Dataset, DataLoader
-from torchvision import transforms
-import requests
-from io import BytesIO
-from PIL import Image
-import warnings
-warnings.filterwarnings('ignore')
-from color_model import ColorCLIP, Tokenizer, ImageEncoder, TextEncoder, collate_batch
-from torch.utils.data import DataLoader
-from transformers import CLIPProcessor, CLIPModel as TransformersCLIPModel
-import config
-
-class ColorDataset(Dataset):
-    """
-    Dataset class for color embedding evaluation.
-    
-    Handles loading images from various sources (local paths, URLs, bytes) and
-    applying appropriate transformations for evaluation.
-    """
-    def __init__(self, dataframe):
-        """
-        Initialize the color dataset.
-        
-        Args:
-            dataframe: DataFrame containing image paths/URLs, text, and color labels
-        """
-        self.dataframe = dataframe
-        self.transform = transforms.Compose([
-            transforms.Resize((224, 224)),
-            transforms.ToTensor(),
-            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
-        ])
-
-    def __len__(self):
-        return len(self.dataframe)
-
-    def __getitem__(self, idx):
-        row = self.dataframe.iloc[idx]
-        
-        # Handle image - it should be in row[config.column_url_image] and contain the image data
-        image_data = row[config.column_url_image]
-        
-        try:
-            # Check if image_data has 'bytes' key or is already PIL Image
-            if isinstance(image_data, dict) and 'bytes' in image_data:
-                image = Image.open(BytesIO(image_data['bytes'])).convert("RGB")
-            elif hasattr(image_data, 'convert'):  # Already a PIL Image
-                image = image_data.convert("RGB")
-            elif isinstance(image_data, str):
-                # It's a file path (local or URL)
-                if image_data.startswith('http'):
-                    # It's a URL - download the image
-                    response = requests.get(image_data, timeout=10)
-                    response.raise_for_status()
-                    image = Image.open(BytesIO(response.content)).convert("RGB")
-                else:
-                    # It's a local file path
-                    image = Image.open(image_data).convert("RGB")
-            else:
-                # Assume it's bytes data
-                image = Image.open(BytesIO(image_data)).convert("RGB")
-            
-            # Apply transform
-            image = self.transform(image)
-
-        except Exception as e:
-            print(f"⚠️ Failed to load image {idx}: {e}")
-            # Return a placeholder image
-            image = torch.zeros(3, 224, 224)
-
-        # Get text and color
-        description = row[config.text_column]  
-        color = row[config.color_column]
-
-        return image, description, color
-
-class EmbeddingEvaluator:
-    """
-    Evaluator for color embeddings generated by the ColorCLIP model.
-    
-    This class provides methods to evaluate the quality of color embeddings by computing
-    similarity metrics, classification accuracies, and generating visualizations.
-    """
-    
-    def __init__(self, model_path, embed_dim):
-        """
-        Initialize the embedding evaluator.
-        
-        Args:
-            model_path: Path to the trained ColorCLIP model checkpoint
-            embed_dim: Embedding dimension for the model
-        """
-        self.device = config.device
-        
-        # Initialize tokenizer first to get vocab size
-        self.tokenizer = Tokenizer()
-        vocab_size = None
-        
-        # Load vocabulary if available to determine vocab_size
-        if os.path.exists(config.tokeniser_path):
-            with open(config.tokeniser_path, 'r') as f:
-                vocab_dict = json.load(f)
-                # Manually load vocabulary
-                self.tokenizer.word2idx = defaultdict(lambda: 0, {k: int(v) for k, v in vocab_dict.items()})
-                self.tokenizer.idx2word = {int(v): k for k, v in vocab_dict.items() if int(v) > 0}
-                self.tokenizer.counter = max(self.tokenizer.word2idx.values(), default=0) + 1
-                vocab_size = self.tokenizer.counter
-                print(f"Tokenizer vocabulary loaded from {config.tokeniser_path}")
-        else:
-            print(f"Warning: {config.tokeniser_path} not found. Using default tokenizer.")
-        
-        # Load checkpoint to get vocab_size and state_dict
-        checkpoint = None
-        if os.path.exists(model_path):
-            checkpoint = torch.load(model_path, map_location=self.device)
-            
-            # Try to get vocab_size from model checkpoint if not already determined
-            if vocab_size is None:
-                # Try to get vocab_size from metadata
-                if isinstance(checkpoint, dict) and 'vocab_size' in checkpoint:
-                    vocab_size = checkpoint['vocab_size']
-                # Otherwise, try to infer from model state dict
-                elif isinstance(checkpoint, dict) and 'model_state_dict' in checkpoint:
-                    state_dict = checkpoint['model_state_dict']
-                    if 'text_encoder.embedding.weight' in state_dict:
-                        vocab_size = state_dict['text_encoder.embedding.weight'].shape[0]
-                elif isinstance(checkpoint, dict) and 'text_encoder.embedding.weight' in checkpoint:
-                    vocab_size = checkpoint['text_encoder.embedding.weight'].shape[0]
-        
-        # Fallback to default if still not determined
-        if vocab_size is None:
-            vocab_size = 39  # Default fallback
-            print(f"Warning: Could not determine vocab_size, using default: {vocab_size}")
-        
-        # Initialize model with determined vocab_size
-        self.model = ColorCLIP(vocab_size=vocab_size, embedding_dim=embed_dim).to(self.device)
-        
-        # Load trained model state dict
-        if checkpoint is not None:
-            state_dict = checkpoint.get('model_state_dict', checkpoint)
-            self.model.load_state_dict(state_dict)
-            print(f"Model loaded from {model_path}")
-        else:
-            print(f"Warning: Model file {model_path} not found. Using untrained model.")
-        
-        self.model.eval()
-    
-    def extract_embeddings(self, dataloader, embedding_type='text'):
-        """
-        Extract embeddings from the model for a given dataloader.
-        
-        Args:
-            dataloader: DataLoader containing images, texts, and colors
-            embedding_type: Type of embeddings to extract ('text', 'image', or 'color')
-            
-        Returns:
-            Tuple of (embeddings array, labels list, texts list)
-        """
-        all_embeddings = []
-        all_labels = []
-        all_texts = []
-        
-        with torch.no_grad():
-            for images, texts, colors in tqdm(dataloader, desc=f"Extracting {embedding_type} embeddings"):
-                if embedding_type == 'text':
-                    # Tokenize texts using the tokenizer
-                    tokenized_texts = [self.tokenizer(text) for text in texts]
-                    # Convert to tensors and pad sequences
-                    text_tensors = [torch.tensor(t, dtype=torch.long) for t in tokenized_texts]
-                    text_tokens = nn.utils.rnn.pad_sequence(text_tensors, batch_first=True, padding_value=0).to(self.device)
-                    lengths = torch.tensor([len(t) for t in tokenized_texts], dtype=torch.long).to(self.device)
-                    embeddings = self.model.text_encoder(text_tokens, lengths)
-                    labels = colors
-                elif embedding_type == 'image':
-                    images = images.to(self.device)
-                    embeddings = self.model.image_encoder(images)
-                    labels = colors
-                elif embedding_type == 'color':
-                    # Tokenize color names using the tokenizer
-                    tokenized_colors = [self.tokenizer(color) for color in colors]
-                    # Convert to tensors and pad sequences
-                    color_tensors = [torch.tensor(t, dtype=torch.long) for t in tokenized_colors]
-                    color_tokens = nn.utils.rnn.pad_sequence(color_tensors, batch_first=True, padding_value=0).to(self.device)
-                    lengths = torch.tensor([len(t) for t in tokenized_colors], dtype=torch.long).to(self.device)
-                    embeddings = self.model.text_encoder(color_tokens, lengths)
-                    labels = colors
-                
-                all_embeddings.append(embeddings.cpu().numpy())
-                all_labels.extend(labels)
-                all_texts.extend(texts)
-        
-        return np.vstack(all_embeddings), all_labels, all_texts
-    
-    def compute_similarity_metrics(self, embeddings, labels):
-        """Compute intra-class and inter-class similarities"""
-        similarities = cosine_similarity(embeddings)
-        
-        # Group embeddings by color
-        color_groups = defaultdict(list)
-        for i, color in enumerate(labels):
-            color_groups[color].append(i)
-        
-        # Calculate intra-class similarities (same color)
-        intra_class_similarities = []
-        for color, indices in color_groups.items():
-            if len(indices) > 1:
-                for i in range(len(indices)):
-                    for j in range(i+1, len(indices)):
-                        sim = similarities[indices[i], indices[j]]
-                        intra_class_similarities.append(sim)
-        
-        # Calculate inter-class similarities (different colors)
-        inter_class_similarities = []
-        colors = list(color_groups.keys())
-        for i in range(len(colors)):
-            for j in range(i+1, len(colors)):
-                color1_indices = color_groups[colors[i]]
-                color2_indices = color_groups[colors[j]]
-                
-                for idx1 in color1_indices:
-                    for idx2 in color2_indices:
-                        sim = similarities[idx1, idx2]
-                        inter_class_similarities.append(sim)
-        
-        # Calculate classification accuracy using nearest neighbor in embedding space
-        nn_accuracy = self.compute_embedding_accuracy(embeddings, labels, similarities)
-        
-        # Calculate classification accuracy using centroids
-        centroid_accuracy = self.compute_centroid_accuracy(embeddings, labels)
-        
-        return {
-            'intra_class_similarities': intra_class_similarities,
-            'inter_class_similarities': inter_class_similarities,
-            'intra_class_mean': np.mean(intra_class_similarities) if intra_class_similarities else 0,
-            'inter_class_mean': np.mean(inter_class_similarities) if inter_class_similarities else 0,
-            'separation_score': np.mean(intra_class_similarities) - np.mean(inter_class_similarities) if intra_class_similarities and inter_class_similarities else 0,
-            'accuracy': nn_accuracy,
-            'centroid_accuracy': centroid_accuracy
-        }
-    
-    def compute_embedding_accuracy(self, embeddings, labels, similarities):
-        """Compute classification accuracy using nearest neighbor in embedding space"""
-        correct_predictions = 0
-        total_predictions = len(labels)
-        
-        for i in range(len(embeddings)):
-            true_label = labels[i]
-            
-            # Find the most similar embedding (excluding itself)
-            similarities_row = similarities[i].copy()
-            similarities_row[i] = -1  # Exclude self-similarity
-            nearest_neighbor_idx = np.argmax(similarities_row)
-            predicted_label = labels[nearest_neighbor_idx]
-            
-            if predicted_label == true_label:
-                correct_predictions += 1
-        
-        return correct_predictions / total_predictions if total_predictions > 0 else 0
-    
-    def compute_centroid_accuracy(self, embeddings, labels):
-        """Compute classification accuracy using color centroids"""
-        # Create centroids for each color
-        unique_colors = list(set(labels))
-        centroids = {}
-        
-        for color in unique_colors:
-            color_indices = [i for i, label in enumerate(labels) if label == color]
-            color_embeddings = embeddings[color_indices]
-            centroids[color] = np.mean(color_embeddings, axis=0)
-        
-        # Classify each embedding to nearest centroid
-        correct_predictions = 0
-        total_predictions = len(labels)
-        
-        for i, embedding in enumerate(embeddings):
-            true_label = labels[i]
-            
-            # Find closest centroid
-            best_similarity = -1
-            predicted_label = None
-            
-            for color, centroid in centroids.items():
-                similarity = cosine_similarity([embedding], [centroid])[0][0]
-                if similarity > best_similarity:
-                    best_similarity = similarity
-                    predicted_label = color
-            
-            if predicted_label == true_label:
-                correct_predictions += 1
-        
-        return correct_predictions / total_predictions if total_predictions > 0 else 0
-    
-    def predict_colors_from_embeddings(self, embeddings, labels):
-        """Predict colors from embeddings using centroid-based classification"""
-        # Create color centroids from training data
-        unique_colors = list(set(labels))
-        centroids = {}
-        
-        for color in unique_colors:
-            color_indices = [i for i, label in enumerate(labels) if label == color]
-            color_embeddings = embeddings[color_indices]
-            centroids[color] = np.mean(color_embeddings, axis=0)
-        
-        # Predict colors for all embeddings
-        predictions = []
-        
-        for i, embedding in enumerate(embeddings):
-            # Find closest centroid
-            best_similarity = -1
-            predicted_color = None
-            
-            for color, centroid in centroids.items():
-                similarity = cosine_similarity([embedding], [centroid])[0][0]
-                if similarity > best_similarity:
-                    best_similarity = similarity
-                    predicted_color = color
-            
-            predictions.append(predicted_color)
-        
-        return predictions
-    
-    def create_confusion_matrix(self, true_labels, predicted_labels, title="Confusion Matrix"):
-        """Create and plot confusion matrix"""
-        # Get unique labels
-        unique_labels = sorted(list(set(true_labels + predicted_labels)))
-        
-        # Create confusion matrix
-        cm = confusion_matrix(true_labels, predicted_labels, labels=unique_labels)
-        
-        # Calculate accuracy
-        accuracy = accuracy_score(true_labels, predicted_labels)
-        
-        # Plot confusion matrix
-        plt.figure(figsize=(12, 10))
-        sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', 
-                   xticklabels=unique_labels, yticklabels=unique_labels)
-        plt.title(f'{title}\nAccuracy: {accuracy:.3f} ({accuracy*100:.1f}%)')
-        plt.ylabel('True Color')
-        plt.xlabel('Predicted Color')
-        plt.xticks(rotation=45)
-        plt.yticks(rotation=0)
-        plt.tight_layout()
-        
-        return plt.gcf(), accuracy, cm
-    
-    def evaluate_classification_performance(self, embeddings, labels, embedding_type="Embeddings"):
-        """Evaluate classification performance and create confusion matrix"""
-        # Predict colors
-        predictions = self.predict_colors_from_embeddings(embeddings, labels)
-        
-        # Calculate accuracy
-        accuracy = accuracy_score(labels, predictions)
-        
-        # Create confusion matrix
-        fig, acc, cm = self.create_confusion_matrix(labels, predictions, 
-                                                   f"{embedding_type} - Color Classification")
-        
-        # Generate classification report
-        unique_labels = sorted(list(set(labels)))
-        report = classification_report(labels, predictions, labels=unique_labels, 
-                                     target_names=unique_labels, output_dict=True)
-        
-        return {
-            'accuracy': accuracy,
-            'predictions': predictions,
-            'confusion_matrix': cm,
-            'classification_report': report,
-            'figure': fig
-        }
-    
-    def evaluate_dataset(self, dataframe, dataset_name="Dataset"):
-        """
-        Evaluate embeddings on a given dataset.
-        
-        This method extracts embeddings for text, image, and color, computes similarity metrics,
-        evaluates classification performance, and saves confusion matrices.
-        
-        Args:
-            dataframe: DataFrame containing the dataset
-            dataset_name: Name of the dataset for display purposes
-            
-        Returns:
-            Dictionary containing evaluation results for text, image, and color embeddings
-        """
-        print(f"\n{'='*60}")
-        print(f"Evaluating {dataset_name}")
-        print(f"{'='*60}")
-        
-        # Create dataset and dataloader - use KaglDataset for kagl data
-        if "kagl" in dataset_name.lower():
-            dataset = KaglDataset(dataframe)
-        else:
-            dataset = ColorDataset(dataframe)
-        # Optimize batch size and workers for faster processing
-        dataloader = DataLoader(dataset, batch_size=64, shuffle=False, num_workers=4, pin_memory=True)
-        
-        results = {}
-        
-        # Evaluate text embeddings
-        text_embeddings, text_labels, texts = self.extract_embeddings(dataloader, 'text')
-        text_metrics = self.compute_similarity_metrics(text_embeddings, text_labels)
-        text_classification = self.evaluate_classification_performance(text_embeddings, text_labels, "Text Embeddings")
-        text_metrics.update(text_classification)
-        results['text'] = text_metrics
-        
-        # Evaluate image embeddings
-        image_embeddings, image_labels, _ = self.extract_embeddings(dataloader, 'image')
-        image_metrics = self.compute_similarity_metrics(image_embeddings, image_labels)
-        image_classification = self.evaluate_classification_performance(image_embeddings, image_labels, "Image Embeddings")
-        image_metrics.update(image_classification)
-        results['image'] = image_metrics
-        
-        # Evaluate color embeddings
-        color_embeddings, color_labels, _ = self.extract_embeddings(dataloader, 'color')
-        color_metrics = self.compute_similarity_metrics(color_embeddings, color_labels)
-        color_classification = self.evaluate_classification_performance(color_embeddings, color_labels, "Color Embeddings")
-        color_metrics.update(color_classification)
-        results['color'] = color_metrics
-        
-        # Print results
-        print(f"\n{dataset_name} Results:")
-        print("-" * 40)
-        for emb_type, metrics in results.items():
-            print(f"{emb_type.capitalize()} Embeddings:")
-            print(f"  Intra-class similarity (same color): {metrics['intra_class_mean']:.4f}")
-            print(f"  Inter-class similarity (diff colors): {metrics['inter_class_mean']:.4f}")
-            print(f"  Separation score: {metrics['separation_score']:.4f}")
-            print(f"  Nearest Neighbor Accuracy: {metrics['accuracy']:.4f} ({metrics['accuracy']*100:.1f}%)")
-            print(f"  Centroid Accuracy: {metrics['centroid_accuracy']:.4f} ({metrics['centroid_accuracy']*100:.1f}%)")
-            
-            # Classification report summary
-            report = metrics['classification_report']
-            print(f"  📊 Classification Performance:")
-            print(f"    • Macro Avg F1-Score: {report['macro avg']['f1-score']:.4f}")
-            print(f"    • Weighted Avg F1-Score: {report['weighted avg']['f1-score']:.4f}")
-            print(f"    • Support: {report['macro avg']['support']:.0f} samples")
-            print()
-        
-        # Create visualizations
-        os.makedirs('embedding_evaluation', exist_ok=True)
-        
-        # Confusion matrices
-        results['text']['figure'].savefig(f'embedding_evaluation/{dataset_name.lower()}_text_confusion_matrix.png', dpi=300, bbox_inches='tight')
-        plt.close(results['text']['figure'])
-        
-        results['image']['figure'].savefig(f'embedding_evaluation/{dataset_name.lower()}_image_confusion_matrix.png', dpi=300, bbox_inches='tight')
-        plt.close(results['image']['figure'])
-        
-        results['color']['figure'].savefig(f'embedding_evaluation/{dataset_name.lower()}_color_confusion_matrix.png', dpi=300, bbox_inches='tight')
-        plt.close(results['color']['figure'])
-        
-        return results
-
-class FashionCLIPDataset(Dataset):
-    """
-    Special dataset for Fashion-CLIP that doesn't normalize images.
-    
-    This dataset is used when evaluating with Fashion-CLIP baseline model,
-    which requires different image preprocessing (no normalization).
-    """
-    def __init__(self, dataframe):
-        """
-        Initialize the Fashion-CLIP dataset.
-        
-        Args:
-            dataframe: DataFrame containing image paths/URLs, text, and color labels
-        """
-        self.dataframe = dataframe
-        # Only resize and convert to tensor, no normalization
-        self.transform = transforms.Compose([
-            transforms.Resize((224, 224)),
-            transforms.ToTensor()
-        ])
-
-    def __len__(self):
-        return len(self.dataframe)
-
-    def __getitem__(self, idx):
-        row = self.dataframe.iloc[idx]
-        
-        # Handle image - it should be in row[config.column_url_image] and contain the image data
-        image_data = row[config.column_url_image]
-        
-        try:
-            # Check if image_data has 'bytes' key or is already PIL Image
-            if isinstance(image_data, dict) and 'bytes' in image_data:
-                image = Image.open(BytesIO(image_data['bytes'])).convert("RGB")
-            elif hasattr(image_data, 'convert'):  # Already a PIL Image
-                image = image_data.convert("RGB")
-            elif isinstance(image_data, str):
-                # It's a file path (local or URL)
-                if image_data.startswith('http'):
-                    # It's a URL - download the image
-                    import requests
-                    response = requests.get(image_data, timeout=10)
-                    response.raise_for_status()
-                    image = Image.open(BytesIO(response.content)).convert("RGB")
-                else:
-                    # It's a local file path
-                    image = Image.open(image_data).convert("RGB")
-            else:
-                # Assume it's bytes data
-                image = Image.open(BytesIO(image_data)).convert("RGB")
-            
-            # Apply minimal transform (no normalization)
-            image = self.transform(image)
-
-        except Exception as e:
-            print(f"⚠️ Failed to load image {idx}: {e}")
-            # Return a placeholder image instead of undefined variable
-            image = torch.zeros(3, 224, 224)
-
-        # Get text and color
-        description = row[config.text_column]  
-        color = row[config.color_column]
-
-        return image, description, color
-
-class FashionCLIPEvaluator:
-    """
-    Evaluator for Fashion-CLIP baseline model.
-    
-    This class provides methods to evaluate embeddings from the Fashion-CLIP model
-    and compare them with the custom ColorCLIP model.
-    """
-    
-    def __init__(self):
-        """
-        Initialize the Fashion-CLIP evaluator.
-        
-        Loads the Fashion-CLIP model from Hugging Face and prepares it for evaluation.
-        """
-        # Load Fashion-CLIP model
-        patrick_model_name = "patrickjohncyh/fashion-clip"
-        print(f"🔄 Loading Fashion-CLIP model: {patrick_model_name}")
-        self.processor = CLIPProcessor.from_pretrained(patrick_model_name)
-        self.device = config.device
-        self.model = TransformersCLIPModel.from_pretrained(patrick_model_name).to(self.device)
-        self.model.eval()
-        print(f"✅ Fashion-CLIP model loaded successfully")
-    
-    def extract_embeddings(self, dataloader, embedding_type='text'):
-        """
-        Extract embeddings from the Fashion-CLIP model.
-        
-        Args:
-            dataloader: DataLoader containing images, texts, and colors
-            embedding_type: Type of embeddings to extract ('text', 'image', or 'color')
-            
-        Returns:
-            Tuple of (embeddings array, labels list, texts list)
-        """
-        all_embeddings = []
-        all_labels = []
-        all_texts = []
-        
-        with torch.no_grad():
-            for images, texts, colors in tqdm(dataloader, desc=f"Extracting {embedding_type} embeddings (Fashion-CLIP)"):
-                if embedding_type == 'text':
-                    # Process text through Fashion-CLIP
-                    inputs = self.processor(text=texts, return_tensors="pt", padding=True, truncation=True, max_length=77)
-                    inputs = {k: v.to(self.device) for k, v in inputs.items()}
-                    text_features = self.model.get_text_features(**inputs)
-                    text_features = text_features / text_features.norm(dim=-1, keepdim=True)
-                    embeddings = text_features.cpu().numpy()
-                    labels = colors
-                elif embedding_type == 'image':
-                    # Convert tensors back to PIL images for CLIP processor
-                    pil_images = []
-                    for i in range(images.shape[0]):
-                        # Convert tensor back to PIL Image
-                        img_tensor = images[i]
-                        # Denormalize if needed (images should be in [0,1] range)
-                        if img_tensor.min() < 0 or img_tensor.max() > 1:
-                            # If normalized, denormalize
-                            img_tensor = (img_tensor + 1) / 2  # Assuming [-1,1] to [0,1]
-                        img_tensor = torch.clamp(img_tensor, 0, 1)
-                        img_pil = transforms.ToPILImage()(img_tensor)
-                        pil_images.append(img_pil)
-                    
-                    # Process images through Fashion-CLIP
-                    inputs = self.processor(images=pil_images, return_tensors="pt", padding=True)
-                    inputs = {k: v.to(self.device) for k, v in inputs.items()}
-                    image_features = self.model.get_image_features(**inputs)
-                    image_features = image_features / image_features.norm(dim=-1, keepdim=True)
-                    embeddings = image_features.cpu().numpy()
-                    labels = colors
-                elif embedding_type == 'color':
-                    # Process color names as text through Fashion-CLIP
-                    inputs = self.processor(text=colors, return_tensors="pt", padding=True, truncation=True, max_length=77)
-                    inputs = {k: v.to(self.device) for k, v in inputs.items()}
-                    text_features = self.model.get_text_features(**inputs)
-                    text_features = text_features / text_features.norm(dim=-1, keepdim=True)
-                    embeddings = text_features.cpu().numpy()
-                    labels = colors
-                
-                all_embeddings.append(embeddings)
-                all_labels.extend(labels)
-                all_texts.extend(texts)
-        
-        return np.vstack(all_embeddings), all_labels, all_texts
-    
-    def compute_similarity_metrics(self, embeddings, labels):
-        """Compute intra-class and inter-class similarities"""
-        similarities = cosine_similarity(embeddings)
-        
-        # Group embeddings by color
-        color_groups = defaultdict(list)
-        for i, color in enumerate(labels):
-            color_groups[color].append(i)
-        
-        # Calculate intra-class similarities (same color)
-        intra_class_similarities = []
-        for color, indices in color_groups.items():
-            if len(indices) > 1:
-                for i in range(len(indices)):
-                    for j in range(i+1, len(indices)):
-                        sim = similarities[indices[i], indices[j]]
-                        intra_class_similarities.append(sim)
-        
-        # Calculate inter-class similarities (different colors)
-        inter_class_similarities = []
-        colors = list(color_groups.keys())
-        for i in range(len(colors)):
-            for j in range(i+1, len(colors)):
-                color1_indices = color_groups[colors[i]]
-                color2_indices = color_groups[colors[j]]
-                
-                for idx1 in color1_indices:
-                    for idx2 in color2_indices:
-                        sim = similarities[idx1, idx2]
-                        inter_class_similarities.append(sim)
-        
-        # Calculate classification accuracy using nearest neighbor in embedding space
-        nn_accuracy = self.compute_embedding_accuracy(embeddings, labels, similarities)
-        
-        # Calculate classification accuracy using centroids
-        centroid_accuracy = self.compute_centroid_accuracy(embeddings, labels)
-        
-        return {
-            'intra_class_similarities': intra_class_similarities,
-            'inter_class_similarities': inter_class_similarities,
-            'intra_class_mean': np.mean(intra_class_similarities) if intra_class_similarities else 0,
-            'inter_class_mean': np.mean(inter_class_similarities) if inter_class_similarities else 0,
-            'separation_score': np.mean(intra_class_similarities) - np.mean(inter_class_similarities) if intra_class_similarities and inter_class_similarities else 0,
-            'accuracy': nn_accuracy,
-            'centroid_accuracy': centroid_accuracy
-        }
-    
-    def compute_embedding_accuracy(self, embeddings, labels, similarities):
-        """Compute classification accuracy using nearest neighbor in embedding space"""
-        correct_predictions = 0
-        total_predictions = len(labels)
-        
-        for i in range(len(embeddings)):
-            true_label = labels[i]
-            
-            # Find the most similar embedding (excluding itself)
-            similarities_row = similarities[i].copy()
-            similarities_row[i] = -1  # Exclude self-similarity
-            nearest_neighbor_idx = np.argmax(similarities_row)
-            predicted_label = labels[nearest_neighbor_idx]
-            
-            if predicted_label == true_label:
-                correct_predictions += 1
-        
-        return correct_predictions / total_predictions if total_predictions > 0 else 0
-    
-    def compute_centroid_accuracy(self, embeddings, labels):
-        """Compute classification accuracy using color centroids"""
-        # Create centroids for each color
-        unique_colors = list(set(labels))
-        centroids = {}
-        
-        for color in unique_colors:
-            color_indices = [i for i, label in enumerate(labels) if label == color]
-            color_embeddings = embeddings[color_indices]
-            centroids[color] = np.mean(color_embeddings, axis=0)
-        
-        # Classify each embedding to nearest centroid
-        correct_predictions = 0
-        total_predictions = len(labels)
-        
-        for i, embedding in enumerate(embeddings):
-            true_label = labels[i]
-            
-            # Find closest centroid
-            best_similarity = -1
-            predicted_label = None
-            
-            for color, centroid in centroids.items():
-                similarity = cosine_similarity([embedding], [centroid])[0][0]
-                if similarity > best_similarity:
-                    best_similarity = similarity
-                    predicted_label = color
-            
-            if predicted_label == true_label:
-                correct_predictions += 1
-        
-        return correct_predictions / total_predictions if total_predictions > 0 else 0
-    
-    def predict_colors_from_embeddings(self, embeddings, labels):
-        """Predict colors from embeddings using centroid-based classification"""
-        # Create color centroids from training data
-        unique_colors = list(set(labels))
-        centroids = {}
-        
-        for color in unique_colors:
-            color_indices = [i for i, label in enumerate(labels) if label == color]
-            color_embeddings = embeddings[color_indices]
-            centroids[color] = np.mean(color_embeddings, axis=0)
-        
-        # Predict colors for all embeddings
-        predictions = []
-        
-        for i, embedding in enumerate(embeddings):
-            # Find closest centroid
-            best_similarity = -1
-            predicted_color = None
-            
-            for color, centroid in centroids.items():
-                similarity = cosine_similarity([embedding], [centroid])[0][0]
-                if similarity > best_similarity:
-                    best_similarity = similarity
-                    predicted_color = color
-            
-            predictions.append(predicted_color)
-        
-        return predictions
-    
-    def create_confusion_matrix(self, true_labels, predicted_labels, title="Confusion Matrix"):
-        """Create and plot confusion matrix"""
-        # Get unique labels
-        unique_labels = sorted(list(set(true_labels + predicted_labels)))
-        
-        # Create confusion matrix
-        cm = confusion_matrix(true_labels, predicted_labels, labels=unique_labels)
-        
-        # Calculate accuracy
-        accuracy = accuracy_score(true_labels, predicted_labels)
-        
-        # Plot confusion matrix
-        plt.figure(figsize=(12, 10))
-        sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', 
-                   xticklabels=unique_labels, yticklabels=unique_labels)
-        plt.title(f'{title}\nAccuracy: {accuracy:.3f} ({accuracy*100:.1f}%)')
-        plt.ylabel('True Color')
-        plt.xlabel('Predicted Color')
-        plt.xticks(rotation=45)
-        plt.yticks(rotation=0)
-        plt.tight_layout()
-        
-        return plt.gcf(), accuracy, cm
-    
-    def evaluate_classification_performance(self, embeddings, labels, embedding_type="Embeddings"):
-        """Evaluate classification performance and create confusion matrix"""
-        # Predict colors
-        predictions = self.predict_colors_from_embeddings(embeddings, labels)
-        
-        # Calculate accuracy
-        accuracy = accuracy_score(labels, predictions)
-        
-        # Create confusion matrix
-        fig, acc, cm = self.create_confusion_matrix(labels, predictions, 
-                                                   f"{embedding_type} - Color Classification (Fashion-CLIP)")
-        
-        # Generate classification report
-        unique_labels = sorted(list(set(labels)))
-        report = classification_report(labels, predictions, labels=unique_labels, 
-                                     target_names=unique_labels, output_dict=True)
-        
-        return {
-            'accuracy': accuracy,
-            'predictions': predictions,
-            'confusion_matrix': cm,
-            'classification_report': report,
-            'figure': fig
-        }
-    
-    def evaluate_dataset(self, dataframe, dataset_name="Dataset"):
-        """
-        Evaluate Fashion-CLIP embeddings on a given dataset.
-        
-        This method extracts embeddings for text, image, and color, computes similarity metrics,
-        evaluates classification performance, and saves confusion matrices.
-        
-        Args:
-            dataframe: DataFrame containing the dataset
-            dataset_name: Name of the dataset for display purposes
-            
-        Returns:
-            Dictionary containing evaluation results for text, image, and color embeddings
-        """
-        print(f"\n{'='*60}")
-        print(f"Evaluating {dataset_name} with Fashion-CLIP")
-        print(f"{'='*60}")
-        
-        # Create dataset and dataloader - use FashionCLIPDataset for Fashion-CLIP
-        if "kagl" in dataset_name.lower():
-            dataset = KaglDataset(dataframe)
-        else:
-            dataset = FashionCLIPDataset(dataframe)  # Use special dataset for Fashion-CLIP
-        # Optimize batch size for Fashion-CLIP
-        dataloader = DataLoader(dataset, batch_size=32, shuffle=False, num_workers=4, pin_memory=True)
-        
-        results = {}
-        
-        # Evaluate text embeddings
-        text_embeddings, text_labels, texts = self.extract_embeddings(dataloader, 'text')
-        text_metrics = self.compute_similarity_metrics(text_embeddings, text_labels)
-        text_classification = self.evaluate_classification_performance(text_embeddings, text_labels, "Text Embeddings")
-        text_metrics.update(text_classification)
-        results['text'] = text_metrics
-        
-        # Evaluate image embeddings
-        image_embeddings, image_labels, _ = self.extract_embeddings(dataloader, 'image')
-        image_metrics = self.compute_similarity_metrics(image_embeddings, image_labels)
-        image_classification = self.evaluate_classification_performance(image_embeddings, image_labels, "Image Embeddings")
-        image_metrics.update(image_classification)
-        results['image'] = image_metrics
-        
-        # Evaluate color embeddings
-        color_embeddings, color_labels, _ = self.extract_embeddings(dataloader, 'color')
-        color_metrics = self.compute_similarity_metrics(color_embeddings, color_labels)
-        color_classification = self.evaluate_classification_performance(color_embeddings, color_labels, "Color Embeddings")
-        color_metrics.update(color_classification)
-        results['color'] = color_metrics
-        
-        # Print results
-        print(f"\n{dataset_name} Results (Fashion-CLIP):")
-        print("-" * 40)
-        for emb_type, metrics in results.items():
-            print(f"{emb_type.capitalize()} Embeddings:")
-            print(f"  Intra-class similarity (same color): {metrics['intra_class_mean']:.4f}")
-            print(f"  Inter-class similarity (diff colors): {metrics['inter_class_mean']:.4f}")
-            print(f"  Separation score: {metrics['separation_score']:.4f}")
-            print(f"  Nearest Neighbor Accuracy: {metrics['accuracy']:.4f} ({metrics['accuracy']*100:.1f}%)")
-            print(f"  Centroid Accuracy: {metrics['centroid_accuracy']:.4f} ({metrics['centroid_accuracy']*100:.1f}%)")
-            
-            # Classification report summary
-            report = metrics['classification_report']
-            print(f"  📊 Classification Performance:")
-            print(f"    • Macro Avg F1-Score: {report['macro avg']['f1-score']:.4f}")
-            print(f"    • Weighted Avg F1-Score: {report['weighted avg']['f1-score']:.4f}")
-            print(f"    • Support: {report['macro avg']['support']:.0f} samples")
-            print()
-        
-        # Create visualizations
-        os.makedirs('embedding_evaluation', exist_ok=True)
-        
-        # Confusion matrices
-        results['text']['figure'].savefig(f'embedding_evaluation/{dataset_name.lower()}_text_confusion_matrix_fashion_clip.png', dpi=300, bbox_inches='tight')
-        plt.close(results['text']['figure'])
-        
-        results['image']['figure'].savefig(f'embedding_evaluation/{dataset_name.lower()}_image_confusion_matrix_fashion_clip.png', dpi=300, bbox_inches='tight')
-        plt.close(results['image']['figure'])
-        
-        results['color']['figure'].savefig(f'embedding_evaluation/{dataset_name.lower()}_color_confusion_matrix_fashion_clip.png', dpi=300, bbox_inches='tight')
-        plt.close(results['color']['figure'])
-        
-        return results
-
-class KaglDataset(Dataset):
-    """
-    Dataset class for KAGL Marqo dataset evaluation.
-    
-    Handles loading images from the KAGL dataset format (with 'bytes' in image_url).
-    """
-    def __init__(self, dataframe):
-        """
-        Initialize the KAGL dataset.
-        
-        Args:
-            dataframe: DataFrame containing image_url (with bytes), text, and color labels
-        """
-        self.dataframe = dataframe
-        self.transform = transforms.Compose([
-            transforms.Resize((224, 224)),
-            transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2),
-            transforms.ToTensor(),
-            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
-        ])
-
-    def __len__(self):
-        return len(self.dataframe)
-
-    def __getitem__(self, idx):
-        row = self.dataframe.iloc[idx]
-        
-        # Handle image - it should be in row['image_url'] and contain the image data
-        image_data = row["image_url"]
-        
-        # Check if image_data has 'bytes' key or is already PIL Image
-        if isinstance(image_data, dict) and 'bytes' in image_data:
-            image = Image.open(BytesIO(image_data['bytes'])).convert("RGB")
-        elif hasattr(image_data, 'convert'):  # Already a PIL Image
-            image = image_data.convert("RGB")
-        else:
-            image = Image.open(BytesIO(image_data)).convert("RGB")
-        
-        image = self.transform(image)
-
-        # Get text and color from kagl
-        description = row['text']  
-        color = row['color']
-
-        return image, description, color
-
-def load_kagl_marqo_dataset():
-    """
-    Load and prepare KAGL Marqo dataset from Hugging Face.
-    
-    This function loads the Marqo/KAGL dataset, filters for valid colors,
-    and formats it for evaluation.
-    
-    Returns:
-        DataFrame with columns: image_url, text, color
-    """
-    from datasets import load_dataset
-    print("Loading kagl KAGL dataset...")
-
-    # Load the dataset
-    dataset = load_dataset("Marqo/KAGL")
-    df = dataset["data"].to_pandas()
-    print(f"✅ Dataset kagl loaded")
-
-    # Prepare data - Replace baseColour
-    df['baseColour'] = df['baseColour'].str.lower().str.replace("grey", "gray")
-    df_test = df[df['baseColour'].notna()].copy()
-
-    print(f"📊 Before filtering: {len(df_test)} samples")
-    
-    # Filter for common colors
-    valid_colors = ['red', 'blue', 'green', 'yellow', 'purple', 'pink', 'orange', 
-                    'brown', 'black', 'white', 'gray', 'navy', 'maroon', 'beige']
-    df_test = df_test[df_test['baseColour'].isin(valid_colors)]
-    
-    print(f"📊 After filtering invalid colors: {len(df_test)} samples")
-    print(f"🎨 Valid colors found: {sorted(df_test['baseColour'].unique())}")
-    
-    if len(df_test) == 0:
-        print("❌ No samples left after color filtering. Using mock dataset.")
-
-    # Map to our expected column names
-    kagl_formatted = pd.DataFrame({
-        'image_url': df_test['image_url'],
-        'text': df_test['text'],
-        'color': df_test['baseColour'].str.lower().str.replace("grey", "gray")
-    })
-    
-    # Additional validation - remove rows with missing data
-    print(f"📊 Before final validation: {len(kagl_formatted)} samples")
-    kagl_formatted = kagl_formatted.dropna(subset=[config.column_url_image, config.text_column, config.color_column])
-    print(f"📊 After removing missing data: {len(kagl_formatted)} samples")
-    
-    # Check for empty strings
-    kagl_formatted = kagl_formatted[
-        (kagl_formatted['text'].str.strip() != '') & 
-        (kagl_formatted['color'].str.strip() != '')
-    ]
-    print(f"📊 After removing empty strings: {len(kagl_formatted)} samples")
-    
-    print(f"📊 Final dataset size: {len(kagl_formatted)} samples")
-    
-    return kagl_formatted
-
-def create_comparison_table(val_results, kagl_results, val_results_fashion_clip, kagl_results_fashion_clip):
-    """
-    Create a structured comparison table between custom model and Fashion-CLIP baseline.
-    
-    Args:
-        val_results: Evaluation results for custom model on validation dataset
-        kagl_results: Evaluation results for custom model on KAGL dataset
-        val_results_fashion_clip: Evaluation results for Fashion-CLIP on validation dataset
-        kagl_results_fashion_clip: Evaluation results for Fashion-CLIP on KAGL dataset
-        
-    Returns:
-        DataFrame containing the comparison table
-    """
-    
-    # Create DataFrame for comparison
-    data = []
-    
-    # Define embedding types and their display names
-    embedding_types = [
-        ('text', 'Text Embeddings'),
-        ('image', 'Image Embeddings'), 
-        ('color', 'Color Embeddings')
-    ]
-    
-    # Define datasets
-    datasets = [
-        ('Validation Dataset', val_results, val_results_fashion_clip),
-        ('kagl Marqo Dataset', kagl_results, kagl_results_fashion_clip)
-    ]
-    
-    for dataset_name, custom_results, baseline_results in datasets:
-        for emb_type, emb_display in embedding_types:
-            # Your custom model results
-            custom_metrics = custom_results[emb_type]
-            # Baseline model results  
-            baseline_metrics = baseline_results[emb_type]
-            
-            data.append({
-                'Dataset': dataset_name,
-                'Embedding Type': emb_display,
-                'Model': 'Your Model',
-                'Separation Score': f"{custom_metrics['separation_score']:.4f}",
-                'NN Accuracy (%)': f"{custom_metrics['accuracy']*100:.1f}%",
-                'Centroid Accuracy (%)': f"{custom_metrics['centroid_accuracy']*100:.1f}%",
-                'Intra-class Similarity': f"{custom_metrics['intra_class_mean']:.4f}",
-                'Inter-class Similarity': f"{custom_metrics['inter_class_mean']:.4f}",
-                'Macro F1-Score': f"{custom_metrics['classification_report']['macro avg']['f1-score']:.4f}",
-                'Weighted F1-Score': f"{custom_metrics['classification_report']['weighted avg']['f1-score']:.4f}"
-            })
-            
-            data.append({
-                'Dataset': dataset_name,
-                'Embedding Type': emb_display,
-                'Model': 'Fashion-CLIP (Baseline)',
-                'Separation Score': f"{baseline_metrics['separation_score']:.4f}",
-                'NN Accuracy (%)': f"{baseline_metrics['accuracy']*100:.1f}%",
-                'Centroid Accuracy (%)': f"{baseline_metrics['centroid_accuracy']*100:.1f}%",
-                'Intra-class Similarity': f"{baseline_metrics['intra_class_mean']:.4f}",
-                'Inter-class Similarity': f"{baseline_metrics['inter_class_mean']:.4f}",
-                'Macro F1-Score': f"{baseline_metrics['classification_report']['macro avg']['f1-score']:.4f}",
-                'Weighted F1-Score': f"{baseline_metrics['classification_report']['weighted avg']['f1-score']:.4f}"
-            })
-    
-    # Create DataFrame
-    df_comparison = pd.DataFrame(data)
-    
-    # Save to CSV
-    df_comparison.to_csv('embedding_evaluation/model_comparison_table.csv', index=False)
-    
-    # Print formatted table
-    print(f"\n{'='*120}")
-    print("📊 COMPREHENSIVE MODEL COMPARISON TABLE")
-    print(f"{'='*120}")
-    
-    # Print table by dataset
-    for dataset_name in df_comparison['Dataset'].unique():
-        print(f"\n🔍 {dataset_name.upper()}")
-        print("-" * 120)
-        
-        dataset_df = df_comparison[df_comparison['Dataset'] == dataset_name]
-        
-        for emb_type in dataset_df['Embedding Type'].unique():
-            print(f"\n📈 {emb_type}:")
-            emb_df = dataset_df[dataset_df['Embedding Type'] == emb_type]
-            
-            # Print header
-            print(f"{'Model':<20} {'Separation':<12} {'NN Acc':<10} {'Centroid Acc':<13} {'Intra-class':<12} {'Inter-class':<12} {'Macro F1':<10} {'Weighted F1':<12}")
-            print("-" * 120)
-            
-            # Print data
-            for _, row in emb_df.iterrows():
-                print(f"{row['Model']:<20} {row['Separation Score']:<12} {row['NN Accuracy (%)']:<10} {row['Centroid Accuracy (%)']:<13} {row['Intra-class Similarity']:<12} {row['Inter-class Similarity']:<12} {row['Macro F1-Score']:<10} {row['Weighted F1-Score']:<12}")
-    
-    return df_comparison
-
-if __name__ == "__main__":
-    
-    # Initialize evaluator for your custom model
-    evaluator = EmbeddingEvaluator(model_path=config.color_model_path, embed_dim=config.color_emb_dim)
-    
-    # Initialize Fashion-CLIP evaluator
-    fashion_clip_evaluator = FashionCLIPEvaluator()
-    
-    # Load datasets
-    print("Loading datasets...")
-    
-    # Load validation dataset
-    df_val = pd.read_csv(config.local_dataset_path)
-    
-    # Filter for better quality data
-    print(f"📊 Original dataset size: {len(df_val)}")
-    samples_to_evaluate = 10000
-    
-
-    # Load kagl Marqo dataset
-    kagl_df = load_kagl_marqo_dataset()
-    
-    # Evaluate your custom model on validation dataset
-    val_results = evaluator.evaluate_dataset(df_val, "Validation Dataset")
-    
-    # Evaluate your custom model on kagl Marqo dataset (reduced sample for speed)
-    kagl_results = evaluator.evaluate_dataset(kagl_df.sample(min(samples_to_evaluate, len(kagl_df)), random_state=42), "kagl Marqo Dataset")
-    
-    # Evaluate Fashion-CLIP on validation dataset
-    val_results_fashion_clip = fashion_clip_evaluator.evaluate_dataset(df_val, "Validation Dataset")
-    
-    # Create comprehensive comparison table
-    comparison_df = create_comparison_table(
-        val_results, kagl_results, 
-        val_results_fashion_clip
-    )
-    
-    print(f"\n{'='*120}")
-    print("✅ Evaluation complete!")
-    print("📁 Confusion matrices saved in 'embedding_evaluation/' folder")
-    print("📁 Comparison table saved as 'model_comparison_table.csv'")
-    print("📁 Fashion-CLIP results are saved with '_fashion_clip' suffix.")
-    print(f"{'='*120}")

Evaluation/fashion_search.py

@@ -1,365 +0,0 @@
-#!/usr/bin/env python3
-"""
-Fashion search system using multi-modal embeddings.
-This file implements a fashion search engine that allows searching for clothing items
-using text queries. It uses embeddings from the main model to calculate cosine similarities
-and return the most relevant items. The system pre-computes embeddings for all items
-in the dataset for fast search.
-"""
-
-import torch
-import numpy as np
-import pandas as pd
-from PIL import Image
-import matplotlib.pyplot as plt
-from sklearn.metrics.pairwise import cosine_similarity
-from transformers import CLIPProcessor, CLIPModel as CLIPModel_transformers
-import warnings
-import os
-from typing import List, Tuple, Union, Optional
-import argparse
-
-# Import custom models
-from color_model import CLIPModel as ColorModel
-from hierarchy_model import Model as HierarchyModel, HierarchyExtractor
-from main_model import CustomDataset
-import config
-
-warnings.filterwarnings("ignore")
-
-class FashionSearchEngine:
-    """
-    Fashion search engine using multi-modal embeddings with category emphasis
-    """
-    
-    def __init__(self, top_k: int = 10, max_items: int = 10000):
-        """
-        Initialize the fashion search engine        
-        Args:
-            top_k: Number of top results to return
-            max_items: Maximum number of items to process (for faster initialization)
-            hierarchy_weight: Weight for hierarchy/category dimensions (default: 2.0)
-            color_weight: Weight for color dimensions (default: 1.0)
-        """
-        self.device = config.device
-        self.top_k = top_k
-        self.max_items = max_items
-        self.color_dim = config.color_emb_dim
-        self.hierarchy_dim = config.hierarchy_emb_dim
-        
-        # Load models
-        self._load_models()
-        
-        # Load dataset
-        self._load_dataset()
-        
-        # Pre-compute embeddings for all items
-        self._precompute_embeddings()
-        
-        print("✅ Fashion Search Engine ready!")
-    
-    def _load_models(self):
-        """Load all required models"""
-        print("📦 Loading models...")
-        
-        # Load color model
-        color_checkpoint = torch.load(config.color_model_path, map_location=self.device, weights_only=True)
-        self.color_model = ColorModel(embed_dim=self.color_dim).to(self.device)
-        self.color_model.load_state_dict(color_checkpoint)
-        self.color_model.eval()
-        
-        # Load hierarchy model
-        hierarchy_checkpoint = torch.load(config.hierarchy_model_path, map_location=self.device)
-        self.hierarchy_classes = hierarchy_checkpoint.get('hierarchy_classes', [])
-        self.hierarchy_model = HierarchyModel(
-            num_hierarchy_classes=len(self.hierarchy_classes),
-            embed_dim=self.hierarchy_dim
-        ).to(self.device)
-        self.hierarchy_model.load_state_dict(hierarchy_checkpoint['model_state'])
-        
-        # Set hierarchy extractor
-        hierarchy_extractor = HierarchyExtractor(self.hierarchy_classes, verbose=False)
-        self.hierarchy_model.set_hierarchy_extractor(hierarchy_extractor)
-        self.hierarchy_model.eval()
-        
-        # Load main CLIP model - Use the trained model directly
-        self.main_model = CLIPModel_transformers.from_pretrained('laion/CLIP-ViT-B-32-laion2B-s34B-b79K')
-        
-        # Load the trained weights
-        checkpoint = torch.load(config.main_model_path, map_location=self.device)
-        if 'model_state_dict' in checkpoint:
-            self.main_model.load_state_dict(checkpoint['model_state_dict'])
-        else:
-            # Fallback: try to load as state dict directly
-            self.main_model.load_state_dict(checkpoint)
-            print("✅ Loaded model weights directly")
-        
-        self.main_model.to(self.device)
-        self.main_model.eval()
-        
-        # Load CLIP processor
-        self.clip_processor = CLIPProcessor.from_pretrained('laion/CLIP-ViT-B-32-laion2B-s34B-b79K')
-        
-        print(f"✅ Models loaded - Colors: {self.color_dim}D, Hierarchy: {self.hierarchy_dim}D")
-    
-    def _load_dataset(self):
-        """Load the fashion dataset"""
-        print("📊 Loading dataset...")
-        
-        # Load dataset
-        self.df = pd.read_csv(config.local_dataset_path)
-        self.df_clean = self.df.dropna(subset=[config.column_local_image_path])
-        
-        # Create dataset object
-        self.dataset = CustomDataset(self.df_clean)
-        self.dataset.set_training_mode(False)  # No augmentation for search
-        
-        print(f"✅ {len(self.df_clean)} items loaded for search")
-    
-    def _precompute_embeddings(self):
-        """Pre-compute embeddings for all items in the dataset"""
-        print("🔄 Pre-computing embeddings...")
-        
-        # OPTIMIZATION: Sample a subset for faster initialization
-        print(f"⚠️ Dataset too large ({len(self.dataset)} items). Using stratified sampling of 10 items per color-category combination.")
-        
-        # Stratified sampling by color-category combinations
-        sampled_df = self.df_clean.groupby([config.color_column, config.hierarchy_column]).sample(n=20, replace=False)
-        
-        # Get the original indices of sampled items
-        sampled_indices = sampled_df.index.tolist()
-        
-        all_embeddings = []
-        all_texts = []
-        all_colors = []
-        all_hierarchies = []
-        all_images = []
-        all_urls = []
-        
-        # Process in batches for efficiency
-        batch_size = 32
-        
-        # Add progress bar
-        from tqdm import tqdm
-        total_batches = (len(sampled_indices) + batch_size - 1) // batch_size
-        
-        for i in tqdm(range(0, len(sampled_indices), batch_size), 
-                     desc="Computing embeddings", 
-                     total=total_batches):
-            batch_end = min(i + batch_size, len(sampled_indices))
-            batch_items = []
-            
-            for j in range(i, batch_end):
-                try:
-                    # Use the original dataset with the sampled index
-                    original_idx = sampled_indices[j]
-                    image, text, color, hierarchy = self.dataset[original_idx]
-                    batch_items.append((image, text, color, hierarchy))
-                    all_texts.append(text)
-                    all_colors.append(color)
-                    all_hierarchies.append(hierarchy)
-                    all_images.append(self.df_clean.iloc[original_idx][config.column_local_image_path])
-                    all_urls.append(self.df_clean.iloc[original_idx][config.column_url_image])
-                except Exception as e:
-                    print(f"⚠️ Skipping item {j}: {e}")
-                    continue
-            
-            if not batch_items:
-                continue
-            
-            # Process batch
-            images = torch.stack([item[0] for item in batch_items]).to(self.device)
-            texts = [item[1] for item in batch_items]
-            
-            with torch.no_grad():
-                # Get embeddings from main model (text embeddings only)
-                text_inputs = self.clip_processor(text=texts, padding=True, return_tensors="pt")
-                text_inputs = {k: v.to(self.device) for k, v in text_inputs.items()}
-                
-                # Create dummy images for the model
-                dummy_images = torch.zeros(len(texts), 3, 224, 224).to(self.device)
-                
-                outputs = self.main_model(**text_inputs, pixel_values=dummy_images)
-                embeddings = outputs.text_embeds.cpu().numpy()
-                
-                all_embeddings.extend(embeddings)
-        
-        self.all_embeddings = np.array(all_embeddings)
-        self.all_texts = all_texts
-        self.all_colors = all_colors
-        self.all_hierarchies = all_hierarchies
-        self.all_images = all_images
-        self.all_urls = all_urls
-        
-        print(f"✅ Pre-computed embeddings for {len(self.all_embeddings)} items")
-    
-    def search_by_text(self, query_text: str, filter_category: str = None) -> List[dict]:
-        """
-        Search for clothing items using text query
-        
-        Args:
-            query_text: Text description to search for
-            
-        Returns:
-            List of dictionaries containing search results
-        """
-        print(f"🔍 Searching for: '{query_text}'")
-        
-        # Get query embedding
-        with torch.no_grad():
-            text_inputs = self.clip_processor(text=[query_text], padding=True, return_tensors="pt")
-            text_inputs = {k: v.to(self.device) for k, v in text_inputs.items()}
-            
-            # Create a dummy image tensor to satisfy the model's requirements
-            dummy_image = torch.zeros(1, 3, 224, 224).to(self.device)
-            
-            outputs = self.main_model(**text_inputs, pixel_values=dummy_image)
-            query_embedding = outputs.text_embeds.cpu().numpy()
-        
-        # Calculate similarities
-        similarities = cosine_similarity(query_embedding, self.all_embeddings)[0]
-
-        # Get top-k results
-        top_indices = np.argsort(similarities)[::-1][:self.top_k * 2]  # Prendre plus de résultats
-        
-        results = []
-        for idx in top_indices:
-            if similarities[idx] > -0.5:
-                # Filter by category if specified
-                if filter_category and filter_category.lower() not in self.all_hierarchies[idx].lower():
-                    continue
-                    
-                results.append({
-                    'rank': len(results) + 1,
-                    'image_path': self.all_images[idx],
-                    'text': self.all_texts[idx],
-                    'color': self.all_colors[idx],
-                    'hierarchy': self.all_hierarchies[idx],
-                    'similarity': float(similarities[idx]),
-                    'index': int(idx),
-                    'url': self.all_urls[idx]
-                })
-                
-                if len(results) >= self.top_k:
-                    break
-        
-        print(f"✅ Found {len(results)} results")
-        return results
-    
-
-    def display_results(self, results: List[dict], query_info: str = ""):
-        """
-        Display search results with images and information
-        
-        Args:
-            results: List of search result dictionaries
-            query_info: Information about the query
-        """
-        if not results:
-            print("❌ No results found")
-            return
-        
-        print(f"\n🎯 Search Results for: {query_info}")
-        print("=" * 80)
-        
-        # Calculate grid layout
-        n_results = len(results)
-        cols = min(5, n_results)
-        rows = (n_results + cols - 1) // cols
-        
-        fig, axes = plt.subplots(rows, cols, figsize=(4*cols, 4*rows))
-        if rows == 1:
-            axes = axes.reshape(1, -1)
-        elif cols == 1:
-            axes = axes.reshape(-1, 1)
-        
-        for i, result in enumerate(results):
-            row = i // cols
-            col = i % cols
-            ax = axes[row, col]
-            
-            try:
-                # Load and display image
-                image = Image.open(result['image_path'])
-                ax.imshow(image)
-                ax.axis('off')
-                
-                # Add title with similarity score
-                title = f"#{result['rank']} (Similarity: {result['similarity']:.3f})\n{result['color']} {result['hierarchy']}"
-                ax.set_title(title, fontsize=10, wrap=True)
-                
-            except Exception as e:
-                ax.text(0.5, 0.5, f"Error loading image\n{result['image_path']}", 
-                       ha='center', va='center', transform=ax.transAxes)
-                ax.axis('off')
-        
-        # Hide empty subplots
-        for i in range(n_results, rows * cols):
-            row = i // cols
-            col = i % cols
-            axes[row, col].axis('off')
-        
-        plt.tight_layout()
-        plt.show()
-        
-        # Print detailed results
-        print("\n📋 Detailed Results:")
-        for result in results:
-            print(f"#{result['rank']:2d} | Similarity: {result['similarity']:.3f} | "
-                  f"Color: {result['color']:12s} | Category: {result['hierarchy']:15s} | "
-                  f"Text: {result['text'][:50]}...")
-            print(f"   🔗 URL: {result['url']}")
-            print()
-
-
-def main():
-    """Main function for command-line usage"""
-    parser = argparse.ArgumentParser(description="Fashion Search Engine with Category Emphasis")
-    parser.add_argument("--query", "-q", type=str, help="Search query")
-    parser.add_argument("--top-k", "-k", type=int, default=10, help="Number of results (default: 10)")
-    parser.add_argument("--fast", "-f", action="store_true", help="Fast mode (less items)")
-    parser.add_argument("--interactive", "-i", action="store_true", help="Interactive mode")
-
-    args = parser.parse_args()
-    
-    print("🎯 Fashion Search Engine with Category Emphasis")
-    
-    search_engine = FashionSearchEngine(
-        top_k=args.top_k, 
-    )
-    print("✅ Ready!")
-    
-    # Single query mode
-    if args.query:
-        print(f"🔍 Search: '{args.query}'...")
-        results = search_engine.search_by_text(args.query)
-        search_engine.display_results(results, args.query)
-
-    
-    # Interactive mode
-    print("Enter your query (e.g. 'red dress') or 'quit' to exit")
-    
-    while True:
-        try:
-            user_input = input("\n🔍 Query: ").strip()
-            if not user_input or user_input.lower() in ['quit', 'exit', 'q']:
-                print("👋 Goodbye!")
-                break
-            
-            if user_input.startswith('verify '):
-                if 'yellow accessories' in user_input:
-                    search_engine.display_yellow_accessories()
-                continue
-
-            print(f"🔍 Search: '{user_input}'...")
-            results = search_engine.search_by_text(user_input)
-            search_engine.display_results(results, user_input)
-            
-        except KeyboardInterrupt:
-            print("\n👋 Goodbye!")
-            break
-        except Exception as e:
-            print(f"❌ Error: {e}")
-
-if __name__ == "__main__":
-    main()

Evaluation/hierarchy_evaluation.py

@@ -1,589 +0,0 @@
-"""
-Hierarchy embedding evaluation for clothing category classification.
-This file evaluates the quality of hierarchy embeddings generated by the hierarchy model
-by calculating intra-class and inter-class similarity metrics, nearest neighbor and centroid-based
-classification accuracies, and generating confusion matrices. It can be used on different datasets
-(local validation, Kagl Marqo) to measure model generalization.
-"""
-
-import torch
-import pandas as pd
-import numpy as np
-import matplotlib.pyplot as plt
-import seaborn as sns
-from sklearn.metrics.pairwise import cosine_similarity
-from sklearn.metrics import confusion_matrix, classification_report, accuracy_score
-from collections import defaultdict
-import os
-from tqdm import tqdm
-from torch.utils.data import Dataset, DataLoader
-from torchvision import transforms
-from sklearn.model_selection import train_test_split
-from io import BytesIO
-from PIL import Image
-import config
-import warnings
-warnings.filterwarnings('ignore')
-from hierarchy_model import Model, HierarchyExtractor, HierarchyDataset, collate_fn
-
-
-class EmbeddingEvaluator:
-    """
-    Evaluator for hierarchy embeddings generated by the hierarchy model.
-    
-    This class provides methods to evaluate the quality of hierarchy embeddings by computing
-    similarity metrics, classification accuracies, and generating visualizations.
-    """
-    
-    def __init__(self, model_path, directory):
-        """
-        Initialize the embedding evaluator.
-        
-        Args:
-            model_path: Path to the trained hierarchy model checkpoint
-            directory: Directory to save evaluation results and visualizations
-        """
-        self.device = config.device
-        self.directory = directory
-
-        # 1. Load the dataset
-        CSV = config.local_dataset_path
-        print(f"📁 Using dataset with local images: {CSV}")
-        df = pd.read_csv(CSV)
-        
-        print(f"📁 Loaded {len(df)} samples")
-        
-        # 2. Get unique hierarchy classes from the dataset
-        hierarchy_classes = sorted(df[config.hierarchy_column].unique().tolist())
-        print(f"📋 Found {len(hierarchy_classes)} hierarchy classes")
-
-        _, self.val_df = train_test_split(df, test_size=0.2, random_state=42, stratify=df[config.hierarchy_column])
-        
-        # 3. Load the model
-        if os.path.exists(model_path):
-            checkpoint = torch.load(model_path, map_location=self.device)
-            
-            # Use model_config to avoid shadowing the imported config module
-            model_config = checkpoint.get('config', {})
-            saved_hierarchy_classes = checkpoint['hierarchy_classes']
-            
-            # Use the saved hierarchy classes
-            self.hierarchy_classes = saved_hierarchy_classes
-            
-            # Create the hierarchy extractor
-            self.vocab = HierarchyExtractor(saved_hierarchy_classes)
-            
-            # Create the model with the saved configuration
-            self.model = Model(
-                num_hierarchy_classes=len(saved_hierarchy_classes),
-                embed_dim=model_config['embed_dim'],
-                dropout=model_config['dropout']
-            ).to(self.device)   
-                
-            self.model.load_state_dict(checkpoint['model_state'])
-            
-            print(f"✅ Model loaded with:")
-            print(f"📋 Hierarchy classes: {len(saved_hierarchy_classes)}")
-            print(f"🎯 Embed dim: {model_config['embed_dim']}")
-            print(f"💧 Dropout: {model_config['dropout']}")
-            print(f"📅 Epoch: {checkpoint.get('epoch', 'unknown')}")
-                
-        else:
-            raise FileNotFoundError(f"Model file {model_path} not found")
-        
-        self.model.eval()
-    
-    def create_dataloader(self, dataframe, batch_size=16):
-        """
-        Create a DataLoader for the hierarchy dataset.
-        
-        Args:
-            dataframe: DataFrame containing the dataset
-            batch_size: Batch size for the DataLoader
-            
-        Returns:
-            DataLoader instance
-        """
-        dataset = HierarchyDataset(dataframe, image_size=224)
-        
-        dataloader = DataLoader(
-            dataset, 
-            batch_size=batch_size, 
-            shuffle=False, 
-            collate_fn=lambda batch: collate_fn(batch, self.vocab), 
-            num_workers=0
-        )
-        
-        return dataloader
-    
-    def extract_embeddings(self, dataloader, embedding_type='text'):
-        """
-        Extract embeddings from the model for a given dataloader.
-        
-        Args:
-            dataloader: DataLoader containing images, texts, and hierarchy labels
-            embedding_type: Type of embeddings to extract ('text' or 'image')
-            
-        Returns:
-            Tuple of (embeddings array, labels list, texts list)
-        """
-        all_embeddings = []
-        all_labels = []
-        all_texts = []
-        
-        with torch.no_grad():
-            for batch in tqdm(dataloader, desc=f"Extracting {embedding_type} embeddings"):
-                images = batch['image'].to(self.device)
-                hierarchy_indices = batch['hierarchy_indices'].to(self.device)
-                hierarchy_labels = batch['hierarchy']
-                
-                # Forward pass
-                out = self.model(image=images, hierarchy_indices=hierarchy_indices)
-                embeddings = out['z_txt'] if embedding_type == 'text' else out['z_img'] if embedding_type == 'image' else out['z_txt']
-                
-                all_embeddings.append(embeddings.cpu().numpy())
-                all_labels.extend(hierarchy_labels)
-                all_texts.extend(hierarchy_labels)
-        
-        return np.vstack(all_embeddings), all_labels, all_texts
-    
-    def compute_similarity_metrics(self, embeddings, labels):
-        """
-        Compute intra-class and inter-class similarity metrics.
-        
-        Args:
-            embeddings: Array of embeddings [N, embed_dim]
-            labels: List of labels for each embedding
-            
-        Returns:
-            Dictionary containing similarity metrics, accuracies, and separation scores
-        """
-        similarities = cosine_similarity(embeddings)
-        
-        # Group embeddings by hierarchy
-        hierarchy_groups = defaultdict(list)
-        for i, hierarchy in enumerate(labels):
-            hierarchy_groups[hierarchy].append(i)
-        
-        # Calculate intra-class similarities (same hierarchy)
-        intra_class_similarities = []
-        for hierarchy, indices in hierarchy_groups.items():
-            if len(indices) > 1:
-                for i in range(len(indices)):
-                    for j in range(i+1, len(indices)):
-                        sim = similarities[indices[i], indices[j]]
-                        intra_class_similarities.append(sim)
-        
-                
-        # Calculate inter-class similarities (different hierarchies)
-        inter_class_similarities = []
-        hierarchies = list(hierarchy_groups.keys())
-        for i in range(len(hierarchies)):
-            for j in range(i+1, len(hierarchies)):
-                hierarchy1_indices = hierarchy_groups[hierarchies[i]]
-                hierarchy2_indices = hierarchy_groups[hierarchies[j]]
-                
-                for idx1 in hierarchy1_indices:
-                    for idx2 in hierarchy2_indices:
-                        sim = similarities[idx1, idx2]
-                        inter_class_similarities.append(sim)
-        
-        # Calculate classification accuracy using nearest neighbor in embedding space
-        nn_accuracy = self.compute_embedding_accuracy(embeddings, labels, similarities)
-        
-        # Calculate classification accuracy using centroids
-        centroid_accuracy = self.compute_centroid_accuracy(embeddings, labels)
-        
-        return {
-            'intra_class_similarities': intra_class_similarities,
-            'inter_class_similarities': inter_class_similarities,
-            'intra_class_mean': np.mean(intra_class_similarities) if intra_class_similarities else 0,
-            'inter_class_mean': np.mean(inter_class_similarities) if inter_class_similarities else 0,
-            'separation_score': np.mean(intra_class_similarities) - np.mean(inter_class_similarities) if intra_class_similarities and inter_class_similarities else 0,
-            'accuracy': nn_accuracy,
-            'centroid_accuracy': centroid_accuracy
-        }
-    
-    def compute_embedding_accuracy(self, embeddings, labels, similarities):
-        """
-        Compute classification accuracy using nearest neighbor in embedding space.
-        
-        Args:
-            embeddings: Array of embeddings [N, embed_dim]
-            labels: List of true labels
-            similarities: Pre-computed similarity matrix [N, N]
-            
-        Returns:
-            Accuracy score (float between 0 and 1)
-        """
-        correct_predictions = 0
-        total_predictions = len(labels)
-        
-        for i in range(len(embeddings)):
-            true_label = labels[i]
-            
-            # Find the most similar embedding (excluding itself)
-            similarities_row = similarities[i].copy()
-            similarities_row[i] = -1  # Exclude self-similarity
-            nearest_neighbor_idx = np.argmax(similarities_row)
-            predicted_label = labels[nearest_neighbor_idx]
-            
-            if predicted_label == true_label:
-                correct_predictions += 1
-        
-        return correct_predictions / total_predictions if total_predictions > 0 else 0
-    
-    def compute_centroid_accuracy(self, embeddings, labels):
-        """
-        Compute classification accuracy using hierarchy centroids.
-        
-        Each hierarchy class is represented by its centroid (mean embedding), and each
-        embedding is classified to the nearest centroid.
-        
-        Args:
-            embeddings: Array of embeddings [N, embed_dim]
-            labels: List of true labels
-            
-        Returns:
-            Accuracy score (float between 0 and 1)
-        """
-        # Create centroids for each hierarchy
-        unique_hierarchies = list(set(labels))
-        centroids = {}
-        
-        for hierarchy in unique_hierarchies:
-            hierarchy_indices = [i for i, label in enumerate(labels) if label == hierarchy]
-            hierarchy_embeddings = embeddings[hierarchy_indices]
-            centroids[hierarchy] = np.mean(hierarchy_embeddings, axis=0)
-        
-        # Classify each embedding to nearest centroid
-        correct_predictions = 0
-        total_predictions = len(labels)
-        
-        for i, embedding in enumerate(embeddings):
-            true_label = labels[i]
-            
-            # Find closest centroid
-            best_similarity = -1
-            predicted_label = None
-            
-            for hierarchy, centroid in centroids.items():
-                similarity = cosine_similarity([embedding], [centroid])[0][0]
-                if similarity > best_similarity:
-                    best_similarity = similarity
-                    predicted_label = hierarchy
-            
-            if predicted_label == true_label:
-                correct_predictions += 1
-        
-        return correct_predictions / total_predictions if total_predictions > 0 else 0
-    
-    def predict_hierarchy_from_embeddings(self, embeddings, labels):
-        """
-        Predict hierarchy from embeddings using centroid-based classification.
-        
-        Args:
-            embeddings: Array of embeddings [N, embed_dim]
-            labels: List of labels used to compute centroids
-            
-        Returns:
-            List of predicted hierarchy labels
-        """
-        # Create hierarchy centroids from training data
-        unique_hierarchies = list(set(labels))
-        centroids = {}
-        
-        for hierarchy in unique_hierarchies:
-            hierarchy_indices = [i for i, label in enumerate(labels) if label == hierarchy]
-            hierarchy_embeddings = embeddings[hierarchy_indices]
-            centroids[hierarchy] = np.mean(hierarchy_embeddings, axis=0)
-        
-        # Predict hierarchy for all embeddings
-        predictions = []
-        
-        for i, embedding in enumerate(embeddings):
-            # Find closest centroid
-            best_similarity = -1
-            predicted_hierarchy = None
-            
-            for hierarchy, centroid in centroids.items():
-                similarity = cosine_similarity([embedding], [centroid])[0][0]
-                if similarity > best_similarity:
-                    best_similarity = similarity
-                    predicted_hierarchy = hierarchy
-            
-            predictions.append(predicted_hierarchy)
-        
-        return predictions
-    
-    def create_confusion_matrix(self, true_labels, predicted_labels, title="Confusion Matrix"):
-        """
-        Create and plot a confusion matrix.
-        
-        Args:
-            true_labels: List of true labels
-            predicted_labels: List of predicted labels
-            title: Title for the confusion matrix plot
-            
-        Returns:
-            Tuple of (figure, accuracy, confusion_matrix)
-        """
-        # Get unique labels
-        unique_labels = sorted(list(set(true_labels + predicted_labels)))
-        
-        # Create confusion matrix
-        cm = confusion_matrix(true_labels, predicted_labels, labels=unique_labels)
-        
-        # Calculate accuracy
-        accuracy = accuracy_score(true_labels, predicted_labels)
-        
-        # Plot confusion matrix
-        plt.figure(figsize=(12, 10))
-        sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', 
-                   xticklabels=unique_labels, yticklabels=unique_labels)
-        plt.title(f'{title}\nAccuracy: {accuracy:.3f} ({accuracy*100:.1f}%)')
-        plt.ylabel('True Hierarchy')
-        plt.xlabel('Predicted Hierarchy')
-        plt.xticks(rotation=45)
-        plt.yticks(rotation=0)
-        plt.tight_layout()
-        
-        return plt.gcf(), accuracy, cm
-    
-    def evaluate_classification_performance(self, embeddings, labels, embedding_type="Embeddings"):
-        """
-        Evaluate classification performance and create confusion matrix.
-        
-        Args:
-            embeddings: Array of embeddings [N, embed_dim]
-            labels: List of true labels
-            embedding_type: Type of embeddings for display purposes
-            
-        Returns:
-            Dictionary containing accuracy, predictions, confusion matrix, and classification report
-        """
-        # Predict hierarchy
-        predictions = self.predict_hierarchy_from_embeddings(embeddings, labels)
-        
-        # Calculate accuracy
-        accuracy = accuracy_score(labels, predictions)
-        
-        # Create confusion matrix
-        fig, acc, cm = self.create_confusion_matrix(labels, predictions, 
-                                                   f"{embedding_type} - Hierarchy Classification")
-        
-        # Generate classification report
-        unique_labels = sorted(list(set(labels)))
-        report = classification_report(labels, predictions, labels=unique_labels, 
-                                     target_names=unique_labels, output_dict=True)
-        
-        return {
-            'accuracy': accuracy,
-            'predictions': predictions,
-            'confusion_matrix': cm,
-            'classification_report': report,
-            'figure': fig
-        }
-    
-    def evaluate_dataset(self, dataframe, dataset_name="Dataset"):
-        """
-        Evaluate embeddings on a given dataset.
-        
-        This method extracts embeddings for text and image, computes similarity metrics,
-        evaluates classification performance, and saves confusion matrices.
-        
-        Args:
-            dataframe: DataFrame containing the dataset
-            dataset_name: Name of the dataset for display purposes
-            
-        Returns:
-            Dictionary containing evaluation results for text and image embeddings
-        """
-        print(f"\n{'='*60}")
-        print(f"Evaluating {dataset_name}")
-        print(f"{'='*60}")
-        
-        # Create dataloader exactly as during training
-        dataloader = self.create_dataloader(dataframe, batch_size=16)
-        
-        results = {}
-        
-        # Evaluate text embeddings
-        text_embeddings, text_labels, texts = self.extract_embeddings(dataloader, 'text')
-        text_metrics = self.compute_similarity_metrics(text_embeddings, text_labels)
-        text_classification = self.evaluate_classification_performance(text_embeddings, text_labels, "Text Embeddings")
-        text_metrics.update(text_classification)
-        results['text'] = text_metrics
-        
-        # Evaluate image embeddings
-        image_embeddings, image_labels, _ = self.extract_embeddings(dataloader, 'image')
-        image_metrics = self.compute_similarity_metrics(image_embeddings, image_labels)
-        image_classification = self.evaluate_classification_performance(image_embeddings, image_labels, "Image Embeddings")
-        image_metrics.update(image_classification)
-        results['image'] = image_metrics
-        
-        # Evaluate hierarchy embeddings
-        hierarchy_embeddings, hierarchy_labels, _ = self.extract_embeddings(dataloader, 'category2')
-        hierarchy_metrics = self.compute_similarity_metrics(hierarchy_embeddings, hierarchy_labels)
-        hierarchy_classification = self.evaluate_classification_performance(hierarchy_embeddings, hierarchy_labels, "hierarchy Embeddings")
-        hierarchy_metrics.update(hierarchy_classification)
-        results['hierarchy'] = hierarchy_metrics
-        
-        # Print results
-        print(f"\n{dataset_name} Results:")
-        print("-" * 40)
-        for emb_type, metrics in results.items():
-            print(f"{emb_type.capitalize()} Embeddings:")
-            print(f"  Intra-class similarity (same hierarchy): {metrics['intra_class_mean']:.4f}")
-            print(f"  Inter-class similarity (diff hierarchy): {metrics['inter_class_mean']:.4f}")
-            print(f"  Separation score: {metrics['separation_score']:.4f}")
-            print(f"  Nearest Neighbor Accuracy: {metrics['accuracy']:.4f} ({metrics['accuracy']*100:.1f}%)")
-            print(f"  Centroid Accuracy: {metrics['centroid_accuracy']:.4f} ({metrics['centroid_accuracy']*100:.1f}%)")
-            
-            # Classification report summary
-            report = metrics['classification_report']
-            print(f"  📊 Classification Performance:")
-            print(f"    • Macro Avg F1-Score: {report['macro avg']['f1-score']:.4f}")
-            print(f"    • Weighted Avg F1-Score: {report['weighted avg']['f1-score']:.4f}")
-            print(f"    • Support: {report['macro avg']['support']:.0f} samples")
-            print()
-        
-        # Create visualizations
-        os.makedirs(f'{self.directory}', exist_ok=True)
-        
-        # Confusion matrices
-        results['text']['figure'].savefig(f'{self.directory}/{dataset_name.lower()}_text_confusion_matrix.png', dpi=300, bbox_inches='tight')
-        plt.close(results['text']['figure'])
-        
-        results['image']['figure'].savefig(f'{self.directory}/{dataset_name.lower()}_image_confusion_matrix.png', dpi=300, bbox_inches='tight')
-        plt.close(results['image']['figure'])
-        
-        results['hierarchy']['figure'].savefig(f'{self.directory}/{dataset_name.lower()}_hierarchy_confusion_matrix.png', dpi=300, bbox_inches='tight')
-        plt.close(results['hierarchy']['figure'])
-        
-        return results
-
-class KaglDataset(Dataset):
-    def __init__(self, dataframe):
-        self.dataframe = dataframe
-        # Use VALIDATION transforms (no augmentation)
-        self.transform = transforms.Compose([
-            transforms.Resize((224, 224)),
-            transforms.ToTensor(),
-            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
-        ])
-
-    def __len__(self):
-        return len(self.dataframe)
-
-    def __getitem__(self, idx):
-        row = self.dataframe.iloc[idx]
-        
-        # Handle image
-        image_data = row['image_url']
-        image = Image.open(BytesIO(image_data['bytes'])).convert("RGB")
-        image = self.transform(image)
-
-        # Get text and hierarchy
-        description = row['text']  
-        hierarchy = row['hierarchy']
-
-        return image, description, hierarchy
-
-def load_Kagl_marqo_dataset(evaluator):
-    """Load and prepare Kagl KAGL dataset"""
-    from datasets import load_dataset
-    print("Loading Kagl KAGL dataset...")
-
-    # Load the dataset
-    dataset = load_dataset("Marqo/KAGL")
-    df = dataset["data"].to_pandas()
-    print(f"✅ Dataset Kagl loaded")
-    print(f"📊 Before filtering: {len(df)} samples")
-    print(f"📋 Available columns: {list(df.columns)}")
-
-    # Check available categories and map them to our hierarchy
-    print(f"🎨 Available categories: {sorted(df['category2'].unique())}")
-    # Apply mapping
-    df['hierarchy'] = df['category2'].str.lower()
-    df['hierarchy'] = df['hierarchy'].replace('bags', 'bag').replace('topwear', 'top').replace('flip flops', 'shoes').replace('sandal', 'shoes')
-
-    # Filter to only include valid hierarchies that exist in our model
-    valid_hierarchies = df['hierarchy'].dropna().unique()
-    print(f"🎯 Valid hierarchies found: {sorted(valid_hierarchies)}")
-    print(f"🎯 Model hierarchies: {sorted(evaluator.hierarchy_classes)}")
-    
-    # Filter to only include hierarchies that exist in our model
-    df = df[df['hierarchy'].isin(evaluator.hierarchy_classes)]
-    print(f"📊 After filtering to model hierarchies: {len(df)} samples")
-    
-    if len(df) == 0:
-        print("❌ No samples left after hierarchy filtering.")
-        return pd.DataFrame()
-    
-    # Ensure we have text and image data
-    df = df.dropna(subset=['text', 'image'])
-    print(f"📊 After removing missing text/image: {len(df)} samples")
-    
-    # Show sample of text data to verify quality
-    print(f"📝 Sample texts:")
-    for i, (text, hierarchy) in enumerate(zip(df['text'].head(3), df['hierarchy'].head(3))):
-        print(f"  {i+1}. [{hierarchy}] {text[:100]}...")
-    
-    print(f"📊 After sampling: {len(df)} samples")
-    print(f"📊 Samples per hierarchy:")
-    for hierarchy in sorted(df['hierarchy'].unique()):
-        count = len(df[df['hierarchy'] == hierarchy])
-        print(f"  {hierarchy}: {count} samples")
-    
-    # Create formatted dataset with proper column names
-    Kagl_formatted = pd.DataFrame({
-        'image_url': df['image'],
-        'text': df['text'],
-        'hierarchy': df['hierarchy']
-    })
-    
-    print(f"📊 Final dataset size: {len(Kagl_formatted)} samples")
-    return Kagl_formatted
-
-if __name__ == "__main__":
-    device = config.device
-    model_path = config.hierarchy_model_path
-    directory = config.evaluation_directory
-
-    print(f"🚀 Starting evaluation with {model_path}")
-    
-    evaluator = EmbeddingEvaluator(model_path, directory)
-    
-    print(f"📊 Final hierarchy classes after initialization: {len(evaluator.vocab.hierarchy_classes)} classes")
-    
-    # Evaluate on validation dataset (same subset as during training)
-    print("\n" + "="*60)
-    print("EVALUATING VALIDATION DATASET")
-    print("="*60)
-    val_results = evaluator.evaluate_dataset(evaluator.val_df, "Validation Dataset")
-    
-    print("\n" + "="*60)
-    print("EVALUATING Kagl MARQO DATASET") 
-    print("="*60)
-    df_Kagl_marqo = load_Kagl_marqo_dataset(evaluator)
-    Kagl_results = evaluator.evaluate_dataset(df_Kagl_marqo, "Kagl Marqo Dataset")
-    
-    # Compare results
-    print(f"\n{'='*60}")
-    print("FINAL EVALUATION SUMMARY")
-    print(f"{'='*60}")
-    
-    print("\n🔍 VALIDATION DATASET RESULTS:")
-    print(f"Text      - Separation: {val_results['text']['separation_score']:.4f} | NN Acc: {val_results['text']['accuracy']*100:.1f}% | Centroid Acc: {val_results['text']['centroid_accuracy']*100:.1f}%")
-    print(f"Image     - Separation: {val_results['image']['separation_score']:.4f} | NN Acc: {val_results['image']['accuracy']*100:.1f}% | Centroid Acc: {val_results['image']['centroid_accuracy']*100:.1f}%")
-    print(f"hierarchy     - Separation: {val_results['hierarchy']['separation_score']:.4f} | NN Acc: {val_results['hierarchy']['accuracy']*100:.1f}% | Centroid Acc: {val_results['hierarchy']['centroid_accuracy']*100:.1f}%")
-    
-    print("\n🌐 Kagl MARQO DATASET RESULTS:")
-    print(f"Text      - Separation: {Kagl_results['text']['separation_score']:.4f} | NN Acc: {Kagl_results['text']['accuracy']*100:.1f}% | Centroid Acc: {Kagl_results['text']['centroid_accuracy']*100:.1f}%")
-    print(f"Image     - Separation: {Kagl_results['image']['separation_score']:.4f} | NN Acc: {Kagl_results['image']['accuracy']*100:.1f}% | Centroid Acc: {Kagl_results['image']['centroid_accuracy']*100:.1f}%")
-    print(f"Hierarchy     - Separation: {Kagl_results['hierarchy']['separation_score']:.4f} | NN Acc: {Kagl_results['hierarchy']['accuracy']*100:.1f}% | Centroid Acc: {Kagl_results['hierarchy']['centroid_accuracy']*100:.1f}%")
-    
-    
-    print(f"\n✅ Evaluation completed! Check 'improved_model_evaluation/' for visualization files.")
-    print(f"📊 Final hierarchy classes used: {len(evaluator.vocab.hierarchy_classes)} classes")

Evaluation/hierarchy_evaluation_with_clip_baseline.py

@@ -1,808 +0,0 @@
-"""
-Hierarchy embedding evaluation with CLIP baseline comparison.
-This file evaluates the quality of hierarchy embeddings from the custom model and compares them
-with a CLIP baseline model (OpenAI CLIP). It calculates similarity metrics, classification accuracies,
-and generates confusion matrices for both models to measure relative performance. It also supports
-evaluation on Fashion-MNIST and kagl Marqo datasets.
-"""
-
-import torch
-import pandas as pd
-import numpy as np
-import matplotlib.pyplot as plt
-import seaborn as sns
-from sklearn.metrics.pairwise import cosine_similarity
-from sklearn.metrics import confusion_matrix, classification_report, accuracy_score, f1_score
-from collections import defaultdict
-import os
-import requests
-from tqdm import tqdm
-from torch.utils.data import Dataset, DataLoader
-from torchvision import transforms
-from sklearn.model_selection import train_test_split
-from io import BytesIO
-from PIL import Image
-import warnings
-warnings.filterwarnings('ignore')
-
-# Import transformers CLIP
-from transformers import CLIPProcessor, CLIPModel as TransformersCLIPModel
-
-# Import your custom model
-from hierarchy_model import Model, HierarchyExtractor, HierarchyDataset, collate_fn
-import config
-
-def convert_fashion_mnist_to_image(pixel_values):
-    """Convert Fashion-MNIST pixel values to PIL image"""
-    # Reshape to 28x28 and convert to PIL Image
-    image_array = np.array(pixel_values).reshape(28, 28).astype(np.uint8)
-    # Convert to RGB by duplicating the grayscale channel
-    image_array = np.stack([image_array] * 3, axis=-1)
-    image = Image.fromarray(image_array)
-    return image
-
-def get_fashion_mnist_labels():
-    """Get Fashion-MNIST class labels"""
-    return {
-        0: "T-shirt/top", 
-        1: "Trouser", 
-        2: "Pullover", 
-        3: "Dress", 
-        4: "Coat", 
-        5: "Sandal", 
-        6: "Shirt", 
-        7: "Sneaker", 
-        8: "Bag", 
-        9: "Ankle boot"
-    }
-
-class FashionMNISTDataset(Dataset):
-    def __init__(self, dataframe, image_size=224):
-        self.dataframe = dataframe
-        self.image_size = image_size
-        self.labels_map = get_fashion_mnist_labels()
-        
-        # Simple transforms for validation/inference
-        self.transform = transforms.Compose([
-            transforms.Resize((image_size, image_size)),
-            transforms.ToTensor(),
-            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
-        ])
-        
-    def __len__(self):
-        return len(self.dataframe)
-
-    def __getitem__(self, idx):
-        row = self.dataframe.iloc[idx]
-        
-        # Get pixel values (columns 1-784)
-        pixel_cols = [f'pixel{i}' for i in range(1, 785)]
-        pixel_values = row[pixel_cols].values
-        
-        # Convert to image
-        image = convert_fashion_mnist_to_image(pixel_values)
-        image = self.transform(image)
-
-        # Get text description
-        text = row['text']
-        
-        # Get hierarchy label
-        hierarchy = row['hierarchy']
-
-        return image, text, hierarchy
-
-class CLIPBaselineEvaluator:
-    def __init__(self, device='mps'):
-        self.device = torch.device(device)
-        
-        # Load CLIP model and processor
-        print("🤗 Loading CLIP baseline model from transformers...")
-        self.clip_model = TransformersCLIPModel.from_pretrained("openai/clip-vit-base-patch32").to(self.device)
-        self.clip_processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
-        self.clip_model.eval()
-        print("✅ CLIP model loaded successfully")
-        
-    def extract_clip_embeddings(self, images, texts):
-        """Extract CLIP embeddings for images and texts"""
-        all_image_embeddings = []
-        all_text_embeddings = []
-        
-        with torch.no_grad():
-            for i in tqdm(range(len(images)), desc="Extracting CLIP embeddings"):
-                # Process image
-                if isinstance(images[i], torch.Tensor):
-                    # Convert tensor back to PIL Image
-                    image_tensor = images[i]
-                    if image_tensor.dim() == 3:
-                        # Denormalize
-                        mean = torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1)
-                        std = torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1)
-                        image_tensor = image_tensor * std + mean
-                        image_tensor = torch.clamp(image_tensor, 0, 1)
-                        
-                        # Convert to PIL
-                        image_pil = transforms.ToPILImage()(image_tensor)
-                elif isinstance(images[i], Image.Image):
-                    image_pil = images[i]
-                else:
-                    raise ValueError(f"Unsupported image type: {type(images[i])}")
-                
-                # Process with CLIP
-                inputs = self.clip_processor(
-                    text=texts[i], 
-                    images=image_pil, 
-                    return_tensors="pt", 
-                    padding=True
-                ).to(self.device)
-                
-                outputs = self.clip_model(**inputs)
-                
-                # Get normalized embeddings
-                image_emb = outputs.image_embeds / outputs.image_embeds.norm(p=2, dim=-1, keepdim=True)
-                text_emb = outputs.text_embeds / outputs.text_embeds.norm(p=2, dim=-1, keepdim=True)
-                
-                all_image_embeddings.append(image_emb.cpu().numpy())
-                all_text_embeddings.append(text_emb.cpu().numpy())
-        
-        return np.vstack(all_image_embeddings), np.vstack(all_text_embeddings)
-
-
-class EmbeddingEvaluator:
-    def __init__(self, model_path, directory):
-        self.device = config.device
-        self.directory = directory
-
-        # 1. Load the dataset
-        CSV = config.local_dataset_path  
-        print(f"📁 Using dataset with local images: {CSV}")
-        df = pd.read_csv(CSV)
-        
-        print(f"📁 Loaded {len(df)} samples")
-        
-        # 2. Get unique hierarchy classes from the dataset
-        hierarchy_classes = sorted(df[config.hierarchy_column].unique().tolist())
-        print(f"📋 Found {len(hierarchy_classes)} hierarchy classes")
-
-        _, self.val_df = train_test_split(df, test_size=0.2, random_state=42, stratify=df['hierarchy'])
-        
-        # 3. Load the model
-        if os.path.exists(model_path):
-            checkpoint = torch.load(model_path, map_location=self.device)
-            
-            config = checkpoint.get('config', {})
-            saved_hierarchy_classes = checkpoint['hierarchy_classes']
-            
-            # Use the saved hierarchy classes
-            self.hierarchy_classes = saved_hierarchy_classes
-            
-            # Create the hierarchy extractor
-            self.vocab = HierarchyExtractor(saved_hierarchy_classes)
-            
-            # Create the model with the saved configuration
-            self.model = Model(
-                num_hierarchy_classes=len(saved_hierarchy_classes),
-                embed_dim=config['embed_dim'],
-                dropout=config['dropout']
-            ).to(self.device)   
-                
-            self.model.load_state_dict(checkpoint['model_state'])
-            
-            print(f"✅ Custom model loaded with:")
-            print(f"📋 Hierarchy classes: {len(saved_hierarchy_classes)}")
-            print(f"🎯 Embed dim: {config['embed_dim']}")
-            print(f"💧 Dropout: {config['dropout']}")
-            print(f"📅 Epoch: {checkpoint.get('epoch', 'unknown')}")
-                
-        else:
-            raise FileNotFoundError(f"Model file {model_path} not found")
-        
-        self.model.eval()
-        
-        # Initialize CLIP baseline
-        self.clip_evaluator = CLIPBaselineEvaluator(device)
-    
-    def create_dataloader(self, dataframe, batch_size=16):
-        """Create a dataloader for custom model"""
-        # Check if this is Fashion-MNIST data (has pixel1 column)
-        if 'pixel1' in dataframe.columns:
-            print("🔍 Detected Fashion-MNIST data, using FashionMNISTDataset")
-            dataset = FashionMNISTDataset(dataframe, image_size=224)
-        else:
-            dataset = HierarchyDataset(dataframe, image_size=224)
-        
-        dataloader = DataLoader(
-            dataset, 
-            batch_size=batch_size, 
-            shuffle=False, 
-            collate_fn=lambda batch: collate_fn(batch, self.vocab), 
-            num_workers=0
-        )
-        
-        return dataloader
-    
-    def create_clip_dataloader(self, dataframe, batch_size=16):
-        """Create a dataloader for CLIP baseline"""
-        # Check if this is Fashion-MNIST data (has pixel1 column)
-        if 'pixel1' in dataframe.columns:
-            print("🔍 Detected Fashion-MNIST data for CLIP, using FashionMNISTDataset")
-            dataset = FashionMNISTDataset(dataframe, image_size=224)
-        else:
-            dataset = CLIPDataset(dataframe)
-        
-        dataloader = DataLoader(
-            dataset, 
-            batch_size=batch_size, 
-            shuffle=False, 
-            num_workers=0
-        )
-        
-        return dataloader
-    
-    def extract_custom_embeddings(self, dataloader, embedding_type='text'):
-        """Extract embeddings from custom model"""
-        all_embeddings = []
-        all_labels = []
-        all_texts = []
-        
-        with torch.no_grad():
-            for batch in tqdm(dataloader, desc=f"Extracting custom {embedding_type} embeddings"):
-                images = batch['image'].to(self.device)
-                hierarchy_indices = batch['hierarchy_indices'].to(self.device)
-                hierarchy_labels = batch['hierarchy']
-                
-                # Forward pass
-                out = self.model(image=images, hierarchy_indices=hierarchy_indices)
-                embeddings = out['z_txt'] if embedding_type == 'text' else out['z_img'] if embedding_type == 'image' else out['z_txt']
-                
-                all_embeddings.append(embeddings.cpu().numpy())
-                all_labels.extend(hierarchy_labels)
-                all_texts.extend(hierarchy_labels)
-        
-        return np.vstack(all_embeddings), all_labels, all_texts
-
-    def compute_similarity_metrics(self, embeddings, labels):
-        """Compute intra-class and inter-class similarities"""
-        similarities = cosine_similarity(embeddings)
-        
-        # Group embeddings by hierarchy
-        hierarchy_groups = defaultdict(list)
-        for i, hierarchy in enumerate(labels):
-            hierarchy_groups[hierarchy].append(i)
-        
-        # Calculate intra-class similarities (same hierarchy)
-        intra_class_similarities = []
-        for hierarchy, indices in hierarchy_groups.items():
-            if len(indices) > 1:
-                for i in range(len(indices)):
-                    for j in range(i+1, len(indices)):
-                        sim = similarities[indices[i], indices[j]]
-                        intra_class_similarities.append(sim)
-        
-        # Calculate inter-class similarities (different hierarchies)
-        inter_class_similarities = []
-        hierarchies = list(hierarchy_groups.keys())
-        for i in range(len(hierarchies)):
-            for j in range(i+1, len(hierarchies)):
-                hierarchy1_indices = hierarchy_groups[hierarchies[i]]
-                hierarchy2_indices = hierarchy_groups[hierarchies[j]]
-                
-                for idx1 in hierarchy1_indices:
-                    for idx2 in hierarchy2_indices:
-                        sim = similarities[idx1, idx2]
-                        inter_class_similarities.append(sim)
-        
-        # Calculate classification accuracy using nearest neighbor in embedding space
-        nn_accuracy = self.compute_embedding_accuracy(embeddings, labels, similarities)
-        
-        # Calculate classification accuracy using centroids
-        centroid_accuracy = self.compute_centroid_accuracy(embeddings, labels)
-        
-        return {
-            'intra_class_similarities': intra_class_similarities,
-            'inter_class_similarities': inter_class_similarities,
-            'intra_class_mean': np.mean(intra_class_similarities) if intra_class_similarities else 0,
-            'inter_class_mean': np.mean(inter_class_similarities) if inter_class_similarities else 0,
-            'separation_score': np.mean(intra_class_similarities) - np.mean(inter_class_similarities) if intra_class_similarities and inter_class_similarities else 0,
-            'accuracy': nn_accuracy,
-            'centroid_accuracy': centroid_accuracy
-        }
-    
-    def compute_embedding_accuracy(self, embeddings, labels, similarities):
-        """Compute classification accuracy using nearest neighbor in embedding space"""
-        correct_predictions = 0
-        total_predictions = len(labels)
-        
-        for i in range(len(embeddings)):
-            true_label = labels[i]
-            
-            # Find the most similar embedding (excluding itself)
-            similarities_row = similarities[i].copy()
-            similarities_row[i] = -1  # Exclude self-similarity
-            nearest_neighbor_idx = np.argmax(similarities_row)
-            predicted_label = labels[nearest_neighbor_idx]
-            
-            if predicted_label == true_label:
-                correct_predictions += 1
-        
-        return correct_predictions / total_predictions if total_predictions > 0 else 0
-    
-    def compute_centroid_accuracy(self, embeddings, labels):
-        """Compute classification accuracy using hierarchy centroids"""
-        # Create centroids for each hierarchy
-        unique_hierarchies = list(set(labels))
-        centroids = {}
-        
-        for hierarchy in unique_hierarchies:
-            hierarchy_indices = [i for i, label in enumerate(labels) if label == hierarchy]
-            hierarchy_embeddings = embeddings[hierarchy_indices]
-            centroids[hierarchy] = np.mean(hierarchy_embeddings, axis=0)
-        
-        # Classify each embedding to nearest centroid
-        correct_predictions = 0
-        total_predictions = len(labels)
-        
-        for i, embedding in enumerate(embeddings):
-            true_label = labels[i]
-            
-            # Find closest centroid
-            best_similarity = -1
-            predicted_label = None
-            
-            for hierarchy, centroid in centroids.items():
-                similarity = cosine_similarity([embedding], [centroid])[0][0]
-                if similarity > best_similarity:
-                    best_similarity = similarity
-                    predicted_label = hierarchy
-            
-            if predicted_label == true_label:
-                correct_predictions += 1
-        
-        return correct_predictions / total_predictions if total_predictions > 0 else 0
-
-    def create_confusion_matrix(self, true_labels, predicted_labels, title="Confusion Matrix"):
-        """Create and plot confusion matrix"""
-        # Get unique labels
-        unique_labels = sorted(list(set(true_labels + predicted_labels)))
-        
-        # Create confusion matrix
-        cm = confusion_matrix(true_labels, predicted_labels, labels=unique_labels)
-        
-        # Calculate accuracy
-        accuracy = accuracy_score(true_labels, predicted_labels)
-        
-        # Plot confusion matrix
-        plt.figure(figsize=(12, 10))
-        sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', 
-                   xticklabels=unique_labels, yticklabels=unique_labels)
-        plt.title(f'{title}\nAccuracy: {accuracy:.3f} ({accuracy*100:.1f}%)')
-        plt.ylabel('True Hierarchy')
-        plt.xlabel('Predicted Hierarchy')
-        plt.xticks(rotation=45)
-        plt.yticks(rotation=0)
-        plt.tight_layout()
-        
-        return plt.gcf(), accuracy, cm
-
-    def predict_hierarchy_from_embeddings(self, embeddings, labels):
-        """Predict hierarchy from embeddings using centroid-based classification"""
-        # Create hierarchy centroids from training data
-        unique_hierarchies = list(set(labels))
-        centroids = {}
-        
-        for hierarchy in unique_hierarchies:
-            hierarchy_indices = [i for i, label in enumerate(labels) if label == hierarchy]
-            hierarchy_embeddings = embeddings[hierarchy_indices]
-            centroids[hierarchy] = np.mean(hierarchy_embeddings, axis=0)
-        
-        # Predict hierarchy for all embeddings
-        predictions = []
-        
-        for i, embedding in enumerate(embeddings):
-            # Find closest centroid
-            best_similarity = -1
-            predicted_hierarchy = None
-            
-            for hierarchy, centroid in centroids.items():
-                similarity = cosine_similarity([embedding], [centroid])[0][0]
-                if similarity > best_similarity:
-                    best_similarity = similarity
-                    predicted_hierarchy = hierarchy
-            
-            predictions.append(predicted_hierarchy)
-        
-        return predictions
-
-    def evaluate_classification_performance(self, embeddings, labels, embedding_type="Embeddings"):
-        """Evaluate classification performance and create confusion matrix"""
-        # Predict hierarchy
-        predictions = self.predict_hierarchy_from_embeddings(embeddings, labels)
-        
-        # Calculate accuracy
-        accuracy = accuracy_score(labels, predictions)
-        
-        # Calculate F1 scores
-        unique_labels = sorted(list(set(labels)))
-        f1_macro = f1_score(labels, predictions, labels=unique_labels, average='macro', zero_division=0)
-        f1_weighted = f1_score(labels, predictions, labels=unique_labels, average='weighted', zero_division=0)
-        f1_per_class = f1_score(labels, predictions, labels=unique_labels, average=None, zero_division=0)
-        
-        # Create confusion matrix
-        fig, acc, cm = self.create_confusion_matrix(labels, predictions, 
-                                                   f"{embedding_type} - Hierarchy Classification")
-        
-        # Generate classification report
-        report = classification_report(labels, predictions, labels=unique_labels, 
-                                     target_names=unique_labels, output_dict=True)
-        
-        return {
-            'accuracy': accuracy,
-            'f1_macro': f1_macro,
-            'f1_weighted': f1_weighted,
-            'f1_per_class': f1_per_class,
-            'predictions': predictions,
-            'confusion_matrix': cm,
-            'classification_report': report,
-            'figure': fig
-        }
-
-    def evaluate_dataset_with_baselines(self, dataframe, dataset_name="Dataset"):
-        """Evaluate embeddings on a given dataset with both custom model and CLIP baseline"""
-        print(f"\n{'='*60}")
-        print(f"Evaluating {dataset_name}")
-        print(f"{'='*60}")
-        
-        results = {}
-        
-        # ===== CUSTOM MODEL EVALUATION =====
-        print(f"\n🔧 Evaluating Custom Model on {dataset_name}")
-        print("-" * 40)
-        
-        # Create dataloader for custom model
-        custom_dataloader = self.create_dataloader(dataframe, batch_size=16)
-        
-        # Evaluate text embeddings
-        text_embeddings, text_labels, texts = self.extract_custom_embeddings(custom_dataloader, 'text')
-        text_metrics = self.compute_similarity_metrics(text_embeddings, text_labels)
-        text_classification = self.evaluate_classification_performance(text_embeddings, text_labels, "Custom Text Embeddings")
-        text_metrics.update(text_classification)
-        results['custom_text'] = text_metrics
-        
-        # Evaluate image embeddings
-        image_embeddings, image_labels, _ = self.extract_custom_embeddings(custom_dataloader, 'image')
-        image_metrics = self.compute_similarity_metrics(image_embeddings, image_labels)
-        image_classification = self.evaluate_classification_performance(image_embeddings, image_labels, "Custom Image Embeddings")
-        image_metrics.update(image_classification)
-        results['custom_image'] = image_metrics
-        
-        # ===== CLIP BASELINE EVALUATION =====
-        print(f"\n🤗 Evaluating CLIP Baseline on {dataset_name}")
-        print("-" * 40)
-        
-        # Create dataloader for CLIP
-        clip_dataloader = self.create_clip_dataloader(dataframe, batch_size=8)  # Smaller batch for CLIP
-        
-        # Extract data for CLIP
-        all_images = []
-        all_texts = []
-        all_labels = []
-        
-        for batch in tqdm(clip_dataloader, desc="Preparing data for CLIP"):
-            images, texts, labels = batch
-            all_images.extend(images)
-            all_texts.extend(texts)
-            all_labels.extend(labels)
-        
-        # Get CLIP embeddings
-        clip_image_embeddings, clip_text_embeddings = self.clip_evaluator.extract_clip_embeddings(all_images, all_texts)
-        
-        # Evaluate CLIP text embeddings
-        clip_text_metrics = self.compute_similarity_metrics(clip_text_embeddings, all_labels)
-        clip_text_classification = self.evaluate_classification_performance(clip_text_embeddings, all_labels, "CLIP Text Embeddings")
-        clip_text_metrics.update(clip_text_classification)
-        results['clip_text'] = clip_text_metrics
-        
-        # Evaluate CLIP image embeddings
-        clip_image_metrics = self.compute_similarity_metrics(clip_image_embeddings, all_labels)
-        clip_image_classification = self.evaluate_classification_performance(clip_image_embeddings, all_labels, "CLIP Image Embeddings")
-        clip_image_metrics.update(clip_image_classification)
-        results['clip_image'] = clip_image_metrics
-        
-        # ===== PRINT COMPARISON RESULTS =====
-        print(f"\n{dataset_name} Results Comparison:")
-        print(f"Dataset size: {len(dataframe)} samples")
-        print("=" * 80)
-        print(f"{'Model':<20} {'Embedding':<10} {'Sep Score':<10} {'NN Acc':<8} {'Centroid Acc':<12} {'F1 Macro':<10}")
-        print("-" * 80)
-        
-        for model_type in ['custom', 'clip']:
-            for emb_type in ['text', 'image']:
-                key = f"{model_type}_{emb_type}"
-                if key in results:
-                    metrics = results[key]
-                    model_name = "Custom Model" if model_type == 'custom' else "CLIP Baseline"
-                    print(f"{model_name:<20} {emb_type.capitalize():<10} {metrics['separation_score']:<10.4f} {metrics['accuracy']*100:<8.1f}% {metrics['centroid_accuracy']*100:<12.1f}% {metrics['f1_macro']*100:<10.1f}%")
-        
-        # ===== SAVE VISUALIZATIONS =====
-        os.makedirs(f'{self.directory}', exist_ok=True)
-        
-        # Save confusion matrices
-        for key, metrics in results.items():
-            if 'figure' in metrics:
-                metrics['figure'].savefig(f'{self.directory}/{dataset_name.lower()}_{key}_confusion_matrix.png', dpi=300, bbox_inches='tight')
-                plt.close(metrics['figure'])
-        
-        return results
-
-
-class CLIPDataset(Dataset):
-    def __init__(self, dataframe):
-        self.dataframe = dataframe
-        # Use VALIDATION transforms (no augmentation)
-        self.transform = transforms.Compose([
-            transforms.Resize((224, 224)),
-            transforms.ToTensor(),
-            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
-        ])
-
-    def __len__(self):
-        return len(self.dataframe)
-
-    def __getitem__(self, idx):
-        row = self.dataframe.iloc[idx]
-        
-        # Handle image loading (same as HierarchyDataset)
-        if config.column_local_image_path in row.index and pd.notna(row[config.column_local_image_path]):
-            local_path = row[config.column_local_image_path]
-            try:
-                if os.path.exists(local_path):
-                    image = Image.open(local_path).convert("RGB")
-                else:
-                    print(f"⚠️ Local image not found: {local_path}")
-                    image = Image.new('RGB', (224, 224), color='gray')
-            except Exception as e:
-                print(f"⚠️ Failed to load local image {idx}: {e}")
-                image = Image.new('RGB', (224, 224), color='gray')
-        elif isinstance(row[config.column_url_image], dict):
-            image = Image.open(BytesIO(row[config.column_url_image]['bytes'])).convert('RGB')
-        elif isinstance(row['image_url'], (list, np.ndarray)):
-            pixels = np.array(row[config.column_url_image]).reshape(28, 28)
-            image = Image.fromarray(pixels.astype(np.uint8)).convert("RGB")
-        elif isinstance(row[config.column_url_image], Image.Image):
-            # Handle PIL Image objects directly (for Fashion-MNIST)
-            image = row[config.column_url_image].convert("RGB")
-        else:
-            try:
-                response = requests.get(row[config.column_url_image], timeout=10)
-                response.raise_for_status()
-                image = Image.open(BytesIO(response.content)).convert("RGB")
-            except Exception as e:
-                print(f"⚠️ Failed to load image {idx}: {e}")
-                image = Image.new('RGB', (224, 224), color='gray')
-        
-        # Apply transforms
-        image_tensor = self.transform(image)
-
-        description = row[config.text_column]  
-        hierarchy = row[config.hierarchy_column]
-
-        return image_tensor, description, hierarchy
-
-
-def load_fashion_mnist_dataset(evaluator):
-    """Load and prepare Fashion-MNIST test dataset"""
-    print("Loading Fashion-MNIST test dataset...")
-    
-    # Load the dataset
-    df = pd.read_csv(config.fashion_mnist_test_path)
-    print(f"✅ Fashion-MNIST dataset loaded")
-    print(f"📊 Total samples: {len(df)}")
-    
-    # Fashion-MNIST class labels mapping
-    fashion_mnist_labels = get_fashion_mnist_labels()
-    
-    # Map labels to hierarchy classes
-    hierarchy_mapping = {
-        'T-shirt/top': 'top',
-        'Trouser': 'bottom', 
-        'Pullover': 'top',
-        'Dress': 'dress',
-        'Coat': 'top',
-        'Sandal': 'shoes',
-        'Shirt': 'top',
-        'Sneaker': 'shoes',
-        'Bag': 'bag',
-        'Ankle boot': 'shoes'
-    }
-    
-    # Apply label mapping
-    df['hierarchy'] = df['label'].map(fashion_mnist_labels).map(hierarchy_mapping)
-    
-    # Filter to only include hierarchies that exist in our model
-    valid_hierarchies = df['hierarchy'].dropna().unique()
-    print(f"🎯 Valid hierarchies found: {sorted(valid_hierarchies)}")
-    print(f"🎯 Model hierarchies: {sorted(evaluator.hierarchy_classes)}")
-    
-    # Filter to only include hierarchies that exist in our model
-    df = df[df['hierarchy'].isin(evaluator.hierarchy_classes)]
-    print(f"📊 After filtering to model hierarchies: {len(df)} samples")
-    
-    if len(df) == 0:
-        print("❌ No samples left after hierarchy filtering.")
-        return pd.DataFrame()
-    
-    # Keep pixel columns as they are (FashionMNISTDataset will handle them)
-    
-    # Create text descriptions based on hierarchy
-    text_descriptions = {
-        'top': 'A top clothing item',
-        'bottom': 'A bottom clothing item', 
-        'dress': 'A dress',
-        'shoes': 'A pair of shoes',
-        'bag': 'A bag'
-    }
-    
-    df['text'] = df['hierarchy'].map(text_descriptions)
-    
-    # Show sample of data
-    print(f"📝 Sample data:")
-    for i, (hierarchy, text) in enumerate(zip(df['hierarchy'].head(3), df['text'].head(3))):
-        print(f"  {i+1}. [{hierarchy}] {text}")
-    
-    df_test = df.copy()
-    
-    print(f"📊 After sampling: {len(df_test)} samples")
-    print(f"📊 Samples per hierarchy:")
-    for hierarchy in sorted(df_test['hierarchy'].unique()):
-        count = len(df_test[df_test['hierarchy'] == hierarchy])
-        print(f"  {hierarchy}: {count} samples")
-    
-    # Create formatted dataset with proper column names
-    # Keep all pixel columns for FashionMNISTDataset
-    pixel_cols = [f'pixel{i}' for i in range(1, 785)]
-    fashion_mnist_formatted = df_test[['label'] + pixel_cols + ['text', 'hierarchy']].copy()
-    
-    print(f"📊 Final dataset size: {len(fashion_mnist_formatted)} samples")
-    return fashion_mnist_formatted
-
-
-def load_kagl_marqo_dataset(evaluator):
-    """Load and prepare kagl dataset"""
-    from datasets import load_dataset
-    print("Loading kagl dataset...")
-
-    # Load the dataset
-    dataset = load_dataset("Marqo/KAGL")
-    df = dataset["data"].to_pandas()
-    print(f"✅ Dataset kagl loaded")
-    print(f"📊 Before filtering: {len(df)} samples")
-    print(f"📋 Available columns: {list(df.columns)}")
-
-    # Check available categories and map them to our hierarchy
-    print(f"🎨 Available categories: {sorted(df['category2'].unique())}")
-    # Apply mapping
-    df['hierarchy'] = df['category2'].str.lower()
-    df['hierarchy'] = df['hierarchy'].replace('bags', 'bag').replace('topwear', 'top').replace('flip flops', 'shoes').replace('sandal', 'shoes')
-
-    # Filter to only include valid hierarchies that exist in our model
-    valid_hierarchies = df['hierarchy'].dropna().unique()
-    print(f"🎯 Valid hierarchies found: {sorted(valid_hierarchies)}")
-    print(f"🎯 Model hierarchies: {sorted(evaluator.hierarchy_classes)}")
-    
-    # Filter to only include hierarchies that exist in our model
-    df = df[df['hierarchy'].isin(evaluator.hierarchy_classes)]
-    print(f"📊 After filtering to model hierarchies: {len(df)} samples")
-    
-    if len(df) == 0:
-        print("❌ No samples left after hierarchy filtering.")
-        return pd.DataFrame()
-    
-    # Ensure we have text and image data
-    df = df.dropna(subset=['text', 'image'])
-    print(f"📊 After removing missing text/image: {len(df)} samples")
-    
-    # Show sample of text data to verify quality
-    print(f"📝 Sample texts:")
-    for i, (text, hierarchy) in enumerate(zip(df['text'].head(3), df['hierarchy'].head(3))):
-        print(f"  {i+1}. [{hierarchy}] {text[:100]}...")
-    
-    print(f"📊 After sampling: {len(df_test)} samples")
-    print(f"📊 Samples per hierarchy:")
-    for hierarchy in sorted(df_test['hierarchy'].unique()):
-        count = len(df_test[df_test['hierarchy'] == hierarchy])
-        print(f"  {hierarchy}: {count} samples")
-    
-    # Create formatted dataset with proper column names
-    kagl_formatted = pd.DataFrame({
-        'image_url': df_test['image'],
-        'text': df_test['text'],
-        'hierarchy': df_test['hierarchy']
-    })
-    
-    print(f"📊 Final dataset size: {len(kagl_formatted)} samples")
-    return kagl_formatted
-
-
-if __name__ == "__main__":
-    device = config.device
-    directory = config.evaluation_directory
-
-    print(f"🚀 Starting evaluation with custom model: {config.hierarchy_model_path}")
-    print(f"🤗 Including CLIP baseline comparison")
-    
-    evaluator = EmbeddingEvaluator(config.hierarchy_model_path, directory, device=device)
-    
-    print(f"📊 Final hierarchy classes after initialization: {len(evaluator.vocab.hierarchy_classes)} classes")
-    
-    # Evaluate on validation dataset (same subset as during training)
-    print("\n" + "="*60)
-    print("EVALUATING VALIDATION DATASET - CUSTOM MODEL vs CLIP BASELINE")
-    print("="*60)
-    val_results = evaluator.evaluate_dataset_with_baselines(evaluator.val_df, "Validation Dataset")
-    
-    print("\n" + "="*60)
-    print("EVALUATING FASHION-MNIST TEST DATASET - CUSTOM MODEL vs CLIP BASELINE") 
-    print("="*60)
-    df_fashion_mnist = load_fashion_mnist_dataset(evaluator)
-    if len(df_fashion_mnist) > 0:
-        fashion_mnist_results = evaluator.evaluate_dataset_with_baselines(df_fashion_mnist, "Fashion-MNIST Test Dataset")
-    else:
-        fashion_mnist_results = {}
-    
-    print("\n" + "="*60)
-    print("EVALUATING kagl MARQO DATASET - CUSTOM MODEL vs CLIP BASELINE") 
-    print("="*60)
-    df_kagl_marqo = load_kagl_marqo_dataset(evaluator)
-    if len(df_kagl_marqo) > 0:
-        kagl_results = evaluator.evaluate_dataset_with_baselines(df_kagl_marqo, "kagl Marqo Dataset")
-    else:
-        kagl_results = {}
-    
-    # Compare results
-    print(f"\n{'='*80}")
-    print("FINAL EVALUATION SUMMARY - CUSTOM MODEL vs CLIP BASELINE")
-    print(f"{'='*80}")
-    
-    print("\n🔍 VALIDATION DATASET RESULTS:")
-    print(f"Dataset size: {len(evaluator.val_df)} samples")
-    print(f"{'Model':<20} {'Embedding':<10} {'Sep Score':<12} {'NN Acc':<10} {'Centroid Acc':<12} {'F1 Macro':<10}")
-    print("-" * 80)
-    
-    for model_type in ['custom', 'clip']:
-        for emb_type in ['text', 'image']:
-            key = f"{model_type}_{emb_type}"
-            if key in val_results:
-                metrics = val_results[key]
-                model_name = "Custom Model" if model_type == 'custom' else "CLIP Baseline"
-                print(f"{model_name:<20} {emb_type.capitalize():<10} {metrics['separation_score']:<12.4f} {metrics['accuracy']*100:<10.1f}% {metrics['centroid_accuracy']*100:<12.1f}% {metrics['f1_macro']*100:<10.1f}%")
-
-    if fashion_mnist_results:
-        print("\n👗 FASHION-MNIST TEST DATASET RESULTS:")
-        print(f"Dataset size: {len(df_fashion_mnist)} samples")
-        print(f"{'Model':<20} {'Embedding':<10} {'Sep Score':<12} {'NN Acc':<10} {'Centroid Acc':<12} {'F1 Macro':<10}")
-        print("-" * 80)
-        
-        for model_type in ['custom', 'clip']:
-            for emb_type in ['text', 'image']:
-                key = f"{model_type}_{emb_type}"
-                if key in fashion_mnist_results:
-                    metrics = fashion_mnist_results[key]
-                    model_name = "Custom Model" if model_type == 'custom' else "CLIP Baseline"
-                    print(f"{model_name:<20} {emb_type.capitalize():<10} {metrics['separation_score']:<12.4f} {metrics['accuracy']*100:<10.1f}% {metrics['centroid_accuracy']*100:<12.1f}% {metrics['f1_macro']*100:<10.1f}%")
-    
-    if kagl_results:
-        print("\n🌐 kagl MARQO DATASET RESULTS:")
-        print(f"Dataset size: {len(df_kagl_marqo)} samples")
-        print(f"{'Model':<20} {'Embedding':<10} {'Sep Score':<12} {'NN Acc':<10} {'Centroid Acc':<12} {'F1 Macro':<10}")
-        print("-" * 80)
-        
-        for model_type in ['custom', 'clip']:
-            for emb_type in ['text', 'image']:
-                key = f"{model_type}_{emb_type}"
-                if key in kagl_results:
-                    metrics = kagl_results[key]
-                    model_name = "Custom Model" if model_type == 'custom' else "CLIP Baseline"
-                    print(f"{model_name:<20} {emb_type.capitalize():<10} {metrics['separation_score']:<12.4f} {metrics['accuracy']*100:<10.1f}% {metrics['centroid_accuracy']*100:<12.1f}% {metrics['f1_macro']*100:<10.1f}%")
-    
-    print(f"\n✅ Evaluation completed! Check '{directory}/' for visualization files.")
-    print(f"📊 Custom model hierarchy classes: {len(evaluator.vocab.hierarchy_classes)} classes")
-    print(f"🤗 CLIP baseline comparison included")

Evaluation/tsne_images.py

@@ -1,569 +0,0 @@
-#!/usr/bin/env python3
-"""
-Outputs several t-SNE visualizations with color and hierarchy overlays to
-verify that the main model separates colors well inside each hierarchy group.
-"""
-
-import math
-
-import matplotlib.pyplot as plt
-import numpy as np
-import pandas as pd
-import seaborn as sns
-import torch
-from matplotlib.patches import Polygon
-from PIL import Image
-from sklearn.manifold import TSNE
-from sklearn.metrics import (
-    silhouette_score,
-    davies_bouldin_score,
-    calinski_harabasz_score,
-)
-from sklearn.preprocessing import normalize
-from sklearn.metrics.pairwise import cosine_similarity
-from torch.utils.data import DataLoader, Dataset
-from torchvision import transforms
-from tqdm import tqdm
-from transformers import CLIPModel as CLIPModel_transformers, CLIPProcessor
-
-try:
-    from scipy.spatial import ConvexHull
-except ImportError:
-    ConvexHull = None
-
-from config import (
-    color_column,
-    color_emb_dim,
-    column_local_image_path,
-    device,
-    hierarchy_column,
-    hierarchy_emb_dim,
-    images_dir,
-    local_dataset_path,
-    main_model_path,
-)
-
-
-class ImageDataset(Dataset):
-    """Lightweight dataset to load local images along with colors and hierarchies."""
-
-    def __init__(self, dataframe: pd.DataFrame, root_dir: str):
-        self.df = dataframe.reset_index(drop=True)
-        self.root_dir = root_dir
-        self.transform = transforms.Compose(
-            [
-                transforms.Resize((224, 224)),
-                transforms.ToTensor(),
-                transforms.Normalize(
-                    mean=[0.485, 0.456, 0.406],
-                    std=[0.229, 0.224, 0.225],
-                ),
-            ]
-        )
-
-    def __len__(self):
-        return len(self.df)
-
-    def __getitem__(self, idx):
-        row = self.df.iloc[idx]
-        img_path = row[column_local_image_path]
-        image = Image.open(img_path).convert("RGB")
-        image = self.transform(image)
-        color = row[color_column]
-        hierarchy = row[hierarchy_column]
-        return image, color, hierarchy
-
-
-
-def load_main_model():
-    """Load the main model with the trained weights."""
-    checkpoint = torch.load(main_model_path, map_location=device)
-    state_dict = checkpoint.get("model_state_dict", checkpoint)
-    model = CLIPModel_transformers.from_pretrained(
-        "laion/CLIP-ViT-B-32-laion2B-s34B-b79K"
-    )
-    model.load_state_dict(state_dict)
-    model.to(device)
-    model.eval()
-    # Load processor for text tokenization
-    processor = CLIPProcessor.from_pretrained("laion/CLIP-ViT-B-32-laion2B-s34B-b79K")
-    return model, processor
-
-
-def load_clip_baseline():
-    """Load the CLIP baseline model from transformers."""
-    print("🤗 Loading CLIP baseline model from transformers...")
-    clip_model = CLIPModel_transformers.from_pretrained("openai/clip-vit-base-patch32").to(device)
-    clip_processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
-    clip_model.eval()
-    print("✅ CLIP baseline model loaded successfully")
-    return clip_model, clip_processor
-
-
-def enforce_min_hierarchy_samples(df, min_per_hierarchy):
-    """Filter out hierarchy groups with fewer than min_per_hierarchy rows."""
-    if not min_per_hierarchy or min_per_hierarchy <= 0:
-        return df
-    counts = df[hierarchy_column].value_counts()
-    keep_values = counts[counts >= min_per_hierarchy].index
-    filtered = df[df[hierarchy_column].isin(keep_values)].reset_index(drop=True)
-    return filtered
-
-
-def prepare_dataframe(df, sample_size, per_color_limit, min_per_hierarchy=None):
-    """Subsample the dataframe to speed up the t-SNE."""
-    if per_color_limit and per_color_limit > 0:
-        df_limited = (
-            df.groupby(color_column)
-            .apply(lambda g: g.sample(min(len(g), per_color_limit), random_state=42))
-            .reset_index(drop=True)
-        )
-    else:
-        df_limited = df
-
-    if sample_size and 0 < sample_size < len(df_limited):
-        df_limited = df_limited.sample(sample_size, random_state=42).reset_index(
-            drop=True
-        )
-    df_limited = enforce_min_hierarchy_samples(df_limited, min_per_hierarchy)
-    return df_limited
-
-
-def compute_embeddings(model, dataloader):
-    """Extract color, hierarchy, and combined embeddings."""
-    color_embeddings = []
-    hierarchy_embeddings = []
-    color_labels = []
-    hierarchy_labels = []
-    with torch.no_grad():
-        for images, colors, hierarchies in tqdm(
-            dataloader, desc="Extracting embeddings"
-        ):
-            images = images.to(device)
-            if images.shape[1] == 1:  # safety in case
-                images = images.expand(-1, 3, -1, -1)
-            image_embeds = model.get_image_features(pixel_values=images)
-            color_part = image_embeds[:, :color_emb_dim]
-            hierarchy_part = image_embeds[
-                :, color_emb_dim : color_emb_dim + hierarchy_emb_dim
-            ]
-            color_embeddings.append(color_part.cpu().numpy())
-            hierarchy_embeddings.append(hierarchy_part.cpu().numpy())
-            color_labels.extend(colors)
-            hierarchy_labels.extend(hierarchies)
-    return (
-        np.concatenate(color_embeddings, axis=0),
-        np.concatenate(hierarchy_embeddings, axis=0),
-        color_labels,
-        hierarchy_labels,
-    )
-
-
-def compute_clip_embeddings(clip_model, clip_processor, dataloader):
-    """Extract CLIP baseline embeddings (full image embeddings, not separated)."""
-    all_embeddings = []
-    color_labels = []
-    hierarchy_labels = []
-    
-    with torch.no_grad():
-        for images, colors, hierarchies in tqdm(
-            dataloader, desc="Extracting CLIP embeddings"
-        ):
-            batch_embeddings = []
-            for i in range(images.shape[0]):
-                # Get single image from batch
-                image_tensor = images[i]  # Shape: (3, 224, 224)
-                
-                # Denormalize on CPU (safer for PIL conversion)
-                mean = torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1)
-                std = torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1)
-                image_tensor = image_tensor * std + mean
-                image_tensor = torch.clamp(image_tensor, 0, 1)
-                
-                # Convert to PIL Image (must be on CPU)
-                image_pil = transforms.ToPILImage()(image_tensor.cpu())
-                
-                # Process with CLIP (using empty text since we only need image embeddings)
-                inputs = clip_processor(
-                    text="", 
-                    images=image_pil, 
-                    return_tensors="pt", 
-                    padding=True
-                ).to(device)
-                
-                outputs = clip_model(**inputs)
-                # Get normalized image embeddings
-                image_emb = outputs.image_embeds / outputs.image_embeds.norm(p=2, dim=-1, keepdim=True)
-                batch_embeddings.append(image_emb.cpu().numpy())
-            
-            all_embeddings.append(np.vstack(batch_embeddings))
-            color_labels.extend(colors)
-            hierarchy_labels.extend(hierarchies)
-    
-    # For CLIP, we use the full embeddings for all visualizations
-    # (no separation into color/hierarchy dimensions)
-    full_embeddings = np.concatenate(all_embeddings, axis=0)
-    return (
-        full_embeddings,  # color_embeddings (using full CLIP embeddings)
-        full_embeddings,  # hierarchy_embeddings (using full CLIP embeddings)
-        full_embeddings,  # color_hier_embeddings (using full CLIP embeddings)
-        color_labels,
-        hierarchy_labels,
-    )
-
-
-def compute_dunn_index(embeddings, labels):
-    """
-    Compute the Dunn Index for clustering evaluation.
-    
-    The Dunn Index is the ratio of the minimum inter-cluster distance
-    to the maximum intra-cluster distance. Higher values indicate better clustering.
-    
-    Args:
-        embeddings: Array of embeddings [N, embed_dim]
-        labels: Array of cluster labels [N]
-        
-    Returns:
-        Dunn Index value (float) or None if calculation fails
-    """
-    try:
-        unique_labels = np.unique(labels)
-        if len(unique_labels) < 2:
-            return None
-        
-        # Calculate intra-cluster distances (maximum within each cluster)
-        max_intra_cluster_dist = 0
-        for label in unique_labels:
-            cluster_points = embeddings[labels == label]
-            if len(cluster_points) > 1:
-                # Calculate pairwise distances within cluster
-                from scipy.spatial.distance import pdist
-                intra_dists = pdist(cluster_points, metric='euclidean')
-                if len(intra_dists) > 0:
-                    max_intra = np.max(intra_dists)
-                    max_intra_cluster_dist = max(max_intra_cluster_dist, max_intra)
-        
-        if max_intra_cluster_dist == 0:
-            return None
-        
-        # Calculate inter-cluster distances (minimum between clusters)
-        min_inter_cluster_dist = float('inf')
-        for i, label1 in enumerate(unique_labels):
-            for label2 in unique_labels[i+1:]:
-                cluster1_points = embeddings[labels == label1]
-                cluster2_points = embeddings[labels == label2]
-                
-                # Calculate distances between clusters
-                from scipy.spatial.distance import cdist
-                inter_dists = cdist(cluster1_points, cluster2_points, metric='euclidean')
-                min_inter = np.min(inter_dists)
-                min_inter_cluster_dist = min(min_inter_cluster_dist, min_inter)
-        
-        if min_inter_cluster_dist == float('inf'):
-            return None
-        
-        # Dunn Index = minimum inter-cluster distance / maximum intra-cluster distance
-        dunn_index = min_inter_cluster_dist / max_intra_cluster_dist
-        return float(dunn_index)
-    except Exception as e:
-        print(f"⚠️ Error computing Dunn Index: {e}")
-        return None
-
-
-def build_color_map(labels, prefer_true_colors=False):
-    """Build a color mapping for labels."""
-    unique_labels = sorted(set(labels))
-    palette = sns.color_palette("husl", len(unique_labels))
-    return {label: palette[idx] for idx, label in enumerate(unique_labels)}
-
-
-def compute_color_similarity_matrix(embeddings, colors, title="Color similarity (image embeddings)"):
-    """Compute and visualize similarity matrix between color centroids."""
-    # Use only the colors from the reference heatmap
-    reference_colors = ['red', 'pink', 'blue', 'green', 'aqua', 'lime', 'yellow', 'orange', 
-                        'purple', 'brown', 'gray', 'black', 'white']
-    # Map 'yelloworange' to 'yellow' or 'orange' if needed
-    color_mapping = {
-        'yelloworange': 'yellow',
-        'grey': 'gray'  # Handle grey/gray variation
-    }
-    
-    # Filter to only include colors that are in the reference list
-    filtered_colors = []
-    filtered_embeddings = []
-    for i, color in enumerate(colors):
-        # Normalize color name
-        normalized_color = color_mapping.get(color.lower(), color.lower())
-        if normalized_color in reference_colors:
-            filtered_colors.append(normalized_color)
-            filtered_embeddings.append(embeddings[i])
-    
-    if len(filtered_colors) == 0:
-        print("⚠️ No matching colors found in reference list")
-        return None
-    
-    # Use only unique colors from reference that exist in data
-    unique_colors = sorted([c for c in reference_colors if c in filtered_colors])
-    
-    # Convert to numpy arrays
-    filtered_embeddings = np.array(filtered_embeddings)
-    filtered_colors = np.array(filtered_colors)
-    
-    # Compute centroids for each color
-    centroids = {}
-    for color in unique_colors:
-        color_mask = np.array([c == color for c in filtered_colors])
-        if color_mask.sum() > 0:
-            centroids[color] = np.mean(filtered_embeddings[color_mask], axis=0)
-    
-    # Compute similarity matrix
-    similarity_matrix = np.zeros((len(unique_colors), len(unique_colors)))
-    for i, color1 in enumerate(unique_colors):
-        for j, color2 in enumerate(unique_colors):
-            if i == j:
-                similarity_matrix[i, j] = 1.0
-            else:
-                if color1 in centroids and color2 in centroids:
-                    similarity = cosine_similarity(
-                        [centroids[color1]], 
-                        [centroids[color2]]
-                    )[0][0]
-                    similarity_matrix[i, j] = similarity
-    
-    # Create heatmap
-    plt.figure(figsize=(12, 10))
-    sns.heatmap(
-        similarity_matrix,
-        annot=True,
-        fmt='.2f',
-        cmap='RdYlBu_r',
-        xticklabels=unique_colors,
-        yticklabels=unique_colors,
-        square=True,
-        cbar_kws={'label': 'Cosine Similarity'},
-        linewidths=0.5,
-        vmin=-0.6,
-        vmax=1.0
-    )
-    
-    plt.title(title, fontsize=16, fontweight='bold', pad=20)
-    plt.xlabel('Colors', fontsize=14, fontweight='bold')
-    plt.ylabel('Colors', fontsize=14, fontweight='bold')
-    plt.xticks(rotation=45, ha='right')
-    plt.yticks(rotation=0)
-    plt.tight_layout()
-    
-    output_path = "color_similarity_image_embeddings.png"
-    plt.savefig(output_path, dpi=300, bbox_inches='tight')
-    plt.close()
-    print(f"✅ Color similarity heatmap saved: {output_path}")
-    
-    return similarity_matrix
-
-
-def run_tsne(embeddings,legend_labels,output_path,perplexity,title,scatter_color_labels=None,prefer_true_colors=False):
-    """Calculate and plot a t-SNE projection."""
-    tsne = TSNE(
-        n_components=2,
-        perplexity=perplexity,
-        init="pca",
-        learning_rate="auto",
-        random_state=42,
-    )
-    reduced = tsne.fit_transform(embeddings)
-
-    label_array = np.array(legend_labels)
-    color_labels = (
-        np.array(scatter_color_labels) if scatter_color_labels is not None else label_array
-    )
-
-    # Calculate silhouette scores
-    unique_labels_list = sorted(set(label_array))
-    if len(unique_labels_list) > 1 and len(label_array) > 1:
-        # Convert labels to numeric indices for silhouette_score
-        label_to_idx = {label: idx for idx, label in enumerate(unique_labels_list)}
-        numeric_labels = np.array([label_to_idx[label] for label in label_array])
-        
-        # Calculate in original embedding space (ground truth - measures real separation)
-        silhouette = silhouette_score(embeddings, numeric_labels, metric='euclidean')
-        davies_bouldin = davies_bouldin_score(embeddings, numeric_labels)
-        calinski_harabasz = calinski_harabasz_score(embeddings, numeric_labels)
-        dunn = compute_dunn_index(embeddings, numeric_labels)
-
-    else:
-        silhouette = None
-        davies_bouldin = None
-        calinski_harabasz = None
-        dunn = None
-
-    # Helpful reference for the reported clustering indices:
-    #   • Silhouette Score ∈ [-1, 1] — closer to 1 means points fit their cluster well, 0 means overlap, < 0 suggests misassignment.
-    #   • Davies–Bouldin Index ∈ [0, +∞) — lower is better; quantifies average similarity between clusters relative to their size.
-    #   • Calinski–Harabasz Index ∈ [0, +∞) — higher is better; ratio of between-cluster dispersion to within-cluster dispersion.
-    #   • Dunn Index ∈ [0, +∞) — higher is better; compares the tightest cluster diameter to the closest distance between clusters.
-
-    # Build color map for visualization
-    color_map = build_color_map(color_labels, prefer_true_colors=prefer_true_colors)
-    color_series = np.array([color_map[label] for label in color_labels])
-
-    plt.figure(figsize=(10, 8))
-    unique_labels = sorted(set(label_array))
-    for label in unique_labels:
-        mask = label_array == label
-        if 'color' in title:
-            c = label
-        else:
-            c = color_series[mask]
-        plt.scatter(
-            reduced[mask, 0],
-            reduced[mask, 1],
-            c=c,
-            s=15,
-            alpha=0.8,
-            label=label,
-        )
-
-    # Add silhouette score to title
-    if silhouette is not None:
-        title_with_score = f"{title}\n(t-SNE Silhouette: {silhouette:.3f} | Davies-Bouldin: {davies_bouldin:.3f} | Calinski-Harabasz: {calinski_harabasz:.3f} | Dunn: {dunn:.3f})"
-    else:
-        title_with_score = title
-    
-    plt.title(title_with_score)
-    plt.xlabel("t-SNE 1")
-    plt.ylabel("t-SNE 2")
-    plt.legend(
-        bbox_to_anchor=(1.05, 1), loc="upper left", fontsize="small", frameon=False
-    )
-    plt.tight_layout()
-    plt.savefig(output_path, dpi=300)
-    plt.close()
-    print(f"✅ Figure saved in {output_path}")
-    print(f"   📊 t-SNE space: {silhouette:.3f} (matches visualization) | Davies-Bouldin: {davies_bouldin:.3f} | Calinski-Harabasz: {calinski_harabasz:.3f} | Dunn: {dunn:.3f}")
-    
-
-
-def filter_valid_rows(dataframe: pd.DataFrame) -> pd.DataFrame:
-    """Keep only rows with valid local image paths and colors."""
-    dataframe = dataframe[dataframe['color'] != 'unknown'].copy()
-    df = dataframe.dropna(
-        subset=[column_local_image_path, color_column, hierarchy_column]
-    ).copy()
-    mask = df[column_local_image_path].apply(lambda x: isinstance(x, str) and len(x.strip()) > 0)
-    return df[mask].reset_index(drop=True)
-
-if __name__ == "__main__":
-    sample_size = None
-    per_color_limit = 500
-    min_per_hierarchy = 200
-    batch_size = 32
-    perplexity = 30
-    output_color = "tsne_color_space.png"
-    output_hierarchy = "tsne_hierarchy_space.png"
-
-    print("📥 Loading the dataset...")
-    df = pd.read_csv("data/data_with_local_paths.csv")
-    df = filter_valid_rows(df)
-    print(f"Total len if the dataset: {len(df)}")
-    df = prepare_dataframe(df, sample_size, per_color_limit, min_per_hierarchy)
-    print(f"✅ {len(df)} samples will be used for the t-SNE")
-    print(f"Number of colors in the dataset: {len(df['color'].unique())}")
-    print(f"Colors in the dataset: {df['color'].unique()}")
-    dataset = ImageDataset(df, images_dir)
-    dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=4)
-
-    # 2) Loading the models
-    print("⚙️ Loading the main model...")
-    model, processor = load_main_model()
-
-    print("⚙️ Loading CLIP baseline model...")
-    clip_model, clip_processor = load_clip_baseline()
-
-    # 3) Extracting the embeddings
-    print("🎯 Extracting the embeddings...")
-    
-    (
-        color_embeddings,
-        hierarchy_embeddings,
-        colors,
-        hierarchies,
-    ) = compute_embeddings(model, dataloader)
-    
-    # 4) Calculating the t-SNE
-    print("🌀 Calculating the color t-SNE...")
-    run_tsne(
-        color_embeddings,
-        colors,
-        output_color,
-        perplexity,
-        "t-SNE of the color embeddings of the main model",
-        scatter_color_labels=colors,
-        prefer_true_colors=True,
-    )
-    
-    print("🎨 Computing color similarity matrix from image embeddings...")
-    compute_color_similarity_matrix(
-        color_embeddings,
-        colors,
-        title="Color similarity (image embeddings - main model)"
-    )
-
-    print("🌀 Calculating the hierarchy t-SNE...")
-    run_tsne(
-        hierarchy_embeddings,
-        hierarchies,
-        output_hierarchy,
-        perplexity,
-        "t-SNE of the hierarchy embeddings of the main model",
-        scatter_color_labels=hierarchies,
-    )
-
-    # ========== CLIP BASELINE EVALUATION ==========
-    print("\n" + "="*60)
-    print("🔄 Starting CLIP Baseline Evaluation")
-    print("="*60)
-
-    print("🎯 Extracting CLIP embeddings...")
-    (
-        clip_color_embeddings,
-        clip_hierarchy_embeddings,
-        clip_color_hier_embeddings,
-        clip_colors,
-        clip_hierarchies,
-    ) = compute_clip_embeddings(clip_model, clip_processor, dataloader)
-
-    # Output paths for CLIP baseline
-    clip_output_color = "clip_baseline_tsne_color_space.png"
-    clip_output_hierarchy = "clip_baseline_tsne_hierarchy_space.png"
-
-    print("🌀 Calculating CLIP baseline color t-SNE...")
-    run_tsne(
-        clip_color_embeddings,
-        clip_colors,
-        clip_output_color,
-        perplexity,
-        "t-SNE of the color embeddings (CLIP Baseline)",
-        scatter_color_labels=clip_colors,
-        prefer_true_colors=True,
-    )
-
-    print("🎨 Computing color similarity matrix from image embeddings...")
-    compute_color_similarity_matrix(
-        clip_color_embeddings,
-        clip_colors,
-        title="Color similarity (image embeddings - CLIP Baseline)"
-    )
-
-    print("🌀 Calculating CLIP baseline hierarchy t-SNE...")
-    run_tsne(
-        clip_hierarchy_embeddings,
-        clip_hierarchies,
-        clip_output_hierarchy,
-        perplexity,
-        "t-SNE of the hierarchy embeddings (CLIP Baseline)",
-        scatter_color_labels=clip_hierarchies,
-    )
-    
-    print("\n✅ All t-SNE visualizations completed!")
-    print("   - Main model: tsne_*.png")
-    print("   - CLIP baseline: clip_baseline_tsne_*.png")