Upload evaluation/heatmap_color_similarities.py with huggingface_hub

evaluation/heatmap_color_similarities.py

@@ -0,0 +1,535 @@
+import os
+import torch
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from sklearn.metrics.pairwise import cosine_similarity
+from sklearn.metrics import confusion_matrix, classification_report, accuracy_score
+from sklearn.model_selection import train_test_split
+from config import local_dataset_path, column_local_image_path, color_emb_dim, main_model_path, device
+from transformers import CLIPProcessor, CLIPModel as CLIPModel_transformers
+import warnings
+warnings.filterwarnings('ignore')
+from torch.utils.data import Dataset, DataLoader
+from torchvision import transforms
+from PIL import Image
+from tqdm import tqdm
+
+
+PRIMARY_COLORS = [
+    'red', 'pink', 'blue', 'green', 'aqua', 'lime', 'yellow', 
+    'orange', 'purple', 'brown', 'gray', 'black', 'white'
+]
+
+class ColorEncoder:
+    def __init__(self, main_model_path, device='mps'):
+        self.device = torch.device(device)
+        self.color_emb_dim = color_emb_dim
+        self.primary_colors = PRIMARY_COLORS
+        
+        print(f"🚀 Loading Main Model from {main_model_path}")
+        
+        # Load the main CLIP model
+        if os.path.exists(main_model_path):
+            checkpoint = torch.load(main_model_path, map_location=self.device)
+            self.main_model = CLIPModel_transformers.from_pretrained('laion/CLIP-ViT-B-32-laion2B-s34B-b79K')
+            self.main_model.load_state_dict(checkpoint['model_state_dict'])
+            self.main_model.to(self.device)
+            self.main_model.eval()
+            print(f"✅ Main model loaded successfully")
+        else:
+            raise FileNotFoundError(f"Main model file {main_model_path} not found")
+        
+        # Create processor
+        self.processor = CLIPProcessor.from_pretrained('laion/CLIP-ViT-B-32-laion2B-s34B-b79K')
+        
+        # Load dataset
+        self._load_dataset()
+        
+    def _load_dataset(self):
+        """Load and prepare dataset with primary colors filtering"""
+        print("📊 Loading dataset...")
+        df = pd.read_csv(local_dataset_path)
+        print(f"📊 Loaded {len(df)} samples")
+        
+        # Filter out rows with NaN values in image path
+        df_clean = df.dropna(subset=[column_local_image_path])
+        print(f"📊 After filtering NaN image paths: {len(df_clean)} samples")
+        
+        # Filter for primary colors only
+        df_primary = df_clean[df_clean['color'].isin(self.primary_colors)]
+        print(f"📊 After filtering for primary colors: {len(df_primary)} samples")
+        
+        # Show color distribution
+        color_counts = df_primary['color'].value_counts()
+        print(f"📊 Color distribution:")
+        for color in self.primary_colors:
+            count = color_counts.get(color, 0)
+            print(f"  {color}: {count} samples")
+        
+        # Split for train/val - Limit to 10000 samples
+        if len(df_primary) > 0:
+            # Limit to 10000 samples maximum
+            if len(df_primary) > 10000:
+                df_primary = df_primary.sample(n=10000, random_state=42)
+                print(f"📊 Limited to 10000 samples for processing")
+            
+            _, self.val_df = train_test_split(df_primary, test_size=0.2, random_state=42, stratify=df_primary['color'])
+            print(f"📊 Validation samples: {len(self.val_df)}")
+        else:
+            print("❌ No samples found for primary colors!")
+            self.val_df = pd.DataFrame()
+
+    def create_dataloader(self, dataframe, batch_size=8):
+        """Create a dataloader for the dataset"""
+        dataset = CustomDataset(dataframe, image_size=224)
+        dataset.set_training_mode(False)  # Use validation transforms
+        
+        dataloader = DataLoader(
+            dataset, 
+            batch_size=batch_size, 
+            shuffle=False, 
+            num_workers=0  # No multiprocessing to avoid memory issues
+        )
+        
+        return dataloader
+    
+    def extract_color_embeddings(self, dataloader, embedding_type='text', max_samples=10000):
+        """Extract color embeddings (first 16 dimensions) from text or image"""
+        all_embeddings = []
+        all_colors = []
+        
+        sample_count = 0
+        
+        with torch.no_grad():
+            for batch in tqdm(dataloader, desc=f"Extracting {embedding_type} color embeddings"):
+                if sample_count >= max_samples:
+                    break
+                    
+                images, texts, colors, hierarchies = batch
+                images = images.to(self.device)
+                images = images.expand(-1, 3, -1, -1)  # Ensure 3 channels
+                
+                # Process text inputs
+                text_inputs = self.processor(text=texts, padding=True, return_tensors="pt")
+                text_inputs = {k: v.to(self.device) for k, v in text_inputs.items()}
+                
+                # Forward pass through main model
+                outputs = self.main_model(**text_inputs, pixel_values=images)
+                
+                # Extract embeddings based on type
+                if embedding_type == 'text':
+                    embeddings = outputs.text_embeds
+                elif embedding_type == 'image':
+                    embeddings = outputs.image_embeds
+                else:
+                    embeddings = outputs.text_embeds
+                
+                # Extract only the first 16 dimensions (color embeddings)
+                color_embeddings = embeddings[:, :self.color_emb_dim]
+                
+                all_embeddings.append(color_embeddings.cpu().numpy())
+                all_colors.extend(colors)
+                
+                sample_count += len(images)
+                
+                # Clear GPU memory
+                del images, text_inputs, outputs, embeddings, color_embeddings
+                torch.cuda.empty_cache() if torch.cuda.is_available() else None
+        
+        return np.vstack(all_embeddings), all_colors
+
+    # Modifiez la méthode predict_colors_from_embeddings
+    def predict_colors_from_embeddings(self, embeddings, colors):
+        """Predict colors from embeddings using centroid-based classification"""
+        # Create color centroids from training data - only for primary colors
+        unique_colors = [c for c in self.primary_colors if c in colors]
+        centroids = {}
+        
+        for color in unique_colors:
+            color_indices = [i for i, c in enumerate(colors) if c == color]
+            if len(color_indices) > 0:
+                color_embeddings = embeddings[color_indices]
+                centroids[color] = np.mean(color_embeddings, axis=0)
+        
+        # Predict colors for all embeddings
+        predictions = []
+        
+        for i, embedding in enumerate(embeddings):
+            # Find closest centroid
+            best_similarity = -1
+            predicted_color = None
+            
+            for color, centroid in centroids.items():
+                similarity = cosine_similarity([embedding], [centroid])[0][0]
+                if similarity > best_similarity:
+                    best_similarity = similarity
+                    predicted_color = color
+            
+            predictions.append(predicted_color)
+        
+        return predictions
+
+    # Modifiez la méthode create_color_confusion_matrix
+    def create_color_confusion_matrix(self, true_colors, predicted_colors, title="Primary Colors Confusion Matrix"):
+        """Create and plot confusion matrix for primary colors"""
+        # Use only the primary colors in the order specified
+        unique_colors = [c for c in self.primary_colors if c in true_colors or c in predicted_colors]
+        
+        # Create confusion matrix
+        cm = confusion_matrix(true_colors, predicted_colors, labels=unique_colors)
+        
+        # Calculate accuracy
+        accuracy = accuracy_score(true_colors, predicted_colors)
+        
+        # Plot confusion matrix with better formatting
+        plt.figure(figsize=(14, 12))
+        sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', 
+                   xticklabels=unique_colors, yticklabels=unique_colors,
+                   cbar_kws={'label': 'Number of Samples'})
+        plt.title(f'{title}\nAccuracy: {accuracy:.3f} ({accuracy*100:.1f}%)', fontsize=16, fontweight='bold')
+        plt.ylabel('True Color', fontsize=14, fontweight='bold')
+        plt.xlabel('Predicted Color', fontsize=14, fontweight='bold')
+        plt.xticks(rotation=45, ha='right')
+        plt.yticks(rotation=0)
+        plt.tight_layout()
+        
+        return plt.gcf(), accuracy, cm
+
+    # Modifiez la méthode evaluate_color_classification
+    def evaluate_color_classification(self, dataframe, max_samples=10000):
+        """Evaluate primary color classification using first 16 dimensions"""
+        if len(dataframe) == 0:
+            print("❌ No data available for evaluation")
+            return None
+            
+        print(f"\n{'='*60}")
+        print(f"Evaluating Primary Color Classification (max {max_samples} samples)")
+        print(f"Target colors: {', '.join(self.primary_colors)}")
+        print(f"{'='*60}")
+        
+        # Create dataloader
+        dataloader = self.create_dataloader(dataframe, batch_size=8)
+        
+        results = {}
+        
+        # Evaluate text embeddings
+        print("🎨 Extracting text color embeddings (first 16 dimensions)...")
+        text_color_embeddings, color_labels = self.extract_color_embeddings(dataloader, 'text', max_samples)
+        text_predictions = self.predict_colors_from_embeddings(text_color_embeddings, color_labels)
+        text_accuracy = accuracy_score(color_labels, text_predictions)
+        
+        # Create confusion matrix for text
+        text_fig, text_acc, text_cm = self.create_color_confusion_matrix(
+            color_labels, text_predictions, "Text Color Embeddings (16D) - Confusion Matrix"
+        )
+        
+        results['text'] = {
+            'embeddings': text_color_embeddings,
+            'true_colors': color_labels,
+            'predicted_colors': text_predictions,
+            'accuracy': text_accuracy,
+            'confusion_matrix': text_cm,
+            'figure': text_fig
+        }
+        
+        # Clear memory
+        del text_color_embeddings
+        torch.cuda.empty_cache() if torch.cuda.is_available() else None
+        
+        # Evaluate image embeddings
+        print("🎨 Extracting image color embeddings (first 16 dimensions)...")
+        image_color_embeddings, color_labels_img = self.extract_color_embeddings(dataloader, 'image', max_samples)
+        image_predictions = self.predict_colors_from_embeddings(image_color_embeddings, color_labels_img)
+        image_accuracy = accuracy_score(color_labels_img, image_predictions)
+        
+        # Create confusion matrix for image
+        image_fig, image_acc, image_cm = self.create_color_confusion_matrix(
+            color_labels_img, image_predictions, "Image Color Embeddings (16D) - Confusion Matrix"
+        )
+        
+        results['image'] = {
+            'embeddings': image_color_embeddings,
+            'true_colors': color_labels_img,
+            'predicted_colors': image_predictions,
+            'accuracy': image_accuracy,
+            'confusion_matrix': image_cm,
+            'figure': image_fig
+        }
+        
+        # Clear memory
+        del image_color_embeddings
+        torch.cuda.empty_cache() if torch.cuda.is_available() else None
+        
+        # Print detailed results
+        print(f"\nPrimary Color Classification Results:")
+        print("-" * 50)
+        print(f"Text Color Embeddings:")
+        print(f"  Accuracy: {text_accuracy:.4f} ({text_accuracy*100:.1f}%)")
+        print(f"Image Color Embeddings:")
+        print(f"  Accuracy: {image_accuracy:.4f} ({image_accuracy*100:.1f}%)")
+        
+        # Show classification report
+        print(f"\n📊 Detailed Classification Report - Text:")
+        text_report = classification_report(color_labels, text_predictions, labels=self.primary_colors, 
+                                          target_names=self.primary_colors, output_dict=True)
+        for color in self.primary_colors:
+            if color in text_report:
+                precision = text_report[color]['precision']
+                recall = text_report[color]['recall']
+                f1 = text_report[color]['f1-score']
+                support = text_report[color]['support']
+                print(f"  {color:>8}: P={precision:.3f} R={recall:.3f} F1={f1:.3f} S={support}")
+        
+        print(f"\n📊 Detailed Classification Report - Image:")
+        image_report = classification_report(color_labels_img, image_predictions, labels=self.primary_colors, 
+                                           target_names=self.primary_colors, output_dict=True)
+        for color in self.primary_colors:
+            if color in image_report:
+                precision = image_report[color]['precision']
+                recall = image_report[color]['recall']
+                f1 = image_report[color]['f1-score']
+                support = image_report[color]['support']
+                print(f"  {color:>8}: P={precision:.3f} R={recall:.3f} F1={f1:.3f} S={support}")
+        
+        # Create visualizations
+        os.makedirs('evaluation/color_evaluation_results', exist_ok=True)
+        results['text']['figure'].savefig('evaluation/color_evaluation_results/text_color_confusion_matrix.png', 
+                                        dpi=300, bbox_inches='tight')
+        results['image']['figure'].savefig('evaluation/color_evaluation_results/image_color_confusion_matrix.png', 
+                                         dpi=300, bbox_inches='tight')
+        plt.close(results['text']['figure'])
+        plt.close(results['image']['figure'])
+        
+        return results
+
+    def create_color_similarity_heatmap(self, embeddings, colors, embedding_type='text', save_path='evaluation/color_similarity_results/color_similarity_heatmap.png'):
+        """
+        Create a heatmap of similarities between encoded colors
+        """
+        print(f"🎨 Creating color similarity heatmap for {embedding_type} embeddings...")
+        
+        unique_colors = [c for c in self.primary_colors if c in colors]
+        centroids = {}
+        
+        for color in unique_colors:
+            color_indices = [i for i, c in enumerate(colors) if c == color]
+            if len(color_indices) > 0:
+                color_embeddings = embeddings[color_indices]
+                centroids[color] = np.mean(color_embeddings, axis=0)
+        
+        similarity_matrix = np.zeros((len(unique_colors), len(unique_colors)))
+        
+        for i, color1 in enumerate(unique_colors):
+            for j, color2 in enumerate(unique_colors):
+                if i == j:
+                    similarity_matrix[i, j] = 1.0
+                else:
+                    similarity = cosine_similarity([centroids[color1]], [centroids[color2]])[0][0]
+                    similarity_matrix[i, j] = similarity
+    
+        plt.figure(figsize=(12, 10))
+        
+        sns.heatmap(
+            similarity_matrix,
+            annot=True,
+            fmt='.3f',
+            cmap='RdYlBu_r',
+            xticklabels=unique_colors,
+            yticklabels=unique_colors,
+            square=True,
+            cbar_kws={'label': 'Cosine Similarity'},
+            linewidths=0.5,
+            vmin=-0.6,
+            vmax=1.0
+        )
+    
+        plt.title(f'Color similarity ({embedding_type} embeddings)', 
+                fontsize=16, fontweight='bold', pad=20)
+        plt.xlabel('Colors', fontsize=14, fontweight='bold')
+        plt.ylabel('Colors', fontsize=14, fontweight='bold')
+        plt.xticks(rotation=45, ha='right')
+        plt.yticks(rotation=0)
+        plt.tight_layout()
+        
+        plt.savefig(save_path, dpi=300, bbox_inches='tight')
+        print(f"💾 Heatmap saved: {save_path}")
+        
+        return plt.gcf(), similarity_matrix
+    
+ 
+
+    def create_color_similarity_analysis(self, results):
+        """
+        Complete analysis of similarities between colors for text and image embeddings
+        """
+        print(f"\n{'='*60}")
+        print("🎨 ANALYSIS OF SIMILARITIES BETWEEN COLORS")
+        print(f"{'='*60}")
+        
+        os.makedirs('evaluation/color_similarity_results', exist_ok=True)
+        
+        similarity_results = {}
+                
+        if 'text' in results:
+            print("\n📝 Analyse des similarités - Text Embeddings:")
+            text_fig, text_similarity_matrix = self.create_color_similarity_heatmap(
+                results['text']['embeddings'],
+                results['text']['true_colors'],
+                'text',
+                'evaluation/color_similarity_results/text_color_similarity_heatmap.png'
+            )
+            similarity_results['text'] = {
+                'similarity_matrix': text_similarity_matrix,
+                'figure': text_fig
+            }
+            plt.close(text_fig)
+        
+        # Analyser les embeddings image
+        if 'image' in results:
+            print("\n🖼️ Analyse des similarités - Image Embeddings:")
+            image_fig, image_similarity_matrix = self.create_color_similarity_heatmap(
+                results['image']['embeddings'],
+                results['image']['true_colors'],
+                'image',
+                'evaluation/color_similarity_results/image_color_similarity_heatmap.png'
+            )
+            similarity_results['image'] = {
+                'similarity_matrix': image_similarity_matrix,
+                'figure': image_fig
+            }
+            plt.close(image_fig)
+        
+        # Analyser les similarités les plus élevées et les plus faibles
+        self._analyze_similarity_patterns(similarity_results)
+        
+        return similarity_results
+
+    def _analyze_similarity_patterns(self, similarity_results):
+        """
+        Analyse les patterns de similarité entre les couleurs
+        """
+        print(f"\n�� ANALYSE DES PATTERNS DE SIMILARITÉ")
+        print("-" * 50)
+        
+        for embedding_type, data in similarity_results.items():
+            matrix = data['similarity_matrix']
+            unique_colors = [c for c in self.primary_colors if c in [f"color_{i}" for i in range(len(matrix))]]
+            
+            print(f"\n{embedding_type.upper()} Embeddings:")
+            
+            # Trouver les paires les plus similaires (hors diagonale)
+            n = len(matrix)
+            similarities = []
+            
+            for i in range(n):
+                for j in range(i+1, n):  # Éviter la diagonale et la redondance
+                    similarities.append((i, j, matrix[i, j]))
+            
+            # Trier par similarité décroissante
+            similarities.sort(key=lambda x: x[2], reverse=True)
+            
+            print("🔗 Couleurs les plus similaires:")
+            for i, (idx1, idx2, sim) in enumerate(similarities[:5]):
+                color1 = self.primary_colors[idx1] if idx1 < len(self.primary_colors) else f"Color_{idx1}"
+                color2 = self.primary_colors[idx2] if idx2 < len(self.primary_colors) else f"Color_{idx2}"
+                print(f"  {i+1}. {color1} ↔ {color2}: {sim:.3f}")
+            
+            print("🔗 Couleurs les moins similaires:")
+            for i, (idx1, idx2, sim) in enumerate(similarities[-5:]):
+                color1 = self.primary_colors[idx1] if idx1 < len(self.primary_colors) else f"Color_{idx1}"
+                color2 = self.primary_colors[idx2] if idx2 < len(self.primary_colors) else f"Color_{idx2}"
+                print(f"  {i+1}. {color1} ↔ {color2}: {sim:.3f}")
+            
+            # Calculer la similarité moyenne
+            off_diagonal = matrix[np.triu_indices_from(matrix, k=1)]
+            mean_similarity = np.mean(off_diagonal)
+            std_similarity = np.std(off_diagonal)
+            
+            print(f"📈 Similarité moyenne: {mean_similarity:.3f} ± {std_similarity:.3f}")
+
+class CustomDataset(Dataset):
+    def __init__(self, dataframe, image_size=224):
+        self.dataframe = dataframe
+        self.image_size = image_size
+        
+        # Transforms for validation (no augmentation)
+        self.val_transform = transforms.Compose([
+            transforms.Resize((image_size, image_size)),
+            transforms.ToTensor(),
+            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+        ])
+        
+        self.training_mode = True
+        
+    def set_training_mode(self, training=True):
+        self.training_mode = training
+
+    def __len__(self):
+        return len(self.dataframe)
+
+    def __getitem__(self, idx):
+        row = self.dataframe.iloc[idx]
+        
+        image_data = row[column_local_image_path]
+        image = Image.open(image_data).convert("RGB")
+        
+        # Apply validation transform
+        image = self.val_transform(image)
+
+        # Get text and labels
+        description = row['text']
+        color = row['color']
+        hierarchy = row['hierarchy']
+
+        return image, description, color, hierarchy
+
+# Modifiez la section main
+if __name__ == "__main__":
+    print("🚀 Starting Primary Color Encoding and Similarity Analysis")
+    print("="*70)
+    print(f"Target Primary Colors: {', '.join(PRIMARY_COLORS)}")
+    print("="*70)
+    
+    # Initialize color encoder
+    color_encoder = ColorEncoder(
+        main_model_path=main_model_path,
+        device=device
+    )
+    
+    # Evaluate primary color classification
+    results = color_encoder.evaluate_color_classification(
+        color_encoder.val_df, 
+        max_samples=10000
+    )
+    
+    if results:
+        print(f"\n✅ Primary color encoding and confusion matrix generation completed!")
+        print(f"📊 Results saved in 'evaluation/color_evaluation_results/' directory")
+        print(f"🎨 Text Primary Color Accuracy: {results['text']['accuracy']*100:.1f}%")
+        print(f"🖼️ Image Primary Color Accuracy: {results['image']['accuracy']*100:.1f}%")
+        
+        # NOUVELLE SECTION: Analyse des similarités
+        print(f"\n🎨 Starting Color Similarity Analysis...")
+        similarity_results = color_encoder.create_color_similarity_analysis(results)
+        
+        print(f"\n✅ Color similarity analysis completed!")
+        print(f"📊 Similarity heatmaps saved in 'evaluation/color_similarity_results/' directory")
+        
+        # Show some sample predictions
+        print(f"\n📝 Sample Text Predictions:")
+        for i in range(min(10, len(results['text']['true_colors']))):
+            true_color = results['text']['true_colors'][i]
+            pred_color = results['text']['predicted_colors'][i]
+            status = "✓" if true_color == pred_color else "✗"
+            print(f"  {status} True: {true_color:>8} | Predicted: {pred_color:>8}")
+        
+        print(f"\n🖼️ Sample Image Predictions:")
+        for i in range(min(10, len(results['image']['true_colors']))):
+            true_color = results['image']['true_colors'][i]
+            pred_color = results['image']['predicted_colors'][i]
+            status = "✓" if true_color == pred_color else "✗"
+            print(f"  {status} True: {true_color:>8} | Predicted: {pred_color:>8}")
+    else:
+        print("❌ No results generated - check if primary colors exist in dataset")