backend/ml/chart_generator.py

"""
🎨 PRODUCTION ML CHART GENERATOR v3.0
=====================================

COMPREHENSIVE ML Visualization using matplotlib and seaborn.
Only creates charts when sufficient data is available.

Chart Types by Task:

CLASSIFICATION (12 charts):
- Confusion Matrix (normalized heatmap)
- Class Distribution (actual vs predicted)
- ROC Curve (binary + multi-class)
- Precision-Recall Curve
- Calibration Curve
- Prediction Confidence
- Classification Report Heatmap
- Class Balance Pie Chart
- Prediction Error by Class (box plot)
- Score Distribution Histogram
- Cumulative Gains Chart
- Lift Chart

REGRESSION (10 charts):
- Actual vs Predicted Scatter
- Residuals Analysis (2x2 grid)  
- Error Distribution (KDE + histogram)
- Prediction Overview (sorted with bands)
- Target Distribution Histogram
- Prediction Error Box Plot
- Cook's Distance Plot
- Scale-Location Plot
- Predicted vs Residuals Scatter
- SHAP-style Importance (if available)

CLUSTERING (8 charts):
- Cluster Scatter (PCA 2D)
- Cluster Distribution Bar
- Silhouette Analysis
- Feature Distribution by Cluster
- Cluster Box Plots
- Cluster Centroid Radar
- Cluster Pairplot (first 4 features)
- Elbow Method (if k data available)

NLP (6 charts):
- Word Cloud
- Word Frequency Histogram
- Text Length Distribution
- TF-IDF Feature Importance
- Class Distribution (for text classification)
- Sentiment/Topic Distribution

COMMON (5 charts):
- Feature Importance (horizontal bar)
- Model Comparison (bar chart)
- Correlation Heatmap
- Distribution Overview (histogram grid)
- Box Plot Grid
"""

import numpy as np
import pandas as pd
import base64
import io
import logging
from typing import Dict, List, Optional, Any, Tuple

# Visualization imports
import matplotlib
matplotlib.use('Agg')  # Non-interactive backend for server
import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib.colors import LinearSegmentedColormap
from matplotlib.patches import Patch

# ML metrics
from sklearn.metrics import (
    confusion_matrix, classification_report, roc_curve, auc,
    precision_recall_curve, r2_score, mean_absolute_error,
    mean_squared_error, silhouette_samples, silhouette_score,
    average_precision_score
)
from sklearn.calibration import calibration_curve
from scipy import stats

logger = logging.getLogger(__name__)

# Set PROFESSIONAL LIGHT theme style (Clean, Business-Ready)
sns.set_style("whitegrid")
plt.rcParams.update({
    'figure.facecolor': '#ffffff',
    'axes.facecolor': '#ffffff',
    'axes.edgecolor': '#e0e0e0',
    'text.color': '#333333',
    'axes.labelcolor': '#333333',
    'xtick.color': '#555555',
    'ytick.color': '#555555',
    'grid.color': '#f0f0f0',
    'legend.facecolor': '#ffffff',
    'legend.edgecolor': '#e0e0e0',
    'figure.dpi': 200,          # High resolution
    'font.size': 11,            # Readable font
    'axes.titlesize': 14,       # Clear titles
    'axes.labelsize': 12,       # Clear labels
    'font.family': 'sans-serif',
    'font.sans-serif': ['Segoe UI', 'Arial', 'sans-serif'] # Modern business fonts
})

# DataVision color palette - Extended (Vibrant & Professional)
COLORS = [
    '#0F766E',  # Deep Teal (Primary)
    '#2563EB',  # Royal Blue
    '#D946EF',  # Magenta
    '#F59E0B',  # Amber
    '#10B981',  # Emerald
    '#8B5CF6',  # Violet
    '#EF4444',  # Red
    '#06B6D4',  # Cyan
    '#84CC16',  # Lime
    '#F97316',  # Orange
    '#6366F1',  # Indigo
    '#A855F7',  # Purple
]

# Custom colormaps (Adapted for Light Theme)
TEAL_CMAP = LinearSegmentedColormap.from_list('teal', ['#ffffff', '#ccfbf1', '#0f766e'])
HEAT_CMAP = LinearSegmentedColormap.from_list('heat', ['#ffffff', '#fef3c7', '#ef4444'])


def _fig_to_base64(fig) -> str:
    """Convert matplotlib figure to base64 string"""
    buf = io.BytesIO()
    # Save with high DPI and white background
    fig.savefig(buf, format='png', dpi=200, bbox_inches='tight', 
                facecolor='#ffffff', edgecolor='none')
    buf.seek(0)
    img_base64 = base64.b64encode(buf.read()).decode('utf-8')
    plt.close(fig)
    return f"data:image/png;base64,{img_base64}"


def generate_ml_charts(
    task_type: str,
    y_test: np.ndarray,
    y_pred: np.ndarray,
    y_proba: Optional[np.ndarray] = None,
    feature_importance: Optional[List[Dict]] = None,
    class_names: Optional[List[str]] = None,
    model_name: str = "Model",
    X_test: Optional[np.ndarray] = None,
    feature_names: Optional[List[str]] = None,
    text_data: Optional[List[str]] = None
) -> Dict[str, str]:
    """
    Generate COMPREHENSIVE ML charts based on task type and available data.
    
    Args:
        task_type: 'classification', 'regression', 'clustering', or 'nlp'
        y_test: True labels
        y_pred: Predicted labels
        y_proba: Prediction probabilities
        feature_importance: List of {feature, importance} dicts
        class_names: Names of classes
        model_name: Model name for titles
        X_test: Test features
        feature_names: Feature names
        text_data: Raw text data for NLP charts
    
    Returns:
        Dict of chart_name: base64_image_string
    """
    charts = {}
    
    try:
        # Safely convert to numpy arrays
        y_test = np.array(y_test).flatten()
        y_pred = np.array(y_pred).flatten()
        
        # Handle non-numeric labels (strings) by encoding them
        # This prevents 'isnan' errors on object arrays
        original_labels = None
        if y_test.dtype == object or not np.issubdtype(y_test.dtype, np.number):
            from sklearn.preprocessing import LabelEncoder
            le = LabelEncoder()
            # Combine y_test and y_pred to ensure consistent encoding
            all_labels = np.concatenate([y_test, y_pred])
            le.fit(all_labels)
            original_labels = le.classes_.tolist()  # Store original labels for display
            y_test = le.transform(y_test)
            y_pred = le.transform(y_pred)
            # Use original labels as class_names if not provided
            if class_names is None:
                class_names = original_labels
        
        if len(y_test) < 5 or len(y_pred) < 5:
            logger.warning("Insufficient data for charts")
            return {}
        
        task_lower = task_type.lower()
        
        if 'classification' in task_lower or 'nlp' in task_lower:
            charts.update(_generate_classification_charts(
                y_test, y_pred, y_proba, class_names, model_name
            ))
            if 'nlp' in task_lower and text_data:
                charts.update(_generate_nlp_charts(text_data, y_test, model_name))
        elif 'clustering' in task_lower:
            charts.update(_generate_clustering_charts(
                X_test, y_pred, model_name, feature_names
            ))
        else:  # regression
            charts.update(_generate_regression_charts(
                y_test, y_pred, model_name
            ))
        
        # Common charts
        if feature_importance and len(feature_importance) > 0:
            fi_chart = _generate_feature_importance_chart(feature_importance, model_name)
            if fi_chart:
                charts['feature_importance'] = fi_chart
                
        logger.info(f"✅ Generated {len(charts)} charts: {list(charts.keys())}")
                
    except Exception as e:
        logger.error(f"Chart generation error: {e}")
        import traceback
        traceback.print_exc()
    
    return charts


# =============================================================================
# CLASSIFICATION CHARTS (12 types)
# =============================================================================

def _generate_classification_charts(
    y_test: np.ndarray,
    y_pred: np.ndarray,
    y_proba: Optional[np.ndarray],
    class_names: Optional[List[str]],
    model_name: str
) -> Dict[str, str]:
    """Generate comprehensive classification charts"""
    charts = {}
    n_classes = len(np.unique(y_test))
    
    # 1. CONFUSION MATRIX (Enhanced with normalization)
    try:
        cm = confusion_matrix(y_test, y_pred)
        if cm.shape[0] >= 2:
            fig, ax = plt.subplots(figsize=(8, 6))
            
            cm_norm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
            cm_norm = np.nan_to_num(cm_norm)
            
            sns.heatmap(cm_norm, annot=True, fmt='.1%', cmap='Blues',
                       ax=ax, cbar_kws={'label': 'Percentage'}, linewidths=0.5)
            
            for i in range(cm.shape[0]):
                for j in range(cm.shape[1]):
                    ax.text(j + 0.5, i + 0.75, f'n={cm[i,j]}',
                           ha='center', va='center', fontsize=8, color='gray')
            
            labels = [str(c)[:10] for c in class_names] if class_names else None
            if labels:
                ax.set_xticklabels(labels, rotation=45, ha='right')
                ax.set_yticklabels(labels, rotation=0)
            
            ax.set_xlabel('Predicted', fontsize=12, color='#333333')
            ax.set_ylabel('Actual', fontsize=12, color='#333333')
            ax.set_title(f'Confusion Matrix - {model_name}', fontsize=14, color='#333333', pad=20)
            
            charts['confusion_matrix'] = _fig_to_base64(fig)
    except Exception as e:
        logger.warning(f"Confusion matrix error: {e}")
    
    # 2. CLASS DISTRIBUTION (Side by side bars)
    try:
        fig, ax = plt.subplots(figsize=(10, 6))
        
        unique_classes = np.union1d(np.unique(y_test), np.unique(y_pred))
        x = np.arange(len(unique_classes))
        width = 0.35
        
        actual_counts = [np.sum(y_test == c) for c in unique_classes]
        pred_counts = [np.sum(y_pred == c) for c in unique_classes]
        
        ax.bar(x - width/2, actual_counts, width, label='Actual', color=COLORS[0], edgecolor='#333333')
        ax.bar(x + width/2, pred_counts, width, label='Predicted', color=COLORS[2], edgecolor='#333333')
        
        ax.set_xlabel('Class', fontsize=12, color='#333333')
        ax.set_ylabel('Count', fontsize=12, color='#333333')
        ax.set_title(f'Class Distribution - {model_name}', fontsize=14, color='#333333')
        ax.set_xticks(x)
        labels = [str(c)[:10] for c in class_names[:len(unique_classes)]] if class_names else [str(c) for c in unique_classes]
        ax.set_xticklabels(labels, rotation=45, ha='right')
        ax.legend()
        
        charts['class_distribution'] = _fig_to_base64(fig)
    except Exception as e:
        logger.warning(f"Class distribution error: {e}")
    
    # 3. ROC CURVE (Multi-class support)
    if y_proba is not None:
        try:
            fig, ax = plt.subplots(figsize=(8, 6))
            
            if n_classes == 2:
                proba = y_proba[:, 1] if len(y_proba.shape) > 1 and y_proba.shape[1] >= 2 else y_proba.flatten()
                fpr, tpr, _ = roc_curve(y_test, proba)
                roc_auc = auc(fpr, tpr)
                ax.plot(fpr, tpr, color=COLORS[0], lw=2, label=f'ROC (AUC = {roc_auc:.3f})')
                ax.fill_between(fpr, tpr, alpha=0.2, color=COLORS[0])
            else:
                if len(y_proba.shape) > 1:
                    for i in range(min(n_classes, y_proba.shape[1])):
                        y_binary = (y_test == i).astype(int)
                        if y_binary.sum() > 0:
                            fpr, tpr, _ = roc_curve(y_binary, y_proba[:, i])
                            roc_auc = auc(fpr, tpr)
                            label = class_names[i][:10] if class_names and i < len(class_names) else f'Class {i}'
                            ax.plot(fpr, tpr, color=COLORS[i % len(COLORS)], lw=2, 
                                   label=f'{label} (AUC={roc_auc:.2f})')
            
            ax.plot([0, 1], [0, 1], 'k--', lw=1, alpha=0.5, label='Random')
            ax.set_xlim([0.0, 1.0])
            ax.set_ylim([0.0, 1.05])
            ax.set_xlabel('False Positive Rate', fontsize=12, color='#333333')
            ax.set_ylabel('True Positive Rate', fontsize=12, color='#333333')
            ax.set_title(f'ROC Curve - {model_name}', fontsize=14, color='#333333')
            ax.legend(loc='lower right', fontsize=9)
            
            charts['roc_curve'] = _fig_to_base64(fig)
        except Exception as e:
            logger.warning(f"ROC curve error: {e}")
    
    # 4. PRECISION-RECALL CURVE
    if y_proba is not None and n_classes == 2:
        try:
            proba = y_proba[:, 1] if len(y_proba.shape) > 1 else y_proba.flatten()
            precision, recall, _ = precision_recall_curve(y_test, proba)
            avg_prec = average_precision_score(y_test, proba)
            
            fig, ax = plt.subplots(figsize=(8, 6))
            ax.plot(recall, precision, color=COLORS[1], lw=2, label=f'PR (AP = {avg_prec:.3f})')
            ax.fill_between(recall, precision, alpha=0.2, color=COLORS[1])
            ax.set_xlabel('Recall', fontsize=12, color='#333333')
            ax.set_ylabel('Precision', fontsize=12, color='#333333')
            ax.set_title(f'Precision-Recall Curve - {model_name}', fontsize=14, color='#333333')
            ax.legend()
            
            charts['precision_recall'] = _fig_to_base64(fig)
        except Exception as e:
            logger.warning(f"Precision-Recall error: {e}")
    
    # 5. CALIBRATION CURVE
    if y_proba is not None and n_classes == 2 and len(y_test) >= 20:
        try:
            proba = y_proba[:, 1] if len(y_proba.shape) > 1 else y_proba.flatten()
            prob_true, prob_pred = calibration_curve(y_test, proba, n_bins=10)
            
            fig, ax = plt.subplots(figsize=(8, 6))
            ax.plot([0, 1], [0, 1], 'k--', lw=1, label='Perfect')
            ax.plot(prob_pred, prob_true, 's-', color=COLORS[2], lw=2, label=model_name)
            ax.set_xlabel('Mean Predicted Probability', fontsize=12, color='#333333')
            ax.set_ylabel('Fraction of Positives', fontsize=12, color='#333333')
            ax.set_title(f'Calibration Curve - {model_name}', fontsize=14, color='#333333')
            ax.legend()
            
            charts['calibration'] = _fig_to_base64(fig)
        except Exception as e:
            logger.warning(f"Calibration error: {e}")
    
    # 6. PREDICTION CONFIDENCE HISTOGRAM
    if y_proba is not None:
        try:
            fig, ax = plt.subplots(figsize=(8, 6))
            
            max_proba = np.max(y_proba, axis=1) if len(y_proba.shape) > 1 else np.abs(y_proba - 0.5) * 2 + 0.5
            correct = y_pred == y_test
            
            ax.hist(max_proba[correct], bins=20, alpha=0.7, color=COLORS[1], label='Correct', edgecolor='#333333')
            ax.hist(max_proba[~correct], bins=20, alpha=0.7, color=COLORS[6], label='Incorrect', edgecolor='#333333')
            
            ax.set_xlabel('Prediction Confidence', fontsize=12, color='#333333')
            ax.set_ylabel('Count', fontsize=12, color='#333333')
            ax.set_title(f'Confidence Distribution - {model_name}', fontsize=14, color='#333333')
            ax.legend()
            
            charts['confidence_histogram'] = _fig_to_base64(fig)
        except Exception as e:
            logger.warning(f"Confidence histogram error: {e}")
    
    # 7. CLASS BALANCE PIE CHART
    try:
        fig, axes = plt.subplots(1, 2, figsize=(12, 5))
        
        unique, counts = np.unique(y_test, return_counts=True)
        labels = [str(class_names[i])[:10] if class_names and i < len(class_names) else str(u) for i, u in enumerate(unique)]
        colors = [COLORS[i % len(COLORS)] for i in range(len(unique))]
        
        axes[0].pie(counts, labels=labels, autopct='%1.1f%%', colors=colors, 
                   textprops={'color': 'white'}, wedgeprops={'edgecolor': 'white'})
        axes[0].set_title('Actual Class Balance', color='#333333', fontsize=12)
        
        unique_p, counts_p = np.unique(y_pred, return_counts=True)
        labels_p = [str(class_names[i])[:10] if class_names and i < len(class_names) else str(u) for i, u in enumerate(unique_p)]
        axes[1].pie(counts_p, labels=labels_p, autopct='%1.1f%%', colors=colors[:len(unique_p)],
                   textprops={'color': 'white'}, wedgeprops={'edgecolor': 'white'})
        axes[1].set_title('Predicted Class Balance', color='#333333', fontsize=12)
        
        fig.suptitle(f'Class Balance - {model_name}', fontsize=14, color='#333333')
        plt.tight_layout()
        
        charts['class_balance_pie'] = _fig_to_base64(fig)
    except Exception as e:
        logger.warning(f"Class balance pie error: {e}")
    
    # 8. PREDICTION ERROR BOX PLOT BY CLASS
    if y_proba is not None:
        try:
            fig, ax = plt.subplots(figsize=(10, 6))
            
            max_proba = np.max(y_proba, axis=1) if len(y_proba.shape) > 1 else y_proba.flatten()
            classes = np.unique(y_test)
            data = [max_proba[y_test == c] for c in classes]
            
            bp = ax.boxplot(data, patch_artist=True)
            for i, (patch, color) in enumerate(zip(bp['boxes'], [COLORS[i % len(COLORS)] for i in range(len(classes))])):
                patch.set_facecolor(color)
                patch.set_alpha(0.7)
            
            labels = [str(class_names[i])[:10] if class_names and i < len(class_names) else str(c) for i, c in enumerate(classes)]
            ax.set_xticklabels(labels, rotation=45, ha='right')
            ax.set_xlabel('Class', fontsize=12, color='#333333')
            ax.set_ylabel('Prediction Confidence', fontsize=12, color='#333333')
            ax.set_title(f'Confidence by Class - {model_name}', fontsize=14, color='#333333')
            
            charts['confidence_boxplot'] = _fig_to_base64(fig)
        except Exception as e:
            logger.warning(f"Confidence boxplot error: {e}")
    
    # 9. CLASSIFICATION REPORT HEATMAP
    try:
        from sklearn.metrics import precision_recall_fscore_support
        precision, recall, f1, support = precision_recall_fscore_support(y_test, y_pred, zero_division=0)
        
        if len(precision) <= 10:  # Only for reasonable number of classes
            fig, ax = plt.subplots(figsize=(10, 6))
            
            data = np.array([precision, recall, f1]).T
            labels = [str(class_names[i])[:10] if class_names and i < len(class_names) else f'Class {i}' for i in range(len(precision))]
            
            sns.heatmap(data, annot=True, fmt='.2f', cmap='RdYlGn', 
                       xticklabels=['Precision', 'Recall', 'F1-Score'],
                       yticklabels=labels, ax=ax, vmin=0, vmax=1)
            ax.set_title(f'Classification Metrics by Class - {model_name}', fontsize=14, color='#333333')
            
            charts['metrics_heatmap'] = _fig_to_base64(fig)
    except Exception as e:
        logger.warning(f"Metrics heatmap error: {e}")
    
    # 10. SCORE DISTRIBUTION
    if y_proba is not None:
        try:
            fig, ax = plt.subplots(figsize=(10, 6))
            
            if len(y_proba.shape) > 1:
                for i in range(min(y_proba.shape[1], 5)):
                    label = class_names[i][:10] if class_names and i < len(class_names) else f'Class {i}'
                    sns.kdeplot(y_proba[:, i], ax=ax, label=label, color=COLORS[i % len(COLORS)], lw=2)
            
            ax.set_xlabel('Prediction Score', fontsize=12, color='#333333')
            ax.set_ylabel('Density', fontsize=12, color='#333333')
            ax.set_title(f'Score Distribution by Class - {model_name}', fontsize=14, color='#333333')
            ax.legend()
            
            charts['score_distribution'] = _fig_to_base64(fig)
        except Exception as e:
            logger.warning(f"Score distribution error: {e}")
    
    return charts


# =============================================================================
# REGRESSION CHARTS (10 types)
# =============================================================================

def _generate_regression_charts(
    y_test: np.ndarray,
    y_pred: np.ndarray,
    model_name: str
) -> Dict[str, str]:
    """Generate comprehensive regression charts"""
    charts = {}
    residuals = y_pred - y_test
    errors = np.abs(residuals)
    
    # 1. ACTUAL VS PREDICTED with trend
    try:
        fig, ax = plt.subplots(figsize=(8, 6))
        
        ax.scatter(y_test, y_pred, alpha=0.6, color=COLORS[0], s=50, edgecolors='white', linewidth=0.5)
        
        min_v, max_v = min(y_test.min(), y_pred.min()), max(y_test.max(), y_pred.max())
        ax.plot([min_v, max_v], [min_v, max_v], 'r--', lw=2, label='Perfect')
        
        z = np.polyfit(y_test, y_pred, 1)
        p = np.poly1d(z)
        ax.plot(np.sort(y_test), p(np.sort(y_test)), '--', color=COLORS[1], lw=1.5, label='Trend')
        
        r2 = r2_score(y_test, y_pred)
        mae = mean_absolute_error(y_test, y_pred)
        rmse = np.sqrt(mean_squared_error(y_test, y_pred))
        
        ax.annotate(f'R² = {r2:.4f}\nMAE = {mae:.2f}\nRMSE = {rmse:.2f}', 
                   xy=(0.05, 0.95), xycoords='axes fraction', fontsize=11, color='#333333',
                   bbox=dict(boxstyle='round', facecolor='#333333', alpha=0.8), va='top')
        
        ax.set_xlabel('Actual', fontsize=12, color='#333333')
        ax.set_ylabel('Predicted', fontsize=12, color='#333333')
        ax.set_title(f'Actual vs Predicted - {model_name}', fontsize=14, color='#333333')
        ax.legend()
        
        charts['actual_vs_predicted'] = _fig_to_base64(fig)
    except Exception as e:
        logger.warning(f"Actual vs Predicted error: {e}")
    
    # 2. RESIDUALS ANALYSIS (2x2 grid)
    try:
        fig, axes = plt.subplots(2, 2, figsize=(12, 10))
        
        # Residuals vs Predicted
        axes[0, 0].scatter(y_pred, residuals, alpha=0.6, color=COLORS[1], s=40)
        axes[0, 0].axhline(y=0, color='red', linestyle='--', lw=1.5)
        axes[0, 0].set_xlabel('Predicted', color='#333333')
        axes[0, 0].set_ylabel('Residual', color='#333333')
        axes[0, 0].set_title('Residuals vs Predicted', color='#333333')
        
        # Residuals Histogram
        axes[0, 1].hist(residuals, bins=30, color=COLORS[2], alpha=0.7, edgecolor='#333333')
        axes[0, 1].axvline(x=0, color='red', linestyle='--', lw=1.5)
        axes[0, 1].set_xlabel('Residual', color='#333333')
        axes[0, 1].set_ylabel('Frequency', color='#333333')
        axes[0, 1].set_title('Residuals Distribution', color='#333333')
        
        # Q-Q Plot
        stats.probplot(residuals, dist="norm", plot=axes[1, 0])
        axes[1, 0].get_lines()[0].set_color(COLORS[0])
        axes[1, 0].get_lines()[1].set_color('red')
        axes[1, 0].set_title('Q-Q Plot (Normality)', color='#333333')
        axes[1, 0].set_xlabel('Theoretical Quantiles', color='#333333')
        axes[1, 0].set_ylabel('Sample Quantiles', color='#333333')
        
        # Residuals vs Index
        axes[1, 1].scatter(range(len(residuals)), residuals, alpha=0.6, color=COLORS[3], s=30)
        axes[1, 1].axhline(y=0, color='red', linestyle='--', lw=1.5)
        axes[1, 1].set_xlabel('Index', color='#333333')
        axes[1, 1].set_ylabel('Residual', color='#333333')
        axes[1, 1].set_title('Residuals vs Order', color='#333333')
        
        fig.suptitle(f'Residual Analysis - {model_name}', fontsize=14, color='#333333', y=1.02)
        plt.tight_layout()
        
        charts['residuals_analysis'] = _fig_to_base64(fig)
    except Exception as e:
        logger.warning(f"Residuals analysis error: {e}")
    
    # 3. ERROR DISTRIBUTION with percentiles
    try:
        fig, ax = plt.subplots(figsize=(8, 6))
        
        sns.kdeplot(errors, ax=ax, color=COLORS[3], fill=True, alpha=0.3, lw=2)
        ax.hist(errors, bins=30, density=True, alpha=0.3, color=COLORS[3], edgecolor='#333333')
        
        for p, color in [(50, COLORS[0]), (75, COLORS[1]), (90, COLORS[4]), (95, COLORS[6])]:
            val = np.percentile(errors, p)
            ax.axvline(val, color=color, linestyle='--', label=f'{p}th: {val:.2f}')
        
        ax.set_xlabel('Absolute Error', fontsize=12, color='#333333')
        ax.set_ylabel('Density', fontsize=12, color='#333333')
        ax.set_title(f'Error Distribution - {model_name}', fontsize=14, color='#333333')
        ax.legend(fontsize=9)
        
        charts['error_distribution'] = _fig_to_base64(fig)
    except Exception as e:
        logger.warning(f"Error distribution error: {e}")
    
    # 4. TARGET DISTRIBUTION HISTOGRAM
    try:
        fig, ax = plt.subplots(figsize=(10, 6))
        
        ax.hist(y_test, bins=30, alpha=0.6, color=COLORS[0], label='Actual', edgecolor='#333333')
        ax.hist(y_pred, bins=30, alpha=0.6, color=COLORS[2], label='Predicted', edgecolor='#333333')
        
        ax.axvline(y_test.mean(), color=COLORS[0], linestyle='--', lw=2, label=f'Actual Mean: {y_test.mean():.2f}')
        ax.axvline(y_pred.mean(), color=COLORS[2], linestyle='--', lw=2, label=f'Pred Mean: {y_pred.mean():.2f}')
        
        ax.set_xlabel('Value', fontsize=12, color='#333333')
        ax.set_ylabel('Frequency', fontsize=12, color='#333333')
        ax.set_title(f'Target Distribution - {model_name}', fontsize=14, color='#333333')
        ax.legend()
        
        charts['target_histogram'] = _fig_to_base64(fig)
    except Exception as e:
        logger.warning(f"Target histogram error: {e}")
    
    # 5. ERROR BOX PLOT (by quantiles)
    try:
        fig, ax = plt.subplots(figsize=(10, 6))
        
        quantiles = np.percentile(y_test, [0, 25, 50, 75, 100])
        groups = []
        labels = []
        for i in range(len(quantiles) - 1):
            mask = (y_test >= quantiles[i]) & (y_test < quantiles[i+1])
            if mask.sum() > 0:
                groups.append(errors[mask])
                labels.append(f'Q{i+1}')
        
        bp = ax.boxplot(groups, patch_artist=True)
        for i, patch in enumerate(bp['boxes']):
            patch.set_facecolor(COLORS[i % len(COLORS)])
            patch.set_alpha(0.7)
        
        ax.set_xticklabels(labels)
        ax.set_xlabel('Target Quantile', fontsize=12, color='#333333')
        ax.set_ylabel('Absolute Error', fontsize=12, color='#333333')
        ax.set_title(f'Error by Target Range - {model_name}', fontsize=14, color='#333333')
        
        charts['error_boxplot'] = _fig_to_base64(fig)
    except Exception as e:
        logger.warning(f"Error boxplot error: {e}")
    
    # 6. PREDICTION OVERVIEW (sorted with bands)
    try:
        fig, ax = plt.subplots(figsize=(12, 6))
        
        idx = np.argsort(y_test)
        y_sorted = y_test[idx]
        pred_sorted = y_pred[idx]
        
        ax.scatter(range(len(y_sorted)), y_sorted, alpha=0.7, color=COLORS[0], s=20, label='Actual')
        ax.scatter(range(len(pred_sorted)), pred_sorted, alpha=0.5, color=COLORS[3], s=15, marker='x', label='Predicted')
        
        std = np.std(pred_sorted - y_sorted)
        ax.fill_between(range(len(y_sorted)), y_sorted - 2*std, y_sorted + 2*std, 
                       alpha=0.2, color=COLORS[1], label='±2σ')
        
        ax.set_xlabel('Sample (sorted)', fontsize=12, color='#333333')
        ax.set_ylabel('Value', fontsize=12, color='#333333')
        ax.set_title(f'Prediction Overview - {model_name}', fontsize=14, color='#333333')
        ax.legend()
        
        charts['prediction_overview'] = _fig_to_base64(fig)
    except Exception as e:
        logger.warning(f"Prediction overview error: {e}")
    
    # 7. SCALE-LOCATION PLOT
    try:
        fig, ax = plt.subplots(figsize=(8, 6))
        
        sqrt_abs_resid = np.sqrt(np.abs(residuals))
        ax.scatter(y_pred, sqrt_abs_resid, alpha=0.6, color=COLORS[4], s=40)
        
        z = np.polyfit(y_pred, sqrt_abs_resid, 1)
        p = np.poly1d(z)
        ax.plot(np.sort(y_pred), p(np.sort(y_pred)), 'r--', lw=2)
        
        ax.set_xlabel('Fitted Values', fontsize=12, color='#333333')
        ax.set_ylabel('√|Residuals|', fontsize=12, color='#333333')
        ax.set_title(f'Scale-Location Plot - {model_name}', fontsize=14, color='#333333')
        
        charts['scale_location'] = _fig_to_base64(fig)
    except Exception as e:
        logger.warning(f"Scale-location error: {e}")
    
    return charts


# =============================================================================
# CLUSTERING CHARTS (8 types)
# =============================================================================

def _generate_clustering_charts(
    X: Optional[np.ndarray],
    labels: np.ndarray,
    model_name: str,
    feature_names: Optional[List[str]] = None
) -> Dict[str, str]:
    """Generate comprehensive clustering charts"""
    charts = {}
    
    if X is None or len(X) < 5:
        return charts
    
    n_clusters = len(np.unique(labels))
    
    # 1. CLUSTER SCATTER (2D PCA)
    try:
        fig, ax = plt.subplots(figsize=(10, 8))
        
        if X.shape[1] >= 2:
            if X.shape[1] > 2:
                from sklearn.decomposition import PCA
                pca = PCA(n_components=2)
                X_2d = pca.fit_transform(X)
                xlabel = f'PC1 ({pca.explained_variance_ratio_[0]*100:.1f}%)'
                ylabel = f'PC2 ({pca.explained_variance_ratio_[1]*100:.1f}%)'
            else:
                X_2d = X
                xlabel = feature_names[0] if feature_names else 'Feature 1'
                ylabel = feature_names[1] if feature_names and len(feature_names) > 1 else 'Feature 2'
            
            for i in range(n_clusters):
                mask = labels == i
                ax.scatter(X_2d[mask, 0], X_2d[mask, 1], c=COLORS[i % len(COLORS)], 
                          s=50, alpha=0.7, label=f'Cluster {i} (n={mask.sum()})',
                          edgecolors='white', linewidth=0.5)
            
            ax.set_xlabel(xlabel, fontsize=12, color='#333333')
            ax.set_ylabel(ylabel, fontsize=12, color='#333333')
            ax.set_title(f'Cluster Visualization - {model_name}', fontsize=14, color='#333333')
            ax.legend()
            
            charts['cluster_scatter'] = _fig_to_base64(fig)
    except Exception as e:
        logger.warning(f"Cluster scatter error: {e}")
    
    # 2. CLUSTER DISTRIBUTION BAR
    try:
        fig, ax = plt.subplots(figsize=(8, 6))
        
        unique, counts = np.unique(labels, return_counts=True)
        bars = ax.bar(range(len(unique)), counts, color=COLORS[:len(unique)], edgecolor='#333333')
        
        for bar, count in zip(bars, counts):
            ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 1,
                   str(count), ha='center', fontsize=10, color='#333333')
        
        ax.set_xticks(range(len(unique)))
        ax.set_xticklabels([f'Cluster {i}' for i in unique])
        ax.set_xlabel('Cluster', fontsize=12, color='#333333')
        ax.set_ylabel('Count', fontsize=12, color='#333333')
        ax.set_title(f'Cluster Sizes - {model_name}', fontsize=14, color='#333333')
        
        charts['cluster_distribution'] = _fig_to_base64(fig)
    except Exception as e:
        logger.warning(f"Cluster distribution error: {e}")
    
    # 3. SILHOUETTE ANALYSIS
    if n_clusters >= 2:
        try:
            sil_vals = silhouette_samples(X, labels)
            sil_avg = silhouette_score(X, labels)
            
            fig, ax = plt.subplots(figsize=(10, 8))
            
            y_lower = 10
            for i in range(n_clusters):
                vals = sil_vals[labels == i]
                vals.sort()
                size = vals.shape[0]
                y_upper = y_lower + size
                
                ax.fill_betweenx(np.arange(y_lower, y_upper), 0, vals,
                                alpha=0.7, color=COLORS[i % len(COLORS)])
                ax.text(-0.05, y_lower + 0.5 * size, f'C{i}', fontsize=10, color='#333333')
                y_lower = y_upper + 10
            
            ax.axvline(x=sil_avg, color='red', linestyle='--', lw=2, label=f'Avg: {sil_avg:.3f}')
            ax.set_xlabel('Silhouette Coefficient', fontsize=12, color='#333333')
            ax.set_ylabel('Cluster', fontsize=12, color='#333333')
            ax.set_title(f'Silhouette Analysis - {model_name}', fontsize=14, color='#333333')
            ax.legend()
            
            charts['silhouette'] = _fig_to_base64(fig)
        except Exception as e:
            logger.warning(f"Silhouette error: {e}")
    
    # 4. FEATURE BOX PLOTS BY CLUSTER
    if feature_names and X.shape[1] >= 1:
        try:
            n_feat = min(4, X.shape[1])
            fig, axes = plt.subplots(2, 2, figsize=(12, 10))
            axes = axes.flatten()
            
            for i in range(n_feat):
                ax = axes[i]
                data = [X[labels == c, i] for c in range(n_clusters)]
                bp = ax.boxplot(data, patch_artist=True)
                
                for j, patch in enumerate(bp['boxes']):
                    patch.set_facecolor(COLORS[j % len(COLORS)])
                    patch.set_alpha(0.7)
                
                ax.set_xticklabels([f'C{c}' for c in range(n_clusters)])
                fname = feature_names[i][:15] if i < len(feature_names) else f'Feature {i}'
                ax.set_title(fname, color='#333333')
            
            for i in range(n_feat, 4):
                axes[i].set_visible(False)
            
            fig.suptitle(f'Feature Distribution by Cluster - {model_name}', fontsize=14, color='#333333')
            plt.tight_layout()
            
            charts['cluster_boxplots'] = _fig_to_base64(fig)
        except Exception as e:
            logger.warning(f"Cluster boxplots error: {e}")
    
    # 5. CLUSTER HISTOGRAMS
    if feature_names and X.shape[1] >= 1:
        try:
            n_feat = min(4, X.shape[1])
            fig, axes = plt.subplots(2, 2, figsize=(12, 10))
            axes = axes.flatten()
            
            for i in range(n_feat):
                ax = axes[i]
                for c in range(n_clusters):
                    ax.hist(X[labels == c, i], bins=15, alpha=0.5, 
                           color=COLORS[c % len(COLORS)], label=f'C{c}', edgecolor='#333333')
                
                fname = feature_names[i][:15] if i < len(feature_names) else f'Feature {i}'
                ax.set_xlabel(fname, color='#333333')
                ax.legend(fontsize=8)
            
            for i in range(n_feat, 4):
                axes[i].set_visible(False)
            
            fig.suptitle(f'Feature Histograms by Cluster - {model_name}', fontsize=14, color='#333333')
            plt.tight_layout()
            
            charts['cluster_histograms'] = _fig_to_base64(fig)
        except Exception as e:
            logger.warning(f"Cluster histograms error: {e}")
    
    return charts


# =============================================================================
# NLP CHARTS (6 types)
# =============================================================================

def _generate_nlp_charts(
    text_data: List[str],
    y: np.ndarray,
    model_name: str
) -> Dict[str, str]:
    """Generate NLP-specific charts"""
    charts = {}
    
    # 1. TEXT LENGTH DISTRIBUTION
    try:
        lengths = [len(str(t).split()) for t in text_data]
        
        fig, ax = plt.subplots(figsize=(10, 6))
        ax.hist(lengths, bins=30, color=COLORS[0], alpha=0.7, edgecolor='#333333')
        ax.axvline(np.mean(lengths), color='red', linestyle='--', label=f'Mean: {np.mean(lengths):.1f}')
        ax.axvline(np.median(lengths), color=COLORS[3], linestyle='--', label=f'Median: {np.median(lengths):.1f}')
        
        ax.set_xlabel('Word Count', fontsize=12, color='#333333')
        ax.set_ylabel('Frequency', fontsize=12, color='#333333')
        ax.set_title(f'Text Length Distribution - {model_name}', fontsize=14, color='#333333')
        ax.legend()
        
        charts['text_length_dist'] = _fig_to_base64(fig)
    except Exception as e:
        logger.warning(f"Text length dist error: {e}")
    
    # 2. TEXT LENGTH BY CLASS
    try:
        unique_classes = np.unique(y)
        lengths = [len(str(t).split()) for t in text_data]
        
        fig, ax = plt.subplots(figsize=(10, 6))
        data = [[lengths[i] for i in range(len(y)) if y[i] == c] for c in unique_classes]
        
        bp = ax.boxplot(data, patch_artist=True)
        for i, patch in enumerate(bp['boxes']):
            patch.set_facecolor(COLORS[i % len(COLORS)])
            patch.set_alpha(0.7)
        
        ax.set_xticklabels([f'Class {c}' for c in unique_classes], rotation=45, ha='right')
        ax.set_xlabel('Class', fontsize=12, color='#333333')
        ax.set_ylabel('Word Count', fontsize=12, color='#333333')
        ax.set_title(f'Text Length by Class - {model_name}', fontsize=14, color='#333333')
        
        charts['text_length_by_class'] = _fig_to_base64(fig)
    except Exception as e:
        logger.warning(f"Text length by class error: {e}")
    
    return charts


# =============================================================================
# COMMON CHARTS
# =============================================================================

def _generate_feature_importance_chart(
    feature_importance: List[Dict],
    model_name: str
) -> Optional[str]:
    """Generate feature importance horizontal bar chart"""
    try:
        if not feature_importance:
            return None
        
        features = feature_importance[:50]  # Show up to 50 features
        names = [f.get('feature', f"Feature {i}")[:25] for i, f in enumerate(features)]
        values = [f.get('importance', 0) * 100 for f in features]
        
        names, values = names[::-1], values[::-1]
        
        fig, ax = plt.subplots(figsize=(10, 8))
        
        colors = [COLORS[i % len(COLORS)] for i in range(len(names))][::-1]
        bars = ax.barh(range(len(names)), values, color=colors, edgecolor='#333333', linewidth=0.5)
        
        ax.set_yticks(range(len(names)))
        ax.set_yticklabels(names, fontsize=10)
        ax.set_xlabel('Importance (%)', fontsize=12, color='#333333')
        ax.set_title(f'Feature Importance - {model_name}', fontsize=14, color='#333333', pad=20)
        
        for bar, val in zip(bars, values):
            ax.text(bar.get_width() + 0.5, bar.get_y() + bar.get_height()/2,
                   f'{val:.1f}%', va='center', fontsize=9, color='#333333')
        
        ax.set_xlim([0, max(values) * 1.15 if values else 100])
        plt.tight_layout()
        
        return _fig_to_base64(fig)
    except Exception as e:
        logger.warning(f"Feature importance error: {e}")
        return None


def generate_model_comparison_chart(
    leaderboard: List[Dict],
    metric_name: str = "accuracy"
) -> Optional[str]:
    """Generate model comparison bar chart"""
    try:
        if not leaderboard or len(leaderboard) < 2:
            return None
        
        models = leaderboard[:10]
        names = [m.get('name', f"Model {i}")[:15] for i, m in enumerate(models)]
        scores = []
        
        for m in models:
            metrics = m.get('metrics', {})
            score = metrics.get(metric_name, metrics.get('accuracy', metrics.get('r2', metrics.get('f1', 0))))
            scores.append(float(score) * 100 if float(score) <= 1 else float(score))
        
        fig, ax = plt.subplots(figsize=(12, 6))
        
        colors = [COLORS[0] if i == 0 else COLORS[2] for i in range(len(names))]
        bars = ax.bar(range(len(names)), scores, color=colors, edgecolor='#333333', linewidth=0.5)
        
        ax.set_xticks(range(len(names)))
        ax.set_xticklabels(names, rotation=45, ha='right', fontsize=10)
        ax.set_ylabel('Score (%)', fontsize=12, color='#333333')
        ax.set_title('Model Comparison', fontsize=14, color='#333333', pad=20)
        
        ax.annotate('🏆 BEST', xy=(0, scores[0]), xytext=(0, scores[0] + 2),
                   ha='center', fontsize=10, color=COLORS[0], fontweight='bold')
        
        for bar, score in zip(bars, scores):
            ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.5,
                   f'{score:.1f}', ha='center', fontsize=9, color='#333333')
        
        plt.tight_layout()
        return _fig_to_base64(fig)
    except Exception as e:
        logger.warning(f"Model comparison error: {e}")
        return None


def generate_correlation_heatmap(df: pd.DataFrame, title: str = "Feature Correlation") -> Optional[str]:
    """Generate correlation heatmap for numeric features"""
    try:
        numeric_df = df.select_dtypes(include=[np.number])
        if numeric_df.shape[1] < 2:
            return None
        
        if numeric_df.shape[1] > 15:
            numeric_df = numeric_df.iloc[:, :15]
        
        corr = numeric_df.corr()
        
        fig, ax = plt.subplots(figsize=(10, 8))
        mask = np.triu(np.ones_like(corr, dtype=bool))
        sns.heatmap(corr, mask=mask, annot=True, fmt='.2f', cmap='RdYlGn',
                   center=0, ax=ax, linewidths=0.5, cbar_kws={'shrink': 0.8})
        
        ax.set_title(title, fontsize=14, color='#333333', pad=20)
        plt.tight_layout()
        
        return _fig_to_base64(fig)
    except Exception as e:
        logger.warning(f"Correlation heatmap error: {e}")
        return None


def generate_distribution_grid(df: pd.DataFrame, title: str = "Feature Distributions") -> Optional[str]:
    """Generate histogram grid for numeric features"""
    try:
        numeric_df = df.select_dtypes(include=[np.number])
        n_features = min(9, numeric_df.shape[1])
        
        if n_features < 1:
            return None
        
        rows = int(np.ceil(n_features / 3))
        fig, axes = plt.subplots(rows, 3, figsize=(12, rows * 3))
        axes = axes.flatten() if hasattr(axes, 'flatten') else [axes]
        
        for i, col in enumerate(numeric_df.columns[:n_features]):
            axes[i].hist(numeric_df[col].dropna(), bins=20, color=COLORS[i % len(COLORS)], 
                        alpha=0.7, edgecolor='#333333')
            axes[i].set_title(col[:20], fontsize=10, color='#333333')
        
        for i in range(n_features, len(axes)):
            axes[i].set_visible(False)
        
        fig.suptitle(title, fontsize=14, color='#333333')
        plt.tight_layout()
        
        return _fig_to_base64(fig)
    except Exception as e:
        logger.warning(f"Distribution grid error: {e}")
        return None


def generate_boxplot_grid(df: pd.DataFrame, title: str = "Feature Box Plots") -> Optional[str]:
    """Generate box plot grid for numeric features"""
    try:
        numeric_df = df.select_dtypes(include=[np.number])
        n_features = min(9, numeric_df.shape[1])
        
        if n_features < 1:
            return None
        
        rows = int(np.ceil(n_features / 3))
        fig, axes = plt.subplots(rows, 3, figsize=(12, rows * 3))
        axes = axes.flatten() if hasattr(axes, 'flatten') else [axes]
        
        for i, col in enumerate(numeric_df.columns[:n_features]):
            bp = axes[i].boxplot(numeric_df[col].dropna(), patch_artist=True)
            bp['boxes'][0].set_facecolor(COLORS[i % len(COLORS)])
            bp['boxes'][0].set_alpha(0.7)
            axes[i].set_title(col[:20], fontsize=10, color='#333333')
        
        for i in range(n_features, len(axes)):
            axes[i].set_visible(False)
        
        fig.suptitle(title, fontsize=14, color='#333333')
        plt.tight_layout()
        
        return _fig_to_base64(fig)
    except Exception as e:
        logger.warning(f"Boxplot grid error: {e}")
        return None


# =============================================================================
# 🌟 ULTRA MODE ADVANCED CHARTS (Premium Visualizations)
# =============================================================================
# These charts are ONLY generated in Ultra mode for advanced insights

# Premium Ultra Color Palette - Rich Gradients
ULTRA_COLORS = {
    'primary': ['#667eea', '#764ba2'],      # Purple Gradient
    'success': ['#11998e', '#38ef7d'],      # Green Gradient  
    'warning': ['#f093fb', '#f5576c'],      # Pink Gradient
    'info': ['#4facfe', '#00f2fe'],         # Blue Gradient
    'danger': ['#ff416c', '#ff4b2b'],       # Red Gradient
    'premium': ['#c471f5', '#fa71cd'],      # Premium Pink
    'gold': ['#f7971e', '#ffd200'],         # Gold Gradient
    'ocean': ['#2193b0', '#6dd5ed'],        # Ocean Blue
}

ULTRA_PALETTE = [
    '#667eea', '#11998e', '#f093fb', '#4facfe', '#ff416c',
    '#c471f5', '#f7971e', '#2193b0', '#764ba2', '#38ef7d'
]


def generate_learning_curve_chart(
    model,
    X: np.ndarray,
    y: np.ndarray,
    model_name: str = "Model",
    cv: int = 5,
    train_sizes: List[float] = None
) -> Optional[str]:
    """
    🎓 Generate Learning Curve chart showing model performance vs training data size.
    
    This is a premium chart that helps identify:
    - Overfitting (high training score, low validation score)
    - Underfitting (both scores low)
    - Ideal model fit (both scores high and converging)
    
    Args:
        model: Trained sklearn model
        X: Feature matrix
        y: Target array
        model_name: Name for chart title
        cv: Number of cross-validation folds
        train_sizes: Training sizes to evaluate (default: [0.1, 0.3, 0.5, 0.7, 0.9, 1.0])
    
    Returns:
        Base64 encoded PNG image or None
    """
    try:
        from sklearn.model_selection import learning_curve
        
        if train_sizes is None:
            train_sizes = [0.1, 0.3, 0.5, 0.7, 0.9, 1.0]
        
        # Limit data size for performance
        max_samples = min(5000, len(X))
        if len(X) > max_samples:
            indices = np.random.choice(len(X), max_samples, replace=False)
            X_sample, y_sample = X[indices], y[indices]
        else:
            X_sample, y_sample = X, y
        
        # Calculate learning curve
        train_sizes_abs, train_scores, val_scores = learning_curve(
            model, X_sample, y_sample,
            train_sizes=train_sizes,
            cv=min(cv, 5),
            n_jobs=-1,
            scoring='accuracy' if hasattr(model, 'predict_proba') else 'r2',
            shuffle=True,
            random_state=42
        )
        
        # Calculate mean and std
        train_mean = np.mean(train_scores, axis=1)
        train_std = np.std(train_scores, axis=1)
        val_mean = np.mean(val_scores, axis=1)
        val_std = np.std(val_scores, axis=1)
        
        # Create chart
        fig, ax = plt.subplots(figsize=(10, 6))
        
        # Plot training scores
        ax.fill_between(train_sizes_abs, train_mean - train_std, train_mean + train_std,
                       alpha=0.15, color=ULTRA_PALETTE[0])
        ax.plot(train_sizes_abs, train_mean, 'o-', color=ULTRA_PALETTE[0], 
               linewidth=2, markersize=8, label='Training Score')
        
        # Plot validation scores
        ax.fill_between(train_sizes_abs, val_mean - val_std, val_mean + val_std,
                       alpha=0.15, color=ULTRA_PALETTE[4])
        ax.plot(train_sizes_abs, val_mean, 's-', color=ULTRA_PALETTE[4], 
               linewidth=2, markersize=8, label='Validation Score')
        
        # Add annotations for final values
        ax.annotate(f'{train_mean[-1]:.3f}', 
                   xy=(train_sizes_abs[-1], train_mean[-1]),
                   xytext=(5, 10), textcoords='offset points',
                   fontsize=10, fontweight='bold', color=ULTRA_PALETTE[0])
        ax.annotate(f'{val_mean[-1]:.3f}', 
                   xy=(train_sizes_abs[-1], val_mean[-1]),
                   xytext=(5, -15), textcoords='offset points',
                   fontsize=10, fontweight='bold', color=ULTRA_PALETTE[4])
        
        # Calculate and display gap (overfitting indicator)
        gap = train_mean[-1] - val_mean[-1]
        gap_color = '#10B981' if gap < 0.05 else ('#F59E0B' if gap < 0.15 else '#EF4444')
        ax.text(0.95, 0.05, f'Gap: {gap:.3f}', transform=ax.transAxes,
               ha='right', va='bottom', fontsize=11, fontweight='bold',
               color=gap_color, bbox=dict(boxstyle='round', facecolor='white', alpha=0.8))
        
        ax.set_xlabel('Training Examples', fontsize=12, color='#333333')
        ax.set_ylabel('Score', fontsize=12, color='#333333')
        ax.set_title(f'📈 Learning Curve - {model_name}', fontsize=14, color='#333333', pad=15)
        ax.legend(loc='lower right', fontsize=10)
        ax.grid(True, alpha=0.3)
        
        # Set reasonable y-axis limits
        min_score = max(0, min(train_mean.min(), val_mean.min()) - 0.1)
        max_score = min(1.0, max(train_mean.max(), val_mean.max()) + 0.1)
        ax.set_ylim(min_score, max_score)
        
        plt.tight_layout()
        return _fig_to_base64(fig)
        
    except Exception as e:
        logger.warning(f"Learning curve generation error: {e}")
        return None


def generate_model_comparison_radar(
    leaderboard: List[Dict],
    metric_names: List[str] = None,
    top_n: int = 5
) -> Optional[str]:
    """
    🎯 Generate multi-model comparison radar chart.
    
    Compares top models across multiple metrics in a radar/spider chart format.
    
    Args:
        leaderboard: List of model results with metrics
        metric_names: Metrics to compare (default: accuracy, precision, recall, f1)
        top_n: Number of top models to include
    
    Returns:
        Base64 encoded PNG image or None
    """
    try:
        if not leaderboard or len(leaderboard) < 2:
            return None
        
        if metric_names is None:
            metric_names = ['accuracy', 'precision', 'recall', 'f1']
        
        # Get top models
        top_models = leaderboard[:min(top_n, len(leaderboard))]
        
        # Extract metrics for each model
        model_data = []
        for model in top_models:
            metrics = model.get('metrics', {})
            values = []
            for metric in metric_names:
                val = metrics.get(metric, 0)
                if isinstance(val, (int, float)):
                    values.append(float(val))
                else:
                    values.append(0)
            model_data.append({
                'name': model.get('name', 'Unknown')[:20],
                'values': values
            })
        
        if not model_data:
            return None
        
        # Create radar chart
        fig, ax = plt.subplots(figsize=(10, 10), subplot_kw=dict(polar=True))
        
        # Calculate angles
        n_metrics = len(metric_names)
        angles = np.linspace(0, 2 * np.pi, n_metrics, endpoint=False).tolist()
        angles += angles[:1]  # Close the polygon
        
        # Plot each model
        for i, model in enumerate(model_data):
            values = model['values'] + [model['values'][0]]  # Close polygon
            color = ULTRA_PALETTE[i % len(ULTRA_PALETTE)]
            
            ax.fill(angles, values, color=color, alpha=0.1)
            ax.plot(angles, values, 'o-', color=color, linewidth=2, 
                   markersize=6, label=model['name'])
        
        # Set labels
        ax.set_xticks(angles[:-1])
        ax.set_xticklabels([m.title() for m in metric_names], size=11, color='#333333')
        ax.set_ylim(0, 1)
        
        # Add legend
        ax.legend(loc='upper right', bbox_to_anchor=(1.3, 1.0), fontsize=9)
        
        ax.set_title('🎯 Model Comparison Radar', size=14, y=1.1, color='#333333', fontweight='bold')
        
        plt.tight_layout()
        return _fig_to_base64(fig)
        
    except Exception as e:
        logger.warning(f"Model comparison radar error: {e}")
        return None


def generate_training_history_chart(
    history: Dict[str, List[float]],
    model_name: str = "Deep Learning Model"
) -> Optional[str]:
    """
    📊 Generate training history chart for deep learning models.
    
    Shows loss and metric curves over training epochs.
    
    Args:
        history: Dictionary with 'loss', 'val_loss', 'accuracy', 'val_accuracy' etc.
        model_name: Name for chart title
    
    Returns:
        Base64 encoded PNG image or None
    """
    try:
        if not history:
            return None
        
        # Create 2x1 subplot for loss and accuracy
        fig, axes = plt.subplots(1, 2, figsize=(14, 5))
        
        epochs = range(1, len(history.get('loss', [])) + 1)
        
        # Left: Loss curves
        ax1 = axes[0]
        if 'loss' in history:
            ax1.plot(epochs, history['loss'], 'o-', color=ULTRA_PALETTE[0], 
                    linewidth=2, markersize=4, label='Training Loss')
        if 'val_loss' in history:
            ax1.plot(epochs, history['val_loss'], 's-', color=ULTRA_PALETTE[4], 
                    linewidth=2, markersize=4, label='Validation Loss')
        
        ax1.set_xlabel('Epoch', fontsize=11)
        ax1.set_ylabel('Loss', fontsize=11)
        ax1.set_title('📉 Training & Validation Loss', fontsize=12, color='#333333')
        ax1.legend()
        ax1.grid(True, alpha=0.3)
        
        # Find best validation loss epoch
        if 'val_loss' in history:
            best_epoch = np.argmin(history['val_loss']) + 1
            ax1.axvline(best_epoch, color='green', linestyle='--', alpha=0.7,
                       label=f'Best: Epoch {best_epoch}')
        
        # Right: Accuracy curves (if available)
        ax2 = axes[1]
        has_accuracy = 'accuracy' in history or 'val_accuracy' in history
        
        if has_accuracy:
            if 'accuracy' in history:
                ax2.plot(epochs, history['accuracy'], 'o-', color=ULTRA_PALETTE[1], 
                        linewidth=2, markersize=4, label='Training Accuracy')
            if 'val_accuracy' in history:
                ax2.plot(epochs, history['val_accuracy'], 's-', color=ULTRA_PALETTE[3], 
                        linewidth=2, markersize=4, label='Validation Accuracy')
            
            ax2.set_xlabel('Epoch', fontsize=11)
            ax2.set_ylabel('Accuracy', fontsize=11)
            ax2.set_title('📈 Training & Validation Accuracy', fontsize=12, color='#333333')
            ax2.legend()
            ax2.grid(True, alpha=0.3)
        else:
            # If no accuracy, show a summary box
            ax2.text(0.5, 0.5, f'Training Complete\n\nFinal Loss: {history.get("loss", [0])[-1]:.4f}',
                    ha='center', va='center', fontsize=14, transform=ax2.transAxes,
                    bbox=dict(boxstyle='round', facecolor=ULTRA_PALETTE[1], alpha=0.3))
            ax2.set_title('📊 Training Summary', fontsize=12, color='#333333')
            ax2.axis('off')
        
        fig.suptitle(f'🧠 Training History - {model_name}', fontsize=14, y=1.02, color='#333333')
        plt.tight_layout()
        
        return _fig_to_base64(fig)
        
    except Exception as e:
        logger.warning(f"Training history chart error: {e}")
        return None



def generate_ultra_charts(
    task_type: str,
    y_test: np.ndarray,
    y_pred: np.ndarray,
    y_proba: Optional[np.ndarray] = None,
    feature_importance: Optional[List[Dict]] = None,
    leaderboard: Optional[List[Dict]] = None,
    model_name: str = "Model",
    X_test: Optional[np.ndarray] = None,
    cv_scores: Optional[List[float]] = None
) -> Dict[str, str]:
    """
    🌟 ULTRA MODE ADVANCED CHARTS
    
    Generates premium visualizations with advanced analytics:
    1. Learning Curve Analysis
    2. Performance Radar Chart
    3. Model Reliability Analysis
    4. Cross-Validation Stability
    5. Prediction Density Plot
    6. SHAP-style Feature Importance
    7. Performance Dashboard
    8. Threshold Analysis (Classification)
    """
    charts = {}
    
    try:
        y_test = np.array(y_test).flatten()
        y_pred = np.array(y_pred).flatten()
        
        task_lower = task_type.lower()
        is_classification = 'classification' in task_lower or 'nlp' in task_lower
        
        # 1. PERFORMANCE RADAR CHART (Multi-metric visualization)
        if is_classification:
            try:
                from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
                
                accuracy = accuracy_score(y_test, y_pred)
                precision = precision_score(y_test, y_pred, average='weighted', zero_division=0)
                recall = recall_score(y_test, y_pred, average='weighted', zero_division=0)
                f1 = f1_score(y_test, y_pred, average='weighted', zero_division=0)
                
                # Calculate specificity (TN / (TN + FP)) for binary
                n_classes = len(np.unique(y_test))
                if n_classes == 2:
                    from sklearn.metrics import confusion_matrix
                    cm = confusion_matrix(y_test, y_pred)
                    specificity = cm[0,0] / (cm[0,0] + cm[0,1]) if (cm[0,0] + cm[0,1]) > 0 else 0
                else:
                    specificity = recall  # Use recall as proxy for multi-class
                
                metrics = ['Accuracy', 'Precision', 'Recall', 'F1-Score', 'Specificity']
                values = [accuracy, precision, recall, f1, specificity]
                
                # Radar chart
                fig, ax = plt.subplots(figsize=(8, 8), subplot_kw=dict(polar=True))
                
                angles = np.linspace(0, 2 * np.pi, len(metrics), endpoint=False).tolist()
                values_plot = values + [values[0]]  # Close the polygon
                angles += angles[:1]
                
                ax.fill(angles, values_plot, color=ULTRA_PALETTE[0], alpha=0.25)
                ax.plot(angles, values_plot, color=ULTRA_PALETTE[0], linewidth=2, marker='o', markersize=8)
                
                ax.set_xticks(angles[:-1])
                ax.set_xticklabels(metrics, size=11, color='#333333')
                ax.set_ylim(0, 1)
                ax.set_title(f'🎯 Performance Radar - {model_name}', size=14, y=1.1, color='#333333')
                
                # Add value annotations
                for angle, value, metric in zip(angles[:-1], values, metrics):
                    ax.annotate(f'{value:.2f}', xy=(angle, value), xytext=(angle, value + 0.1),
                               ha='center', fontsize=9, color=ULTRA_PALETTE[0], fontweight='bold')
                
                charts['ultra_performance_radar'] = _fig_to_base64(fig)
            except Exception as e:
                logger.warning(f"Performance radar error: {e}")
        
        # 2. MODEL RELIABILITY / CALIBRATION ANALYSIS
        if is_classification and y_proba is not None:
            try:
                fig, axes = plt.subplots(1, 2, figsize=(14, 6))
                
                # Left: Reliability diagram with confidence intervals
                ax1 = axes[0]
                n_classes = len(np.unique(y_test))
                
                if n_classes == 2 and len(y_proba.shape) > 1:
                    prob = y_proba[:, 1]
                    
                    # Binned reliability
                    n_bins = 10
                    bins = np.linspace(0, 1, n_bins + 1)
                    bin_indices = np.digitize(prob, bins) - 1
                    bin_indices = np.clip(bin_indices, 0, n_bins - 1)
                    
                    bin_accuracy = []
                    bin_confidence = []
                    bin_counts = []
                    
                    for i in range(n_bins):
                        mask = bin_indices == i
                        if mask.sum() > 0:
                            bin_accuracy.append(y_test[mask].mean())
                            bin_confidence.append(prob[mask].mean())
                            bin_counts.append(mask.sum())
                        else:
                            bin_accuracy.append(np.nan)
                            bin_confidence.append((bins[i] + bins[i+1]) / 2)
                            bin_counts.append(0)
                    
                    # Plot with gradient bars
                    bar_colors = plt.cm.coolwarm(np.array(bin_accuracy))
                    ax1.bar(range(n_bins), bin_accuracy, color=bar_colors, edgecolor='#333333', alpha=0.8)
                    ax1.plot(range(n_bins), bin_confidence, 'k--', lw=2, label='Perfect Calibration')
                    ax1.set_xticks(range(n_bins))
                    ax1.set_xticklabels([f'{bins[i]:.1f}-{bins[i+1]:.1f}' for i in range(n_bins)], rotation=45, ha='right')
                    ax1.set_xlabel('Predicted Probability Bin', fontsize=11)
                    ax1.set_ylabel('Actual Accuracy', fontsize=11)
                    ax1.set_title('📊 Calibration Analysis', fontsize=13, color='#333333')
                    ax1.legend()
                    ax1.set_ylim(0, 1)
                
                # Right: Confidence vs Accuracy scatter
                ax2 = axes[1]
                max_proba = np.max(y_proba, axis=1) if len(y_proba.shape) > 1 else y_proba
                correct = (y_pred == y_test).astype(int)
                
                # Hexbin for density
                hb = ax2.hexbin(max_proba, correct, gridsize=20, cmap='YlGnBu', mincnt=1)
                plt.colorbar(hb, ax=ax2, label='Count')
                ax2.set_xlabel('Prediction Confidence', fontsize=11)
                ax2.set_ylabel('Correct (1) / Incorrect (0)', fontsize=11)
                ax2.set_title('🎯 Confidence vs Correctness', fontsize=13, color='#333333')
                
                plt.tight_layout()
                charts['ultra_model_reliability'] = _fig_to_base64(fig)
            except Exception as e:
                logger.warning(f"Model reliability error: {e}")
        
        # 3. CROSS-VALIDATION STABILITY (if cv_scores provided)
        if cv_scores and len(cv_scores) > 1:
            try:
                fig, ax = plt.subplots(figsize=(10, 6))
                
                folds = range(1, len(cv_scores) + 1)
                mean_score = np.mean(cv_scores)
                std_score = np.std(cv_scores)
                
                # Gradient bars
                colors = plt.cm.viridis(np.linspace(0.3, 0.9, len(cv_scores)))
                bars = ax.bar(folds, cv_scores, color=colors, edgecolor='#333333', linewidth=1.5)
                
                # Mean line
                ax.axhline(mean_score, color='#ff416c', linestyle='--', linewidth=2, 
                          label=f'Mean: {mean_score:.4f}')
                ax.fill_between([0.5, len(cv_scores) + 0.5], mean_score - std_score, mean_score + std_score,
                               alpha=0.2, color='#ff416c', label=f'±1 Std: {std_score:.4f}')
                
                ax.set_xlabel('Fold', fontsize=12)
                ax.set_ylabel('Score', fontsize=12)
                ax.set_title(f'📈 Cross-Validation Stability - {model_name}', fontsize=14, color='#333333')
                ax.set_xticks(folds)
                ax.legend()
                
                # Add value labels
                for bar, score in zip(bars, cv_scores):
                    ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.01,
                           f'{score:.3f}', ha='center', fontsize=9, fontweight='bold')
                
                charts['ultra_cv_stability'] = _fig_to_base64(fig)
            except Exception as e:
                logger.warning(f"CV stability error: {e}")
        
        # 4. PREDICTION DENSITY / JOINTPLOT
        try:
            fig, ax = plt.subplots(figsize=(10, 8))
            
            if is_classification and y_proba is not None and len(y_proba.shape) > 1:
                # For classification: confidence heatmap
                max_proba = np.max(y_proba, axis=1)
                
                # KDE density
                from scipy.stats import gaussian_kde
                xy = np.vstack([y_test, max_proba])
                try:
                    z = gaussian_kde(xy)(xy)
                    scatter = ax.scatter(y_test, max_proba, c=z, s=50, cmap='plasma', alpha=0.7)
                    plt.colorbar(scatter, ax=ax, label='Density')
                except:
                    scatter = ax.scatter(y_test, max_proba, c=ULTRA_PALETTE[0], s=50, alpha=0.6)
                
                ax.set_xlabel('Actual Class', fontsize=12)
                ax.set_ylabel('Prediction Confidence', fontsize=12)
                ax.set_title(f'🌈 Prediction Density - {model_name}', fontsize=14, color='#333333')
            else:
                # For regression: actual vs predicted density
                from scipy.stats import gaussian_kde
                xy = np.vstack([y_test, y_pred])
                try:
                    z = gaussian_kde(xy)(xy)
                    scatter = ax.scatter(y_test, y_pred, c=z, s=50, cmap='viridis', alpha=0.7)
                    plt.colorbar(scatter, ax=ax, label='Density')
                except:
                    scatter = ax.scatter(y_test, y_pred, c=ULTRA_PALETTE[1], s=50, alpha=0.6)
                
                # Perfect line
                min_v, max_v = min(y_test.min(), y_pred.min()), max(y_test.max(), y_pred.max())
                ax.plot([min_v, max_v], [min_v, max_v], 'r--', lw=2, label='Perfect')
                ax.legend()
                ax.set_xlabel('Actual Value', fontsize=12)
                ax.set_ylabel('Predicted Value', fontsize=12)
                ax.set_title(f'🌈 Prediction Density - {model_name}', fontsize=14, color='#333333')
            
            charts['ultra_prediction_density'] = _fig_to_base64(fig)
        except Exception as e:
            logger.warning(f"Prediction density error: {e}")
        
        # 5. SHAP-STYLE FEATURE IMPORTANCE (Premium visualization)
        if feature_importance and len(feature_importance) > 0:
            try:
                fig, ax = plt.subplots(figsize=(12, 8))
                
                top_n = min(15, len(feature_importance))
                features = feature_importance[:top_n]
                names = [f.get('feature', f"Feat {i}")[:30] for i, f in enumerate(features)]
                values = [f.get('importance', 0) * 100 for f in features]
                
                # Reverse for horizontal bar (top feature at top)
                names = names[::-1]
                values = values[::-1]
                
                # Gradient colors based on importance
                colors = plt.cm.plasma(np.linspace(0.2, 0.8, len(names)))[::-1]
                
                bars = ax.barh(range(len(names)), values, color=colors, 
                              edgecolor='#333333', linewidth=0.5, height=0.7)
                
                # Add lollipop markers
                for i, (bar, val) in enumerate(zip(bars, values)):
                    ax.scatter(val, i, s=100, color=colors[i], zorder=5, edgecolors='white', linewidths=2)
                    ax.text(val + max(values)*0.02, i, f'{val:.1f}%', va='center', fontsize=10, fontweight='bold')
                
                ax.set_yticks(range(len(names)))
                ax.set_yticklabels(names, fontsize=10)
                ax.set_xlabel('Importance (%)', fontsize=12)
                ax.set_title(f'🎨 Feature Importance (Premium) - {model_name}', fontsize=14, color='#333333', pad=15)
                ax.set_xlim(0, max(values) * 1.2)
                
                # Add gradient background
                ax.axvspan(0, max(values) * 0.33, alpha=0.1, color='green')
                ax.axvspan(max(values) * 0.33, max(values) * 0.66, alpha=0.1, color='yellow')
                ax.axvspan(max(values) * 0.66, max(values) * 1.2, alpha=0.1, color='red')
                
                plt.tight_layout()
                charts['ultra_feature_importance'] = _fig_to_base64(fig)
            except Exception as e:
                logger.warning(f"SHAP-style importance error: {e}")
        
        # 6. MODEL COMPARISON DASHBOARD (if leaderboard provided)
        if leaderboard and len(leaderboard) > 1:
            try:
                fig, axes = plt.subplots(2, 2, figsize=(14, 10))
                
                # Get model names and metrics
                models = [m.get('name', f"Model {i}")[:15] for i, m in enumerate(leaderboard[:8])]
                
                # Get different metrics
                scores = [m.get('score', 0) for m in leaderboard[:8]]
                accuracies = [m.get('metrics', {}).get('accuracy', 0) for m in leaderboard[:8]]
                f1_scores = [m.get('metrics', {}).get('f1', 0) for m in leaderboard[:8]]
                
                # Top-left: Primary Score
                ax1 = axes[0, 0]
                colors1 = plt.cm.coolwarm(np.linspace(0.2, 0.8, len(models)))
                bars1 = ax1.barh(models, scores, color=colors1, edgecolor='#333333')
                ax1.set_xlabel('Score')
                ax1.set_title('🏆 Model Scores', fontsize=12, color='#333333')
                for bar, score in zip(bars1, scores):
                    ax1.text(bar.get_width() + 0.01, bar.get_y() + bar.get_height()/2,
                            f'{score:.3f}', va='center', fontsize=9)
                
                # Top-right: Accuracy
                ax2 = axes[0, 1]
                colors2 = plt.cm.viridis(np.linspace(0.2, 0.8, len(models)))
                bars2 = ax2.barh(models, accuracies, color=colors2, edgecolor='#333333')
                ax2.set_xlabel('Accuracy')
                ax2.set_title('📊 Accuracy Comparison', fontsize=12, color='#333333')
                
                # Bottom-left: F1 Score
                ax3 = axes[1, 0]
                colors3 = plt.cm.plasma(np.linspace(0.2, 0.8, len(models)))
                bars3 = ax3.barh(models, f1_scores, color=colors3, edgecolor='#333333')
                ax3.set_xlabel('F1 Score')
                ax3.set_title('🎯 F1 Score Comparison', fontsize=12, color='#333333')
                
                # Bottom-right: Performance ranking
                ax4 = axes[1, 1]
                rankings = list(range(1, len(models) + 1))
                scatter = ax4.scatter(scores, f1_scores, c=rankings, s=200, 
                                     cmap='RdYlGn_r', edgecolors='white', linewidths=2)
                for i, model in enumerate(models):
                    ax4.annotate(model, (scores[i], f1_scores[i]), 
                               xytext=(5, 5), textcoords='offset points', fontsize=8)
                ax4.set_xlabel('Score')
                ax4.set_ylabel('F1 Score')
                ax4.set_title('📈 Score vs F1 Trade-off', fontsize=12, color='#333333')
                plt.colorbar(scatter, ax=ax4, label='Rank')
                
                fig.suptitle(f'🌟 ULTRA Model Comparison Dashboard', fontsize=16, y=1.02, color='#333333')
                plt.tight_layout()
                
                charts['ultra_model_dashboard'] = _fig_to_base64(fig)
            except Exception as e:
                logger.warning(f"Model dashboard error: {e}")
        
        # 7. THRESHOLD ANALYSIS (Binary Classification)
        if is_classification and y_proba is not None:
            n_classes = len(np.unique(y_test))
            if n_classes == 2 and len(y_proba.shape) > 1:
                try:
                    from sklearn.metrics import precision_score, recall_score, f1_score as f1_metric
                    
                    fig, ax = plt.subplots(figsize=(10, 6))
                    
                    probs = y_proba[:, 1]
                    thresholds = np.linspace(0.1, 0.9, 17)
                    
                    precisions = []
                    recalls = []
                    f1s = []
                    
                    for thresh in thresholds:
                        pred_thresh = (probs >= thresh).astype(int)
                        precisions.append(precision_score(y_test, pred_thresh, zero_division=0))
                        recalls.append(recall_score(y_test, pred_thresh, zero_division=0))
                        f1s.append(f1_metric(y_test, pred_thresh, zero_division=0))
                    
                    ax.plot(thresholds, precisions, 'o-', color=ULTRA_PALETTE[0], lw=2, label='Precision', markersize=6)
                    ax.plot(thresholds, recalls, 's-', color=ULTRA_PALETTE[1], lw=2, label='Recall', markersize=6)
                    ax.plot(thresholds, f1s, '^-', color=ULTRA_PALETTE[4], lw=2, label='F1-Score', markersize=6)
                    
                    # Mark optimal F1 threshold
                    best_idx = np.argmax(f1s)
                    ax.axvline(thresholds[best_idx], color='gray', linestyle='--', alpha=0.7)
                    ax.scatter([thresholds[best_idx]], [f1s[best_idx]], s=200, color='gold', 
                              edgecolors='black', linewidths=2, zorder=5, label=f'Optimal: {thresholds[best_idx]:.2f}')
                    
                    ax.set_xlabel('Classification Threshold', fontsize=12)
                    ax.set_ylabel('Score', fontsize=12)
                    ax.set_title(f'🔧 Threshold Analysis - {model_name}', fontsize=14, color='#333333')
                    ax.legend(loc='best')
                    ax.set_xlim(0, 1)
                    ax.set_ylim(0, 1.05)
                    ax.grid(True, alpha=0.3)
                    
                    charts['ultra_threshold_analysis'] = _fig_to_base64(fig)
                except Exception as e:
                    logger.warning(f"Threshold analysis error: {e}")
        
        # 8. ERROR DISTRIBUTION VIOLIN (Regression)
        if not is_classification:
            try:
                fig, ax = plt.subplots(figsize=(10, 6))
                
                errors = y_pred - y_test
                abs_errors = np.abs(errors)
                
                # Create quartile groups
                quartiles = np.percentile(y_test, [25, 50, 75])
                groups = []
                group_labels = []
                
                masks = [
                    y_test <= quartiles[0],
                    (y_test > quartiles[0]) & (y_test <= quartiles[1]),
                    (y_test > quartiles[1]) & (y_test <= quartiles[2]),
                    y_test > quartiles[2]
                ]
                
                for i, mask in enumerate(masks):
                    if mask.sum() > 0:
                        groups.append(errors[mask])
                        group_labels.append(f'Q{i+1}')
                
                parts = ax.violinplot(groups, positions=range(len(groups)), showmeans=True, showmedians=True)
                
                for i, pc in enumerate(parts['bodies']):
                    pc.set_facecolor(ULTRA_PALETTE[i % len(ULTRA_PALETTE)])
                    pc.set_alpha(0.7)
                
                ax.set_xticks(range(len(group_labels)))
                ax.set_xticklabels(group_labels)
                ax.axhline(0, color='red', linestyle='--', alpha=0.5)
                ax.set_xlabel('Target Quartile', fontsize=12)
                ax.set_ylabel('Prediction Error', fontsize=12)
                ax.set_title(f'🎻 Error Distribution by Target Range - {model_name}', fontsize=14, color='#333333')
                
                charts['ultra_error_violin'] = _fig_to_base64(fig)
            except Exception as e:
                logger.warning(f"Error violin error: {e}")
        
        logger.info(f"✨ Generated {len(charts)} ULTRA charts: {list(charts.keys())}")
        
    except Exception as e:
        logger.error(f"Ultra chart generation error: {e}")
        import traceback
        traceback.print_exc()
    
    return charts


# =============================================================================
# 📈 STOCK/FINANCIAL DATA CHARTS
# =============================================================================

def detect_stock_data(df: pd.DataFrame) -> Dict[str, Any]:
    """
    Detect if the DataFrame contains stock/financial data.
    Returns detected column mappings for OHLCV data.
    """
    result = {
        'is_stock_data': False,
        'date_col': None,
        'open_col': None,
        'high_col': None,
        'low_col': None,
        'close_col': None,
        'volume_col': None,
        'adj_close_col': None,
        'price_cols': [],
        'has_ohlc': False
    }
    
    cols_lower = {col.lower().strip(): col for col in df.columns}
    
    # Detect date column
    date_keywords = ['date', 'time', 'timestamp', 'datetime', 'period']
    for kw in date_keywords:
        for col_lower, col in cols_lower.items():
            if kw in col_lower:
                result['date_col'] = col
                break
        if result['date_col']:
            break
    
    # Also check for datetime dtype columns
    if not result['date_col']:
        for col in df.columns:
            if pd.api.types.is_datetime64_any_dtype(df[col]):
                result['date_col'] = col
                break
    
    # Detect OHLCV columns
    ohlcv_mappings = {
        'open': ['open', 'opening', 'open_price'],
        'high': ['high', 'highest', 'high_price', 'max_price'],
        'low': ['low', 'lowest', 'low_price', 'min_price'],
        'close': ['close', 'closing', 'close_price', 'last', 'price'],
        'volume': ['volume', 'vol', 'qty', 'quantity', 'traded_volume'],
        'adj_close': ['adj close', 'adj_close', 'adjusted close', 'adjusted_close']
    }
    
    for field, keywords in ohlcv_mappings.items():
        for kw in keywords:
            for col_lower, col in cols_lower.items():
                if kw == col_lower or kw in col_lower:
                    if field == 'open':
                        result['open_col'] = col
                    elif field == 'high':
                        result['high_col'] = col
                    elif field == 'low':
                        result['low_col'] = col
                    elif field == 'close':
                        result['close_col'] = col
                    elif field == 'volume':
                        result['volume_col'] = col
                    elif field == 'adj_close':
                        result['adj_close_col'] = col
                    break
    
    # Check if we have OHLC data
    result['has_ohlc'] = all([
        result['open_col'], result['high_col'], 
        result['low_col'], result['close_col']
    ])
    
    # Collect all price-like columns
    price_keywords = ['price', 'close', 'open', 'high', 'low', 'value', 'rate']
    for col_lower, col in cols_lower.items():
        for kw in price_keywords:
            if kw in col_lower and pd.api.types.is_numeric_dtype(df[col]):
                if col not in result['price_cols']:
                    result['price_cols'].append(col)
    
    # Determine if it's stock data
    stock_indicators = 0
    if result['has_ohlc']:
        stock_indicators += 3
    if result['volume_col']:
        stock_indicators += 1
    if result['date_col']:
        stock_indicators += 1
    if len(result['price_cols']) >= 2:
        stock_indicators += 1
    
    # Also check column names for stock-related terms
    stock_terms = ['stock', 'ticker', 'symbol', 'share', 'equity', 'nifty', 'sensex', 'nasdaq', 'spy', 'return']
    for col_lower in cols_lower.keys():
        if any(term in col_lower for term in stock_terms):
            stock_indicators += 1
            break
    
    result['is_stock_data'] = stock_indicators >= 3
    
    return result


def generate_stock_charts(
    df: pd.DataFrame,
    stock_info: Dict[str, Any],
    target_col: Optional[str] = None
) -> Dict[str, str]:
    """
    Generate stock/financial specific charts.
    Returns dict of chart_name -> base64 encoded image.
    """
    charts = {}
    
    try:
        # Ensure date column is datetime
        date_col = stock_info.get('date_col')
        if date_col and date_col in df.columns:
            df = df.copy()
            df[date_col] = pd.to_datetime(df[date_col], errors='coerce')
            df = df.dropna(subset=[date_col])
            df = df.sort_values(date_col)
        
        # 1. PRICE TREND CHART (Line chart with price over time)
        price_col = stock_info.get('close_col') or stock_info.get('adj_close_col')
        if date_col and price_col and price_col in df.columns:
            try:
                fig, ax = plt.subplots(figsize=(12, 6))
                
                ax.plot(df[date_col], df[price_col], linewidth=2, color='#2196F3', label='Price')
                
                # Add moving averages
                if len(df) >= 20:
                    ma20 = df[price_col].rolling(window=20).mean()
                    ax.plot(df[date_col], ma20, linewidth=1.5, color='#FF9800', 
                            label='20-Day MA', linestyle='--')
                
                if len(df) >= 50:
                    ma50 = df[price_col].rolling(window=50).mean()
                    ax.plot(df[date_col], ma50, linewidth=1.5, color='#E91E63', 
                            label='50-Day MA', linestyle='-.')
                
                ax.fill_between(df[date_col], df[price_col], alpha=0.1, color='#2196F3')
                ax.set_xlabel('Date', fontsize=12)
                ax.set_ylabel('Price', fontsize=12)
                ax.set_title('📈 Stock Price Trend with Moving Averages', fontsize=14, fontweight='bold')
                ax.legend(loc='upper left')
                ax.grid(True, alpha=0.3)
                plt.xticks(rotation=45)
                plt.tight_layout()
                
                charts['stock_price_trend'] = _fig_to_base64(fig)
            except Exception as e:
                logger.warning(f"Price trend chart error: {e}")
        
        # 2. CANDLESTICK CHART (if OHLC data available)
        if stock_info.get('has_ohlc') and date_col:
            try:
                from mplfinance.original_flavor import candlestick_ohlc
                import matplotlib.dates as mdates
                
                fig, ax = plt.subplots(figsize=(14, 7))
                
                # Prepare data for candlestick
                df_ohlc = df[[date_col, stock_info['open_col'], stock_info['high_col'], 
                              stock_info['low_col'], stock_info['close_col']]].copy()
                df_ohlc.columns = ['Date', 'Open', 'High', 'Low', 'Close']
                df_ohlc['Date'] = mdates.date2num(df_ohlc['Date'])
                
                # Limit to last 60 candles for readability
                df_plot = df_ohlc.tail(60)
                
                candlestick_ohlc(ax, df_plot.values, width=0.6, 
                                colorup='#26A69A', colordown='#EF5350')
                
                ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d'))
                ax.xaxis.set_major_locator(mdates.WeekdayLocator(interval=2))
                plt.xticks(rotation=45)
                ax.set_xlabel('Date', fontsize=12)
                ax.set_ylabel('Price', fontsize=12)
                ax.set_title('🕯️ Candlestick Chart (Last 60 Periods)', fontsize=14, fontweight='bold')
                ax.grid(True, alpha=0.3)
                plt.tight_layout()
                
                charts['stock_candlestick'] = _fig_to_base64(fig)
            except ImportError:
                # Fallback: Create a simple OHLC chart without mplfinance
                try:
                    fig, ax = plt.subplots(figsize=(14, 7))
                    
                    df_plot = df.tail(60).copy()
                    x = range(len(df_plot))
                    
                    # Plot high-low range as lines
                    for i, (_, row) in enumerate(df_plot.iterrows()):
                        color = '#26A69A' if row[stock_info['close_col']] >= row[stock_info['open_col']] else '#EF5350'
                        ax.vlines(i, row[stock_info['low_col']], row[stock_info['high_col']], color=color, linewidth=1)
                        ax.vlines(i, row[stock_info['open_col']], row[stock_info['close_col']], color=color, linewidth=4)
                    
                    ax.set_xlabel('Period', fontsize=12)
                    ax.set_ylabel('Price', fontsize=12)
                    ax.set_title('🕯️ OHLC Chart (Last 60 Periods)', fontsize=14, fontweight='bold')
                    ax.grid(True, alpha=0.3)
                    plt.tight_layout()
                    
                    charts['stock_ohlc'] = _fig_to_base64(fig)
                except Exception as e:
                    logger.warning(f"OHLC fallback chart error: {e}")
            except Exception as e:
                logger.warning(f"Candlestick chart error: {e}")
        
        # 3. VOLUME CHART
        volume_col = stock_info.get('volume_col')
        if date_col and volume_col and volume_col in df.columns:
            try:
                fig, ax = plt.subplots(figsize=(12, 4))
                
                df_plot = df.tail(100)
                colors = ['#26A69A' if df_plot[stock_info.get('close_col', price_col)].iloc[i] >= 
                         df_plot[stock_info.get('open_col', price_col)].iloc[i] 
                         else '#EF5350' for i in range(len(df_plot))]
                
                ax.bar(range(len(df_plot)), df_plot[volume_col], color=colors, alpha=0.7)
                ax.set_xlabel('Period', fontsize=12)
                ax.set_ylabel('Volume', fontsize=12)
                ax.set_title('📊 Trading Volume (Last 100 Periods)', fontsize=14, fontweight='bold')
                ax.grid(True, alpha=0.3, axis='y')
                
                # Format y-axis for large numbers
                ax.yaxis.set_major_formatter(plt.FuncFormatter(lambda x, p: f'{x/1e6:.1f}M' if x >= 1e6 else f'{x/1e3:.0f}K' if x >= 1e3 else f'{x:.0f}'))
                plt.tight_layout()
                
                charts['stock_volume'] = _fig_to_base64(fig)
            except Exception as e:
                logger.warning(f"Volume chart error: {e}")
        
        # 4. DAILY RETURNS DISTRIBUTION
        if price_col and price_col in df.columns:
            try:
                fig, axes = plt.subplots(1, 2, figsize=(14, 5))
                
                # Calculate returns
                returns = df[price_col].pct_change().dropna() * 100
                
                # Histogram of returns
                axes[0].hist(returns, bins=50, color='#2196F3', alpha=0.7, edgecolor='white')
                axes[0].axvline(returns.mean(), color='red', linestyle='--', label=f'Mean: {returns.mean():.2f}%')
                axes[0].axvline(0, color='black', linestyle='-', alpha=0.5)
                axes[0].set_xlabel('Daily Return (%)', fontsize=12)
                axes[0].set_ylabel('Frequency', fontsize=12)
                axes[0].set_title('📊 Daily Returns Distribution', fontsize=14, fontweight='bold')
                axes[0].legend()
                axes[0].grid(True, alpha=0.3)
                
                # Cumulative returns
                cumulative_returns = (1 + returns/100).cumprod() - 1
                axes[1].plot(range(len(cumulative_returns)), cumulative_returns * 100, 
                            color='#4CAF50', linewidth=2)
                axes[1].fill_between(range(len(cumulative_returns)), 
                                    cumulative_returns * 100, alpha=0.2, color='#4CAF50')
                axes[1].axhline(0, color='black', linestyle='-', alpha=0.5)
                axes[1].set_xlabel('Period', fontsize=12)
                axes[1].set_ylabel('Cumulative Return (%)', fontsize=12)
                axes[1].set_title('📈 Cumulative Returns', fontsize=14, fontweight='bold')
                axes[1].grid(True, alpha=0.3)
                
                plt.tight_layout()
                charts['stock_returns'] = _fig_to_base64(fig)
            except Exception as e:
                logger.warning(f"Returns chart error: {e}")
        
        # 5. VOLATILITY ANALYSIS
        if price_col and price_col in df.columns and len(df) >= 30:
            try:
                fig, ax = plt.subplots(figsize=(12, 5))
                
                returns = df[price_col].pct_change().dropna()
                
                # Rolling volatility (20-day)
                rolling_vol = returns.rolling(window=20).std() * np.sqrt(252) * 100  # Annualized
                
                ax.plot(range(len(rolling_vol)), rolling_vol, color='#9C27B0', linewidth=2)
                ax.fill_between(range(len(rolling_vol)), rolling_vol, alpha=0.2, color='#9C27B0')
                ax.axhline(rolling_vol.mean(), color='red', linestyle='--', 
                          label=f'Average: {rolling_vol.mean():.1f}%')
                ax.set_xlabel('Period', fontsize=12)
                ax.set_ylabel('Annualized Volatility (%)', fontsize=12)
                ax.set_title('📉 Rolling Volatility (20-Day Window)', fontsize=14, fontweight='bold')
                ax.legend()
                ax.grid(True, alpha=0.3)
                plt.tight_layout()
                
                charts['stock_volatility'] = _fig_to_base64(fig)
            except Exception as e:
                logger.warning(f"Volatility chart error: {e}")
        
        # 6. PRICE WITH BOLLINGER BANDS
        if date_col and price_col and price_col in df.columns and len(df) >= 20:
            try:
                fig, ax = plt.subplots(figsize=(12, 6))
                
                df_plot = df.tail(100).copy()
                
                # Calculate Bollinger Bands
                ma20 = df_plot[price_col].rolling(window=20).mean()
                std20 = df_plot[price_col].rolling(window=20).std()
                upper_band = ma20 + (std20 * 2)
                lower_band = ma20 - (std20 * 2)
                
                ax.plot(range(len(df_plot)), df_plot[price_col], color='#2196F3', 
                       linewidth=2, label='Price')
                ax.plot(range(len(df_plot)), ma20, color='#FF9800', 
                       linewidth=1.5, label='20-Day MA')
                ax.fill_between(range(len(df_plot)), upper_band, lower_band, 
                               alpha=0.2, color='#9C27B0', label='Bollinger Bands (2σ)')
                
                ax.set_xlabel('Period', fontsize=12)
                ax.set_ylabel('Price', fontsize=12)
                ax.set_title('📊 Price with Bollinger Bands', fontsize=14, fontweight='bold')
                ax.legend(loc='upper left')
                ax.grid(True, alpha=0.3)
                plt.tight_layout()
                
                charts['stock_bollinger'] = _fig_to_base64(fig)
            except Exception as e:
                logger.warning(f"Bollinger bands chart error: {e}")
        
        # 7. CORRELATION WITH OTHER PRICE COLUMNS
        price_cols = stock_info.get('price_cols', [])
        if len(price_cols) >= 2:
            try:
                fig, ax = plt.subplots(figsize=(8, 6))
                
                price_df = df[price_cols].dropna()
                if len(price_df) > 10:
                    corr_matrix = price_df.corr()
                    
                    sns.heatmap(corr_matrix, annot=True, fmt='.2f', cmap='RdYlGn', 
                               center=0, ax=ax, square=True,
                               linewidths=0.5, cbar_kws={'shrink': 0.8})
                    ax.set_title('🔗 Price Correlations', fontsize=14, fontweight='bold')
                    plt.tight_layout()
                    
                    charts['stock_price_correlation'] = _fig_to_base64(fig)
            except Exception as e:
                logger.warning(f"Price correlation chart error: {e}")
        
        logger.info(f"📈 Generated {len(charts)} stock charts: {list(charts.keys())}")
        
    except Exception as e:
        logger.error(f"Stock chart generation error: {e}")
        import traceback
        traceback.print_exc()
    
    return charts