backend/api/v1/endpoints/clustering_api.py

"""
🎯 Clustering API - Unsupervised Machine Learning (Production AutoML)
=====================================================================
Real AutoML Unsupervised Learning with:
- Multiple algorithm comparison (KMeans, DBSCAN, GMM, Hierarchical, Spectral, K-Prototypes)
- Automatic optimal k detection (elbow + silhouette)
- Model persistence for predictions
- Comprehensive visualization & charts
- Production intelligence & reliability scoring

🛡️ PRODUCTION INTELLIGENCE INTEGRATED:
- Data quality assessment
- Feature validation
- Missing data handling
- Reliability scoring for cluster quality
- Validation warnings
"""

from fastapi import APIRouter, HTTPException, UploadFile, File, Form, Header
from pydantic import BaseModel
from typing import Optional, Dict, Any, List
import pandas as pd
import numpy as np
from sklearn.cluster import KMeans, DBSCAN, SpectralClustering, AgglomerativeClustering
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.metrics import silhouette_score, davies_bouldin_score, calinski_harabasz_score
import io
import logging
import pickle
import uuid
import json
from pathlib import Path
from datetime import datetime
import base64

# Import production clustering engine
try:
    from ml.clustering_engine import ProductionClusteringEngine, clustering_engine
except ImportError:
    clustering_engine = None

# Import user paths utility
try:
    from utils.paths import get_user_paths
except ImportError:
    def get_user_paths(user_id):
        base = Path("storage/users") / user_id
        paths = {"base": base, "files": base / "files", "models": base / "models"}
        for p in paths.values():
            p.mkdir(parents=True, exist_ok=True)
        return paths

logger = logging.getLogger(__name__)
router = APIRouter()


# ===========================================================================
# REQUEST/RESPONSE MODELS
# ===========================================================================

class ClusteringRequest(BaseModel):
    """Request model for clustering from file_id"""
    file_id: str
    user_id: Optional[str] = None
    algorithm: str = "auto"  # auto, kmeans, dbscan, hierarchical, gmm, spectral
    n_clusters: Optional[int] = None  # None = auto-detect
    compare_all: bool = False  # Compare all algorithms
    exclude_columns: Optional[List[str]] = None


class ClusterPredictRequest(BaseModel):
    """Request model for predicting cluster of new data point"""
    user_id: Optional[str] = None
    model_id: str
    features: Dict[str, float]


# ===========================================================================
# 🎯 MAIN CLUSTERING ENDPOINT - JSON API (Used by frontend)
# ===========================================================================

@router.post("/clustering")
async def run_clustering_analysis(
    request: ClusteringRequest,
    x_user_id: Optional[str] = Header(None, alias="X-User-ID")
):
    """
    🎯 PRODUCTION AutoML Unsupervised Learning
    
    Takes a file_id and runs real clustering analysis with:
    - Automatic algorithm selection based on data type
    - Optimal k detection via silhouette analysis
    - Multiple algorithm comparison (optional)
    - Model persistence for predictions
    - Comprehensive visualizations
    
    Request Body:
    - file_id: User's file name/id
    - algorithm: 'auto', 'kmeans', 'dbscan', 'hierarchical', 'gmm', 'spectral'
    - n_clusters: Number of clusters (auto-detect if None)
    - compare_all: Run all algorithms and compare
    - exclude_columns: Columns to exclude from clustering
    """
    user_id = request.user_id or x_user_id
    if not user_id:
        raise HTTPException(status_code=400, detail="User ID required")
    
    logger.info(f"🎯 Clustering request: user={user_id}, file={request.file_id}, algo={request.algorithm}")
    
    try:
        # 1. Load user's file
        paths = get_user_paths(user_id)
        files_dir = paths.get("files", paths["base"] / "files")
        
        # Try multiple file path patterns (CSV + Excel)
        file_path = None
        search_patterns = [
            files_dir / request.file_id,
            files_dir / f"{request.file_id}.csv",
            files_dir / f"{request.file_id}.xlsx",
            files_dir / f"{request.file_id}.xls",
            Path(f"storage/users/{user_id}/files/{request.file_id}"),
            Path(f"storage/users/{user_id}/files/{request.file_id}.csv"),
            Path(f"storage/users/{user_id}/files/{request.file_id}.xlsx"),
            Path(f"storage/users/{user_id}/files/{request.file_id}.xls"),
            Path(f"backend/storage/users/{user_id}/files/{request.file_id}"),
            Path(f"backend/storage/users/{user_id}/files/{request.file_id}.csv"),
            Path(f"backend/storage/users/{user_id}/files/{request.file_id}.xlsx"),
        ]
        
        for pattern in search_patterns:
            if pattern.exists():
                file_path = pattern
                break
        
        # Fallback: glob search in user's files directory for partial match
        if not file_path and files_dir.exists():
            file_stem = Path(request.file_id).stem  # Remove extension if present
            for ext in ['*.csv', '*.xlsx', '*.xls', '*.tsv']:
                matches = list(files_dir.glob(ext))
                for m in matches:
                    if file_stem.lower() in m.stem.lower() or m.name == request.file_id:
                        file_path = m
                        break
                if file_path:
                    break
            # Last resort: just pick any data file if only one exists
            if not file_path:
                all_data_files = [f for f in files_dir.iterdir() 
                                  if f.is_file() and f.suffix.lower() in ('.csv', '.xlsx', '.xls', '.tsv')
                                  and not f.name.startswith('cleaned_') and not f.name.startswith('clustered_')]
                if len(all_data_files) == 1:
                    file_path = all_data_files[0]
                    logger.info(f"📂 Auto-selected only available file: {file_path.name}")
        
        if not file_path:
            raise HTTPException(
                status_code=404, 
                detail=f"File not found: {request.file_id}. Upload a file in DataHub first."
            )
        
        logger.info(f"📂 Found file: {file_path}")
        
        # 2. Read data — support CSV and Excel
        file_ext = file_path.suffix.lower()
        if file_ext in ('.xlsx', '.xls'):
            df = pd.read_excel(file_path, engine='openpyxl' if file_ext == '.xlsx' else None)
        elif file_ext == '.tsv':
            df = pd.read_csv(file_path, sep='\t')
        else:
            df = pd.read_csv(file_path)
        logger.info(f"📊 Loaded {len(df)} rows x {len(df.columns)} columns from {file_ext}")
        
        if len(df) < 10:
            raise HTTPException(status_code=400, detail="Need at least 10 rows for clustering")
        
        # 3. Prepare exclude columns
        exclude_cols = request.exclude_columns or []
        
        # Auto-exclude obvious ID/datetime columns
        for col in df.columns:
            col_lower = col.lower()
            if any(x in col_lower for x in ['id', 'index', 'key', 'timestamp', 'date', 'time', 'created', 'updated']):
                if col not in exclude_cols:
                    exclude_cols.append(col)
        
        # 4. Run clustering with production engine or fallback
        result = await _run_production_clustering(
            df=df,
            algorithm=request.algorithm,
            n_clusters=request.n_clusters,
            exclude_columns=exclude_cols,
            compare_all=request.compare_all,
            user_id=user_id
        )
        
        return result
        
    except HTTPException:
        raise
    except FileNotFoundError as e:
        raise HTTPException(status_code=404, detail=str(e))
    except Exception as e:
        logger.error(f"❌ Clustering failed: {e}")
        import traceback
        traceback.print_exc()
        raise HTTPException(status_code=500, detail=f"Clustering error: {str(e)}")


async def _run_production_clustering(
    df: pd.DataFrame,
    algorithm: str,
    n_clusters: Optional[int],
    exclude_columns: List[str],
    compare_all: bool,
    user_id: str
) -> Dict[str, Any]:
    """
    Core clustering logic using production-grade algorithms.
    """
    import math
    
    # Prepare numeric data
    df_clean = df.drop(columns=exclude_columns, errors='ignore')
    
    # Drop datetime columns
    datetime_cols = df_clean.select_dtypes(include=['datetime64']).columns.tolist()
    df_clean = df_clean.drop(columns=datetime_cols, errors='ignore')
    
    # Get numeric columns only for clustering
    numeric_df = df_clean.select_dtypes(include=[np.number])
    
    if numeric_df.empty or len(numeric_df.columns) < 2:
        raise HTTPException(status_code=400, detail="Need at least 2 numeric columns for clustering")
    
    # Fill missing values
    numeric_df = numeric_df.fillna(numeric_df.median())
    
    # Drop zero-variance columns
    zero_var_cols = numeric_df.columns[numeric_df.std() == 0].tolist()
    numeric_df = numeric_df.drop(columns=zero_var_cols, errors='ignore')
    
    feature_columns = numeric_df.columns.tolist()
    X = numeric_df.values
    
    logger.info(f"📊 Clustering {len(X)} samples with {len(feature_columns)} features")
    
    # Scale data
    scaler = StandardScaler()
    X_scaled = scaler.fit_transform(X)
    
    # Initialize k_scores for elbow method chart
    k_scores = None
    
    # =======================================================================
    # AUTO-DETECT OPTIMAL K (if not provided)
    # =======================================================================
    if n_clusters is None and algorithm not in ['dbscan']:
        n_clusters, k_scores = _find_optimal_k(X_scaled, max_k=min(15, len(X) // 10))
        logger.info(f"✅ Auto-detected optimal k={n_clusters}")
    elif n_clusters is None:
        n_clusters = 3  # Default for DBSCAN (will auto-detect anyway)
    
    # =======================================================================
    # RUN CLUSTERING (compare all or single algorithm)
    # =======================================================================
    if compare_all:
        # Run all algorithms and compare
        all_results = _compare_all_algorithms(X_scaled, n_clusters)
        best_algo = max(all_results, key=lambda x: all_results[x]['silhouette_score'])
        best_result = all_results[best_algo]
        labels = np.array(best_result['labels'])
        algorithm = best_algo
        comparison_results = all_results
    else:
        # Single algorithm
        if algorithm == 'auto':
            algorithm = 'kmeans'  # Default to kmeans for numeric data
        
        labels, model, metrics = _run_single_clustering(X_scaled, algorithm, n_clusters)
        comparison_results = None
    
    # =======================================================================
    # CALCULATE METRICS
    # =======================================================================
    metrics = _calculate_clustering_metrics(X_scaled, labels)
    
    # Cluster distribution
    unique_labels, counts = np.unique(labels, return_counts=True)
    cluster_distribution = {
        f"Cluster {int(k)}" if k != -1 else "Noise": int(v)
        for k, v in zip(unique_labels, counts)
    }
    
    actual_n_clusters = len([l for l in unique_labels if l != -1])
    
    # =======================================================================
    # PCA FOR VISUALIZATION
    # =======================================================================
    pca = PCA(n_components=2)
    X_2d = pca.fit_transform(X_scaled)
    pca_variance = sum(pca.explained_variance_ratio_)
    
    # =======================================================================
    # COMPUTE FEATURE STATISTICS PER CLUSTER
    # =======================================================================
    feature_stats = {}
    for col in feature_columns:
        col_idx = feature_columns.index(col)
        stats = {
            'mean': float(numeric_df[col].mean()),
            'std': float(numeric_df[col].std()),
            'min': float(numeric_df[col].min()),
            'max': float(numeric_df[col].max()),
        }
        feature_stats[col] = stats
    
    # Cluster centroids in original scale
    cluster_profiles = {}
    for cluster_id in unique_labels:
        if cluster_id == -1:
            continue
        mask = labels == cluster_id
        cluster_data = numeric_df.values[mask]
        profile = {
            'size': int(mask.sum()),
            'percentage': float(mask.sum() / len(labels) * 100),
            'characteristics': {},
        }
        for i, col in enumerate(feature_columns):
            profile['characteristics'][col] = {
                'mean': float(np.mean(cluster_data[:, i])),
                'std': float(np.std(cluster_data[:, i])),
            }
        cluster_profiles[f"Cluster {cluster_id}"] = profile
    
    # =======================================================================
    # GENERATE COMPREHENSIVE CHARTS
    # =======================================================================
    charts = _generate_clustering_charts(
        X_2d=X_2d,
        labels=labels,
        algorithm=algorithm,
        n_clusters=actual_n_clusters,
        silhouette=metrics['silhouette_score'],
        feature_names=feature_columns,
        cluster_profiles=cluster_profiles,
        X_scaled=X_scaled,
        X_original=numeric_df.values,
        k_scores=k_scores
    )
    
    # =======================================================================
    # RELIABILITY SCORE (Production Intelligence)
    # =======================================================================
    reliability_score, validation_warnings = _compute_reliability_score(
        n_samples=len(X),
        n_features=len(feature_columns),
        silhouette=metrics['silhouette_score'],
        calinski=metrics.get('calinski_harabasz_score', 0),
        davies=metrics.get('davies_bouldin_score', float('inf')),
        df=df
    )
    
    # =======================================================================
    # SAVE MODEL FOR PREDICTIONS
    # =======================================================================
    model_id = f"clustering_{uuid.uuid4().hex[:8]}"
    model_data = {
        'algorithm': algorithm,
        'n_clusters': actual_n_clusters,
        'scaler_mean': scaler.mean_.tolist(),
        'scaler_scale': scaler.scale_.tolist(),
        'feature_columns': feature_columns,
        'labels': labels.tolist(),
        'centroids_scaled': _get_cluster_centroids(X_scaled, labels).tolist() if actual_n_clusters > 0 else [],
        'created_at': datetime.now().isoformat(),
        'silhouette_score': metrics['silhouette_score'],
    }
    
    # Save to user's model directory
    _save_clustering_model(user_id, model_id, model_data)
    
    # =======================================================================
    # SAVE CLEANED DATA WITH CLUSTER LABELS + PKL MODEL
    # =======================================================================
    cleaned_file = None
    model_pkl_file = None
    
    # Get user paths (must call here since this is a separate function)
    paths = get_user_paths(user_id)
    
    try:
        # Create cleaned dataframe with cluster assignments
        cleaned_df = df.copy()
        cleaned_df['Cluster'] = labels
        cleaned_df['Cluster_Name'] = [f'Cluster_{l}' if l >= 0 else 'Noise' for l in labels]
        
        # Add PCA components for visualization
        cleaned_df['PCA_1'] = X_2d[:, 0]
        cleaned_df['PCA_2'] = X_2d[:, 1]
        
        # Save cleaned CSV
        files_dir = paths.get("files", paths["base"] / "files")
        files_dir.mkdir(parents=True, exist_ok=True)
        cleaned_filename = f"clustered_data_{model_id}.csv"
        cleaned_path = files_dir / cleaned_filename
        cleaned_df.to_csv(cleaned_path, index=False)
        cleaned_file = cleaned_filename
        logger.info(f"✅ Saved cleaned data: {cleaned_path}")
        
        # Save PKL model file
        import pickle
        pkl_filename = f"clustering_model_{model_id}.pkl"
        models_dir = paths.get("models", paths["base"] / "models")
        models_dir.mkdir(parents=True, exist_ok=True)
        pkl_path = models_dir / pkl_filename
        
        pkl_data = {
            'algorithm': algorithm,
            'n_clusters': actual_n_clusters,
            'scaler': scaler,
            'feature_columns': feature_columns,
            'centroids_scaled': _get_cluster_centroids(X_scaled, labels) if actual_n_clusters > 0 else None,
            'labels': labels,
            'model_id': model_id,
            'created_at': datetime.now().isoformat(),
            'silhouette_score': metrics['silhouette_score'],
            'cluster_profiles': cluster_profiles,
        }
        
        with open(pkl_path, 'wb') as f:
            pickle.dump(pkl_data, f)
        model_pkl_file = pkl_filename
        logger.info(f"✅ Saved PKL model: {pkl_path}")
        
    except Exception as e:
        logger.error(f"❌ Failed to save cleaned data/PKL: {e}")
        import traceback
        traceback.print_exc()
    
    # =======================================================================
    # SAVE CLUSTERING CHARTS TO DISK (for ZIP download)
    # =======================================================================
    if charts:
        try:
            charts_json_path = paths.get("models", paths["base"] / "models") / "active_clustering_charts.json"
            charts_json_path.parent.mkdir(parents=True, exist_ok=True)
            with open(charts_json_path, 'w') as f:
                json.dump(charts, f)
            logger.info(f"✅ Saved {len(charts)} clustering charts to {charts_json_path}")
        except Exception as e:
            logger.warning(f"Could not save clustering charts: {e}")
    
    # =======================================================================
    # 🔬 PRODUCTION ML ENGINEERING FEATURES
    # =======================================================================
    
    # --- Anomaly Detection (Isolation Forest) ---
    anomaly_results = None
    try:
        from sklearn.ensemble import IsolationForest
        iso_forest = IsolationForest(n_estimators=100, contamination=0.05, random_state=42)
        anomaly_labels = iso_forest.fit_predict(X_scaled)
        anomaly_scores = iso_forest.decision_function(X_scaled)
        n_anomalies = int(np.sum(anomaly_labels == -1))
        anomaly_results = {
            'n_anomalies': n_anomalies,
            'anomaly_percentage': round(n_anomalies / len(X_scaled) * 100, 2),
            'anomaly_indices': np.where(anomaly_labels == -1)[0].tolist()[:50],  # Limit to 50
            'anomaly_scores_summary': {
                'min': float(np.min(anomaly_scores)),
                'max': float(np.max(anomaly_scores)),
                'mean': float(np.mean(anomaly_scores)),
                'threshold': float(np.percentile(anomaly_scores, 5)),
            }
        }
        logger.info(f"🔍 Anomaly Detection: {n_anomalies} anomalies ({anomaly_results['anomaly_percentage']}%)")
    except Exception as ae:
        logger.warning(f"Anomaly detection skipped: {ae}")
    
    # --- t-SNE Visualization (2D) ---
    tsne_visualization = None
    try:
        from sklearn.manifold import TSNE
        if len(X_scaled) <= 5000:  # t-SNE is expensive for large datasets
            perplexity = min(30, max(5, len(X_scaled) // 5))
            tsne = TSNE(n_components=2, perplexity=perplexity, random_state=42, n_iter=500)
            X_tsne = tsne.fit_transform(X_scaled)
            tsne_visualization = {
                'x': X_tsne[:, 0].tolist(),
                'y': X_tsne[:, 1].tolist(),
            }
            logger.info(f"📊 t-SNE computed for {len(X_scaled)} samples")
        else:
            logger.info(f"⏭️ t-SNE skipped: dataset too large ({len(X_scaled)} samples)")
    except Exception as te:
        logger.warning(f"t-SNE skipped: {te}")
    
    # --- Feature Importance per Cluster ---
    feature_importance = None
    try:
        from sklearn.ensemble import RandomForestClassifier
        if actual_n_clusters >= 2 and len(feature_columns) >= 2:
            # Use cluster labels as targets, features as input
            valid_mask = labels >= 0  # Exclude noise points
            if np.sum(valid_mask) > 20:
                rf = RandomForestClassifier(n_estimators=50, max_depth=10, random_state=42, n_jobs=-1)
                rf.fit(X_scaled[valid_mask], labels[valid_mask])
                importances = rf.feature_importances_
                sorted_idx = np.argsort(importances)[::-1]
                feature_importance = {
                    'features': [feature_columns[i] for i in sorted_idx],
                    'importance_scores': [round(float(importances[i]), 4) for i in sorted_idx],
                    'top_3_features': [feature_columns[i] for i in sorted_idx[:3]],
                }
                logger.info(f"🎯 Feature importance: top features = {feature_importance['top_3_features']}")
    except Exception as fie:
        logger.warning(f"Feature importance skipped: {fie}")
    
    # --- Cluster Stability (Bootstrap Silhouette Variance) ---
    cluster_stability = None
    try:
        if actual_n_clusters >= 2 and len(X_scaled) >= 50:
            n_bootstrap = 5
            bootstrap_scores = []
            for _ in range(n_bootstrap):
                sample_idx = np.random.choice(len(X_scaled), size=min(len(X_scaled), 500), replace=True)
                X_sample = X_scaled[sample_idx]
                try:
                    km_boot = KMeans(n_clusters=actual_n_clusters, random_state=np.random.randint(1000), n_init=5)
                    boot_labels = km_boot.fit_predict(X_sample)
                    if len(set(boot_labels)) > 1:
                        score = silhouette_score(X_sample, boot_labels)
                        bootstrap_scores.append(score)
                except Exception:
                    pass
            if bootstrap_scores:
                cluster_stability = {
                    'mean_silhouette': round(float(np.mean(bootstrap_scores)), 4),
                    'std_silhouette': round(float(np.std(bootstrap_scores)), 4),
                    'stability_rating': 'High' if np.std(bootstrap_scores) < 0.03 else 'Medium' if np.std(bootstrap_scores) < 0.08 else 'Low',
                    'n_bootstrap_runs': len(bootstrap_scores),
                }
                logger.info(f"📈 Cluster stability: {cluster_stability['stability_rating']} (σ={cluster_stability['std_silhouette']})")
    except Exception as cse:
        logger.warning(f"Cluster stability skipped: {cse}")
    
    # --- Per-Sample Silhouette Scores ---
    sample_silhouettes = None
    try:
        from sklearn.metrics import silhouette_samples
        if actual_n_clusters >= 2:
            valid_mask = labels >= 0
            if np.sum(valid_mask) > 10 and len(set(labels[valid_mask])) > 1:
                sil_samples = silhouette_samples(X_scaled[valid_mask], labels[valid_mask])
                # Per-cluster average silhouette
                per_cluster_sil = {}
                for cl in range(actual_n_clusters):
                    mask = labels[valid_mask] == cl
                    if np.sum(mask) > 0:
                        per_cluster_sil[f"Cluster {cl}"] = round(float(np.mean(sil_samples[mask])), 4)
                sample_silhouettes = {
                    'per_cluster': per_cluster_sil,
                    'overall_mean': round(float(np.mean(sil_samples)), 4),
                    'n_negative': int(np.sum(sil_samples < 0)),  # Misclassified samples
                    'negative_percentage': round(float(np.sum(sil_samples < 0) / len(sil_samples) * 100), 2),
                }
                logger.info(f"📊 Silhouette analysis: {sample_silhouettes['n_negative']} potentially misclassified ({sample_silhouettes['negative_percentage']}%)")
    except Exception as sse:
        logger.warning(f"Sample silhouettes skipped: {sse}")
    
    # =======================================================================
    # BUILD RESPONSE
    # =======================================================================
    response = {
        'success': True,
        'model_id': model_id,
        'algorithm': algorithm,
        'n_clusters': actual_n_clusters,
        'n_samples': len(X),
        'n_features': len(feature_columns),
        'feature_columns': feature_columns,
        'feature_stats': feature_stats,
        'silhouette_score': metrics['silhouette_score'],
        'calinski_harabasz_score': metrics.get('calinski_harabasz_score', 0),
        'davies_bouldin_score': metrics.get('davies_bouldin_score', 0),
        'cluster_distribution': cluster_distribution,
        'cluster_profiles': cluster_profiles,
        'pca_variance_explained': pca_variance,
        'labels': labels.tolist(),
        'visualization': {
            'x': X_2d[:, 0].tolist(),
            'y': X_2d[:, 1].tolist(),
        },
        'charts': charts,
        'reliability_score': reliability_score,
        'validation_warnings': validation_warnings if validation_warnings else None,
        'comparison_results': comparison_results,
        'cleaned_file': cleaned_file,
        'model_pkl_file': model_pkl_file,
        'k_scores': k_scores,  # For elbow chart on frontend
        # 🔬 Production ML Engineering Features
        'anomaly_detection': anomaly_results,
        'tsne_visualization': tsne_visualization,
        'feature_importance': feature_importance,
        'cluster_stability': cluster_stability,
        'sample_silhouettes': sample_silhouettes,
    }
    
    logger.info(f"✅ Clustering complete: {actual_n_clusters} clusters, silhouette={metrics['silhouette_score']:.3f}")
    
    return response


def _find_optimal_k(X_scaled: np.ndarray, max_k: int = 10) -> tuple:
    """
    Find optimal number of clusters using multiple metrics:
    - Silhouette Score (primary)
    - Elbow Method (inertia)
    - Calinski-Harabasz Index
    """
    max_k = min(max_k, len(X_scaled) - 1, 15)
    max_k = max(max_k, 3)
    
    scores = {}
    inertias = []
    calinski_scores = []
    best_k = 2
    best_score = -1
    
    for k in range(2, max_k + 1):
        try:
            # Use k-means++ initialization with multiple runs
            kmeans = KMeans(
                n_clusters=k, 
                random_state=42, 
                n_init=15,  # More initializations for stability
                max_iter=500,  # More iterations for convergence
                init='k-means++',
                algorithm='lloyd'
            )
            labels = kmeans.fit_predict(X_scaled)
            
            if len(set(labels)) > 1:
                score = silhouette_score(X_scaled, labels)
                scores[k] = score
                inertias.append(kmeans.inertia_)
                
                try:
                    calinski = calinski_harabasz_score(X_scaled, labels)
                    calinski_scores.append(calinski)
                except:
                    calinski_scores.append(0)
                
                if score > best_score:
                    best_score = score
                    best_k = k
        except Exception as e:
            logger.warning(f"Optimal k search failed for k={k}: {e}")
    
    # If silhouette fails, try elbow method
    if best_score < 0 and inertias:
        # Find elbow using rate of change
        deltas = np.diff(inertias)
        if len(deltas) > 1:
            delta2 = np.diff(deltas)
            elbow_idx = np.argmax(delta2) + 2
            best_k = elbow_idx + 2  # Adjust for range starting at 2
    
    logger.info(f"🎯 Optimal k detection: best_k={best_k}, silhouette={best_score:.3f}")
    return best_k, scores


def _run_single_clustering(X_scaled: np.ndarray, algorithm: str, n_clusters: int):
    """
    Run a single clustering algorithm with PRODUCTION-QUALITY settings.
    Enhanced with:
    - Better hyperparameters
    - Multiple initializations
    - Adaptive parameters based on data size
    """
    metrics = {}
    n_samples = len(X_scaled)
    
    if algorithm == 'kmeans':
        # Production K-Means with k-means++ and stability settings
        model = KMeans(
            n_clusters=n_clusters, 
            random_state=42, 
            n_init=20,  # More initializations for better results
            max_iter=500,  # More iterations
            init='k-means++',  # Smart initialization
            algorithm='lloyd',
            tol=1e-5  # Stricter convergence
        )
        labels = model.fit_predict(X_scaled)
        metrics['inertia'] = model.inertia_
        metrics['n_iter'] = model.n_iter_
        
    elif algorithm == 'dbscan':
        # Auto-detect eps using k-distance graph with better heuristics
        from sklearn.neighbors import NearestNeighbors
        
        # Adaptive k based on data size
        k = min(max(5, n_samples // 50), 15, n_samples - 1)
        nn = NearestNeighbors(n_neighbors=k)
        nn.fit(X_scaled)
        distances, _ = nn.kneighbors(X_scaled)
        
        # Use multiple percentiles and pick best
        sorted_distances = np.sort(distances[:, -1])
        
        # Try different eps values and pick best silhouette
        best_eps = np.percentile(sorted_distances, 90)
        best_labels = None
        best_sil = -1
        
        for pct in [80, 85, 90, 95]:
            try:
                eps = np.percentile(sorted_distances, pct)
                min_samples = max(3, n_samples // 100)
                
                model = DBSCAN(eps=eps, min_samples=min_samples)
                test_labels = model.fit_predict(X_scaled)
                
                n_clusters_found = len(set(test_labels)) - (1 if -1 in test_labels else 0)
                if n_clusters_found >= 2:
                    mask = test_labels != -1
                    if mask.sum() > 10:
                        sil = silhouette_score(X_scaled[mask], test_labels[mask])
                        if sil > best_sil:
                            best_sil = sil
                            best_eps = eps
                            best_labels = test_labels
            except:
                pass
        
        if best_labels is not None:
            labels = best_labels
        else:
            model = DBSCAN(eps=best_eps, min_samples=max(3, n_samples // 100))
            labels = model.fit_predict(X_scaled)
        
        metrics['eps'] = best_eps
        
    elif algorithm == 'hierarchical':
        # Use ward linkage for compactness, try different linkage if fails
        try:
            model = AgglomerativeClustering(
                n_clusters=n_clusters, 
                linkage='ward',
                metric='euclidean'
            )
            labels = model.fit_predict(X_scaled)
        except Exception:
            # Fallback to average linkage
            model = AgglomerativeClustering(
                n_clusters=n_clusters, 
                linkage='average'
            )
            labels = model.fit_predict(X_scaled)
        
        metrics['linkage'] = 'ward'
        
    elif algorithm == 'gmm':
        # Gaussian Mixture with multiple covariance types and select best
        from sklearn.mixture import GaussianMixture
        
        best_model = None
        best_bic = float('inf')
        
        for cov_type in ['full', 'tied', 'diag', 'spherical']:
            try:
                model = GaussianMixture(
                    n_components=n_clusters, 
                    random_state=42, 
                    n_init=10,
                    max_iter=200,
                    covariance_type=cov_type,
                    init_params='k-means++'
                )
                model.fit(X_scaled)
                bic = model.bic(X_scaled)
                
                if bic < best_bic:
                    best_bic = bic
                    best_model = model
            except Exception:
                pass
        
        if best_model is None:
            # Fallback to simple GMM
            best_model = GaussianMixture(
                n_components=n_clusters, 
                random_state=42, 
                n_init=5
            )
            best_model.fit(X_scaled)
        
        labels = best_model.predict(X_scaled)
        metrics['bic'] = best_model.bic(X_scaled)
        metrics['aic'] = best_model.aic(X_scaled)
        
    elif algorithm == 'spectral':
        # Spectral clustering with adaptive neighbors
        n_neighbors = min(max(10, n_samples // 50), 30, n_samples - 1)
        
        try:
            model = SpectralClustering(
                n_clusters=n_clusters, 
                random_state=42, 
                affinity='nearest_neighbors',
                n_neighbors=n_neighbors,
                assign_labels='cluster_qr'  # Better assignment
            )
            labels = model.fit_predict(X_scaled)
        except Exception:
            # Fallback with simpler settings
            model = SpectralClustering(
                n_clusters=n_clusters, 
                random_state=42, 
                affinity='rbf'
            )
            labels = model.fit_predict(X_scaled)
        
    else:
        raise HTTPException(status_code=400, detail=f"Unknown algorithm: {algorithm}")
    
    return labels, model, metrics


def _compare_all_algorithms(X_scaled: np.ndarray, n_clusters: int) -> Dict[str, Any]:
    """Compare all clustering algorithms."""
    results = {}
    algorithms = ['kmeans', 'hierarchical', 'gmm', 'spectral', 'dbscan']
    
    for algo in algorithms:
        try:
            labels, model, _ = _run_single_clustering(X_scaled, algo, n_clusters)
            metrics = _calculate_clustering_metrics(X_scaled, labels)
            
            actual_k = len(set(labels)) - (1 if -1 in labels else 0)
            
            results[algo] = {
                'labels': labels.tolist(),
                'n_clusters': actual_k,
                **metrics
            }
        except Exception as e:
            logger.warning(f"Algorithm {algo} failed: {e}")
    
    return results


def _calculate_clustering_metrics(X_scaled: np.ndarray, labels: np.ndarray) -> Dict[str, float]:
    """Calculate clustering quality metrics."""
    mask = labels != -1
    unique_labels = set(labels[mask])
    
    metrics = {
        'silhouette_score': 0.0,
        'calinski_harabasz_score': 0.0,
        'davies_bouldin_score': 0.0,
    }
    
    if len(unique_labels) <= 1 or mask.sum() < 2:
        return metrics
    
    try:
        metrics['silhouette_score'] = float(silhouette_score(X_scaled[mask], labels[mask]))
    except Exception:
        pass
    
    try:
        metrics['calinski_harabasz_score'] = float(calinski_harabasz_score(X_scaled[mask], labels[mask]))
    except Exception:
        pass
    
    try:
        metrics['davies_bouldin_score'] = float(davies_bouldin_score(X_scaled[mask], labels[mask]))
    except Exception:
        pass
    
    return metrics


def _get_cluster_centroids(X_scaled: np.ndarray, labels: np.ndarray) -> np.ndarray:
    """Calculate cluster centroids."""
    unique_labels = [l for l in np.unique(labels) if l != -1]
    centroids = []
    for label in unique_labels:
        mask = labels == label
        centroid = X_scaled[mask].mean(axis=0)
        centroids.append(centroid)
    return np.array(centroids) if centroids else np.array([])


def _compute_reliability_score(
    n_samples: int,
    n_features: int,
    silhouette: float,
    calinski: float,
    davies: float,
    df: pd.DataFrame
) -> tuple:
    """Compute reliability score and validation warnings."""
    import math
    
    validation_warnings = []
    
    # Data quality checks
    missing_ratio = df.isna().sum().sum() / df.size if df.size > 0 else 0
    duplicate_ratio = df.duplicated().sum() / n_samples if n_samples > 0 else 0
    
    if missing_ratio > 0.2:
        validation_warnings.append(f"⚠️ High missing data: {missing_ratio:.1%}")
    if duplicate_ratio > 0.1:
        validation_warnings.append(f"⚠️ Many duplicates: {duplicate_ratio:.1%}")
    if n_samples < 100:
        validation_warnings.append(f"⚠️ Small dataset ({n_samples} samples)")
    if n_features > 50:
        validation_warnings.append(f"⚠️ High dimensionality ({n_features} features)")
    if silhouette < 0.2:
        validation_warnings.append(f"⚠️ Low silhouette score ({silhouette:.3f}) - clusters may overlap")
    
    # Compute reliability score
    silhouette_points = max(0, (silhouette + 1) / 2 * 40)
    calinski_points = min(25, math.log(calinski + 1) * 3) if calinski > 0 else 0
    davies_points = max(0, 20 - davies * 5) if davies < float('inf') else 10
    
    if n_samples >= 1000:
        size_points = 15
    elif n_samples >= 500:
        size_points = 12
    elif n_samples >= 100:
        size_points = 8
    else:
        size_points = 5
    
    reliability_score = min(100, silhouette_points + calinski_points + davies_points + size_points)
    
    return reliability_score, validation_warnings


def _generate_clustering_charts(
    X_2d: np.ndarray,
    labels: np.ndarray,
    algorithm: str,
    n_clusters: int,
    silhouette: float,
    feature_names: List[str],
    cluster_profiles: Dict,
    X_scaled: np.ndarray = None,
    X_original: np.ndarray = None,
    k_scores: Dict[int, float] = None
) -> Dict[str, str]:
    """
    Generate COMPREHENSIVE visualization charts for unsupervised learning.
    Only generates charts when data is sufficient and suitable.
    
    Charts included (when data permits):
    1. Cluster Scatter Plot (PCA 2D)
    2. Elbow Method Graph
    3. Silhouette Score Plot
    4. Cluster Distribution Bar Chart
    5. Cluster Profile Heatmap
    6. PCA Explained Variance Plot
    7. Pairplot (feature relationships) - up to 4 features
    8. 3D Cluster Visualization
    9. Dendrogram (Hierarchical only)
    10. t-SNE Visualization (if samples < 5000)
    11. Boxplot per Cluster
    12. Violin Plot per Cluster
    13. Correlation Heatmap
    14. Radar Chart for Cluster Comparison
    """
    charts = {}
    
    try:
        import matplotlib
        matplotlib.use('Agg')
        import matplotlib.pyplot as plt
        import seaborn as sns
        from matplotlib.colors import LinearSegmentedColormap
        
        # Set style
        plt.style.use('seaborn-v0_8-whitegrid')
        
        # Professional color palette
        colors = ['#2563eb', '#16a34a', '#dc2626', '#ea580c', '#9333ea', 
                  '#0891b2', '#db2777', '#d97706', '#0d9488', '#4f46e5',
                  '#84cc16', '#06b6d4', '#f43f5e', '#8b5cf6', '#14b8a6']
        
        unique_labels = sorted([l for l in set(labels) if l >= 0])
        n_samples = len(labels)
        n_features = len(feature_names) if feature_names else 0
        
        # ==================================================================
        # 1. CLUSTER SCATTER PLOT (PCA 2D) - Always generate if possible
        # ==================================================================
        if X_2d is not None and len(X_2d) > 0:
            try:
                fig, ax = plt.subplots(figsize=(10, 8))
                
                for i, label in enumerate(sorted(set(labels))):
                    mask = labels == label
                    color = colors[i % len(colors)] if label >= 0 else '#6b7280'
                    label_name = f'Cluster {label}' if label >= 0 else 'Noise'
                    ax.scatter(X_2d[mask, 0], X_2d[mask, 1], 
                              c=color, label=label_name, alpha=0.7, s=50,
                              edgecolors='white', linewidth=0.5)
                
                # Add cluster centroids
                for i, label in enumerate(unique_labels):
                    mask = labels == label
                    centroid = X_2d[mask].mean(axis=0)
                    ax.scatter(centroid[0], centroid[1], c=colors[i % len(colors)], 
                              s=200, marker='X', edgecolors='black', linewidth=2, zorder=10)
                
                ax.set_xlabel('PCA Component 1', fontweight='bold', fontsize=12)
                ax.set_ylabel('PCA Component 2', fontweight='bold', fontsize=12)
                ax.set_title(f'🔮 {algorithm.upper()} Clustering (Silhouette: {silhouette:.3f})', 
                            fontweight='bold', pad=15, fontsize=14)
                ax.legend(loc='best', fontsize=10)
                ax.grid(alpha=0.3)
                fig.tight_layout()
                
                charts['cluster_scatter'] = _fig_to_base64(fig)
                plt.close(fig)
            except Exception as e:
                logger.warning(f"Cluster scatter plot failed: {e}")
        
        # ==================================================================
        # 2. ELBOW METHOD GRAPH - Only if k_scores provided
        # ==================================================================
        if k_scores and len(k_scores) >= 3:
            try:
                fig, ax = plt.subplots(figsize=(10, 6))
                
                k_values = sorted(k_scores.keys())
                scores = [k_scores[k] for k in k_values]
                
                ax.plot(k_values, scores, 'bo-', linewidth=2, markersize=10)
                
                # Highlight optimal k
                optimal_k = k_values[np.argmax(scores)]
                optimal_score = max(scores)
                ax.scatter([optimal_k], [optimal_score], c='red', s=200, zorder=10, 
                          marker='*', label=f'Optimal k={optimal_k}')
                ax.axvline(x=optimal_k, color='red', linestyle='--', alpha=0.5)
                
                ax.set_xlabel('Number of Clusters (k)', fontweight='bold', fontsize=12)
                ax.set_ylabel('Silhouette Score', fontweight='bold', fontsize=12)
                ax.set_title('📈 Elbow Method - Optimal Cluster Selection', fontweight='bold', pad=15, fontsize=14)
                ax.legend(loc='best')
                ax.grid(alpha=0.3)
                fig.tight_layout()
                
                charts['elbow_method'] = _fig_to_base64(fig)
                plt.close(fig)
            except Exception as e:
                logger.warning(f"Elbow method chart failed: {e}")
        
        # ==================================================================
        # 3. SILHOUETTE SCORE PLOT - Need sklearn and enough samples
        # ==================================================================
        if X_scaled is not None and n_clusters >= 2 and n_samples >= 50:
            try:
                from sklearn.metrics import silhouette_samples
                
                fig, ax = plt.subplots(figsize=(10, 8))
                
                sample_silhouette_values = silhouette_samples(X_scaled, labels)
                
                y_lower = 10
                for i, label in enumerate(unique_labels):
                    cluster_silhouette_vals = sample_silhouette_values[labels == label]
                    cluster_silhouette_vals.sort()
                    
                    cluster_size = cluster_silhouette_vals.shape[0]
                    y_upper = y_lower + cluster_size
                    
                    ax.fill_betweenx(np.arange(y_lower, y_upper),
                                    0, cluster_silhouette_vals,
                                    facecolor=colors[i % len(colors)], alpha=0.7)
                    ax.text(-0.05, y_lower + 0.5 * cluster_size, f'Cluster {label}',
                           fontsize=10, fontweight='bold')
                    
                    y_lower = y_upper + 10
                
                ax.axvline(x=silhouette, color='red', linestyle='--', linewidth=2,
                          label=f'Mean: {silhouette:.3f}')
                ax.set_xlabel('Silhouette Coefficient', fontweight='bold', fontsize=12)
                ax.set_ylabel('Cluster', fontweight='bold', fontsize=12)
                ax.set_title('📊 Silhouette Analysis - Cluster Quality', fontweight='bold', pad=15, fontsize=14)
                ax.legend(loc='best')
                ax.grid(alpha=0.3, axis='x')
                fig.tight_layout()
                
                charts['silhouette_plot'] = _fig_to_base64(fig)
                plt.close(fig)
            except Exception as e:
                logger.warning(f"Silhouette plot failed: {e}")
        
        # ==================================================================
        # 4. CLUSTER DISTRIBUTION BAR CHART - Always generate
        # ==================================================================
        try:
            fig, ax = plt.subplots(figsize=(10, 6))
            
            cluster_names = [f'Cluster {l}' if l >= 0 else 'Noise' for l in sorted(set(labels))]
            cluster_counts = [np.sum(labels == l) for l in sorted(set(labels))]
            cluster_colors = [colors[i % len(colors)] if l >= 0 else '#6b7280' 
                             for i, l in enumerate(sorted(set(labels)))]
            
            bars = ax.bar(cluster_names, cluster_counts, color=cluster_colors, 
                         edgecolor='white', linewidth=2)
            
            # Add counts on bars
            for bar, count in zip(bars, cluster_counts):
                pct = count / n_samples * 100
                ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.5, 
                       f'{count}\n({pct:.1f}%)', ha='center', va='bottom', 
                       fontweight='bold', fontsize=10)
            
            ax.set_xlabel('Cluster', fontweight='bold', fontsize=12)
            ax.set_ylabel('Number of Samples', fontweight='bold', fontsize=12)
            ax.set_title('📊 Cluster Size Distribution', fontweight='bold', pad=15, fontsize=14)
            ax.grid(axis='y', alpha=0.3)
            fig.tight_layout()
            
            charts['cluster_distribution'] = _fig_to_base64(fig)
            plt.close(fig)
        except Exception as e:
            logger.warning(f"Distribution chart failed: {e}")
        
        # ==================================================================
        # 5. CLUSTER PROFILE HEATMAP - Need profiles and features
        # ==================================================================
        if cluster_profiles and len(feature_names) >= 3 and len(unique_labels) >= 2:
            try:
                fig, ax = plt.subplots(figsize=(12, 8))
                
                # Build heatmap data
                top_features = feature_names[:min(15, len(feature_names))]
                heatmap_data = []
                cluster_labels_list = []
                
                for cluster_name, profile in sorted(cluster_profiles.items()):
                    row = [profile.get(f, {}).get('mean', 0) for f in top_features]
                    heatmap_data.append(row)
                    cluster_labels_list.append(cluster_name)
                
                heatmap_array = np.array(heatmap_data)
                
                # Normalize per feature (column)
                for j in range(heatmap_array.shape[1]):
                    col_min = heatmap_array[:, j].min()
                    col_max = heatmap_array[:, j].max()
                    if col_max - col_min > 0:
                        heatmap_array[:, j] = (heatmap_array[:, j] - col_min) / (col_max - col_min)
                
                sns.heatmap(heatmap_array, annot=True, fmt='.2f', cmap='RdYlGn',
                           xticklabels=top_features, yticklabels=cluster_labels_list,
                           ax=ax, linewidths=0.5, cbar_kws={'label': 'Normalized Value'})
                
                ax.set_title('🔥 Cluster Profile Heatmap', fontweight='bold', pad=15, fontsize=14)
                ax.set_xlabel('Features', fontweight='bold', fontsize=12)
                ax.set_ylabel('Clusters', fontweight='bold', fontsize=12)
                plt.xticks(rotation=45, ha='right')
                fig.tight_layout()
                
                charts['cluster_heatmap'] = _fig_to_base64(fig)
                plt.close(fig)
            except Exception as e:
                logger.warning(f"Heatmap failed: {e}")
        
        # ==================================================================
        # 6. PCA EXPLAINED VARIANCE PLOT - Need enough features
        # ==================================================================
        if X_scaled is not None and n_features >= 3:
            try:
                from sklearn.decomposition import PCA as PCAViz
                
                n_components = min(10, n_features, n_samples)
                pca_full = PCAViz(n_components=n_components)
                pca_full.fit(X_scaled)
                
                fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))
                
                # Individual variance
                variance = pca_full.explained_variance_ratio_
                components = range(1, len(variance) + 1)
                ax1.bar(components, variance * 100, color=colors[0], edgecolor='white', linewidth=1.5)
                ax1.set_xlabel('Principal Component', fontweight='bold', fontsize=12)
                ax1.set_ylabel('Variance Explained (%)', fontweight='bold', fontsize=12)
                ax1.set_title('📊 Individual Variance per Component', fontweight='bold', pad=15, fontsize=14)
                ax1.grid(axis='y', alpha=0.3)
                
                # Cumulative variance
                cumulative = np.cumsum(variance) * 100
                ax2.plot(components, cumulative, 'bo-', linewidth=2, markersize=8)
                ax2.fill_between(components, cumulative, alpha=0.3, color=colors[0])
                ax2.axhline(y=90, color='red', linestyle='--', label='90% threshold')
                ax2.set_xlabel('Number of Components', fontweight='bold', fontsize=12)
                ax2.set_ylabel('Cumulative Variance (%)', fontweight='bold', fontsize=12)
                ax2.set_title('📈 Cumulative Explained Variance', fontweight='bold', pad=15, fontsize=14)
                ax2.legend()
                ax2.grid(alpha=0.3)
                
                fig.tight_layout()
                charts['pca_variance'] = _fig_to_base64(fig)
                plt.close(fig)
            except Exception as e:
                logger.warning(f"PCA variance plot failed: {e}")
        
        # ==================================================================
        # 7. PAIRPLOT (Feature Relationships) - Need 2-4 features, <2000 samples
        # ==================================================================
        if X_original is not None and 2 <= n_features <= 6 and n_samples <= 2000:
            try:
                import pandas as pd
                
                # Select top 4 features
                plot_features = feature_names[:min(4, len(feature_names))]
                df_plot = pd.DataFrame(X_original[:, :len(plot_features)], columns=plot_features)
                df_plot['Cluster'] = [f'C{l}' for l in labels]
                
                palette = {f'C{l}': colors[i % len(colors)] for i, l in enumerate(unique_labels)}
                palette['C-1'] = '#6b7280'  # Noise
                
                g = sns.pairplot(df_plot, hue='Cluster', palette=palette, 
                                diag_kind='kde', plot_kws={'alpha': 0.6, 's': 30})
                g.fig.suptitle('🔍 Feature Relationships by Cluster', fontweight='bold', y=1.02, fontsize=14)
                
                charts['pairplot'] = _fig_to_base64(g.fig)
                plt.close(g.fig)
            except Exception as e:
                logger.warning(f"Pairplot failed: {e}")
        
        # ==================================================================
        # 8. 3D CLUSTER VISUALIZATION - Need 3+ features
        # ==================================================================
        if X_scaled is not None and n_features >= 3 and n_samples <= 5000:
            try:
                from sklearn.decomposition import PCA as PCA3D
                from mpl_toolkits.mplot3d import Axes3D
                
                pca_3d = PCA3D(n_components=3)
                X_3d = pca_3d.fit_transform(X_scaled)
                
                fig = plt.figure(figsize=(12, 10))
                ax = fig.add_subplot(111, projection='3d')
                
                for i, label in enumerate(sorted(set(labels))):
                    mask = labels == label
                    color = colors[i % len(colors)] if label >= 0 else '#6b7280'
                    label_name = f'Cluster {label}' if label >= 0 else 'Noise'
                    ax.scatter(X_3d[mask, 0], X_3d[mask, 1], X_3d[mask, 2],
                              c=color, label=label_name, alpha=0.6, s=30)
                
                ax.set_xlabel('PC1', fontweight='bold')
                ax.set_ylabel('PC2', fontweight='bold')
                ax.set_zlabel('PC3', fontweight='bold')
                ax.set_title('🌐 3D Cluster Visualization', fontweight='bold', pad=20, fontsize=14)
                ax.legend(loc='best')
                
                charts['cluster_3d'] = _fig_to_base64(fig)
                plt.close(fig)
            except Exception as e:
                logger.warning(f"3D plot failed: {e}")
        
        # ==================================================================
        # 9. DENDROGRAM - For hierarchical clustering (works with any dataset size)
        # ==================================================================
        if algorithm == 'hierarchical' and X_scaled is not None:
            try:
                from scipy.cluster.hierarchy import dendrogram, linkage
                
                # Sample data if too large (for performance)
                max_dendrogram_samples = min(500, n_samples)
                if n_samples > max_dendrogram_samples:
                    # Random sample for dendrogram visualization
                    sample_indices = np.random.choice(n_samples, max_dendrogram_samples, replace=False)
                    X_dendro = X_scaled[sample_indices]
                else:
                    X_dendro = X_scaled
                
                # Use ward linkage for best visual hierarchy
                Z = linkage(X_dendro, method='ward')
                
                fig, ax = plt.subplots(figsize=(14, 8))
                
                # Truncate for readability
                dendrogram(Z, ax=ax, truncate_mode='lastp', p=min(50, max_dendrogram_samples), 
                          leaf_rotation=90, leaf_font_size=9,
                          color_threshold=0.7*max(Z[:,2]),
                          above_threshold_color='gray')
                
                ax.set_xlabel('Sample Index / Cluster Size', fontweight='bold', fontsize=12)
                ax.set_ylabel('Distance (Ward)', fontweight='bold', fontsize=12)
                ax.set_title(f'🌳 Hierarchical Clustering Dendrogram ({n_clusters} Clusters)', 
                            fontweight='bold', pad=15, fontsize=14)
                ax.axhline(y=0.7*max(Z[:,2]), color='red', linestyle='--', linewidth=2,
                          label=f'Suggested cut-off for {n_clusters} clusters')
                ax.legend(loc='upper right')
                ax.grid(axis='y', alpha=0.3)
                fig.tight_layout()
                
                charts['dendrogram'] = _fig_to_base64(fig)
                plt.close(fig)
                logger.info(f"✅ Dendrogram generated for hierarchical clustering")
            except Exception as e:
                logger.warning(f"Dendrogram failed: {e}")
        
        # ==================================================================
        # 10. t-SNE VISUALIZATION - Best for <5000 samples
        # ==================================================================
        if X_scaled is not None and 100 <= n_samples <= 3000 and n_features >= 3:
            try:
                from sklearn.manifold import TSNE
                
                perplexity = min(30, n_samples // 5)
                tsne = TSNE(n_components=2, perplexity=perplexity, random_state=42, n_iter=500)
                X_tsne = tsne.fit_transform(X_scaled)
                
                fig, ax = plt.subplots(figsize=(10, 8))
                
                for i, label in enumerate(sorted(set(labels))):
                    mask = labels == label
                    color = colors[i % len(colors)] if label >= 0 else '#6b7280'
                    label_name = f'Cluster {label}' if label >= 0 else 'Noise'
                    ax.scatter(X_tsne[mask, 0], X_tsne[mask, 1],
                              c=color, label=label_name, alpha=0.7, s=40)
                
                ax.set_xlabel('t-SNE 1', fontweight='bold', fontsize=12)
                ax.set_ylabel('t-SNE 2', fontweight='bold', fontsize=12)
                ax.set_title('🔬 t-SNE Visualization', fontweight='bold', pad=15, fontsize=14)
                ax.legend(loc='best')
                ax.grid(alpha=0.3)
                fig.tight_layout()
                
                charts['tsne'] = _fig_to_base64(fig)
                plt.close(fig)
            except Exception as e:
                logger.warning(f"t-SNE plot failed: {e}")
        
        # ==================================================================
        # 10b. UMAP VISUALIZATION - Optional (requires umap-learn package)
        # ==================================================================
        if X_scaled is not None and 100 <= n_samples <= 5000 and n_features >= 3:
            try:
                import umap
                
                reducer = umap.UMAP(n_components=2, random_state=42, n_neighbors=15, min_dist=0.1)
                X_umap = reducer.fit_transform(X_scaled)
                
                fig, ax = plt.subplots(figsize=(10, 8))
                
                for i, label in enumerate(sorted(set(labels))):
                    mask = labels == label
                    color = colors[i % len(colors)] if label >= 0 else '#6b7280'
                    label_name = f'Cluster {label}' if label >= 0 else 'Noise'
                    ax.scatter(X_umap[mask, 0], X_umap[mask, 1],
                              c=color, label=label_name, alpha=0.7, s=40)
                
                ax.set_xlabel('UMAP 1', fontweight='bold', fontsize=12)
                ax.set_ylabel('UMAP 2', fontweight='bold', fontsize=12)
                ax.set_title('🗺️ UMAP Visualization', fontweight='bold', pad=15, fontsize=14)
                ax.legend(loc='best')
                ax.grid(alpha=0.3)
                fig.tight_layout()
                
                charts['umap'] = _fig_to_base64(fig)
                plt.close(fig)
            except ImportError:
                logger.info("UMAP not installed - skipping UMAP visualization")
            except Exception as e:
                logger.warning(f"UMAP plot failed: {e}")
        
        # ==================================================================
        # 11. BOXPLOT PER CLUSTER - Need features
        # ==================================================================
        if X_original is not None and n_features >= 2 and len(unique_labels) >= 2:
            try:
                import pandas as pd
                
                # Select top 4 features for box plots
                top_features = feature_names[:min(4, len(feature_names))]
                n_plot = len(top_features)
                
                fig, axes = plt.subplots(1, n_plot, figsize=(4*n_plot, 6))
                if n_plot == 1:
                    axes = [axes]
                
                for idx, feature in enumerate(top_features):
                    feature_idx = feature_names.index(feature)
                    data_by_cluster = [X_original[labels == l, feature_idx] for l in unique_labels]
                    
                    bp = axes[idx].boxplot(data_by_cluster, patch_artist=True,
                                          labels=[f'C{l}' for l in unique_labels])
                    
                    for i, patch in enumerate(bp['boxes']):
                        patch.set_facecolor(colors[i % len(colors)])
                        patch.set_alpha(0.7)
                    
                    axes[idx].set_xlabel('Cluster', fontweight='bold', fontsize=11)
                    axes[idx].set_ylabel(feature, fontweight='bold', fontsize=11)
                    axes[idx].set_title(f'{feature}', fontweight='bold', fontsize=12)
                    axes[idx].grid(axis='y', alpha=0.3)
                
                fig.suptitle('📦 Feature Distribution by Cluster (Boxplots)', fontweight='bold', fontsize=14, y=1.02)
                fig.tight_layout()
                
                charts['boxplots'] = _fig_to_base64(fig)
                plt.close(fig)
            except Exception as e:
                logger.warning(f"Boxplot failed: {e}")
        
        # ==================================================================
        # 12. VIOLIN PLOT PER CLUSTER - Alternative to boxplot
        # ==================================================================
        if X_original is not None and n_features >= 2 and len(unique_labels) >= 2 and n_samples >= 50:
            try:
                import pandas as pd
                
                # Select top 3 features for violin plots
                top_features = feature_names[:min(3, len(feature_names))]
                
                fig, axes = plt.subplots(1, len(top_features), figsize=(5*len(top_features), 6))
                if len(top_features) == 1:
                    axes = [axes]
                
                for idx, feature in enumerate(top_features):
                    feature_idx = feature_names.index(feature)
                    
                    # Create dataframe for seaborn
                    df_violin = pd.DataFrame({
                        'Value': X_original[:, feature_idx],
                        'Cluster': [f'C{l}' for l in labels]
                    })
                    
                    palette = {f'C{l}': colors[i % len(colors)] for i, l in enumerate(unique_labels)}
                    
                    sns.violinplot(data=df_violin, x='Cluster', y='Value', 
                                  palette=palette, ax=axes[idx])
                    axes[idx].set_xlabel('Cluster', fontweight='bold', fontsize=11)
                    axes[idx].set_ylabel(feature, fontweight='bold', fontsize=11)
                    axes[idx].set_title(f'{feature}', fontweight='bold', fontsize=12)
                    axes[idx].grid(axis='y', alpha=0.3)
                
                fig.suptitle('🎻 Feature Distribution by Cluster (Violin Plots)', fontweight='bold', fontsize=14, y=1.02)
                fig.tight_layout()
                
                charts['violin_plots'] = _fig_to_base64(fig)
                plt.close(fig)
            except Exception as e:
                logger.warning(f"Violin plot failed: {e}")
        
        # ==================================================================
        # 13. CORRELATION HEATMAP - Need enough features
        # ==================================================================
        if X_original is not None and n_features >= 3:
            try:
                import pandas as pd
                
                df_corr = pd.DataFrame(X_original, columns=feature_names)
                corr_matrix = df_corr.corr()
                
                fig, ax = plt.subplots(figsize=(10, 8))
                
                mask = np.triu(np.ones_like(corr_matrix, dtype=bool))
                sns.heatmap(corr_matrix, mask=mask, annot=True, fmt='.2f', 
                           cmap='coolwarm', center=0, ax=ax,
                           linewidths=0.5, cbar_kws={'label': 'Correlation'})
                
                ax.set_title('🔗 Feature Correlation Heatmap', fontweight='bold', pad=15, fontsize=14)
                plt.xticks(rotation=45, ha='right')
                fig.tight_layout()
                
                charts['correlation_heatmap'] = _fig_to_base64(fig)
                plt.close(fig)
            except Exception as e:
                logger.warning(f"Correlation heatmap failed: {e}")
        
        # ==================================================================
        # 14. RADAR CHART FOR CLUSTER COMPARISON - Need profiles
        # ==================================================================
        if cluster_profiles and len(feature_names) >= 3 and len(unique_labels) >= 2:
            try:
                fig, ax = plt.subplots(figsize=(10, 10), subplot_kw=dict(polar=True))
                
                top_features = feature_names[:min(8, len(feature_names))]
                n_features_plot = len(top_features)
                angles = np.linspace(0, 2 * np.pi, n_features_plot, endpoint=False).tolist()
                angles += angles[:1]
                
                for cluster_name, profile in sorted(cluster_profiles.items()):
                    values = [profile.get(f, {}).get('mean', 0) for f in top_features]
                    
                    # Normalize to 0-1 range across all clusters
                    all_vals = [cluster_profiles[cn].get(f, {}).get('mean', 0) 
                               for cn in cluster_profiles for f in top_features]
                    val_min, val_max = min(all_vals), max(all_vals)
                    if val_max - val_min > 0:
                        values = [(v - val_min) / (val_max - val_min) for v in values]
                    values += values[:1]
                    
                    cluster_idx = int(cluster_name.split()[-1])
                    ax.plot(angles, values, 'o-', linewidth=2, 
                           label=cluster_name, color=colors[cluster_idx % len(colors)])
                    ax.fill(angles, values, alpha=0.15, color=colors[cluster_idx % len(colors)])
                
                ax.set_xticks(angles[:-1])
                ax.set_xticklabels(top_features, fontsize=10)
                ax.set_title('🎯 Radar Chart - Cluster Comparison', fontweight='bold', pad=25, fontsize=14)
                ax.legend(loc='upper right', bbox_to_anchor=(1.3, 1.1))
                fig.tight_layout()
                
                charts['radar_chart'] = _fig_to_base64(fig)
                plt.close(fig)
            except Exception as e:
                logger.warning(f"Radar chart failed: {e}")
        
        # ==================================================================
        # 15. GMM BIC/AIC SCORES - Only for GMM algorithm
        # ==================================================================
        if algorithm == 'gmm' and X_scaled is not None:
            try:
                from sklearn.mixture import GaussianMixture
                
                k_range = range(2, min(11, n_samples // 10 + 1))
                bic_scores = []
                aic_scores = []
                
                for k in k_range:
                    gmm = GaussianMixture(n_components=k, random_state=42, n_init=3)
                    gmm.fit(X_scaled)
                    bic_scores.append(gmm.bic(X_scaled))
                    aic_scores.append(gmm.aic(X_scaled))
                
                fig, ax = plt.subplots(figsize=(10, 6))
                
                ax.plot(list(k_range), bic_scores, 'bo-', linewidth=2, markersize=8, label='BIC')
                ax.plot(list(k_range), aic_scores, 'rs-', linewidth=2, markersize=8, label='AIC')
                
                # Highlight optimal k (lowest BIC)
                optimal_k = list(k_range)[np.argmin(bic_scores)]
                ax.axvline(x=optimal_k, color='green', linestyle='--', alpha=0.7,
                          label=f'Optimal k={optimal_k}')
                ax.scatter([optimal_k], [min(bic_scores)], c='green', s=200, zorder=10, marker='*')
                
                ax.set_xlabel('Number of Components (k)', fontweight='bold', fontsize=12)
                ax.set_ylabel('Score (lower is better)', fontweight='bold', fontsize=12)
                ax.set_title('📊 GMM Model Selection - BIC/AIC Scores', fontweight='bold', pad=15, fontsize=14)
                ax.legend(loc='best')
                ax.grid(alpha=0.3)
                fig.tight_layout()
                
                charts['gmm_bic_aic'] = _fig_to_base64(fig)
                plt.close(fig)
                logger.info(f"✅ GMM BIC/AIC chart generated")
            except Exception as e:
                logger.warning(f"GMM BIC/AIC chart failed: {e}")
        
        # ==================================================================
        # 16. DBSCAN K-DISTANCE PLOT - Only for DBSCAN algorithm
        # ==================================================================
        if algorithm == 'dbscan' and X_scaled is not None and n_samples >= 20:
            try:
                from sklearn.neighbors import NearestNeighbors
                
                # Calculate k-distance (k=min_samples typically 5 or n/100)
                k = max(5, min(n_samples // 100, 20))
                nbrs = NearestNeighbors(n_neighbors=k).fit(X_scaled)
                distances, _ = nbrs.kneighbors(X_scaled)
                k_distances = np.sort(distances[:, k-1])[::-1]
                
                fig, ax = plt.subplots(figsize=(10, 6))
                
                ax.plot(range(1, len(k_distances) + 1), k_distances, 'b-', linewidth=2)
                
                # Find elbow point (approximate)
                kneedle_idx = len(k_distances) // 4
                if kneedle_idx > 0:
                    suggested_eps = k_distances[kneedle_idx]
                    ax.axhline(y=suggested_eps, color='red', linestyle='--', 
                              label=f'Suggested eps ≈ {suggested_eps:.3f}')
                    ax.scatter([kneedle_idx], [suggested_eps], c='red', s=100, zorder=10)
                
                ax.set_xlabel('Points (sorted by distance)', fontweight='bold', fontsize=12)
                ax.set_ylabel(f'{k}-th Nearest Neighbor Distance', fontweight='bold', fontsize=12)
                ax.set_title('📈 DBSCAN Epsilon Selection (K-Distance Graph)', fontweight='bold', pad=15, fontsize=14)
                ax.legend(loc='best')
                ax.grid(alpha=0.3)
                fig.tight_layout()
                
                charts['dbscan_kdist'] = _fig_to_base64(fig)
                plt.close(fig)
                logger.info(f"✅ DBSCAN k-distance chart generated")
            except Exception as e:
                logger.warning(f"DBSCAN k-distance chart failed: {e}")
        
        # ==================================================================
        # 17. SPECTRAL AFFINITY MATRIX - Only for Spectral algorithm
        # ==================================================================
        if algorithm == 'spectral' and X_scaled is not None and n_samples <= 500:
            try:
                from sklearn.metrics import pairwise_distances
                
                # Sample if too large
                sample_size = min(200, n_samples)
                if n_samples > sample_size:
                    indices = np.random.choice(n_samples, sample_size, replace=False)
                    X_sample = X_scaled[indices]
                    labels_sample = labels[indices]
                else:
                    X_sample = X_scaled
                    labels_sample = labels
                
                # Compute affinity matrix (RBF kernel)
                gamma = 1.0 / X_sample.shape[1]
                distances = pairwise_distances(X_sample)
                affinity = np.exp(-gamma * distances ** 2)
                
                # Sort by cluster labels for better visualization
                sorted_indices = np.argsort(labels_sample)
                affinity_sorted = affinity[sorted_indices][:, sorted_indices]
                
                fig, ax = plt.subplots(figsize=(10, 8))
                
                sns.heatmap(affinity_sorted, cmap='viridis', ax=ax,
                           xticklabels=False, yticklabels=False,
                           cbar_kws={'label': 'Affinity (similarity)'})
                
                ax.set_title('🔗 Spectral Clustering - Affinity Matrix', fontweight='bold', pad=15, fontsize=14)
                ax.set_xlabel('Samples (sorted by cluster)', fontweight='bold', fontsize=12)
                ax.set_ylabel('Samples (sorted by cluster)', fontweight='bold', fontsize=12)
                fig.tight_layout()
                
                charts['spectral_affinity'] = _fig_to_base64(fig)
                plt.close(fig)
                logger.info(f"✅ Spectral affinity matrix chart generated")
            except Exception as e:
                logger.warning(f"Spectral affinity chart failed: {e}")
        
        # ==================================================================
        # 19. FEATURE IMPORTANCE - Variance-based importance for clustering
        # ==================================================================
        if X_scaled is not None and feature_names and n_clusters >= 2:
            try:
                fig, ax = plt.subplots(figsize=(10, 6))
                
                # Compute feature importance using between-cluster variance ratio
                importances = []
                for j in range(X_scaled.shape[1]):
                    overall_mean = X_scaled[:, j].mean()
                    between_var = sum(
                        np.sum(labels == label) * (X_scaled[labels == label, j].mean() - overall_mean) ** 2
                        for label in unique_labels
                    )
                    total_var = np.var(X_scaled[:, j]) * len(X_scaled)
                    importances.append(between_var / max(total_var, 1e-10))
                
                importances = np.array(importances)
                sorted_idx = np.argsort(importances)[::-1]
                top_n = min(15, len(feature_names))
                top_idx = sorted_idx[:top_n]
                
                feat_labels = [feature_names[i] if i < len(feature_names) else f"F{i}" for i in top_idx]
                feat_values = importances[top_idx]
                
                bars = ax.barh(range(top_n), feat_values[::-1], color=plt.cm.viridis(np.linspace(0.3, 0.9, top_n)))
                ax.set_yticks(range(top_n))
                ax.set_yticklabels(feat_labels[::-1], fontsize=10)
                ax.set_xlabel('Cluster Separation Importance', fontweight='bold', fontsize=12)
                ax.set_title('⭐ Feature Importance for Cluster Separation', fontweight='bold', pad=15, fontsize=14)
                ax.grid(alpha=0.3, axis='x')
                fig.tight_layout()
                
                charts['feature_importance'] = _fig_to_base64(fig)
                plt.close(fig)
                logger.info("✅ Feature importance chart generated")
            except Exception as e:
                logger.warning(f"Feature importance chart failed: {e}")
        
        # ==================================================================
        # 20. SILHOUETTE COMPARISON - Compare silhouette across k values
        # ==================================================================
        if k_scores and len(k_scores) >= 3:
            try:
                fig, ax = plt.subplots(figsize=(10, 6))
                
                k_vals = sorted(k_scores.keys())
                s_scores = [k_scores[k] for k in k_vals]
                
                colors_bar = ['#ef4444' if s < 0.25 else '#f59e0b' if s < 0.5 else '#22c55e' if s < 0.7 else '#3b82f6' for s in s_scores]
                bars = ax.bar(k_vals, s_scores, color=colors_bar, edgecolor='white', linewidth=1.5, width=0.6)
                
                # Add value labels
                for bar, score in zip(bars, s_scores):
                    ax.text(bar.get_x() + bar.get_width() / 2., bar.get_height() + 0.01,
                            f'{score:.3f}', ha='center', va='bottom', fontweight='bold', fontsize=10)
                
                # Add quality zones
                ax.axhline(y=0.25, color='#ef4444', linestyle='--', alpha=0.4, label='Poor (<0.25)')
                ax.axhline(y=0.5, color='#f59e0b', linestyle='--', alpha=0.4, label='Fair (0.25-0.5)')
                ax.axhline(y=0.7, color='#22c55e', linestyle='--', alpha=0.4, label='Good (0.5-0.7)')
                
                ax.set_xlabel('Number of Clusters (k)', fontweight='bold', fontsize=12)
                ax.set_ylabel('Silhouette Score', fontweight='bold', fontsize=12)
                ax.set_title('📊 Silhouette Score Comparison Across k', fontweight='bold', pad=15, fontsize=14)
                ax.set_xticks(k_vals)
                ax.legend(loc='best', fontsize=9)
                ax.set_ylim(0, max(s_scores) * 1.2 if s_scores else 1)
                ax.grid(alpha=0.3, axis='y')
                fig.tight_layout()
                
                charts['silhouette_comparison'] = _fig_to_base64(fig)
                plt.close(fig)
                logger.info("✅ Silhouette comparison chart generated")
            except Exception as e:
                logger.warning(f"Silhouette comparison chart failed: {e}")
        
        logger.info(f"✅ Generated {len(charts)} clustering charts")
        
    except Exception as e:
        logger.error(f"Chart generation error: {e}")
        import traceback
        traceback.print_exc()
    
    return charts


def _fig_to_base64(fig) -> str:
    """Convert matplotlib figure to base64 string."""
    buf = io.BytesIO()
    fig.savefig(buf, format='png', dpi=150, bbox_inches='tight', facecolor='white')
    buf.seek(0)
    return f"data:image/png;base64,{base64.b64encode(buf.getvalue()).decode('utf-8')}"


def _save_clustering_model(user_id: str, model_id: str, model_data: Dict):
    """Save clustering model for predictions."""
    try:
        paths = get_user_paths(user_id)
        models_dir = paths.get("models", paths["base"] / "models")
        models_dir.mkdir(parents=True, exist_ok=True)
        
        model_path = models_dir / f"{model_id}.json"
        with open(model_path, 'w') as f:
            json.dump(model_data, f)
        
        # Also save as active clustering model
        active_path = models_dir / "active_clustering.json"
        with open(active_path, 'w') as f:
            json.dump({'model_id': model_id, **model_data}, f)
        
        logger.info(f"✅ Saved clustering model: {model_path}")
    except Exception as e:
        logger.warning(f"Failed to save model: {e}")


# ===========================================================================
# 🔮 CLUSTER PREDICTION ENDPOINT
# ===========================================================================

@router.post("/clustering/predict")
async def predict_cluster(
    request: ClusterPredictRequest,
    x_user_id: Optional[str] = Header(None, alias="X-User-ID")
):
    """
    Predict which cluster a new data point belongs to.
    
    Uses the saved model to find the nearest cluster centroid.
    """
    user_id = request.user_id or x_user_id
    if not user_id:
        raise HTTPException(status_code=400, detail="User ID required")
    
    try:
        # Load model
        paths = get_user_paths(user_id)
        models_dir = paths.get("models", paths["base"] / "models")
        
        model_path = models_dir / f"{request.model_id}.json"
        if not model_path.exists():
            # Try active model
            model_path = models_dir / "active_clustering.json"
        
        if not model_path.exists():
            raise HTTPException(status_code=404, detail="Clustering model not found. Run clustering first.")
        
        with open(model_path, 'r') as f:
            model_data = json.load(f)
        
        # Prepare input features
        feature_columns = model_data['feature_columns']
        features_array = np.array([request.features.get(col, 0) for col in feature_columns]).reshape(1, -1)
        
        # Scale features
        scaler_mean = np.array(model_data['scaler_mean'])
        scaler_scale = np.array(model_data['scaler_scale'])
        features_scaled = (features_array - scaler_mean) / scaler_scale
        
        # Find nearest centroid
        centroids = np.array(model_data['centroids_scaled'])
        if len(centroids) == 0:
            raise HTTPException(status_code=400, detail="No cluster centroids available")
        
        distances = np.linalg.norm(centroids - features_scaled, axis=1)
        predicted_cluster = int(np.argmin(distances))
        confidence = float(1.0 / (1.0 + distances[predicted_cluster]))
        
        # Get cluster characteristics
        cluster_name = f"Cluster {predicted_cluster}"
        
        return {
            'success': True,
            'cluster': predicted_cluster,
            'cluster_name': cluster_name,
            'confidence': confidence,
            'distances_to_centroids': distances.tolist(),
            'algorithm': model_data.get('algorithm', 'unknown'),
            'n_clusters': model_data.get('n_clusters', len(centroids)),
        }
        
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Cluster prediction failed: {e}")
        raise HTTPException(status_code=500, detail=str(e))


@router.get("/clustering/models/{user_id}/active")
async def get_active_clustering_model(user_id: str):
    """Get the active clustering model for a user."""
    try:
        paths = get_user_paths(user_id)
        models_dir = paths.get("models", paths["base"] / "models")
        active_path = models_dir / "active_clustering.json"
        
        if not active_path.exists():
            return {'success': True, 'has_model': False}
        
        with open(active_path, 'r') as f:
            model_data = json.load(f)
        
        return {
            'success': True,
            'has_model': True,
            'model_id': model_data.get('model_id'),
            'algorithm': model_data.get('algorithm'),
            'n_clusters': model_data.get('n_clusters'),
            'created_at': model_data.get('created_at'),
        }
    except Exception as e:
        logger.error(f"Failed to get active model: {e}")
        return {'success': False, 'error': str(e)}


# ===========================================================================
# LEGACY FILE UPLOAD ENDPOINTS (Keep for backward compatibility)
# ===========================================================================


@router.post("/cluster")
async def perform_clustering(
    file: UploadFile = File(...),
    algorithm: str = Form("kmeans"),
    n_clusters: Optional[int] = Form(3),
    features: Optional[str] = Form(None),
    eps: Optional[float] = Form(0.5),
    min_samples: Optional[int] = Form(5),
    normalize: bool = Form(True)
):
    """
    Perform clustering analysis on uploaded data
    
    Parameters:
    - algorithm: 'kmeans', 'dbscan', 'gmm', 'spectral'
    - n_clusters: Number of clusters (for kmeans, gmm, spectral)
    - features: Comma-separated feature names (optional)
    - eps: DBSCAN epsilon parameter
    - min_samples: DBSCAN min samples parameter
    - normalize: Whether to normalize features
    """
    try:
        # Read uploaded file
        contents = await file.read()
        df = pd.read_csv(io.BytesIO(contents))
        
        logger.info(f"📊 Clustering {len(df)} rows with {algorithm.upper()}")
        
        # Select features
        if features:
            feature_list = [f.strip() for f in features.split(",")]
            X = df[feature_list].select_dtypes(include=[np.number])
        else:
            X = df.select_dtypes(include=[np.number])
        
        if X.empty:
            raise HTTPException(status_code=400, detail="No numeric features found")
        
        # Handle missing values
        X = X.fillna(X.mean())
        
        # Normalize if requested
        if normalize:
            scaler = StandardScaler()
            X_scaled = scaler.fit_transform(X)
        else:
            X_scaled = X.values
        
        # Perform clustering
        if algorithm == "kmeans":
            model = KMeans(n_clusters=n_clusters, random_state=42, n_init=10)
            labels = model.fit_predict(X_scaled)
            cluster_centers = model.cluster_centers_
            
        elif algorithm == "dbscan":
            model = DBSCAN(eps=eps, min_samples=min_samples)
            labels = model.fit_predict(X_scaled)
            cluster_centers = None
            n_clusters = len(set(labels)) - (1 if -1 in labels else 0)
            
        elif algorithm == "gmm":
            model = GaussianMixture(n_components=n_clusters, random_state=42)
            labels = model.fit_predict(X_scaled)
            cluster_centers = model.means_
            
        elif algorithm == "spectral":
            model = SpectralClustering(n_clusters=n_clusters, random_state=42)
            labels = model.fit_predict(X_scaled)
            cluster_centers = None
            
        else:
            raise HTTPException(status_code=400, detail=f"Unknown algorithm: {algorithm}")
        
        # Calculate metrics (only if more than 1 cluster)
        metrics = {}
        if len(set(labels)) > 1 and len(set(labels)) < len(X_scaled):
            try:
                metrics["silhouette_score"] = float(silhouette_score(X_scaled, labels))
                metrics["davies_bouldin_score"] = float(davies_bouldin_score(X_scaled, labels))
                metrics["calinski_harabasz_score"] = float(calinski_harabasz_score(X_scaled, labels))
            except Exception as e:
                logger.warning(f"Could not calculate metrics: {e}")
        
        # PCA for visualization (2D)
        pca = PCA(n_components=2)
        X_pca = pca.fit_transform(X_scaled)
        
        # Prepare results
        cluster_counts = pd.Series(labels).value_counts().to_dict()
        
        # =============================================================
        # 🛡️ PRODUCTION INTELLIGENCE: Data quality & reliability
        # =============================================================
        reliability_score = 75  # Default
        validation_warnings = []
        data_quality = {}
        
        try:
            # 1. Check data quality
            n_samples = len(df)
            n_features = X.shape[1]
            missing_ratio = df.isna().sum().sum() / df.size if df.size > 0 else 0
            duplicate_ratio = df.duplicated().sum() / n_samples if n_samples > 0 else 0
            
            data_quality = {
                'n_samples': n_samples,
                'n_features': n_features,
                'missing_ratio': float(missing_ratio),
                'duplicate_ratio': float(duplicate_ratio),
                'size_category': 'small' if n_samples < 500 else 'medium' if n_samples < 5000 else 'large'
            }
            
            # Data quality warnings
            if missing_ratio > 0.2:
                validation_warnings.append(f"⚠️ High missing data: {missing_ratio:.1%} - may affect cluster quality")
            if duplicate_ratio > 0.1:
                validation_warnings.append(f"⚠️ Many duplicates: {duplicate_ratio:.1%} - consider removing")
            if n_samples < 100:
                validation_warnings.append(f"⚠️ Small dataset ({n_samples} samples) - clusters may be unreliable")
            if n_features > 50:
                validation_warnings.append(f"⚠️ High dimensionality ({n_features} features) - consider dimensionality reduction")
            
            # 2. Compute reliability score for clustering
            # Based on silhouette score, cluster separation, and sample size
            silhouette = metrics.get('silhouette_score', 0)
            calinski = metrics.get('calinski_harabasz_score', 0)
            davies = metrics.get('davies_bouldin_score', float('inf'))
            
            # Silhouette contributes 40 points (scaled from -1 to 1)
            silhouette_points = max(0, (silhouette + 1) / 2 * 40)
            
            # Calinski-Harabasz contributes 25 points (log-scaled)
            import math
            calinski_points = min(25, math.log(calinski + 1) * 3) if calinski > 0 else 0
            
            # Davies-Bouldin contributes 20 points (lower is better)
            davies_points = max(0, 20 - davies * 5) if davies < float('inf') else 10
            
            # Sample size contributes 15 points
            if n_samples >= 1000:
                size_points = 15
            elif n_samples >= 500:
                size_points = 12
            elif n_samples >= 100:
                size_points = 8
            else:
                size_points = 5
            
            reliability_score = min(100, silhouette_points + calinski_points + davies_points + size_points)
            
            # Add cluster quality assessment
            if silhouette < 0.2:
                validation_warnings.append(f"⚠️ Low silhouette score ({silhouette:.3f}) - clusters may overlap significantly")
            if silhouette > 0.7:
                logger.info(f"✅ Excellent cluster separation (silhouette={silhouette:.3f})")
            
            logger.info(f"🛡️ Clustering Reliability Score: {reliability_score:.1f}/100")
            
        except Exception as intel_err:
            logger.warning(f"Production Intelligence check failed: {intel_err}")
        
        result = {
            "algorithm": algorithm,
            "n_clusters": n_clusters if algorithm != "dbscan" else len(set(labels)) - (1 if -1 in labels else 0),
            "n_samples": len(df),
            "n_features": X.shape[1],
            "features_used": list(X.columns),
            "labels": labels.tolist(),
            "cluster_counts": {str(k): int(v) for k, v in cluster_counts.items()},
            "metrics": metrics,
            "visualization": {
                "x": X_pca[:, 0].tolist(),
                "y": X_pca[:, 1].tolist(),
                "explained_variance": pca.explained_variance_ratio_.tolist()
            },
            # 🛡️ PRODUCTION INTELLIGENCE outputs
            "reliability_score": reliability_score,
            "validation_warnings": validation_warnings if validation_warnings else None,
            "data_quality": data_quality,
        }
        
        # Add cluster centers if available
        if cluster_centers is not None:
            if normalize:
                cluster_centers = scaler.inverse_transform(cluster_centers)
            result["cluster_centers"] = cluster_centers.tolist()
        
        logger.info(f"✅ Clustering complete: {result['n_clusters']} clusters found")
        
        return result
        
    except pd.errors.EmptyDataError:
        raise HTTPException(status_code=400, detail="Empty CSV file")
    except KeyError as e:
        raise HTTPException(status_code=400, detail=f"Feature not found: {str(e)}")
    except Exception as e:
        logger.error(f"❌ Clustering error: {e}")
        raise HTTPException(status_code=500, detail=str(e))


@router.post("/cluster/optimal")
async def find_optimal_clusters(
    file: UploadFile = File(...),
    algorithm: str = Form("kmeans"),
    max_clusters: int = Form(10),
    features: Optional[str] = Form(None),
    normalize: bool = Form(True)
):
    """
    Find optimal number of clusters using elbow method and silhouette analysis
    """
    try:
        # Read uploaded file
        contents = await file.read()
        df = pd.read_csv(io.BytesIO(contents))
        
        # Select features
        if features:
            feature_list = [f.strip() for f in features.split(",")]
            X = df[feature_list].select_dtypes(include=[np.number])
        else:
            X = df.select_dtypes(include=[np.number])
        
        if X.empty:
            raise HTTPException(status_code=400, detail="No numeric features found")
        
        # Handle missing values
        X = X.fillna(X.mean())
        
        # Normalize if requested
        if normalize:
            scaler = StandardScaler()
            X_scaled = scaler.fit_transform(X)
        else:
            X_scaled = X.values
        
        # Test different numbers of clusters
        inertias = []
        silhouettes = []
        k_range = range(2, min(max_clusters + 1, len(X_scaled)))
        
        for k in k_range:
            if algorithm == "kmeans":
                model = KMeans(n_clusters=k, random_state=42, n_init=10)
                labels = model.fit_predict(X_scaled)
                inertias.append(model.inertia_)
            elif algorithm == "gmm":
                model = GaussianMixture(n_components=k, random_state=42)
                labels = model.fit_predict(X_scaled)
                inertias.append(-model.bic(X_scaled))  # Negative BIC
            else:
                raise HTTPException(status_code=400, detail=f"Optimal search not supported for {algorithm}")
            
            # Calculate silhouette score
            silhouette = silhouette_score(X_scaled, labels)
            silhouettes.append(silhouette)
        
        # Find optimal k (highest silhouette)
        optimal_k = list(k_range)[np.argmax(silhouettes)]
        
        return {
            "algorithm": algorithm,
            "k_range": list(k_range),
            "inertias": inertias,
            "silhouette_scores": silhouettes,
            "optimal_k": optimal_k,
            "max_silhouette": max(silhouettes)
        }
        
    except Exception as e:
        logger.error(f"❌ Optimal clustering error: {e}")
        raise HTTPException(status_code=500, detail=str(e))


@router.get("/cluster/info")
async def get_clustering_info():
    """
    Get information about available clustering algorithms
    """
    return {
        "algorithms": {
            "kmeans": {
                "name": "K-Means",
                "description": "Partition-based clustering that minimizes within-cluster variance",
                "parameters": ["n_clusters"],
                "pros": ["Fast", "Scalable", "Works well with spherical clusters"],
                "cons": ["Requires specifying k", "Sensitive to outliers", "Assumes spherical clusters"]
            },
            "dbscan": {
                "name": "DBSCAN",
                "description": "Density-based clustering that can find arbitrarily shaped clusters",
                "parameters": ["eps", "min_samples"],
                "pros": ["No need to specify k", "Finds arbitrary shapes", "Identifies outliers"],
                "cons": ["Sensitive to parameters", "Struggles with varying densities"]
            },
            "gmm": {
                "name": "Gaussian Mixture Model",
                "description": "Probabilistic model that assumes data is generated from mixture of Gaussians",
                "parameters": ["n_clusters"],
                "pros": ["Soft clustering", "Provides probabilities", "Flexible cluster shapes"],
                "cons": ["Computationally expensive", "Sensitive to initialization"]
            },
            "spectral": {
                "name": "Spectral Clustering",
                "description": "Uses eigenvalues of similarity matrix to reduce dimensions before clustering",
                "parameters": ["n_clusters"],
                "pros": ["Works with non-convex clusters", "Graph-based approach"],
                "cons": ["Computationally expensive", "Memory intensive for large datasets"]
            }
        },
        "metrics": {
            "silhouette_score": "Measures how similar a point is to its own cluster vs other clusters (-1 to 1, higher is better)",
            "davies_bouldin_score": "Average similarity ratio of each cluster with most similar cluster (lower is better)",
            "calinski_harabasz_score": "Ratio of between-cluster to within-cluster dispersion (higher is better)"
        }
    }


@router.get("/clustering/download-model/{user_id}")
async def download_clustering_model(user_id: str):
    """
    Download the trained clustering model as PKL file.
    If no PKL exists, creates one from the JSON model data.
    """
    from fastapi.responses import FileResponse, Response
    
    try:
        paths = get_user_paths(user_id)
        models_dir = paths.get("models", paths["base"] / "models")
        models_dir.mkdir(parents=True, exist_ok=True)
        
        # Try to find existing PKL
        pkl_files = list(models_dir.glob("clustering_model_*.pkl"))
        
        if pkl_files:
            latest_pkl = max(pkl_files, key=lambda p: p.stat().st_mtime)
            return FileResponse(
                path=str(latest_pkl),
                media_type='application/octet-stream',
                filename=latest_pkl.name
            )
        
        # No PKL found — create one from the active JSON model
        active_json = models_dir / "active_clustering.json"
        if not active_json.exists():
            # Try any clustering JSON
            json_files = list(models_dir.glob("clustering_*.json"))
            if json_files:
                active_json = max(json_files, key=lambda p: p.stat().st_mtime)
            else:
                raise HTTPException(status_code=404, detail="No clustering model found. Train a model first.")
        
        with open(active_json, 'r') as f:
            model_json = json.load(f)
        
        # Reconstruct a PKL with scaler from JSON data
        scaler_mean = model_json.get('scaler_mean')
        scaler_scale = model_json.get('scaler_scale')
        reconstructed_scaler = None
        if scaler_mean and scaler_scale:
            reconstructed_scaler = StandardScaler()
            reconstructed_scaler.mean_ = np.array(scaler_mean)
            reconstructed_scaler.scale_ = np.array(scaler_scale)
            reconstructed_scaler.var_ = np.array(scaler_scale) ** 2
            reconstructed_scaler.n_features_in_ = len(scaler_mean)
        
        pkl_data = {
            'algorithm': model_json.get('algorithm', 'kmeans'),
            'n_clusters': model_json.get('n_clusters', 3),
            'scaler': reconstructed_scaler,
            'feature_columns': model_json.get('feature_columns', []),
            'centroids_scaled': np.array(model_json.get('centroids_scaled', [])) if model_json.get('centroids_scaled') else None,
            'labels': np.array(model_json.get('labels', [])),
            'model_id': model_json.get('model_id', 'unknown'),
            'created_at': model_json.get('created_at', ''),
            'silhouette_score': model_json.get('silhouette_score', 0),
        }
        
        # Save as PKL for future use
        model_id = model_json.get('model_id', f"clustering_{uuid.uuid4().hex[:8]}")
        pkl_filename = f"clustering_model_{model_id}.pkl"
        pkl_path = models_dir / pkl_filename
        with open(pkl_path, 'wb') as f:
            pickle.dump(pkl_data, f)
        logger.info(f"✅ Created PKL from JSON: {pkl_path}")
        
        return FileResponse(
            path=str(pkl_path),
            media_type='application/octet-stream',
            filename=pkl_filename
        )
        
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"❌ Download model error: {e}")
        raise HTTPException(status_code=500, detail=str(e))


@router.get("/clustering/download-data/{user_id}")
async def download_clustered_data(user_id: str):
    """
    Download the cleaned data with cluster assignments.
    If no clustered CSV exists, creates one from the original data + model labels.
    """
    from fastapi.responses import FileResponse, Response
    
    try:
        paths = get_user_paths(user_id)
        files_dir = paths.get("files", paths["base"] / "files")
        models_dir = paths.get("models", paths["base"] / "models")
        files_dir.mkdir(parents=True, exist_ok=True)
        models_dir.mkdir(parents=True, exist_ok=True)
        
        # Try to find existing clustered data
        csv_files = list(files_dir.glob("clustered_data_*.csv"))
        
        if csv_files:
            latest_csv = max(csv_files, key=lambda p: p.stat().st_mtime)
            return FileResponse(
                path=str(latest_csv),
                media_type='text/csv',
                filename=latest_csv.name
            )
        
        # No clustered data found — try to reconstruct from model labels + original data
        active_json = models_dir / "active_clustering.json"
        if not active_json.exists():
            json_files = list(models_dir.glob("clustering_*.json"))
            if json_files:
                active_json = max(json_files, key=lambda p: p.stat().st_mtime)
            else:
                raise HTTPException(status_code=404, detail="No clustered data found. Run clustering first.")
        
        with open(active_json, 'r') as f:
            model_json = json.load(f)
        
        labels = model_json.get('labels', [])
        feature_columns = model_json.get('feature_columns', [])
        
        if not labels:
            raise HTTPException(status_code=404, detail="No cluster labels found. Run clustering first.")
        
        # Try to find the most recent uploaded CSV to attach labels to
        all_csv = list(files_dir.glob("*.csv"))
        # Exclude already-clustered files
        original_csvs = [f for f in all_csv if not f.name.startswith("clustered_data_") and not f.name.startswith("cleaned_")]
        
        if original_csvs:
            # Use the most recent original CSV
            data_file = max(original_csvs, key=lambda p: p.stat().st_mtime)
            df = pd.read_csv(data_file)
            
            if len(df) == len(labels):
                df['Cluster'] = labels
                df['Cluster_Name'] = [f'Cluster_{l}' if l >= 0 else 'Noise' for l in labels]
                
                # Create and save clustered CSV
                model_id = model_json.get('model_id', f"clustering_{uuid.uuid4().hex[:8]}")
                clustered_filename = f"clustered_data_{model_id}.csv"
                clustered_path = files_dir / clustered_filename
                df.to_csv(clustered_path, index=False)
                logger.info(f"✅ Reconstructed clustered data: {clustered_path}")
                
                return FileResponse(
                    path=str(clustered_path),
                    media_type='text/csv',
                    filename=clustered_filename
                )
        
        # Last resort: just return labels as CSV
        labels_df = pd.DataFrame({'sample_index': range(len(labels)), 'cluster': labels})
        csv_content = labels_df.to_csv(index=False)
        return Response(
            content=csv_content,
            media_type='text/csv',
            headers={"Content-Disposition": "attachment; filename=cluster_labels.csv"}
        )
        
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"❌ Download data error: {e}")
        raise HTTPException(status_code=500, detail=str(e))


@router.get("/clustering/download-code/{user_id}")
async def download_clustering_code(user_id: str):
    """
    Download a complete clustering project as a ZIP file.
    Includes: model.pkl, clustered_data.csv, predict_cluster.py, train_clustering.py,
    visualize_clusters.py, api_server.py, charts/, README.md, Dockerfile, requirements.txt
    """
    from fastapi.responses import StreamingResponse

    try:
        paths = get_user_paths(user_id)
        models_dir = paths.get("models", paths["base"] / "models")
        files_dir = paths.get("files", paths["base"] / "files")
        models_dir.mkdir(parents=True, exist_ok=True)
        files_dir.mkdir(parents=True, exist_ok=True)

        # Find latest clustering model PKL
        pkl_files = list(models_dir.glob("clustering_model_*.pkl"))
        latest_pkl = max(pkl_files, key=lambda p: p.stat().st_mtime) if pkl_files else None

        # If no PKL, try to create from JSON
        if not latest_pkl:
            active_json = models_dir / "active_clustering.json"
            if not active_json.exists():
                json_files = list(models_dir.glob("clustering_*.json"))
                if json_files:
                    active_json = max(json_files, key=lambda p: p.stat().st_mtime)
                else:
                    raise HTTPException(status_code=404, detail="No clustering model found. Train a model first.")
            
            with open(active_json, 'r') as f:
                model_json = json.load(f)
            
            # Reconstruct PKL from JSON
            scaler_mean = model_json.get('scaler_mean')
            scaler_scale = model_json.get('scaler_scale')
            reconstructed_scaler = None
            if scaler_mean and scaler_scale:
                reconstructed_scaler = StandardScaler()
                reconstructed_scaler.mean_ = np.array(scaler_mean)
                reconstructed_scaler.scale_ = np.array(scaler_scale)
                reconstructed_scaler.var_ = np.array(scaler_scale) ** 2
                reconstructed_scaler.n_features_in_ = len(scaler_mean)
            
            pkl_data_dict = {
                'algorithm': model_json.get('algorithm', 'kmeans'),
                'n_clusters': model_json.get('n_clusters', 3),
                'scaler': reconstructed_scaler,
                'feature_columns': model_json.get('feature_columns', []),
                'centroids_scaled': np.array(model_json.get('centroids_scaled', [])) if model_json.get('centroids_scaled') else None,
                'labels': np.array(model_json.get('labels', [])),
                'model_id': model_json.get('model_id', 'unknown'),
                'created_at': model_json.get('created_at', ''),
                'silhouette_score': model_json.get('silhouette_score', 0),
            }
            
            model_id = model_json.get('model_id', f"clustering_{uuid.uuid4().hex[:8]}")
            pkl_filename = f"clustering_model_{model_id}.pkl"
            latest_pkl = models_dir / pkl_filename
            with open(latest_pkl, 'wb') as f:
                pickle.dump(pkl_data_dict, f)
            logger.info(f"✅ Created PKL from JSON for ZIP: {latest_pkl}")

        # Load model metadata from PKL
        clustering_meta = {}
        pkl_data = {}
        try:
            with open(latest_pkl, 'rb') as f:
                pkl_data = pickle.load(f)
            clustering_meta = {
                'algorithm': pkl_data.get('algorithm', 'kmeans'),
                'n_clusters': pkl_data.get('n_clusters', 3),
                'silhouette_score': pkl_data.get('silhouette_score', 0),
                'feature_columns': pkl_data.get('feature_columns', []),
                'cluster_profiles': pkl_data.get('cluster_profiles', {}),
            }
        except Exception as e:
            logger.warning(f"Could not load PKL metadata: {e}")

        # Find latest clustered data CSV
        csv_files = list(files_dir.glob("clustered_data_*.csv"))
        cleaned_data_path = max(csv_files, key=lambda p: p.stat().st_mtime) if csv_files else None
        
        # If no clustered CSV, try to reconstruct
        if not cleaned_data_path:
            try:
                labels = pkl_data.get('labels', []) if pkl_data else []
                if hasattr(labels, 'tolist'):
                    labels = labels.tolist()
                all_csv = list(files_dir.glob("*.csv"))
                original_csvs = [f for f in all_csv if not f.name.startswith("clustered_data_") and not f.name.startswith("cleaned_")]
                if original_csvs and labels:
                    data_file = max(original_csvs, key=lambda p: p.stat().st_mtime)
                    df = pd.read_csv(data_file)
                    if len(df) == len(labels):
                        df['Cluster'] = labels
                        df['Cluster_Name'] = [f'Cluster_{l}' if l >= 0 else 'Noise' for l in labels]
                        clustered_filename = f"clustered_data_reconstructed.csv"
                        cleaned_data_path = files_dir / clustered_filename
                        df.to_csv(cleaned_data_path, index=False)
                        logger.info(f"✅ Reconstructed clustered data for ZIP: {cleaned_data_path}")
            except Exception as e:
                logger.warning(f"Could not reconstruct clustered data: {e}")

        if cleaned_data_path:
            try:
                import pandas as _pd
                _df = _pd.read_csv(cleaned_data_path)
                clustering_meta['n_samples'] = len(_df)
            except Exception:
                pass

        # Load charts if available
        charts_data = None
        try:
            # Check for active clustering charts stored as JSON
            charts_json = models_dir / "active_clustering_charts.json"
            if charts_json.exists():
                with open(charts_json, 'r') as f:
                    charts_data = json.load(f)
            else:
                # Try model_persistence charts
                try:
                    from ml.model_persistence import model_persistence
                    charts_data = model_persistence.get_charts(user_id)
                except Exception:
                    pass
        except Exception as e:
            logger.warning(f"Could not load charts: {e}")

        # Generate ZIP
        from ml.ml_code_generator import generate_clustering_code_zip
        zip_buffer = generate_clustering_code_zip(
            pkl_path=latest_pkl,
            cleaned_data_path=cleaned_data_path,
            charts_data=charts_data,
            clustering_meta=clustering_meta,
        )

        return StreamingResponse(
            zip_buffer,
            media_type="application/zip",
            headers={"Content-Disposition": "attachment; filename=clustering_project.zip"}
        )

    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"❌ Download clustering code error: {e}")
        raise HTTPException(status_code=500, detail=str(e))