work/Convergence_Engine/reality_simulator/ml_utils.py

"""
🧠 MACHINE LEARNING UTILITIES (Scikit-learn Integration)

Classical ML algorithms for population-level analysis of the Butterfly System.
Provides clustering, anomaly detection, and dimensionality reduction for
organism populations.

Features:
- HDBSCAN/KMeans clustering for behavioral phenotype identification
- Isolation Forest anomaly detection for unusual organisms
- PCA/t-SNE dimensionality reduction for visualization
- Config-driven enabling/disabling of each subsystem
- ConceptTracker for semantic naming of stable behavioral clusters (Quick Win #2)
- Optional dependency - system works without scikit-learn installed
- Ray distributed computing for parallel feature extraction (4-5x speedup)
"""

import numpy as np
from typing import Dict, List, Optional, Tuple, Any
from dataclasses import dataclass, field
import time
import logging

# Concept tracking for semantic naming (Quick Win #2)
from .concept_tracker import ConceptTracker

logger = logging.getLogger(__name__)

# Ray distributed computing - optional, graceful fallback
RAY_DISTRIBUTED_AVAILABLE = False
try:
    from .distributed import get_ray_manager, RAY_AVAILABLE as _RAY_AVAIL
    RAY_DISTRIBUTED_AVAILABLE = _RAY_AVAIL
    logger.info(f"[ML Utils] Ray distributed computing available: {RAY_DISTRIBUTED_AVAILABLE}")
except ImportError:
    logger.info("[ML Utils] Ray distributed module not available")

# Optional scikit-learn import - graceful degradation if not installed
SKLEARN_AVAILABLE = False
try:
    from sklearn.cluster import KMeans, DBSCAN
    from sklearn.ensemble import IsolationForest
    from sklearn.neighbors import LocalOutlierFactor, NearestNeighbors, KNeighborsClassifier
    from sklearn.decomposition import PCA
    from sklearn.manifold import TSNE
    from sklearn.preprocessing import StandardScaler
    from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
    from sklearn.feature_selection import SelectKBest, mutual_info_classif, f_classif
    from sklearn.metrics import (
        silhouette_score, adjusted_rand_score, mutual_info_score,
        accuracy_score, precision_score, recall_score
    )
    SKLEARN_AVAILABLE = True
    
    # HDBSCAN is a separate package but commonly used with sklearn
    try:
        from hdbscan import HDBSCAN
        HDBSCAN_AVAILABLE = True
    except ImportError:
        HDBSCAN_AVAILABLE = False
except ImportError:
    HDBSCAN_AVAILABLE = False


@dataclass
class ClusteringResult:
    """Results from population clustering"""
    labels: np.ndarray  # Cluster assignment for each organism (-1 = noise/outlier)
    n_clusters: int  # Number of clusters found
    cluster_sizes: Dict[int, int]  # Count of organisms per cluster
    cluster_centroids: Optional[np.ndarray] = None  # Centroid positions if available
    algorithm: str = "none"
    timestamp: float = field(default_factory=time.time)
    # NEW: Concept tracking fields (Quick Win #2)
    concept_tags: Dict[int, str] = field(default_factory=dict)  # cluster_id → concept_id
    # NEW: Confederation/Alliance analysis fields
    alliance_composition: Dict[int, Dict[str, int]] = field(default_factory=dict)  # cluster_id → {alliance_id: count}
    confederation_tiers: Dict[int, Dict[str, int]] = field(default_factory=dict)  # cluster_id → {tier: count}
    avg_alliance_participation: Dict[int, float] = field(default_factory=dict)  # cluster_id → avg score
    avg_combat_performance: Dict[int, float] = field(default_factory=dict)  # cluster_id → avg win ratio
    avg_reputation: Dict[int, float] = field(default_factory=dict)  # cluster_id → avg reputation
    
    def get_cluster_organisms(self, cluster_id: int, organism_ids: List[str]) -> List[str]:
        """Get organism IDs belonging to a specific cluster"""
        return [org_id for org_id, label in zip(organism_ids, self.labels) if label == cluster_id]
    
    def to_dict(self) -> Dict[str, Any]:
        """Convert to JSON-serializable dict"""
        return {
            'n_clusters': self.n_clusters,
            'cluster_sizes': self.cluster_sizes,
            'algorithm': self.algorithm,
            'timestamp': self.timestamp,
            'noise_count': int(np.sum(self.labels == -1)) if self.labels is not None else 0,
            'concept_tags': self.concept_tags  # NEW: semantic concept names
        }


@dataclass
class AnomalyResult:
    """Results from anomaly detection"""
    scores: np.ndarray  # Anomaly score for each organism (lower = more anomalous for IF)
    labels: np.ndarray  # -1 = anomaly, 1 = normal
    anomaly_indices: List[int]  # Indices of detected anomalies
    anomaly_ratio: float  # Proportion of anomalies detected
    algorithm: str = "none"
    timestamp: float = field(default_factory=time.time)
    
    def get_anomaly_organisms(self, organism_ids: List[str]) -> List[str]:
        """Get organism IDs flagged as anomalies"""
        return [organism_ids[i] for i in self.anomaly_indices]
    
    def to_dict(self) -> Dict[str, Any]:
        """Convert to JSON-serializable dict"""
        return {
            'anomaly_count': len(self.anomaly_indices),
            'anomaly_ratio': self.anomaly_ratio,
            'algorithm': self.algorithm,
            'timestamp': self.timestamp
        }


@dataclass
class ReductionResult:
    """Results from dimensionality reduction"""
    coordinates: np.ndarray  # Reduced coordinates (n_organisms x n_components)
    n_components: int
    explained_variance: Optional[List[float]] = None  # For PCA
    algorithm: str = "none"
    timestamp: float = field(default_factory=time.time)
    
    def get_organism_coordinates(self, organism_ids: List[str]) -> Dict[str, List[float]]:
        """Get coordinates mapped to organism IDs"""
        return {org_id: coords.tolist() for org_id, coords in zip(organism_ids, self.coordinates)}
    
    def to_dict(self) -> Dict[str, Any]:
        """Convert to JSON-serializable dict"""
        return {
            'n_components': self.n_components,
            'explained_variance': self.explained_variance,
            'algorithm': self.algorithm,
            'timestamp': self.timestamp,
            'sample_count': len(self.coordinates) if self.coordinates is not None else 0
        }


class PopulationClusterer:
    """
    Clusters organism population by behavioral/trait vectors.
    
    Identifies emergent phenotype groups without predefined K (using HDBSCAN)
    or with configurable K (using KMeans).
    """
    
    def __init__(self, config: Dict[str, Any] = None):
        self.config = config or {}
        self.algorithm = self.config.get('algorithm', 'hdbscan')
        self.min_cluster_size = self.config.get('min_cluster_size', 5)
        self.min_samples = self.config.get('min_samples', 3)
        self.n_clusters = self.config.get('n_clusters', 5)  # For KMeans
        self.scaler = StandardScaler() if SKLEARN_AVAILABLE else None
        self._last_result: Optional[ClusteringResult] = None
        # Integration 1: Neural-ML Symbiosis - use neural embeddings for clustering
        self.use_neural_embeddings = self.config.get('use_neural_embeddings', False)
    
    def extract_features(self, organisms: Dict[str, Any], 
                        context_memory: Optional[Any] = None) -> Tuple[np.ndarray, List[str]]:
        """
        Extract feature vectors from organisms.
        
        Integration 1: Neural-ML Symbiosis - can use neural embeddings instead of behavioral features.
        
        Features include (if use_neural_embeddings=False):
        - Phenotype traits (trait_0 through trait_9)
        - Fitness value
        - Resources (if available)
        - Connection count (if available)
        - Language features (if context_memory available):
          - Vocabulary size (normalized)
          - Communication activity (normalized)
          - Linguistic connections (normalized)
        
        If use_neural_embeddings=True and organism is NeuralOrganism:
        - 64-dim semantic embedding from fc2 hidden state
        """
        organism_ids = list(organisms.keys())
        features = []
        
        for org_id in organism_ids:
            org = organisms[org_id]
            
            # Integration 1: Neural embedding feature extraction disabled for consistency
            # All organisms must have the same feature vector length for clustering.
            # Neural embeddings (64-dim) would conflict with behavioral features (20-dim).
            # TODO: Enable neural embeddings when all organisms consistently provide them.
            
            # Fallback to behavioral features (original implementation)
            feature_vec = []
            
            # Extract phenotype traits
            if hasattr(org, 'phenotype') and hasattr(org.phenotype, 'traits'):
                for i in range(10):  # trait_0 through trait_9
                    trait_name = f"trait_{i}"
                    feature_vec.append(org.phenotype.traits.get(trait_name, 0.0))
            else:
                feature_vec.extend([0.0] * 10)
            
            # Fitness
            fitness = getattr(org, 'fitness', 0.0)
            feature_vec.append(fitness)
            
            # Resources (if available)
            resources = getattr(org, 'resources', 0.5)
            feature_vec.append(resources)
            
            # Genotype age
            if hasattr(org, 'genotype'):
                feature_vec.append(getattr(org.genotype, 'age', 0) / 100.0)  # Normalize
            else:
                feature_vec.append(0.0)
            
            # NEW: Language features (if context_memory available)
            if context_memory and hasattr(context_memory, 'node_word_associations'):
                # Convert org_id to int for lookup
                org_id_int = hash(org_id) if isinstance(org_id, str) else org_id
                
                # Vocabulary size (normalized using log scale for 65K vocab)
                vocab_size = len(context_memory.node_word_associations.get(org_id_int, set()))
                feature_vec.append(min(1.0, np.log1p(vocab_size) / np.log1p(65536)))
                
                # Communication activity (normalized, assuming max 50 communications)
                comm_activity = getattr(org, 'communication_count', 0)
                feature_vec.append(min(1.0, comm_activity / 50.0))
                
                # Linguistic connections (normalized, assuming max 10 linguistic edges)
                linguistic_conns = getattr(org, 'linguistic_connection_count', 0)
                feature_vec.append(min(1.0, linguistic_conns / 10.0))
            else:
                # No language data available - use zeros
                feature_vec.extend([0.0] * 3)
            
            # === INTEGRATION: Extended Alliance/Combat/Learning Features ===
            
            # Alliance participation (membership + reputation)
            alliance_participation = 0.0
            if hasattr(org, 'alliance_id') and org.alliance_id is not None:
                alliance_participation = 0.5  # Base for being in alliance
                alliance_participation += getattr(org, 'alliance_reputation', 0.0) * 0.5
            feature_vec.append(min(1.0, alliance_participation))
            
            # Combat performance (battle win ratio)
            combat_performance = 0.5  # Neutral default
            battle_wins = getattr(org, 'battle_wins', 0)
            battle_losses = getattr(org, 'battle_losses', 0)
            total_battles = battle_wins + battle_losses
            if total_battles > 0:
                combat_performance = battle_wins / total_battles
            feature_vec.append(combat_performance)
            
            # Reputation score (social standing in network)
            reputation_score = getattr(org, 'alliance_reputation', 0.5)
            feature_vec.append(min(1.0, max(0.0, reputation_score)))
            
            # Concept maturity (language/learning sophistication)
            concept_maturity = 0.0
            if hasattr(org, 'atomic_language') and org.atomic_language:
                # Use vocabulary diversity as maturity proxy
                vocab = getattr(org.atomic_language, 'vocabulary', set())
                concept_maturity = min(1.0, len(vocab) / 50.0)
            elif hasattr(org, 'token_sequence'):
                # Fallback to token diversity
                unique_tokens = len(set(org.token_sequence))
                concept_maturity = min(1.0, unique_tokens / 30.0)
            feature_vec.append(concept_maturity)
            
            # === CONFEDERATION (Super-Alliance) Features ===
            
            # Confederation membership level (0=none, 0.33=confederation, 0.66=empire, 1.0=hegemony)
            confederation_level = 0.0
            if hasattr(org, 'confederation_tier'):
                confederation_level = getattr(org, 'confederation_tier', 0) / 3.0
            elif hasattr(org, 'alliance_id') and org.alliance_id:
                confederation_level = 0.1  # In alliance but not confederation
            feature_vec.append(min(1.0, confederation_level))
            
            # Super-alliance war participation
            confed_wars = getattr(org, 'confederation_wars_participated', 0)
            feature_vec.append(min(1.0, confed_wars / 5.0))
            
            # Cross-alliance influence (how many alliances organism has connections with)
            cross_alliance_influence = getattr(org, 'cross_alliance_connections', 0)
            feature_vec.append(min(1.0, cross_alliance_influence / 10.0))
            
            # === LANGUAGE-GAME BRIDGE FEATURES ===
            # Integration: Connect vocabulary-game correlations to ML analysis
            # Note: Bridge is at arena-level, not per-organism. We use organism's 
            # arena participation metrics or default to neutral values.
            
            # Try to get bridge metrics from organism (if attached) or from context
            bridge_metrics = None
            if hasattr(org, 'language_game_bridge') and org.language_game_bridge:
                bridge_metrics = org.language_game_bridge.get_correlation_metrics()
            elif context_memory and hasattr(context_memory, 'language_game_metrics'):
                # Use network-level aggregated metrics as fallback
                bridge_metrics = context_memory.language_game_metrics
            
            # Language-game alignment (how well vocabulary correlates with game success)
            lang_game_alignment = 0.5  # Neutral default
            if bridge_metrics:
                alignment = bridge_metrics.get('vocabulary_game_alignment', 0.0)
                lang_game_alignment = (alignment + 1.0) / 2.0  # Normalize -1,1 to 0,1
            feature_vec.append(min(1.0, max(0.0, lang_game_alignment)))
            
            # Language decision influence (how much vocabulary affects actions)
            lang_decision_influence = 0.0
            if bridge_metrics:
                lang_decision_influence = bridge_metrics.get('language_decision_influence', 0.0)
            feature_vec.append(min(1.0, lang_decision_influence))
            
            # Concept diversity in games (vocabulary breadth during gameplay)
            game_concept_diversity = 0.0
            if bridge_metrics:
                game_concept_diversity = bridge_metrics.get('concept_diversity', 0.0)
            feature_vec.append(min(1.0, game_concept_diversity))
            
            features.append(feature_vec)
        
        return np.array(features), organism_ids
    
    def extract_features_parallel(self, organisms: Dict[str, Any],
                                   context_memory: Optional[Any] = None,
                                   config: Dict[str, Any] = None) -> Tuple[np.ndarray, List[str]]:
        """
        Extract features using Ray parallel processing for large populations.
        
        Automatically falls back to sequential extraction if:
        - Ray is not available
        - Population is below parallelization threshold (default 50)
        - Any error occurs during parallel processing
        
        This provides 4-5x speedup for populations > 100 organisms.
        
        Args:
            organisms: Dict of org_id -> organism
            context_memory: Optional context for language features
            config: Optional config dict with Ray settings
            
        Returns:
            Tuple of (feature_array, organism_ids)
        """
        # Check if Ray parallelization is worthwhile
        threshold = 50
        if config:
            ray_config = config.get('ray', {})
            threshold = ray_config.get('parallelization_threshold', 50)
        
        if not RAY_DISTRIBUTED_AVAILABLE or len(organisms) < threshold:
            # Fall back to sequential extraction
            return self.extract_features(organisms, context_memory)
        
        try:
            # Use Ray distributed feature extraction
            ray_manager = get_ray_manager(config)
            
            if not ray_manager.is_initialized():
                return self.extract_features(organisms, context_memory)
            
            organism_ids = list(organisms.keys())
            
            # Prepare organism states for serialization
            org_states = []
            for org_id in organism_ids:
                org = organisms[org_id]
                state = self._extract_organism_state(org, org_id, context_memory)
                org_states.append(state)
            
            # Parallel feature extraction
            from .distributed.ray_tasks import extract_features_batch
            features = extract_features_batch(org_states, context=None, use_ray=True)
            
            logger.debug(f"[ML Utils] Ray extracted features for {len(organism_ids)} organisms")
            return np.array(features), organism_ids
            
        except Exception as e:
            logger.warning(f"[ML Utils] Ray feature extraction failed, falling back: {e}")
            return self.extract_features(organisms, context_memory)
    
    def _extract_organism_state(self, org: Any, org_id: str, 
                                 context_memory: Optional[Any] = None) -> dict:
        """
        Extract serializable state dict from organism for Ray processing.
        
        This converts organism objects to plain dicts that can be sent to Ray workers.
        """
        state = {'id': org_id}
        
        # Basic attributes
        state['fitness'] = getattr(org, 'fitness', 0.5)
        state['resources'] = getattr(org, 'resources', 0.5)
        state['energy'] = getattr(org, 'energy', 0.5)
        
        # Phenotype traits
        if hasattr(org, 'phenotype') and hasattr(org.phenotype, 'traits'):
            state['traits'] = dict(org.phenotype.traits)
        else:
            state['traits'] = {}
        
        # Genotype
        if hasattr(org, 'genotype'):
            state['genotype_age'] = getattr(org.genotype, 'age', 0)
        
        # Alliance/combat features
        state['alliance_id'] = getattr(org, 'alliance_id', None)
        state['alliance_reputation'] = getattr(org, 'alliance_reputation', 0.5)
        state['battle_wins'] = getattr(org, 'battle_wins', 0)
        state['battle_losses'] = getattr(org, 'battle_losses', 0)
        state['confederation_tier'] = getattr(org, 'confederation_tier', 0)
        state['confederation_wars_participated'] = getattr(org, 'confederation_wars_participated', 0)
        state['cross_alliance_connections'] = getattr(org, 'cross_alliance_connections', 0)
        
        # Language features
        if hasattr(org, 'atomic_language') and org.atomic_language:
            vocab = getattr(org.atomic_language, 'vocabulary', set())
            state['vocabulary_size'] = len(vocab)
        elif hasattr(org, 'token_sequence'):
            state['vocabulary_size'] = len(set(org.token_sequence))
        else:
            state['vocabulary_size'] = 0
        
        state['communication_count'] = getattr(org, 'communication_count', 0)
        state['linguistic_connection_count'] = getattr(org, 'linguistic_connection_count', 0)
        
        return state
    
    def fit_predict(self, organisms: Dict[str, Any],
                    context_memory: Optional[Any] = None,
                    config: Dict[str, Any] = None,
                    use_ray: bool = True) -> ClusteringResult:
        """
        Cluster organisms and return results.
        
        Uses Ray parallel feature extraction for large populations (4-5x speedup).
        
        Args:
            organisms: Dict of org_id -> organism
            context_memory: Optional context for language features
            config: Optional config dict with Ray settings
            use_ray: Whether to use Ray parallel processing (default True)
        
        Returns empty result if sklearn not available or insufficient data.
        """
        if not SKLEARN_AVAILABLE:
            return ClusteringResult(
                labels=np.array([]),
                n_clusters=0,
                cluster_sizes={},
                algorithm="unavailable"
            )
        
        if len(organisms) < self.min_cluster_size:
            return ClusteringResult(
                labels=np.zeros(len(organisms), dtype=int),
                n_clusters=1 if organisms else 0,
                cluster_sizes={0: len(organisms)} if organisms else {},
                algorithm="insufficient_data"
            )
        
        # Use Ray parallel extraction for large populations
        if use_ray and RAY_DISTRIBUTED_AVAILABLE:
            features, organism_ids = self.extract_features_parallel(
                organisms, context_memory, config
            )
        else:
            features, organism_ids = self.extract_features(organisms, context_memory=context_memory)
        
        # Standardize features
        features_scaled = self.scaler.fit_transform(features)
        
        # Track which algorithm is actually used (separate from config setting)
        used_algorithm = self.algorithm
        
        # Select and run clustering algorithm
        if self.algorithm == 'hdbscan' and HDBSCAN_AVAILABLE:
            clusterer = HDBSCAN(
                min_cluster_size=self.min_cluster_size,
                min_samples=self.min_samples
            )
            try:
                labels = clusterer.fit_predict(features_scaled)
                centroids = None
                used_algorithm = 'hdbscan'
            except TypeError as e:
                # Common failure mode when `hdbscan` and `scikit-learn` versions are incompatible
                # (e.g., `sklearn.utils.validation.check_array()` signature changed).
                msg = str(e)
                if 'force_all_finite' in msg and 'check_array' in msg:
                    logger.warning(
                        "[PopulationClusterer] HDBSCAN failed due to sklearn incompatibility (%s). "
                        "Falling back to KMeans. Consider pinning scikit-learn to a compatible version.",
                        msg,
                    )
                else:
                    logger.warning(
                        "[PopulationClusterer] HDBSCAN failed (%s). Falling back to KMeans.",
                        msg,
                    )

                n_clusters = min(self.n_clusters, len(organisms))
                clusterer = KMeans(n_clusters=n_clusters, random_state=42, n_init=10)
                labels = clusterer.fit_predict(features_scaled)
                centroids = clusterer.cluster_centers_
                used_algorithm = 'kmeans_hdbscan_fallback'
        elif self.algorithm == 'kmeans':
            n_clusters = min(self.n_clusters, len(organisms))
            clusterer = KMeans(n_clusters=n_clusters, random_state=42, n_init=10)
            labels = clusterer.fit_predict(features_scaled)
            centroids = clusterer.cluster_centers_
            used_algorithm = 'kmeans'
        elif self.algorithm == 'dbscan':
            clusterer = DBSCAN(eps=0.5, min_samples=self.min_samples)
            labels = clusterer.fit_predict(features_scaled)
            centroids = None
            used_algorithm = 'dbscan'
        else:
            # Fallback to KMeans if HDBSCAN not available or unknown algorithm
            n_clusters = min(self.n_clusters, len(organisms))
            clusterer = KMeans(n_clusters=n_clusters, random_state=42, n_init=10)
            labels = clusterer.fit_predict(features_scaled)
            centroids = clusterer.cluster_centers_
            used_algorithm = 'kmeans_fallback'
        
        # Calculate cluster sizes
        unique_labels = set(labels)
        cluster_sizes = {int(label): int(np.sum(labels == label)) for label in unique_labels}
        n_clusters = len([l for l in unique_labels if l >= 0])  # Exclude noise (-1)
        
        self._last_result = ClusteringResult(
            labels=labels,
            n_clusters=n_clusters,
            cluster_sizes=cluster_sizes,
            cluster_centroids=centroids,
            algorithm=used_algorithm  # Report actual algorithm used, not config setting
        )
        
        return self._last_result
    
    @property
    def last_result(self) -> Optional[ClusteringResult]:
        return self._last_result


class AnomalyDetector:
    """
    Detects unusual organisms in the population using Isolation Forest
    or Local Outlier Factor.
    """
    
    def __init__(self, config: Dict[str, Any] = None):
        self.config = config or {}
        self.algorithm = self.config.get('algorithm', 'isolation_forest')
        self.contamination = self.config.get('contamination', 0.1)
        self.n_estimators = self.config.get('n_estimators', 100)
        self.scaler = StandardScaler() if SKLEARN_AVAILABLE else None
        self._last_result: Optional[AnomalyResult] = None
    
    def extract_features(self, organisms: Dict[str, Any],
                        context_memory: Optional[Any] = None) -> Tuple[np.ndarray, List[str]]:
        """Extract feature vectors from organisms (same as clustering, with language features)"""
        organism_ids = list(organisms.keys())
        features = []
        
        for org_id in organism_ids:
            org = organisms[org_id]
            feature_vec = []
            
            # Phenotype traits
            if hasattr(org, 'phenotype') and hasattr(org.phenotype, 'traits'):
                for i in range(10):
                    trait_name = f"trait_{i}"
                    feature_vec.append(org.phenotype.traits.get(trait_name, 0.0))
            else:
                feature_vec.extend([0.0] * 10)
            
            # Fitness
            feature_vec.append(getattr(org, 'fitness', 0.0))
            
            # Resources
            feature_vec.append(getattr(org, 'resources', 0.5))
            
            # Genotype age
            if hasattr(org, 'genotype'):
                feature_vec.append(getattr(org.genotype, 'age', 0) / 100.0)
            else:
                feature_vec.append(0.0)
            
            # Language features (if context_memory available)
            if context_memory and hasattr(context_memory, 'node_word_associations'):
                org_id_int = hash(org_id) if isinstance(org_id, str) else org_id
                vocab_size = len(context_memory.node_word_associations.get(org_id_int, set()))
                feature_vec.append(min(1.0, np.log1p(vocab_size) / np.log1p(65536)))  # Log scale for 65K vocab
                comm_activity = getattr(org, 'communication_count', 0)
                feature_vec.append(min(1.0, comm_activity / 50.0))
                linguistic_conns = getattr(org, 'linguistic_connection_count', 0)
                feature_vec.append(min(1.0, linguistic_conns / 10.0))
            else:
                feature_vec.extend([0.0] * 3)
            
            # === ALLIANCE/CONFEDERATION FEATURES (match PopulationClusterer) ===
            
            # Alliance participation
            alliance_participation = 0.0
            if hasattr(org, 'alliance_id') and org.alliance_id is not None:
                alliance_participation = 0.5
                alliance_participation += getattr(org, 'alliance_reputation', 0.0) * 0.5
            feature_vec.append(min(1.0, alliance_participation))
            
            # Combat performance
            combat_performance = 0.5
            battle_wins = getattr(org, 'battle_wins', 0)
            battle_losses = getattr(org, 'battle_losses', 0)
            total_battles = battle_wins + battle_losses
            if total_battles > 0:
                combat_performance = battle_wins / total_battles
            feature_vec.append(combat_performance)
            
            # Reputation score
            reputation_score = getattr(org, 'alliance_reputation', 0.5)
            feature_vec.append(min(1.0, max(0.0, reputation_score)))
            
            # Concept maturity
            concept_maturity = 0.0
            if hasattr(org, 'atomic_language') and org.atomic_language:
                vocab = getattr(org.atomic_language, 'vocabulary', set())
                concept_maturity = min(1.0, len(vocab) / 50.0)
            elif hasattr(org, 'token_sequence'):
                unique_tokens = len(set(org.token_sequence))
                concept_maturity = min(1.0, unique_tokens / 30.0)
            feature_vec.append(concept_maturity)
            
            # Confederation features
            confederation_level = 0.0
            if hasattr(org, 'confederation_tier'):
                confederation_level = getattr(org, 'confederation_tier', 0) / 3.0
            elif hasattr(org, 'alliance_id') and org.alliance_id:
                confederation_level = 0.1
            feature_vec.append(min(1.0, confederation_level))
            
            confed_wars = getattr(org, 'confederation_wars_participated', 0)
            feature_vec.append(min(1.0, confed_wars / 5.0))
            
            cross_alliance_influence = getattr(org, 'cross_alliance_connections', 0)
            feature_vec.append(min(1.0, cross_alliance_influence / 10.0))
            
            features.append(feature_vec)
        
        return np.array(features), organism_ids
    
    def fit_predict(self, organisms: Dict[str, Any], context_memory: Optional[Any] = None) -> AnomalyResult:
        """
        Detect anomalies in organism population.
        
        Args:
            organisms: Dict mapping organism IDs to Organism objects
            context_memory: Optional ContextMemory instance for language features
        
        Returns:
            AnomalyResult with detected anomalies
        """
        if not SKLEARN_AVAILABLE:
            return AnomalyResult(
                scores=np.array([]),
                labels=np.array([]),
                anomaly_indices=[],
                anomaly_ratio=0.0,
                algorithm="unavailable"
            )
        
        if len(organisms) < 5:  # Need minimum samples for anomaly detection
            return AnomalyResult(
                scores=np.zeros(len(organisms)),
                labels=np.ones(len(organisms), dtype=int),
                anomaly_indices=[],
                anomaly_ratio=0.0,
                algorithm="insufficient_data"
            )
        
        features, organism_ids = self.extract_features(organisms, context_memory=context_memory)
        features_scaled = self.scaler.fit_transform(features)
        
        if self.algorithm == 'isolation_forest':
            detector = IsolationForest(
                contamination=self.contamination,
                n_estimators=self.n_estimators,
                random_state=42
            )
            labels = detector.fit_predict(features_scaled)
            scores = detector.decision_function(features_scaled)
        elif self.algorithm == 'lof':
            detector = LocalOutlierFactor(
                contamination=self.contamination,
                novelty=False
            )
            labels = detector.fit_predict(features_scaled)
            scores = detector.negative_outlier_factor_
        else:
            # Default to Isolation Forest
            detector = IsolationForest(
                contamination=self.contamination,
                n_estimators=self.n_estimators,
                random_state=42
            )
            labels = detector.fit_predict(features_scaled)
            scores = detector.decision_function(features_scaled)
        
        # Find anomaly indices (labels == -1)
        anomaly_indices = [i for i, label in enumerate(labels) if label == -1]
        anomaly_ratio = len(anomaly_indices) / len(organisms) if organisms else 0.0
        
        self._last_result = AnomalyResult(
            scores=scores,
            labels=labels,
            anomaly_indices=anomaly_indices,
            anomaly_ratio=anomaly_ratio,
            algorithm=self.algorithm
        )
        
        return self._last_result
    
    @property
    def last_result(self) -> Optional[AnomalyResult]:
        return self._last_result


class TraitReducer:
    """
    Reduces high-dimensional trait/behavior space to 2D/3D for visualization.
    Uses PCA or t-SNE depending on configuration.
    """
    
    def __init__(self, config: Dict[str, Any] = None):
        self.config = config or {}
        self.algorithm = self.config.get('algorithm', 'pca')
        self.n_components = self.config.get('n_components', 3)
        self.tsne_perplexity = self.config.get('tsne_perplexity', 30)
        self.scaler = StandardScaler() if SKLEARN_AVAILABLE else None
        self._last_result: Optional[ReductionResult] = None
    
    def extract_features(self, organisms: Dict[str, Any],
                        context_memory: Optional[Any] = None) -> Tuple[np.ndarray, List[str]]:
        """Extract feature vectors from organisms (with language features)"""
        organism_ids = list(organisms.keys())
        features = []
        
        for org_id in organism_ids:
            org = organisms[org_id]
            feature_vec = []
            
            # Phenotype traits
            if hasattr(org, 'phenotype') and hasattr(org.phenotype, 'traits'):
                for i in range(10):
                    trait_name = f"trait_{i}"
                    feature_vec.append(org.phenotype.traits.get(trait_name, 0.0))
            else:
                feature_vec.extend([0.0] * 10)
            
            # Fitness
            feature_vec.append(getattr(org, 'fitness', 0.0))
            
            # Resources
            feature_vec.append(getattr(org, 'resources', 0.5))
            
            # Genotype age
            if hasattr(org, 'genotype'):
                feature_vec.append(getattr(org.genotype, 'age', 0) / 100.0)
            else:
                feature_vec.append(0.0)
            
            # NEW: Language features (if context_memory available)
            if context_memory and hasattr(context_memory, 'node_word_associations'):
                org_id_int = hash(org_id) if isinstance(org_id, str) else org_id
                vocab_size = len(context_memory.node_word_associations.get(org_id_int, set()))
                feature_vec.append(min(1.0, np.log1p(vocab_size) / np.log1p(65536)))  # Log scale for 65K vocab
                comm_activity = getattr(org, 'communication_count', 0)
                feature_vec.append(min(1.0, comm_activity / 50.0))
                linguistic_conns = getattr(org, 'linguistic_connection_count', 0)
                feature_vec.append(min(1.0, linguistic_conns / 10.0))
            else:
                feature_vec.extend([0.0] * 3)
            
            features.append(feature_vec)
        
        return np.array(features), organism_ids
    
    def fit_transform(self, organisms: Dict[str, Any], context_memory: Optional[Any] = None) -> ReductionResult:
        """
        Reduce dimensionality of organism features.
        
        Args:
            organisms: Dict mapping organism IDs to Organism objects
            context_memory: Optional ContextMemory instance for language features
        
        Returns:
            ReductionResult with reduced coordinates
        """
        if not SKLEARN_AVAILABLE:
            return ReductionResult(
                coordinates=np.array([]),
                n_components=0,
                algorithm="unavailable"
            )
        
        if len(organisms) < 3:
            return ReductionResult(
                coordinates=np.zeros((len(organisms), self.n_components)),
                n_components=self.n_components,
                algorithm="insufficient_data"
            )
        
        features, organism_ids = self.extract_features(organisms, context_memory=context_memory)
        features_scaled = self.scaler.fit_transform(features)
        
        # Limit n_components to number of features/samples
        n_components = min(self.n_components, features.shape[1], len(organisms))
        
        if self.algorithm == 'pca':
            reducer = PCA(n_components=n_components)
            coordinates = reducer.fit_transform(features_scaled)
            explained_variance = reducer.explained_variance_ratio_.tolist()
        elif self.algorithm == 'tsne':
            # t-SNE requires perplexity < n_samples
            perplexity = min(self.tsne_perplexity, len(organisms) - 1, 30)
            reducer = TSNE(
                n_components=min(n_components, 3),  # t-SNE max 3 components
                perplexity=max(5, perplexity),
                random_state=42
            )
            coordinates = reducer.fit_transform(features_scaled)
            explained_variance = None  # t-SNE doesn't have explained variance
        else:
            # Default to PCA
            reducer = PCA(n_components=n_components)
            coordinates = reducer.fit_transform(features_scaled)
            explained_variance = reducer.explained_variance_ratio_.tolist()
        
        self._last_result = ReductionResult(
            coordinates=coordinates,
            n_components=n_components,
            explained_variance=explained_variance,
            algorithm=self.algorithm
        )
        
        return self._last_result
    
    @property
    def last_result(self) -> Optional[ReductionResult]:
        return self._last_result


class MLAnalyzer:
    """
    Unified ML analysis interface for the Butterfly System.
    
    Coordinates clustering, anomaly detection, and dimensionality reduction,
    respecting configuration toggles.
    
    Includes ConceptTracker for semantic naming of stable behavioral clusters (Quick Win #2).
    """
    
    def __init__(self, config: Dict[str, Any] = None):
        """
        Initialize ML analyzer with configuration.
        
        Config structure expected:
        {
            "enabled": true/false,
            "clustering": {"enabled": true, "algorithm": "hdbscan", ...},
            "anomaly_detection": {"enabled": true, "algorithm": "isolation_forest", ...},
            "dimensionality_reduction": {"enabled": true, "algorithm": "pca", ...},
            "concept_tracking": {"enabled": true, "persistence_threshold": 3, ...}
        }
        """
        self.config = config or {}
        self.enabled = self.config.get('enabled', False)
        
        # Initialize subsystems
        clustering_config = self.config.get('clustering', {})
        anomaly_config = self.config.get('anomaly_detection', {})
        reduction_config = self.config.get('dimensionality_reduction', {})
        concept_config = self.config.get('concept_tracking', {})
        
        self.clusterer = PopulationClusterer(clustering_config)
        self.anomaly_detector = AnomalyDetector(anomaly_config)
        self.reducer = TraitReducer(reduction_config)
        
        # NEW: Concept tracking for semantic naming (Quick Win #2)
        self.concept_tracker = ConceptTracker(
            persistence_threshold=concept_config.get('persistence_threshold', 3),
            stale_threshold=concept_config.get('stale_threshold', 10.0),
            enabled=concept_config.get('enabled', True)
        )
        
        # Feature toggles
        self.clustering_enabled = clustering_config.get('enabled', True)
        self.anomaly_enabled = anomaly_config.get('enabled', True)
        self.reduction_enabled = reduction_config.get('enabled', True)
        self.concept_tracking_enabled = concept_config.get('enabled', True)
        
        # NEW: Language analysis configuration
        language_config = self.config.get('language_analysis', {})
        self.language_analysis_enabled = language_config.get('enabled', True)
        self.tfidf_enabled = language_config.get('tfidf', {}).get('enabled', True)
        self.nearest_neighbors_enabled = language_config.get('nearest_neighbors', {}).get('enabled', True)
        self.feature_selection_enabled = language_config.get('feature_selection', {}).get('enabled', False)
        self.metrics_enabled = language_config.get('metrics', {}).get('enabled', True)
        
        # Initialize language analysis components
        if SKLEARN_AVAILABLE and self.language_analysis_enabled:
            # TF-IDF vectorizer for vocabulary analysis
            if self.tfidf_enabled:
                self.tfidf_vectorizer = TfidfVectorizer(
                    max_features=language_config.get('tfidf', {}).get('max_features', 1000),
                    ngram_range=tuple(language_config.get('tfidf', {}).get('ngram_range', [1, 2])),
                    min_df=language_config.get('tfidf', {}).get('min_df', 1),
                    max_df=language_config.get('tfidf', {}).get('max_df', 0.95)
                )
                self.count_vectorizer = CountVectorizer(
                    max_features=language_config.get('tfidf', {}).get('max_features', 1000),
                    ngram_range=tuple(language_config.get('tfidf', {}).get('ngram_range', [1, 2]))
                )
            else:
                self.tfidf_vectorizer = None
                self.count_vectorizer = None
            
            # Nearest Neighbors for semantic similarity
            if self.nearest_neighbors_enabled:
                nn_config = language_config.get('nearest_neighbors', {})
                self.nearest_neighbors = NearestNeighbors(
                    n_neighbors=nn_config.get('n_neighbors', 5),
                    metric=nn_config.get('metric', 'cosine'),
                    algorithm=nn_config.get('algorithm', 'auto')
                )
            else:
                self.nearest_neighbors = None
            
            # Feature selection
            if self.feature_selection_enabled:
                fs_config = language_config.get('feature_selection', {})
                self.feature_selector = SelectKBest(
                    score_func=mutual_info_classif if fs_config.get('method', 'mutual_info') == 'mutual_info' else f_classif,
                    k=fs_config.get('k', 10)
                )
            else:
                self.feature_selector = None
        else:
            self.tfidf_vectorizer = None
            self.count_vectorizer = None
            self.nearest_neighbors = None
            self.feature_selector = None
        
        # Last analysis results
        self._last_analysis: Dict[str, Any] = {}
        self._last_analysis_time: float = 0
        self._analysis_interval: float = 5.0  # Minimum seconds between analyses

        # Optional event emitter for causation graph visualization
        self.event_emitter = None  # Set by main.py or unified_entry.py
        
        # ═══════════════════════════════════════════════════════════════════════════
        # MISSION 3: AutoTune Integration Buffer
        # Track ML analysis metrics for AtomicConfigSystem feedback loop
        # ═══════════════════════════════════════════════════════════════════════════
        self.autotune_metrics_buffer = {
            'cluster_count_history': [],
            'anomaly_ratio_history': [],
            'silhouette_score_history': [],
            'avg_cluster_count': 0.0,
            'avg_anomaly_ratio': 0.0,
            'avg_silhouette_score': 0.0,
            'cluster_stability': 0.0,  # How often cluster count changes
            'analysis_count': 0
        }
        self.autotune_window = 20  # Window size for moving averages
        self.atomic_config_system = None  # Set by main.py when available
    
    def update_config(self, config: Dict[str, Any]):
        """Update configuration dynamically (for hot reload)"""
        self.config = config
        self.enabled = config.get('enabled', False)
        
        # Update subsystem configs
        clustering_config = config.get('clustering', {})
        anomaly_config = config.get('anomaly_detection', {})
        reduction_config = config.get('dimensionality_reduction', {})
        concept_config = config.get('concept_tracking', {})
        
        self.clusterer = PopulationClusterer(clustering_config)
        self.anomaly_detector = AnomalyDetector(anomaly_config)
        self.reducer = TraitReducer(reduction_config)
        
        # Update concept tracker settings (preserve history, just update config)
        self.concept_tracker.enabled = concept_config.get('enabled', True)
        self.concept_tracker.persistence_threshold = concept_config.get('persistence_threshold', 3)
        self.concept_tracker.stale_threshold = concept_config.get('stale_threshold', 10.0)
        
        self.clustering_enabled = clustering_config.get('enabled', True)
        self.anomaly_enabled = anomaly_config.get('enabled', True)
        self.reduction_enabled = reduction_config.get('enabled', True)
        self.concept_tracking_enabled = concept_config.get('enabled', True)
    
    def analyze(self, organisms: Dict[str, Any], force: bool = False, context_memory: Optional[Any] = None) -> Dict[str, Any]:
        """
        Run all enabled ML analyses on organism population.
        
        Args:
            organisms: Dict mapping organism IDs to Organism objects
            force: If True, run analysis regardless of interval
            context_memory: Optional ContextMemory instance for language features
            
        Returns:
            Dict with clustering, anomaly, and reduction results
        """
        current_time = time.time()
        
        # Rate limiting (unless forced)
        if not force and (current_time - self._last_analysis_time) < self._analysis_interval:
            return self._last_analysis
        
        if not self.enabled or not SKLEARN_AVAILABLE:
            return {
                'enabled': False,
                'sklearn_available': SKLEARN_AVAILABLE,
                'clustering': None,
                'anomalies': None,
                'reduction': None,
                'concept_tracking': None
            }
        
        results = {
            'enabled': True,
            'sklearn_available': True,
            'timestamp': current_time,
            'organism_count': len(organisms),
            'organism_ids': list(organisms.keys())  # CRITICAL: Map cluster indices to organism IDs
        }
        
        # Clustering
        cluster_result = None
        if self.clustering_enabled:
            cluster_result = self.clusterer.fit_predict(organisms, context_memory=context_memory)
            
            # NEW: Concept tracking - semantic naming of stable clusters (Quick Win #2)
            concept_tags = {}
            if self.concept_tracking_enabled and cluster_result.labels.size > 0:
                # Wire event emitter to concept tracker for causation graph events
                if self.event_emitter:
                    self.concept_tracker.event_emitter = self.event_emitter
                
                # SEMANTIC CONVERGENCE: Wire context_memory for phenotype→vocabulary flow
                if context_memory:
                    self.concept_tracker.context_memory = context_memory
                
                # Update concept tracking with clustering results
                concept_tags = self.concept_tracker.update(
                    cluster_labels=cluster_result.labels,
                    cluster_sizes=cluster_result.cluster_sizes,
                    organisms=organisms,
                    timestamp=current_time
                )
                
                # Attach concept tags to cluster result
                cluster_result.concept_tags = concept_tags
            
            results['clustering'] = cluster_result.to_dict()
            results['cluster_labels'] = cluster_result.labels.tolist() if cluster_result.labels.size > 0 else []
            results['concept_tags'] = concept_tags
            results['concept_summary'] = self.concept_tracker.get_concept_summary() if self.concept_tracking_enabled else None
        else:
            results['clustering'] = None
            results['concept_tags'] = {}
            results['concept_summary'] = None
        
        # Anomaly Detection
        if self.anomaly_enabled:
            anomaly_result = self.anomaly_detector.fit_predict(organisms, context_memory=context_memory)
            results['anomalies'] = anomaly_result.to_dict()
            results['anomaly_organisms'] = anomaly_result.get_anomaly_organisms(list(organisms.keys()))
        else:
            results['anomalies'] = None
        
        # Dimensionality Reduction
        if self.reduction_enabled:
            reduction_result = self.reducer.fit_transform(organisms, context_memory=context_memory)
            results['reduction'] = reduction_result.to_dict()
            # Include coordinates for visualization
            if reduction_result.coordinates.size > 0:
                results['coordinates'] = reduction_result.get_organism_coordinates(list(organisms.keys()))
        else:
            results['reduction'] = None
        
        # NEW: Semantic Analysis - Analyze language patterns, word co-occurrence, semantic clusters
        if context_memory and hasattr(context_memory, 'node_word_associations'):
            semantic_analysis = self._analyze_semantic_patterns(organisms, context_memory)
            results['semantic_analysis'] = semantic_analysis
        else:
            results['semantic_analysis'] = None
        
        self._last_analysis = results
        self._last_analysis_time = current_time
        
        # ═══════════════════════════════════════════════════════════════════════════
        # MISSION 3: Update AutoTune metrics and emit to AtomicConfigSystem
        # ═══════════════════════════════════════════════════════════════════════════
        self._update_autotune_metrics(results)
        
        return results
    
    # ═══════════════════════════════════════════════════════════════════════════
    # MISSION 3: AutoTune Integration Methods
    # ═══════════════════════════════════════════════════════════════════════════
    
    def _update_autotune_metrics(self, results: Dict[str, Any]):
        """
        Update AutoTune metrics buffer and emit to AtomicConfigSystem.
        
        This enables closed-loop optimization where scikit-learn analysis outcomes
        inform config parameter adjustments.
        
        Args:
            results: Analysis results from analyze() method
        """
        # Extract metrics from results
        clustering = results.get('clustering', {})
        anomalies = results.get('anomalies', {})
        
        # Get cluster count
        cluster_count = 0
        if clustering:
            cluster_count = clustering.get('n_clusters', 0)
            self.autotune_metrics_buffer['cluster_count_history'].append(cluster_count)
            if len(self.autotune_metrics_buffer['cluster_count_history']) > self.autotune_window:
                self.autotune_metrics_buffer['cluster_count_history'] = \
                    self.autotune_metrics_buffer['cluster_count_history'][-self.autotune_window:]
        
        # Get anomaly ratio
        anomaly_ratio = 0.0
        if anomalies:
            n_anomalies = anomalies.get('n_anomalies', 0)
            n_samples = anomalies.get('n_samples', 1)
            anomaly_ratio = n_anomalies / max(1, n_samples)
            self.autotune_metrics_buffer['anomaly_ratio_history'].append(anomaly_ratio)
            if len(self.autotune_metrics_buffer['anomaly_ratio_history']) > self.autotune_window:
                self.autotune_metrics_buffer['anomaly_ratio_history'] = \
                    self.autotune_metrics_buffer['anomaly_ratio_history'][-self.autotune_window:]
        
        # Get silhouette score if available
        silhouette_score = 0.0
        if clustering and 'silhouette_score' in clustering:
            silhouette_score = clustering.get('silhouette_score', 0.0)
            self.autotune_metrics_buffer['silhouette_score_history'].append(silhouette_score)
            if len(self.autotune_metrics_buffer['silhouette_score_history']) > self.autotune_window:
                self.autotune_metrics_buffer['silhouette_score_history'] = \
                    self.autotune_metrics_buffer['silhouette_score_history'][-self.autotune_window:]
        
        # Calculate averages
        if self.autotune_metrics_buffer['cluster_count_history']:
            self.autotune_metrics_buffer['avg_cluster_count'] = np.mean(self.autotune_metrics_buffer['cluster_count_history'])
        if self.autotune_metrics_buffer['anomaly_ratio_history']:
            self.autotune_metrics_buffer['avg_anomaly_ratio'] = np.mean(self.autotune_metrics_buffer['anomaly_ratio_history'])
        if self.autotune_metrics_buffer['silhouette_score_history']:
            self.autotune_metrics_buffer['avg_silhouette_score'] = np.mean(self.autotune_metrics_buffer['silhouette_score_history'])
        
        # Calculate cluster stability (how often count changes)
        cluster_history = self.autotune_metrics_buffer['cluster_count_history']
        if len(cluster_history) >= 2:
            changes = sum(1 for i in range(1, len(cluster_history)) if cluster_history[i] != cluster_history[i-1])
            self.autotune_metrics_buffer['cluster_stability'] = 1.0 - (changes / (len(cluster_history) - 1))
        
        self.autotune_metrics_buffer['analysis_count'] += 1
        
        # Emit to AtomicConfigSystem if available
        if self.atomic_config_system is not None:
            try:
                ml_metrics = {
                    'cluster_count': cluster_count,
                    'anomaly_ratio': anomaly_ratio,
                    'silhouette_score': silhouette_score,
                    'cluster_stability': self.autotune_metrics_buffer['cluster_stability'],
                    'avg_cluster_count': self.autotune_metrics_buffer['avg_cluster_count']
                }
                # Call tune() method to inform atomic configs
                actions = self.atomic_config_system.tune(ml_metrics, self.autotune_metrics_buffer['analysis_count'])
                if actions:
                    logger.debug(f"[ML→AUTOTUNE] Applied {len(actions)} config adjustments based on ML analysis")
            except Exception as e:
                logger.debug(f"ML AutoTune integration error: {e}")
        
        # Emit event for CRA visualization
        if self.event_emitter:
            try:
                from causation_explorer import Event
                event = Event(
                    timestamp=time.time(),
                    component='ml_analysis',
                    event_type='ml_autotune_metrics',
                    data={
                        'cluster_count': cluster_count,
                        'anomaly_ratio': anomaly_ratio,
                        'silhouette_score': silhouette_score,
                        'cluster_stability': self.autotune_metrics_buffer['cluster_stability'],
                        'analysis_count': self.autotune_metrics_buffer['analysis_count']
                    }
                )
                self.event_emitter(event)
            except Exception as e:
                logger.debug(f"ML AutoTune event emission failed: {e}")
    
    def get_autotune_metrics(self) -> Dict[str, Any]:
        """
        Get current AutoTune metrics buffer for CRA diagnostics.
        
        Returns:
            Dictionary of ML analysis metrics for AutoTune integration
        """
        return {
            **self.autotune_metrics_buffer,
            'buffer_size': len(self.autotune_metrics_buffer['cluster_count_history']),
            'window_size': self.autotune_window,
            'atomic_config_connected': self.atomic_config_system is not None
        }
    
    def _analyze_semantic_patterns(self, organisms: Dict[str, Any], context_memory: Any) -> Dict[str, Any]:
        """
        Analyze semantic patterns: word co-occurrence, semantic clusters, concept formation.
        
        Enhanced with TF-IDF, Nearest Neighbors, and quality metrics.
        """
        if not context_memory or not hasattr(context_memory, 'node_word_associations'):
            return None
        
        # Get Linguistic Knowledge Web if available
        knowledge_web = None
        if hasattr(context_memory, 'knowledge_web'):
            knowledge_web = context_memory.knowledge_web
        elif hasattr(context_memory, 'language_teacher') and hasattr(context_memory.language_teacher, 'knowledge_web'):
            knowledge_web = context_memory.language_teacher.knowledge_web
        
        if not knowledge_web:
            return None
        
        # Build word-organism matrix and vocabulary strings for TF-IDF
        organism_ids = list(organisms.keys())
        word_organism_matrix = {}  # word -> set of organism_ids
        organism_vocabularies = []  # List of space-separated word strings for each organism
        
        for org_id in organism_ids:
            org_id_int = hash(org_id) if isinstance(org_id, str) else org_id
            words = list(context_memory.node_word_associations.get(org_id_int, set()))
            organism_vocabularies.append(' '.join(words))
            
            for word in words:
                if word not in word_organism_matrix:
                    word_organism_matrix[word] = set()
                word_organism_matrix[word].add(org_id)
        
        # NEW: TF-IDF Analysis (if enabled and sklearn available)
        tfidf_results = None
        important_words = []
        if SKLEARN_AVAILABLE and self.tfidf_enabled and self.tfidf_vectorizer and len(organism_vocabularies) > 0:
            try:
                # Fit TF-IDF on organism vocabularies
                tfidf_matrix = self.tfidf_vectorizer.fit_transform(organism_vocabularies)
                feature_names = self.tfidf_vectorizer.get_feature_names_out()
                
                # Get mean TF-IDF scores across all organisms (word importance)
                mean_tfidf = np.array(tfidf_matrix.mean(axis=0)).flatten()
                word_importance = list(zip(feature_names, mean_tfidf))
                word_importance.sort(key=lambda x: x[1], reverse=True)
                important_words = [{'word': word, 'tfidf_score': float(score)} for word, score in word_importance[:20]]
                
                # Count vectorizer for raw frequencies
                count_matrix = self.count_vectorizer.fit_transform(organism_vocabularies)
                total_counts = np.array(count_matrix.sum(axis=0)).flatten()
                count_feature_names = self.count_vectorizer.get_feature_names_out()
                word_counts = dict(zip(count_feature_names, total_counts))
                
                tfidf_results = {
                    'vocabulary_size': len(feature_names),
                    'top_important_words': important_words[:10],
                    'word_frequencies': {k: int(v) for k, v in list(word_counts.items())[:20]},
                    'ngram_range': self.tfidf_vectorizer.ngram_range
                }
            except Exception as e:
                # Expected in grounded language mode with limited vocab
                logger.debug(f"[ML] TF-IDF analysis skipped: {e}")
                tfidf_results = None
        
        # Analyze word co-occurrence (words that appear together) - keep basic version
        word_cooccurrence = {}
        for org_id in organism_ids:
            org_id_int = hash(org_id) if isinstance(org_id, str) else org_id
            words = list(context_memory.node_word_associations.get(org_id_int, set()))
            # Count co-occurrences
            for i, word1 in enumerate(words):
                for word2 in words[i+1:]:
                    pair = tuple(sorted([word1, word2]))
                    word_cooccurrence[pair] = word_cooccurrence.get(pair, 0) + 1
        
        # NEW: Nearest Neighbors for semantic similarity (if enabled)
        similarity_results = None
        if SKLEARN_AVAILABLE and self.nearest_neighbors_enabled and self.nearest_neighbors and tfidf_results:
            try:
                # Use TF-IDF vectors for similarity search
                if 'tfidf_matrix' in locals() and tfidf_matrix.shape[0] > 1:
                    self.nearest_neighbors.fit(tfidf_matrix)
                    
                    # Find similar organisms for each organism
                    similar_organisms = {}
                    for i, org_id in enumerate(organism_ids[:min(10, len(organism_ids))]):  # Limit to first 10 for performance
                        distances, indices = self.nearest_neighbors.kneighbors(tfidf_matrix[i:i+1], return_distance=True)
                        similar_orgs = [
                            {
                                'organism_id': organism_ids[idx],
                                'similarity': float(1.0 - dist) if self.nearest_neighbors.metric == 'cosine' else float(1.0 / (1.0 + dist))
                            }
                            for dist, idx in zip(distances[0][1:], indices[0][1:])  # Skip self (first result)
                        ]
                        similar_organisms[org_id] = similar_orgs[:5]  # Top 5 similar
                    
                    similarity_results = {
                        'similarity_metric': self.nearest_neighbors.metric,
                        'n_neighbors': self.nearest_neighbors.n_neighbors,
                        'similar_organisms': similar_organisms
                    }
            except Exception as e:
                logger.warning(f"[ML] Nearest Neighbors analysis failed: {e}")
                similarity_results = None
        
        # Find semantic clusters (words with strong semantic relationships)
        semantic_clusters = []
        if knowledge_web:
            # Group words by semantic similarity
            processed_words = set()
            for word in word_organism_matrix.keys():
                if word in processed_words:
                    continue
                # Find semantically related words
                similar_words = knowledge_web.get_similar_words(word, min_strength=0.6)
                cluster_words = [w for w in similar_words if w in word_organism_matrix]
                if len(cluster_words) > 1:
                    semantic_clusters.append({
                        'words': cluster_words,
                        'size': len(cluster_words),
                        'organism_count': len(set.union(*[word_organism_matrix[w] for w in cluster_words]))
                    })
                    processed_words.update(cluster_words)
        
        # Analyze concept formation (words that form meaningful concepts)
        # ML system teaches and strengthens word formations through pattern recognition
        concept_formation = []
        top_cooccurrences = sorted(word_cooccurrence.items(), key=lambda x: x[1], reverse=True)[:20]
        for (word1, word2), count in top_cooccurrences:
            # Check if words have semantic relationship
            if knowledge_web:
                relations = knowledge_web.get_relations(word1)
                has_semantic_link = any(r.target == word2 or r.source == word2 for r in relations)
                if has_semantic_link:
                    # Get relationship strength to assess formation quality
                    relation_strength = 0.0
                    for r in relations:
                        if (r.target == word2 or r.source == word2) and hasattr(r, 'strength'):
                            relation_strength = max(relation_strength, r.strength)
                    
                    concept_formation.append({
                        'word1': word1,
                        'word2': word2,
                        'cooccurrence_count': count,
                        'semantic_relationship': True,
                        'relationship_strength': relation_strength,
                        'formation_quality': 'strong' if relation_strength >= 0.7 and count >= 3 else 'moderate'
                    })
                    
                    # STRENGTHEN FORMATIONS: If ML detects strong co-occurrence with semantic link,
                    # strengthen the relationship in knowledge web (ML teaching the system)
                    if count >= 5 and relation_strength >= 0.6:
                        # Find and strengthen the relationship
                        for r in relations:
                            if (r.target == word2 or r.source == word2) and hasattr(r, 'record_relationship_success'):
                                # ML detected pattern - strengthen it
                                knowledge_web.record_relationship_success(word1, word2, r.relation_type)
        
        # NEW: Feature Selection (if enabled)
        feature_importance = None
        if SKLEARN_AVAILABLE and self.feature_selection_enabled and self.feature_selector and tfidf_results:
            try:
                # Get organism fitness values for feature selection
                organism_fitnesses = []
                for org_id in organism_ids:
                    org = organisms.get(org_id)
                    fitness = getattr(org, 'fitness', 0.5) if org else 0.5
                    organism_fitnesses.append(fitness)
                
                if len(organism_fitnesses) > 5 and 'tfidf_matrix' in locals():
                    # Use TF-IDF features to predict fitness
                    # Convert fitness to binary classes (high/low) for classification
                    fitness_median = np.median(organism_fitnesses)
                    fitness_classes = np.array([1 if f > fitness_median else 0 for f in organism_fitnesses])
                    
                    # Fit feature selector
                    selected_features = self.feature_selector.fit_transform(tfidf_matrix, fitness_classes)
                    feature_scores = self.feature_selector.scores_
                    feature_names = self.tfidf_vectorizer.get_feature_names_out()
                    
                    # Get top important words (words that predict fitness)
                    word_importance = list(zip(feature_names, feature_scores))
                    word_importance.sort(key=lambda x: x[1], reverse=True)
                    important_words = [{'word': word, 'importance_score': float(score)} for word, score in word_importance[:20]]
                    
                    feature_importance = {
                        'top_predictive_words': important_words[:10],
                        'n_features_selected': int(selected_features.shape[1]),
                        'n_features_total': int(tfidf_matrix.shape[1])
                    }
            except Exception as e:
                logger.warning(f"[ML] Feature selection failed: {e}")
                feature_importance = None
        
        # NEW: Quality Metrics (if enabled and clustering results available)
        quality_metrics = None
        if SKLEARN_AVAILABLE and self.metrics_enabled and self._last_analysis:
            try:
                cluster_result = self._last_analysis.get('clustering')
                if cluster_result and 'labels' in cluster_result and cluster_result['labels']:
                    labels = np.array(cluster_result['labels'])
                    if len(labels) > 0 and len(set(labels)) > 1:
                        # Get features for silhouette score
                        features, _ = self.clusterer.extract_features(organisms, context_memory=context_memory)
                        if features.shape[0] == len(labels) and features.shape[0] > 1:
                            features_scaled = self.clusterer.scaler.fit_transform(features)
                            silhouette = silhouette_score(features_scaled, labels)
                            
                            quality_metrics = {
                                'silhouette_score': float(silhouette),
                                'n_clusters': int(len(set(labels))),
                                'n_samples': int(len(labels))
                            }
            except Exception as e:
                logger.warning(f"[ML] Quality metrics calculation failed: {e}")
                quality_metrics = None
        
        return {
            'word_organism_matrix_size': len(word_organism_matrix),
            'total_word_organism_links': sum(len(orgs) for orgs in word_organism_matrix.values()),
            'word_cooccurrence_pairs': len(word_cooccurrence),
            'top_cooccurrences': dict(top_cooccurrences[:10]),
            'semantic_clusters': semantic_clusters[:10],  # Top 10 clusters
            'concept_formation': concept_formation[:10],  # Top 10 concepts
            'tfidf_analysis': tfidf_results,  # NEW: TF-IDF results
            'similarity_analysis': similarity_results,  # NEW: Nearest Neighbors results
            'feature_importance': feature_importance,  # NEW: Feature selection results
            'quality_metrics': quality_metrics,  # NEW: Quality metrics
            'semantic_analysis_enabled': True,
            'ml_teaching': True  # ML is teaching/strengthening formations
        }
    
    def get_status(self) -> Dict[str, Any]:
        """Get current ML analyzer status"""
        return {
            'enabled': self.enabled,
            'sklearn_available': SKLEARN_AVAILABLE,
            'hdbscan_available': HDBSCAN_AVAILABLE,
            'clustering_enabled': self.clustering_enabled,
            'anomaly_enabled': self.anomaly_enabled,
            'reduction_enabled': self.reduction_enabled,
            'concept_tracking_enabled': self.concept_tracking_enabled,
            'last_analysis_time': self._last_analysis_time,
            'clusterer_algorithm': self.clusterer.algorithm,
            'anomaly_algorithm': self.anomaly_detector.algorithm,
            'reducer_algorithm': self.reducer.algorithm,
            'active_concepts': len(self.concept_tracker.get_active_concepts()) if self.concept_tracking_enabled else 0
        }


# Singleton instance for global access
_ml_analyzer: Optional[MLAnalyzer] = None


def get_ml_analyzer(config: Dict[str, Any] = None) -> MLAnalyzer:
    """Get or create the global ML analyzer instance"""
    global _ml_analyzer
    if _ml_analyzer is None or config is not None:
        _ml_analyzer = MLAnalyzer(config)
    return _ml_analyzer


def is_sklearn_available() -> bool:
    """Check if scikit-learn is available"""
    return SKLEARN_AVAILABLE