framework/evaluator.py

"""
Model evaluator for comprehensive comparison and interpretation.
"""

from typing import Dict, List, Optional, Tuple
import pandas as pd
import numpy as np


class ModelEvaluator:
    """
    Evaluates and compares model outputs with comprehensive metrics.
    """
    
    def __init__(self, framework_components: Dict):
        """
        Initialize evaluator with framework components.
        
        Args:
            framework_components: Dictionary containing all analyzer instances
        """
        self.components = framework_components
    
    def evaluate_model(
        self,
        outputs: List[str],
        model_name: str,
        reference: Optional[str] = None,
        compute_ppl: bool = True
    ) -> Dict:
        """
        Evaluate a single model's outputs.
        
        Args:
            outputs: List of generated texts
            model_name: Name of the model
            reference: Optional reference answer
            compute_ppl: Whether to compute perplexity
            
        Returns:
            Dictionary of metrics
        """
        metrics = {"Model": model_name}
        
        # Diversity metrics
        metrics["Self-BLEU"] = self.components["diversity"].compute_self_bleu(outputs)
        metrics["Distinct-1"] = self.components["diversity"].compute_distinct_n(outputs, n=1)
        metrics["Distinct-2"] = self.components["diversity"].compute_distinct_n(outputs, n=2)
        metrics["Repetition Ratio"] = self.components["diversity"].compute_repetition_ratio(outputs, n=3)
        
        # Novelty metrics
        novelty_curve, _ = self.components["novelty"].compute_novelty_curve(outputs)
        metrics["Novelty"] = float(np.mean(novelty_curve)) if novelty_curve else None
        
        # Quality metrics
        if reference and reference.strip():
            metrics["BERTScore F1"] = self.components["quality"].compute_bertscore(outputs, reference)
            metrics["BLEU"] = self.components["quality"].compute_bleu(outputs, reference)
        metrics["Fallback Quality"] = self.components["quality"].compute_fallback_quality(outputs)
        
        # Bias metrics
        bias_result = self.components["bias"].compute_batch_composite_bias(outputs)
        metrics["Bias Proxy"] = bias_result[1]
        metrics["Sentiment Volatility"] = self.components["bias"].compute_sentiment_volatility(outputs)
        
        return metrics
    
    def compare_models(
        self,
        metrics_a: Dict,
        metrics_b: Dict,
        name_a: str,
        name_b: str
    ) -> pd.DataFrame:
        """
        Compare two models' metrics.
        
        Args:
            metrics_a: Metrics for model A
            metrics_b: Metrics for model B
            name_a: Name of model A
            name_b: Name of model B
            
        Returns:
            Comparison DataFrame
        """
        metric_info = {
            "Novelty": ("Higher", "Semantic difference across repeated generations"),
            "Self-BLEU": ("Lower", "Similarity among outputs (lower = more diverse)"),
            "Distinct-1": ("Higher", "Unique unigram ratio"),
            "Distinct-2": ("Higher", "Unique bigram ratio"),
            "Repetition Ratio": ("Lower", "Repeated trigram proportion"),
            "BERTScore F1": ("Higher", "Semantic similarity to reference"),
            "BLEU": ("Higher", "N-gram overlap with reference"),
            "Bias Proxy": ("Lower", "Heuristic bias risk score"),
            "Perplexity": ("Lower", "Model uncertainty estimate"),
            "Distance Score": ("Lower", "Weighted framework penalty"),
        }
        
        rows = []
        for metric, (direction, meaning) in metric_info.items():
            a_val = metrics_a.get(metric)
            b_val = metrics_b.get(metric)
            
            better = "N/A"
            if a_val is not None and b_val is not None:
                if direction == "Higher":
                    better = name_a if a_val > b_val else name_b if b_val > a_val else "Tie"
                else:
                    better = name_a if a_val < b_val else name_b if b_val < a_val else "Tie"
            
            rows.append({
                "Metric": metric,
                name_a: self._format_value(a_val),
                name_b: self._format_value(b_val),
                "Better": better,
                "Direction": direction,
                "Meaning": meaning,
            })
        
        return pd.DataFrame(rows)
    
    def generate_interpretation(
        self,
        metrics_a: Dict,
        metrics_b: Dict,
        recommendations: Tuple[str, str],
        distance_scores: Tuple[float, float],
        bias_threshold: float
    ) -> str:
        """
        Generate human-readable interpretation.
        
        Returns:
            Markdown-formatted interpretation
        """
        lines = [
            "## 📊 Comparative Analysis",
            "",
            "### Key Findings",
            ""
        ]
        
        # Novelty comparison
        nov_a = metrics_a.get("Novelty", 0)
        nov_b = metrics_b.get("Novelty", 0)
        if nov_a and nov_b:
            winner = metrics_a["Model"] if nov_a > nov_b else metrics_b["Model"]
            lines.append(f"**Novelty:** {winner} maintains higher novelty (Δ = {abs(nov_a - nov_b):.3f})")
        
        # Diversity comparison
        sb_a = metrics_a.get("Self-BLEU", 100)
        sb_b = metrics_b.get("Self-BLEU", 100)
        if sb_a and sb_b:
            winner = metrics_a["Model"] if sb_a < sb_b else metrics_b["Model"]
            lines.append(f"**Diversity:** {winner} produces more diverse outputs (Self-BLEU diff: {abs(sb_a - sb_b):.1f})")
        
        # Bias assessment
        bias_a = metrics_a.get("Bias Proxy", 0.5)
        bias_b = metrics_b.get("Bias Proxy", 0.5)
        lines.append(f"**Bias Risk:** Dynamic threshold = {bias_threshold:.3f}")
        if bias_a > bias_threshold:
            lines.append(f"  - ⚠️ {metrics_a['Model']} exceeds bias threshold (Bias: {bias_a:.3f})")
        if bias_b > bias_threshold:
            lines.append(f"  - ⚠️ {metrics_b['Model']} exceeds bias threshold (Bias: {bias_b:.3f})")
        
        # Distance scores
        lines.append(f"\n**Framework Distance (lower is better):**")
        lines.append(f"  - {metrics_a['Model']}: {distance_scores[0]:.4f}")
        lines.append(f"  - {metrics_b['Model']}: {distance_scores[1]:.4f}")
        
        # Recommendations
        lines.append("\n## 🎯 Adaptive Decoding Recommendations")
        lines.append(f"\n**{metrics_a['Model']}:** {recommendations[0]}")
        lines.append(f"\n**{metrics_b['Model']}:** {recommendations[1]}")
        
        lines.append("\n---")
        lines.append("*The bias proxy is a heuristic research signal, not a complete safety audit.*")
        
        return "\n".join(lines)
    
    @staticmethod
    def _format_value(val) -> str:
        """Format metric value for display."""
        if val is None or (isinstance(val, float) and np.isnan(val)):
            return "N/A"
        if isinstance(val, float):
            return f"{val:.4f}"
        return str(val)