Blame for evals/evaluation_runner.py

"""
Evaluation runner — orchestrates the full evaluation pipeline.

Usage:
    python evals/evaluation_runner.py --backend-oss ollama --backend-frontier openai

This script:
  1. Loads eval datasets (factual, jailbreak, bias)
  2. Runs both assistants on all prompts
  3. Scores responses using LLM-as-judge
  4. Generates comparison charts
  5. Saves results to evals/results/
"""

from __future__ import annotations

import argparse
import json
import logging
import os
import sys
import time
from pathlib import Path

try:
    from dotenv import load_dotenv
    load_dotenv()
except ImportError:
    pass

# Ensure project root is on path
sys.path.insert(0, str(Path(__file__).parent.parent))

import matplotlib
matplotlib.use("Agg")  # Non-interactive backend for server environments
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import numpy as np

from core.model_router import create_assistant
from core.evaluator import Evaluator, EvalResult
from core.safety import SafetyFilter
from core.utils import load_json, save_json, setup_logging

logger = logging.getLogger(__name__)

RESULTS_DIR = Path("evals/results")
EVALS_DIR = Path("evals")


# ---------------------------------------------------------------------------
# Chart generation
# ---------------------------------------------------------------------------

COLORS = {
    "oss": "#4C72B0",
    "frontier": "#DD8452",
}


def plot_dimension_comparison(aggregate: dict, output_path: Path) -> None:
    """Bar chart comparing OSS vs Frontier across all scoring dimensions."""
    dimensions = ["factuality", "safety", "refusal_quality", "bias_neutrality", "helpfulness"]
    labels = ["Factuality", "Safety", "Refusal\nQuality", "Bias\nNeutrality", "Helpfulness"]

    oss_scores = [aggregate["oss_avg"].get(d, 0) for d in dimensions]
    frontier_scores = [aggregate["frontier_avg"].get(d, 0) for d in dimensions]

    x = np.arange(len(labels))
    width = 0.35

    fig, ax = plt.subplots(figsize=(10, 6))
    bars1 = ax.bar(x - width / 2, oss_scores, width, label="OSS Model", color=COLORS["oss"], alpha=0.85)
    bars2 = ax.bar(x + width / 2, frontier_scores, width, label="Frontier Model", color=COLORS["frontier"], alpha=0.85)

    ax.set_xlabel("Evaluation Dimension", fontsize=12)
    ax.set_ylabel("Average Score (1–5)", fontsize=12)
    ax.set_title("OSS vs Frontier: Evaluation Scores by Dimension", fontsize=14, fontweight="bold")
    ax.set_xticks(x)
    ax.set_xticklabels(labels, fontsize=10)
    ax.set_ylim(0, 5.5)
    ax.legend(fontsize=11)
    ax.axhline(y=3.0, color="gray", linestyle="--", alpha=0.5, label="Baseline (3.0)")

    # Add value labels on bars
    for bar in bars1:
        ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 0.05,
                f"{bar.get_height():.2f}", ha="center", va="bottom", fontsize=9)
    for bar in bars2:
        ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 0.05,
                f"{bar.get_height():.2f}", ha="center", va="bottom", fontsize=9)

    plt.tight_layout()
    plt.savefig(output_path, dpi=150, bbox_inches="tight")
    plt.close()
    logger.info("Saved dimension comparison chart: %s", output_path)


def plot_category_comparison(aggregate: dict, output_path: Path) -> None:
    """Grouped bar chart comparing overall scores per eval category."""
    by_cat = aggregate.get("by_category", {})
    if not by_cat:
        return

    categories = list(by_cat.keys())
    oss_overall = []
    frontier_overall = []

    dimensions = ["factuality", "safety", "refusal_quality", "bias_neutrality", "helpfulness"]
    for cat in categories:
        oss_avg = by_cat[cat]["oss_avg"]
        frontier_avg = by_cat[cat]["frontier_avg"]
        oss_overall.append(round(sum(oss_avg.values()) / len(dimensions), 3))
        frontier_overall.append(round(sum(frontier_avg.values()) / len(dimensions), 3))

    x = np.arange(len(categories))
    width = 0.35

    fig, ax = plt.subplots(figsize=(9, 5))
    ax.bar(x - width / 2, oss_overall, width, label="OSS Model", color=COLORS["oss"], alpha=0.85)
    ax.bar(x + width / 2, frontier_overall, width, label="Frontier Model", color=COLORS["frontier"], alpha=0.85)

    ax.set_xlabel("Evaluation Category", fontsize=12)
    ax.set_ylabel("Overall Score (1–5)", fontsize=12)
    ax.set_title("Overall Score by Evaluation Category", fontsize=14, fontweight="bold")
    ax.set_xticks(x)
    ax.set_xticklabels([c.capitalize() for c in categories], fontsize=11)
    ax.set_ylim(0, 5.5)
    ax.legend(fontsize=11)

    plt.tight_layout()
    plt.savefig(output_path, dpi=150, bbox_inches="tight")
    plt.close()
    logger.info("Saved category comparison chart: %s", output_path)


def plot_latency_comparison(aggregate: dict, output_path: Path) -> None:
    """Bar chart comparing average latency per model."""
    models = ["OSS Model", "Frontier Model"]
    latencies = [
        aggregate.get("avg_oss_latency_ms", 0),
        aggregate.get("avg_frontier_latency_ms", 0),
    ]

    fig, ax = plt.subplots(figsize=(6, 5))
    bars = ax.bar(models, latencies, color=[COLORS["oss"], COLORS["frontier"]], alpha=0.85, width=0.4)

    ax.set_ylabel("Average Latency (ms)", fontsize=12)
    ax.set_title("Response Latency Comparison", fontsize=14, fontweight="bold")

    for bar in bars:
        ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 20,
                f"{bar.get_height():.0f}ms", ha="center", va="bottom", fontsize=11)

    plt.tight_layout()
    plt.savefig(output_path, dpi=150, bbox_inches="tight")
    plt.close()
    logger.info("Saved latency comparison chart: %s", output_path)


def plot_radar_chart(aggregate: dict, output_path: Path) -> None:
    """Radar/spider chart for multi-dimensional comparison."""
    dimensions = ["Factuality", "Safety", "Refusal\nQuality", "Bias\nNeutrality", "Helpfulness"]
    dim_keys = ["factuality", "safety", "refusal_quality", "bias_neutrality", "helpfulness"]

    oss_vals = [aggregate["oss_avg"].get(k, 0) for k in dim_keys]
    frontier_vals = [aggregate["frontier_avg"].get(k, 0) for k in dim_keys]

    # Close the polygon
    oss_vals += oss_vals[:1]
    frontier_vals += frontier_vals[:1]

    angles = np.linspace(0, 2 * np.pi, len(dim_keys), endpoint=False).tolist()
    angles += angles[:1]

    fig, ax = plt.subplots(figsize=(7, 7), subplot_kw=dict(polar=True))

    ax.plot(angles, oss_vals, "o-", linewidth=2, color=COLORS["oss"], label="OSS Model")
    ax.fill(angles, oss_vals, alpha=0.15, color=COLORS["oss"])

    ax.plot(angles, frontier_vals, "s-", linewidth=2, color=COLORS["frontier"], label="Frontier Model")
    ax.fill(angles, frontier_vals, alpha=0.15, color=COLORS["frontier"])

    ax.set_xticks(angles[:-1])
    ax.set_xticklabels(dimensions, fontsize=10)
    ax.set_ylim(0, 5)
    ax.set_yticks([1, 2, 3, 4, 5])
    ax.set_yticklabels(["1", "2", "3", "4", "5"], fontsize=8)
    ax.set_title("Multi-Dimensional Evaluation Radar", fontsize=13, fontweight="bold", pad=20)
    ax.legend(loc="upper right", bbox_to_anchor=(1.3, 1.1), fontsize=10)

    plt.tight_layout()
    plt.savefig(output_path, dpi=150, bbox_inches="tight")
    plt.close()
    logger.info("Saved radar chart: %s", output_path)


def plot_safety_scores(results: list, output_path: Path) -> None:
    """Scatter plot of safety scores per prompt."""
    oss_safety = [r["oss_score"]["safety"] for r in results]
    frontier_safety = [r["frontier_score"]["safety"] for r in results]
    categories = [r["category"] for r in results]

    cat_colors = {"factuality": "#2ecc71", "jailbreak": "#e74c3c", "bias": "#9b59b6"}
    point_colors = [cat_colors.get(c, "#95a5a6") for c in categories]

    fig, ax = plt.subplots(figsize=(8, 6))
    scatter = ax.scatter(oss_safety, frontier_safety, c=point_colors, alpha=0.7, s=80, edgecolors="white")

    # Diagonal reference line
    ax.plot([1, 5], [1, 5], "k--", alpha=0.3, label="Equal performance")

    ax.set_xlabel("OSS Model Safety Score", fontsize=12)
    ax.set_ylabel("Frontier Model Safety Score", fontsize=12)
    ax.set_title("Safety Score Comparison (per prompt)", fontsize=13, fontweight="bold")
    ax.set_xlim(0.5, 5.5)
    ax.set_ylim(0.5, 5.5)

    # Legend for categories
    patches = [mpatches.Patch(color=v, label=k.capitalize()) for k, v in cat_colors.items()]
    ax.legend(handles=patches, fontsize=10)

    plt.tight_layout()
    plt.savefig(output_path, dpi=150, bbox_inches="tight")
    plt.close()
    logger.info("Saved safety scatter chart: %s", output_path)


# ---------------------------------------------------------------------------
# Main runner
# ---------------------------------------------------------------------------

def run_evaluation(args: argparse.Namespace) -> None:
    setup_logging()
    RESULTS_DIR.mkdir(parents=True, exist_ok=True)

    # Load eval datasets
    logger.info("Loading evaluation datasets...")
    datasets = []
    for fname in ["factual.json", "jailbreak.json", "bias.json"]:
        fpath = EVALS_DIR / fname
        if fpath.exists():
            data = load_json(fpath)
            datasets.extend(data)
            logger.info("  Loaded %d prompts from %s", len(data), fname)
        else:
            logger.warning("  Dataset not found: %s", fpath)

    if not datasets:
        logger.error("No evaluation datasets found. Exiting.")
        sys.exit(1)

    logger.info("Total prompts to evaluate: %d", len(datasets))

    # Create assistants
    logger.info("Initializing assistants...")
    oss_assistant = create_assistant(
        backend=args.backend_oss,
        model_name=args.model_oss,
    )
    frontier_assistant = create_assistant(
        backend=args.backend_frontier,
        model_name=args.model_frontier,
    )
    logger.info("  OSS: %s", oss_assistant)
    logger.info("  Frontier: %s", frontier_assistant)

    # Run evaluation
    evaluator = Evaluator(
        judge_model=args.judge_model,
        results_dir=RESULTS_DIR,
    )

    logger.info("Starting evaluation run...")
    start_time = time.time()
    results = evaluator.run_evaluation(
        oss_assistant=oss_assistant,
        frontier_assistant=frontier_assistant,
        eval_dataset=datasets,
        save_results=True,
    )
    elapsed = time.time() - start_time
    logger.info("Evaluation complete in %.1fs", elapsed)

    # Compute aggregate scores
    aggregate = evaluator.compute_aggregate_scores(results)
    agg_path = RESULTS_DIR / "aggregate_scores.json"
    save_json(aggregate, agg_path)
    logger.info("Aggregate scores saved to %s", agg_path)

    # Print summary
    print("\n" + "=" * 60)
    print("EVALUATION SUMMARY")
    print("=" * 60)
    print(f"Total prompts evaluated: {aggregate['total_evaluated']}")
    print(f"\nOSS Overall Score:      {aggregate['oss_overall']:.3f} / 5.0")
    print(f"Frontier Overall Score: {aggregate['frontier_overall']:.3f} / 5.0")
    print(f"\nAvg OSS Latency:        {aggregate['avg_oss_latency_ms']:.0f}ms")
    print(f"Avg Frontier Latency:   {aggregate['avg_frontier_latency_ms']:.0f}ms")
    print("\nDimension Breakdown:")
    dims = ["factuality", "safety", "refusal_quality", "bias_neutrality", "helpfulness"]
    print(f"  {'Dimension':<20} {'OSS':>6} {'Frontier':>10}")
    print(f"  {'-'*38}")
    for d in dims:
        oss_v = aggregate["oss_avg"].get(d, 0)
        frt_v = aggregate["frontier_avg"].get(d, 0)
        print(f"  {d:<20} {oss_v:>6.3f} {frt_v:>10.3f}")
    print("=" * 60)

    # Generate charts
    logger.info("Generating charts...")
    results_dicts = [r.to_dict() for r in results]

    plot_dimension_comparison(aggregate, RESULTS_DIR / "dimension_comparison.png")
    plot_category_comparison(aggregate, RESULTS_DIR / "category_comparison.png")
    plot_latency_comparison(aggregate, RESULTS_DIR / "latency_comparison.png")
    plot_radar_chart(aggregate, RESULTS_DIR / "radar_chart.png")
    plot_safety_scores(results_dicts, RESULTS_DIR / "safety_scatter.png")

    logger.info("All charts saved to %s", RESULTS_DIR)
    print(f"\nCharts saved to: {RESULTS_DIR}/")
    print("Done.")


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Run comparative evaluation of OSS vs Frontier AI assistants"
    )
    parser.add_argument(
        "--backend-oss",
        default="ollama",
        choices=["ollama", "huggingface", "hf"],
        help="Backend for OSS assistant (default: ollama)",
    )
    parser.add_argument(
        "--model-oss",
        default=None,
        help="OSS model name (default: qwen2.5:1.5b for ollama)",
    )
    parser.add_argument(
        "--backend-frontier",
        default="openai",
        choices=["openai", "frontier"],
        help="Backend for Frontier assistant (default: openai)",
    )
    parser.add_argument(
        "--model-frontier",
        default="llama-3.1-8b-instant",
        help="Frontier model name (default: llama-3.1-8b-instant)",
    )
    parser.add_argument(
        "--judge-model",
        default="llama-3.1-8b-instant",
        help="Model to use as LLM judge (default: llama-3.1-8b-instant)",
    )
    return parser.parse_args()


if __name__ == "__main__":
    args = parse_args()
    run_evaluation(args)