tests/eval/test_model_comparison.py

"""
Eval: model comparison harness.

Two test classes:

  ModelComparisonMocked (always runs in CI)
  ─────────────────────────────────────────
  Validates that the routing and response-parsing layer works correctly
  for each model config using simulated LLM responses. Fast, no API key
  needed.

  ModelComparisonReal (runs only when RUN_REAL_EVAL=1)
  ─────────────────────────────────────────────────────
  Actually routes through each model via OpenRouter and records
  precision / recall / false-positive-rate scores to
  tests/eval/results/model_comparison_{timestamp}.json.

  Run with:
      RUN_REAL_EVAL=1 pytest tests/eval/test_model_comparison.py -v -s

  Requires:
      OPENROUTER_API_KEY set in environment
"""
from __future__ import annotations

import json
import os
import time
from datetime import datetime, timezone
from pathlib import Path
from unittest.mock import AsyncMock, patch

import pytest

from app.agents import bug_detection_agent, code_review_agent
from app.models.repository import RepositoryMetadata
from tests.eval.benchmark_dataset import BENCHMARK_FIXTURES

RESULTS_DIR = Path(__file__).parent / "results"

SRP_CODE = """\
class GodClass:
    def save_to_db(self): ...
    def send_email(self): ...
    def generate_report(self): ...
    def authenticate_user(self): ...
"""

MODEL_CONFIGS = [
    {
        "id": "llama-3.3-70b",
        "model": "meta-llama/llama-3.3-70b-instruct",
        "simulated_score": 4.5,
        "simulated_violations": ["GodClass violates SRP"],
    },
    {
        "id": "claude-sonnet",
        "model": "anthropic/claude-sonnet-4-5",
        "simulated_score": 3.5,
        "simulated_violations": ["GodClass handles DB, email, reporting, auth — clear SRP violation"],
    },
    {
        "id": "gpt-4o",
        "model": "openai/gpt-4o",
        "simulated_score": 4.0,
        "simulated_violations": ["GodClass violates Single Responsibility Principle"],
    },
]

BUGGY_FIXTURES = [f for f in BENCHMARK_FIXTURES if not f["clean"]]
CLEAN_FIXTURES  = [f for f in BENCHMARK_FIXTURES if f["clean"]]

RUN_REAL_EVAL = os.getenv("RUN_REAL_EVAL", "0") == "1"


# ── helpers ────────────────────────────────────────────────────────────────

def _make_review_response(score: float, violations: list[str]) -> str:
    return json.dumps({
        "solid_violations":     violations,
        "duplicate_code":       [],
        "refactor_suggestions": ["Split into focused classes"],
        "overall_score":        score,
        "summary":              f"Model detected {len(violations)} violation(s).",
    })


def _make_metadata(tmp_path: Path) -> RepositoryMetadata:
    return RepositoryMetadata(
        url="https://github.com/example/repo",
        name="repo",
        local_path=str(tmp_path),
        language="Python",
        frameworks=[],
        architecture="",
        entry_points=[],
        summary="",
    )


def _detection_hit(report_text: str, expected_bugs: list[str]) -> bool:
    lower = report_text.lower()
    return any(kw.lower() in lower for kw in expected_bugs)


def _save_results(results: list[dict]) -> Path:
    RESULTS_DIR.mkdir(parents=True, exist_ok=True)
    timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
    path = RESULTS_DIR / f"model_comparison_{timestamp}.json"
    path.write_text(json.dumps(results, indent=2))
    return path


# ── mocked suite (always runs) ─────────────────────────────────────────────

class TestModelComparisonMocked:
    """Fast mocked tests — validate routing and parsing for each model config."""

    @pytest.mark.parametrize("config", MODEL_CONFIGS, ids=[c["id"] for c in MODEL_CONFIGS])
    @pytest.mark.asyncio
    async def test_model_detects_srp_violation(self, config: dict, tmp_path: Path) -> None:
        (tmp_path / "god_class.py").write_text(SRP_CODE)
        metadata     = _make_metadata(tmp_path)
        mock_response = _make_review_response(
            config["simulated_score"], config["simulated_violations"]
        )

        with (
            patch("app.core.llm._call_llm_with_model", new=AsyncMock(return_value=mock_response)),
            patch("app.core.config.get_settings") as mock_settings,
        ):
            mock_settings.return_value.llm_model          = config["model"]
            mock_settings.return_value.llm_model_cheap    = config["model"]
            mock_settings.return_value.llm_model_expensive = config["model"]
            mock_settings.return_value.openrouter_api_key = "test-key"
            review = await code_review_agent.run(str(tmp_path), metadata)

        assert review.overall_score <= 6.0, (
            f"[{config['id']}] Expected score ≤ 6.0 for SRP violation, got {review.overall_score}"
        )
        assert len(review.solid_violations) > 0, (
            f"[{config['id']}] Expected SRP violations to be detected"
        )

    @pytest.mark.asyncio
    async def test_comparison_summary_prints_table(self, tmp_path: Path) -> None:
        (tmp_path / "god_class.py").write_text(SRP_CODE)
        metadata = _make_metadata(tmp_path)
        rows     = []

        for config in MODEL_CONFIGS:
            mock_response = _make_review_response(
                config["simulated_score"], config["simulated_violations"]
            )
            with (
                patch("app.core.llm._call_llm_with_model", new=AsyncMock(return_value=mock_response)),
                patch("app.core.config.get_settings") as mock_settings,
            ):
                mock_settings.return_value.llm_model          = config["model"]
                mock_settings.return_value.llm_model_cheap    = config["model"]
                mock_settings.return_value.llm_model_expensive = config["model"]
                mock_settings.return_value.openrouter_api_key = "test-key"
                review = await code_review_agent.run(str(tmp_path), metadata)

            rows.append({
                "model":      config["id"],
                "score":      review.overall_score,
                "violations": len(review.solid_violations),
            })

        print(f"\n{'='*55}")
        print(f"{'Model':<20} {'Score':>8} {'Violations':>12}")
        print(f"{'-'*55}")
        for r in rows:
            print(f"{r['model']:<20} {r['score']:>8.1f} {r['violations']:>12}")
        print(f"{'='*55}")

        assert len(rows) == len(MODEL_CONFIGS)


# ── real eval suite (RUN_REAL_EVAL=1 only) ────────────────────────────────

@pytest.mark.skipif(not RUN_REAL_EVAL, reason="Set RUN_REAL_EVAL=1 to run real model evals")
class TestModelComparisonReal:
    """
    Calls actual LLM endpoints via OpenRouter and records precision/recall
    scores per model to tests/eval/results/model_comparison_{ts}.json.
    """

    @pytest.mark.asyncio
    @pytest.mark.parametrize("config", MODEL_CONFIGS, ids=[c["id"] for c in MODEL_CONFIGS])
    async def test_real_model_precision_recall(
        self, config: dict, tmp_path: Path
    ) -> None:
        true_positives  = 0
        false_negatives = 0
        false_positives = 0
        latencies: list[float] = []

        with patch("app.core.config.get_settings") as mock_settings:
            mock_settings.return_value.llm_model           = config["model"]
            mock_settings.return_value.llm_model_cheap     = config["model"]
            mock_settings.return_value.llm_model_expensive = config["model"]
            mock_settings.return_value.openrouter_api_key  = os.environ["OPENROUTER_API_KEY"]

            # Run buggy fixtures — expect detections
            for fixture in BUGGY_FIXTURES:
                code_file = tmp_path / "sample.py"
                code_file.write_text(fixture["code"])

                t0     = time.monotonic()
                report = await bug_detection_agent.run(str(tmp_path))
                latencies.append(time.monotonic() - t0)

                report_text = " ".join(
                    [b.description for b in report.critical]
                    + [w.description for w in report.warnings]
                )
                if _detection_hit(report_text, fixture["expected_bugs"]):
                    true_positives += 1
                else:
                    false_negatives += 1

            # Run clean fixtures — expect no detections (false positive check)
            for fixture in CLEAN_FIXTURES:
                code_file = tmp_path / "sample.py"
                code_file.write_text(fixture["code"])

                report = await bug_detection_agent.run(str(tmp_path))
                if report.total_critical > 0:
                    false_positives += 1

        total_buggy = len(BUGGY_FIXTURES)
        total_clean = len(CLEAN_FIXTURES)
        recall      = true_positives / total_buggy if total_buggy else 0.0
        fpr         = false_positives / total_clean if total_clean else 0.0
        avg_latency = sum(latencies) / len(latencies) if latencies else 0.0

        # Precision requires knowing true negatives — approximate here
        # as TP / (TP + FP) across the full fixture set
        precision = (
            true_positives / (true_positives + false_positives)
            if (true_positives + false_positives) > 0
            else 0.0
        )

        result = {
            "model":           config["id"],
            "model_string":    config["model"],
            "timestamp":       datetime.now(timezone.utc).isoformat(),
            "fixtures": {
                "buggy": total_buggy,
                "clean": total_clean,
            },
            "metrics": {
                "true_positives":  true_positives,
                "false_negatives": false_negatives,
                "false_positives": false_positives,
                "recall":          round(recall, 3),
                "precision":       round(precision, 3),
                "false_positive_rate": round(fpr, 3),
                "avg_latency_s":   round(avg_latency, 2),
            },
        }

        out_path = _save_results([result])
        print(f"\n[{config['id']}] recall={recall:.1%}  precision={precision:.1%}  fpr={fpr:.1%}  → {out_path}")

        # Minimum quality bar — fail the eval if the model is too noisy or misses too much
        assert recall    >= 0.5, f"[{config['id']}] Recall {recall:.1%} below 50% threshold"
        assert fpr       <= 0.5, f"[{config['id']}] False positive rate {fpr:.1%} above 50% threshold"