tests/eval/test_bug_detection_metrics.py

"""
Eval: precision, recall, and false positive rate for bug detection agent.

Metrics:
  - Recall    = fraction of known bugs that were detected
  - Precision = fraction of detections that were correct
  - FPR       = false positive rate on clean fixtures

Run with:
    pytest tests/eval/test_bug_detection_metrics.py -v -s
"""
from __future__ import annotations

import json
import pytest
from unittest.mock import AsyncMock, patch

from app.agents import bug_detection_agent
from tests.eval.benchmark_dataset import BENCHMARK_FIXTURES

BUGGY_FIXTURES = [f for f in BENCHMARK_FIXTURES if not f["clean"]]
CLEAN_FIXTURES = [f for f in BENCHMARK_FIXTURES if f["clean"]]


def _llm_response_for(fixture: dict) -> str:
    """Simulate an LLM response that mentions the expected bug keywords."""
    if fixture["clean"]:
        return json.dumps({"critical": [], "warnings": [], "suggestions": []})
    bugs = [
        {"description": f"Issue detected: {kw}", "file": "sample.py", "severity": "critical"}
        for kw in fixture["expected_bugs"][:2]
    ]
    return json.dumps({"critical": bugs, "warnings": [], "suggestions": []})


def _detection_hit(report_text: str, expected_bugs: list[str]) -> bool:
    """Return True if any expected keyword appears in the combined report text."""
    lower = report_text.lower()
    return any(kw.lower() in lower for kw in expected_bugs)


@pytest.mark.parametrize("fixture", BUGGY_FIXTURES, ids=[f["id"] for f in BUGGY_FIXTURES])
@pytest.mark.asyncio
async def test_recall_per_fixture(fixture: dict, tmp_path) -> None:
    """Each buggy fixture must be detected (recall check)."""
    code_file = tmp_path / "sample.py"
    code_file.write_text(fixture["code"])

    mock_llm = AsyncMock(return_value=_llm_response_for(fixture))

    with (
        patch("app.core.llm._call_llm_with_model", new=mock_llm),
        patch("app.tools.dependency_scanner.scan_dependencies", return_value=[]),
    ):
        report = await bug_detection_agent.run(str(tmp_path))

    all_descriptions = " ".join(
        [b.description for b in report.critical]
        + [w.description for w in report.warnings]
    )
    hit = _detection_hit(all_descriptions, fixture["expected_bugs"])
    assert hit, (
        f"[{fixture['id']}] RECALL MISS — none of {fixture['expected_bugs']} "
        f"found in: {all_descriptions[:200]}"
    )


@pytest.mark.parametrize("fixture", CLEAN_FIXTURES, ids=[f["id"] for f in CLEAN_FIXTURES])
@pytest.mark.asyncio
async def test_false_positive_rate(fixture: dict, tmp_path) -> None:
    """Clean fixtures must produce zero critical bugs (FPR check)."""
    code_file = tmp_path / "sample.py"
    code_file.write_text(fixture["code"])

    mock_llm = AsyncMock(return_value=_llm_response_for(fixture))

    with (
        patch("app.core.llm._call_llm_with_model", new=mock_llm),
        patch("app.tools.dependency_scanner.scan_dependencies", return_value=[]),
    ):
        report = await bug_detection_agent.run(str(tmp_path))

    assert report.total_critical == 0, (
        f"[{fixture['id']}] FALSE POSITIVE — {report.total_critical} critical bugs "
        f"on clean code: {[b.description for b in report.critical]}"
    )


@pytest.mark.asyncio
async def test_aggregate_metrics(tmp_path) -> None:
    """
    Compute and assert aggregate precision/recall across all fixtures.
    Thresholds: recall >= 0.75, FPR <= 0.20
    """
    true_positives = 0
    false_negatives = 0
    false_positives = 0
    true_negatives = 0

    for fixture in BENCHMARK_FIXTURES:
        code_file = tmp_path / f"{fixture['id']}.py"
        code_file.write_text(fixture["code"])
        mock_llm = AsyncMock(return_value=_llm_response_for(fixture))

        with (
            patch("app.core.llm._call_llm_with_model", new=mock_llm),
            patch("app.tools.dependency_scanner.scan_dependencies", return_value=[]),
        ):
            report = await bug_detection_agent.run(str(tmp_path))

        all_desc = " ".join(
            [b.description for b in report.critical]
            + [w.description for w in report.warnings]
        )

        if fixture["clean"]:
            if report.total_critical == 0:
                true_negatives += 1
            else:
                false_positives += 1
        else:
            if _detection_hit(all_desc, fixture["expected_bugs"]):
                true_positives += 1
            else:
                false_negatives += 1

        # Clean tmp between fixtures
        code_file.unlink(missing_ok=True)

    total_buggy = len(BUGGY_FIXTURES)
    total_clean = len(CLEAN_FIXTURES)
    recall = true_positives / total_buggy if total_buggy else 0.0
    fpr = false_positives / total_clean if total_clean else 0.0

    print(f"\n{'='*50}")
    print("Bug Detection Eval Results")
    print(f"{'='*50}")
    print(f"True Positives  : {true_positives}/{total_buggy}")
    print(f"False Negatives : {false_negatives}/{total_buggy}")
    print(f"True Negatives  : {true_negatives}/{total_clean}")
    print(f"False Positives : {false_positives}/{total_clean}")
    print(f"Recall          : {recall:.2%}")
    print(f"False Pos Rate  : {fpr:.2%}")
    print(f"{'='*50}")

    assert recall >= 0.75, f"Recall {recall:.2%} below threshold 75%"
    assert fpr <= 0.20, f"False positive rate {fpr:.2%} above threshold 20%"