tests/unit/test_evaluation.py

from __future__ import annotations

import json
import tempfile
import unittest
from pathlib import Path

from training.evaluation import (
    attach_perplexity,
    compare_base_vs_tuned,
    default_prompt_cases,
    evaluate_responses,
    load_prompt_cases,
    log_eval_report,
    perplexity_from_losses,
)


class EvaluationTest(unittest.TestCase):
    def test_evaluates_exact_match_rate(self) -> None:
        cases = default_prompt_cases()

        report = evaluate_responses(cases, ["field note", "wrong", "no"])

        self.assertEqual(report.exact_match_rate, 2 / 3)
        self.assertEqual(report.rows[1].notes, "review")

    def test_compares_base_vs_tuned_reports(self) -> None:
        cases = default_prompt_cases()
        base = evaluate_responses(cases, ["wrong", "wrong", "wrong"])
        tuned = evaluate_responses(cases, ["field note", "jsonl", "no"])

        comparison = compare_base_vs_tuned(base, tuned)

        self.assertEqual(comparison.delta, 1.0)

    def test_loads_prompt_cases_from_jsonl(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            path = Path(tmp) / "cases.jsonl"
            path.write_text(
                json.dumps({"prompt": "Prompt", "expected": "Answer"}) + "\n",
                encoding="utf-8",
            )

            cases = load_prompt_cases(path)

            self.assertEqual(cases[0].prompt, "Prompt")
            self.assertEqual(cases[0].expected, "Answer")

    def test_logs_eval_report(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            path = Path(tmp) / "eval.jsonl"
            report = evaluate_responses(default_prompt_cases(), ["field note"])

            saved = log_eval_report(report, path)

            self.assertEqual(saved, path)
            self.assertIn("exact_match_rate", path.read_text(encoding="utf-8"))

    def test_calculates_perplexity_from_average_loss(self) -> None:
        perplexity = perplexity_from_losses([0.0, 0.0])

        self.assertEqual(perplexity, 1.0)
        self.assertIsNone(perplexity_from_losses([]))

    def test_attaches_perplexity_to_eval_report(self) -> None:
        report = evaluate_responses(default_prompt_cases(), ["field note"])

        updated = attach_perplexity(report, [0.0])

        self.assertEqual(updated.exact_match_rate, report.exact_match_rate)
        self.assertEqual(updated.perplexity, 1.0)


if __name__ == "__main__":
    unittest.main()