tests/test_evaluation.py

from __future__ import annotations

import pytest

from deberta_ime.evaluation import EvaluationItem, Prediction, evaluate_predictions


def test_evaluation_uses_input_as_effective_output_when_prediction_abstains() -> None:
    report = evaluate_predictions(
        (
            EvaluationItem(
                item_id="typo-1",
                input_text="犬でし",
                references=("犬です",),
                label="typo",
            ),
        ),
        (),
        candidate_limit=8,
    )

    assert report["metrics"]["effective_acc_at_1"] == 0.0
    assert report["metrics"]["mean_min_cer"] == pytest.approx(1 / 3)
    assert report["metrics"]["abstention_rate"] == 1.0


def test_overcorrection_requires_an_explicit_clean_label() -> None:
    clean = EvaluationItem(
        item_id="clean-1",
        input_text="猫です",
        references=("猫です",),
        label="clean",
    )
    changed = Prediction(item_id="clean-1", candidates=("犬です",))

    labelled = evaluate_predictions((clean,), (changed,), candidate_limit=8)
    unspecified = evaluate_predictions(
        (
            EvaluationItem(
                item_id="clean-1",
                input_text="猫です",
                references=("猫です",),
            ),
        ),
        (changed,),
        candidate_limit=8,
    )

    assert labelled["metrics"]["overcorrection_rate"] == 1.0
    assert labelled["metrics"]["clean_rows"] == 1
    assert unspecified["metrics"]["overcorrection_rate"] is None
    assert unspecified["metrics"]["clean_rows"] == 0


def test_evaluation_separates_candidate_recall_acceptance_and_quality_changes() -> None:
    items = (
        EvaluationItem("clean", "猫です", ("猫です",), "clean"),
        EvaluationItem("typo", "犬でし", ("犬です",), "typo"),
        EvaluationItem("unknown", "鳥でし", ("鳥です",)),
    )
    predictions = (
        Prediction("clean", ("犬です", "猫です")),
        Prediction("typo", ("犬です",)),
    )

    report = evaluate_predictions(items, predictions, candidate_limit=2)
    metrics = report["metrics"]

    assert metrics["candidate_recall_at_k"] == pytest.approx(2 / 3)
    assert metrics["accepted_rows"] == 2
    assert metrics["accepted_accuracy"] == 0.5
    assert metrics["typo_rows"] == 1
    assert metrics["typo_accuracy"] == 1.0
    assert metrics["improved_rows"] == 1
    assert metrics["worsened_rows"] == 1
    assert metrics["unchanged_rows"] == 1


def test_evaluation_reports_candidate_misses_selection_errors_and_provenance() -> None:
    items = (
        EvaluationItem("generator-miss", "犬でし", ("犬です",), "typo"),
        EvaluationItem("selection-miss", "橋です", ("箸です",), "typo"),
        EvaluationItem("accepted", "猫でし", ("猫です",), "typo"),
    )
    predictions = (
        Prediction(
            "generator-miss",
            ("犬でした",),
            provenance="provider",
            reason="baseline_best",
            margin=0.0,
        ),
        Prediction(
            "selection-miss",
            ("橋です", "箸です"),
            provenance="deberta",
            reason="low_margin",
            margin=0.2,
        ),
        Prediction(
            "accepted",
            ("猫です",),
            provenance="deberta",
            reason="accepted",
            margin=1.2,
        ),
    )

    metrics = evaluate_predictions(items, predictions, candidate_limit=8)["metrics"]

    assert metrics["accepted_candidate_miss_rows"] == 1
    assert metrics["selection_error_rows"] == 1
    assert metrics["declared_provenance_counts"] == {"deberta": 2, "provider": 1}
    assert metrics["selection_reason_counts"] == {
        "accepted": 1,
        "baseline_best": 1,
        "low_margin": 1,
    }
    assert metrics["reported_margin_rows"] == 3
    assert metrics["mean_reported_margin"] == pytest.approx(1.4 / 3)


def test_evaluation_rejects_ambiguous_or_unmatched_ids() -> None:
    item = EvaluationItem("one", "猫", ("猫",), "clean")

    with pytest.raises(ValueError, match="duplicate item_id"):
        evaluate_predictions((item, item), (), candidate_limit=8)
    with pytest.raises(ValueError, match="duplicate prediction item_id"):
        evaluate_predictions(
            (item,),
            (Prediction("one", ("猫",)), Prediction("one", ("猫",))),
            candidate_limit=8,
        )
    with pytest.raises(ValueError, match="unknown item_id"):
        evaluate_predictions(
            (item,),
            (Prediction("other", ("猫",)),),
            candidate_limit=8,
        )


def test_clean_rows_must_name_the_input_as_an_acceptable_reference() -> None:
    with pytest.raises(ValueError, match="clean item input must be a reference"):
        evaluate_predictions(
            (EvaluationItem("bad-clean", "猫", ("犬",), "clean"),),
            (),
            candidate_limit=8,
        )


def test_public_evaluation_seam_rejects_empty_or_malformed_artifacts() -> None:
    with pytest.raises(ValueError, match="evaluation items must not be empty"):
        evaluate_predictions((), (), candidate_limit=8)
    with pytest.raises(ValueError, match="prediction item_id must not be empty"):
        Prediction("", ())
    with pytest.raises(ValueError, match="prediction candidates must be unique"):
        Prediction("one", ("猫", "猫"))
    with pytest.raises(ValueError, match="prediction candidates must not contain empty"):
        Prediction("one", ("",))