evals/tests/test_run_eval.py

"""Unit tests for the pure (torch-free, app-free) eval helpers.

These import only ``run_eval`` — never ``app`` or torch — because the heavy
model imports live inside ``run_eval.run()``. That keeps this suite fast and
lets it run anywhere sklearn is installed.
"""

import pytest

import run_eval


# --- Slice A: label-compatibility guard (review fix #12) ---------------------


def test_label_compat_subset_passes():
    # 3-class dataset against a 3-class model: nothing to complain about.
    run_eval.validate_label_compatibility(
        {"positive", "negative"}, {"negative", "neutral", "positive"}, allow_mismatch=False
    )


def test_label_compat_mismatch_raises_system_exit():
    # `neutral` cannot be predicted by a binary model -> hard fail by default.
    with pytest.raises(SystemExit) as exc:
        run_eval.validate_label_compatibility(
            {"negative", "neutral", "positive"}, {"negative", "positive"}, allow_mismatch=False
        )
    assert "neutral" in str(exc.value)
    assert "--allow-label-mismatch" in str(exc.value)


def test_label_compat_mismatch_allowed_warns_and_returns(capsys):
    run_eval.validate_label_compatibility(
        {"negative", "neutral", "positive"}, {"negative", "positive"}, allow_mismatch=True
    )
    out = capsys.readouterr().out
    assert "WARNING" in out
    assert "neutral" in out


# --- Slice B: CSV parsing + row validation -----------------------------------

_GOOD_CSV = (
    "id,text,true_label,category,notes\n"
    "1,The battery life is incredible,positive,product_review,clear positive\n"
    '2,"Not bad, not great",neutral,ambiguous,mixed phrase\n'
)


def test_parse_eval_rows_reads_all_fields():
    rows = run_eval.parse_eval_rows(_GOOD_CSV)
    assert len(rows) == 2
    assert rows[0].id == "1"
    assert rows[0].text == "The battery life is incredible"
    assert rows[0].true_label == "positive"
    assert rows[0].category == "product_review"
    # Quoted field with an embedded comma survives intact.
    assert rows[1].text == "Not bad, not great"


def test_parse_eval_rows_missing_required_column_raises():
    bad = "id,text,category\n1,hello,greeting\n"  # no true_label
    with pytest.raises(ValueError) as exc:
        run_eval.parse_eval_rows(bad)
    assert "true_label" in str(exc.value)


def test_parse_eval_rows_rejects_overlong_text_without_truncating():
    long_text = "x" * (run_eval.MAX_TEXT_CHARS + 1)
    bad = f"id,text,true_label\n1,{long_text},positive\n"
    with pytest.raises(ValueError) as exc:
        run_eval.parse_eval_rows(bad)
    msg = str(exc.value)
    assert str(run_eval.MAX_TEXT_CHARS) in msg
    # Error names the offending row so the author can fix it.
    assert "1" in msg


def test_parse_eval_rows_skips_blank_text_rows():
    csv_text = "id,text,true_label\n1,,positive\n2,real,negative\n"
    rows = run_eval.parse_eval_rows(csv_text)
    assert [r.text for r in rows] == ["real"]


def test_parse_eval_rows_defaults_optional_metadata():
    rows = run_eval.parse_eval_rows("id,text,true_label\n1,hi,positive\n")
    assert rows[0].category == ""
    assert rows[0].notes == ""


# --- Slice C: metrics + latency percentiles ----------------------------------

_LABELS = ["negative", "neutral", "positive"]
_Y_TRUE = ["positive", "positive", "negative", "neutral"]
_Y_PRED = ["positive", "negative", "negative", "neutral"]


def test_compute_metrics_accuracy():
    m = run_eval.compute_metrics(_Y_TRUE, _Y_PRED, _LABELS)
    assert m["accuracy"] == pytest.approx(0.75)


def test_compute_metrics_has_macro_f1_in_unit_range():
    m = run_eval.compute_metrics(_Y_TRUE, _Y_PRED, _LABELS)
    assert 0.0 <= m["macro_f1"] <= 1.0


def test_compute_metrics_confusion_matrix_ordered_by_labels():
    m = run_eval.compute_metrics(_Y_TRUE, _Y_PRED, _LABELS)
    # rows = true class, cols = predicted class, both in _LABELS order.
    assert m["confusion_matrix"] == [[1, 0, 0], [0, 1, 0], [1, 0, 1]]


def test_compute_metrics_per_class_support():
    m = run_eval.compute_metrics(_Y_TRUE, _Y_PRED, _LABELS)
    support = {k: v["support"] for k, v in m["per_class"].items()}
    assert support == {"negative": 1, "neutral": 1, "positive": 2}


def test_latency_percentiles_p50_is_median_and_p95_ge_p50():
    data = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0]
    p = run_eval.latency_percentiles(data)
    assert p["p50"] == pytest.approx(55.0)
    assert p["p95"] >= p["p50"]
    assert p["p95"] <= max(data)


def test_latency_percentiles_single_sample():
    p = run_eval.latency_percentiles([42.0])
    assert p["p50"] == pytest.approx(42.0)
    assert p["p95"] == pytest.approx(42.0)


def test_compute_metrics_handles_class_never_predicted():
    # The --allow-label-mismatch path: a class present in y_true that the model
    # can never emit (e.g. `neutral` under a binary model). zero_division=0 must
    # keep it scoreless instead of raising, and the matrix must stay square so
    # the report renderer (which iterates `labels`) never hits a KeyError.
    labels = ["negative", "positive", "neutral"]
    m = run_eval.compute_metrics(["neutral", "positive"], ["positive", "positive"], labels)
    assert len(m["confusion_matrix"]) == 3
    assert all(len(row) == 3 for row in m["confusion_matrix"])
    assert m["per_class"]["neutral"]["support"] == 1
    assert m["per_class"]["neutral"]["precision"] == 0
    assert m["per_class"]["neutral"]["recall"] == 0


# --- Slice D: wrong-example analysis + summary + report ----------------------


def _rows():
    return [
        run_eval.EvalRow("1", "The battery life is incredible", "positive",
                         "product_review", "clear positive"),
        run_eval.EvalRow("4", "Yeah, amazing, another crash", "negative",
                         "sarcasm", "sarcastic negative"),
        run_eval.EvalRow("3", "I love the design but it crashes", "neutral",
                         "mixed", "multi-sentiment"),
    ]


def test_collect_wrong_examples_only_mismatches():
    rows = _rows()
    y_pred = ["positive", "positive", "neutral"]  # row 4 sarcasm misread
    conf = [0.98, 0.91, 0.55]
    wrong = run_eval.collect_wrong_examples(rows, y_pred, conf)
    assert len(wrong) == 1
    w = wrong[0]
    assert w["true"] == "negative"
    assert w["predicted"] == "positive"
    assert w["category"] == "sarcasm"
    assert w["confidence"] == pytest.approx(0.91)
    assert "amazing" in w["text"]


def test_build_summary_shape():
    rows = _rows()
    y_pred = ["positive", "positive", "neutral"]
    conf = [0.98, 0.91, 0.55]
    metrics = run_eval.compute_metrics(
        [r.true_label for r in rows], y_pred, _LABELS
    )
    summary = run_eval.build_summary(
        model_id="twitter-roberta",
        model_name="cardiffnlp/twitter-roberta-base-sentiment-latest",
        data_file="evals/data/sentiment_eval.csv",
        rows=rows,
        y_pred=y_pred,
        confidences=conf,
        latencies_ms=[70.0, 72.0, 80.0],
        metrics=metrics,
        labels=_LABELS,
    )
    for key in (
        "model_id", "accuracy", "macro_f1", "latency_p50_ms",
        "latency_p95_ms", "confusion_matrix", "wrong_examples", "labels",
        "n_examples",
    ):
        assert key in summary
    assert summary["n_examples"] == 3
    assert isinstance(summary["latency_p95_ms"], float)


def test_render_report_contains_all_required_sections():
    rows = _rows()
    y_pred = ["positive", "positive", "neutral"]
    conf = [0.98, 0.91, 0.55]
    metrics = run_eval.compute_metrics(
        [r.true_label for r in rows], y_pred, _LABELS
    )
    summary = run_eval.build_summary(
        "twitter-roberta", "cardiffnlp/twitter-roberta", "d.csv",
        rows, y_pred, conf, [70.0, 72.0, 80.0], metrics, _LABELS,
    )
    md = run_eval.render_report(summary)
    # Honest-scope disclaimer (required verbatim theme).
    assert "not creativity judges" in md
    # Metric sections.
    assert "Accuracy" in md and "Macro F1" in md
    assert "Confusion" in md
    assert "p50" in md and "p95" in md
    # Wrong-example table columns.
    for col in ("Text", "Category", "True", "Predicted", "Confidence"):
        assert col in md
    # Top failure modes.
    for mode in ("Sarcasm", "Mixed sentiment", "finance", "context"):
        assert mode in md
    # The sarcasm miss should surface in the table.
    assert "amazing" in md