tests/test_eval_plumbing.py

"""Smoke #7: full eval plumbing load -> generate -> parse -> metric with a tiny
model on 3 synthetic examples (output is garbage; the pipeline is what's tested).
Requires network (tiny model download)."""
import pytest

from mathcompose.eval.processbench import score_subset
from mathcompose.eval.parse import majority_index

TINY = "trl-internal-testing/tiny-Qwen2ForCausalLM-2.5"


@pytest.mark.slow
@pytest.mark.network
def test_eval_pipeline_runs():
    pytest.importorskip("torch")
    from mathcompose.eval.runners import HFRunner, make_verify_fn

    runner = HFRunner(TINY, max_context=512)
    verify_fn = make_verify_fn(runner, n=1, max_new_tokens=8)

    examples = [
        ("What is 2+2?", ["We add.", "2+2=5."], 1),
        ("What is 3*3?", ["Multiply.", "3*3=9."], -1),
        ("Derivative of x^2?", ["Power rule.", "= x."], 1),
    ]
    golds, preds = [], []
    for problem, steps, gold in examples:
        preds.append(majority_index(verify_fn(problem, steps)))
        golds.append(gold)

    s = score_subset("smoke", golds, preds)   # tiny model -> garbage preds, but scoring runs
    assert 0.0 <= s.f1 <= 1.0
    assert s.n_error + s.n_correct == 3


@pytest.mark.slow
@pytest.mark.network
def test_batched_processbench_eval_runs():
    """The fast batched path: load -> chat_many -> parse -> score over a tiny slice."""
    pytest.importorskip("torch")
    from mathcompose.eval.runners import HFRunner
    from mathcompose.eval.processbench import evaluate_batched

    runner = HFRunner(TINY, max_context=512)
    res = evaluate_batched(runner, splits=["gsm8k"], limit=4, n=1, batch_size=2, max_new_tokens=6)
    assert 0.0 <= res["average_f1"] <= 100.0   # now reported on the 0-100 scale
    assert "gsm8k" in res["subsets"]