tests/test_eval_numeric_correctness.py

"""Tests for the numeric-correctness eval harness.

The load-bearing property: the scorer rewards a correct *number*, not correct
*format*. A perfectly-templated answer with the wrong value must fail — that is
the whole reason this harness exists alongside the perplexity benchmarks.
"""

from __future__ import annotations

import pytest

from civil_safety import read_jsonl
from eval_numeric_correctness import (
    DEFAULT_SUITE,
    answer_matches,
    extract_numbers,
    ground_truth,
    score_answers,
    self_check,
)

PERIOD_ITEM = {
    "id": "t-period",
    "category": "reactor_kinetics",
    "prompt": "period for +100 pcm?",
    "check": {
        "tool": "reactor_period",
        "args": {"rho_pcm": 100.0},
        "field": "period_s",
        "rel_tol": 0.02,
    },
}
DECAY_ITEM = {
    "id": "t-decay",
    "category": "decay_heat",
    "prompt": "decay heat percent at 1 h?",
    "check": {
        "tool": "decay_heat",
        "args": {"t_seconds": 3600.0},
        "field": "percent",
        "rel_tol": 0.02,
    },
}


def test_extract_numbers_handles_signs_and_scientific():
    values = extract_numbers("worth -1949.4 pcm; Xe at 7.687e+14 atoms; ratio 1.185")
    assert any(v == pytest.approx(-1949.4) for v in values)
    assert any(v == pytest.approx(7.687e14) for v in values)
    assert any(v == pytest.approx(1.185) for v in values)


def test_extract_numbers_masks_isotope_labels():
    # The mass numbers in Xe-135 / I-135 / U-235 must not be read as answers.
    assert extract_numbers("Xe-135 and I-135 and U-235") == []


def test_answer_matches_within_and_outside_tolerance():
    ok, matched = answer_matches(100.0, "about 101 units", rel_tol=0.02, abs_tol=1e-9)
    assert ok and matched == pytest.approx(101.0)
    ok, matched = answer_matches(100.0, "about 130 units", rel_tol=0.02, abs_tol=1e-9)
    assert not ok and matched is None


def test_answer_matches_abs_tol_near_zero():
    ok, _ = answer_matches(0.0, "the value is 0.0000001", rel_tol=0.02, abs_tol=1e-6)
    assert ok


def test_ground_truth_matches_engine():
    assert ground_truth(PERIOD_ITEM) == pytest.approx(53.5875, rel=1e-4)


def test_ground_truth_raises_on_unknown_field():
    bad = {
        "id": "x",
        "check": {
            "tool": "reactor_period",
            "args": {"rho_pcm": 100.0},
            "field": "nope",
        },
    }
    with pytest.raises(ValueError):
        ground_truth(bad)


def test_score_answers_counts_correct_wrong_and_missing():
    items = [PERIOD_ITEM, DECAY_ITEM]
    answers = {
        "t-period": "The stable period is 53.6 s.",  # correct
        # t-decay: intentionally omitted -> missing
    }
    report = score_answers(items, answers)
    assert report.total == 2
    assert report.passed == 1
    by_cat = report.by_category()
    assert by_cat["reactor_kinetics"] == (1, 1)
    assert by_cat["decay_heat"] == (0, 1)


def test_correct_format_wrong_number_fails():
    # A fully-templated answer whose number is wrong must score as a failure.
    templated_but_wrong = (
        "## System Boundaries & Assumptions\nCivil education only.\n"
        "## Governing Physics / Logic\nInhour equation.\n"
        "## Civil Analysis\nThe stable reactor period is about 12.0 s.\n"
        "## Reliability & Safety Considerations\nEducational.\n"
        "## Limitations\nPoint model.\n"
    )
    report = score_answers([PERIOD_ITEM], {"t-period": templated_but_wrong})
    assert report.passed == 0
    assert report.accuracy == 0.0


def test_real_suite_self_check_passes():
    items = read_jsonl(DEFAULT_SUITE)
    report = self_check(items)
    assert report.total >= 12
    assert report.passed == report.total, report.render()