tests/test_reader_eval.py

#!/usr/bin/env python3
"""Tests for reader evaluation metrics. Pure python, no data, no GPU."""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.evaluate_reader import ece, exact_match, token_f1

PASS = FAIL = 0
def chk(c, l, d=""):
    global PASS, FAIL
    if c: PASS += 1; print(f"  ok   {l}")
    else:  FAIL += 1; print(f"  FAIL {l}  {d}")

print("[token F1]")
chk(token_f1("30 to 55 mph", "30 to 55 mph") == 1.0, "identical -> 1.0")
chk(token_f1("30 to 55 mph.", "30 to 55 mph") == 1.0, "punctuation normalised")
chk(token_f1("Eagles fly 30 to 55 mph", "30 to 55 mph") > 0.6, "superset scores high",
    token_f1("Eagles fly 30 to 55 mph", "30 to 55 mph"))
chk(token_f1("blue whales", "30 to 55 mph") == 0.0, "no overlap -> 0.0")
chk(token_f1("", "answer") == 0.0, "empty prediction")
chk(token_f1("", "") == 1.0, "both empty is trivially equal")
f = token_f1("the bald eagle", "bald eagle is the national bird")
chk(0 < f < 1, "partial overlap between 0 and 1", f)

print("\n[exact match]")
chk(exact_match("1782", "1782.") == 1.0, "punctuation-insensitive")
chk(exact_match("THE Bald Eagle", "the bald eagle") == 1.0, "case-insensitive")
chk(exact_match("bald eagle", "the bald eagle") == 0.0, "extra token breaks EM")

print("\n[unicode / indic]")
chk(token_f1("७३.३ डिग्री", "७३.३ डिग्री") == 1.0, "devanagari identical")
chk(token_f1("மணிக்கு 30 மைல்", "30 மைல்") > 0.5, "tamil partial overlap")

print("\n[ECE]")
# perfectly calibrated: confidence == accuracy in every bin
conf = [0.05]*100 + [0.95]*100
corr = [False]*95 + [True]*5 + [False]*5 + [True]*95
chk(ece(conf, corr) < 0.02, "perfect calibration -> ~0", ece(conf, corr))
# overconfident: says 0.95, is right 50% of the time
chk(ece([0.95]*100, [True]*50 + [False]*50) > 0.4, "overconfidence detected",
    ece([0.95]*100, [True]*50 + [False]*50))
chk(ece([], []) == 0.0, "empty input")

print("\n[unanswerable queries always count as errors]")
# mirrors the rule in evaluate_reader: correct = answerable and f1 >= 0.5
def is_correct(answerable, f1): return bool(answerable and f1 >= 0.50)
chk(is_correct(True, 0.9), "answerable + good span -> correct")
chk(not is_correct(True, 0.2), "answerable + bad span -> wrong")
chk(not is_correct(False, 1.0), "UNANSWERABLE -> wrong even with a perfect span")

print(f"\n{'='*50}\n  {PASS} passed, {FAIL} failed\n{'='*50}")
sys.exit(1 if FAIL else 0)