File size: 2,486 Bytes
11ecc5b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | #!/usr/bin/env python3
"""Tests for reader evaluation metrics. Pure python, no data, no GPU."""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.evaluate_reader import ece, exact_match, token_f1
PASS = FAIL = 0
def chk(c, l, d=""):
global PASS, FAIL
if c: PASS += 1; print(f" ok {l}")
else: FAIL += 1; print(f" FAIL {l} {d}")
print("[token F1]")
chk(token_f1("30 to 55 mph", "30 to 55 mph") == 1.0, "identical -> 1.0")
chk(token_f1("30 to 55 mph.", "30 to 55 mph") == 1.0, "punctuation normalised")
chk(token_f1("Eagles fly 30 to 55 mph", "30 to 55 mph") > 0.6, "superset scores high",
token_f1("Eagles fly 30 to 55 mph", "30 to 55 mph"))
chk(token_f1("blue whales", "30 to 55 mph") == 0.0, "no overlap -> 0.0")
chk(token_f1("", "answer") == 0.0, "empty prediction")
chk(token_f1("", "") == 1.0, "both empty is trivially equal")
f = token_f1("the bald eagle", "bald eagle is the national bird")
chk(0 < f < 1, "partial overlap between 0 and 1", f)
print("\n[exact match]")
chk(exact_match("1782", "1782.") == 1.0, "punctuation-insensitive")
chk(exact_match("THE Bald Eagle", "the bald eagle") == 1.0, "case-insensitive")
chk(exact_match("bald eagle", "the bald eagle") == 0.0, "extra token breaks EM")
print("\n[unicode / indic]")
chk(token_f1("७३.३ डिग्री", "७३.३ डिग्री") == 1.0, "devanagari identical")
chk(token_f1("மணிக்கு 30 மைல்", "30 மைல்") > 0.5, "tamil partial overlap")
print("\n[ECE]")
# perfectly calibrated: confidence == accuracy in every bin
conf = [0.05]*100 + [0.95]*100
corr = [False]*95 + [True]*5 + [False]*5 + [True]*95
chk(ece(conf, corr) < 0.02, "perfect calibration -> ~0", ece(conf, corr))
# overconfident: says 0.95, is right 50% of the time
chk(ece([0.95]*100, [True]*50 + [False]*50) > 0.4, "overconfidence detected",
ece([0.95]*100, [True]*50 + [False]*50))
chk(ece([], []) == 0.0, "empty input")
print("\n[unanswerable queries always count as errors]")
# mirrors the rule in evaluate_reader: correct = answerable and f1 >= 0.5
def is_correct(answerable, f1): return bool(answerable and f1 >= 0.50)
chk(is_correct(True, 0.9), "answerable + good span -> correct")
chk(not is_correct(True, 0.2), "answerable + bad span -> wrong")
chk(not is_correct(False, 1.0), "UNANSWERABLE -> wrong even with a perfect span")
print(f"\n{'='*50}\n {PASS} passed, {FAIL} failed\n{'='*50}")
sys.exit(1 if FAIL else 0)
|