laya-ledgar / code /tests /test_metrics.py
VibeHash's picture
Release laya-ledgar, run ce-seed0
779ff0b verified
Raw History Blame Contribute Delete
4.9 kB
"""The scores the README reports, checked against values worked out by hand."""
from __future__ import annotations
import unittest
from layatasks import metrics
class WilsonTests(unittest.TestCase):
def test_half_of_ten(self):
lo, hi = metrics.wilson(5, 10)
self.assertAlmostEqual(lo, 0.2366, places=4)
self.assertAlmostEqual(hi, 0.7634, places=4)
def test_stays_inside_zero_and_one(self):
self.assertEqual(metrics.wilson(0, 20)[0], 0.0)
self.assertEqual(metrics.wilson(20, 20)[1], 1.0)
self.assertGreater(metrics.wilson(20, 20)[0], 0.8)
def test_refuses_bad_counts(self):
with self.assertRaises(ValueError):
metrics.wilson(3, 0)
with self.assertRaises(ValueError):
metrics.wilson(5, 4)
class EceTests(unittest.TestCase):
def test_honest_confidence_scores_zero(self):
conf = [0.75] * 4
correct = [True, True, True, False]
self.assertAlmostEqual(metrics.ece(conf, correct), 0.0)
def test_overconfidence_shows_as_the_gap(self):
conf = [0.9] * 10
correct = [True] * 6 + [False] * 4
self.assertAlmostEqual(metrics.ece(conf, correct), 0.3)
def test_confidence_one_falls_in_the_last_bin(self):
self.assertAlmostEqual(metrics.ece([1.0, 1.0], [True, True]), 0.0)
class SelectiveTests(unittest.TestCase):
def test_threshold_keeps_only_confident_answers(self):
conf = [0.99, 0.95, 0.6, 0.4]
correct = [True, True, False, True]
s = metrics.selective(conf, correct, 0.9)
self.assertEqual(s["answered"], 2)
self.assertEqual(s["coverage"], 0.5)
self.assertEqual(s["accuracy"], 1.0)
self.assertEqual(s["errors"], 0)
def test_nothing_answered_has_no_accuracy(self):
self.assertIsNone(metrics.selective([0.2], [True], 0.9)["accuracy"])
class CoverageTests(unittest.TestCase):
def test_largest_prefix_at_target(self):
conf = [0.99, 0.98, 0.97, 0.5, 0.4]
correct = [True, True, True, False, True]
c = metrics.coverage_at_accuracy(conf, correct, 0.95)
self.assertEqual(c["answered"], 3)
self.assertEqual(c["threshold"], 0.97)
def test_a_later_prefix_can_recover(self):
conf = [0.9, 0.8, 0.7, 0.6, 0.5]
correct = [True, False, True, True, True]
self.assertEqual(metrics.coverage_at_accuracy(conf, correct, 0.8)["answered"], 5)
def test_ties_are_kept_or_dropped_together(self):
conf = [0.9, 0.9, 0.9, 0.5]
correct = [True, True, False, True]
self.assertIsNone(metrics.coverage_at_accuracy(conf, correct, 0.95))
self.assertEqual(metrics.coverage_at_accuracy(conf, correct, 0.75)["answered"], 4)
def test_none_when_unreachable(self):
self.assertIsNone(metrics.coverage_at_accuracy([0.9, 0.8], [False, False], 0.5))
class McNemarTests(unittest.TestCase):
def test_no_disagreement(self):
self.assertEqual(metrics.mcnemar_exact(0, 0), 1.0)
def test_lopsided_disagreement(self):
self.assertAlmostEqual(metrics.mcnemar_exact(0, 10), 2 / 1024)
def test_symmetric(self):
self.assertEqual(metrics.mcnemar_exact(3, 9), metrics.mcnemar_exact(9, 3))
class CompareTests(unittest.TestCase):
def test_counts_only_the_disagreements(self):
from layatasks.compare import compare
a = [{"gold": "x", "pred": "x"}, {"gold": "y", "pred": "x"}, {"gold": "z", "pred": "z"}]
b = [{"gold": "x", "pred": "x"}, {"gold": "y", "pred": "y"}, {"gold": "z", "pred": "y"}]
r = compare(a, b)
self.assertEqual((r["a_right"], r["b_right"], r["only_a"], r["only_b"]), (2, 2, 1, 1))
def test_refuses_runs_on_different_tests(self):
from layatasks.compare import compare
with self.assertRaises(ValueError):
compare([{"gold": "x", "pred": "x"}], [{"gold": "y", "pred": "y"}])
class F1Tests(unittest.TestCase):
def test_micro_f1_is_accuracy_for_one_label_each(self):
self.assertAlmostEqual(metrics.micro_f1(["a", "b", "b", "c"], ["a", "b", "c", "c"]), 0.75)
def test_macro_f1_by_hand(self):
# a: tp 1, fp 0, fn 0 -> 1. b: tp 1, fp 0, fn 1 -> 2/3. c: tp 1, fp 1, fn 0 -> 2/3.
golds = ["a", "b", "b", "c"]
preds = ["a", "b", "c", "c"]
self.assertAlmostEqual(metrics.macro_f1(golds, preds), (1 + 2 / 3 + 2 / 3) / 3)
def test_a_label_only_ever_predicted_counts_as_zero(self):
# a: tp 1, fn 1 -> 2/3. z: never gold, predicted once and wrong -> 0.
self.assertAlmostEqual(metrics.macro_f1(["a", "a"], ["a", "z"]), (2 / 3 + 0) / 2)
def test_refuses_mismatched_lengths(self):
with self.assertRaises(ValueError):
metrics.macro_f1(["a"], ["a", "b"])
with self.assertRaises(ValueError):
metrics.micro_f1([], [])
if __name__ == "__main__":
unittest.main()