File size: 4,903 Bytes
779ff0b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
"""The scores the README reports, checked against values worked out by hand."""

from __future__ import annotations

import unittest

from layatasks import metrics


class WilsonTests(unittest.TestCase):
    def test_half_of_ten(self):
        lo, hi = metrics.wilson(5, 10)
        self.assertAlmostEqual(lo, 0.2366, places=4)
        self.assertAlmostEqual(hi, 0.7634, places=4)

    def test_stays_inside_zero_and_one(self):
        self.assertEqual(metrics.wilson(0, 20)[0], 0.0)
        self.assertEqual(metrics.wilson(20, 20)[1], 1.0)
        self.assertGreater(metrics.wilson(20, 20)[0], 0.8)

    def test_refuses_bad_counts(self):
        with self.assertRaises(ValueError):
            metrics.wilson(3, 0)
        with self.assertRaises(ValueError):
            metrics.wilson(5, 4)


class EceTests(unittest.TestCase):
    def test_honest_confidence_scores_zero(self):
        conf = [0.75] * 4
        correct = [True, True, True, False]
        self.assertAlmostEqual(metrics.ece(conf, correct), 0.0)

    def test_overconfidence_shows_as_the_gap(self):
        conf = [0.9] * 10
        correct = [True] * 6 + [False] * 4
        self.assertAlmostEqual(metrics.ece(conf, correct), 0.3)

    def test_confidence_one_falls_in_the_last_bin(self):
        self.assertAlmostEqual(metrics.ece([1.0, 1.0], [True, True]), 0.0)


class SelectiveTests(unittest.TestCase):
    def test_threshold_keeps_only_confident_answers(self):
        conf = [0.99, 0.95, 0.6, 0.4]
        correct = [True, True, False, True]
        s = metrics.selective(conf, correct, 0.9)
        self.assertEqual(s["answered"], 2)
        self.assertEqual(s["coverage"], 0.5)
        self.assertEqual(s["accuracy"], 1.0)
        self.assertEqual(s["errors"], 0)

    def test_nothing_answered_has_no_accuracy(self):
        self.assertIsNone(metrics.selective([0.2], [True], 0.9)["accuracy"])


class CoverageTests(unittest.TestCase):
    def test_largest_prefix_at_target(self):
        conf = [0.99, 0.98, 0.97, 0.5, 0.4]
        correct = [True, True, True, False, True]
        c = metrics.coverage_at_accuracy(conf, correct, 0.95)
        self.assertEqual(c["answered"], 3)
        self.assertEqual(c["threshold"], 0.97)

    def test_a_later_prefix_can_recover(self):
        conf = [0.9, 0.8, 0.7, 0.6, 0.5]
        correct = [True, False, True, True, True]
        self.assertEqual(metrics.coverage_at_accuracy(conf, correct, 0.8)["answered"], 5)

    def test_ties_are_kept_or_dropped_together(self):
        conf = [0.9, 0.9, 0.9, 0.5]
        correct = [True, True, False, True]
        self.assertIsNone(metrics.coverage_at_accuracy(conf, correct, 0.95))
        self.assertEqual(metrics.coverage_at_accuracy(conf, correct, 0.75)["answered"], 4)

    def test_none_when_unreachable(self):
        self.assertIsNone(metrics.coverage_at_accuracy([0.9, 0.8], [False, False], 0.5))


class McNemarTests(unittest.TestCase):
    def test_no_disagreement(self):
        self.assertEqual(metrics.mcnemar_exact(0, 0), 1.0)

    def test_lopsided_disagreement(self):
        self.assertAlmostEqual(metrics.mcnemar_exact(0, 10), 2 / 1024)

    def test_symmetric(self):
        self.assertEqual(metrics.mcnemar_exact(3, 9), metrics.mcnemar_exact(9, 3))


class CompareTests(unittest.TestCase):
    def test_counts_only_the_disagreements(self):
        from layatasks.compare import compare

        a = [{"gold": "x", "pred": "x"}, {"gold": "y", "pred": "x"}, {"gold": "z", "pred": "z"}]
        b = [{"gold": "x", "pred": "x"}, {"gold": "y", "pred": "y"}, {"gold": "z", "pred": "y"}]
        r = compare(a, b)
        self.assertEqual((r["a_right"], r["b_right"], r["only_a"], r["only_b"]), (2, 2, 1, 1))

    def test_refuses_runs_on_different_tests(self):
        from layatasks.compare import compare

        with self.assertRaises(ValueError):
            compare([{"gold": "x", "pred": "x"}], [{"gold": "y", "pred": "y"}])


class F1Tests(unittest.TestCase):
    def test_micro_f1_is_accuracy_for_one_label_each(self):
        self.assertAlmostEqual(metrics.micro_f1(["a", "b", "b", "c"], ["a", "b", "c", "c"]), 0.75)

    def test_macro_f1_by_hand(self):
        # a: tp 1, fp 0, fn 0 -> 1.  b: tp 1, fp 0, fn 1 -> 2/3.  c: tp 1, fp 1, fn 0 -> 2/3.
        golds = ["a", "b", "b", "c"]
        preds = ["a", "b", "c", "c"]
        self.assertAlmostEqual(metrics.macro_f1(golds, preds), (1 + 2 / 3 + 2 / 3) / 3)

    def test_a_label_only_ever_predicted_counts_as_zero(self):
        # a: tp 1, fn 1 -> 2/3.  z: never gold, predicted once and wrong -> 0.
        self.assertAlmostEqual(metrics.macro_f1(["a", "a"], ["a", "z"]), (2 / 3 + 0) / 2)

    def test_refuses_mismatched_lengths(self):
        with self.assertRaises(ValueError):
            metrics.macro_f1(["a"], ["a", "b"])
        with self.assertRaises(ValueError):
            metrics.micro_f1([], [])


if __name__ == "__main__":
    unittest.main()