File size: 2,545 Bytes
0e9d0a1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
"""CLINC150's option texts and its two benchmark numbers, worked out by hand on made-up predictions."""

from __future__ import annotations

import unittest

from layatasks.tasks import clinc150
from layatasks.tasks.clinc150 import OOS_OPTION, TASK, extra_scores


def p(gold, pred, confidence):
    return {"gold": gold, "pred": pred, "confidence": confidence}


class OptionTests(unittest.TestCase):
    def test_out_of_scope_reads_as_words(self):
        self.assertEqual(clinc150.option_text("oos"), "none of these")
        self.assertEqual(clinc150.option_text("what_is_your_name"), "what is your name")
        self.assertEqual(TASK.question(["oos", "w2", "pto_balance"])["criteria"], ["none of these", "w2", "pto balance"])

    def test_settings(self):
        self.assertEqual((TASK.max_len, TASK.head_max_len, TASK.epochs, TASK.target_error), (512, 512, 4, 0.01))


class ScoreTests(unittest.TestCase):
    PREDS = [
        p("timer", "timer", 0.99),        # in scope, right, sure
        p("timer", "alarm", 0.95),        # in scope, wrong, sure
        p("alarm", OOS_OPTION, 0.80),     # in scope, wrongly called out of scope
        p("alarm", "alarm", 0.50),        # in scope, right, unsure
        p(OOS_OPTION, OOS_OPTION, 0.90),  # out of scope, caught
        p(OOS_OPTION, "timer", 0.97),     # out of scope, let through by a sure answer
        p(OOS_OPTION, "alarm", 0.40),     # out of scope, named an intent but unsure: stopped by the threshold
    ]

    def test_benchmark_numbers_without_the_threshold(self):
        s = extra_scores(self.PREDS, None)
        self.assertEqual((s["in_scope"]["n"], s["in_scope"]["right"]), (4, 2))
        self.assertEqual(s["in_scope"]["answered_none_of_these"], 1)
        self.assertEqual((s["out_of_scope"]["n"], s["out_of_scope"]["caught"]), (3, 1))
        self.assertAlmostEqual(s["out_of_scope"]["recall"], 1 / 3)
        self.assertNotIn("with_gate", s)

    def test_threshold_stops_unsure_answers(self):
        g = extra_scores(self.PREDS, 0.9)["with_gate"]
        self.assertEqual(g["in_scope_acted"], 2)              # timer 0.99 and timer->alarm 0.95
        self.assertAlmostEqual(g["in_scope_acted_accuracy"], 0.5)
        self.assertEqual(g["out_of_scope_let_through"], 1)    # only the 0.97 answer
        self.assertAlmostEqual(g["out_of_scope_recall"], 2 / 3)

    def test_needs_both_kinds_of_query(self):
        with self.assertRaises(ValueError):
            extra_scores([p("timer", "timer", 0.9)], None)


if __name__ == "__main__":
    unittest.main()