"""CLINC150's option texts and its two benchmark numbers, worked out by hand on made-up predictions.""" from __future__ import annotations import unittest from layatasks.tasks import clinc150 from layatasks.tasks.clinc150 import OOS_OPTION, TASK, extra_scores def p(gold, pred, confidence): return {"gold": gold, "pred": pred, "confidence": confidence} class OptionTests(unittest.TestCase): def test_out_of_scope_reads_as_words(self): self.assertEqual(clinc150.option_text("oos"), "none of these") self.assertEqual(clinc150.option_text("what_is_your_name"), "what is your name") self.assertEqual(TASK.question(["oos", "w2", "pto_balance"])["criteria"], ["none of these", "w2", "pto balance"]) def test_settings(self): self.assertEqual((TASK.max_len, TASK.head_max_len, TASK.epochs, TASK.target_error), (512, 512, 4, 0.01)) class ScoreTests(unittest.TestCase): PREDS = [ p("timer", "timer", 0.99), # in scope, right, sure p("timer", "alarm", 0.95), # in scope, wrong, sure p("alarm", OOS_OPTION, 0.80), # in scope, wrongly called out of scope p("alarm", "alarm", 0.50), # in scope, right, unsure p(OOS_OPTION, OOS_OPTION, 0.90), # out of scope, caught p(OOS_OPTION, "timer", 0.97), # out of scope, let through by a sure answer p(OOS_OPTION, "alarm", 0.40), # out of scope, named an intent but unsure: stopped by the threshold ] def test_benchmark_numbers_without_the_threshold(self): s = extra_scores(self.PREDS, None) self.assertEqual((s["in_scope"]["n"], s["in_scope"]["right"]), (4, 2)) self.assertEqual(s["in_scope"]["answered_none_of_these"], 1) self.assertEqual((s["out_of_scope"]["n"], s["out_of_scope"]["caught"]), (3, 1)) self.assertAlmostEqual(s["out_of_scope"]["recall"], 1 / 3) self.assertNotIn("with_gate", s) def test_threshold_stops_unsure_answers(self): g = extra_scores(self.PREDS, 0.9)["with_gate"] self.assertEqual(g["in_scope_acted"], 2) # timer 0.99 and timer->alarm 0.95 self.assertAlmostEqual(g["in_scope_acted_accuracy"], 0.5) self.assertEqual(g["out_of_scope_let_through"], 1) # only the 0.97 answer self.assertAlmostEqual(g["out_of_scope_recall"], 2 / 3) def test_needs_both_kinds_of_query(self): with self.assertRaises(ValueError): extra_scores([p("timer", "timer", 0.9)], None) if __name__ == "__main__": unittest.main()