laya-clinc150 / code /tests /test_task_clinc150.py
VibeHash's picture
Release laya-clinc150, seed 0
0e9d0a1 verified
Raw History Blame Contribute Delete
2.55 kB
"""CLINC150's option texts and its two benchmark numbers, worked out by hand on made-up predictions."""
from __future__ import annotations
import unittest
from layatasks.tasks import clinc150
from layatasks.tasks.clinc150 import OOS_OPTION, TASK, extra_scores
def p(gold, pred, confidence):
return {"gold": gold, "pred": pred, "confidence": confidence}
class OptionTests(unittest.TestCase):
def test_out_of_scope_reads_as_words(self):
self.assertEqual(clinc150.option_text("oos"), "none of these")
self.assertEqual(clinc150.option_text("what_is_your_name"), "what is your name")
self.assertEqual(TASK.question(["oos", "w2", "pto_balance"])["criteria"], ["none of these", "w2", "pto balance"])
def test_settings(self):
self.assertEqual((TASK.max_len, TASK.head_max_len, TASK.epochs, TASK.target_error), (512, 512, 4, 0.01))
class ScoreTests(unittest.TestCase):
PREDS = [
p("timer", "timer", 0.99), # in scope, right, sure
p("timer", "alarm", 0.95), # in scope, wrong, sure
p("alarm", OOS_OPTION, 0.80), # in scope, wrongly called out of scope
p("alarm", "alarm", 0.50), # in scope, right, unsure
p(OOS_OPTION, OOS_OPTION, 0.90), # out of scope, caught
p(OOS_OPTION, "timer", 0.97), # out of scope, let through by a sure answer
p(OOS_OPTION, "alarm", 0.40), # out of scope, named an intent but unsure: stopped by the threshold
]
def test_benchmark_numbers_without_the_threshold(self):
s = extra_scores(self.PREDS, None)
self.assertEqual((s["in_scope"]["n"], s["in_scope"]["right"]), (4, 2))
self.assertEqual(s["in_scope"]["answered_none_of_these"], 1)
self.assertEqual((s["out_of_scope"]["n"], s["out_of_scope"]["caught"]), (3, 1))
self.assertAlmostEqual(s["out_of_scope"]["recall"], 1 / 3)
self.assertNotIn("with_gate", s)
def test_threshold_stops_unsure_answers(self):
g = extra_scores(self.PREDS, 0.9)["with_gate"]
self.assertEqual(g["in_scope_acted"], 2) # timer 0.99 and timer->alarm 0.95
self.assertAlmostEqual(g["in_scope_acted_accuracy"], 0.5)
self.assertEqual(g["out_of_scope_let_through"], 1) # only the 0.97 answer
self.assertAlmostEqual(g["out_of_scope_recall"], 2 / 3)
def test_needs_both_kinds_of_query(self):
with self.assertRaises(ValueError):
extra_scores([p("timer", "timer", 0.9)], None)
if __name__ == "__main__":
unittest.main()