Text Classification
Laya
Safetensors
English
clinc150
intent-classification
out-of-scope-detection
calibrated
selective-prediction
Eval Results (legacy)
Instructions to use VibeHash/laya-clinc150 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Laya
How to use VibeHash/laya-clinc150 with Laya:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
File size: 2,545 Bytes
0e9d0a1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | """CLINC150's option texts and its two benchmark numbers, worked out by hand on made-up predictions."""
from __future__ import annotations
import unittest
from layatasks.tasks import clinc150
from layatasks.tasks.clinc150 import OOS_OPTION, TASK, extra_scores
def p(gold, pred, confidence):
return {"gold": gold, "pred": pred, "confidence": confidence}
class OptionTests(unittest.TestCase):
def test_out_of_scope_reads_as_words(self):
self.assertEqual(clinc150.option_text("oos"), "none of these")
self.assertEqual(clinc150.option_text("what_is_your_name"), "what is your name")
self.assertEqual(TASK.question(["oos", "w2", "pto_balance"])["criteria"], ["none of these", "w2", "pto balance"])
def test_settings(self):
self.assertEqual((TASK.max_len, TASK.head_max_len, TASK.epochs, TASK.target_error), (512, 512, 4, 0.01))
class ScoreTests(unittest.TestCase):
PREDS = [
p("timer", "timer", 0.99), # in scope, right, sure
p("timer", "alarm", 0.95), # in scope, wrong, sure
p("alarm", OOS_OPTION, 0.80), # in scope, wrongly called out of scope
p("alarm", "alarm", 0.50), # in scope, right, unsure
p(OOS_OPTION, OOS_OPTION, 0.90), # out of scope, caught
p(OOS_OPTION, "timer", 0.97), # out of scope, let through by a sure answer
p(OOS_OPTION, "alarm", 0.40), # out of scope, named an intent but unsure: stopped by the threshold
]
def test_benchmark_numbers_without_the_threshold(self):
s = extra_scores(self.PREDS, None)
self.assertEqual((s["in_scope"]["n"], s["in_scope"]["right"]), (4, 2))
self.assertEqual(s["in_scope"]["answered_none_of_these"], 1)
self.assertEqual((s["out_of_scope"]["n"], s["out_of_scope"]["caught"]), (3, 1))
self.assertAlmostEqual(s["out_of_scope"]["recall"], 1 / 3)
self.assertNotIn("with_gate", s)
def test_threshold_stops_unsure_answers(self):
g = extra_scores(self.PREDS, 0.9)["with_gate"]
self.assertEqual(g["in_scope_acted"], 2) # timer 0.99 and timer->alarm 0.95
self.assertAlmostEqual(g["in_scope_acted_accuracy"], 0.5)
self.assertEqual(g["out_of_scope_let_through"], 1) # only the 0.97 answer
self.assertAlmostEqual(g["out_of_scope_recall"], 2 / 3)
def test_needs_both_kinds_of_query(self):
with self.assertRaises(ValueError):
extra_scores([p("timer", "timer", 0.9)], None)
if __name__ == "__main__":
unittest.main()
|