Text Classification
Laya
Safetensors
English
clinc150
intent-classification
out-of-scope-detection
calibrated
selective-prediction
Eval Results (legacy)
Instructions to use VibeHash/laya-clinc150 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Laya
How to use VibeHash/laya-clinc150 with Laya:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
Download code/tests/test_task_clinc150.py from VibeHash/laya-clinc150: direct link, hf CLI and curl.
- Browser
- Download file 2.55 kB
-
https://huggingface.co/VibeHash/laya-clinc150/resolve/main/code/tests/test_task_clinc150.py
- Command line
-
hf download hf://VibeHash/laya-clinc150/code/tests/test_task_clinc150.py
-
curl -L -o test_task_clinc150.py https://huggingface.co/VibeHash/laya-clinc150/resolve/main/code/tests/test_task_clinc150.py
2.55 kB
| """CLINC150's option texts and its two benchmark numbers, worked out by hand on made-up predictions.""" | |
| from __future__ import annotations | |
| import unittest | |
| from layatasks.tasks import clinc150 | |
| from layatasks.tasks.clinc150 import OOS_OPTION, TASK, extra_scores | |
| def p(gold, pred, confidence): | |
| return {"gold": gold, "pred": pred, "confidence": confidence} | |
| class OptionTests(unittest.TestCase): | |
| def test_out_of_scope_reads_as_words(self): | |
| self.assertEqual(clinc150.option_text("oos"), "none of these") | |
| self.assertEqual(clinc150.option_text("what_is_your_name"), "what is your name") | |
| self.assertEqual(TASK.question(["oos", "w2", "pto_balance"])["criteria"], ["none of these", "w2", "pto balance"]) | |
| def test_settings(self): | |
| self.assertEqual((TASK.max_len, TASK.head_max_len, TASK.epochs, TASK.target_error), (512, 512, 4, 0.01)) | |
| class ScoreTests(unittest.TestCase): | |
| PREDS = [ | |
| p("timer", "timer", 0.99), # in scope, right, sure | |
| p("timer", "alarm", 0.95), # in scope, wrong, sure | |
| p("alarm", OOS_OPTION, 0.80), # in scope, wrongly called out of scope | |
| p("alarm", "alarm", 0.50), # in scope, right, unsure | |
| p(OOS_OPTION, OOS_OPTION, 0.90), # out of scope, caught | |
| p(OOS_OPTION, "timer", 0.97), # out of scope, let through by a sure answer | |
| p(OOS_OPTION, "alarm", 0.40), # out of scope, named an intent but unsure: stopped by the threshold | |
| ] | |
| def test_benchmark_numbers_without_the_threshold(self): | |
| s = extra_scores(self.PREDS, None) | |
| self.assertEqual((s["in_scope"]["n"], s["in_scope"]["right"]), (4, 2)) | |
| self.assertEqual(s["in_scope"]["answered_none_of_these"], 1) | |
| self.assertEqual((s["out_of_scope"]["n"], s["out_of_scope"]["caught"]), (3, 1)) | |
| self.assertAlmostEqual(s["out_of_scope"]["recall"], 1 / 3) | |
| self.assertNotIn("with_gate", s) | |
| def test_threshold_stops_unsure_answers(self): | |
| g = extra_scores(self.PREDS, 0.9)["with_gate"] | |
| self.assertEqual(g["in_scope_acted"], 2) # timer 0.99 and timer->alarm 0.95 | |
| self.assertAlmostEqual(g["in_scope_acted_accuracy"], 0.5) | |
| self.assertEqual(g["out_of_scope_let_through"], 1) # only the 0.97 answer | |
| self.assertAlmostEqual(g["out_of_scope_recall"], 2 / 3) | |
| def test_needs_both_kinds_of_query(self): | |
| with self.assertRaises(ValueError): | |
| extra_scores([p("timer", "timer", 0.9)], None) | |
| if __name__ == "__main__": | |
| unittest.main() | |