Text Classification
Laya
Safetensors
English
clinc150
intent-classification
out-of-scope-detection
calibrated
selective-prediction
Eval Results (legacy)
Instructions to use VibeHash/laya-clinc150 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Laya
How to use VibeHash/laya-clinc150 with Laya:
# No code snippets available yet for this library. # To use this model, check the repository files and the library's documentation. # Want to help? PRs adding snippets are welcome at: # https://github.com/huggingface/huggingface.js
- Notebooks
- Google Colab
- Kaggle
Download eval.json from VibeHash/laya-clinc150: direct link, hf CLI and curl.
- Browser
- Download file 4.22 kB
-
https://huggingface.co/VibeHash/laya-clinc150/resolve/main/eval.json
- Command line
-
hf download hf://VibeHash/laya-clinc150/eval.json
-
curl -L -o eval.json https://huggingface.co/VibeHash/laya-clinc150/resolve/main/eval.json
4.22 kB
| { | |
| "task": "clinc150", | |
| "name": "seed0", | |
| "test_sha256": "470ec9495a074a3c8ab3a738ee99497f334f2a2e5caa9d214673bbeba9c6316c", | |
| "max_len": 512, | |
| "head_max_len": 512, | |
| "batched": { | |
| "batch_size": 32, | |
| "seconds": 70.32565169999725, | |
| "ms_per_text": 12.786482127272228 | |
| }, | |
| "n": 5500, | |
| "right": 4993, | |
| "accuracy": 0.9078181818181819, | |
| "ci95": [ | |
| 0.8998857114771881, | |
| 0.9151813709531285 | |
| ], | |
| "mean_confidence": 0.9547379454545454, | |
| "ece": 0.049853000000000224, | |
| "gate": { | |
| "threshold": 0.9687, | |
| "answered": 4141, | |
| "coverage": 0.7529090909090909, | |
| "accuracy": 0.966191741125332, | |
| "errors": 140 | |
| }, | |
| "curve": [ | |
| { | |
| "target": 0.95, | |
| "threshold": 0.9506, | |
| "answered": 4844, | |
| "coverage": 0.8807272727272727, | |
| "accuracy": 0.9500412881915772 | |
| }, | |
| { | |
| "target": 0.98, | |
| "threshold": 0.9795, | |
| "answered": 2001, | |
| "coverage": 0.3638181818181818, | |
| "accuracy": 0.9800099950024987 | |
| }, | |
| { | |
| "target": 0.99, | |
| "threshold": 0.9921, | |
| "answered": 13, | |
| "coverage": 0.0023636363636363638, | |
| "accuracy": 1.0 | |
| } | |
| ], | |
| "weakest_labels": [ | |
| [ | |
| "none of these", | |
| { | |
| "accuracy": 0.716, | |
| "n": 1000 | |
| } | |
| ], | |
| [ | |
| "transactions", | |
| { | |
| "accuracy": 0.7666666666666667, | |
| "n": 30 | |
| } | |
| ], | |
| [ | |
| "calendar", | |
| { | |
| "accuracy": 0.7666666666666667, | |
| "n": 30 | |
| } | |
| ], | |
| [ | |
| "yes", | |
| { | |
| "accuracy": 0.8, | |
| "n": 30 | |
| } | |
| ], | |
| [ | |
| "restaurant reviews", | |
| { | |
| "accuracy": 0.8, | |
| "n": 30 | |
| } | |
| ], | |
| [ | |
| "how busy", | |
| { | |
| "accuracy": 0.8, | |
| "n": 30 | |
| } | |
| ], | |
| [ | |
| "pin change", | |
| { | |
| "accuracy": 0.8, | |
| "n": 30 | |
| } | |
| ], | |
| [ | |
| "report fraud", | |
| { | |
| "accuracy": 0.8, | |
| "n": 30 | |
| } | |
| ], | |
| [ | |
| "replacement card duration", | |
| { | |
| "accuracy": 0.8333333333333334, | |
| "n": 30 | |
| } | |
| ], | |
| [ | |
| "spending history", | |
| { | |
| "accuracy": 0.8333333333333334, | |
| "n": 30 | |
| } | |
| ] | |
| ], | |
| "top_confusions": [ | |
| { | |
| "gold": "none of these", | |
| "pred": "directions", | |
| "count": 16 | |
| }, | |
| { | |
| "gold": "none of these", | |
| "pred": "vaccines", | |
| "count": 13 | |
| }, | |
| { | |
| "gold": "none of these", | |
| "pred": "order", | |
| "count": 11 | |
| }, | |
| { | |
| "gold": "none of these", | |
| "pred": "todo list update", | |
| "count": 9 | |
| }, | |
| { | |
| "gold": "none of these", | |
| "pred": "text", | |
| "count": 8 | |
| }, | |
| { | |
| "gold": "none of these", | |
| "pred": "smart home", | |
| "count": 8 | |
| }, | |
| { | |
| "gold": "none of these", | |
| "pred": "oil change how", | |
| "count": 7 | |
| }, | |
| { | |
| "gold": "none of these", | |
| "pred": "mpg", | |
| "count": 7 | |
| }, | |
| { | |
| "gold": "transactions", | |
| "pred": "spending history", | |
| "count": 6 | |
| }, | |
| { | |
| "gold": "none of these", | |
| "pred": "car rental", | |
| "count": 6 | |
| } | |
| ], | |
| "by_seen": { | |
| "seen_in_train": { | |
| "n": 2, | |
| "right": 1, | |
| "accuracy": 0.5 | |
| }, | |
| "unseen": { | |
| "n": 5498, | |
| "right": 4992, | |
| "accuracy": 0.9079665332848309 | |
| } | |
| }, | |
| "by_truncation": { | |
| "cut_short": { | |
| "n": 0, | |
| "right": 0, | |
| "accuracy": null | |
| }, | |
| "whole": { | |
| "n": 5500, | |
| "right": 4993, | |
| "accuracy": 0.9078181818181819 | |
| } | |
| }, | |
| "task_scores": { | |
| "in_scope": { | |
| "n": 4500, | |
| "right": 4277, | |
| "accuracy": 0.9504444444444444, | |
| "ci95": [ | |
| 0.9437104113763727, | |
| 0.9564100828875434 | |
| ], | |
| "answered_none_of_these": 44 | |
| }, | |
| "out_of_scope": { | |
| "n": 1000, | |
| "recall": 0.716, | |
| "caught": 716, | |
| "ci95": [ | |
| 0.6872658654245983, | |
| 0.7430809749096418 | |
| ] | |
| }, | |
| "with_gate": { | |
| "threshold": 0.9687, | |
| "in_scope_acted": 3795, | |
| "in_scope_coverage": 0.8433333333333334, | |
| "in_scope_acted_accuracy": 0.9857707509881423, | |
| "out_of_scope_let_through": 83, | |
| "out_of_scope_recall": 0.917 | |
| } | |
| }, | |
| "single": { | |
| "n": 200, | |
| "median_ms": 23.872100005974062, | |
| "p95_ms": 26.412899998831563 | |
| } | |
| } | |