{ "task": "clinc150", "name": "seed0", "test_sha256": "470ec9495a074a3c8ab3a738ee99497f334f2a2e5caa9d214673bbeba9c6316c", "max_len": 512, "head_max_len": 512, "batched": { "batch_size": 32, "seconds": 70.32565169999725, "ms_per_text": 12.786482127272228 }, "n": 5500, "right": 4993, "accuracy": 0.9078181818181819, "ci95": [ 0.8998857114771881, 0.9151813709531285 ], "mean_confidence": 0.9547379454545454, "ece": 0.049853000000000224, "gate": { "threshold": 0.9687, "answered": 4141, "coverage": 0.7529090909090909, "accuracy": 0.966191741125332, "errors": 140 }, "curve": [ { "target": 0.95, "threshold": 0.9506, "answered": 4844, "coverage": 0.8807272727272727, "accuracy": 0.9500412881915772 }, { "target": 0.98, "threshold": 0.9795, "answered": 2001, "coverage": 0.3638181818181818, "accuracy": 0.9800099950024987 }, { "target": 0.99, "threshold": 0.9921, "answered": 13, "coverage": 0.0023636363636363638, "accuracy": 1.0 } ], "weakest_labels": [ [ "none of these", { "accuracy": 0.716, "n": 1000 } ], [ "transactions", { "accuracy": 0.7666666666666667, "n": 30 } ], [ "calendar", { "accuracy": 0.7666666666666667, "n": 30 } ], [ "yes", { "accuracy": 0.8, "n": 30 } ], [ "restaurant reviews", { "accuracy": 0.8, "n": 30 } ], [ "how busy", { "accuracy": 0.8, "n": 30 } ], [ "pin change", { "accuracy": 0.8, "n": 30 } ], [ "report fraud", { "accuracy": 0.8, "n": 30 } ], [ "replacement card duration", { "accuracy": 0.8333333333333334, "n": 30 } ], [ "spending history", { "accuracy": 0.8333333333333334, "n": 30 } ] ], "top_confusions": [ { "gold": "none of these", "pred": "directions", "count": 16 }, { "gold": "none of these", "pred": "vaccines", "count": 13 }, { "gold": "none of these", "pred": "order", "count": 11 }, { "gold": "none of these", "pred": "todo list update", "count": 9 }, { "gold": "none of these", "pred": "text", "count": 8 }, { "gold": "none of these", "pred": "smart home", "count": 8 }, { "gold": "none of these", "pred": "oil change how", "count": 7 }, { "gold": "none of these", "pred": "mpg", "count": 7 }, { "gold": "transactions", "pred": "spending history", "count": 6 }, { "gold": "none of these", "pred": "car rental", "count": 6 } ], "by_seen": { "seen_in_train": { "n": 2, "right": 1, "accuracy": 0.5 }, "unseen": { "n": 5498, "right": 4992, "accuracy": 0.9079665332848309 } }, "by_truncation": { "cut_short": { "n": 0, "right": 0, "accuracy": null }, "whole": { "n": 5500, "right": 4993, "accuracy": 0.9078181818181819 } }, "task_scores": { "in_scope": { "n": 4500, "right": 4277, "accuracy": 0.9504444444444444, "ci95": [ 0.9437104113763727, 0.9564100828875434 ], "answered_none_of_these": 44 }, "out_of_scope": { "n": 1000, "recall": 0.716, "caught": 716, "ci95": [ 0.6872658654245983, 0.7430809749096418 ] }, "with_gate": { "threshold": 0.9687, "in_scope_acted": 3795, "in_scope_coverage": 0.8433333333333334, "in_scope_acted_accuracy": 0.9857707509881423, "out_of_scope_let_through": 83, "out_of_scope_recall": 0.917 } }, "single": { "n": 200, "median_ms": 23.872100005974062, "p95_ms": 26.412899998831563 } }