laya-clinc150 / eval.json
VibeHash's picture
Release laya-clinc150, seed 0
0e9d0a1 verified
Raw History Blame Contribute Delete
4.22 kB
{
"task": "clinc150",
"name": "seed0",
"test_sha256": "470ec9495a074a3c8ab3a738ee99497f334f2a2e5caa9d214673bbeba9c6316c",
"max_len": 512,
"head_max_len": 512,
"batched": {
"batch_size": 32,
"seconds": 70.32565169999725,
"ms_per_text": 12.786482127272228
},
"n": 5500,
"right": 4993,
"accuracy": 0.9078181818181819,
"ci95": [
0.8998857114771881,
0.9151813709531285
],
"mean_confidence": 0.9547379454545454,
"ece": 0.049853000000000224,
"gate": {
"threshold": 0.9687,
"answered": 4141,
"coverage": 0.7529090909090909,
"accuracy": 0.966191741125332,
"errors": 140
},
"curve": [
{
"target": 0.95,
"threshold": 0.9506,
"answered": 4844,
"coverage": 0.8807272727272727,
"accuracy": 0.9500412881915772
},
{
"target": 0.98,
"threshold": 0.9795,
"answered": 2001,
"coverage": 0.3638181818181818,
"accuracy": 0.9800099950024987
},
{
"target": 0.99,
"threshold": 0.9921,
"answered": 13,
"coverage": 0.0023636363636363638,
"accuracy": 1.0
}
],
"weakest_labels": [
[
"none of these",
{
"accuracy": 0.716,
"n": 1000
}
],
[
"transactions",
{
"accuracy": 0.7666666666666667,
"n": 30
}
],
[
"calendar",
{
"accuracy": 0.7666666666666667,
"n": 30
}
],
[
"yes",
{
"accuracy": 0.8,
"n": 30
}
],
[
"restaurant reviews",
{
"accuracy": 0.8,
"n": 30
}
],
[
"how busy",
{
"accuracy": 0.8,
"n": 30
}
],
[
"pin change",
{
"accuracy": 0.8,
"n": 30
}
],
[
"report fraud",
{
"accuracy": 0.8,
"n": 30
}
],
[
"replacement card duration",
{
"accuracy": 0.8333333333333334,
"n": 30
}
],
[
"spending history",
{
"accuracy": 0.8333333333333334,
"n": 30
}
]
],
"top_confusions": [
{
"gold": "none of these",
"pred": "directions",
"count": 16
},
{
"gold": "none of these",
"pred": "vaccines",
"count": 13
},
{
"gold": "none of these",
"pred": "order",
"count": 11
},
{
"gold": "none of these",
"pred": "todo list update",
"count": 9
},
{
"gold": "none of these",
"pred": "text",
"count": 8
},
{
"gold": "none of these",
"pred": "smart home",
"count": 8
},
{
"gold": "none of these",
"pred": "oil change how",
"count": 7
},
{
"gold": "none of these",
"pred": "mpg",
"count": 7
},
{
"gold": "transactions",
"pred": "spending history",
"count": 6
},
{
"gold": "none of these",
"pred": "car rental",
"count": 6
}
],
"by_seen": {
"seen_in_train": {
"n": 2,
"right": 1,
"accuracy": 0.5
},
"unseen": {
"n": 5498,
"right": 4992,
"accuracy": 0.9079665332848309
}
},
"by_truncation": {
"cut_short": {
"n": 0,
"right": 0,
"accuracy": null
},
"whole": {
"n": 5500,
"right": 4993,
"accuracy": 0.9078181818181819
}
},
"task_scores": {
"in_scope": {
"n": 4500,
"right": 4277,
"accuracy": 0.9504444444444444,
"ci95": [
0.9437104113763727,
0.9564100828875434
],
"answered_none_of_these": 44
},
"out_of_scope": {
"n": 1000,
"recall": 0.716,
"caught": 716,
"ci95": [
0.6872658654245983,
0.7430809749096418
]
},
"with_gate": {
"threshold": 0.9687,
"in_scope_acted": 3795,
"in_scope_coverage": 0.8433333333333334,
"in_scope_acted_accuracy": 0.9857707509881423,
"out_of_scope_let_through": 83,
"out_of_scope_recall": 0.917
}
},
"single": {
"n": 200,
"median_ms": 23.872100005974062,
"p95_ms": 26.412899998831563
}
}