laya-ledgar / eval.json
VibeHash's picture
Release laya-ledgar, run ce-seed0
779ff0b verified
Raw History Blame Contribute Delete
3.72 kB
{
"task": "ledgar",
"name": "ce-seed0",
"test_sha256": "8c3aa107260838655efebe9f319a60258b0a6e2ee3ec156314bd91b544c5a2e0",
"rows": "test",
"max_len": 768,
"head_max_len": 512,
"batched": {
"batch_size": 32,
"seconds": 195.57866689999355,
"ms_per_text": 19.557866689999354
},
"n": 10000,
"right": 8700,
"accuracy": 0.87,
"ci95": [
0.8632662286896442,
0.876449612516055
],
"mean_confidence": 0.9072139800000001,
"ece": 0.041833199999995394,
"gate": {
"threshold": 0.9806,
"answered": 3639,
"coverage": 0.3639,
"accuracy": 0.9843363561417972,
"errors": 57
},
"curve": [
{
"target": 0.95,
"threshold": 0.908,
"answered": 7603,
"coverage": 0.7603,
"accuracy": 0.9500197290543206
},
{
"target": 0.98,
"threshold": 0.9763,
"answered": 4417,
"coverage": 0.4417,
"accuracy": 0.9800769753226172
},
{
"target": 0.99,
"threshold": 0.9864,
"answered": 2167,
"coverage": 0.2167,
"accuracy": 0.9903091832025842
}
],
"weakest_labels": [
[
"applicable laws",
{
"accuracy": 0.18867924528301888,
"n": 53
}
],
[
"assigns",
{
"accuracy": 0.25,
"n": 4
}
],
[
"venues",
{
"accuracy": 0.4,
"n": 20
}
],
[
"indemnity",
{
"accuracy": 0.4827586206896552,
"n": 29
}
],
[
"miscellaneous",
{
"accuracy": 0.5316455696202531,
"n": 79
}
],
[
"agreements",
{
"accuracy": 0.5416666666666666,
"n": 48
}
],
[
"general",
{
"accuracy": 0.541871921182266,
"n": 203
}
],
[
"submission to jurisdiction",
{
"accuracy": 0.5517241379310345,
"n": 29
}
],
[
"approvals",
{
"accuracy": 0.5769230769230769,
"n": 26
}
],
[
"enforceability",
{
"accuracy": 0.5777777777777777,
"n": 45
}
]
],
"top_confusions": [
{
"gold": "definitions",
"pred": "defined terms",
"count": 37
},
{
"gold": "applicable laws",
"pred": "governing laws",
"count": 35
},
{
"gold": "tax withholdings",
"pred": "withholdings",
"count": 20
},
{
"gold": "no waivers",
"pred": "waivers",
"count": 20
},
{
"gold": "waivers",
"pred": "no waivers",
"count": 19
},
{
"gold": "integration",
"pred": "entire agreements",
"count": 16
},
{
"gold": "indemnity",
"pred": "indemnifications",
"count": 15
},
{
"gold": "modifications",
"pred": "amendments",
"count": 15
},
{
"gold": "interpretations",
"pred": "construction",
"count": 13
},
{
"gold": "successors",
"pred": "binding effects",
"count": 13
}
],
"by_seen": {
"seen_in_train": {
"n": 282,
"right": 276,
"accuracy": 0.9787234042553191
},
"unseen": {
"n": 9718,
"right": 8424,
"accuracy": 0.8668450298415312
}
},
"by_truncation": {
"cut_short": {
"n": 253,
"right": 223,
"accuracy": 0.8814229249011858
},
"whole": {
"n": 9747,
"right": 8477,
"accuracy": 0.8697034985123627
}
},
"task_scores": {
"micro_f1": 0.87,
"macro_f1": 0.8141006672917114
},
"single": {
"n": 200,
"median_ms": 46.5544999751728,
"p95_ms": 54.286799975670874
}
}