{ "task": "ledgar", "name": "ce-seed0", "test_sha256": "8c3aa107260838655efebe9f319a60258b0a6e2ee3ec156314bd91b544c5a2e0", "rows": "test", "max_len": 768, "head_max_len": 512, "batched": { "batch_size": 32, "seconds": 195.57866689999355, "ms_per_text": 19.557866689999354 }, "n": 10000, "right": 8700, "accuracy": 0.87, "ci95": [ 0.8632662286896442, 0.876449612516055 ], "mean_confidence": 0.9072139800000001, "ece": 0.041833199999995394, "gate": { "threshold": 0.9806, "answered": 3639, "coverage": 0.3639, "accuracy": 0.9843363561417972, "errors": 57 }, "curve": [ { "target": 0.95, "threshold": 0.908, "answered": 7603, "coverage": 0.7603, "accuracy": 0.9500197290543206 }, { "target": 0.98, "threshold": 0.9763, "answered": 4417, "coverage": 0.4417, "accuracy": 0.9800769753226172 }, { "target": 0.99, "threshold": 0.9864, "answered": 2167, "coverage": 0.2167, "accuracy": 0.9903091832025842 } ], "weakest_labels": [ [ "applicable laws", { "accuracy": 0.18867924528301888, "n": 53 } ], [ "assigns", { "accuracy": 0.25, "n": 4 } ], [ "venues", { "accuracy": 0.4, "n": 20 } ], [ "indemnity", { "accuracy": 0.4827586206896552, "n": 29 } ], [ "miscellaneous", { "accuracy": 0.5316455696202531, "n": 79 } ], [ "agreements", { "accuracy": 0.5416666666666666, "n": 48 } ], [ "general", { "accuracy": 0.541871921182266, "n": 203 } ], [ "submission to jurisdiction", { "accuracy": 0.5517241379310345, "n": 29 } ], [ "approvals", { "accuracy": 0.5769230769230769, "n": 26 } ], [ "enforceability", { "accuracy": 0.5777777777777777, "n": 45 } ] ], "top_confusions": [ { "gold": "definitions", "pred": "defined terms", "count": 37 }, { "gold": "applicable laws", "pred": "governing laws", "count": 35 }, { "gold": "tax withholdings", "pred": "withholdings", "count": 20 }, { "gold": "no waivers", "pred": "waivers", "count": 20 }, { "gold": "waivers", "pred": "no waivers", "count": 19 }, { "gold": "integration", "pred": "entire agreements", "count": 16 }, { "gold": "indemnity", "pred": "indemnifications", "count": 15 }, { "gold": "modifications", "pred": "amendments", "count": 15 }, { "gold": "interpretations", "pred": "construction", "count": 13 }, { "gold": "successors", "pred": "binding effects", "count": 13 } ], "by_seen": { "seen_in_train": { "n": 282, "right": 276, "accuracy": 0.9787234042553191 }, "unseen": { "n": 9718, "right": 8424, "accuracy": 0.8668450298415312 } }, "by_truncation": { "cut_short": { "n": 253, "right": 223, "accuracy": 0.8814229249011858 }, "whole": { "n": 9747, "right": 8477, "accuracy": 0.8697034985123627 } }, "task_scores": { "micro_f1": 0.87, "macro_f1": 0.8141006672917114 }, "single": { "n": 200, "median_ms": 46.5544999751728, "p95_ms": 54.286799975670874 } }