| { |
| "ood": { |
| "overall": { |
| "n": 4500, |
| "acc": 0.47555555555555556, |
| "nll": 1.268476909899631, |
| "ece": 0.1325593729014183, |
| "mae": 0.7127850154762857 |
| }, |
| "macro_acc": 0.49183333333333334, |
| "by_kind": { |
| "choice": { |
| "n": 2500, |
| "acc": 0.4276, |
| "nll": 1.4152444272854594, |
| "ece": 0.128002237490073 |
| }, |
| "noul": { |
| "n": 1000, |
| "acc": 0.589, |
| "nll": 1.0108116476597444, |
| "ece": 0.26637219234174647 |
| }, |
| "score": { |
| "n": 1000, |
| "acc": 0.482, |
| "nll": 1.1592233786749255, |
| "ece": 0.07211618516297272, |
| "mae": 0.7127850154762857 |
| } |
| }, |
| "by_task": { |
| "amazon_stars": { |
| "n": 1000, |
| "acc": 0.482, |
| "nll": 1.1592233786749255, |
| "ece": 0.07211618516297272, |
| "mae": 0.7127850154762857 |
| }, |
| "emotion": { |
| "n": 1000, |
| "acc": 0.551, |
| "nll": 1.2441813441969587, |
| "ece": 0.08441541454068684 |
| }, |
| "enron_spam": { |
| "n": 1000, |
| "acc": 0.589, |
| "nll": 1.0108116476597444, |
| "ece": 0.26637219234174647 |
| }, |
| "mmlu": { |
| "n": 1500, |
| "acc": 0.3453333333333333, |
| "nll": 1.5292864826777985, |
| "ece": 0.16617672448084847 |
| } |
| } |
| }, |
| "order_invariance_max_logit_delta": 0.000008940696716308594, |
| "val": { |
| "overall": { |
| "n": 8127, |
| "acc": 0.801894918173988, |
| "nll": 0.5053446978458112, |
| "ece": 0.007099048157460054, |
| "mae": 0.5636936249569889 |
| }, |
| "macro_acc": 0.7992572561996758, |
| "by_kind": { |
| "choice": { |
| "n": 4745, |
| "acc": 0.7869336143308746, |
| "nll": 0.5689453066911272, |
| "ece": 0.010992410314786963 |
| }, |
| "noul": { |
| "n": 2703, |
| "acc": 0.8953015168331484, |
| "nll": 0.25909313340379886, |
| "ece": 0.008562793730987436 |
| }, |
| "score": { |
| "n": 679, |
| "acc": 0.5346097201767305, |
| "nll": 1.0411813542754798, |
| "ece": 0.03380929975803491, |
| "mae": 0.5636936249569889 |
| } |
| }, |
| "by_task": { |
| "amazon_polarity": { |
| "n": 300, |
| "acc": 0.94, |
| "nll": 0.1780728226299769, |
| "ece": 0.03494007905033705 |
| }, |
| "arc": { |
| "n": 200, |
| "acc": 0.49, |
| "nll": 1.2722349869251168, |
| "ece": 0.12446486240223996 |
| }, |
| "arc_easy": { |
| "n": 200, |
| "acc": 0.66, |
| "nll": 0.8571748012168005, |
| "ece": 0.0790243305106568 |
| }, |
| "banking77": { |
| "n": 300, |
| "acc": 0.9666666666666667, |
| "nll": 0.09405613306326095, |
| "ece": 0.031746609200941224 |
| }, |
| "bitext_support": { |
| "n": 300, |
| "acc": 1, |
| "nll": 0.010971368929279203, |
| "ece": 0.010659385842467978 |
| }, |
| "boolq": { |
| "n": 300, |
| "acc": 0.8333333333333334, |
| "nll": 0.41208993970329466, |
| "ece": 0.04317514312264036 |
| }, |
| "civil_comments": { |
| "n": 156, |
| "acc": 0.7692307692307693, |
| "nll": 0.5426374327890833, |
| "ece": 0.08561364800510647, |
| "mae": 0.4794184904520537 |
| }, |
| "clinc150": { |
| "n": 300, |
| "acc": 0.9833333333333333, |
| "nll": 0.058032448735896636, |
| "ece": 0.019049882616036785 |
| }, |
| "cola": { |
| "n": 200, |
| "acc": 0.79, |
| "nll": 0.4192314168142291, |
| "ece": 0.08574232134607189 |
| }, |
| "commonsense_qa": { |
| "n": 300, |
| "acc": 0.65, |
| "nll": 0.8803430494105633, |
| "ece": 0.042060654356154394 |
| }, |
| "dbpedia14": { |
| "n": 300, |
| "acc": 0.7, |
| "nll": 1.0272166475279698, |
| "ece": 0.16287307424578817 |
| }, |
| "go_emotions": { |
| "n": 256, |
| "acc": 0.84375, |
| "nll": 0.4625858118782197, |
| "ece": 0.07856337055037782 |
| }, |
| "hellaswag": { |
| "n": 300, |
| "acc": 0.6866666666666666, |
| "nll": 0.8184342990797175, |
| "ece": 0.065393505705077 |
| }, |
| "massive_intent": { |
| "n": 300, |
| "acc": 0.97, |
| "nll": 0.11297520330888683, |
| "ece": 0.03747435978798287 |
| }, |
| "massive_scenario": { |
| "n": 300, |
| "acc": 0.9633333333333334, |
| "nll": 0.14092133711034824, |
| "ece": 0.040415555363053655 |
| }, |
| "mnli": { |
| "n": 300, |
| "acc": 0.8733333333333333, |
| "nll": 0.3591158164193244, |
| "ece": 0.06122319501006837 |
| }, |
| "mrpc": { |
| "n": 200, |
| "acc": 0.905, |
| "nll": 0.2340552890847502, |
| "ece": 0.0475103336919146 |
| }, |
| "newsgroups": { |
| "n": 296, |
| "acc": 0.8547297297297297, |
| "nll": 0.4397821279908795, |
| "ece": 0.05815265497795075 |
| }, |
| "openbookqa": { |
| "n": 250, |
| "acc": 0.596, |
| "nll": 0.9566964109317068, |
| "ece": 0.05584956464154654 |
| }, |
| "qasc": { |
| "n": 250, |
| "acc": 0.496, |
| "nll": 1.3698718341800415, |
| "ece": 0.05324690519921278 |
| }, |
| "qnli": { |
| "n": 200, |
| "acc": 0.905, |
| "nll": 0.22825732034313698, |
| "ece": 0.05859514918248248 |
| }, |
| "qqp": { |
| "n": 200, |
| "acc": 0.87, |
| "nll": 0.3305964280804749, |
| "ece": 0.05731508099827738 |
| }, |
| "rte": { |
| "n": 200, |
| "acc": 0.82, |
| "nll": 0.3908081576970799, |
| "ece": 0.03793475295601129 |
| }, |
| "sms_spam": { |
| "n": 300, |
| "acc": 1, |
| "nll": 0.009443218337208322, |
| "ece": 0.009181943972248042 |
| }, |
| "snli": { |
| "n": 295, |
| "acc": 0.8745762711864407, |
| "nll": 0.34642039268536257, |
| "ece": 0.0733830186710087 |
| }, |
| "sst2": { |
| "n": 300, |
| "acc": 0.95, |
| "nll": 0.17723819437605498, |
| "ece": 0.04834600996257101 |
| }, |
| "sst5": { |
| "n": 300, |
| "acc": 0.5466666666666666, |
| "nll": 1.034445504429844, |
| "ece": 0.06904864172392965, |
| "mae": 0.5531891951338506 |
| }, |
| "stsb": { |
| "n": 300, |
| "acc": 0.5133333333333333, |
| "nll": 1.0930674410680064, |
| "ece": 0.048921423078040656, |
| "mae": 0.5963905068664275 |
| }, |
| "toxic_conversations": { |
| "n": 124, |
| "acc": 0.9193548387096774, |
| "nll": 0.25773286368862236, |
| "ece": 0.05372284727427989 |
| }, |
| "wanli": { |
| "n": 300, |
| "acc": 0.7466666666666667, |
| "nll": 0.5828687631711043, |
| "ece": 0.04179919773420856 |
| }, |
| "winogrande": { |
| "n": 300, |
| "acc": 0.66, |
| "nll": 0.7154438457054739, |
| "ece": 0.11005624853642657 |
| } |
| } |
| }, |
| "val_uncalibrated": { |
| "overall": { |
| "n": 8127, |
| "acc": 0.801894918173988, |
| "nll": 0.6020770461865659, |
| "ece": 0.08041752615679162, |
| "mae": 0.559288462760455 |
| }, |
| "macro_acc": 0.7992572561996758, |
| "by_kind": { |
| "choice": { |
| "n": 4745, |
| "acc": 0.7869336143308746, |
| "nll": 0.713495906024384, |
| "ece": 0.10104285618425683 |
| }, |
| "noul": { |
| "n": 2703, |
| "acc": 0.8953015168331484, |
| "nll": 0.2901019780777808, |
| "ece": 0.04708815584016769 |
| }, |
| "score": { |
| "n": 679, |
| "acc": 0.5346097201767305, |
| "nll": 1.0653850272875651, |
| "ece": 0.07348444458021999, |
| "mae": 0.559288462760455 |
| } |
| }, |
| "by_task": { |
| "amazon_polarity": { |
| "n": 300, |
| "acc": 0.94, |
| "nll": 0.21491849340648492, |
| "ece": 0.041034815786236065 |
| }, |
| "arc": { |
| "n": 200, |
| "acc": 0.49, |
| "nll": 1.7225013280848958, |
| "ece": 0.2666289663771016 |
| }, |
| "arc_easy": { |
| "n": 200, |
| "acc": 0.66, |
| "nll": 1.0581853665538725, |
| "ece": 0.16953352736225474 |
| }, |
| "banking77": { |
| "n": 300, |
| "acc": 0.9666666666666667, |
| "nll": 0.08512822269970122, |
| "ece": 0.019191444769018 |
| }, |
| "bitext_support": { |
| "n": 300, |
| "acc": 1, |
| "nll": 0.0006662884404404625, |
| "ece": 0.0006564032820199905 |
| }, |
| "boolq": { |
| "n": 300, |
| "acc": 0.8333333333333334, |
| "nll": 0.46327447408189254, |
| "ece": 0.09105915635074 |
| }, |
| "civil_comments": { |
| "n": 156, |
| "acc": 0.7692307692307693, |
| "nll": 0.48259455349648295, |
| "ece": 0.044267076511289985, |
| "mae": 0.4782040600423823 |
| }, |
| "clinc150": { |
| "n": 300, |
| "acc": 0.9833333333333333, |
| "nll": 0.050976034228883325, |
| "ece": 0.015613015942019744 |
| }, |
| "cola": { |
| "n": 200, |
| "acc": 0.79, |
| "nll": 0.4599676350741479, |
| "ece": 0.10238053149913356 |
| }, |
| "commonsense_qa": { |
| "n": 300, |
| "acc": 0.65, |
| "nll": 1.0999147304745942, |
| "ece": 0.1701816034928976 |
| }, |
| "dbpedia14": { |
| "n": 300, |
| "acc": 0.7, |
| "nll": 1.7263676027906165, |
| "ece": 0.227155404042094 |
| }, |
| "go_emotions": { |
| "n": 256, |
| "acc": 0.84375, |
| "nll": 0.44446812826453613, |
| "ece": 0.06292944279854824 |
| }, |
| "hellaswag": { |
| "n": 300, |
| "acc": 0.6866666666666666, |
| "nll": 1.0690494049258386, |
| "ece": 0.16647033490244928 |
| }, |
| "massive_intent": { |
| "n": 300, |
| "acc": 0.97, |
| "nll": 0.10690323173654996, |
| "ece": 0.026559626387008747 |
| }, |
| "massive_scenario": { |
| "n": 300, |
| "acc": 0.9633333333333334, |
| "nll": 0.12833034262109785, |
| "ece": 0.023441601714425293 |
| }, |
| "mnli": { |
| "n": 300, |
| "acc": 0.8733333333333333, |
| "nll": 0.35470827238044367, |
| "ece": 0.054108644890771114 |
| }, |
| "mrpc": { |
| "n": 200, |
| "acc": 0.905, |
| "nll": 0.22430776762422472, |
| "ece": 0.04434373851108637 |
| }, |
| "newsgroups": { |
| "n": 296, |
| "acc": 0.8547297297297297, |
| "nll": 0.48702157440611676, |
| "ece": 0.07240345282869842 |
| }, |
| "openbookqa": { |
| "n": 250, |
| "acc": 0.596, |
| "nll": 1.1414455726990549, |
| "ece": 0.16806808919887872 |
| }, |
| "qasc": { |
| "n": 250, |
| "acc": 0.496, |
| "nll": 1.716193180040507, |
| "ece": 0.2341949557110443 |
| }, |
| "qnli": { |
| "n": 200, |
| "acc": 0.905, |
| "nll": 0.2502223326943523, |
| "ece": 0.05328244333552107 |
| }, |
| "qqp": { |
| "n": 200, |
| "acc": 0.87, |
| "nll": 0.36164874486477994, |
| "ece": 0.08291620754257616 |
| }, |
| "rte": { |
| "n": 200, |
| "acc": 0.82, |
| "nll": 0.44405080029545163, |
| "ece": 0.09511886647669884 |
| }, |
| "sms_spam": { |
| "n": 300, |
| "acc": 1, |
| "nll": 0.0015762756577773233, |
| "ece": 0.0015227966663197983 |
| }, |
| "snli": { |
| "n": 295, |
| "acc": 0.8745762711864407, |
| "nll": 0.34305959824971366, |
| "ece": 0.0675852776100721 |
| }, |
| "sst2": { |
| "n": 300, |
| "acc": 0.95, |
| "nll": 0.1810899283111296, |
| "ece": 0.037267897728623225 |
| }, |
| "sst5": { |
| "n": 300, |
| "acc": 0.5466666666666666, |
| "nll": 1.056353078896091, |
| "ece": 0.07267637287258515, |
| "mae": 0.5517158101363502 |
| }, |
| "stsb": { |
| "n": 300, |
| "acc": 0.5133333333333333, |
| "nll": 1.1312887822663569, |
| "ece": 0.10057246284204428, |
| "mae": 0.5882133414336538 |
| }, |
| "toxic_conversations": { |
| "n": 124, |
| "acc": 0.9193548387096774, |
| "nll": 0.27868695477230904, |
| "ece": 0.060428655478112825 |
| }, |
| "wanli": { |
| "n": 300, |
| "acc": 0.7466666666666667, |
| "nll": 0.6654510619859225, |
| "ece": 0.12142146453678089 |
| }, |
| "winogrande": { |
| "n": 300, |
| "acc": 0.66, |
| "nll": 1.0156935419433384, |
| "ece": 0.19326918465280768 |
| } |
| } |
| } |
| } |
