{
  "ood": {
    "overall": {
      "n": 4500,
      "acc": 0.47555555555555556,
      "nll": 1.268476909899631,
      "ece": 0.1325593729014183,
      "mae": 0.7127850154762857
    },
    "macro_acc": 0.49183333333333334,
    "by_kind": {
      "choice": {
        "n": 2500,
        "acc": 0.4276,
        "nll": 1.4152444272854594,
        "ece": 0.128002237490073
      },
      "noul": {
        "n": 1000,
        "acc": 0.589,
        "nll": 1.0108116476597444,
        "ece": 0.26637219234174647
      },
      "score": {
        "n": 1000,
        "acc": 0.482,
        "nll": 1.1592233786749255,
        "ece": 0.07211618516297272,
        "mae": 0.7127850154762857
      }
    },
    "by_task": {
      "amazon_stars": {
        "n": 1000,
        "acc": 0.482,
        "nll": 1.1592233786749255,
        "ece": 0.07211618516297272,
        "mae": 0.7127850154762857
      },
      "emotion": {
        "n": 1000,
        "acc": 0.551,
        "nll": 1.2441813441969587,
        "ece": 0.08441541454068684
      },
      "enron_spam": {
        "n": 1000,
        "acc": 0.589,
        "nll": 1.0108116476597444,
        "ece": 0.26637219234174647
      },
      "mmlu": {
        "n": 1500,
        "acc": 0.3453333333333333,
        "nll": 1.5292864826777985,
        "ece": 0.16617672448084847
      }
    }
  },
  "order_invariance_max_logit_delta": 0.000008940696716308594,
  "val": {
    "overall": {
      "n": 8127,
      "acc": 0.801894918173988,
      "nll": 0.5053446978458112,
      "ece": 0.007099048157460054,
      "mae": 0.5636936249569889
    },
    "macro_acc": 0.7992572561996758,
    "by_kind": {
      "choice": {
        "n": 4745,
        "acc": 0.7869336143308746,
        "nll": 0.5689453066911272,
        "ece": 0.010992410314786963
      },
      "noul": {
        "n": 2703,
        "acc": 0.8953015168331484,
        "nll": 0.25909313340379886,
        "ece": 0.008562793730987436
      },
      "score": {
        "n": 679,
        "acc": 0.5346097201767305,
        "nll": 1.0411813542754798,
        "ece": 0.03380929975803491,
        "mae": 0.5636936249569889
      }
    },
    "by_task": {
      "amazon_polarity": {
        "n": 300,
        "acc": 0.94,
        "nll": 0.1780728226299769,
        "ece": 0.03494007905033705
      },
      "arc": {
        "n": 200,
        "acc": 0.49,
        "nll": 1.2722349869251168,
        "ece": 0.12446486240223996
      },
      "arc_easy": {
        "n": 200,
        "acc": 0.66,
        "nll": 0.8571748012168005,
        "ece": 0.0790243305106568
      },
      "banking77": {
        "n": 300,
        "acc": 0.9666666666666667,
        "nll": 0.09405613306326095,
        "ece": 0.031746609200941224
      },
      "bitext_support": {
        "n": 300,
        "acc": 1,
        "nll": 0.010971368929279203,
        "ece": 0.010659385842467978
      },
      "boolq": {
        "n": 300,
        "acc": 0.8333333333333334,
        "nll": 0.41208993970329466,
        "ece": 0.04317514312264036
      },
      "civil_comments": {
        "n": 156,
        "acc": 0.7692307692307693,
        "nll": 0.5426374327890833,
        "ece": 0.08561364800510647,
        "mae": 0.4794184904520537
      },
      "clinc150": {
        "n": 300,
        "acc": 0.9833333333333333,
        "nll": 0.058032448735896636,
        "ece": 0.019049882616036785
      },
      "cola": {
        "n": 200,
        "acc": 0.79,
        "nll": 0.4192314168142291,
        "ece": 0.08574232134607189
      },
      "commonsense_qa": {
        "n": 300,
        "acc": 0.65,
        "nll": 0.8803430494105633,
        "ece": 0.042060654356154394
      },
      "dbpedia14": {
        "n": 300,
        "acc": 0.7,
        "nll": 1.0272166475279698,
        "ece": 0.16287307424578817
      },
      "go_emotions": {
        "n": 256,
        "acc": 0.84375,
        "nll": 0.4625858118782197,
        "ece": 0.07856337055037782
      },
      "hellaswag": {
        "n": 300,
        "acc": 0.6866666666666666,
        "nll": 0.8184342990797175,
        "ece": 0.065393505705077
      },
      "massive_intent": {
        "n": 300,
        "acc": 0.97,
        "nll": 0.11297520330888683,
        "ece": 0.03747435978798287
      },
      "massive_scenario": {
        "n": 300,
        "acc": 0.9633333333333334,
        "nll": 0.14092133711034824,
        "ece": 0.040415555363053655
      },
      "mnli": {
        "n": 300,
        "acc": 0.8733333333333333,
        "nll": 0.3591158164193244,
        "ece": 0.06122319501006837
      },
      "mrpc": {
        "n": 200,
        "acc": 0.905,
        "nll": 0.2340552890847502,
        "ece": 0.0475103336919146
      },
      "newsgroups": {
        "n": 296,
        "acc": 0.8547297297297297,
        "nll": 0.4397821279908795,
        "ece": 0.05815265497795075
      },
      "openbookqa": {
        "n": 250,
        "acc": 0.596,
        "nll": 0.9566964109317068,
        "ece": 0.05584956464154654
      },
      "qasc": {
        "n": 250,
        "acc": 0.496,
        "nll": 1.3698718341800415,
        "ece": 0.05324690519921278
      },
      "qnli": {
        "n": 200,
        "acc": 0.905,
        "nll": 0.22825732034313698,
        "ece": 0.05859514918248248
      },
      "qqp": {
        "n": 200,
        "acc": 0.87,
        "nll": 0.3305964280804749,
        "ece": 0.05731508099827738
      },
      "rte": {
        "n": 200,
        "acc": 0.82,
        "nll": 0.3908081576970799,
        "ece": 0.03793475295601129
      },
      "sms_spam": {
        "n": 300,
        "acc": 1,
        "nll": 0.009443218337208322,
        "ece": 0.009181943972248042
      },
      "snli": {
        "n": 295,
        "acc": 0.8745762711864407,
        "nll": 0.34642039268536257,
        "ece": 0.0733830186710087
      },
      "sst2": {
        "n": 300,
        "acc": 0.95,
        "nll": 0.17723819437605498,
        "ece": 0.04834600996257101
      },
      "sst5": {
        "n": 300,
        "acc": 0.5466666666666666,
        "nll": 1.034445504429844,
        "ece": 0.06904864172392965,
        "mae": 0.5531891951338506
      },
      "stsb": {
        "n": 300,
        "acc": 0.5133333333333333,
        "nll": 1.0930674410680064,
        "ece": 0.048921423078040656,
        "mae": 0.5963905068664275
      },
      "toxic_conversations": {
        "n": 124,
        "acc": 0.9193548387096774,
        "nll": 0.25773286368862236,
        "ece": 0.05372284727427989
      },
      "wanli": {
        "n": 300,
        "acc": 0.7466666666666667,
        "nll": 0.5828687631711043,
        "ece": 0.04179919773420856
      },
      "winogrande": {
        "n": 300,
        "acc": 0.66,
        "nll": 0.7154438457054739,
        "ece": 0.11005624853642657
      }
    }
  },
  "val_uncalibrated": {
    "overall": {
      "n": 8127,
      "acc": 0.801894918173988,
      "nll": 0.6020770461865659,
      "ece": 0.08041752615679162,
      "mae": 0.559288462760455
    },
    "macro_acc": 0.7992572561996758,
    "by_kind": {
      "choice": {
        "n": 4745,
        "acc": 0.7869336143308746,
        "nll": 0.713495906024384,
        "ece": 0.10104285618425683
      },
      "noul": {
        "n": 2703,
        "acc": 0.8953015168331484,
        "nll": 0.2901019780777808,
        "ece": 0.04708815584016769
      },
      "score": {
        "n": 679,
        "acc": 0.5346097201767305,
        "nll": 1.0653850272875651,
        "ece": 0.07348444458021999,
        "mae": 0.559288462760455
      }
    },
    "by_task": {
      "amazon_polarity": {
        "n": 300,
        "acc": 0.94,
        "nll": 0.21491849340648492,
        "ece": 0.041034815786236065
      },
      "arc": {
        "n": 200,
        "acc": 0.49,
        "nll": 1.7225013280848958,
        "ece": 0.2666289663771016
      },
      "arc_easy": {
        "n": 200,
        "acc": 0.66,
        "nll": 1.0581853665538725,
        "ece": 0.16953352736225474
      },
      "banking77": {
        "n": 300,
        "acc": 0.9666666666666667,
        "nll": 0.08512822269970122,
        "ece": 0.019191444769018
      },
      "bitext_support": {
        "n": 300,
        "acc": 1,
        "nll": 0.0006662884404404625,
        "ece": 0.0006564032820199905
      },
      "boolq": {
        "n": 300,
        "acc": 0.8333333333333334,
        "nll": 0.46327447408189254,
        "ece": 0.09105915635074
      },
      "civil_comments": {
        "n": 156,
        "acc": 0.7692307692307693,
        "nll": 0.48259455349648295,
        "ece": 0.044267076511289985,
        "mae": 0.4782040600423823
      },
      "clinc150": {
        "n": 300,
        "acc": 0.9833333333333333,
        "nll": 0.050976034228883325,
        "ece": 0.015613015942019744
      },
      "cola": {
        "n": 200,
        "acc": 0.79,
        "nll": 0.4599676350741479,
        "ece": 0.10238053149913356
      },
      "commonsense_qa": {
        "n": 300,
        "acc": 0.65,
        "nll": 1.0999147304745942,
        "ece": 0.1701816034928976
      },
      "dbpedia14": {
        "n": 300,
        "acc": 0.7,
        "nll": 1.7263676027906165,
        "ece": 0.227155404042094
      },
      "go_emotions": {
        "n": 256,
        "acc": 0.84375,
        "nll": 0.44446812826453613,
        "ece": 0.06292944279854824
      },
      "hellaswag": {
        "n": 300,
        "acc": 0.6866666666666666,
        "nll": 1.0690494049258386,
        "ece": 0.16647033490244928
      },
      "massive_intent": {
        "n": 300,
        "acc": 0.97,
        "nll": 0.10690323173654996,
        "ece": 0.026559626387008747
      },
      "massive_scenario": {
        "n": 300,
        "acc": 0.9633333333333334,
        "nll": 0.12833034262109785,
        "ece": 0.023441601714425293
      },
      "mnli": {
        "n": 300,
        "acc": 0.8733333333333333,
        "nll": 0.35470827238044367,
        "ece": 0.054108644890771114
      },
      "mrpc": {
        "n": 200,
        "acc": 0.905,
        "nll": 0.22430776762422472,
        "ece": 0.04434373851108637
      },
      "newsgroups": {
        "n": 296,
        "acc": 0.8547297297297297,
        "nll": 0.48702157440611676,
        "ece": 0.07240345282869842
      },
      "openbookqa": {
        "n": 250,
        "acc": 0.596,
        "nll": 1.1414455726990549,
        "ece": 0.16806808919887872
      },
      "qasc": {
        "n": 250,
        "acc": 0.496,
        "nll": 1.716193180040507,
        "ece": 0.2341949557110443
      },
      "qnli": {
        "n": 200,
        "acc": 0.905,
        "nll": 0.2502223326943523,
        "ece": 0.05328244333552107
      },
      "qqp": {
        "n": 200,
        "acc": 0.87,
        "nll": 0.36164874486477994,
        "ece": 0.08291620754257616
      },
      "rte": {
        "n": 200,
        "acc": 0.82,
        "nll": 0.44405080029545163,
        "ece": 0.09511886647669884
      },
      "sms_spam": {
        "n": 300,
        "acc": 1,
        "nll": 0.0015762756577773233,
        "ece": 0.0015227966663197983
      },
      "snli": {
        "n": 295,
        "acc": 0.8745762711864407,
        "nll": 0.34305959824971366,
        "ece": 0.0675852776100721
      },
      "sst2": {
        "n": 300,
        "acc": 0.95,
        "nll": 0.1810899283111296,
        "ece": 0.037267897728623225
      },
      "sst5": {
        "n": 300,
        "acc": 0.5466666666666666,
        "nll": 1.056353078896091,
        "ece": 0.07267637287258515,
        "mae": 0.5517158101363502
      },
      "stsb": {
        "n": 300,
        "acc": 0.5133333333333333,
        "nll": 1.1312887822663569,
        "ece": 0.10057246284204428,
        "mae": 0.5882133414336538
      },
      "toxic_conversations": {
        "n": 124,
        "acc": 0.9193548387096774,
        "nll": 0.27868695477230904,
        "ece": 0.060428655478112825
      },
      "wanli": {
        "n": 300,
        "acc": 0.7466666666666667,
        "nll": 0.6654510619859225,
        "ece": 0.12142146453678089
      },
      "winogrande": {
        "n": 300,
        "acc": 0.66,
        "nll": 1.0156935419433384,
        "ece": 0.19326918465280768
      }
    }
  }
}