| { |
| "overall": { |
| "count": 6113, |
| "accuracy": 0.6679208244724358, |
| "brier": 0.44247259100885217, |
| "brier_soft": 0.4170925201205985, |
| "nll": 0.8056102230141763, |
| "nll_soft": 0.8172796926231564, |
| "ece": 0.03653387851957047, |
| "mean_confidence": 0.6869317729442912, |
| "confident_error_rate": 0.019466710289546867, |
| "score_mae": 0.5729998939493892, |
| "evidence_auc": 0.9074727460810058 |
| }, |
| "by_type": { |
| "bool": { |
| "count": 1898, |
| "accuracy": 0.7787144362486829, |
| "brier": 0.3124533326117464, |
| "brier_soft": 0.30893223729451025, |
| "nll": 0.47617615577105815, |
| "nll_soft": 0.4805717663652272, |
| "ece": 0.04409051484755141, |
| "mean_confidence": 0.8090805982576155, |
| "confident_error_rate": 0.026343519494204427, |
| "evidence_auc": 0.9039322111530974 |
| }, |
| "choice": { |
| "count": 3248, |
| "accuracy": 0.625615763546798, |
| "brier": 0.5012455732095099, |
| "brier_soft": 0.4763148831509033, |
| "nll": 0.9690121828222874, |
| "nll_soft": 0.9798443434525502, |
| "ece": 0.06097491611004007, |
| "mean_confidence": 0.6575062932025669, |
| "confident_error_rate": 0.021243842364532018, |
| "evidence_auc": 0.9208606009429315 |
| }, |
| "score": { |
| "count": 967, |
| "accuracy": 0.5925542916235781, |
| "brier": 0.5002615323221618, |
| "brier_soft": 0.4304678889742543, |
| "nll": 0.9033726782062063, |
| "nll_soft": 0.9321314590592249, |
| "ece": 0.06700552232571198, |
| "mean_confidence": 0.546017075691376, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.5729998939493892 |
| } |
| }, |
| "by_source": { |
| "ag_news": { |
| "count": 100, |
| "accuracy": 0.88, |
| "brier": 0.17829138819549176, |
| "brier_soft": 0.17829138819549176, |
| "nll": 0.30741759767501897, |
| "nll_soft": 0.30741759767501897, |
| "ece": 0.09543892771005633, |
| "mean_confidence": 0.9135349568724632, |
| "confident_error_rate": 0.02 |
| }, |
| "amazon_polarity": { |
| "count": 100, |
| "accuracy": 0.97, |
| "brier": 0.06162859960932998, |
| "brier_soft": 0.06162859960932998, |
| "nll": 0.13110608215243919, |
| "nll_soft": 0.13110608215243919, |
| "ece": 0.03008812248706818, |
| "mean_confidence": 0.9551744455099106, |
| "confident_error_rate": 0.03 |
| }, |
| "amazon_stars": { |
| "count": 100, |
| "accuracy": 0.53, |
| "brier": 0.5461979259182673, |
| "brier_soft": 0.5461979259182673, |
| "nll": 0.9164086895783538, |
| "nll_soft": 0.9164086895783538, |
| "ece": 0.11809196531772617, |
| "mean_confidence": 0.5753743493556976, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.5362134778685868 |
| }, |
| "anli": { |
| "count": 100, |
| "accuracy": 0.39, |
| "brier": 0.9722760942391393, |
| "brier_soft": 0.9722760942391393, |
| "nll": 1.8542452552224933, |
| "nll_soft": 1.8542452552224933, |
| "ece": 0.44132104188203813, |
| "mean_confidence": 0.7956731787323952, |
| "confident_error_rate": 0.23 |
| }, |
| "arc_challenge": { |
| "count": 100, |
| "accuracy": 0.36, |
| "brier": 0.778481672987934, |
| "brier_soft": 0.778481672987934, |
| "nll": 1.4337150394138498, |
| "nll_soft": 1.4337150394138498, |
| "ece": 0.20447848409414293, |
| "mean_confidence": 0.5459534320235252, |
| "confident_error_rate": 0.0 |
| }, |
| "arc_easy": { |
| "count": 100, |
| "accuracy": 0.45, |
| "brier": 0.6407232552939703, |
| "brier_soft": 0.6407232552939703, |
| "nll": 1.2136572969992905, |
| "nll_soft": 1.2136572969992905, |
| "ece": 0.12803085684776305, |
| "mean_confidence": 0.5452363789081573, |
| "confident_error_rate": 0.01 |
| }, |
| "banking77": { |
| "count": 100, |
| "accuracy": 0.81, |
| "brier": 0.3814467708771144, |
| "brier_soft": 0.3814467708771144, |
| "nll": 1.0443211262890961, |
| "nll_soft": 1.0443211262890961, |
| "ece": 0.3126603952050208, |
| "mean_confidence": 0.4973396047949791, |
| "confident_error_rate": 0.0 |
| }, |
| "boolq": { |
| "count": 125, |
| "accuracy": 0.568, |
| "brier": 0.6168742406476001, |
| "brier_soft": 0.6168742406476001, |
| "nll": 0.8592622690829735, |
| "nll_soft": 0.8592622690829735, |
| "ece": 0.2938120722770691, |
| "mean_confidence": 0.8219961438179016, |
| "confident_error_rate": 0.032, |
| "evidence_auc": 0.9706666666666667 |
| }, |
| "cb": { |
| "count": 28, |
| "accuracy": 0.9285714285714286, |
| "brier": 0.13537618609721863, |
| "brier_soft": 0.13537618609721863, |
| "nll": 0.2816298666367622, |
| "nll_soft": 0.2816298666367622, |
| "ece": 0.11475402755396709, |
| "mean_confidence": 0.9356890469789505, |
| "confident_error_rate": 0.03571428571428571 |
| }, |
| "civil_comments_severity": { |
| "count": 100, |
| "accuracy": 0.81, |
| "brier": 0.28384330937032604, |
| "brier_soft": 0.17998165817214293, |
| "nll": 0.5338759677740593, |
| "nll_soft": 0.6298814779743555, |
| "ece": 0.14874677449464796, |
| "mean_confidence": 0.6996492806077004, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.39768598369671965 |
| }, |
| "civil_comments_toxic": { |
| "count": 100, |
| "accuracy": 0.94, |
| "brier": 0.09920447825135499, |
| "brier_soft": 0.03237408913021391, |
| "nll": 0.19394092520219666, |
| "nll_soft": 0.2773696142795253, |
| "ece": 0.08483366131782531, |
| "mean_confidence": 0.8621529483795166, |
| "confident_error_rate": 0.0 |
| }, |
| "clinc150": { |
| "count": 100, |
| "accuracy": 0.8, |
| "brier": 0.4848727215372612, |
| "brier_soft": 0.4848727215372612, |
| "nll": 1.4845052623480186, |
| "nll_soft": 1.4845052623480186, |
| "ece": 0.4221790521405637, |
| "mean_confidence": 0.37782094785943626, |
| "confident_error_rate": 0.0 |
| }, |
| "cola": { |
| "count": 100, |
| "accuracy": 0.58, |
| "brier": 0.4671416305229769, |
| "brier_soft": 0.4671416305229769, |
| "nll": 0.6581302967113183, |
| "nll_soft": 0.6581302967113183, |
| "ece": 0.10910327613353732, |
| "mean_confidence": 0.6739702433347702, |
| "confident_error_rate": 0.0 |
| }, |
| "commonsense_qa": { |
| "count": 100, |
| "accuracy": 0.42, |
| "brier": 0.7026358240551468, |
| "brier_soft": 0.7026358240551468, |
| "nll": 1.3662261855444524, |
| "nll_soft": 1.3662261855444524, |
| "ece": 0.13439485043287275, |
| "mean_confidence": 0.510890220105648, |
| "confident_error_rate": 0.0 |
| }, |
| "cosmos_qa": { |
| "count": 120, |
| "accuracy": 0.4, |
| "brier": 0.674820119653572, |
| "brier_soft": 0.674820119653572, |
| "nll": 1.2437770795447451, |
| "nll_soft": 1.2437770795447451, |
| "ece": 0.14261294777194658, |
| "mean_confidence": 0.5219608165323735, |
| "confident_error_rate": 0.0, |
| "evidence_auc": 0.836875 |
| }, |
| "dates": { |
| "count": 300, |
| "accuracy": 0.7633333333333333, |
| "brier": 0.2819022563327028, |
| "brier_soft": 0.2819022563327028, |
| "nll": 0.4583224548625654, |
| "nll_soft": 0.4583224548625654, |
| "ece": 0.06106376767158511, |
| "mean_confidence": 0.7644337038199107, |
| "confident_error_rate": 0.0033333333333333335, |
| "score_mae": 0.45998074431250346 |
| }, |
| "dbpedia": { |
| "count": 100, |
| "accuracy": 0.99, |
| "brier": 0.015198128030094994, |
| "brier_soft": 0.015198128030094994, |
| "nll": 0.06724704860844338, |
| "nll_soft": 0.06724704860844338, |
| "ece": 0.057107390761375415, |
| "mean_confidence": 0.9441341024637222, |
| "confident_error_rate": 0.0 |
| }, |
| "dynasent": { |
| "count": 100, |
| "accuracy": 0.53, |
| "brier": 0.5435009917424021, |
| "brier_soft": 0.40578378452816594, |
| "nll": 0.9203439216635296, |
| "nll_soft": 0.95189629595177, |
| "ece": 0.12150101184844973, |
| "mean_confidence": 0.5809222316741943, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.590812379643321 |
| }, |
| "enron_spam": { |
| "count": 100, |
| "accuracy": 0.94, |
| "brier": 0.08228255110990408, |
| "brier_soft": 0.08228255110990408, |
| "nll": 0.15493662205431416, |
| "nll_soft": 0.15493662205431416, |
| "ece": 0.043621286749839804, |
| "mean_confidence": 0.9467802709341049, |
| "confident_error_rate": 0.02 |
| }, |
| "go_emotions": { |
| "count": 100, |
| "accuracy": 0.54, |
| "brier": 0.6768268805616047, |
| "brier_soft": 0.25233950625259904, |
| "nll": 1.7380711253413985, |
| "nll_soft": 2.0785070377760406, |
| "ece": 0.24022679880261422, |
| "mean_confidence": 0.29977320119738576, |
| "confident_error_rate": 0.0 |
| }, |
| "hellaswag": { |
| "count": 100, |
| "accuracy": 0.49, |
| "brier": 0.6818457284598648, |
| "brier_soft": 0.6818457284598648, |
| "nll": 1.27200660086301, |
| "nll_soft": 1.27200660086301, |
| "ece": 0.1305046308040619, |
| "mean_confidence": 0.5641709685325622, |
| "confident_error_rate": 0.0 |
| }, |
| "imdb": { |
| "count": 100, |
| "accuracy": 0.95, |
| "brier": 0.08363715307563817, |
| "brier_soft": 0.08363715307563817, |
| "nll": 0.16551026463952032, |
| "nll_soft": 0.16551026463952032, |
| "ece": 0.026311837434768674, |
| "mean_confidence": 0.9574666857719422, |
| "confident_error_rate": 0.03 |
| }, |
| "ledgar": { |
| "count": 100, |
| "accuracy": 0.75, |
| "brier": 0.42560135728608983, |
| "brier_soft": 0.42560135728608983, |
| "nll": 1.3302157486233597, |
| "nll_soft": 1.3302157486233597, |
| "ece": 0.23709699064493178, |
| "mean_confidence": 0.5129030093550682, |
| "confident_error_rate": 0.0 |
| }, |
| "liar": { |
| "count": 100, |
| "accuracy": 0.32, |
| "brier": 0.7427052798517789, |
| "brier_soft": 0.7427052798517789, |
| "nll": 1.4964320047893989, |
| "nll_soft": 1.4964320047893989, |
| "ece": 0.055253805071115485, |
| "mean_confidence": 0.3017434249818325, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.9733948946371674 |
| }, |
| "measuring_hate_speech": { |
| "count": 100, |
| "accuracy": 0.83, |
| "brier": 0.3177919283014086, |
| "brier_soft": 0.16750927245698513, |
| "nll": 0.5942829577379074, |
| "nll_soft": 0.7062310284569752, |
| "ece": 0.21628696441650394, |
| "mean_confidence": 0.6214796364307403, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.6045454696938396 |
| }, |
| "medqa": { |
| "count": 100, |
| "accuracy": 0.34, |
| "brier": 0.7671574185751032, |
| "brier_soft": 0.7671574185751032, |
| "nll": 1.4356347250931116, |
| "nll_soft": 1.4356347250931116, |
| "ece": 0.17744469463825224, |
| "mean_confidence": 0.48414860367774964, |
| "confident_error_rate": 0.0 |
| }, |
| "mnli": { |
| "count": 100, |
| "accuracy": 0.78, |
| "brier": 0.3263025000138572, |
| "brier_soft": 0.3263025000138572, |
| "nll": 0.5982821287502831, |
| "nll_soft": 0.5982821287502831, |
| "ece": 0.10382685691118237, |
| "mean_confidence": 0.8509401044249535, |
| "confident_error_rate": 0.06 |
| }, |
| "mrpc": { |
| "count": 100, |
| "accuracy": 0.81, |
| "brier": 0.27720433744929446, |
| "brier_soft": 0.27720433744929446, |
| "nll": 0.43150783049960595, |
| "nll_soft": 0.43150783049960595, |
| "ece": 0.1067017036676407, |
| "mean_confidence": 0.856079712510109, |
| "confident_error_rate": 0.03 |
| }, |
| "multirc": { |
| "count": 125, |
| "accuracy": 0.664, |
| "brier": 0.5262665529453465, |
| "brier_soft": 0.5262665529453465, |
| "nll": 0.7910576977190109, |
| "nll_soft": 0.7910576977190109, |
| "ece": 0.20780441761016843, |
| "mean_confidence": 0.833528317451477, |
| "confident_error_rate": 0.136, |
| "evidence_auc": 0.6994666666666667 |
| }, |
| "openbookqa": { |
| "count": 100, |
| "accuracy": 0.52, |
| "brier": 0.6059996266277589, |
| "brier_soft": 0.6059996266277589, |
| "nll": 1.1493342555724386, |
| "nll_soft": 1.1493342555724386, |
| "ece": 0.08220899760723116, |
| "mean_confidence": 0.5423529654741287, |
| "confident_error_rate": 0.01 |
| }, |
| "piqa": { |
| "count": 100, |
| "accuracy": 0.61, |
| "brier": 0.5129767773118842, |
| "brier_soft": 0.5129767773118842, |
| "nll": 0.7296690239966398, |
| "nll_soft": 0.7296690239966398, |
| "ece": 0.14158266067504877, |
| "mean_confidence": 0.6558456552028656, |
| "confident_error_rate": 0.02 |
| }, |
| "policy_application": { |
| "count": 150, |
| "accuracy": 0.7333333333333333, |
| "brier": 0.3508971131550242, |
| "brier_soft": 0.3508971131550242, |
| "nll": 0.5516997947737176, |
| "nll_soft": 0.5516997947737176, |
| "ece": 0.044622519016265856, |
| "mean_confidence": 0.7177301915486654, |
| "confident_error_rate": 0.0, |
| "evidence_auc": 0.5696533682145193 |
| }, |
| "policy_hard": { |
| "count": 300, |
| "accuracy": 0.6966666666666667, |
| "brier": 0.38472149078986073, |
| "brier_soft": 0.38472149078986073, |
| "nll": 0.6128855879532513, |
| "nll_soft": 0.6128855879532513, |
| "ece": 0.053346475064754505, |
| "mean_confidence": 0.7120696005225181, |
| "confident_error_rate": 0.0, |
| "evidence_auc": 0.7445023148148148 |
| }, |
| "qqp": { |
| "count": 100, |
| "accuracy": 0.8, |
| "brier": 0.3034980053199748, |
| "brier_soft": 0.3034980053199748, |
| "nll": 0.4749638870383975, |
| "nll_soft": 0.4749638870383975, |
| "ece": 0.10882365942001343, |
| "mean_confidence": 0.8906454944610596, |
| "confident_error_rate": 0.07 |
| }, |
| "race": { |
| "count": 125, |
| "accuracy": 0.464, |
| "brier": 0.670541267736231, |
| "brier_soft": 0.670541267736231, |
| "nll": 1.2785957499648075, |
| "nll_soft": 1.2785957499648075, |
| "ece": 0.09217529821395876, |
| "mean_confidence": 0.4943129940032959, |
| "confident_error_rate": 0.0, |
| "evidence_auc": 0.7856 |
| }, |
| "rotten_tomatoes": { |
| "count": 100, |
| "accuracy": 0.89, |
| "brier": 0.26950262705978917, |
| "brier_soft": 0.26950262705978917, |
| "nll": 0.43769943345994206, |
| "nll_soft": 0.43769943345994206, |
| "ece": 0.18772910118103028, |
| "mean_confidence": 0.7022708988189698, |
| "confident_error_rate": 0.0 |
| }, |
| "rte": { |
| "count": 100, |
| "accuracy": 0.7, |
| "brier": 0.4417545164022784, |
| "brier_soft": 0.4417545164022784, |
| "nll": 0.6803569597273692, |
| "nll_soft": 0.6803569597273692, |
| "ece": 0.166305216550827, |
| "mean_confidence": 0.8475300312042237, |
| "confident_error_rate": 0.07 |
| }, |
| "scotus": { |
| "count": 100, |
| "accuracy": 0.63, |
| "brier": 0.57591555494716, |
| "brier_soft": 0.57591555494716, |
| "nll": 1.4268808895864786, |
| "nll_soft": 1.4268808895864786, |
| "ece": 0.1622882364690304, |
| "mean_confidence": 0.479786828905344, |
| "confident_error_rate": 0.0 |
| }, |
| "shp": { |
| "count": 100, |
| "accuracy": 0.5, |
| "brier": 0.553928533015661, |
| "brier_soft": 0.16866709422112394, |
| "nll": 0.7553610522259013, |
| "nll_soft": 0.7667537170621911, |
| "ece": 0.22228121280670163, |
| "mean_confidence": 0.6585486650466919, |
| "confident_error_rate": 0.0 |
| }, |
| "sms_spam": { |
| "count": 100, |
| "accuracy": 0.96, |
| "brier": 0.14873415443052604, |
| "brier_soft": 0.14873415443052604, |
| "nll": 0.29604541382027116, |
| "nll_soft": 0.29604541382027116, |
| "ece": 0.18373807787895202, |
| "mean_confidence": 0.7762619221210479, |
| "confident_error_rate": 0.0 |
| }, |
| "snli": { |
| "count": 100, |
| "accuracy": 0.79, |
| "brier": 0.3657109430484185, |
| "brier_soft": 0.3657109430484185, |
| "nll": 0.6824574908252256, |
| "nll_soft": 0.6824574908252256, |
| "ece": 0.1515702518820763, |
| "mean_confidence": 0.8926358130574227, |
| "confident_error_rate": 0.1 |
| }, |
| "social_iqa": { |
| "count": 100, |
| "accuracy": 0.5, |
| "brier": 0.6701157287052262, |
| "brier_soft": 0.6701157287052262, |
| "nll": 1.2110427461546294, |
| "nll_soft": 1.2110427461546294, |
| "ece": 0.21912074834108355, |
| "mean_confidence": 0.68725660353899, |
| "confident_error_rate": 0.04 |
| }, |
| "sst2": { |
| "count": 100, |
| "accuracy": 0.9, |
| "brier": 0.25762914103330403, |
| "brier_soft": 0.25762914103330403, |
| "nll": 0.42289569521733256, |
| "nll_soft": 0.42289569521733256, |
| "ece": 0.182864231467247, |
| "mean_confidence": 0.717135768532753, |
| "confident_error_rate": 0.0 |
| }, |
| "sst5": { |
| "count": 100, |
| "accuracy": 0.56, |
| "brier": 0.5750112386200766, |
| "brier_soft": 0.5750112386200766, |
| "nll": 1.0410791576793528, |
| "nll_soft": 1.0410791576793528, |
| "ece": 0.13715695679187778, |
| "mean_confidence": 0.4760879367589951, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.569280381789431 |
| }, |
| "stance_abortion": { |
| "count": 60, |
| "accuracy": 0.6333333333333333, |
| "brier": 0.4775618932889926, |
| "brier_soft": 0.4775618932889926, |
| "nll": 0.772960340218129, |
| "nll_soft": 0.772960340218129, |
| "ece": 0.14922663221756619, |
| "mean_confidence": 0.7609471683700879, |
| "confident_error_rate": 0.05 |
| }, |
| "stance_atheism": { |
| "count": 60, |
| "accuracy": 0.7, |
| "brier": 0.39241280131421197, |
| "brier_soft": 0.39241280131421197, |
| "nll": 0.6474431235930713, |
| "nll_soft": 0.6474431235930713, |
| "ece": 0.15528117964665092, |
| "mean_confidence": 0.8136728132764498, |
| "confident_error_rate": 0.016666666666666666 |
| }, |
| "stance_feminist": { |
| "count": 60, |
| "accuracy": 0.6833333333333333, |
| "brier": 0.46696109115797474, |
| "brier_soft": 0.46696109115797474, |
| "nll": 0.7586194826479168, |
| "nll_soft": 0.7586194826479168, |
| "ece": 0.17778196881214778, |
| "mean_confidence": 0.7421373461683591, |
| "confident_error_rate": 0.05 |
| }, |
| "stance_hillary": { |
| "count": 60, |
| "accuracy": 0.6666666666666666, |
| "brier": 0.47286586681141635, |
| "brier_soft": 0.47286586681141635, |
| "nll": 0.8005868818543999, |
| "nll_soft": 0.8005868818543999, |
| "ece": 0.15976376285155613, |
| "mean_confidence": 0.7363345429301262, |
| "confident_error_rate": 0.03333333333333333 |
| }, |
| "stsb": { |
| "count": 100, |
| "accuracy": 0.45, |
| "brier": 0.6576776158521066, |
| "brier_soft": 0.3746345989346838, |
| "nll": 1.2299245867552293, |
| "nll_soft": 1.2685160423963167, |
| "ece": 0.10637196481227874, |
| "mean_confidence": 0.523349083662033, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.7103589298564476 |
| }, |
| "subjectivity": { |
| "count": 100, |
| "accuracy": 0.94, |
| "brier": 0.07403801155263033, |
| "brier_soft": 0.07403801155263033, |
| "nll": 0.12967172016931594, |
| "nll_soft": 0.12967172016931594, |
| "ece": 0.06520502805709842, |
| "mean_confidence": 0.9230094337463379, |
| "confident_error_rate": 0.0 |
| }, |
| "swag": { |
| "count": 100, |
| "accuracy": 0.4, |
| "brier": 0.7323198866636677, |
| "brier_soft": 0.7323198866636677, |
| "nll": 1.3764612983642295, |
| "nll_soft": 1.3764612983642295, |
| "ece": 0.2521471258997917, |
| "mean_confidence": 0.6150306752324104, |
| "confident_error_rate": 0.02 |
| }, |
| "trec": { |
| "count": 100, |
| "accuracy": 0.96, |
| "brier": 0.06216060648768522, |
| "brier_soft": 0.06216060648768522, |
| "nll": 0.14523731763009665, |
| "nll_soft": 0.14523731763009665, |
| "ece": 0.07524359405040741, |
| "mean_confidence": 0.9198704010248184, |
| "confident_error_rate": 0.0 |
| }, |
| "tweet_hate": { |
| "count": 100, |
| "accuracy": 0.54, |
| "brier": 0.4864483510328587, |
| "brier_soft": 0.4864483510328587, |
| "nll": 0.6796653519155663, |
| "nll_soft": 0.6796653519155663, |
| "ece": 0.08831145048141475, |
| "mean_confidence": 0.6011694383621216, |
| "confident_error_rate": 0.0 |
| }, |
| "tweet_sentiment": { |
| "count": 100, |
| "accuracy": 0.67, |
| "brier": 0.4380984174109661, |
| "brier_soft": 0.4380984174109661, |
| "nll": 0.729035173618918, |
| "nll_soft": 0.729035173618918, |
| "ece": 0.13779532372951508, |
| "mean_confidence": 0.5552096617221832, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.4073845294490457 |
| }, |
| "vitaminc": { |
| "count": 100, |
| "accuracy": 0.55, |
| "brier": 0.6059178299160775, |
| "brier_soft": 0.6059178299160775, |
| "nll": 1.0501780738072244, |
| "nll_soft": 1.0501780738072244, |
| "ece": 0.20950709015131, |
| "mean_confidence": 0.75950709015131, |
| "confident_error_rate": 0.07 |
| }, |
| "wic": { |
| "count": 100, |
| "accuracy": 0.61, |
| "brier": 0.502306829144201, |
| "brier_soft": 0.502306829144201, |
| "nll": 0.7209994732293087, |
| "nll_soft": 0.7209994732293087, |
| "ece": 0.16384762406349185, |
| "mean_confidence": 0.7613926827907562, |
| "confident_error_rate": 0.04 |
| }, |
| "winogrande": { |
| "count": 100, |
| "accuracy": 0.57, |
| "brier": 0.5063494004660651, |
| "brier_soft": 0.5063494004660651, |
| "nll": 0.7075272167886625, |
| "nll_soft": 0.7075272167886625, |
| "ece": 0.09909610986709594, |
| "mean_confidence": 0.6518526268005371, |
| "confident_error_rate": 0.0 |
| }, |
| "yelp": { |
| "count": 100, |
| "accuracy": 0.62, |
| "brier": 0.43856781301069964, |
| "brier_soft": 0.43856781301069964, |
| "nll": 0.7669727900329802, |
| "nll_soft": 0.7669727900329802, |
| "ece": 0.15332692295312883, |
| "mean_confidence": 0.5684746101498603, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.44304582916665824 |
| } |
| }, |
| "reliability": [ |
| { |
| "bin": "0.0-0.1", |
| "count": 17, |
| "confidence": 0.06687228787032996, |
| "accuracy": 0.35294117647058826 |
| }, |
| { |
| "bin": "0.1-0.2", |
| "count": 42, |
| "confidence": 0.16727064549922943, |
| "accuracy": 0.2619047619047619 |
| }, |
| { |
| "bin": "0.2-0.3", |
| "count": 183, |
| "confidence": 0.25834722043386577, |
| "accuracy": 0.3770491803278688 |
| }, |
| { |
| "bin": "0.3-0.4", |
| "count": 348, |
| "confidence": 0.3574332141841965, |
| "accuracy": 0.3994252873563218 |
| }, |
| { |
| "bin": "0.4-0.5", |
| "count": 677, |
| "confidence": 0.4517646449971586, |
| "accuracy": 0.4564254062038405 |
| }, |
| { |
| "bin": "0.5-0.6", |
| "count": 1026, |
| "confidence": 0.5484959817769235, |
| "accuracy": 0.5487329434697856 |
| }, |
| { |
| "bin": "0.6-0.7", |
| "count": 832, |
| "confidence": 0.6478659043518397, |
| "accuracy": 0.6334134615384616 |
| }, |
| { |
| "bin": "0.7-0.8", |
| "count": 756, |
| "confidence": 0.7482921844594693, |
| "accuracy": 0.7314814814814815 |
| }, |
| { |
| "bin": "0.8-0.9", |
| "count": 814, |
| "confidence": 0.8511582021338348, |
| "accuracy": 0.7457002457002457 |
| }, |
| { |
| "bin": "0.9-1.0", |
| "count": 1418, |
| "confidence": 0.9543087374844571, |
| "accuracy": 0.9160789844851904 |
| } |
| ] |
| } |
