{ "overall": { "count": 6113, "accuracy": 0.6679208244724358, "brier": 0.44247259100885217, "brier_soft": 0.4170925201205985, "nll": 0.8056102230141763, "nll_soft": 0.8172796926231564, "ece": 0.03653387851957047, "mean_confidence": 0.6869317729442912, "confident_error_rate": 0.019466710289546867, "score_mae": 0.5729998939493892, "evidence_auc": 0.9074727460810058 }, "by_type": { "bool": { "count": 1898, "accuracy": 0.7787144362486829, "brier": 0.3124533326117464, "brier_soft": 0.30893223729451025, "nll": 0.47617615577105815, "nll_soft": 0.4805717663652272, "ece": 0.04409051484755141, "mean_confidence": 0.8090805982576155, "confident_error_rate": 0.026343519494204427, "evidence_auc": 0.9039322111530974 }, "choice": { "count": 3248, "accuracy": 0.625615763546798, "brier": 0.5012455732095099, "brier_soft": 0.4763148831509033, "nll": 0.9690121828222874, "nll_soft": 0.9798443434525502, "ece": 0.06097491611004007, "mean_confidence": 0.6575062932025669, "confident_error_rate": 0.021243842364532018, "evidence_auc": 0.9208606009429315 }, "score": { "count": 967, "accuracy": 0.5925542916235781, "brier": 0.5002615323221618, "brier_soft": 0.4304678889742543, "nll": 0.9033726782062063, "nll_soft": 0.9321314590592249, "ece": 0.06700552232571198, "mean_confidence": 0.546017075691376, "confident_error_rate": 0.0, "score_mae": 0.5729998939493892 } }, "by_source": { "ag_news": { "count": 100, "accuracy": 0.88, "brier": 0.17829138819549176, "brier_soft": 0.17829138819549176, "nll": 0.30741759767501897, "nll_soft": 0.30741759767501897, "ece": 0.09543892771005633, "mean_confidence": 0.9135349568724632, "confident_error_rate": 0.02 }, "amazon_polarity": { "count": 100, "accuracy": 0.97, "brier": 0.06162859960932998, "brier_soft": 0.06162859960932998, "nll": 0.13110608215243919, "nll_soft": 0.13110608215243919, "ece": 0.03008812248706818, "mean_confidence": 0.9551744455099106, "confident_error_rate": 0.03 }, "amazon_stars": { "count": 100, "accuracy": 0.53, "brier": 0.5461979259182673, "brier_soft": 0.5461979259182673, "nll": 0.9164086895783538, "nll_soft": 0.9164086895783538, "ece": 0.11809196531772617, "mean_confidence": 0.5753743493556976, "confident_error_rate": 0.0, "score_mae": 0.5362134778685868 }, "anli": { "count": 100, "accuracy": 0.39, "brier": 0.9722760942391393, "brier_soft": 0.9722760942391393, "nll": 1.8542452552224933, "nll_soft": 1.8542452552224933, "ece": 0.44132104188203813, "mean_confidence": 0.7956731787323952, "confident_error_rate": 0.23 }, "arc_challenge": { "count": 100, "accuracy": 0.36, "brier": 0.778481672987934, "brier_soft": 0.778481672987934, "nll": 1.4337150394138498, "nll_soft": 1.4337150394138498, "ece": 0.20447848409414293, "mean_confidence": 0.5459534320235252, "confident_error_rate": 0.0 }, "arc_easy": { "count": 100, "accuracy": 0.45, "brier": 0.6407232552939703, "brier_soft": 0.6407232552939703, "nll": 1.2136572969992905, "nll_soft": 1.2136572969992905, "ece": 0.12803085684776305, "mean_confidence": 0.5452363789081573, "confident_error_rate": 0.01 }, "banking77": { "count": 100, "accuracy": 0.81, "brier": 0.3814467708771144, "brier_soft": 0.3814467708771144, "nll": 1.0443211262890961, "nll_soft": 1.0443211262890961, "ece": 0.3126603952050208, "mean_confidence": 0.4973396047949791, "confident_error_rate": 0.0 }, "boolq": { "count": 125, "accuracy": 0.568, "brier": 0.6168742406476001, "brier_soft": 0.6168742406476001, "nll": 0.8592622690829735, "nll_soft": 0.8592622690829735, "ece": 0.2938120722770691, "mean_confidence": 0.8219961438179016, "confident_error_rate": 0.032, "evidence_auc": 0.9706666666666667 }, "cb": { "count": 28, "accuracy": 0.9285714285714286, "brier": 0.13537618609721863, "brier_soft": 0.13537618609721863, "nll": 0.2816298666367622, "nll_soft": 0.2816298666367622, "ece": 0.11475402755396709, "mean_confidence": 0.9356890469789505, "confident_error_rate": 0.03571428571428571 }, "civil_comments_severity": { "count": 100, "accuracy": 0.81, "brier": 0.28384330937032604, "brier_soft": 0.17998165817214293, "nll": 0.5338759677740593, "nll_soft": 0.6298814779743555, "ece": 0.14874677449464796, "mean_confidence": 0.6996492806077004, "confident_error_rate": 0.0, "score_mae": 0.39768598369671965 }, "civil_comments_toxic": { "count": 100, "accuracy": 0.94, "brier": 0.09920447825135499, "brier_soft": 0.03237408913021391, "nll": 0.19394092520219666, "nll_soft": 0.2773696142795253, "ece": 0.08483366131782531, "mean_confidence": 0.8621529483795166, "confident_error_rate": 0.0 }, "clinc150": { "count": 100, "accuracy": 0.8, "brier": 0.4848727215372612, "brier_soft": 0.4848727215372612, "nll": 1.4845052623480186, "nll_soft": 1.4845052623480186, "ece": 0.4221790521405637, "mean_confidence": 0.37782094785943626, "confident_error_rate": 0.0 }, "cola": { "count": 100, "accuracy": 0.58, "brier": 0.4671416305229769, "brier_soft": 0.4671416305229769, "nll": 0.6581302967113183, "nll_soft": 0.6581302967113183, "ece": 0.10910327613353732, "mean_confidence": 0.6739702433347702, "confident_error_rate": 0.0 }, "commonsense_qa": { "count": 100, "accuracy": 0.42, "brier": 0.7026358240551468, "brier_soft": 0.7026358240551468, "nll": 1.3662261855444524, "nll_soft": 1.3662261855444524, "ece": 0.13439485043287275, "mean_confidence": 0.510890220105648, "confident_error_rate": 0.0 }, "cosmos_qa": { "count": 120, "accuracy": 0.4, "brier": 0.674820119653572, "brier_soft": 0.674820119653572, "nll": 1.2437770795447451, "nll_soft": 1.2437770795447451, "ece": 0.14261294777194658, "mean_confidence": 0.5219608165323735, "confident_error_rate": 0.0, "evidence_auc": 0.836875 }, "dates": { "count": 300, "accuracy": 0.7633333333333333, "brier": 0.2819022563327028, "brier_soft": 0.2819022563327028, "nll": 0.4583224548625654, "nll_soft": 0.4583224548625654, "ece": 0.06106376767158511, "mean_confidence": 0.7644337038199107, "confident_error_rate": 0.0033333333333333335, "score_mae": 0.45998074431250346 }, "dbpedia": { "count": 100, "accuracy": 0.99, "brier": 0.015198128030094994, "brier_soft": 0.015198128030094994, "nll": 0.06724704860844338, "nll_soft": 0.06724704860844338, "ece": 0.057107390761375415, "mean_confidence": 0.9441341024637222, "confident_error_rate": 0.0 }, "dynasent": { "count": 100, "accuracy": 0.53, "brier": 0.5435009917424021, "brier_soft": 0.40578378452816594, "nll": 0.9203439216635296, "nll_soft": 0.95189629595177, "ece": 0.12150101184844973, "mean_confidence": 0.5809222316741943, "confident_error_rate": 0.0, "score_mae": 0.590812379643321 }, "enron_spam": { "count": 100, "accuracy": 0.94, "brier": 0.08228255110990408, "brier_soft": 0.08228255110990408, "nll": 0.15493662205431416, "nll_soft": 0.15493662205431416, "ece": 0.043621286749839804, "mean_confidence": 0.9467802709341049, "confident_error_rate": 0.02 }, "go_emotions": { "count": 100, "accuracy": 0.54, "brier": 0.6768268805616047, "brier_soft": 0.25233950625259904, "nll": 1.7380711253413985, "nll_soft": 2.0785070377760406, "ece": 0.24022679880261422, "mean_confidence": 0.29977320119738576, "confident_error_rate": 0.0 }, "hellaswag": { "count": 100, "accuracy": 0.49, "brier": 0.6818457284598648, "brier_soft": 0.6818457284598648, "nll": 1.27200660086301, "nll_soft": 1.27200660086301, "ece": 0.1305046308040619, "mean_confidence": 0.5641709685325622, "confident_error_rate": 0.0 }, "imdb": { "count": 100, "accuracy": 0.95, "brier": 0.08363715307563817, "brier_soft": 0.08363715307563817, "nll": 0.16551026463952032, "nll_soft": 0.16551026463952032, "ece": 0.026311837434768674, "mean_confidence": 0.9574666857719422, "confident_error_rate": 0.03 }, "ledgar": { "count": 100, "accuracy": 0.75, "brier": 0.42560135728608983, "brier_soft": 0.42560135728608983, "nll": 1.3302157486233597, "nll_soft": 1.3302157486233597, "ece": 0.23709699064493178, "mean_confidence": 0.5129030093550682, "confident_error_rate": 0.0 }, "liar": { "count": 100, "accuracy": 0.32, "brier": 0.7427052798517789, "brier_soft": 0.7427052798517789, "nll": 1.4964320047893989, "nll_soft": 1.4964320047893989, "ece": 0.055253805071115485, "mean_confidence": 0.3017434249818325, "confident_error_rate": 0.0, "score_mae": 0.9733948946371674 }, "measuring_hate_speech": { "count": 100, "accuracy": 0.83, "brier": 0.3177919283014086, "brier_soft": 0.16750927245698513, "nll": 0.5942829577379074, "nll_soft": 0.7062310284569752, "ece": 0.21628696441650394, "mean_confidence": 0.6214796364307403, "confident_error_rate": 0.0, "score_mae": 0.6045454696938396 }, "medqa": { "count": 100, "accuracy": 0.34, "brier": 0.7671574185751032, "brier_soft": 0.7671574185751032, "nll": 1.4356347250931116, "nll_soft": 1.4356347250931116, "ece": 0.17744469463825224, "mean_confidence": 0.48414860367774964, "confident_error_rate": 0.0 }, "mnli": { "count": 100, "accuracy": 0.78, "brier": 0.3263025000138572, "brier_soft": 0.3263025000138572, "nll": 0.5982821287502831, "nll_soft": 0.5982821287502831, "ece": 0.10382685691118237, "mean_confidence": 0.8509401044249535, "confident_error_rate": 0.06 }, "mrpc": { "count": 100, "accuracy": 0.81, "brier": 0.27720433744929446, "brier_soft": 0.27720433744929446, "nll": 0.43150783049960595, "nll_soft": 0.43150783049960595, "ece": 0.1067017036676407, "mean_confidence": 0.856079712510109, "confident_error_rate": 0.03 }, "multirc": { "count": 125, "accuracy": 0.664, "brier": 0.5262665529453465, "brier_soft": 0.5262665529453465, "nll": 0.7910576977190109, "nll_soft": 0.7910576977190109, "ece": 0.20780441761016843, "mean_confidence": 0.833528317451477, "confident_error_rate": 0.136, "evidence_auc": 0.6994666666666667 }, "openbookqa": { "count": 100, "accuracy": 0.52, "brier": 0.6059996266277589, "brier_soft": 0.6059996266277589, "nll": 1.1493342555724386, "nll_soft": 1.1493342555724386, "ece": 0.08220899760723116, "mean_confidence": 0.5423529654741287, "confident_error_rate": 0.01 }, "piqa": { "count": 100, "accuracy": 0.61, "brier": 0.5129767773118842, "brier_soft": 0.5129767773118842, "nll": 0.7296690239966398, "nll_soft": 0.7296690239966398, "ece": 0.14158266067504877, "mean_confidence": 0.6558456552028656, "confident_error_rate": 0.02 }, "policy_application": { "count": 150, "accuracy": 0.7333333333333333, "brier": 0.3508971131550242, "brier_soft": 0.3508971131550242, "nll": 0.5516997947737176, "nll_soft": 0.5516997947737176, "ece": 0.044622519016265856, "mean_confidence": 0.7177301915486654, "confident_error_rate": 0.0, "evidence_auc": 0.5696533682145193 }, "policy_hard": { "count": 300, "accuracy": 0.6966666666666667, "brier": 0.38472149078986073, "brier_soft": 0.38472149078986073, "nll": 0.6128855879532513, "nll_soft": 0.6128855879532513, "ece": 0.053346475064754505, "mean_confidence": 0.7120696005225181, "confident_error_rate": 0.0, "evidence_auc": 0.7445023148148148 }, "qqp": { "count": 100, "accuracy": 0.8, "brier": 0.3034980053199748, "brier_soft": 0.3034980053199748, "nll": 0.4749638870383975, "nll_soft": 0.4749638870383975, "ece": 0.10882365942001343, "mean_confidence": 0.8906454944610596, "confident_error_rate": 0.07 }, "race": { "count": 125, "accuracy": 0.464, "brier": 0.670541267736231, "brier_soft": 0.670541267736231, "nll": 1.2785957499648075, "nll_soft": 1.2785957499648075, "ece": 0.09217529821395876, "mean_confidence": 0.4943129940032959, "confident_error_rate": 0.0, "evidence_auc": 0.7856 }, "rotten_tomatoes": { "count": 100, "accuracy": 0.89, "brier": 0.26950262705978917, "brier_soft": 0.26950262705978917, "nll": 0.43769943345994206, "nll_soft": 0.43769943345994206, "ece": 0.18772910118103028, "mean_confidence": 0.7022708988189698, "confident_error_rate": 0.0 }, "rte": { "count": 100, "accuracy": 0.7, "brier": 0.4417545164022784, "brier_soft": 0.4417545164022784, "nll": 0.6803569597273692, "nll_soft": 0.6803569597273692, "ece": 0.166305216550827, "mean_confidence": 0.8475300312042237, "confident_error_rate": 0.07 }, "scotus": { "count": 100, "accuracy": 0.63, "brier": 0.57591555494716, "brier_soft": 0.57591555494716, "nll": 1.4268808895864786, "nll_soft": 1.4268808895864786, "ece": 0.1622882364690304, "mean_confidence": 0.479786828905344, "confident_error_rate": 0.0 }, "shp": { "count": 100, "accuracy": 0.5, "brier": 0.553928533015661, "brier_soft": 0.16866709422112394, "nll": 0.7553610522259013, "nll_soft": 0.7667537170621911, "ece": 0.22228121280670163, "mean_confidence": 0.6585486650466919, "confident_error_rate": 0.0 }, "sms_spam": { "count": 100, "accuracy": 0.96, "brier": 0.14873415443052604, "brier_soft": 0.14873415443052604, "nll": 0.29604541382027116, "nll_soft": 0.29604541382027116, "ece": 0.18373807787895202, "mean_confidence": 0.7762619221210479, "confident_error_rate": 0.0 }, "snli": { "count": 100, "accuracy": 0.79, "brier": 0.3657109430484185, "brier_soft": 0.3657109430484185, "nll": 0.6824574908252256, "nll_soft": 0.6824574908252256, "ece": 0.1515702518820763, "mean_confidence": 0.8926358130574227, "confident_error_rate": 0.1 }, "social_iqa": { "count": 100, "accuracy": 0.5, "brier": 0.6701157287052262, "brier_soft": 0.6701157287052262, "nll": 1.2110427461546294, "nll_soft": 1.2110427461546294, "ece": 0.21912074834108355, "mean_confidence": 0.68725660353899, "confident_error_rate": 0.04 }, "sst2": { "count": 100, "accuracy": 0.9, "brier": 0.25762914103330403, "brier_soft": 0.25762914103330403, "nll": 0.42289569521733256, "nll_soft": 0.42289569521733256, "ece": 0.182864231467247, "mean_confidence": 0.717135768532753, "confident_error_rate": 0.0 }, "sst5": { "count": 100, "accuracy": 0.56, "brier": 0.5750112386200766, "brier_soft": 0.5750112386200766, "nll": 1.0410791576793528, "nll_soft": 1.0410791576793528, "ece": 0.13715695679187778, "mean_confidence": 0.4760879367589951, "confident_error_rate": 0.0, "score_mae": 0.569280381789431 }, "stance_abortion": { "count": 60, "accuracy": 0.6333333333333333, "brier": 0.4775618932889926, "brier_soft": 0.4775618932889926, "nll": 0.772960340218129, "nll_soft": 0.772960340218129, "ece": 0.14922663221756619, "mean_confidence": 0.7609471683700879, "confident_error_rate": 0.05 }, "stance_atheism": { "count": 60, "accuracy": 0.7, "brier": 0.39241280131421197, "brier_soft": 0.39241280131421197, "nll": 0.6474431235930713, "nll_soft": 0.6474431235930713, "ece": 0.15528117964665092, "mean_confidence": 0.8136728132764498, "confident_error_rate": 0.016666666666666666 }, "stance_feminist": { "count": 60, "accuracy": 0.6833333333333333, "brier": 0.46696109115797474, "brier_soft": 0.46696109115797474, "nll": 0.7586194826479168, "nll_soft": 0.7586194826479168, "ece": 0.17778196881214778, "mean_confidence": 0.7421373461683591, "confident_error_rate": 0.05 }, "stance_hillary": { "count": 60, "accuracy": 0.6666666666666666, "brier": 0.47286586681141635, "brier_soft": 0.47286586681141635, "nll": 0.8005868818543999, "nll_soft": 0.8005868818543999, "ece": 0.15976376285155613, "mean_confidence": 0.7363345429301262, "confident_error_rate": 0.03333333333333333 }, "stsb": { "count": 100, "accuracy": 0.45, "brier": 0.6576776158521066, "brier_soft": 0.3746345989346838, "nll": 1.2299245867552293, "nll_soft": 1.2685160423963167, "ece": 0.10637196481227874, "mean_confidence": 0.523349083662033, "confident_error_rate": 0.0, "score_mae": 0.7103589298564476 }, "subjectivity": { "count": 100, "accuracy": 0.94, "brier": 0.07403801155263033, "brier_soft": 0.07403801155263033, "nll": 0.12967172016931594, "nll_soft": 0.12967172016931594, "ece": 0.06520502805709842, "mean_confidence": 0.9230094337463379, "confident_error_rate": 0.0 }, "swag": { "count": 100, "accuracy": 0.4, "brier": 0.7323198866636677, "brier_soft": 0.7323198866636677, "nll": 1.3764612983642295, "nll_soft": 1.3764612983642295, "ece": 0.2521471258997917, "mean_confidence": 0.6150306752324104, "confident_error_rate": 0.02 }, "trec": { "count": 100, "accuracy": 0.96, "brier": 0.06216060648768522, "brier_soft": 0.06216060648768522, "nll": 0.14523731763009665, "nll_soft": 0.14523731763009665, "ece": 0.07524359405040741, "mean_confidence": 0.9198704010248184, "confident_error_rate": 0.0 }, "tweet_hate": { "count": 100, "accuracy": 0.54, "brier": 0.4864483510328587, "brier_soft": 0.4864483510328587, "nll": 0.6796653519155663, "nll_soft": 0.6796653519155663, "ece": 0.08831145048141475, "mean_confidence": 0.6011694383621216, "confident_error_rate": 0.0 }, "tweet_sentiment": { "count": 100, "accuracy": 0.67, "brier": 0.4380984174109661, "brier_soft": 0.4380984174109661, "nll": 0.729035173618918, "nll_soft": 0.729035173618918, "ece": 0.13779532372951508, "mean_confidence": 0.5552096617221832, "confident_error_rate": 0.0, "score_mae": 0.4073845294490457 }, "vitaminc": { "count": 100, "accuracy": 0.55, "brier": 0.6059178299160775, "brier_soft": 0.6059178299160775, "nll": 1.0501780738072244, "nll_soft": 1.0501780738072244, "ece": 0.20950709015131, "mean_confidence": 0.75950709015131, "confident_error_rate": 0.07 }, "wic": { "count": 100, "accuracy": 0.61, "brier": 0.502306829144201, "brier_soft": 0.502306829144201, "nll": 0.7209994732293087, "nll_soft": 0.7209994732293087, "ece": 0.16384762406349185, "mean_confidence": 0.7613926827907562, "confident_error_rate": 0.04 }, "winogrande": { "count": 100, "accuracy": 0.57, "brier": 0.5063494004660651, "brier_soft": 0.5063494004660651, "nll": 0.7075272167886625, "nll_soft": 0.7075272167886625, "ece": 0.09909610986709594, "mean_confidence": 0.6518526268005371, "confident_error_rate": 0.0 }, "yelp": { "count": 100, "accuracy": 0.62, "brier": 0.43856781301069964, "brier_soft": 0.43856781301069964, "nll": 0.7669727900329802, "nll_soft": 0.7669727900329802, "ece": 0.15332692295312883, "mean_confidence": 0.5684746101498603, "confident_error_rate": 0.0, "score_mae": 0.44304582916665824 } }, "reliability": [ { "bin": "0.0-0.1", "count": 17, "confidence": 0.06687228787032996, "accuracy": 0.35294117647058826 }, { "bin": "0.1-0.2", "count": 42, "confidence": 0.16727064549922943, "accuracy": 0.2619047619047619 }, { "bin": "0.2-0.3", "count": 183, "confidence": 0.25834722043386577, "accuracy": 0.3770491803278688 }, { "bin": "0.3-0.4", "count": 348, "confidence": 0.3574332141841965, "accuracy": 0.3994252873563218 }, { "bin": "0.4-0.5", "count": 677, "confidence": 0.4517646449971586, "accuracy": 0.4564254062038405 }, { "bin": "0.5-0.6", "count": 1026, "confidence": 0.5484959817769235, "accuracy": 0.5487329434697856 }, { "bin": "0.6-0.7", "count": 832, "confidence": 0.6478659043518397, "accuracy": 0.6334134615384616 }, { "bin": "0.7-0.8", "count": 756, "confidence": 0.7482921844594693, "accuracy": 0.7314814814814815 }, { "bin": "0.8-0.9", "count": 814, "confidence": 0.8511582021338348, "accuracy": 0.7457002457002457 }, { "bin": "0.9-1.0", "count": 1418, "confidence": 0.9543087374844571, "accuracy": 0.9160789844851904 } ] }