diff --git "a/evaluation/results.json" "b/evaluation/results.json" new file mode 100644--- /dev/null +++ "b/evaluation/results.json" @@ -0,0 +1,7968 @@ +{ + "selected_epoch": 3, + "epochs": [ + { + "epoch": 1, + "development": { + "raw": { + "questions": 14912, + "accuracy": 0.8959898068669528, + "mean_class_recall": 0.8086847041598828, + "nll": 0.2676199905412643, + "brier": 0.14581168647550916, + "ece15": 0.024724176262894793, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.773806330472103, + "error_rate_at_confidence_0_9": 0.0240055464078343, + "ordinal_expected_level_mae": 0.00949861612167232, + "tasks": { + "agentdojo_tool_outputs:agentdojo_boundary_action": { + "questions": 222, + "accuracy": 0.9819819819819819, + "mean_class_recall": 0.9670930232558139, + "nll": 0.06952124509948641, + "brier": 0.033360936223710204, + "ece15": 0.01963763286720348, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9774774774774775, + "error_rate_at_confidence_0_9": 0.013824884792626729, + "ordinal_expected_level_mae": null + }, + "agentdojo_tool_outputs:agentdojo_embedded_injection": { + "questions": 222, + "accuracy": 0.9864864864864865, + "mean_class_recall": 0.97, + "nll": 0.06796588080667874, + "brier": 0.02442578405456454, + "ece15": 0.013534851962772593, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.972972972972973, + "error_rate_at_confidence_0_9": 0.009259259259259259, + "ordinal_expected_level_mae": null + }, + "byzfl_mnist:fl_attack_generated_update": { + "questions": 1400, + "accuracy": 0.9742857142857143, + "mean_class_recall": 0.9504464285714286, + "nll": 0.07440929350732509, + "brier": 0.03861493679124129, + "ece15": 0.01061565077999396, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.925, + "error_rate_at_confidence_0_9": 0.008494208494208495, + "ordinal_expected_level_mae": null + }, + "ciciot2023:ciciot_attack": { + "questions": 5000, + "accuracy": 0.8334, + "mean_class_recall": 0.7313333333333334, + "nll": 0.4357438451410003, + "brier": 0.2420765591030422, + "ece15": 0.05574236954650583, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.644, + "error_rate_at_confidence_0_9": 0.06677018633540373, + "ordinal_expected_level_mae": null + }, + "injecagent:injecagent_requested_operation_v13": { + "questions": 124, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.004390672226438891, + "brier": 0.0015490359962901127, + "ece15": 0.003915493134373699, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9919354838709677, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_known_logon_action_v16": { + "questions": 328, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.004780661486616687, + "brier": 0.0026501860965206722, + "ece15": 0.0038321931288341792, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9908536585365854, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_observable_policy": { + "questions": 328, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 2.8436047490948972e-05, + "brier": 2.213459113159646e-07, + "ece15": 2.8380504783511674e-05, + "temperature": 1.0, + "coverage_at_confidence_0_9": 1.0, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "ton_iot_network:ton_attack": { + "questions": 2500, + "accuracy": 0.9832, + "mean_class_recall": 0.9856, + "nll": 0.0716936772308886, + "brier": 0.032718644101254024, + "ece15": 0.01640596644430267, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9948, + "error_rate_at_confidence_0_9": 0.01648572577402493, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_evidence_level": { + "questions": 797, + "accuracy": 0.9937264742785445, + "mean_class_recall": 0.9934913409721132, + "nll": 0.0153080496936535, + "brier": 0.009627399097334195, + "ece15": 0.004599932218083785, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9912170639899623, + "error_rate_at_confidence_0_9": 0.0037974683544303796, + "ordinal_expected_level_mae": 0.00949861612167232 + }, + "twins_streamlet:twins_policy_action": { + "questions": 797, + "accuracy": 0.9924717691342535, + "mean_class_recall": 0.9923419156847569, + "nll": 0.017883068917921716, + "brier": 0.011294880553062277, + "ece15": 0.0038184765498114437, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9811794228356336, + "error_rate_at_confidence_0_9": 0.0012787723785166241, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_quorum": { + "questions": 797, + "accuracy": 0.9962358845671268, + "mean_class_recall": 0.995834315034425, + "nll": 0.011957269820791244, + "brier": 0.007393224542639707, + "ece15": 0.00670405372457154, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9836888331242158, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_visible_conflict": { + "questions": 797, + "accuracy": 0.9974905897114178, + "mean_class_recall": 0.9969442527413541, + "nll": 0.010437285979484219, + "brier": 0.0038211519865233618, + "ece15": 0.0009184477173700703, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9962358845671268, + "error_rate_at_confidence_0_9": 0.0012594458438287153, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_message_manipulation": { + "questions": 800, + "accuracy": 0.63, + "mean_class_recall": 0.6354549163606691, + "nll": 0.5876157448193948, + "brier": 0.4142276501377162, + "ece15": 0.031917999077751745, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.10375, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_supported_attack_type_v16": { + "questions": 800, + "accuracy": 0.59875, + "mean_class_recall": 0.4115791153670771, + "nll": 1.2269806519983115, + "brier": 0.571523062943736, + "ece15": 0.08060690424281568, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.11875, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + } + }, + "macro_task_accuracy": 0.9262877786032521, + "macro_task_balanced_accuracy": 0.9021513315229265 + }, + "recalibrated": { + "questions": 14912, + "accuracy": 0.8959898068669528, + "mean_class_recall": 0.8086847041598828, + "nll": 0.2562444405938866, + "brier": 0.1441282781781607, + "ece15": 0.016168446150980925, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.7631437768240343, + "error_rate_at_confidence_0_9": 0.020913884007029877, + "ordinal_expected_level_mae": 0.010086743500138846, + "tasks": { + "agentdojo_tool_outputs:agentdojo_boundary_action": { + "questions": 222, + "accuracy": 0.9819819819819819, + "mean_class_recall": 0.9670930232558139, + "nll": 0.062064251368360995, + "brier": 0.03274641091592082, + "ece15": 0.018974706155035728, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.9774774774774775, + "error_rate_at_confidence_0_9": 0.013824884792626729, + "ordinal_expected_level_mae": null + }, + "agentdojo_tool_outputs:agentdojo_embedded_injection": { + "questions": 222, + "accuracy": 0.9864864864864865, + "mean_class_recall": 0.97, + "nll": 0.06063186595056717, + "brier": 0.02487828836064738, + "ece15": 0.013603842355616328, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.972972972972973, + "error_rate_at_confidence_0_9": 0.009259259259259259, + "ordinal_expected_level_mae": null + }, + "byzfl_mnist:fl_attack_generated_update": { + "questions": 1400, + "accuracy": 0.9742857142857143, + "mean_class_recall": 0.9504464285714286, + "nll": 0.08099031193469938, + "brier": 0.03952213251707266, + "ece15": 0.021719508126191233, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.9007142857142857, + "error_rate_at_confidence_0_9": 0.007137192704203013, + "ordinal_expected_level_mae": null + }, + "ciciot2023:ciciot_attack": { + "questions": 5000, + "accuracy": 0.8334, + "mean_class_recall": 0.7313333333333334, + "nll": 0.4055190039113824, + "brier": 0.23828084117194667, + "ece15": 0.047374338949895636, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.6276, + "error_rate_at_confidence_0_9": 0.058954748247291265, + "ordinal_expected_level_mae": null + }, + "injecagent:injecagent_requested_operation_v13": { + "questions": 124, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.0058641097080354134, + "brier": 0.002007000102514143, + "ece15": 0.0052393541355612895, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.9838709677419355, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_known_logon_action_v16": { + "questions": 328, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.005681917468602561, + "brier": 0.002881636759121932, + "ece15": 0.004649618743350316, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.9908536585365854, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_observable_policy": { + "questions": 328, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 7.967277164193898e-05, + "brier": 1.0603152088487418e-06, + "ece15": 7.940563040823767e-05, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 1.0, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "ton_iot_network:ton_attack": { + "questions": 2500, + "accuracy": 0.9832, + "mean_class_recall": 0.9856, + "nll": 0.06283986398882055, + "brier": 0.031969827800453, + "ece15": 0.015381335792818867, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.992, + "error_rate_at_confidence_0_9": 0.014516129032258065, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_evidence_level": { + "questions": 797, + "accuracy": 0.9937264742785445, + "mean_class_recall": 0.9934913409721132, + "nll": 0.014979010411112437, + "brier": 0.009232024594375448, + "ece15": 0.0059751389669031076, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.986198243412798, + "error_rate_at_confidence_0_9": 0.001272264631043257, + "ordinal_expected_level_mae": 0.010086743500138846 + }, + "twins_streamlet:twins_policy_action": { + "questions": 797, + "accuracy": 0.9924717691342535, + "mean_class_recall": 0.9923419156847569, + "nll": 0.017946444333002753, + "brier": 0.011016825969457151, + "ece15": 0.005758410373299677, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.9799247176913425, + "error_rate_at_confidence_0_9": 0.0012804097311139564, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_quorum": { + "questions": 797, + "accuracy": 0.9962358845671268, + "mean_class_recall": 0.995834315034425, + "nll": 0.012786451464898273, + "brier": 0.00742639513463676, + "ece15": 0.007511975455200797, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.9824341279799247, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_visible_conflict": { + "questions": 797, + "accuracy": 0.9974905897114178, + "mean_class_recall": 0.9969442527413541, + "nll": 0.010010236373401123, + "brier": 0.003879830332877274, + "ece15": 0.0006442096172233073, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.9962358845671268, + "error_rate_at_confidence_0_9": 0.0012594458438287153, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_message_manipulation": { + "questions": 800, + "accuracy": 0.63, + "mean_class_recall": 0.6354549163606691, + "nll": 0.5899608657261348, + "brier": 0.41478182050444184, + "ece15": 0.02797725458606776, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.095, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_supported_attack_type_v16": { + "questions": 800, + "accuracy": 0.59875, + "mean_class_recall": 0.4115791153670771, + "nll": 1.22100156554001, + "brier": 0.5645239942686594, + "ece15": 0.03015245360912201, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.09125, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + } + }, + "macro_task_accuracy": 0.9262877786032521, + "macro_task_balanced_accuracy": 0.9021513315229265 + } + }, + "test": { + "raw": { + "questions": 21634, + "accuracy": 0.8587870943884626, + "mean_class_recall": 0.8070634668409765, + "nll": 0.38882363258516517, + "brier": 0.19821716542849313, + "ece15": 0.010114739193927335, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.6294259036701488, + "error_rate_at_confidence_0_9": 0.016596900932657706, + "ordinal_expected_level_mae": 0.010228305213524986, + "tasks": { + "agentdojo_tool_outputs:agentdojo_boundary_action": { + "questions": 426, + "accuracy": 0.9953051643192489, + "mean_class_recall": 0.9895833333333333, + "nll": 0.025162438436180455, + "brier": 0.012711490628916015, + "ece15": 0.009157764610888798, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9694835680751174, + "error_rate_at_confidence_0_9": 0.002421307506053269, + "ordinal_expected_level_mae": null + }, + "agentdojo_tool_outputs:agentdojo_embedded_injection": { + "questions": 426, + "accuracy": 0.9835680751173709, + "mean_class_recall": 0.9635416666666667, + "nll": 0.039729487219800165, + "brier": 0.024078530771277282, + "ece15": 0.008718119493403046, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9765258215962441, + "error_rate_at_confidence_0_9": 0.007211538461538462, + "ordinal_expected_level_mae": null + }, + "byzfl_mnist:fl_attack_generated_update": { + "questions": 800, + "accuracy": 0.93875, + "mean_class_recall": 0.8984375, + "nll": 0.19453941595728141, + "brier": 0.09689065974959639, + "ece15": 0.027271759841056658, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.87875, + "error_rate_at_confidence_0_9": 0.034139402560455195, + "ordinal_expected_level_mae": null + }, + "ciciot2023:ciciot_attack": { + "questions": 6000, + "accuracy": 0.9085, + "mean_class_recall": 0.8498888888888889, + "nll": 0.30026072378768554, + "brier": 0.1436264364242417, + "ece15": 0.037464827151113794, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.6986666666666667, + "error_rate_at_confidence_0_9": 0.029818702290076337, + "ordinal_expected_level_mae": null + }, + "injecagent:injecagent_requested_operation_v13": { + "questions": 124, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.0009360610136724553, + "brier": 5.531806653087891e-05, + "ece15": 0.0009217275317422124, + "temperature": 1.0, + "coverage_at_confidence_0_9": 1.0, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_known_logon_action_v16": { + "questions": 540, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.0029619998489780613, + "brier": 0.0010650062308243538, + "ece15": 0.0026318040347967494, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9925925925925926, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_observable_policy": { + "questions": 540, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 4.009888988278539e-05, + "brier": 8.791167963328194e-07, + "ece15": 3.98768557623178e-05, + "temperature": 1.0, + "coverage_at_confidence_0_9": 1.0, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "ton_iot_network:ton_attack": { + "questions": 3500, + "accuracy": 0.9882857142857143, + "mean_class_recall": 0.9807619047619047, + "nll": 0.0658259406128984, + "brier": 0.02104448067811733, + "ece15": 0.0069241929022703305, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9845714285714285, + "error_rate_at_confidence_0_9": 0.00783517121300058, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_evidence_level": { + "questions": 659, + "accuracy": 0.9908952959028832, + "mean_class_recall": 0.9899995024628091, + "nll": 0.022081750206618136, + "brier": 0.01158884274796682, + "ece15": 0.006288646106945627, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9954476479514416, + "error_rate_at_confidence_0_9": 0.004573170731707317, + "ordinal_expected_level_mae": 0.010228305213524986 + }, + "twins_streamlet:twins_policy_action": { + "questions": 659, + "accuracy": 0.9939301972685888, + "mean_class_recall": 0.9932998324958123, + "nll": 0.017255345447197507, + "brier": 0.008938281239516815, + "ece15": 0.004558085120676935, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9893778452200304, + "error_rate_at_confidence_0_9": 0.003067484662576687, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_quorum": { + "questions": 659, + "accuracy": 0.9969650986342944, + "mean_class_recall": 0.9966867772750125, + "nll": 0.005807395652175951, + "brier": 0.0034083401253427324, + "ece15": 0.0031822580740001574, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9893778452200304, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_visible_conflict": { + "questions": 659, + "accuracy": 0.9969650986342944, + "mean_class_recall": 0.9949748743718593, + "nll": 0.009270762829542105, + "brier": 0.00610585714141352, + "ece15": 0.0051900114831071945, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9878603945371776, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_message_manipulation": { + "questions": 3321, + "accuracy": 0.6546221017765733, + "mean_class_recall": 0.6604367829458773, + "nll": 0.6040832398786956, + "brier": 0.4230681455327693, + "ece15": 0.012144611565586284, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.08220415537488708, + "error_rate_at_confidence_0_9": 0.014652014652014652, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_supported_attack_type_v16": { + "questions": 3321, + "accuracy": 0.6248118036735922, + "mean_class_recall": 0.3856953745435135, + "nll": 1.250473514719575, + "brier": 0.5523161094538271, + "ece15": 0.04328702573140643, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.1093044263775971, + "error_rate_at_confidence_0_9": 0.10192837465564739, + "ordinal_expected_level_mae": null + } + }, + "macro_task_accuracy": 0.9337570392580401, + "macro_task_balanced_accuracy": 0.9073790312675484 + }, + "recalibrated": { + "questions": 21634, + "accuracy": 0.8587870943884626, + "mean_class_recall": 0.8070634668409765, + "nll": 0.3819588765605003, + "brier": 0.1988268469084605, + "ece15": 0.019530087212558746, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.6154663954885828, + "error_rate_at_confidence_0_9": 0.014344723995493804, + "ordinal_expected_level_mae": 0.01072047331602824, + "tasks": { + "agentdojo_tool_outputs:agentdojo_boundary_action": { + "questions": 426, + "accuracy": 0.9953051643192489, + "mean_class_recall": 0.9895833333333333, + "nll": 0.028102839471687808, + "brier": 0.013352375860260531, + "ece15": 0.011924724016462706, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.9624413145539906, + "error_rate_at_confidence_0_9": 0.0024390243902439024, + "ordinal_expected_level_mae": null + }, + "agentdojo_tool_outputs:agentdojo_embedded_injection": { + "questions": 426, + "accuracy": 0.9835680751173709, + "mean_class_recall": 0.9635416666666667, + "nll": 0.03830754803404818, + "brier": 0.02344796092548543, + "ece15": 0.009297528928361507, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.9694835680751174, + "error_rate_at_confidence_0_9": 0.004842615012106538, + "ordinal_expected_level_mae": null + }, + "byzfl_mnist:fl_attack_generated_update": { + "questions": 800, + "accuracy": 0.93875, + "mean_class_recall": 0.8984375, + "nll": 0.18853618602333774, + "brier": 0.09702707273078204, + "ece15": 0.01936307032837748, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.84125, + "error_rate_at_confidence_0_9": 0.03417533432392273, + "ordinal_expected_level_mae": null + }, + "ciciot2023:ciciot_attack": { + "questions": 6000, + "accuracy": 0.9085, + "mean_class_recall": 0.8498888888888889, + "nll": 0.2843389317505148, + "brier": 0.14586901269531594, + "ece15": 0.04356789248344108, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.6861666666666667, + "error_rate_at_confidence_0_9": 0.028175856205975226, + "ordinal_expected_level_mae": null + }, + "injecagent:injecagent_requested_operation_v13": { + "questions": 124, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.0018322563352568268, + "brier": 0.0001319762188887517, + "ece15": 0.0017975613028342768, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 1.0, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_known_logon_action_v16": { + "questions": 540, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.004015391153194959, + "brier": 0.0013677195263880856, + "ece15": 0.0035859188244034786, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.9888888888888889, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_observable_policy": { + "questions": 540, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 9.895528751872936e-05, + "brier": 3.024028962806258e-06, + "ece15": 9.818509110354407e-05, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 1.0, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "ton_iot_network:ton_attack": { + "questions": 3500, + "accuracy": 0.9882857142857143, + "mean_class_recall": 0.9807619047619047, + "nll": 0.059841713174448447, + "brier": 0.021127495151853498, + "ece15": 0.005242220550012023, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.976, + "error_rate_at_confidence_0_9": 0.007318501170960188, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_evidence_level": { + "questions": 659, + "accuracy": 0.9908952959028832, + "mean_class_recall": 0.9899995024628091, + "nll": 0.020306967381801818, + "brier": 0.011159005956953514, + "ece15": 0.005337269869592108, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.9908952959028832, + "error_rate_at_confidence_0_9": 0.0030627871362940277, + "ordinal_expected_level_mae": 0.01072047331602824 + }, + "twins_streamlet:twins_policy_action": { + "questions": 659, + "accuracy": 0.9939301972685888, + "mean_class_recall": 0.9932998324958123, + "nll": 0.016710459187929856, + "brier": 0.008886576198762068, + "ece15": 0.0032207054419096, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.9893778452200304, + "error_rate_at_confidence_0_9": 0.003067484662576687, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_quorum": { + "questions": 659, + "accuracy": 0.9969650986342944, + "mean_class_recall": 0.9966867772750125, + "nll": 0.006802759775612477, + "brier": 0.003478846491952278, + "ece15": 0.004168012801242965, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.9893778452200304, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_visible_conflict": { + "questions": 659, + "accuracy": 0.9969650986342944, + "mean_class_recall": 0.9949748743718593, + "nll": 0.009405080434222151, + "brier": 0.006029858296029803, + "ece15": 0.004873038962556261, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.9863429438543247, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_message_manipulation": { + "questions": 3321, + "accuracy": 0.6546221017765733, + "mean_class_recall": 0.6604367829458773, + "nll": 0.6050451348525548, + "brier": 0.4236393366516397, + "ece15": 0.022216352122328073, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.0671484492622704, + "error_rate_at_confidence_0_9": 0.008968609865470852, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_supported_attack_type_v16": { + "questions": 3321, + "accuracy": 0.6248118036735922, + "mean_class_recall": 0.3856953745435135, + "nll": 1.2411381163756228, + "brier": 0.5515874786124301, + "ece15": 0.032556668330635724, + "temperature": 1.1759291426711453, + "coverage_at_confidence_0_9": 0.07768744354110207, + "error_rate_at_confidence_0_9": 0.06976744186046512, + "ordinal_expected_level_mae": null + } + }, + "macro_task_accuracy": 0.9337570392580401, + "macro_task_balanced_accuracy": 0.9073790312675484 + } + }, + "general": { + "decision-v7": { + "raw_clean": { + "n": 1200, + "nll": 0.5075688394241022, + "acc": 0.8666666666666667, + "ece": 0.07962904500464703, + "brier": 0.2102660845468813, + "mean_conf": 0.9440260985257297, + "confident_error_rate": 0.0575, + "coverage_at_0_9": 0.8266666666666667, + "accuracy_at_0_9": 0.9304435483870968, + "coverage_at_5pct_error": 0.7891666666666667, + "coverage_at_1pct_error": 0.6058333333333333, + "aurc": 0.028250531410146766, + "error_rate_at_0_9": 0.06955645161290322, + "confidence_bias": 0.07735943185906302, + "top_bins": { + "0.9": { + "n": 992, + "errors": 69, + "error_rate": 0.06955645161290322 + }, + "0.95": { + "n": 931, + "errors": 44, + "error_rate": 0.047261009667024706 + }, + "0.99": { + "n": 853, + "errors": 25, + "error_rate": 0.029308323563892145 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9916666666666667, + "confidence_cutoff": 0.9999431212544798 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.9458333333333333, + "confidence_cutoff": 0.9297460863402224 + } + }, + "score_mae": 0.4607497712522395, + "ranked_probability_score": 0.08554771406167339 + }, + "as_shipped_clean": { + "n": 1200, + "nll": 0.454544651852342, + "acc": 0.8666666666666667, + "ece": 0.06910317787787193, + "brier": 0.20468221280246773, + "mean_conf": 0.9344856098346994, + "confident_error_rate": 0.041666666666666664, + "coverage_at_0_9": 0.795, + "accuracy_at_0_9": 0.9475890985324947, + "coverage_at_5pct_error": 0.7866666666666666, + "coverage_at_1pct_error": 0.6041666666666666, + "aurc": 0.02824430039449196, + "error_rate_at_0_9": 0.05241090146750524, + "confidence_bias": 0.06781894316803272, + "top_bins": { + "0.9": { + "n": 954, + "errors": 50, + "error_rate": 0.05241090146750524 + }, + "0.95": { + "n": 901, + "errors": 37, + "error_rate": 0.041065482796892344 + }, + "0.99": { + "n": 820, + "errors": 16, + "error_rate": 0.01951219512195122 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9916666666666667, + "confidence_cutoff": 0.9997511018421836 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.9458333333333333, + "confidence_cutoff": 0.894833765474474 + } + }, + "score_mae": 0.46716677242673305, + "ranked_probability_score": 0.08353281985680432 + }, + "sources": { + "agnews": { + "n": 240, + "nll": 0.2969571491022943, + "acc": 0.9166666666666666, + "ece": 0.05669702215129781, + "brier": 0.13610080496920665, + "mean_conf": 0.9733636888179642, + "confident_error_rate": 0.04583333333333333, + "coverage_at_0_9": 0.9291666666666667, + "accuracy_at_0_9": 0.9506726457399103, + "coverage_at_5pct_error": 0.9291666666666667, + "coverage_at_1pct_error": 0.7333333333333333, + "aurc": 0.011459961623908478, + "error_rate_at_0_9": 0.04932735426008968, + "confidence_bias": 0.05669702215129757, + "top_bins": { + "0.9": { + "n": 223, + "errors": 11, + "error_rate": 0.04932735426008968 + }, + "0.95": { + "n": 216, + "errors": 10, + "error_rate": 0.046296296296296294 + }, + "0.99": { + "n": 197, + "errors": 4, + "error_rate": 0.02030456852791878 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.999603972973965 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.984375, + "confidence_cutoff": 0.9927218344770156 + } + } + }, + "amazon": { + "n": 80, + "nll": 1.1950196702821447, + "acc": 0.6125, + "ece": 0.13013344623625947, + "brier": 0.5588490719296325, + "mean_conf": 0.7426334462362595, + "confident_error_rate": 0.0625, + "coverage_at_0_9": 0.2625, + "accuracy_at_0_9": 0.7619047619047619, + "coverage_at_5pct_error": 0.075, + "coverage_at_1pct_error": 0.075, + "aurc": 0.2609215557981968, + "error_rate_at_0_9": 0.23809523809523808, + "confidence_bias": 0.13013344623625944, + "top_bins": { + "0.9": { + "n": 21, + "errors": 5, + "error_rate": 0.23809523809523808 + }, + "0.95": { + "n": 13, + "errors": 2, + "error_rate": 0.15384615384615385 + }, + "0.99": { + "n": 4, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.775, + "confidence_cutoff": 0.7941759769280257 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.65625, + "confidence_cutoff": 0.5659829933652708 + } + }, + "score_mae": 0.4892758257511954, + "ranked_probability_score": 0.08858888411388968 + }, + "banking77": { + "n": 80, + "nll": 0.5758902612345038, + "acc": 0.85, + "ece": 0.08719569564847485, + "brier": 0.2407008008106888, + "mean_conf": 0.898073693405655, + "confident_error_rate": 0.0375, + "coverage_at_0_9": 0.75, + "accuracy_at_0_9": 0.95, + "coverage_at_5pct_error": 0.7625, + "coverage_at_1pct_error": 0.625, + "aurc": 0.03238808988486375, + "error_rate_at_0_9": 0.05, + "confidence_bias": 0.04807369340565504, + "top_bins": { + "0.9": { + "n": 60, + "errors": 3, + "error_rate": 0.05 + }, + "0.95": { + "n": 55, + "errors": 2, + "error_rate": 0.03636363636363636 + }, + "0.99": { + "n": 47, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9967902498867545 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.90625, + "confidence_cutoff": 0.8441430830009494 + } + } + }, + "boolq": { + "n": 80, + "nll": 0.5929837671372773, + "acc": 0.875, + "ece": 0.12193170422191033, + "brier": 0.2235795473979305, + "mean_conf": 0.9525887301568842, + "confident_error_rate": 0.0875, + "coverage_at_0_9": 0.85, + "accuracy_at_0_9": 0.8970588235294118, + "coverage_at_5pct_error": 0.6875, + "coverage_at_1pct_error": 0.1, + "aurc": 0.05282386578594207, + "error_rate_at_0_9": 0.10294117647058823, + "confidence_bias": 0.07758873015688417, + "top_bins": { + "0.9": { + "n": 68, + "errors": 7, + "error_rate": 0.10294117647058823 + }, + "0.95": { + "n": 67, + "errors": 7, + "error_rate": 0.1044776119402985 + }, + "0.99": { + "n": 61, + "errors": 4, + "error_rate": 0.06557377049180328 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.975, + "confidence_cutoff": 0.999911295185737 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.921875, + "confidence_cutoff": 0.9794008835333886 + } + } + }, + "contrastive": { + "n": 160, + "nll": 0.05996336403593442, + "acc": 0.96875, + "ece": 0.02417385494863131, + "brier": 0.04256566388357948, + "mean_conf": 0.9929238549486312, + "confident_error_rate": 0.0125, + "coverage_at_0_9": 0.98125, + "accuracy_at_0_9": 0.9872611464968153, + "coverage_at_5pct_error": 1.0, + "coverage_at_1pct_error": 0.975, + "aurc": 0.0005908980274374722, + "error_rate_at_0_9": 0.012738853503184714, + "confidence_bias": 0.024173854948631224, + "top_bins": { + "0.9": { + "n": 157, + "errors": 2, + "error_rate": 0.012738853503184714 + }, + "0.95": { + "n": 155, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 148, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.999992580361535 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 1.0, + "confidence_cutoff": 0.9997815066400773 + } + } + }, + "dbpedia14": { + "n": 80, + "nll": 0.018296334655618373, + "acc": 0.9875, + "ece": 0.010929643397143786, + "brier": 0.013547082606835407, + "mean_conf": 0.9948529231687339, + "confident_error_rate": 0.0, + "coverage_at_0_9": 0.9875, + "accuracy_at_0_9": 1.0, + "coverage_at_5pct_error": 1.0, + "coverage_at_1pct_error": 0.9875, + "aurc": 0.00015625, + "error_rate_at_0_9": 0.0, + "confidence_bias": 0.007352923168733816, + "top_bins": { + "0.9": { + "n": 79, + "errors": 0, + "error_rate": 0.0 + }, + "0.95": { + "n": 77, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 77, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9999968083760991 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 1.0, + "confidence_cutoff": 0.999980508233233 + } + } + }, + "imdb": { + "n": 80, + "nll": 0.06650112598274324, + "acc": 0.9625, + "ece": 0.021127312041626652, + "brier": 0.047311917135916516, + "mean_conf": 0.9836273120416268, + "confident_error_rate": 0.0125, + "coverage_at_0_9": 0.9625, + "accuracy_at_0_9": 0.987012987012987, + "coverage_at_5pct_error": 1.0, + "coverage_at_1pct_error": 0.95, + "aurc": 0.0012680561790530144, + "error_rate_at_0_9": 0.012987012987012988, + "confidence_bias": 0.021127312041626767, + "top_bins": { + "0.9": { + "n": 77, + "errors": 1, + "error_rate": 0.012987012987012988 + }, + "0.95": { + "n": 75, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 71, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9999868991306584 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 1.0, + "confidence_cutoff": 0.999740450053584 + } + } + }, + "mnli": { + "n": 80, + "nll": 0.4800709042620615, + "acc": 0.9125, + "ece": 0.07753013500835038, + "brier": 0.16332522643400862, + "mean_conf": 0.9812306735501618, + "confident_error_rate": 0.0625, + "coverage_at_0_9": 0.925, + "accuracy_at_0_9": 0.9324324324324325, + "coverage_at_5pct_error": 0.8875, + "coverage_at_1pct_error": 0.6375, + "aurc": 0.016122981848020253, + "error_rate_at_0_9": 0.06756756756756757, + "confidence_bias": 0.06873067355016182, + "top_bins": { + "0.9": { + "n": 74, + "errors": 5, + "error_rate": 0.06756756756756757 + }, + "0.95": { + "n": 74, + "errors": 5, + "error_rate": 0.06756756756756757 + }, + "0.99": { + "n": 71, + "errors": 3, + "error_rate": 0.04225352112676056 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.999991586558766 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.96875, + "confidence_cutoff": 0.9998147211094783 + } + } + }, + "sst5": { + "n": 80, + "nll": 1.2448974820413372, + "acc": 0.6125, + "ece": 0.22505814089878196, + "brier": 0.5919475867934173, + "mean_conf": 0.8141449665678422, + "confident_error_rate": 0.0625, + "coverage_at_0_9": 0.2875, + "accuracy_at_0_9": 0.782608695652174, + "coverage_at_5pct_error": 0.0875, + "coverage_at_1pct_error": 0.0875, + "aurc": 0.30028061666521133, + "error_rate_at_0_9": 0.21739130434782608, + "confidence_bias": 0.20164496656784214, + "top_bins": { + "0.9": { + "n": 23, + "errors": 5, + "error_rate": 0.21739130434782608 + }, + "0.95": { + "n": 8, + "errors": 1, + "error_rate": 0.125 + }, + "0.99": { + "n": 0, + "errors": 0, + "error_rate": null + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.6, + "confidence_cutoff": 0.855339285968297 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.625, + "confidence_cutoff": 0.7133087914054365 + } + }, + "score_mae": 0.4884492482076522, + "ranked_probability_score": 0.08971730097335581 + }, + "trec": { + "n": 80, + "nll": 0.28480951015184036, + "acc": 0.9625, + "ece": 0.030856363792743223, + "brier": 0.058704890234289875, + "mean_conf": 0.9903723975747134, + "confident_error_rate": 0.025, + "coverage_at_0_9": 0.975, + "accuracy_at_0_9": 0.9743589743589743, + "coverage_at_5pct_error": 1.0, + "coverage_at_1pct_error": 0.325, + "aurc": 0.015881097479045377, + "error_rate_at_0_9": 0.02564102564102564, + "confidence_bias": 0.02787239757471338, + "top_bins": { + "0.9": { + "n": 78, + "errors": 2, + "error_rate": 0.02564102564102564 + }, + "0.95": { + "n": 78, + "errors": 2, + "error_rate": 0.02564102564102564 + }, + "0.99": { + "n": 77, + "errors": 2, + "error_rate": 0.025974025974025976 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.975, + "confidence_cutoff": 0.9999910962505757 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.984375, + "confidence_cutoff": 0.9999026419356177 + } + } + }, + "yelp": { + "n": 160, + "nll": 0.6744512733294261, + "acc": 0.76875, + "ece": 0.11010641434949675, + "brier": 0.33941666300975865, + "mean_conf": 0.8769106142337293, + "confident_error_rate": 0.05625, + "coverage_at_0_9": 0.5875, + "accuracy_at_0_9": 0.9042553191489362, + "coverage_at_5pct_error": 0.425, + "coverage_at_1pct_error": 0.16875, + "aurc": 0.08692888800847512, + "error_rate_at_0_9": 0.09574468085106383, + "confidence_bias": 0.10816061423372925, + "top_bins": { + "0.9": { + "n": 94, + "errors": 9, + "error_rate": 0.09574468085106383 + }, + "0.95": { + "n": 83, + "errors": 8, + "error_rate": 0.0963855421686747 + }, + "0.99": { + "n": 67, + "errors": 3, + "error_rate": 0.04477611940298507 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.925, + "confidence_cutoff": 0.9744081429606993 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.84375, + "confidence_cutoff": 0.7335309795017884 + } + }, + "score_mae": 0.4237752433213515, + "ranked_probability_score": 0.07229227448316741 + } + }, + "types": { + "choice": { + "n": 480, + "nll": 0.3107149818460764, + "acc": 0.93125, + "ece": 0.04989938310601072, + "brier": 0.11702630095906619, + "mean_conf": 0.9725961663793784, + "confident_error_rate": 0.03333333333333333, + "coverage_at_0_9": 0.925, + "accuracy_at_0_9": 0.963963963963964, + "coverage_at_5pct_error": 0.95, + "coverage_at_1pct_error": 0.8333333333333334, + "aurc": 0.011065724252754237, + "error_rate_at_0_9": 0.036036036036036036, + "confidence_bias": 0.04134616637937838, + "top_bins": { + "0.9": { + "n": 444, + "errors": 16, + "error_rate": 0.036036036036036036 + }, + "0.95": { + "n": 436, + "errors": 15, + "error_rate": 0.034403669724770644 + }, + "0.99": { + "n": 415, + "errors": 7, + "error_rate": 0.016867469879518072 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9958333333333333, + "confidence_cutoff": 0.9999830700690301 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.9895833333333334, + "confidence_cutoff": 0.9987196152255894 + } + } + }, + "noul": { + "n": 480, + "nll": 0.2523830232779272, + "acc": 0.925, + "ece": 0.04746414517655202, + "brier": 0.115343937114107, + "mean_conf": 0.9724641451765519, + "confident_error_rate": 0.0375, + "coverage_at_0_9": 0.91875, + "accuracy_at_0_9": 0.9591836734693877, + "coverage_at_5pct_error": 0.94375, + "coverage_at_1pct_error": 0.7041666666666667, + "aurc": 0.015342193433417054, + "error_rate_at_0_9": 0.04081632653061224, + "confidence_bias": 0.04746414517655184, + "top_bins": { + "0.9": { + "n": 441, + "errors": 18, + "error_rate": 0.04081632653061224 + }, + "0.95": { + "n": 427, + "errors": 14, + "error_rate": 0.03278688524590164 + }, + "0.99": { + "n": 395, + "errors": 8, + "error_rate": 0.020253164556962026 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9958333333333333, + "confidence_cutoff": 0.9998511569646503 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.9791666666666666, + "confidence_cutoff": 0.9945634588215609 + } + } + }, + "score": { + "n": 240, + "nll": 1.146527249013703, + "acc": 0.6208333333333333, + "ece": 0.16277510247546034, + "brier": 0.5586705878659922, + "mean_conf": 0.7823074260616364, + "confident_error_rate": 0.06666666666666667, + "coverage_at_0_9": 0.2875, + "accuracy_at_0_9": 0.7681159420289855, + "coverage_at_5pct_error": 0.016666666666666666, + "coverage_at_1pct_error": 0.016666666666666666, + "aurc": 0.2783513871168549, + "error_rate_at_0_9": 0.2318840579710145, + "confidence_bias": 0.16147409272830304, + "top_bins": { + "0.9": { + "n": 69, + "errors": 16, + "error_rate": 0.2318840579710145 + }, + "0.95": { + "n": 38, + "errors": 8, + "error_rate": 0.21052631578947367 + }, + "0.99": { + "n": 10, + "errors": 1, + "error_rate": 0.1 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.7166666666666667, + "confidence_cutoff": 0.8132439678266241 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.6614583333333334, + "confidence_cutoff": 0.6191827053278941 + } + }, + "score_mae": 0.46716677242673305, + "ranked_probability_score": 0.08353281985680432 + } + } + }, + "transfer-v4": { + "raw_clean": { + "n": 656, + "nll": 0.7060605455091327, + "acc": 0.8384146341463414, + "ece": 0.10225794465378485, + "brier": 0.26913580782629415, + "mean_conf": 0.936023112192006, + "confident_error_rate": 0.08689024390243902, + "coverage_at_0_9": 0.8246951219512195, + "accuracy_at_0_9": 0.8946395563770795, + "coverage_at_5pct_error": 0.5975609756097561, + "coverage_at_1pct_error": 0.38109756097560976, + "aurc": 0.04670337348873893, + "error_rate_at_0_9": 0.10536044362292052, + "confidence_bias": 0.09760847804566453, + "top_bins": { + "0.9": { + "n": 541, + "errors": 57, + "error_rate": 0.10536044362292052 + }, + "0.95": { + "n": 508, + "errors": 46, + "error_rate": 0.09055118110236221 + }, + "0.99": { + "n": 449, + "errors": 28, + "error_rate": 0.062360801781737196 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9695121951219512, + "confidence_cutoff": 0.9996513990871982 + }, + "0.8": { + "coverage": 0.8003048780487805, + "accuracy": 0.900952380952381, + "confidence_cutoff": 0.9260639080978317 + } + }, + "score_mae": 0.28732928759135146, + "ranked_probability_score": 0.12507059546399885 + }, + "as_shipped_clean": { + "n": 656, + "nll": 0.6270891070988179, + "acc": 0.8384146341463414, + "ece": 0.09035981505437185, + "brier": 0.26355975613405186, + "mean_conf": 0.9253405484079532, + "confident_error_rate": 0.07469512195121951, + "coverage_at_0_9": 0.7896341463414634, + "accuracy_at_0_9": 0.9054054054054054, + "coverage_at_5pct_error": 0.5960365853658537, + "coverage_at_1pct_error": 0.3795731707317073, + "aurc": 0.046713600417941994, + "error_rate_at_0_9": 0.0945945945945946, + "confidence_bias": 0.08692591426161178, + "top_bins": { + "0.9": { + "n": 518, + "errors": 49, + "error_rate": 0.0945945945945946 + }, + "0.95": { + "n": 494, + "errors": 44, + "error_rate": 0.08906882591093117 + }, + "0.99": { + "n": 413, + "errors": 23, + "error_rate": 0.05569007263922518 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9695121951219512, + "confidence_cutoff": 0.9988538098904463 + }, + "0.8": { + "coverage": 0.8003048780487805, + "accuracy": 0.900952380952381, + "confidence_cutoff": 0.8926384237014877 + } + }, + "score_mae": 0.28661481608517747, + "ranked_probability_score": 0.1219924564403267 + }, + "sources": { + "composition_holdout": { + "n": 96, + "nll": 0.7716759343811087, + "acc": 0.8854166666666666, + "ece": 0.13161705342878371, + "brier": 0.2418445379901801, + "mean_conf": 0.9798888855966136, + "confident_error_rate": 0.11458333333333333, + "coverage_at_0_9": 0.9479166666666666, + "accuracy_at_0_9": 0.8791208791208791, + "coverage_at_5pct_error": 0.6875, + "coverage_at_1pct_error": 0.6666666666666666, + "aurc": 0.02787241712770423, + "error_rate_at_0_9": 0.12087912087912088, + "confidence_bias": 0.09447221892994695, + "top_bins": { + "0.9": { + "n": 91, + "errors": 11, + "error_rate": 0.12087912087912088 + }, + "0.95": { + "n": 90, + "errors": 11, + "error_rate": 0.12222222222222222 + }, + "0.99": { + "n": 86, + "errors": 11, + "error_rate": 0.12790697674418605 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.999944022537299 + }, + "0.8": { + "coverage": 0.8020833333333334, + "accuracy": 0.948051948051948, + "confidence_cutoff": 0.9994106055377326 + } + } + }, + "contrastive": { + "n": 80, + "nll": 0.5084316515078089, + "acc": 0.85, + "ece": 0.11914860470823861, + "brier": 0.24157018604381048, + "mean_conf": 0.9691486047082385, + "confident_error_rate": 0.0875, + "coverage_at_0_9": 0.9, + "accuracy_at_0_9": 0.9027777777777778, + "coverage_at_5pct_error": 0.85, + "coverage_at_1pct_error": 0.8, + "aurc": 0.018225744933534453, + "error_rate_at_0_9": 0.09722222222222222, + "confidence_bias": 0.11914860470823851, + "top_bins": { + "0.9": { + "n": 72, + "errors": 7, + "error_rate": 0.09722222222222222 + }, + "0.95": { + "n": 72, + "errors": 7, + "error_rate": 0.09722222222222222 + }, + "0.99": { + "n": 68, + "errors": 3, + "error_rate": 0.04411764705882353 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9999840553223158 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 1.0, + "confidence_cutoff": 0.9992294841723544 + } + }, + "score_mae": 0.28661481608517747, + "ranked_probability_score": 0.1219924564403267 + }, + "emotion": { + "n": 80, + "nll": 1.1842140096136198, + "acc": 0.6625, + "ece": 0.16344433654532525, + "brier": 0.49695454986128207, + "mean_conf": 0.7770470897943043, + "confident_error_rate": 0.075, + "coverage_at_0_9": 0.4375, + "accuracy_at_0_9": 0.8285714285714286, + "coverage_at_5pct_error": 0.075, + "coverage_at_1pct_error": 0.075, + "aurc": 0.18204891186299516, + "error_rate_at_0_9": 0.17142857142857143, + "confidence_bias": 0.11454708979430428, + "top_bins": { + "0.9": { + "n": 35, + "errors": 6, + "error_rate": 0.17142857142857143 + }, + "0.95": { + "n": 30, + "errors": 4, + "error_rate": 0.13333333333333333 + }, + "0.99": { + "n": 12, + "errors": 1, + "error_rate": 0.08333333333333333 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.8, + "confidence_cutoff": 0.8455177052439261 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.734375, + "confidence_cutoff": 0.5895563704278984 + } + } + }, + "mmlu": { + "n": 80, + "nll": 0.8811102729379066, + "acc": 0.7125, + "ece": 0.1128354245761854, + "brier": 0.4333401015309152, + "mean_conf": 0.8253354245761854, + "confident_error_rate": 0.05, + "coverage_at_0_9": 0.475, + "accuracy_at_0_9": 0.8947368421052632, + "coverage_at_5pct_error": 0.3625, + "coverage_at_1pct_error": 0.3125, + "aurc": 0.11284549915448887, + "error_rate_at_0_9": 0.10526315789473684, + "confidence_bias": 0.11283542457618534, + "top_bins": { + "0.9": { + "n": 38, + "errors": 4, + "error_rate": 0.10526315789473684 + }, + "0.95": { + "n": 32, + "errors": 2, + "error_rate": 0.0625 + }, + "0.99": { + "n": 19, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9, + "confidence_cutoff": 0.8934639334502172 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.796875, + "confidence_cutoff": 0.6529787827885164 + } + } + }, + "paws": { + "n": 80, + "nll": 0.9149710000217235, + "acc": 0.8, + "ece": 0.15742064884121132, + "brier": 0.3163214505434944, + "mean_conf": 0.9481862302428528, + "confident_error_rate": 0.125, + "coverage_at_0_9": 0.8625, + "accuracy_at_0_9": 0.855072463768116, + "coverage_at_5pct_error": 0.275, + "coverage_at_1pct_error": 0.175, + "aurc": 0.09027125896066751, + "error_rate_at_0_9": 0.14492753623188406, + "confidence_bias": 0.1481862302428527, + "top_bins": { + "0.9": { + "n": 69, + "errors": 10, + "error_rate": 0.14492753623188406 + }, + "0.95": { + "n": 69, + "errors": 10, + "error_rate": 0.14492753623188406 + }, + "0.99": { + "n": 61, + "errors": 6, + "error_rate": 0.09836065573770492 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9, + "confidence_cutoff": 0.9997018073599228 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.890625, + "confidence_cutoff": 0.9816976299326602 + } + } + }, + "qnli": { + "n": 80, + "nll": 0.3010750024235448, + "acc": 0.925, + "ece": 0.08174310999100654, + "brier": 0.13992055357407857, + "mean_conf": 0.9736296740922775, + "confident_error_rate": 0.0625, + "coverage_at_0_9": 0.925, + "accuracy_at_0_9": 0.9324324324324325, + "coverage_at_5pct_error": 0.8625, + "coverage_at_1pct_error": 0.625, + "aurc": 0.014993418347854443, + "error_rate_at_0_9": 0.06756756756756757, + "confidence_bias": 0.048629674092277475, + "top_bins": { + "0.9": { + "n": 74, + "errors": 5, + "error_rate": 0.06756756756756757 + }, + "0.95": { + "n": 71, + "errors": 4, + "error_rate": 0.056338028169014086 + }, + "0.99": { + "n": 64, + "errors": 2, + "error_rate": 0.03125 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9996265539820621 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.96875, + "confidence_cutoff": 0.9930251162586794 + } + } + }, + "sciq": { + "n": 80, + "nll": 0.007314621146129305, + "acc": 1.0, + "ece": 0.006816864281393114, + "brier": 0.0015733066903292278, + "mean_conf": 0.993183135718607, + "confident_error_rate": 0.0, + "coverage_at_0_9": 0.975, + "accuracy_at_0_9": 1.0, + "coverage_at_5pct_error": 1.0, + "coverage_at_1pct_error": 1.0, + "aurc": 0.0, + "error_rate_at_0_9": 0.0, + "confidence_bias": -0.0068168642813930536, + "top_bins": { + "0.9": { + "n": 78, + "errors": 0, + "error_rate": 0.0 + }, + "0.95": { + "n": 77, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 72, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9999608059905641 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 1.0, + "confidence_cutoff": 0.9995954309046333 + } + } + }, + "tweet_offensive": { + "n": 80, + "nll": 0.4190029993022442, + "acc": 0.8625, + "ece": 0.08458362708432104, + "brier": 0.2412964064670991, + "mean_conf": 0.9253956750968154, + "confident_error_rate": 0.075, + "coverage_at_0_9": 0.7625, + "accuracy_at_0_9": 0.9016393442622951, + "coverage_at_5pct_error": 0.575, + "coverage_at_1pct_error": 0.5, + "aurc": 0.04870527959080802, + "error_rate_at_0_9": 0.09836065573770492, + "confidence_bias": 0.06289567509681537, + "top_bins": { + "0.9": { + "n": 61, + "errors": 6, + "error_rate": 0.09836065573770492 + }, + "0.95": { + "n": 53, + "errors": 6, + "error_rate": 0.11320754716981132 + }, + "0.99": { + "n": 31, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9833703279193756 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.890625, + "confidence_cutoff": 0.891081054907394 + } + } + } + }, + "types": { + "choice": { + "n": 336, + "nll": 0.7139642916559491, + "acc": 0.8184523809523809, + "ece": 0.08621197602515834, + "brier": 0.29097176277874826, + "mean_conf": 0.8979600744773888, + "confident_error_rate": 0.0625, + "coverage_at_0_9": 0.7202380952380952, + "accuracy_at_0_9": 0.9132231404958677, + "coverage_at_5pct_error": 0.5208333333333334, + "coverage_at_1pct_error": 0.3958333333333333, + "aurc": 0.05304090653277752, + "error_rate_at_0_9": 0.08677685950413223, + "confidence_bias": 0.07950769352500786, + "top_bins": { + "0.9": { + "n": 242, + "errors": 21, + "error_rate": 0.08677685950413223 + }, + "0.95": { + "n": 229, + "errors": 17, + "error_rate": 0.07423580786026202 + }, + "0.99": { + "n": 189, + "errors": 12, + "error_rate": 0.06349206349206349 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9702380952380952, + "confidence_cutoff": 0.9970506958588666 + }, + "0.8": { + "coverage": 0.8005952380952381, + "accuracy": 0.895910780669145, + "confidence_cutoff": 0.7834280359430882 + } + } + }, + "noul": { + "n": 280, + "nll": 0.4671583132891337, + "acc": 0.8821428571428571, + "ece": 0.07465765160771948, + "brier": 0.1992966887782935, + "mean_conf": 0.9563447099149558, + "confident_error_rate": 0.075, + "coverage_at_0_9": 0.8714285714285714, + "accuracy_at_0_9": 0.9139344262295082, + "coverage_at_5pct_error": 0.7642857142857142, + "coverage_at_1pct_error": 0.26785714285714285, + "aurc": 0.03774979623265823, + "error_rate_at_0_9": 0.0860655737704918, + "confidence_bias": 0.07420185277209868, + "top_bins": { + "0.9": { + "n": 244, + "errors": 21, + "error_rate": 0.0860655737704918 + }, + "0.95": { + "n": 233, + "errors": 20, + "error_rate": 0.08583690987124463 + }, + "0.99": { + "n": 196, + "errors": 8, + "error_rate": 0.04081632653061224 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9642857142857143, + "confidence_cutoff": 0.9991586798493963 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.9375, + "confidence_cutoff": 0.964939495122677 + } + } + }, + "score": { + "n": 40, + "nll": 1.0168531134867067, + "acc": 0.7, + "ece": 0.23830739887567784, + "brier": 0.48314037180891034, + "mean_conf": 0.9383073988756777, + "confident_error_rate": 0.175, + "coverage_at_0_9": 0.8, + "accuracy_at_0_9": 0.78125, + "coverage_at_5pct_error": 0.65, + "coverage_at_1pct_error": 0.6, + "aurc": 0.07895709505044536, + "error_rate_at_0_9": 0.21875, + "confidence_bias": 0.23830739887567776, + "top_bins": { + "0.9": { + "n": 32, + "errors": 7, + "error_rate": 0.21875 + }, + "0.95": { + "n": 32, + "errors": 7, + "error_rate": 0.21875 + }, + "0.99": { + "n": 28, + "errors": 3, + "error_rate": 0.10714285714285714 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9997651676938517 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.78125, + "confidence_cutoff": 0.9590969197346936 + } + }, + "score_mae": 0.28661481608517747, + "ranked_probability_score": 0.1219924564403267 + } + } + } + }, + "temperature": 1.1759291426711453 + }, + { + "epoch": 2, + "development": { + "raw": { + "questions": 14912, + "accuracy": 0.9046405579399142, + "mean_class_recall": 0.8351706914084861, + "nll": 0.2609479185046837, + "brier": 0.13871271598030968, + "ece15": 0.02912533581539033, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.811762339055794, + "error_rate_at_confidence_0_9": 0.030318050392399834, + "ordinal_expected_level_mae": 0.007142014228908907, + "tasks": { + "agentdojo_tool_outputs:agentdojo_boundary_action": { + "questions": 222, + "accuracy": 0.9774774774774775, + "mean_class_recall": 0.9570930232558139, + "nll": 0.0857490162426134, + "brier": 0.03780111867446018, + "ece15": 0.020326742033581136, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9864864864864865, + "error_rate_at_confidence_0_9": 0.0136986301369863, + "ordinal_expected_level_mae": null + }, + "agentdojo_tool_outputs:agentdojo_embedded_injection": { + "questions": 222, + "accuracy": 0.9819819819819819, + "mean_class_recall": 0.96, + "nll": 0.08583505640070498, + "brier": 0.03170617674969949, + "ece15": 0.018778574788420183, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9819819819819819, + "error_rate_at_confidence_0_9": 0.009174311926605505, + "ordinal_expected_level_mae": null + }, + "byzfl_mnist:fl_attack_generated_update": { + "questions": 1400, + "accuracy": 0.9828571428571429, + "mean_class_recall": 0.9678571428571429, + "nll": 0.05038447430212686, + "brier": 0.026644462415551518, + "ece15": 0.006643072262975827, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9757142857142858, + "error_rate_at_confidence_0_9": 0.008784773060029283, + "ordinal_expected_level_mae": null + }, + "ciciot2023:ciciot_attack": { + "questions": 5000, + "accuracy": 0.839, + "mean_class_recall": 0.7132000000000001, + "nll": 0.4567278707638309, + "brier": 0.24289094068797595, + "ece15": 0.07089882316707959, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.7078, + "error_rate_at_confidence_0_9": 0.08194405199208817, + "ordinal_expected_level_mae": null + }, + "injecagent:injecagent_requested_operation_v13": { + "questions": 124, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.0030323687345377927, + "brier": 0.0010147751519150774, + "ece15": 0.0027304101493378944, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9919354838709677, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_known_logon_action_v16": { + "questions": 328, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.0013590718776517526, + "brier": 0.0005853758373684321, + "ece15": 0.001174090997847941, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9969512195121951, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_observable_policy": { + "questions": 328, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 2.706087239227311e-05, + "brier": 4.4631636950527135e-07, + "ece15": 2.6948652833984242e-05, + "temperature": 1.0, + "coverage_at_confidence_0_9": 1.0, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "ton_iot_network:ton_attack": { + "questions": 2500, + "accuracy": 0.9788, + "mean_class_recall": 0.98, + "nll": 0.08755100200107269, + "brier": 0.038158588944742826, + "ece15": 0.017829224144769444, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9896, + "error_rate_at_confidence_0_9": 0.016976556184316895, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_evidence_level": { + "questions": 797, + "accuracy": 0.9949811794228356, + "mean_class_recall": 0.9949406163344321, + "nll": 0.020100883811983803, + "brier": 0.009532705405460478, + "ece15": 0.004664870689987746, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9962358845671268, + "error_rate_at_confidence_0_9": 0.003778337531486146, + "ordinal_expected_level_mae": 0.007142014228908907 + }, + "twins_streamlet:twins_policy_action": { + "questions": 797, + "accuracy": 0.9887076537013801, + "mean_class_recall": 0.9889237691623501, + "nll": 0.029531627570397425, + "brier": 0.01682083664455817, + "ece15": 0.008194444119065863, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9887076537013801, + "error_rate_at_confidence_0_9": 0.0038071065989847717, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_quorum": { + "questions": 797, + "accuracy": 0.9937264742785445, + "mean_class_recall": 0.9922118380062306, + "nll": 0.019942020828876864, + "brier": 0.011733672902765547, + "ece15": 0.006519660642330038, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9912170639899623, + "error_rate_at_confidence_0_9": 0.0037974683544303796, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_visible_conflict": { + "questions": 797, + "accuracy": 0.9974905897114178, + "mean_class_recall": 0.998236331569665, + "nll": 0.016769983022509982, + "brier": 0.0049992664597086935, + "ece15": 0.0021278163552494573, + "temperature": 1.0, + "coverage_at_confidence_0_9": 1.0, + "error_rate_at_confidence_0_9": 0.002509410288582183, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_message_manipulation": { + "questions": 800, + "accuracy": 0.6875, + "mean_class_recall": 0.6966034271725826, + "nll": 0.5142628800857739, + "brier": 0.36155138388149327, + "ece15": 0.02205690591199928, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.22125, + "error_rate_at_confidence_0_9": 0.011299435028248588, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_supported_attack_type_v16": { + "questions": 800, + "accuracy": 0.67375, + "mean_class_recall": 0.5089660282717738, + "nll": 0.998813782244008, + "brier": 0.4775012041931279, + "ece15": 0.03873448735113525, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.20625, + "error_rate_at_confidence_0_9": 0.030303030303030304, + "ordinal_expected_level_mae": null + } + }, + "macro_task_accuracy": 0.9354480356736271, + "macro_task_balanced_accuracy": 0.9112880126164279 + }, + "recalibrated": { + "questions": 14912, + "accuracy": 0.9046405579399142, + "mean_class_recall": 0.8351706914084861, + "nll": 0.24022147566124724, + "brier": 0.13631204830326463, + "ece15": 0.023735839802210217, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.790973712446352, + "error_rate_at_confidence_0_9": 0.02467147096227215, + "ordinal_expected_level_mae": 0.007897347093607133, + "tasks": { + "agentdojo_tool_outputs:agentdojo_boundary_action": { + "questions": 222, + "accuracy": 0.9774774774774775, + "mean_class_recall": 0.9570930232558139, + "nll": 0.06980345492279078, + "brier": 0.036072899559119326, + "ece15": 0.020348852979949178, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.9819819819819819, + "error_rate_at_confidence_0_9": 0.013761467889908258, + "ordinal_expected_level_mae": null + }, + "agentdojo_tool_outputs:agentdojo_embedded_injection": { + "questions": 222, + "accuracy": 0.9819819819819819, + "mean_class_recall": 0.96, + "nll": 0.0697534508038578, + "brier": 0.030632585867415095, + "ece15": 0.018874774396367578, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.9819819819819819, + "error_rate_at_confidence_0_9": 0.009174311926605505, + "ordinal_expected_level_mae": null + }, + "byzfl_mnist:fl_attack_generated_update": { + "questions": 1400, + "accuracy": 0.9828571428571429, + "mean_class_recall": 0.9678571428571429, + "nll": 0.05022700385733144, + "brier": 0.026160310719648357, + "ece15": 0.005133274295293359, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.9642857142857143, + "error_rate_at_confidence_0_9": 0.007407407407407408, + "ordinal_expected_level_mae": null + }, + "ciciot2023:ciciot_attack": { + "questions": 5000, + "accuracy": 0.839, + "mean_class_recall": 0.7132000000000001, + "nll": 0.4007512736870225, + "brier": 0.23542102872463042, + "ece15": 0.05680924935578224, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.6622, + "error_rate_at_confidence_0_9": 0.0664451827242525, + "ordinal_expected_level_mae": null + }, + "injecagent:injecagent_requested_operation_v13": { + "questions": 124, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.00473204279698464, + "brier": 0.0015928649780981342, + "ece15": 0.004243332776407401, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.9919354838709677, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_known_logon_action_v16": { + "questions": 328, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.0022032517878559357, + "brier": 0.0007796053221569698, + "ece15": 0.0019494525692642349, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.9969512195121951, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_observable_policy": { + "questions": 328, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 9.041263403421138e-05, + "brier": 3.609861838766288e-06, + "ece15": 8.949527479169929e-05, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 1.0, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "ton_iot_network:ton_attack": { + "questions": 2500, + "accuracy": 0.9788, + "mean_class_recall": 0.98, + "nll": 0.07409278691965634, + "brier": 0.03695008653192076, + "ece15": 0.013363996653905837, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.984, + "error_rate_at_confidence_0_9": 0.016666666666666666, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_evidence_level": { + "questions": 797, + "accuracy": 0.9949811794228356, + "mean_class_recall": 0.9949406163344321, + "nll": 0.017338830453877865, + "brier": 0.00923947768144349, + "ece15": 0.0037098432118081266, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.9899623588456713, + "error_rate_at_confidence_0_9": 0.0038022813688212928, + "ordinal_expected_level_mae": 0.007897347093607133 + }, + "twins_streamlet:twins_policy_action": { + "questions": 797, + "accuracy": 0.9887076537013801, + "mean_class_recall": 0.9889237691623501, + "nll": 0.025810322060464137, + "brier": 0.015590356305548642, + "ece15": 0.0063347054801968505, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.9824341279799247, + "error_rate_at_confidence_0_9": 0.002554278416347382, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_quorum": { + "questions": 797, + "accuracy": 0.9937264742785445, + "mean_class_recall": 0.9922118380062306, + "nll": 0.017984967092368, + "brier": 0.011131458808574956, + "ece15": 0.005638113127633191, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.9912170639899623, + "error_rate_at_confidence_0_9": 0.0037974683544303796, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_visible_conflict": { + "questions": 797, + "accuracy": 0.9974905897114178, + "mean_class_recall": 0.998236331569665, + "nll": 0.013810181529074162, + "brier": 0.004981534821676031, + "ece15": 0.0021482520070447817, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.998745294855709, + "error_rate_at_confidence_0_9": 0.002512562814070352, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_message_manipulation": { + "questions": 800, + "accuracy": 0.6875, + "mean_class_recall": 0.6966034271725826, + "nll": 0.5181534061242398, + "brier": 0.36342749885940506, + "ece15": 0.019537157326473344, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.205, + "error_rate_at_confidence_0_9": 0.012195121951219513, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_supported_attack_type_v16": { + "questions": 800, + "accuracy": 0.67375, + "mean_class_recall": 0.5089660282717738, + "nll": 1.020378053391093, + "brier": 0.48493001129627705, + "ece15": 0.07743873861233927, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.1725, + "error_rate_at_confidence_0_9": 0.021739130434782608, + "ordinal_expected_level_mae": null + } + }, + "macro_task_accuracy": 0.9354480356736271, + "macro_task_balanced_accuracy": 0.9112880126164279 + } + }, + "test": { + "raw": { + "questions": 21634, + "accuracy": 0.8712212258482019, + "mean_class_recall": 0.8343950403877007, + "nll": 0.36437325393469283, + "brier": 0.1826183728339598, + "ece15": 0.016790325586147077, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.6702412868632708, + "error_rate_at_confidence_0_9": 0.01903448275862069, + "ordinal_expected_level_mae": 0.005114573364023923, + "tasks": { + "agentdojo_tool_outputs:agentdojo_boundary_action": { + "questions": 426, + "accuracy": 0.9953051643192489, + "mean_class_recall": 0.9895833333333333, + "nll": 0.020419671491798054, + "brier": 0.008500766418781964, + "ece15": 0.004676929181256299, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9765258215962441, + "error_rate_at_confidence_0_9": 0.002403846153846154, + "ordinal_expected_level_mae": null + }, + "agentdojo_tool_outputs:agentdojo_embedded_injection": { + "questions": 426, + "accuracy": 0.9835680751173709, + "mean_class_recall": 0.9635416666666667, + "nll": 0.03064858581307839, + "brier": 0.020119931286614087, + "ece15": 0.009967851623295965, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9788732394366197, + "error_rate_at_confidence_0_9": 0.004796163069544364, + "ordinal_expected_level_mae": null + }, + "byzfl_mnist:fl_attack_generated_update": { + "questions": 800, + "accuracy": 0.9525, + "mean_class_recall": 0.91640625, + "nll": 0.17731219866650544, + "brier": 0.08433190754386494, + "ece15": 0.03454275208230118, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.95, + "error_rate_at_confidence_0_9": 0.03289473684210526, + "ordinal_expected_level_mae": null + }, + "ciciot2023:ciciot_attack": { + "questions": 6000, + "accuracy": 0.8995, + "mean_class_recall": 0.8221111111111111, + "nll": 0.30893788807797395, + "brier": 0.15122747877892262, + "ece15": 0.028859158451860423, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.737, + "error_rate_at_confidence_0_9": 0.03640886476707372, + "ordinal_expected_level_mae": null + }, + "injecagent:injecagent_requested_operation_v13": { + "questions": 124, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.00019733561912440662, + "brier": 1.9375590740059706e-06, + "ece15": 0.0001968485399507404, + "temperature": 1.0, + "coverage_at_confidence_0_9": 1.0, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_known_logon_action_v16": { + "questions": 540, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.0017246613457991484, + "brier": 0.0007621008009601763, + "ece15": 0.001458779457079616, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9981481481481481, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_observable_policy": { + "questions": 540, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 1.1611401866187557e-06, + "brier": 5.747739403769056e-11, + "ece15": 1.1611258165194832e-06, + "temperature": 1.0, + "coverage_at_confidence_0_9": 1.0, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "ton_iot_network:ton_attack": { + "questions": 3500, + "accuracy": 0.9894285714285714, + "mean_class_recall": 0.9826666666666667, + "nll": 0.059799522199125266, + "brier": 0.02016997152334037, + "ece15": 0.00639039716386258, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9728571428571429, + "error_rate_at_confidence_0_9": 0.0076358296622613805, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_evidence_level": { + "questions": 659, + "accuracy": 0.9984825493171472, + "mean_class_recall": 0.9983249581239532, + "nll": 0.0075671027444564045, + "brier": 0.0038579171936140126, + "ece15": 0.0029747801235328448, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9939301972685888, + "error_rate_at_confidence_0_9": 0.0015267175572519084, + "ordinal_expected_level_mae": 0.005114573364023923 + }, + "twins_streamlet:twins_policy_action": { + "questions": 659, + "accuracy": 0.9969650986342944, + "mean_class_recall": 0.9966499162479062, + "nll": 0.007401075027874373, + "brier": 0.004494834346902429, + "ece15": 0.0034418284412443194, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9939301972685888, + "error_rate_at_confidence_0_9": 0.0015267175572519084, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_quorum": { + "questions": 659, + "accuracy": 0.9984825493171472, + "mean_class_recall": 0.9978632478632479, + "nll": 0.0026045927984607647, + "brier": 0.00133772985585407, + "ece15": 0.002039732462895639, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9954476479514416, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_visible_conflict": { + "questions": 659, + "accuracy": 0.9969650986342944, + "mean_class_recall": 0.9978260869565218, + "nll": 0.007434124878991116, + "brier": 0.004449797406116125, + "ece15": 0.002637941445288803, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9969650986342944, + "error_rate_at_confidence_0_9": 0.0015220700152207, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_message_manipulation": { + "questions": 3321, + "accuracy": 0.6838301716350497, + "mean_class_recall": 0.6880928586224566, + "nll": 0.5412193961251557, + "brier": 0.3797960822714433, + "ece15": 0.028068617701296383, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.19451972297500753, + "error_rate_at_confidence_0_9": 0.01393188854489164, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_supported_attack_type_v16": { + "questions": 3321, + "accuracy": 0.6859379704908161, + "mean_class_recall": 0.4840299509564737, + "nll": 1.1567283681183516, + "brier": 0.4884417807744329, + "ece15": 0.038297669894539775, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.18307738632941886, + "error_rate_at_confidence_0_9": 0.0805921052631579, + "ordinal_expected_level_mae": null + } + }, + "macro_task_accuracy": 0.9414975177781386, + "macro_task_balanced_accuracy": 0.9169354318963097 + }, + "recalibrated": { + "questions": 21634, + "accuracy": 0.8712212258482019, + "mean_class_recall": 0.8343950403877007, + "nll": 0.35395590855273523, + "brier": 0.18501029425464544, + "ece15": 0.030017031519889922, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.6506887307016733, + "error_rate_at_confidence_0_9": 0.016054557078923067, + "ordinal_expected_level_mae": 0.006011043764813851, + "tasks": { + "agentdojo_tool_outputs:agentdojo_boundary_action": { + "questions": 426, + "accuracy": 0.9953051643192489, + "mean_class_recall": 0.9895833333333333, + "nll": 0.023835787531761698, + "brier": 0.009460218069430947, + "ece15": 0.00990292779030044, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.9671361502347418, + "error_rate_at_confidence_0_9": 0.0024271844660194173, + "ordinal_expected_level_mae": null + }, + "agentdojo_tool_outputs:agentdojo_embedded_injection": { + "questions": 426, + "accuracy": 0.9835680751173709, + "mean_class_recall": 0.9635416666666667, + "nll": 0.02937486871225543, + "brier": 0.01905460565156044, + "ece15": 0.011296185892511, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.9694835680751174, + "error_rate_at_confidence_0_9": 0.002421307506053269, + "ordinal_expected_level_mae": null + }, + "byzfl_mnist:fl_attack_generated_update": { + "questions": 800, + "accuracy": 0.9525, + "mean_class_recall": 0.91640625, + "nll": 0.15527362489298477, + "brier": 0.08190930430000586, + "ece15": 0.02574101560783749, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.92625, + "error_rate_at_confidence_0_9": 0.029689608636977057, + "ordinal_expected_level_mae": null + }, + "ciciot2023:ciciot_attack": { + "questions": 6000, + "accuracy": 0.8995, + "mean_class_recall": 0.8221111111111111, + "nll": 0.2798275436837799, + "brier": 0.15134523246061166, + "ece15": 0.029238705666976703, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.7075, + "error_rate_at_confidence_0_9": 0.0303886925795053, + "ordinal_expected_level_mae": null + }, + "injecagent:injecagent_requested_operation_v13": { + "questions": 124, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.0008001907223348217, + "brier": 1.793746254505973e-05, + "ece15": 0.0007956440534692383, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 1.0, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_known_logon_action_v16": { + "questions": 540, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.0027723209709698475, + "brier": 0.0009529345145823382, + "ece15": 0.0024453488801810066, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.9925925925925926, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_observable_policy": { + "questions": 540, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 1.8070166383123626e-05, + "brier": 5.357685907682576e-09, + "ece15": 1.806882660338892e-05, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 1.0, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "ton_iot_network:ton_attack": { + "questions": 3500, + "accuracy": 0.9894285714285714, + "mean_class_recall": 0.9826666666666667, + "nll": 0.05655522961511102, + "brier": 0.021374549663926688, + "ece15": 0.009425764157233412, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.95, + "error_rate_at_confidence_0_9": 0.0069172932330827065, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_evidence_level": { + "questions": 659, + "accuracy": 0.9984825493171472, + "mean_class_recall": 0.9983249581239532, + "nll": 0.007520014659349455, + "brier": 0.003991311156523451, + "ece15": 0.00377357794140357, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.992412746585736, + "error_rate_at_confidence_0_9": 0.0015290519877675841, + "ordinal_expected_level_mae": 0.006011043764813851 + }, + "twins_streamlet:twins_policy_action": { + "questions": 659, + "accuracy": 0.9969650986342944, + "mean_class_recall": 0.9966499162479062, + "nll": 0.008102634890372476, + "brier": 0.004356473026911708, + "ece15": 0.004867738330078791, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.9908952959028832, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_quorum": { + "questions": 659, + "accuracy": 0.9984825493171472, + "mean_class_recall": 0.9978632478632479, + "nll": 0.0038576227479580737, + "brier": 0.0014748958385226618, + "ece15": 0.003282715492202558, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.9939301972685888, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_visible_conflict": { + "questions": 659, + "accuracy": 0.9969650986342944, + "mean_class_recall": 0.9978260869565218, + "nll": 0.006788751578147029, + "brier": 0.004171759480970426, + "ece15": 0.003177496987382077, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.9939301972685888, + "error_rate_at_confidence_0_9": 0.0015267175572519084, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_message_manipulation": { + "questions": 3321, + "accuracy": 0.6838301716350497, + "mean_class_recall": 0.6880928586224566, + "nll": 0.5429719234394278, + "brier": 0.3809937714731844, + "ece15": 0.02889956013524373, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.17765733212887685, + "error_rate_at_confidence_0_9": 0.010169491525423728, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_supported_attack_type_v16": { + "questions": 3321, + "accuracy": 0.6859379704908161, + "mean_class_recall": 0.4840299509564737, + "nll": 1.1477144959529286, + "brier": 0.5019380206266735, + "ece15": 0.11401240417222197, + "temperature": 1.2873604198505975, + "coverage_at_confidence_0_9": 0.16079494128274616, + "error_rate_at_confidence_0_9": 0.07865168539325842, + "ordinal_expected_level_mae": null + } + }, + "macro_task_accuracy": 0.9414975177781386, + "macro_task_balanced_accuracy": 0.9169354318963097 + } + }, + "general": { + "decision-v7": { + "raw_clean": { + "n": 1200, + "nll": 0.5237394417093835, + "acc": 0.865, + "ece": 0.08203125833030098, + "brier": 0.21310411533941315, + "mean_conf": 0.9455011120677191, + "confident_error_rate": 0.06, + "coverage_at_0_9": 0.8308333333333333, + "accuracy_at_0_9": 0.9277833500501504, + "coverage_at_5pct_error": 0.7791666666666667, + "coverage_at_1pct_error": 0.6158333333333333, + "aurc": 0.028219153802824883, + "error_rate_at_0_9": 0.07221664994984955, + "confidence_bias": 0.08050111206771915, + "top_bins": { + "0.9": { + "n": 997, + "errors": 72, + "error_rate": 0.07221664994984955 + }, + "0.95": { + "n": 937, + "errors": 47, + "error_rate": 0.05016008537886873 + }, + "0.99": { + "n": 861, + "errors": 27, + "error_rate": 0.0313588850174216 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9933333333333333, + "confidence_cutoff": 0.9999626357556778 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.9458333333333333, + "confidence_cutoff": 0.9342652506942892 + } + }, + "score_mae": 0.4647882331825802, + "ranked_probability_score": 0.08637141204655961 + }, + "as_shipped_clean": { + "n": 1200, + "nll": 0.44174931033716425, + "acc": 0.865, + "ece": 0.0702397073006123, + "brier": 0.20431799318642196, + "mean_conf": 0.930435938959295, + "confident_error_rate": 0.04, + "coverage_at_0_9": 0.7858333333333334, + "accuracy_at_0_9": 0.9490986214209968, + "coverage_at_5pct_error": 0.7783333333333333, + "coverage_at_1pct_error": 0.6125, + "aurc": 0.028152549316370398, + "error_rate_at_0_9": 0.05090137857900318, + "confidence_bias": 0.065435938959295, + "top_bins": { + "0.9": { + "n": 943, + "errors": 48, + "error_rate": 0.05090137857900318 + }, + "0.95": { + "n": 893, + "errors": 35, + "error_rate": 0.03919372900335946 + }, + "0.99": { + "n": 803, + "errors": 15, + "error_rate": 0.0186799501867995 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9916666666666667, + "confidence_cutoff": 0.9996192297027195 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.9447916666666667, + "confidence_cutoff": 0.8818964099133315 + } + }, + "score_mae": 0.47431418086986715, + "ranked_probability_score": 0.08319761613591657 + }, + "sources": { + "agnews": { + "n": 240, + "nll": 0.2796999395688286, + "acc": 0.9125, + "ece": 0.05892681778016237, + "brier": 0.1350754514620047, + "mean_conf": 0.9714268177801624, + "confident_error_rate": 0.04583333333333333, + "coverage_at_0_9": 0.9208333333333333, + "accuracy_at_0_9": 0.9502262443438914, + "coverage_at_5pct_error": 0.9333333333333333, + "coverage_at_1pct_error": 0.75, + "aurc": 0.011330835078134182, + "error_rate_at_0_9": 0.049773755656108594, + "confidence_bias": 0.0589268177801624, + "top_bins": { + "0.9": { + "n": 221, + "errors": 11, + "error_rate": 0.049773755656108594 + }, + "0.95": { + "n": 214, + "errors": 9, + "error_rate": 0.04205607476635514 + }, + "0.99": { + "n": 189, + "errors": 3, + "error_rate": 0.015873015873015872 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9994019817408221 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.984375, + "confidence_cutoff": 0.9890606274392002 + } + } + }, + "amazon": { + "n": 80, + "nll": 1.1725112640711228, + "acc": 0.6, + "ece": 0.14900986362097515, + "brier": 0.5611978409181134, + "mean_conf": 0.7279331238481536, + "confident_error_rate": 0.0375, + "coverage_at_0_9": 0.2125, + "accuracy_at_0_9": 0.8235294117647058, + "coverage_at_5pct_error": 0.075, + "coverage_at_1pct_error": 0.075, + "aurc": 0.268674359040746, + "error_rate_at_0_9": 0.17647058823529413, + "confidence_bias": 0.12793312384815358, + "top_bins": { + "0.9": { + "n": 17, + "errors": 3, + "error_rate": 0.17647058823529413 + }, + "0.95": { + "n": 12, + "errors": 2, + "error_rate": 0.16666666666666666 + }, + "0.99": { + "n": 2, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.725, + "confidence_cutoff": 0.7565342654626949 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.640625, + "confidence_cutoff": 0.5366925117472928 + } + }, + "score_mae": 0.5012771753189752, + "ranked_probability_score": 0.0890286871906647 + }, + "banking77": { + "n": 80, + "nll": 0.5995652320511431, + "acc": 0.85, + "ece": 0.08817712605098262, + "brier": 0.24932641415691287, + "mean_conf": 0.8916570456057876, + "confident_error_rate": 0.05, + "coverage_at_0_9": 0.75, + "accuracy_at_0_9": 0.9333333333333333, + "coverage_at_5pct_error": 0.6875, + "coverage_at_1pct_error": 0.6125, + "aurc": 0.03525784971074972, + "error_rate_at_0_9": 0.06666666666666667, + "confidence_bias": 0.041657045605787646, + "top_bins": { + "0.9": { + "n": 60, + "errors": 4, + "error_rate": 0.06666666666666667 + }, + "0.95": { + "n": 55, + "errors": 2, + "error_rate": 0.03636363636363636 + }, + "0.99": { + "n": 46, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9933430562867247 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.90625, + "confidence_cutoff": 0.8362446757691954 + } + } + }, + "boolq": { + "n": 80, + "nll": 0.5712250447804231, + "acc": 0.875, + "ece": 0.11692367054964475, + "brier": 0.22310665871912266, + "mean_conf": 0.948749077509772, + "confident_error_rate": 0.0875, + "coverage_at_0_9": 0.85, + "accuracy_at_0_9": 0.8970588235294118, + "coverage_at_5pct_error": 0.6875, + "coverage_at_1pct_error": 0.1, + "aurc": 0.05278244760094704, + "error_rate_at_0_9": 0.10294117647058823, + "confidence_bias": 0.073749077509772, + "top_bins": { + "0.9": { + "n": 68, + "errors": 7, + "error_rate": 0.10294117647058823 + }, + "0.95": { + "n": 66, + "errors": 6, + "error_rate": 0.09090909090909091 + }, + "0.99": { + "n": 60, + "errors": 4, + "error_rate": 0.06666666666666667 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.975, + "confidence_cutoff": 0.9998615919766226 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.921875, + "confidence_cutoff": 0.979302462267494 + } + } + }, + "contrastive": { + "n": 160, + "nll": 0.06554428006012027, + "acc": 0.96875, + "ece": 0.024615155693287222, + "brier": 0.04485687620328682, + "mean_conf": 0.9933651556932872, + "confident_error_rate": 0.0125, + "coverage_at_0_9": 0.98125, + "accuracy_at_0_9": 0.9872611464968153, + "coverage_at_5pct_error": 1.0, + "coverage_at_1pct_error": 0.975, + "aurc": 0.000671805023667049, + "error_rate_at_0_9": 0.012738853503184714, + "confidence_bias": 0.024615155693287205, + "top_bins": { + "0.9": { + "n": 157, + "errors": 2, + "error_rate": 0.012738853503184714 + }, + "0.95": { + "n": 154, + "errors": 1, + "error_rate": 0.006493506493506494 + }, + "0.99": { + "n": 148, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9999882930499685 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 1.0, + "confidence_cutoff": 0.9997675703852573 + } + } + }, + "dbpedia14": { + "n": 80, + "nll": 0.01926796153789536, + "acc": 0.9875, + "ece": 0.011278233290686156, + "brier": 0.014217088541729535, + "mean_conf": 0.9949340220579689, + "confident_error_rate": 0.0, + "coverage_at_0_9": 0.9875, + "accuracy_at_0_9": 1.0, + "coverage_at_5pct_error": 1.0, + "coverage_at_1pct_error": 0.9875, + "aurc": 0.00015625, + "error_rate_at_0_9": 0.0, + "confidence_bias": 0.0074340220579688365, + "top_bins": { + "0.9": { + "n": 79, + "errors": 0, + "error_rate": 0.0 + }, + "0.95": { + "n": 77, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 76, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9999950740905034 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 1.0, + "confidence_cutoff": 0.9999745354541041 + } + } + }, + "imdb": { + "n": 80, + "nll": 0.06446545489140978, + "acc": 0.9625, + "ece": 0.025460341159037583, + "brier": 0.045829990154110004, + "mean_conf": 0.9836424817249945, + "confident_error_rate": 0.0, + "coverage_at_0_9": 0.9375, + "accuracy_at_0_9": 1.0, + "coverage_at_5pct_error": 1.0, + "coverage_at_1pct_error": 0.95, + "aurc": 0.0012680561790530144, + "error_rate_at_0_9": 0.0, + "confidence_bias": 0.021142481724994444, + "top_bins": { + "0.9": { + "n": 75, + "errors": 0, + "error_rate": 0.0 + }, + "0.95": { + "n": 75, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 71, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9999788743849367 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 1.0, + "confidence_cutoff": 0.9996364092230372 + } + } + }, + "mnli": { + "n": 80, + "nll": 0.46179093448986874, + "acc": 0.9125, + "ece": 0.07967308878997346, + "brier": 0.16221252791471635, + "mean_conf": 0.985103225410972, + "confident_error_rate": 0.0625, + "coverage_at_0_9": 0.925, + "accuracy_at_0_9": 0.9324324324324325, + "coverage_at_5pct_error": 0.8875, + "coverage_at_1pct_error": 0.65, + "aurc": 0.015873702018260973, + "error_rate_at_0_9": 0.06756756756756757, + "confidence_bias": 0.07260322541097197, + "top_bins": { + "0.9": { + "n": 74, + "errors": 5, + "error_rate": 0.06756756756756757 + }, + "0.95": { + "n": 74, + "errors": 5, + "error_rate": 0.06756756756756757 + }, + "0.99": { + "n": 71, + "errors": 3, + "error_rate": 0.04225352112676056 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.999987939363811 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.96875, + "confidence_cutoff": 0.9997555402093751 + } + } + }, + "sst5": { + "n": 80, + "nll": 1.1930808545317393, + "acc": 0.6125, + "ece": 0.23848723097529123, + "brier": 0.5810003072599018, + "mean_conf": 0.7987986774228062, + "confident_error_rate": 0.0625, + "coverage_at_0_9": 0.2625, + "accuracy_at_0_9": 0.7619047619047619, + "coverage_at_5pct_error": 0.0875, + "coverage_at_1pct_error": 0.0875, + "aurc": 0.2998766100005863, + "error_rate_at_0_9": 0.23809523809523808, + "confidence_bias": 0.18629867742280615, + "top_bins": { + "0.9": { + "n": 21, + "errors": 5, + "error_rate": 0.23809523809523808 + }, + "0.95": { + "n": 5, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 0, + "errors": 0, + "error_rate": null + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.625, + "confidence_cutoff": 0.8363874325149153 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.625, + "confidence_cutoff": 0.6881083881339022 + } + }, + "score_mae": 0.4905379704011906, + "ranked_probability_score": 0.08831634129641223 + }, + "trec": { + "n": 80, + "nll": 0.268396899685072, + "acc": 0.9625, + "ece": 0.02818577556782262, + "brier": 0.058300946365123306, + "mean_conf": 0.9906857755678227, + "confident_error_rate": 0.025, + "coverage_at_0_9": 0.9875, + "accuracy_at_0_9": 0.9746835443037974, + "coverage_at_5pct_error": 1.0, + "coverage_at_1pct_error": 0.325, + "aurc": 0.015881097479045377, + "error_rate_at_0_9": 0.02531645569620253, + "confidence_bias": 0.028185775567822646, + "top_bins": { + "0.9": { + "n": 79, + "errors": 2, + "error_rate": 0.02531645569620253 + }, + "0.95": { + "n": 78, + "errors": 2, + "error_rate": 0.02564102564102564 + }, + "0.99": { + "n": 77, + "errors": 2, + "error_rate": 0.025974025974025976 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.975, + "confidence_cutoff": 0.9999834449407883 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.984375, + "confidence_cutoff": 0.9998382341930544 + } + } + }, + "yelp": { + "n": 160, + "nll": 0.6528738150960317, + "acc": 0.76875, + "ece": 0.10032078827768093, + "brier": 0.3373190084870059, + "mean_conf": 0.867012445257043, + "confident_error_rate": 0.05625, + "coverage_at_0_9": 0.575, + "accuracy_at_0_9": 0.9021739130434783, + "coverage_at_5pct_error": 0.41875, + "coverage_at_1pct_error": 0.18125, + "aurc": 0.08667681796341256, + "error_rate_at_0_9": 0.09782608695652174, + "confidence_bias": 0.09826244525704297, + "top_bins": { + "0.9": { + "n": 92, + "errors": 9, + "error_rate": 0.09782608695652174 + }, + "0.95": { + "n": 83, + "errors": 8, + "error_rate": 0.0963855421686747 + }, + "0.99": { + "n": 63, + "errors": 3, + "error_rate": 0.047619047619047616 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.925, + "confidence_cutoff": 0.9656164925493048 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.84375, + "confidence_cutoff": 0.7039733738891296 + } + }, + "score_mae": 0.4311273968894357, + "ranked_probability_score": 0.07224781992067277 + } + }, + "types": { + "choice": { + "n": 480, + "nll": 0.3037822176153947, + "acc": 0.93125, + "ece": 0.050099480347752155, + "brier": 0.11775456449644288, + "mean_conf": 0.9718507476012318, + "confident_error_rate": 0.035416666666666666, + "coverage_at_0_9": 0.9270833333333334, + "accuracy_at_0_9": 0.9617977528089887, + "coverage_at_5pct_error": 0.94375, + "coverage_at_1pct_error": 0.725, + "aurc": 0.010673395731944459, + "error_rate_at_0_9": 0.038202247191011236, + "confidence_bias": 0.04060074760123178, + "top_bins": { + "0.9": { + "n": 445, + "errors": 17, + "error_rate": 0.038202247191011236 + }, + "0.95": { + "n": 435, + "errors": 15, + "error_rate": 0.034482758620689655 + }, + "0.99": { + "n": 408, + "errors": 6, + "error_rate": 0.014705882352941176 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9958333333333333, + "confidence_cutoff": 0.9999737433376136 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.9895833333333334, + "confidence_cutoff": 0.9984835061671959 + } + } + }, + "noul": { + "n": 480, + "nll": 0.24523830247044484, + "acc": 0.9229166666666667, + "ece": 0.04796953512867931, + "brier": 0.11605074761778626, + "mean_conf": 0.970886201795346, + "confident_error_rate": 0.035416666666666666, + "coverage_at_0_9": 0.9104166666666667, + "accuracy_at_0_9": 0.9610983981693364, + "coverage_at_5pct_error": 0.94375, + "coverage_at_1pct_error": 0.71875, + "aurc": 0.01472209854520942, + "error_rate_at_0_9": 0.038901601830663615, + "confidence_bias": 0.0479695351286793, + "top_bins": { + "0.9": { + "n": 437, + "errors": 17, + "error_rate": 0.038901601830663615 + }, + "0.95": { + "n": 424, + "errors": 13, + "error_rate": 0.030660377358490566 + }, + "0.99": { + "n": 389, + "errors": 8, + "error_rate": 0.02056555269922879 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9958333333333333, + "confidence_cutoff": 0.9997753303513292 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.9791666666666666, + "confidence_cutoff": 0.9930519257681523 + } + } + }, + "score": { + "n": 240, + "nll": 1.110705511514142, + "acc": 0.6166666666666667, + "ece": 0.16184498803254416, + "brier": 0.5539793417036516, + "mean_conf": 0.7667057960033195, + "confident_error_rate": 0.058333333333333334, + "coverage_at_0_9": 0.25416666666666665, + "accuracy_at_0_9": 0.7704918032786885, + "coverage_at_5pct_error": 0.016666666666666666, + "coverage_at_1pct_error": 0.016666666666666666, + "aurc": 0.28057303182426463, + "error_rate_at_0_9": 0.22950819672131148, + "confidence_bias": 0.15003912933665275, + "top_bins": { + "0.9": { + "n": 61, + "errors": 14, + "error_rate": 0.22950819672131148 + }, + "0.95": { + "n": 34, + "errors": 7, + "error_rate": 0.20588235294117646 + }, + "0.99": { + "n": 6, + "errors": 1, + "error_rate": 0.16666666666666666 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.6916666666666667, + "confidence_cutoff": 0.8026983558729256 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.6666666666666666, + "confidence_cutoff": 0.6054379586272015 + } + }, + "score_mae": 0.47431418086986715, + "ranked_probability_score": 0.08319761613591657 + } + } + }, + "transfer-v4": { + "raw_clean": { + "n": 656, + "nll": 0.7491303449341485, + "acc": 0.836890243902439, + "ece": 0.10484780065311013, + "brier": 0.2706697225616099, + "mean_conf": 0.9369188798045006, + "confident_error_rate": 0.08689024390243902, + "coverage_at_0_9": 0.8277439024390244, + "accuracy_at_0_9": 0.8950276243093923, + "coverage_at_5pct_error": 0.5701219512195121, + "coverage_at_1pct_error": 0.3567073170731707, + "aurc": 0.048474736987579994, + "error_rate_at_0_9": 0.10497237569060773, + "confidence_bias": 0.10002863590206157, + "top_bins": { + "0.9": { + "n": 543, + "errors": 57, + "error_rate": 0.10497237569060773 + }, + "0.95": { + "n": 512, + "errors": 48, + "error_rate": 0.09375 + }, + "0.99": { + "n": 455, + "errors": 30, + "error_rate": 0.06593406593406594 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9634146341463414, + "confidence_cutoff": 0.9997756567114997 + }, + "0.8": { + "coverage": 0.8003048780487805, + "accuracy": 0.8990476190476191, + "confidence_cutoff": 0.9312810361300458 + } + }, + "score_mae": 0.2771201607516616, + "ranked_probability_score": 0.12002374674647091 + }, + "as_shipped_clean": { + "n": 656, + "nll": 0.6222520301084513, + "acc": 0.836890243902439, + "ece": 0.09048309685573616, + "brier": 0.2622492394102412, + "mean_conf": 0.9203846154028757, + "confident_error_rate": 0.07317073170731707, + "coverage_at_0_9": 0.7820121951219512, + "accuracy_at_0_9": 0.9064327485380117, + "coverage_at_5pct_error": 0.5670731707317073, + "coverage_at_1pct_error": 0.3551829268292683, + "aurc": 0.0484684761006514, + "error_rate_at_0_9": 0.0935672514619883, + "confidence_bias": 0.0834943715004367, + "top_bins": { + "0.9": { + "n": 513, + "errors": 48, + "error_rate": 0.0935672514619883 + }, + "0.95": { + "n": 477, + "errors": 37, + "error_rate": 0.07756813417190776 + }, + "0.99": { + "n": 405, + "errors": 23, + "error_rate": 0.056790123456790124 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9634146341463414, + "confidence_cutoff": 0.9985297137731126 + }, + "0.8": { + "coverage": 0.8003048780487805, + "accuracy": 0.900952380952381, + "confidence_cutoff": 0.8749540187432164 + } + }, + "score_mae": 0.2768980570793622, + "ranked_probability_score": 0.11577841556824224 + }, + "sources": { + "composition_holdout": { + "n": 96, + "nll": 0.8314301766731748, + "acc": 0.8645833333333334, + "ece": 0.12259156340261834, + "brier": 0.25260603356221595, + "mean_conf": 0.9783762896019588, + "confident_error_rate": 0.11458333333333333, + "coverage_at_0_9": 0.9479166666666666, + "accuracy_at_0_9": 0.8791208791208791, + "coverage_at_5pct_error": 0.7291666666666666, + "coverage_at_1pct_error": 0.6666666666666666, + "aurc": 0.029361211962366768, + "error_rate_at_0_9": 0.12087912087912088, + "confidence_bias": 0.1137929562686254, + "top_bins": { + "0.9": { + "n": 91, + "errors": 11, + "error_rate": 0.12087912087912088 + }, + "0.95": { + "n": 90, + "errors": 11, + "error_rate": 0.12222222222222222 + }, + "0.99": { + "n": 86, + "errors": 11, + "error_rate": 0.12790697674418605 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.99991410687893 + }, + "0.8": { + "coverage": 0.8020833333333334, + "accuracy": 0.948051948051948, + "confidence_cutoff": 0.9986476483509216 + } + } + }, + "contrastive": { + "n": 80, + "nll": 0.46595606789147936, + "acc": 0.85, + "ece": 0.11588755121828759, + "brier": 0.22911401994889413, + "mean_conf": 0.9658875512182877, + "confident_error_rate": 0.0875, + "coverage_at_0_9": 0.9, + "accuracy_at_0_9": 0.9027777777777778, + "coverage_at_5pct_error": 0.85, + "coverage_at_1pct_error": 0.8, + "aurc": 0.017400442212878862, + "error_rate_at_0_9": 0.09722222222222222, + "confidence_bias": 0.11588755121828775, + "top_bins": { + "0.9": { + "n": 72, + "errors": 7, + "error_rate": 0.09722222222222222 + }, + "0.95": { + "n": 70, + "errors": 5, + "error_rate": 0.07142857142857142 + }, + "0.99": { + "n": 68, + "errors": 3, + "error_rate": 0.04411764705882353 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9999823513518187 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 1.0, + "confidence_cutoff": 0.9993279314433398 + } + }, + "score_mae": 0.2768980570793622, + "ranked_probability_score": 0.11577841556824224 + }, + "emotion": { + "n": 80, + "nll": 1.1680421729742996, + "acc": 0.6625, + "ece": 0.15215129647550107, + "brier": 0.49482559482198313, + "mean_conf": 0.767505586427657, + "confident_error_rate": 0.0625, + "coverage_at_0_9": 0.425, + "accuracy_at_0_9": 0.8529411764705882, + "coverage_at_5pct_error": 0.075, + "coverage_at_1pct_error": 0.075, + "aurc": 0.18480906195062094, + "error_rate_at_0_9": 0.14705882352941177, + "confidence_bias": 0.105005586427657, + "top_bins": { + "0.9": { + "n": 34, + "errors": 5, + "error_rate": 0.14705882352941177 + }, + "0.95": { + "n": 26, + "errors": 3, + "error_rate": 0.11538461538461539 + }, + "0.99": { + "n": 8, + "errors": 1, + "error_rate": 0.125 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.8, + "confidence_cutoff": 0.8315422519453946 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.734375, + "confidence_cutoff": 0.5608119629701582 + } + } + }, + "mmlu": { + "n": 80, + "nll": 0.8706892848634225, + "acc": 0.725, + "ece": 0.14396914879993838, + "brier": 0.42715412327538493, + "mean_conf": 0.8115029447218166, + "confident_error_rate": 0.0625, + "coverage_at_0_9": 0.475, + "accuracy_at_0_9": 0.868421052631579, + "coverage_at_5pct_error": 0.375, + "coverage_at_1pct_error": 0.3375, + "aurc": 0.11356864595903862, + "error_rate_at_0_9": 0.13157894736842105, + "confidence_bias": 0.08650294472181663, + "top_bins": { + "0.9": { + "n": 38, + "errors": 5, + "error_rate": 0.13157894736842105 + }, + "0.95": { + "n": 28, + "errors": 1, + "error_rate": 0.03571428571428571 + }, + "0.99": { + "n": 18, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.85, + "confidence_cutoff": 0.8706443942318668 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.796875, + "confidence_cutoff": 0.5685435872401945 + } + } + }, + "paws": { + "n": 80, + "nll": 0.8953763773747229, + "acc": 0.8, + "ece": 0.16414062885679848, + "brier": 0.31529713556607303, + "mean_conf": 0.947798906827078, + "confident_error_rate": 0.125, + "coverage_at_0_9": 0.8625, + "accuracy_at_0_9": 0.855072463768116, + "coverage_at_5pct_error": 0.3, + "coverage_at_1pct_error": 0.15, + "aurc": 0.09544667963478717, + "error_rate_at_0_9": 0.14492753623188406, + "confidence_bias": 0.14779890682707797, + "top_bins": { + "0.9": { + "n": 69, + "errors": 10, + "error_rate": 0.14492753623188406 + }, + "0.95": { + "n": 69, + "errors": 10, + "error_rate": 0.14492753623188406 + }, + "0.99": { + "n": 60, + "errors": 6, + "error_rate": 0.1 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.875, + "confidence_cutoff": 0.9995333943902521 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.890625, + "confidence_cutoff": 0.9804900927788639 + } + } + }, + "qnli": { + "n": 80, + "nll": 0.2866833743812309, + "acc": 0.925, + "ece": 0.07720126969420893, + "brier": 0.1353782597318704, + "mean_conf": 0.969044048844119, + "confident_error_rate": 0.05, + "coverage_at_0_9": 0.9, + "accuracy_at_0_9": 0.9444444444444444, + "coverage_at_5pct_error": 0.8625, + "coverage_at_1pct_error": 0.6125, + "aurc": 0.01474092888988553, + "error_rate_at_0_9": 0.05555555555555555, + "confidence_bias": 0.044044048844118944, + "top_bins": { + "0.9": { + "n": 72, + "errors": 4, + "error_rate": 0.05555555555555555 + }, + "0.95": { + "n": 69, + "errors": 3, + "error_rate": 0.043478260869565216 + }, + "0.99": { + "n": 64, + "errors": 2, + "error_rate": 0.03125 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9994500997969659 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.96875, + "confidence_cutoff": 0.9906893719466282 + } + } + }, + "sciq": { + "n": 80, + "nll": 0.007797247790721174, + "acc": 1.0, + "ece": 0.007304169605922202, + "brier": 0.00159066044034253, + "mean_conf": 0.9926958303940776, + "confident_error_rate": 0.0, + "coverage_at_0_9": 0.975, + "accuracy_at_0_9": 1.0, + "coverage_at_5pct_error": 1.0, + "coverage_at_1pct_error": 1.0, + "aurc": 0.0, + "error_rate_at_0_9": 0.0, + "confidence_bias": -0.007304169605922395, + "top_bins": { + "0.9": { + "n": 78, + "errors": 0, + "error_rate": 0.0 + }, + "0.95": { + "n": 76, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 71, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9999411463988364 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 1.0, + "confidence_cutoff": 0.999496345294104 + } + } + }, + "tweet_offensive": { + "n": 80, + "nll": 0.4102059096056144, + "acc": 0.8625, + "ece": 0.082947906119192, + "brier": 0.24395672910477048, + "mean_conf": 0.9186674303481943, + "confident_error_rate": 0.075, + "coverage_at_0_9": 0.7375, + "accuracy_at_0_9": 0.8983050847457628, + "coverage_at_5pct_error": 0.575, + "coverage_at_1pct_error": 0.5125, + "aurc": 0.05023960594359948, + "error_rate_at_0_9": 0.1016949152542373, + "confidence_bias": 0.05616743034819427, + "top_bins": { + "0.9": { + "n": 59, + "errors": 6, + "error_rate": 0.1016949152542373 + }, + "0.95": { + "n": 49, + "errors": 4, + "error_rate": 0.08163265306122448 + }, + "0.99": { + "n": 30, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9765148863831499 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.875, + "confidence_cutoff": 0.8834322098576286 + } + } + } + }, + "types": { + "choice": { + "n": 336, + "nll": 0.724820218484822, + "acc": 0.8154761904761905, + "ece": 0.09541834159449251, + "brier": 0.29207086162199275, + "mean_conf": 0.8918466450156909, + "confident_error_rate": 0.0625, + "coverage_at_0_9": 0.7172619047619048, + "accuracy_at_0_9": 0.9128630705394191, + "coverage_at_5pct_error": 0.48214285714285715, + "coverage_at_1pct_error": 0.36904761904761907, + "aurc": 0.05557065380684749, + "error_rate_at_0_9": 0.08713692946058091, + "confidence_bias": 0.07637045453950042, + "top_bins": { + "0.9": { + "n": 241, + "errors": 21, + "error_rate": 0.08713692946058091 + }, + "0.95": { + "n": 220, + "errors": 15, + "error_rate": 0.06818181818181818 + }, + "0.99": { + "n": 183, + "errors": 12, + "error_rate": 0.06557377049180328 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9464285714285714, + "confidence_cutoff": 0.9967131547386102 + }, + "0.8": { + "coverage": 0.8005952380952381, + "accuracy": 0.8921933085501859, + "confidence_cutoff": 0.7705104187347486 + } + } + }, + "noul": { + "n": 280, + "nll": 0.4549349162962509, + "acc": 0.8821428571428571, + "ece": 0.08452238489264785, + "brier": 0.19846632133861988, + "mean_conf": 0.9530010972612158, + "confident_error_rate": 0.07142857142857142, + "coverage_at_0_9": 0.8571428571428571, + "accuracy_at_0_9": 0.9166666666666666, + "coverage_at_5pct_error": 0.7607142857142857, + "coverage_at_1pct_error": 0.2571428571428571, + "aurc": 0.03904590143532812, + "error_rate_at_0_9": 0.08333333333333333, + "confidence_bias": 0.07085824011835873, + "top_bins": { + "0.9": { + "n": 240, + "errors": 20, + "error_rate": 0.08333333333333333 + }, + "0.95": { + "n": 227, + "errors": 17, + "error_rate": 0.07488986784140969 + }, + "0.99": { + "n": 194, + "errors": 8, + "error_rate": 0.041237113402061855 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9571428571428572, + "confidence_cutoff": 0.9989211416259292 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.9330357142857143, + "confidence_cutoff": 0.95900292163327 + } + } + }, + "score": { + "n": 40, + "nll": 0.9318990444323394, + "acc": 0.7, + "ece": 0.2317881936468471, + "brier": 0.45822803933287715, + "mean_conf": 0.9317881936468471, + "confident_error_rate": 0.175, + "coverage_at_0_9": 0.8, + "accuracy_at_0_9": 0.78125, + "coverage_at_5pct_error": 0.65, + "coverage_at_1pct_error": 0.6, + "aurc": 0.07545101908306909, + "error_rate_at_0_9": 0.21875, + "confidence_bias": 0.2317881936468471, + "top_bins": { + "0.9": { + "n": 32, + "errors": 7, + "error_rate": 0.21875 + }, + "0.95": { + "n": 30, + "errors": 5, + "error_rate": 0.16666666666666666 + }, + "0.99": { + "n": 28, + "errors": 3, + "error_rate": 0.10714285714285714 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9997762256879271 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.78125, + "confidence_cutoff": 0.9360940664483365 + } + }, + "score_mae": 0.2768980570793622, + "ranked_probability_score": 0.11577841556824224 + } + } + } + }, + "temperature": 1.2873604198505975 + }, + { + "epoch": 3, + "development": { + "raw": { + "questions": 14912, + "accuracy": 0.9075912017167382, + "mean_class_recall": 0.8447857522159936, + "nll": 0.24807737680415345, + "brier": 0.12998082994334342, + "ece15": 0.027028147820359012, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.8103540772532188, + "error_rate_at_confidence_0_9": 0.023502151605428664, + "ordinal_expected_level_mae": 0.00481932257817722, + "tasks": { + "agentdojo_tool_outputs:agentdojo_boundary_action": { + "questions": 222, + "accuracy": 0.9774774774774775, + "mean_class_recall": 0.9570930232558139, + "nll": 0.10407641066616782, + "brier": 0.039347876403718875, + "ece15": 0.021432863016197557, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9864864864864865, + "error_rate_at_confidence_0_9": 0.0136986301369863, + "ordinal_expected_level_mae": null + }, + "agentdojo_tool_outputs:agentdojo_embedded_injection": { + "questions": 222, + "accuracy": 0.9819819819819819, + "mean_class_recall": 0.96, + "nll": 0.10023555924012383, + "brier": 0.031563944714978254, + "ece15": 0.015911253108198964, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9864864864864865, + "error_rate_at_confidence_0_9": 0.0136986301369863, + "ordinal_expected_level_mae": null + }, + "byzfl_mnist:fl_attack_generated_update": { + "questions": 1400, + "accuracy": 0.9885714285714285, + "mean_class_recall": 0.9821428571428572, + "nll": 0.0442702602822601, + "brier": 0.02139513973607548, + "ece15": 0.007710400104216533, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9842857142857143, + "error_rate_at_confidence_0_9": 0.008708272859216255, + "ordinal_expected_level_mae": null + }, + "ciciot2023:ciciot_attack": { + "questions": 5000, + "accuracy": 0.8418, + "mean_class_recall": 0.7537333333333334, + "nll": 0.4173917752941542, + "brier": 0.22451704505631584, + "ece15": 0.06284084146027137, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.6928, + "error_rate_at_confidence_0_9": 0.06293302540415704, + "ordinal_expected_level_mae": null + }, + "injecagent:injecagent_requested_operation_v13": { + "questions": 124, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.0014205610073774013, + "brier": 0.0002211290260091384, + "ece15": 0.0013613576932818498, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9919354838709677, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_known_logon_action_v16": { + "questions": 328, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.000610657881540054, + "brier": 8.099314527924365e-05, + "ece15": 0.0005891478461821947, + "temperature": 1.0, + "coverage_at_confidence_0_9": 1.0, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_observable_policy": { + "questions": 328, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 1.5480302068279906e-07, + "brier": 4.4217021643060835e-12, + "ece15": 1.548019148098009e-07, + "temperature": 1.0, + "coverage_at_confidence_0_9": 1.0, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "ton_iot_network:ton_attack": { + "questions": 2500, + "accuracy": 0.9792, + "mean_class_recall": 0.9818666666666667, + "nll": 0.11218220058914397, + "brier": 0.03724045269615108, + "ece15": 0.01842697861877182, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9928, + "error_rate_at_confidence_0_9": 0.015310233682514102, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_evidence_level": { + "questions": 797, + "accuracy": 0.9962358845671268, + "mean_class_recall": 0.9963898916967509, + "nll": 0.012217254972570101, + "brier": 0.006818829719414822, + "ece15": 0.004133398699556409, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9949811794228356, + "error_rate_at_confidence_0_9": 0.0012610340479192938, + "ordinal_expected_level_mae": 0.00481932257817722 + }, + "twins_streamlet:twins_policy_action": { + "questions": 797, + "accuracy": 0.9949811794228356, + "mean_class_recall": 0.9951865222623345, + "nll": 0.013078119892699127, + "brier": 0.007721112955228842, + "ece15": 0.003423998250058524, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9899623588456713, + "error_rate_at_confidence_0_9": 0.0012674271229404308, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_quorum": { + "questions": 797, + "accuracy": 0.9949811794228356, + "mean_class_recall": 0.994276682635671, + "nll": 0.01027058602627885, + "brier": 0.007209131739335666, + "ece15": 0.004201163796495765, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9912170639899623, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_visible_conflict": { + "questions": 797, + "accuracy": 0.9974905897114178, + "mean_class_recall": 0.998236331569665, + "nll": 0.017287182530912663, + "brier": 0.004953169208636321, + "ece15": 0.0025266183132433223, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.998745294855709, + "error_rate_at_confidence_0_9": 0.002512562814070352, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_message_manipulation": { + "questions": 800, + "accuracy": 0.69, + "mean_class_recall": 0.7116483068135455, + "nll": 0.5092641106435474, + "brier": 0.3569714849542106, + "ece15": 0.03650030376694629, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.2425, + "error_rate_at_confidence_0_9": 0.010309278350515464, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_supported_attack_type_v16": { + "questions": 800, + "accuracy": 0.68875, + "mean_class_recall": 0.529392069171139, + "nll": 0.96833512912194, + "brier": 0.46247412161427887, + "ece15": 0.02731423542286039, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.22625, + "error_rate_at_confidence_0_9": 0.022099447513812154, + "ordinal_expected_level_mae": null + } + }, + "macro_task_accuracy": 0.9379621229396502, + "macro_task_balanced_accuracy": 0.9185689774676985 + }, + "recalibrated": { + "questions": 14912, + "accuracy": 0.9075912017167382, + "mean_class_recall": 0.8447857522159936, + "nll": 0.2269545932235271, + "brier": 0.12792407717450321, + "ece15": 0.01858051585491712, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.7862124463519313, + "error_rate_at_confidence_0_9": 0.018679631525076765, + "ordinal_expected_level_mae": 0.005329911013904147, + "tasks": { + "agentdojo_tool_outputs:agentdojo_boundary_action": { + "questions": 222, + "accuracy": 0.9774774774774775, + "mean_class_recall": 0.9570930232558139, + "nll": 0.08092683597443519, + "brier": 0.037546344393533104, + "ece15": 0.021320738677048364, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.9819819819819819, + "error_rate_at_confidence_0_9": 0.013761467889908258, + "ordinal_expected_level_mae": null + }, + "agentdojo_tool_outputs:agentdojo_embedded_injection": { + "questions": 222, + "accuracy": 0.9819819819819819, + "mean_class_recall": 0.96, + "nll": 0.07809919245507242, + "brier": 0.030623326048867618, + "ece15": 0.015019163882298042, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.9819819819819819, + "error_rate_at_confidence_0_9": 0.009174311926605505, + "ordinal_expected_level_mae": null + }, + "byzfl_mnist:fl_attack_generated_update": { + "questions": 1400, + "accuracy": 0.9885714285714285, + "mean_class_recall": 0.9821428571428572, + "nll": 0.04094699270351949, + "brier": 0.021018406686723334, + "ece15": 0.0038434318338614677, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.9678571428571429, + "error_rate_at_confidence_0_9": 0.006642066420664207, + "ordinal_expected_level_mae": null + }, + "ciciot2023:ciciot_attack": { + "questions": 5000, + "accuracy": 0.8418, + "mean_class_recall": 0.7537333333333334, + "nll": 0.36679030702727994, + "brier": 0.21757064336558796, + "ece15": 0.043357678681868704, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.6394, + "error_rate_at_confidence_0_9": 0.05004691898654989, + "ordinal_expected_level_mae": null + }, + "injecagent:injecagent_requested_operation_v13": { + "questions": 124, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.0028701157894541487, + "brier": 0.0006216354903482641, + "ece15": 0.0026971564840139535, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.9919354838709677, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_known_logon_action_v16": { + "questions": 328, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.001405432942745028, + "brier": 0.00025083300955486444, + "ece15": 0.0013358908781109478, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.9939024390243902, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_observable_policy": { + "questions": 328, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 4.6272880953518815e-06, + "brier": 9.796066038293659e-10, + "ece15": 4.627043141135246e-06, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 1.0, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "ton_iot_network:ton_attack": { + "questions": 2500, + "accuracy": 0.9792, + "mean_class_recall": 0.9818666666666667, + "nll": 0.0877031817842779, + "brier": 0.03599036114227489, + "ece15": 0.01538869285344686, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.988, + "error_rate_at_confidence_0_9": 0.014574898785425101, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_evidence_level": { + "questions": 797, + "accuracy": 0.9962358845671268, + "mean_class_recall": 0.9963898916967509, + "nll": 0.010603240403009088, + "brier": 0.006373682001028304, + "ece15": 0.004218405852177047, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.9937264742785445, + "error_rate_at_confidence_0_9": 0.0012626262626262627, + "ordinal_expected_level_mae": 0.005329911013904147 + }, + "twins_streamlet:twins_policy_action": { + "questions": 797, + "accuracy": 0.9949811794228356, + "mean_class_recall": 0.9951865222623345, + "nll": 0.01230072195659841, + "brier": 0.007337903591834537, + "ece15": 0.0031241457310850957, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.986198243412798, + "error_rate_at_confidence_0_9": 0.001272264631043257, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_quorum": { + "questions": 797, + "accuracy": 0.9949811794228356, + "mean_class_recall": 0.994276682635671, + "nll": 0.009584146250019401, + "brier": 0.00659904677807738, + "ece15": 0.005028933100804552, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.9899623588456713, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_visible_conflict": { + "questions": 797, + "accuracy": 0.9974905897114178, + "mean_class_recall": 0.998236331569665, + "nll": 0.013437428285294059, + "brier": 0.004858731838764845, + "ece15": 0.0022785888551786117, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.998745294855709, + "error_rate_at_confidence_0_9": 0.002512562814070352, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_message_manipulation": { + "questions": 800, + "accuracy": 0.69, + "mean_class_recall": 0.7116483068135455, + "nll": 0.5102124840713312, + "brier": 0.35753050242002155, + "ece15": 0.03811996421246104, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.22125, + "error_rate_at_confidence_0_9": 0.011299435028248588, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_supported_attack_type_v16": { + "questions": 800, + "accuracy": 0.68875, + "mean_class_recall": 0.529392069171139, + "nll": 0.9911460027249259, + "brier": 0.4737144267281807, + "ece15": 0.09647043512079573, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.18625, + "error_rate_at_confidence_0_9": 0.020134228187919462, + "ordinal_expected_level_mae": null + } + }, + "macro_task_accuracy": 0.9379621229396502, + "macro_task_balanced_accuracy": 0.9185689774676985 + } + }, + "test": { + "raw": { + "questions": 21634, + "accuracy": 0.8801423684940372, + "mean_class_recall": 0.8422636415920437, + "nll": 0.35756736839135106, + "brier": 0.17329397786588344, + "ece15": 0.018370944185424988, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.681750947582509, + "error_rate_at_confidence_0_9": 0.017289307749677945, + "ordinal_expected_level_mae": 0.004830545729071066, + "tasks": { + "agentdojo_tool_outputs:agentdojo_boundary_action": { + "questions": 426, + "accuracy": 0.9953051643192489, + "mean_class_recall": 0.9895833333333333, + "nll": 0.019578322598972707, + "brier": 0.008470972915844083, + "ece15": 0.0032192125141472924, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9882629107981221, + "error_rate_at_confidence_0_9": 0.0023752969121140144, + "ordinal_expected_level_mae": null + }, + "agentdojo_tool_outputs:agentdojo_embedded_injection": { + "questions": 426, + "accuracy": 0.9835680751173709, + "mean_class_recall": 0.9635416666666667, + "nll": 0.03323309985013501, + "brier": 0.020700086530374747, + "ece15": 0.012858446963474107, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9835680751173709, + "error_rate_at_confidence_0_9": 0.007159904534606206, + "ordinal_expected_level_mae": null + }, + "byzfl_mnist:fl_attack_generated_update": { + "questions": 800, + "accuracy": 0.95875, + "mean_class_recall": 0.9296875, + "nll": 0.14798192410675134, + "brier": 0.07208296294789789, + "ece15": 0.031237402444916955, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9575, + "error_rate_at_confidence_0_9": 0.02741514360313316, + "ordinal_expected_level_mae": null + }, + "ciciot2023:ciciot_attack": { + "questions": 6000, + "accuracy": 0.9171666666666667, + "mean_class_recall": 0.8741111111111111, + "nll": 0.2872065840367128, + "brier": 0.1291518236859529, + "ece15": 0.02997403903668212, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.7491666666666666, + "error_rate_at_confidence_0_9": 0.03181312569521691, + "ordinal_expected_level_mae": null + }, + "injecagent:injecagent_requested_operation_v13": { + "questions": 124, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 6.416181236130915e-05, + "brier": 3.664856836177276e-07, + "ece15": 6.406994017019252e-05, + "temperature": 1.0, + "coverage_at_confidence_0_9": 1.0, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_known_logon_action_v16": { + "questions": 540, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.0003909349271697467, + "brier": 5.411140895974333e-05, + "ece15": 0.000376397356322764, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9981481481481481, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_observable_policy": { + "questions": 540, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 1.9301873747749096e-07, + "brier": 4.461551704214049e-12, + "ece15": 1.9301762319156735e-07, + "temperature": 1.0, + "coverage_at_confidence_0_9": 1.0, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "ton_iot_network:ton_attack": { + "questions": 3500, + "accuracy": 0.988, + "mean_class_recall": 0.9820952380952381, + "nll": 0.052315917826989855, + "brier": 0.021002166306533596, + "ece15": 0.005742982405630235, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9757142857142858, + "error_rate_at_confidence_0_9": 0.007320644216691069, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_evidence_level": { + "questions": 659, + "accuracy": 0.9954476479514416, + "mean_class_recall": 0.9949748743718594, + "nll": 0.006489090778116727, + "brier": 0.004985928434486637, + "ece15": 0.0035010905751582675, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9954476479514416, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": 0.004830545729071066 + }, + "twins_streamlet:twins_policy_action": { + "questions": 659, + "accuracy": 0.9984825493171472, + "mean_class_recall": 0.9983249581239532, + "nll": 0.007759166929265847, + "brier": 0.0031279766576844436, + "ece15": 0.0017302646891685911, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9969650986342944, + "error_rate_at_confidence_0_9": 0.0015220700152207, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_quorum": { + "questions": 659, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.0012916521858483854, + "brier": 0.0006719742504699832, + "ece15": 0.0010543087689309648, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9969650986342944, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_visible_conflict": { + "questions": 659, + "accuracy": 0.9984825493171472, + "mean_class_recall": 0.9989130434782609, + "nll": 0.0030258654215359524, + "brier": 0.0020473738981638956, + "ece15": 0.0002839890142556098, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9954476479514416, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_message_manipulation": { + "questions": 3321, + "accuracy": 0.7034025895814514, + "mean_class_recall": 0.7106693655944849, + "nll": 0.5271449745997652, + "brier": 0.3679898300956757, + "ece15": 0.03254066478455375, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.21499548328816623, + "error_rate_at_confidence_0_9": 0.011204481792717087, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_supported_attack_type_v16": { + "questions": 3321, + "accuracy": 0.692261367058115, + "mean_class_recall": 0.49711800080440777, + "nll": 1.181957709780574, + "brier": 0.48216427161657194, + "ece15": 0.04703079874679226, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.20776874435411022, + "error_rate_at_confidence_0_9": 0.07681159420289856, + "ordinal_expected_level_mae": null + } + }, + "macro_task_accuracy": 0.9450619006663279, + "macro_task_balanced_accuracy": 0.9242156493985226 + }, + "recalibrated": { + "questions": 21634, + "accuracy": 0.8801423684940372, + "mean_class_recall": 0.8422636415920437, + "nll": 0.3441350952603644, + "brier": 0.1767857045083663, + "ece15": 0.035627089616313175, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.6567440140519553, + "error_rate_at_confidence_0_9": 0.014358108108108109, + "ordinal_expected_level_mae": 0.004973955616734103, + "tasks": { + "agentdojo_tool_outputs:agentdojo_boundary_action": { + "questions": 426, + "accuracy": 0.9953051643192489, + "mean_class_recall": 0.9895833333333333, + "nll": 0.02085470758455793, + "brier": 0.009033562612123906, + "ece15": 0.005062121140742335, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.9741784037558685, + "error_rate_at_confidence_0_9": 0.0024096385542168677, + "ordinal_expected_level_mae": null + }, + "agentdojo_tool_outputs:agentdojo_embedded_injection": { + "questions": 426, + "accuracy": 0.9835680751173709, + "mean_class_recall": 0.9635416666666667, + "nll": 0.029576905474440387, + "brier": 0.019371229609764937, + "ece15": 0.0128149515297424, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.9741784037558685, + "error_rate_at_confidence_0_9": 0.004819277108433735, + "ordinal_expected_level_mae": null + }, + "byzfl_mnist:fl_attack_generated_update": { + "questions": 800, + "accuracy": 0.95875, + "mean_class_recall": 0.9296875, + "nll": 0.1252495255860491, + "brier": 0.06891977027707019, + "ece15": 0.02206119521855819, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.93375, + "error_rate_at_confidence_0_9": 0.0214190093708166, + "ordinal_expected_level_mae": null + }, + "ciciot2023:ciciot_attack": { + "questions": 6000, + "accuracy": 0.9171666666666667, + "mean_class_recall": 0.8741111111111111, + "nll": 0.258923302286342, + "brier": 0.1323925339560934, + "ece15": 0.040786104751424794, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.6995, + "error_rate_at_confidence_0_9": 0.027162258756254467, + "ordinal_expected_level_mae": null + }, + "injecagent:injecagent_requested_operation_v13": { + "questions": 124, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.00036120828507376734, + "brier": 6.118819643947097e-06, + "ece15": 0.0003596634947089362, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 1.0, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_known_logon_action_v16": { + "questions": 540, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.0010259839180316675, + "brier": 0.00015355821904600842, + "ece15": 0.0009831313211751498, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.9981481481481481, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_observable_policy": { + "questions": 540, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 5.4752266593721125e-06, + "brier": 1.1349090161477055e-09, + "ece15": 5.474942881804694e-06, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 1.0, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "ton_iot_network:ton_attack": { + "questions": 3500, + "accuracy": 0.988, + "mean_class_recall": 0.9820952380952381, + "nll": 0.0471904212042291, + "brier": 0.02136284177729358, + "ece15": 0.004930332206950378, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.9597142857142857, + "error_rate_at_confidence_0_9": 0.006251860672819291, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_evidence_level": { + "questions": 659, + "accuracy": 0.9954476479514416, + "mean_class_recall": 0.9949748743718594, + "nll": 0.005989405513587896, + "brier": 0.004363630530423605, + "ece15": 0.003686325313576514, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.9954476479514416, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": 0.004973955616734103 + }, + "twins_streamlet:twins_policy_action": { + "questions": 659, + "accuracy": 0.9984825493171472, + "mean_class_recall": 0.9983249581239532, + "nll": 0.0069631222649601515, + "brier": 0.003204452557959292, + "ece15": 0.001531985329986691, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.9969650986342944, + "error_rate_at_confidence_0_9": 0.0015220700152207, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_quorum": { + "questions": 659, + "accuracy": 1.0, + "mean_class_recall": 1.0, + "nll": 0.0018952623564185236, + "brier": 0.0008114028234666237, + "ece15": 0.0016108880631044219, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.9954476479514416, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_visible_conflict": { + "questions": 659, + "accuracy": 0.9984825493171472, + "mean_class_recall": 0.9989130434782609, + "nll": 0.0033168872089549845, + "brier": 0.0020980871703588605, + "ece15": 0.0012518453771527926, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.9954476479514416, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_message_manipulation": { + "questions": 3321, + "accuracy": 0.7034025895814514, + "mean_class_recall": 0.7106693655944849, + "nll": 0.5278136514287293, + "brier": 0.3689683794121852, + "ece15": 0.03240031037986953, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.1969286359530262, + "error_rate_at_confidence_0_9": 0.009174311926605505, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_supported_attack_type_v16": { + "questions": 3321, + "accuracy": 0.692261367058115, + "mean_class_recall": 0.49711800080440777, + "nll": 1.1560335557868109, + "brier": 0.498611302076152, + "ece15": 0.12617627958797675, + "temperature": 1.334968103387446, + "coverage_at_confidence_0_9": 0.17856067449563384, + "error_rate_at_confidence_0_9": 0.07251264755480608, + "ordinal_expected_level_mae": null + } + }, + "macro_task_accuracy": 0.9450619006663279, + "macro_task_balanced_accuracy": 0.9242156493985226 + } + }, + "general": { + "decision-v7": { + "raw_clean": { + "n": 1200, + "nll": 0.5734957671127382, + "acc": 0.8658333333333333, + "ece": 0.08617509900940432, + "brier": 0.21792288610472857, + "mean_conf": 0.9509841477415141, + "confident_error_rate": 0.0675, + "coverage_at_0_9": 0.8525, + "accuracy_at_0_9": 0.9208211143695014, + "coverage_at_5pct_error": 0.7833333333333333, + "coverage_at_1pct_error": 0.6166666666666667, + "aurc": 0.0281999905676559, + "error_rate_at_0_9": 0.07917888563049853, + "confidence_bias": 0.08515081440818073, + "top_bins": { + "0.9": { + "n": 1023, + "errors": 81, + "error_rate": 0.07917888563049853 + }, + "0.95": { + "n": 962, + "errors": 53, + "error_rate": 0.0550935550935551 + }, + "0.99": { + "n": 885, + "errors": 34, + "error_rate": 0.0384180790960452 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9933333333333333, + "confidence_cutoff": 0.9999894749108045 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.9447916666666667, + "confidence_cutoff": 0.9524106736120255 + } + }, + "score_mae": 0.4625639513458072, + "ranked_probability_score": 0.08810095245487483 + }, + "as_shipped_clean": { + "n": 1200, + "nll": 0.46540017253913407, + "acc": 0.8658333333333333, + "ece": 0.07412868458003202, + "brier": 0.20783721757683873, + "mean_conf": 0.9351787338923708, + "confident_error_rate": 0.041666666666666664, + "coverage_at_0_9": 0.7983333333333333, + "accuracy_at_0_9": 0.9478079331941545, + "coverage_at_5pct_error": 0.7783333333333333, + "coverage_at_1pct_error": 0.6141666666666666, + "aurc": 0.02814862771926311, + "error_rate_at_0_9": 0.05219206680584551, + "confidence_bias": 0.06934540055903748, + "top_bins": { + "0.9": { + "n": 958, + "errors": 50, + "error_rate": 0.05219206680584551 + }, + "0.95": { + "n": 912, + "errors": 40, + "error_rate": 0.043859649122807015 + }, + "0.99": { + "n": 822, + "errors": 18, + "error_rate": 0.021897810218978103 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9933333333333333, + "confidence_cutoff": 0.9998006494793819 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.9458333333333333, + "confidence_cutoff": 0.8959586216727804 + } + }, + "score_mae": 0.47318044838075235, + "ranked_probability_score": 0.08431774694216215 + }, + "sources": { + "agnews": { + "n": 240, + "nll": 0.2925312694754093, + "acc": 0.9166666666666666, + "ece": 0.05790914903059574, + "brier": 0.1357836869201196, + "mean_conf": 0.9733704974572437, + "confident_error_rate": 0.04583333333333333, + "coverage_at_0_9": 0.9291666666666667, + "accuracy_at_0_9": 0.9506726457399103, + "coverage_at_5pct_error": 0.9333333333333333, + "coverage_at_1pct_error": 0.7583333333333333, + "aurc": 0.011635752911899793, + "error_rate_at_0_9": 0.04932735426008968, + "confidence_bias": 0.05670383079057706, + "top_bins": { + "0.9": { + "n": 223, + "errors": 11, + "error_rate": 0.04932735426008968 + }, + "0.95": { + "n": 217, + "errors": 9, + "error_rate": 0.041474654377880185 + }, + "0.99": { + "n": 197, + "errors": 4, + "error_rate": 0.02030456852791878 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9996851444568575 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.984375, + "confidence_cutoff": 0.9921275226980688 + } + } + }, + "amazon": { + "n": 80, + "nll": 1.2207591140196519, + "acc": 0.6125, + "ece": 0.1615943676102411, + "brier": 0.5707631816876242, + "mean_conf": 0.7431377036571911, + "confident_error_rate": 0.0375, + "coverage_at_0_9": 0.2375, + "accuracy_at_0_9": 0.8421052631578947, + "coverage_at_5pct_error": 0.075, + "coverage_at_1pct_error": 0.075, + "aurc": 0.272315413996461, + "error_rate_at_0_9": 0.15789473684210525, + "confidence_bias": 0.13063770365719107, + "top_bins": { + "0.9": { + "n": 19, + "errors": 3, + "error_rate": 0.15789473684210525 + }, + "0.95": { + "n": 14, + "errors": 2, + "error_rate": 0.14285714285714285 + }, + "0.99": { + "n": 3, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.7, + "confidence_cutoff": 0.7847517191083221 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.640625, + "confidence_cutoff": 0.5534749021853737 + } + }, + "score_mae": 0.5042438581009662, + "ranked_probability_score": 0.09073105215000309 + }, + "banking77": { + "n": 80, + "nll": 0.6278787803563556, + "acc": 0.8375, + "ece": 0.1237375681167378, + "brier": 0.25438988606734264, + "mean_conf": 0.8981608539632144, + "confident_error_rate": 0.05, + "coverage_at_0_9": 0.7625, + "accuracy_at_0_9": 0.9344262295081968, + "coverage_at_5pct_error": 0.6875, + "coverage_at_1pct_error": 0.6125, + "aurc": 0.035783745285874746, + "error_rate_at_0_9": 0.06557377049180328, + "confidence_bias": 0.06066085396321441, + "top_bins": { + "0.9": { + "n": 61, + "errors": 4, + "error_rate": 0.06557377049180328 + }, + "0.95": { + "n": 57, + "errors": 4, + "error_rate": 0.07017543859649122 + }, + "0.99": { + "n": 48, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9966194161989207 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.90625, + "confidence_cutoff": 0.8503200856915479 + } + } + }, + "boolq": { + "n": 80, + "nll": 0.6125951836430432, + "acc": 0.875, + "ece": 0.11756712414134643, + "brier": 0.22505069986723222, + "mean_conf": 0.9514740522819943, + "confident_error_rate": 0.0875, + "coverage_at_0_9": 0.85, + "accuracy_at_0_9": 0.8970588235294118, + "coverage_at_5pct_error": 0.6875, + "coverage_at_1pct_error": 0.1125, + "aurc": 0.0511510509173725, + "error_rate_at_0_9": 0.10294117647058823, + "confidence_bias": 0.07647405228199433, + "top_bins": { + "0.9": { + "n": 68, + "errors": 7, + "error_rate": 0.10294117647058823 + }, + "0.95": { + "n": 67, + "errors": 7, + "error_rate": 0.1044776119402985 + }, + "0.99": { + "n": 61, + "errors": 4, + "error_rate": 0.06557377049180328 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.975, + "confidence_cutoff": 0.9999426387743003 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.921875, + "confidence_cutoff": 0.9819850746275051 + } + } + }, + "contrastive": { + "n": 160, + "nll": 0.07962468888836484, + "acc": 0.96875, + "ece": 0.026633113578874344, + "brier": 0.05007247893674288, + "mean_conf": 0.9953831135788744, + "confident_error_rate": 0.0125, + "coverage_at_0_9": 0.98125, + "accuracy_at_0_9": 0.9872611464968153, + "coverage_at_5pct_error": 1.0, + "coverage_at_1pct_error": 0.96875, + "aurc": 0.000752718799433766, + "error_rate_at_0_9": 0.012738853503184714, + "confidence_bias": 0.026633113578874434, + "top_bins": { + "0.9": { + "n": 157, + "errors": 2, + "error_rate": 0.012738853503184714 + }, + "0.95": { + "n": 156, + "errors": 2, + "error_rate": 0.01282051282051282 + }, + "0.99": { + "n": 148, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9999957907736378 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 1.0, + "confidence_cutoff": 0.9999070137203582 + } + } + }, + "dbpedia14": { + "n": 80, + "nll": 0.02165084849307894, + "acc": 0.9875, + "ece": 0.01152608625473029, + "brier": 0.016104080808397182, + "mean_conf": 0.9959462330871315, + "confident_error_rate": 0.0, + "coverage_at_0_9": 0.9875, + "accuracy_at_0_9": 1.0, + "coverage_at_5pct_error": 1.0, + "coverage_at_1pct_error": 0.9875, + "aurc": 0.00015625, + "error_rate_at_0_9": 0.0, + "confidence_bias": 0.008446233087131438, + "top_bins": { + "0.9": { + "n": 79, + "errors": 0, + "error_rate": 0.0 + }, + "0.95": { + "n": 78, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 77, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9999981627832084 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 1.0, + "confidence_cutoff": 0.9999896312065648 + } + } + }, + "imdb": { + "n": 80, + "nll": 0.06521915089368877, + "acc": 0.9625, + "ece": 0.02850319056358286, + "brier": 0.04639252879503235, + "mean_conf": 0.985401588169408, + "confident_error_rate": 0.0, + "coverage_at_0_9": 0.95, + "accuracy_at_0_9": 1.0, + "coverage_at_5pct_error": 1.0, + "coverage_at_1pct_error": 0.95, + "aurc": 0.0012680561790530144, + "error_rate_at_0_9": 0.0, + "confidence_bias": 0.022901588169408016, + "top_bins": { + "0.9": { + "n": 76, + "errors": 0, + "error_rate": 0.0 + }, + "0.95": { + "n": 75, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 71, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9999903501374642 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 1.0, + "confidence_cutoff": 0.9998069111372814 + } + } + }, + "mnli": { + "n": 80, + "nll": 0.5051164719750952, + "acc": 0.9125, + "ece": 0.08269743141349266, + "brier": 0.16426794867102618, + "mean_conf": 0.9883215126078937, + "confident_error_rate": 0.075, + "coverage_at_0_9": 0.95, + "accuracy_at_0_9": 0.9210526315789473, + "coverage_at_5pct_error": 0.8875, + "coverage_at_1pct_error": 0.65, + "aurc": 0.015551107945666897, + "error_rate_at_0_9": 0.07894736842105263, + "confidence_bias": 0.07582151260789372, + "top_bins": { + "0.9": { + "n": 76, + "errors": 6, + "error_rate": 0.07894736842105263 + }, + "0.95": { + "n": 74, + "errors": 5, + "error_rate": 0.06756756756756757 + }, + "0.99": { + "n": 72, + "errors": 4, + "error_rate": 0.05555555555555555 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9999963968506167 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.96875, + "confidence_cutoff": 0.9998914656545862 + } + } + }, + "sst5": { + "n": 80, + "nll": 1.2493438944839201, + "acc": 0.6125, + "ece": 0.24936946322063214, + "brier": 0.5917259991138234, + "mean_conf": 0.8126564640723977, + "confident_error_rate": 0.075, + "coverage_at_0_9": 0.3, + "accuracy_at_0_9": 0.75, + "coverage_at_5pct_error": 0.075, + "coverage_at_1pct_error": 0.075, + "aurc": 0.3025049973121495, + "error_rate_at_0_9": 0.25, + "confidence_bias": 0.20015646407239762, + "top_bins": { + "0.9": { + "n": 24, + "errors": 6, + "error_rate": 0.25 + }, + "0.95": { + "n": 10, + "errors": 1, + "error_rate": 0.1 + }, + "0.99": { + "n": 0, + "errors": 0, + "error_rate": null + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.625, + "confidence_cutoff": 0.8548749588336553 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.625, + "confidence_cutoff": 0.7024572592103087 + } + }, + "score_mae": 0.48716165684142193, + "ranked_probability_score": 0.08942157579848162 + }, + "trec": { + "n": 80, + "nll": 0.2837433451436081, + "acc": 0.9625, + "ece": 0.028515907037196753, + "brier": 0.057680311047061936, + "mean_conf": 0.9910159070371967, + "confident_error_rate": 0.025, + "coverage_at_0_9": 0.9875, + "accuracy_at_0_9": 0.9746835443037974, + "coverage_at_5pct_error": 1.0, + "coverage_at_1pct_error": 0.275, + "aurc": 0.01757155053741791, + "error_rate_at_0_9": 0.02531645569620253, + "confidence_bias": 0.02851590703719664, + "top_bins": { + "0.9": { + "n": 79, + "errors": 2, + "error_rate": 0.02531645569620253 + }, + "0.95": { + "n": 78, + "errors": 2, + "error_rate": 0.02564102564102564 + }, + "0.99": { + "n": 77, + "errors": 2, + "error_rate": 0.025974025974025976 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.975, + "confidence_cutoff": 0.9999924765263039 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.984375, + "confidence_cutoff": 0.9999100942077365 + } + } + }, + "yelp": { + "n": 160, + "nll": 0.6789263064378058, + "acc": 0.76875, + "ece": 0.10810206862028397, + "brier": 0.34184380448059815, + "mean_conf": 0.8753444869898281, + "confident_error_rate": 0.05625, + "coverage_at_0_9": 0.6, + "accuracy_at_0_9": 0.90625, + "coverage_at_5pct_error": 0.41875, + "coverage_at_1pct_error": 0.1875, + "aurc": 0.08672761133002724, + "error_rate_at_0_9": 0.09375, + "confidence_bias": 0.1065944869898281, + "top_bins": { + "0.9": { + "n": 96, + "errors": 9, + "error_rate": 0.09375 + }, + "0.95": { + "n": 86, + "errors": 8, + "error_rate": 0.09302325581395349 + }, + "0.99": { + "n": 68, + "errors": 4, + "error_rate": 0.058823529411764705 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.925, + "confidence_cutoff": 0.971684991366676 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.84375, + "confidence_cutoff": 0.7267623543739319 + } + }, + "score_mae": 0.428135830199869, + "ranked_probability_score": 0.07280061287800174 + } + }, + "types": { + "choice": { + "n": 480, + "nll": 0.3225147096064765, + "acc": 0.9291666666666667, + "ece": 0.05269727298309242, + "brier": 0.11939209773780447, + "mean_conf": 0.9740635473132214, + "confident_error_rate": 0.0375, + "coverage_at_0_9": 0.9333333333333333, + "accuracy_at_0_9": 0.9598214285714286, + "coverage_at_5pct_error": 0.9479166666666666, + "coverage_at_1pct_error": 0.7354166666666667, + "aurc": 0.010523805656224608, + "error_rate_at_0_9": 0.04017857142857143, + "confidence_bias": 0.04489688064655473, + "top_bins": { + "0.9": { + "n": 448, + "errors": 18, + "error_rate": 0.04017857142857143 + }, + "0.95": { + "n": 439, + "errors": 17, + "error_rate": 0.0387243735763098 + }, + "0.99": { + "n": 416, + "errors": 8, + "error_rate": 0.019230769230769232 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9958333333333333, + "confidence_cutoff": 0.999989087291256 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.9895833333333334, + "confidence_cutoff": 0.9991522595300467 + } + } + }, + "noul": { + "n": 480, + "nll": 0.2630675588624098, + "acc": 0.925, + "ece": 0.048426932319500904, + "brier": 0.11894188754077414, + "mean_conf": 0.9734269323195011, + "confident_error_rate": 0.035416666666666666, + "coverage_at_0_9": 0.91875, + "accuracy_at_0_9": 0.9614512471655329, + "coverage_at_5pct_error": 0.9416666666666667, + "coverage_at_1pct_error": 0.71875, + "aurc": 0.014292966145900442, + "error_rate_at_0_9": 0.03854875283446712, + "confidence_bias": 0.048426932319501015, + "top_bins": { + "0.9": { + "n": 441, + "errors": 17, + "error_rate": 0.03854875283446712 + }, + "0.95": { + "n": 431, + "errors": 15, + "error_rate": 0.03480278422273782 + }, + "0.99": { + "n": 397, + "errors": 9, + "error_rate": 0.022670025188916875 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9958333333333333, + "confidence_cutoff": 0.9999067506240331 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.9791666666666666, + "confidence_cutoff": 0.9950087776054062 + } + } + }, + "score": { + "n": 240, + "nll": 1.1558363257578976, + "acc": 0.6208333333333333, + "ece": 0.1746159580142597, + "brier": 0.5625181173270365, + "mean_conf": 0.7809127101964097, + "confident_error_rate": 0.0625, + "coverage_at_0_9": 0.2875, + "accuracy_at_0_9": 0.782608695652174, + "coverage_at_5pct_error": 0.016666666666666666, + "coverage_at_1pct_error": 0.016666666666666666, + "aurc": 0.28163895271092954, + "error_rate_at_0_9": 0.21739130434782608, + "confidence_bias": 0.16007937686307638, + "top_bins": { + "0.9": { + "n": 69, + "errors": 15, + "error_rate": 0.21739130434782608 + }, + "0.95": { + "n": 42, + "errors": 8, + "error_rate": 0.19047619047619047 + }, + "0.99": { + "n": 9, + "errors": 1, + "error_rate": 0.1111111111111111 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.6916666666666667, + "confidence_cutoff": 0.8195696673851967 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.6614583333333334, + "confidence_cutoff": 0.6215591950077792 + } + }, + "score_mae": 0.47318044838075235, + "ranked_probability_score": 0.08431774694216215 + } + } + }, + "transfer-v4": { + "raw_clean": { + "n": 656, + "nll": 0.840653344789253, + "acc": 0.8384146341463414, + "ece": 0.11570395215040587, + "brier": 0.2772648298322645, + "mean_conf": 0.9433821481191502, + "confident_error_rate": 0.0899390243902439, + "coverage_at_0_9": 0.8429878048780488, + "accuracy_at_0_9": 0.8933092224231465, + "coverage_at_5pct_error": 0.5777439024390244, + "coverage_at_1pct_error": 0.3551829268292683, + "aurc": 0.0494438083179723, + "error_rate_at_0_9": 0.10669077757685352, + "confidence_bias": 0.10496751397280879, + "top_bins": { + "0.9": { + "n": 553, + "errors": 59, + "error_rate": 0.10669077757685352 + }, + "0.95": { + "n": 527, + "errors": 54, + "error_rate": 0.10246679316888045 + }, + "0.99": { + "n": 472, + "errors": 34, + "error_rate": 0.07203389830508475 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9634146341463414, + "confidence_cutoff": 0.9999382193883597 + }, + "0.8": { + "coverage": 0.8003048780487805, + "accuracy": 0.900952380952381, + "confidence_cutoff": 0.951274711387887 + } + }, + "score_mae": 0.2762329771748361, + "ranked_probability_score": 0.12228321967451691 + }, + "as_shipped_clean": { + "n": 656, + "nll": 0.6710659810048166, + "acc": 0.8384146341463414, + "ece": 0.09231046175305074, + "brier": 0.26721979560494846, + "mean_conf": 0.9260829686930101, + "confident_error_rate": 0.07926829268292683, + "coverage_at_0_9": 0.7987804878048781, + "accuracy_at_0_9": 0.9007633587786259, + "coverage_at_5pct_error": 0.5746951219512195, + "coverage_at_1pct_error": 0.3551829268292683, + "aurc": 0.049407167334717506, + "error_rate_at_0_9": 0.09923664122137404, + "confidence_bias": 0.08766833454666867, + "top_bins": { + "0.9": { + "n": 524, + "errors": 52, + "error_rate": 0.09923664122137404 + }, + "0.95": { + "n": 493, + "errors": 42, + "error_rate": 0.08519269776876268 + }, + "0.99": { + "n": 422, + "errors": 23, + "error_rate": 0.054502369668246446 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9573170731707317, + "confidence_cutoff": 0.9992490894147191 + }, + "0.8": { + "coverage": 0.8003048780487805, + "accuracy": 0.8990476190476191, + "confidence_cutoff": 0.8992331757610149 + } + }, + "score_mae": 0.27589619803682164, + "ranked_probability_score": 0.11777542530670457 + }, + "sources": { + "composition_holdout": { + "n": 96, + "nll": 0.946778069631199, + "acc": 0.8645833333333334, + "ece": 0.14288844371789414, + "brier": 0.2646184933261653, + "mean_conf": 0.9787006948112301, + "confident_error_rate": 0.11458333333333333, + "coverage_at_0_9": 0.9479166666666666, + "accuracy_at_0_9": 0.8791208791208791, + "coverage_at_5pct_error": 0.6875, + "coverage_at_1pct_error": 0.65625, + "aurc": 0.032783504799663264, + "error_rate_at_0_9": 0.12087912087912088, + "confidence_bias": 0.11411736147789675, + "top_bins": { + "0.9": { + "n": 91, + "errors": 11, + "error_rate": 0.12087912087912088 + }, + "0.95": { + "n": 90, + "errors": 11, + "error_rate": 0.12222222222222222 + }, + "0.99": { + "n": 89, + "errors": 11, + "error_rate": 0.12359550561797752 + } + }, + "selective": { + "0.5": { + "coverage": 0.5104166666666666, + "accuracy": 1.0, + "confidence_cutoff": 0.9999696869828117 + }, + "0.8": { + "coverage": 0.8125, + "accuracy": 0.8974358974358975, + "confidence_cutoff": 0.9992490894147191 + } + } + }, + "contrastive": { + "n": 80, + "nll": 0.5115728584657595, + "acc": 0.875, + "ece": 0.13006292045346943, + "brier": 0.23406192561698297, + "mean_conf": 0.9705133073135578, + "confident_error_rate": 0.0875, + "coverage_at_0_9": 0.9125, + "accuracy_at_0_9": 0.9041095890410958, + "coverage_at_5pct_error": 0.85, + "coverage_at_1pct_error": 0.8, + "aurc": 0.0169297143647776, + "error_rate_at_0_9": 0.0958904109589041, + "confidence_bias": 0.09551330731355778, + "top_bins": { + "0.9": { + "n": 73, + "errors": 7, + "error_rate": 0.0958904109589041 + }, + "0.95": { + "n": 71, + "errors": 6, + "error_rate": 0.08450704225352113 + }, + "0.99": { + "n": 68, + "errors": 3, + "error_rate": 0.04411764705882353 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9999941701311645 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 1.0, + "confidence_cutoff": 0.9997393381382442 + } + }, + "score_mae": 0.27589619803682164, + "ranked_probability_score": 0.11777542530670457 + }, + "emotion": { + "n": 80, + "nll": 1.2094004500158717, + "acc": 0.675, + "ece": 0.19812590622433582, + "brier": 0.4998620899742045, + "mean_conf": 0.7808203087114324, + "confident_error_rate": 0.075, + "coverage_at_0_9": 0.4375, + "accuracy_at_0_9": 0.8285714285714286, + "coverage_at_5pct_error": 0.075, + "coverage_at_1pct_error": 0.075, + "aurc": 0.1832973887490438, + "error_rate_at_0_9": 0.17142857142857143, + "confidence_bias": 0.10582030871143233, + "top_bins": { + "0.9": { + "n": 35, + "errors": 6, + "error_rate": 0.17142857142857143 + }, + "0.95": { + "n": 30, + "errors": 4, + "error_rate": 0.13333333333333333 + }, + "0.99": { + "n": 12, + "errors": 1, + "error_rate": 0.08333333333333333 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.8, + "confidence_cutoff": 0.8500919215367448 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.734375, + "confidence_cutoff": 0.5761070047579089 + } + } + }, + "mmlu": { + "n": 80, + "nll": 0.9027907855123264, + "acc": 0.6875, + "ece": 0.14251964152159943, + "brier": 0.4317941195669751, + "mean_conf": 0.8238540468148077, + "confident_error_rate": 0.075, + "coverage_at_0_9": 0.5, + "accuracy_at_0_9": 0.85, + "coverage_at_5pct_error": 0.375, + "coverage_at_1pct_error": 0.325, + "aurc": 0.11589530494156297, + "error_rate_at_0_9": 0.15, + "confidence_bias": 0.13635404681480767, + "top_bins": { + "0.9": { + "n": 40, + "errors": 6, + "error_rate": 0.15 + }, + "0.95": { + "n": 30, + "errors": 1, + "error_rate": 0.03333333333333333 + }, + "0.99": { + "n": 19, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.85, + "confidence_cutoff": 0.9017783630155174 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.8125, + "confidence_cutoff": 0.613701094142181 + } + } + }, + "paws": { + "n": 80, + "nll": 0.9771542282927816, + "acc": 0.8125, + "ece": 0.1651648000763565, + "brier": 0.31695123775026923, + "mean_conf": 0.9524627544936155, + "confident_error_rate": 0.125, + "coverage_at_0_9": 0.8625, + "accuracy_at_0_9": 0.855072463768116, + "coverage_at_5pct_error": 0.15, + "coverage_at_1pct_error": 0.15, + "aurc": 0.0997884943666906, + "error_rate_at_0_9": 0.14492753623188406, + "confidence_bias": 0.1399627544936155, + "top_bins": { + "0.9": { + "n": 69, + "errors": 10, + "error_rate": 0.14492753623188406 + }, + "0.95": { + "n": 69, + "errors": 10, + "error_rate": 0.14492753623188406 + }, + "0.99": { + "n": 61, + "errors": 6, + "error_rate": 0.09836065573770492 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.875, + "confidence_cutoff": 0.9997721781585782 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.890625, + "confidence_cutoff": 0.9846682768581441 + } + } + }, + "qnli": { + "n": 80, + "nll": 0.32010583310414403, + "acc": 0.925, + "ece": 0.06578826678776872, + "brier": 0.13874220003757964, + "mean_conf": 0.9719094562930305, + "confident_error_rate": 0.05, + "coverage_at_0_9": 0.925, + "accuracy_at_0_9": 0.9459459459459459, + "coverage_at_5pct_error": 0.8625, + "coverage_at_1pct_error": 0.6, + "aurc": 0.015384772492900507, + "error_rate_at_0_9": 0.05405405405405406, + "confidence_bias": 0.04690945629303045, + "top_bins": { + "0.9": { + "n": 74, + "errors": 4, + "error_rate": 0.05405405405405406 + }, + "0.95": { + "n": 72, + "errors": 4, + "error_rate": 0.05555555555555555 + }, + "0.99": { + "n": 66, + "errors": 2, + "error_rate": 0.030303030303030304 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9997714445133897 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.96875, + "confidence_cutoff": 0.9937610824296935 + } + } + }, + "sciq": { + "n": 80, + "nll": 0.00696809639784113, + "acc": 1.0, + "ece": 0.006546558428673463, + "brier": 0.0013901896311441294, + "mean_conf": 0.9934534415713265, + "confident_error_rate": 0.0, + "coverage_at_0_9": 0.975, + "accuracy_at_0_9": 1.0, + "coverage_at_5pct_error": 1.0, + "coverage_at_1pct_error": 1.0, + "aurc": 0.0, + "error_rate_at_0_9": 0.0, + "confidence_bias": -0.006546558428673466, + "top_bins": { + "0.9": { + "n": 78, + "errors": 0, + "error_rate": 0.0 + }, + "0.95": { + "n": 76, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 72, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9999699221173486 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 1.0, + "confidence_cutoff": 0.9997295605534583 + } + } + }, + "tweet_offensive": { + "n": 80, + "nll": 0.4386151088933339, + "acc": 0.8625, + "ece": 0.10954228653253775, + "brier": 0.2508583693920234, + "mean_conf": 0.926426194311437, + "confident_error_rate": 0.1, + "coverage_at_0_9": 0.8, + "accuracy_at_0_9": 0.875, + "coverage_at_5pct_error": 0.575, + "coverage_at_1pct_error": 0.4875, + "aurc": 0.05180455375068149, + "error_rate_at_0_9": 0.125, + "confidence_bias": 0.06392619431143698, + "top_bins": { + "0.9": { + "n": 64, + "errors": 8, + "error_rate": 0.125 + }, + "0.95": { + "n": 55, + "errors": 6, + "error_rate": 0.10909090909090909 + }, + "0.99": { + "n": 35, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.975, + "confidence_cutoff": 0.9842379652550892 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.875, + "confidence_cutoff": 0.902661270668069 + } + } + } + }, + "types": { + "choice": { + "n": 336, + "nll": 0.7750697655913044, + "acc": 0.8095238095238095, + "ece": 0.09777001761032751, + "brier": 0.2977591883723148, + "mean_conf": 0.8982306263978672, + "confident_error_rate": 0.06845238095238096, + "coverage_at_0_9": 0.7261904761904762, + "accuracy_at_0_9": 0.9057377049180327, + "coverage_at_5pct_error": 0.4583333333333333, + "coverage_at_1pct_error": 0.3630952380952381, + "aurc": 0.05730883047817376, + "error_rate_at_0_9": 0.0942622950819672, + "confidence_bias": 0.08870681687405768, + "top_bins": { + "0.9": { + "n": 244, + "errors": 23, + "error_rate": 0.0942622950819672 + }, + "0.95": { + "n": 226, + "errors": 16, + "error_rate": 0.07079646017699115 + }, + "0.99": { + "n": 192, + "errors": 12, + "error_rate": 0.0625 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9464285714285714, + "confidence_cutoff": 0.9979451630958165 + }, + "0.8": { + "coverage": 0.8005952380952381, + "accuracy": 0.8921933085501859, + "confidence_cutoff": 0.805212947551472 + } + } + }, + "noul": { + "n": 280, + "nll": 0.4959650021746835, + "acc": 0.8857142857142857, + "ece": 0.08836865713253901, + "brier": 0.20187194492263405, + "mean_conf": 0.9573703050826868, + "confident_error_rate": 0.07857142857142857, + "coverage_at_0_9": 0.8821428571428571, + "accuracy_at_0_9": 0.9109311740890689, + "coverage_at_5pct_error": 0.7535714285714286, + "coverage_at_1pct_error": 0.25, + "aurc": 0.039651831256480245, + "error_rate_at_0_9": 0.08906882591093117, + "confidence_bias": 0.07165601936840116, + "top_bins": { + "0.9": { + "n": 247, + "errors": 22, + "error_rate": 0.08906882591093117 + }, + "0.95": { + "n": 236, + "errors": 20, + "error_rate": 0.0847457627118644 + }, + "0.99": { + "n": 202, + "errors": 8, + "error_rate": 0.039603960396039604 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9571428571428572, + "confidence_cutoff": 0.9995423463157226 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.9330357142857143, + "confidence_cutoff": 0.9697351853044945 + } + } + }, + "score": { + "n": 40, + "nll": 1.0231410422892537, + "acc": 0.75, + "ece": 0.26013051552429656, + "brier": 0.46812385113527205, + "mean_conf": 0.9410312892444732, + "confident_error_rate": 0.175, + "coverage_at_0_9": 0.825, + "accuracy_at_0_9": 0.7878787878787878, + "coverage_at_5pct_error": 0.65, + "coverage_at_1pct_error": 0.6, + "aurc": 0.07355999344204343, + "error_rate_at_0_9": 0.21212121212121213, + "confidence_bias": 0.1910312892444732, + "top_bins": { + "0.9": { + "n": 33, + "errors": 7, + "error_rate": 0.21212121212121213 + }, + "0.95": { + "n": 31, + "errors": 6, + "error_rate": 0.1935483870967742 + }, + "0.99": { + "n": 28, + "errors": 3, + "error_rate": 0.10714285714285714 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9999132212992297 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.78125, + "confidence_cutoff": 0.9461605303081507 + } + }, + "score_mae": 0.27589619803682164, + "ranked_probability_score": 0.11777542530670457 + } + } + } + }, + "temperature": 1.334968103387446 + } + ], + "kev_4b_security": { + "test": { + "raw": { + "questions": 21634, + "accuracy": 0.5048996949246556, + "mean_class_recall": 0.5647195198307282, + "nll": 1.2328084492499014, + "brier": 0.7147920651499965, + "ece15": 0.3370024824919156, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.44924655634649163, + "error_rate_at_confidence_0_9": 0.34736083959255065, + "ordinal_expected_level_mae": 0.2811370953407191, + "tasks": { + "agentdojo_tool_outputs:agentdojo_boundary_action": { + "questions": 426, + "accuracy": 0.5422535211267606, + "mean_class_recall": 0.7045454545454546, + "nll": 1.2120829748020316, + "brier": 0.6804576908439421, + "ece15": 0.2992356768561654, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.5187793427230047, + "error_rate_at_confidence_0_9": 0.3936651583710407, + "ordinal_expected_level_mae": null + }, + "agentdojo_tool_outputs:agentdojo_embedded_injection": { + "questions": 426, + "accuracy": 0.7300469483568075, + "mean_class_recall": 0.8257575757575757, + "nll": 0.6169123027998439, + "brier": 0.3918536224381035, + "ece15": 0.1421670820222778, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.57981220657277, + "error_rate_at_confidence_0_9": 0.15384615384615385, + "ordinal_expected_level_mae": null + }, + "byzfl_mnist:fl_attack_generated_update": { + "questions": 800, + "accuracy": 0.79875, + "mean_class_recall": 0.49921875, + "nll": 0.5895657949467273, + "brier": 0.35429205012331033, + "ece15": 0.1133528731473528, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.2175, + "error_rate_at_confidence_0_9": 0.4942528735632184, + "ordinal_expected_level_mae": null + }, + "ciciot2023:ciciot_attack": { + "questions": 6000, + "accuracy": 0.2628333333333333, + "mean_class_recall": 0.5085555555555555, + "nll": 1.2759496387732032, + "brier": 0.9560488943280168, + "ece15": 0.5505303178793823, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.12266666666666666, + "error_rate_at_confidence_0_9": 0.453804347826087, + "ordinal_expected_level_mae": null + }, + "injecagent:injecagent_requested_operation_v13": { + "questions": 124, + "accuracy": 0.6290322580645161, + "mean_class_recall": 0.6714285714285715, + "nll": 1.0817458611434574, + "brier": 0.6236238714203711, + "ece15": 0.30916032986760006, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.75, + "error_rate_at_confidence_0_9": 0.34408602150537637, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_known_logon_action_v16": { + "questions": 540, + "accuracy": 0.7981481481481482, + "mean_class_recall": 0.7528344671201813, + "nll": 0.4871733371313632, + "brier": 0.29625578551796805, + "ece15": 0.15242774183988111, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.7962962962962963, + "error_rate_at_confidence_0_9": 0.08837209302325581, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_observable_policy": { + "questions": 540, + "accuracy": 0.9944444444444445, + "mean_class_recall": 0.9886363636363636, + "nll": 0.01641532065146085, + "brier": 0.007136931607478901, + "ece15": 0.013212748100349914, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9777777777777777, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "ton_iot_network:ton_attack": { + "questions": 3500, + "accuracy": 0.5065714285714286, + "mean_class_recall": 0.6710476190476191, + "nll": 1.1359565067657027, + "brier": 0.6406729439019092, + "ece15": 0.23932739622085006, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.38971428571428574, + "error_rate_at_confidence_0_9": 0.3878299120234604, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_evidence_level": { + "questions": 659, + "accuracy": 0.8118361153262519, + "mean_class_recall": 0.794717762064859, + "nll": 0.47493573588084764, + "brier": 0.2794985686136285, + "ece15": 0.05201558678606951, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.4931714719271624, + "error_rate_at_confidence_0_9": 0.04923076923076923, + "ordinal_expected_level_mae": 0.2811370953407191 + }, + "twins_streamlet:twins_policy_action": { + "questions": 659, + "accuracy": 0.7177541729893778, + "mean_class_recall": 0.6923577773226661, + "nll": 0.5581275002335633, + "brier": 0.3443282053616745, + "ece15": 0.09062810464195983, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.44613050075872535, + "error_rate_at_confidence_0_9": 0.04421768707482993, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_quorum": { + "questions": 659, + "accuracy": 0.7814871016691958, + "mean_class_recall": 0.7019105077928607, + "nll": 0.4980710054489271, + "brier": 0.31257573538065736, + "ece15": 0.09857831652926007, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.590288315629742, + "error_rate_at_confidence_0_9": 0.09511568123393316, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_visible_conflict": { + "questions": 659, + "accuracy": 0.9711684370257967, + "mean_class_recall": 0.9693685820406379, + "nll": 0.10259254119815524, + "brier": 0.05091487551434137, + "ece15": 0.030811660924900536, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.8679817905918058, + "error_rate_at_confidence_0_9": 0.006993006993006993, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_message_manipulation": { + "questions": 3321, + "accuracy": 0.47756699789220114, + "mean_class_recall": 0.5, + "nll": 1.2951885811608232, + "brier": 0.8663028363232907, + "ece15": 0.42951687019840074, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.7633242999096658, + "error_rate_at_confidence_0_9": 0.504930966469428, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_supported_attack_type_v16": { + "questions": 3321, + "accuracy": 0.48087925323697683, + "mean_class_recall": 0.12233932942660039, + "nll": 2.41017951541039, + "brier": 0.8961545633575424, + "ece15": 0.3893600279043549, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.5453176753989762, + "error_rate_at_confidence_0_9": 0.4870237437879624, + "ordinal_expected_level_mae": null + } + }, + "macro_task_accuracy": 0.6787694400132313, + "macro_task_balanced_accuracy": 0.671622736838496 + }, + "recalibrated": { + "questions": 21634, + "accuracy": 0.5048996949246556, + "mean_class_recall": 0.5647195198307282, + "nll": 1.2328084492499014, + "brier": 0.7147920651499965, + "ece15": 0.3370024824919156, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.44924655634649163, + "error_rate_at_confidence_0_9": 0.34736083959255065, + "ordinal_expected_level_mae": 0.2811370953407191, + "tasks": { + "agentdojo_tool_outputs:agentdojo_boundary_action": { + "questions": 426, + "accuracy": 0.5422535211267606, + "mean_class_recall": 0.7045454545454546, + "nll": 1.2120829748020316, + "brier": 0.6804576908439421, + "ece15": 0.2992356768561654, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.5187793427230047, + "error_rate_at_confidence_0_9": 0.3936651583710407, + "ordinal_expected_level_mae": null + }, + "agentdojo_tool_outputs:agentdojo_embedded_injection": { + "questions": 426, + "accuracy": 0.7300469483568075, + "mean_class_recall": 0.8257575757575757, + "nll": 0.6169123027998439, + "brier": 0.3918536224381035, + "ece15": 0.1421670820222778, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.57981220657277, + "error_rate_at_confidence_0_9": 0.15384615384615385, + "ordinal_expected_level_mae": null + }, + "byzfl_mnist:fl_attack_generated_update": { + "questions": 800, + "accuracy": 0.79875, + "mean_class_recall": 0.49921875, + "nll": 0.5895657949467273, + "brier": 0.35429205012331033, + "ece15": 0.1133528731473528, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.2175, + "error_rate_at_confidence_0_9": 0.4942528735632184, + "ordinal_expected_level_mae": null + }, + "ciciot2023:ciciot_attack": { + "questions": 6000, + "accuracy": 0.2628333333333333, + "mean_class_recall": 0.5085555555555555, + "nll": 1.2759496387732032, + "brier": 0.9560488943280168, + "ece15": 0.5505303178793823, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.12266666666666666, + "error_rate_at_confidence_0_9": 0.453804347826087, + "ordinal_expected_level_mae": null + }, + "injecagent:injecagent_requested_operation_v13": { + "questions": 124, + "accuracy": 0.6290322580645161, + "mean_class_recall": 0.6714285714285715, + "nll": 1.0817458611434574, + "brier": 0.6236238714203711, + "ece15": 0.30916032986760006, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.75, + "error_rate_at_confidence_0_9": 0.34408602150537637, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_known_logon_action_v16": { + "questions": 540, + "accuracy": 0.7981481481481482, + "mean_class_recall": 0.7528344671201813, + "nll": 0.4871733371313632, + "brier": 0.29625578551796805, + "ece15": 0.15242774183988111, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.7962962962962963, + "error_rate_at_confidence_0_9": 0.08837209302325581, + "ordinal_expected_level_mae": null + }, + "lanl_auth_policy:lanl_observable_policy": { + "questions": 540, + "accuracy": 0.9944444444444445, + "mean_class_recall": 0.9886363636363636, + "nll": 0.01641532065146085, + "brier": 0.007136931607478901, + "ece15": 0.013212748100349914, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.9777777777777777, + "error_rate_at_confidence_0_9": 0.0, + "ordinal_expected_level_mae": null + }, + "ton_iot_network:ton_attack": { + "questions": 3500, + "accuracy": 0.5065714285714286, + "mean_class_recall": 0.6710476190476191, + "nll": 1.1359565067657027, + "brier": 0.6406729439019092, + "ece15": 0.23932739622085006, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.38971428571428574, + "error_rate_at_confidence_0_9": 0.3878299120234604, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_evidence_level": { + "questions": 659, + "accuracy": 0.8118361153262519, + "mean_class_recall": 0.794717762064859, + "nll": 0.47493573588084764, + "brier": 0.2794985686136285, + "ece15": 0.05201558678606951, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.4931714719271624, + "error_rate_at_confidence_0_9": 0.04923076923076923, + "ordinal_expected_level_mae": 0.2811370953407191 + }, + "twins_streamlet:twins_policy_action": { + "questions": 659, + "accuracy": 0.7177541729893778, + "mean_class_recall": 0.6923577773226661, + "nll": 0.5581275002335633, + "brier": 0.3443282053616745, + "ece15": 0.09062810464195983, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.44613050075872535, + "error_rate_at_confidence_0_9": 0.04421768707482993, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_quorum": { + "questions": 659, + "accuracy": 0.7814871016691958, + "mean_class_recall": 0.7019105077928607, + "nll": 0.4980710054489271, + "brier": 0.31257573538065736, + "ece15": 0.09857831652926007, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.590288315629742, + "error_rate_at_confidence_0_9": 0.09511568123393316, + "ordinal_expected_level_mae": null + }, + "twins_streamlet:twins_visible_conflict": { + "questions": 659, + "accuracy": 0.9711684370257967, + "mean_class_recall": 0.9693685820406379, + "nll": 0.10259254119815524, + "brier": 0.05091487551434137, + "ece15": 0.030811660924900536, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.8679817905918058, + "error_rate_at_confidence_0_9": 0.006993006993006993, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_message_manipulation": { + "questions": 3321, + "accuracy": 0.47756699789220114, + "mean_class_recall": 0.5, + "nll": 1.2951885811608232, + "brier": 0.8663028363232907, + "ece15": 0.42951687019840074, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.7633242999096658, + "error_rate_at_confidence_0_9": 0.504930966469428, + "ordinal_expected_level_mae": null + }, + "veremi_nextgen:veremi_supported_attack_type_v16": { + "questions": 3321, + "accuracy": 0.48087925323697683, + "mean_class_recall": 0.12233932942660039, + "nll": 2.41017951541039, + "brier": 0.8961545633575424, + "ece15": 0.3893600279043549, + "temperature": 1.0, + "coverage_at_confidence_0_9": 0.5453176753989762, + "error_rate_at_confidence_0_9": 0.4870237437879624, + "ordinal_expected_level_mae": null + } + }, + "macro_task_accuracy": 0.6787694400132313, + "macro_task_balanced_accuracy": 0.671622736838496 + } + }, + "test_used_for_selection": false + }, + "kev_4b_general": { + "decision-v7": { + "raw_clean": { + "n": 1200, + "nll": 0.46776271359373983, + "acc": 0.8708333333333333, + "ece": 0.07324011844924935, + "brier": 0.2068727227531146, + "mean_conf": 0.9391164907697294, + "confident_error_rate": 0.04833333333333333, + "coverage_at_0_9": 0.8166666666666667, + "accuracy_at_0_9": 0.9408163265306122, + "coverage_at_5pct_error": 0.7791666666666667, + "coverage_at_1pct_error": 0.6133333333333333, + "aurc": 0.02837551275061399, + "error_rate_at_0_9": 0.05918367346938776, + "confidence_bias": 0.06828315743639601, + "top_bins": { + "0.9": { + "n": 980, + "errors": 58, + "error_rate": 0.05918367346938776 + }, + "0.95": { + "n": 915, + "errors": 39, + "error_rate": 0.04262295081967213 + }, + "0.99": { + "n": 829, + "errors": 21, + "error_rate": 0.025331724969843185 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9916666666666667, + "confidence_cutoff": 0.999598385219589 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.94375, + "confidence_cutoff": 0.9220693757732149 + } + }, + "score_mae": 0.4635506535819828, + "ranked_probability_score": 0.08574066842436183 + }, + "as_shipped_clean": { + "n": 1200, + "nll": 0.35957326104404586, + "acc": 0.8708333333333333, + "ece": 0.020190532051177417, + "brier": 0.18914513739887015, + "mean_conf": 0.8656556446393923, + "confident_error_rate": 0.013333333333333334, + "coverage_at_0_9": 0.6583333333333333, + "accuracy_at_0_9": 0.979746835443038, + "coverage_at_5pct_error": 0.7883333333333333, + "coverage_at_1pct_error": 0.585, + "aurc": 0.02903042619721546, + "error_rate_at_0_9": 0.020253164556962026, + "confidence_bias": -0.005177688693941085, + "top_bins": { + "0.9": { + "n": 790, + "errors": 16, + "error_rate": 0.020253164556962026 + }, + "0.95": { + "n": 697, + "errors": 6, + "error_rate": 0.00860832137733142 + }, + "0.99": { + "n": 320, + "errors": 2, + "error_rate": 0.00625 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9916666666666667, + "confidence_cutoff": 0.9690438255888087 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.946875, + "confidence_cutoff": 0.7097948924008157 + } + }, + "score_mae": 0.496047231607107, + "ranked_probability_score": 0.07900573734715889 + }, + "sources": { + "agnews": { + "n": 240, + "nll": 0.2188339468959484, + "acc": 0.9208333333333333, + "ece": 0.03458717155438394, + "brier": 0.11967225736034369, + "mean_conf": 0.9210081021968138, + "confident_error_rate": 0.0125, + "coverage_at_0_9": 0.7875, + "accuracy_at_0_9": 0.9841269841269841, + "coverage_at_5pct_error": 0.9291666666666667, + "coverage_at_1pct_error": 0.7208333333333333, + "aurc": 0.010595808181650849, + "error_rate_at_0_9": 0.015873015873015872, + "confidence_bias": 0.00017476886348055665, + "top_bins": { + "0.9": { + "n": 189, + "errors": 3, + "error_rate": 0.015873015873015872 + }, + "0.95": { + "n": 151, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 8, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.962887554254757 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.9791666666666666, + "confidence_cutoff": 0.8982351380482543 + } + } + }, + "amazon": { + "n": 80, + "nll": 1.019521275107054, + "acc": 0.6125, + "ece": 0.06021888255181947, + "brier": 0.5421117796889148, + "mean_conf": 0.5878388840884237, + "confident_error_rate": 0.0, + "coverage_at_0_9": 0.0375, + "accuracy_at_0_9": 1.0, + "coverage_at_5pct_error": 0.0625, + "coverage_at_1pct_error": 0.0625, + "aurc": 0.26205124362469656, + "error_rate_at_0_9": 0.0, + "confidence_bias": -0.02466111591157638, + "top_bins": { + "0.9": { + "n": 3, + "errors": 0, + "error_rate": 0.0 + }, + "0.95": { + "n": 1, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 0, + "errors": 0, + "error_rate": null + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.75, + "confidence_cutoff": 0.5617743497724099 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.640625, + "confidence_cutoff": 0.44733349967779523 + } + }, + "score_mae": 0.5169737287164902, + "ranked_probability_score": 0.08574565040577714 + }, + "banking77": { + "n": 80, + "nll": 0.6434976771895786, + "acc": 0.875, + "ece": 0.16139951556317184, + "brier": 0.23632755413887385, + "mean_conf": 0.7261215293796409, + "confident_error_rate": 0.0, + "coverage_at_0_9": 0.4, + "accuracy_at_0_9": 1.0, + "coverage_at_5pct_error": 0.75, + "coverage_at_1pct_error": 0.675, + "aurc": 0.025355014288770995, + "error_rate_at_0_9": 0.0, + "confidence_bias": -0.14887847062035908, + "top_bins": { + "0.9": { + "n": 32, + "errors": 0, + "error_rate": 0.0 + }, + "0.95": { + "n": 24, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 2, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.853940262270331 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.921875, + "confidence_cutoff": 0.4421809473945611 + } + } + }, + "boolq": { + "n": 80, + "nll": 0.3692803969173892, + "acc": 0.875, + "ece": 0.0748529395052494, + "brier": 0.2143912889240845, + "mean_conf": 0.9094612721753246, + "confident_error_rate": 0.0625, + "coverage_at_0_9": 0.75, + "accuracy_at_0_9": 0.9166666666666666, + "coverage_at_5pct_error": 0.6625, + "coverage_at_1pct_error": 0.075, + "aurc": 0.059468554539862016, + "error_rate_at_0_9": 0.08333333333333333, + "confidence_bias": 0.034461272175324575, + "top_bins": { + "0.9": { + "n": 60, + "errors": 5, + "error_rate": 0.08333333333333333 + }, + "0.95": { + "n": 50, + "errors": 1, + "error_rate": 0.02 + }, + "0.99": { + "n": 26, + "errors": 1, + "error_rate": 0.038461538461538464 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.975, + "confidence_cutoff": 0.9805121330496657 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.890625, + "confidence_cutoff": 0.8395732170279804 + } + } + }, + "contrastive": { + "n": 160, + "nll": 0.06053811563664827, + "acc": 0.96875, + "ece": 0.03645679005941199, + "brier": 0.032513686428614616, + "mean_conf": 0.9645709125518511, + "confident_error_rate": 0.0, + "coverage_at_0_9": 0.90625, + "accuracy_at_0_9": 1.0, + "coverage_at_5pct_error": 1.0, + "coverage_at_1pct_error": 0.975, + "aurc": 0.0006312206080826334, + "error_rate_at_0_9": 0.0, + "confidence_bias": -0.004179087448148877, + "top_bins": { + "0.9": { + "n": 145, + "errors": 0, + "error_rate": 0.0 + }, + "0.95": { + "n": 139, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 83, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9906152293058859 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 1.0, + "confidence_cutoff": 0.9714574090050602 + } + } + }, + "dbpedia14": { + "n": 80, + "nll": 0.031074258317027002, + "acc": 1.0, + "ece": 0.02597689933514287, + "brier": 0.010683464293698132, + "mean_conf": 0.9740231006648571, + "confident_error_rate": 0.0, + "coverage_at_0_9": 0.95, + "accuracy_at_0_9": 1.0, + "coverage_at_5pct_error": 1.0, + "coverage_at_1pct_error": 1.0, + "aurc": 0.0, + "error_rate_at_0_9": 0.0, + "confidence_bias": -0.02597689933514291, + "top_bins": { + "0.9": { + "n": 76, + "errors": 0, + "error_rate": 0.0 + }, + "0.95": { + "n": 75, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 57, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9936484074458076 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 1.0, + "confidence_cutoff": 0.9825304423953378 + } + } + }, + "imdb": { + "n": 80, + "nll": 0.0720718883732508, + "acc": 0.95, + "ece": 0.044527070446157675, + "brier": 0.0446787843047757, + "mean_conf": 0.9624677353009415, + "confident_error_rate": 0.0, + "coverage_at_0_9": 0.9, + "accuracy_at_0_9": 1.0, + "coverage_at_5pct_error": 1.0, + "coverage_at_1pct_error": 0.9375, + "aurc": 0.0017470077113648063, + "error_rate_at_0_9": 0.0, + "confidence_bias": 0.012467735300941585, + "top_bins": { + "0.9": { + "n": 72, + "errors": 0, + "error_rate": 0.0 + }, + "0.95": { + "n": 70, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 50, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9920587271734113 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 1.0, + "confidence_cutoff": 0.979826985587298 + } + } + }, + "mnli": { + "n": 80, + "nll": 0.27465339009465656, + "acc": 0.9125, + "ece": 0.09412857766900207, + "brier": 0.15288319760619568, + "mean_conf": 0.94639379824153, + "confident_error_rate": 0.0375, + "coverage_at_0_9": 0.875, + "accuracy_at_0_9": 0.9571428571428572, + "coverage_at_5pct_error": 0.875, + "coverage_at_1pct_error": 0.675, + "aurc": 0.01561228714610165, + "error_rate_at_0_9": 0.04285714285714286, + "confidence_bias": 0.03389379824153005, + "top_bins": { + "0.9": { + "n": 70, + "errors": 3, + "error_rate": 0.04285714285714286 + }, + "0.95": { + "n": 67, + "errors": 2, + "error_rate": 0.029850746268656716 + }, + "0.99": { + "n": 45, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9920616734287709 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.96875, + "confidence_cutoff": 0.9661281848866967 + } + } + }, + "sst5": { + "n": 80, + "nll": 0.9636808958022751, + "acc": 0.6125, + "ece": 0.09386529578800333, + "brier": 0.5310525330540982, + "mean_conf": 0.6617158029994963, + "confident_error_rate": 0.0, + "coverage_at_0_9": 0.0, + "accuracy_at_0_9": null, + "coverage_at_5pct_error": 0.1625, + "coverage_at_1pct_error": 0.1625, + "aurc": 0.26405329260422244, + "error_rate_at_0_9": null, + "confidence_bias": 0.04921580299949624, + "top_bins": { + "0.9": { + "n": 0, + "errors": 0, + "error_rate": null + }, + "0.95": { + "n": 0, + "errors": 0, + "error_rate": null + }, + "0.99": { + "n": 0, + "errors": 0, + "error_rate": null + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.65, + "confidence_cutoff": 0.6682213711318762 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.625, + "confidence_cutoff": 0.5940210248093071 + } + }, + "score_mae": 0.5227774115005795, + "ranked_probability_score": 0.08371156976927716 + }, + "trec": { + "n": 80, + "nll": 0.16488315000308515, + "acc": 0.9625, + "ece": 0.02170728565832692, + "brier": 0.06057653876714909, + "mean_conf": 0.9707615704179906, + "confident_error_rate": 0.025, + "coverage_at_0_9": 0.9625, + "accuracy_at_0_9": 0.974025974025974, + "coverage_at_5pct_error": 1.0, + "coverage_at_1pct_error": 0.2625, + "aurc": 0.018315788693627898, + "error_rate_at_0_9": 0.025974025974025976, + "confidence_bias": 0.008261570417990582, + "top_bins": { + "0.9": { + "n": 77, + "errors": 2, + "error_rate": 0.025974025974025976 + }, + "0.95": { + "n": 73, + "errors": 2, + "error_rate": 0.0273972602739726 + }, + "0.99": { + "n": 47, + "errors": 1, + "error_rate": 0.02127659574468085 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.975, + "confidence_cutoff": 0.990957410950485 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.984375, + "confidence_cutoff": 0.9755548475921001 + } + } + }, + "yelp": { + "n": 160, + "nll": 0.5386789559476151, + "acc": 0.78125, + "ece": 0.06183692449171378, + "brier": 0.3102138876335011, + "mean_conf": 0.7769424223142675, + "confident_error_rate": 0.01875, + "coverage_at_0_9": 0.4125, + "accuracy_at_0_9": 0.9545454545454546, + "coverage_at_5pct_error": 0.4375, + "coverage_at_1pct_error": 0.14375, + "aurc": 0.0856358031592544, + "error_rate_at_0_9": 0.045454545454545456, + "confidence_bias": -0.004307577685732533, + "top_bins": { + "0.9": { + "n": 66, + "errors": 3, + "error_rate": 0.045454545454545456 + }, + "0.95": { + "n": 47, + "errors": 1, + "error_rate": 0.02127659574468085 + }, + "0.99": { + "n": 2, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9125, + "confidence_cutoff": 0.8216206137745724 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.8515625, + "confidence_cutoff": 0.577455875568763 + } + }, + "score_mae": 0.44839055460425137, + "ranked_probability_score": 0.0675599918664224 + } + }, + "types": { + "choice": { + "n": 480, + "nll": 0.2482065801683337, + "acc": 0.9375, + "ece": 0.03877078391929752, + "brier": 0.10960480604725589, + "mean_conf": 0.9179948652716078, + "confident_error_rate": 0.0125, + "coverage_at_0_9": 0.8145833333333333, + "accuracy_at_0_9": 0.9846547314578005, + "coverage_at_5pct_error": 0.9583333333333334, + "coverage_at_1pct_error": 0.69375, + "aurc": 0.01188927247802621, + "error_rate_at_0_9": 0.015345268542199489, + "confidence_bias": -0.019505134728392215, + "top_bins": { + "0.9": { + "n": 391, + "errors": 6, + "error_rate": 0.015345268542199489 + }, + "0.95": { + "n": 361, + "errors": 4, + "error_rate": 0.0110803324099723 + }, + "0.99": { + "n": 202, + "errors": 1, + "error_rate": 0.0049504950495049506 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9958333333333333, + "confidence_cutoff": 0.9867396184306336 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.9869791666666666, + "confidence_cutoff": 0.9197849614177179 + } + } + }, + "noul": { + "n": 480, + "nll": 0.18011051231286856, + "acc": 0.9270833333333334, + "ece": 0.014056851970226658, + "brier": 0.10328956268313709, + "mean_conf": 0.9306022039720875, + "confident_error_rate": 0.01875, + "coverage_at_0_9": 0.8166666666666667, + "accuracy_at_0_9": 0.9770408163265306, + "coverage_at_5pct_error": 0.9416666666666667, + "coverage_at_1pct_error": 0.7041666666666667, + "aurc": 0.015597964078219255, + "error_rate_at_0_9": 0.02295918367346939, + "confidence_bias": 0.0035188706387541124, + "top_bins": { + "0.9": { + "n": 392, + "errors": 9, + "error_rate": 0.02295918367346939 + }, + "0.95": { + "n": 334, + "errors": 2, + "error_rate": 0.005988023952095809 + }, + "0.99": { + "n": 118, + "errors": 1, + "error_rate": 0.00847457627118644 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9916666666666667, + "confidence_cutoff": 0.9755983279973393 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.9765625, + "confidence_cutoff": 0.9172471488342538 + } + } + }, + "score": { + "n": 240, + "nll": 0.9412321202578248, + "acc": 0.625, + "ece": 0.048867889504322116, + "brier": 0.519936949533565, + "mean_conf": 0.6310840847095708, + "confident_error_rate": 0.004166666666666667, + "coverage_at_0_9": 0.029166666666666667, + "accuracy_at_0_9": 0.8571428571428571, + "coverage_at_5pct_error": 0.016666666666666666, + "coverage_at_1pct_error": 0.016666666666666666, + "aurc": 0.2774440601733873, + "error_rate_at_0_9": 0.14285714285714285, + "confidence_bias": 0.006084084709570781, + "top_bins": { + "0.9": { + "n": 7, + "errors": 1, + "error_rate": 0.14285714285714285 + }, + "0.95": { + "n": 2, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 0, + "errors": 0, + "error_rate": null + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.7166666666666667, + "confidence_cutoff": 0.6329725841576003 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.6614583333333334, + "confidence_cutoff": 0.4922131178494099 + } + }, + "score_mae": 0.496047231607107, + "ranked_probability_score": 0.07900573734715889 + } + } + }, + "transfer-v4": { + "raw_clean": { + "n": 656, + "nll": 0.5640777093137941, + "acc": 0.836890243902439, + "ece": 0.10195634226245451, + "brier": 0.25517441623108567, + "mean_conf": 0.9301396868913653, + "confident_error_rate": 0.06707317073170732, + "coverage_at_0_9": 0.8094512195121951, + "accuracy_at_0_9": 0.9171374764595104, + "coverage_at_5pct_error": 0.6829268292682927, + "coverage_at_1pct_error": 0.2820121951219512, + "aurc": 0.043201879169005745, + "error_rate_at_0_9": 0.08286252354048965, + "confidence_bias": 0.09324944298892623, + "top_bins": { + "0.9": { + "n": 531, + "errors": 44, + "error_rate": 0.08286252354048965 + }, + "0.95": { + "n": 487, + "errors": 33, + "error_rate": 0.06776180698151951 + }, + "0.99": { + "n": 406, + "errors": 14, + "error_rate": 0.034482758620689655 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9725609756097561, + "confidence_cutoff": 0.9980492531927935 + }, + "0.8": { + "coverage": 0.8003048780487805, + "accuracy": 0.92, + "confidence_cutoff": 0.9074193401309704 + } + }, + "score_mae": 0.24004523406424622, + "ranked_probability_score": 0.09767451376793074 + }, + "as_shipped_clean": { + "n": 656, + "nll": 0.42587070132965826, + "acc": 0.836890243902439, + "ece": 0.0314949650736429, + "brier": 0.23187082308763576, + "mean_conf": 0.844962353152835, + "confident_error_rate": 0.018292682926829267, + "coverage_at_0_9": 0.5853658536585366, + "accuracy_at_0_9": 0.96875, + "coverage_at_5pct_error": 0.6570121951219512, + "coverage_at_1pct_error": 0.2530487804878049, + "aurc": 0.04507835320830427, + "error_rate_at_0_9": 0.03125, + "confidence_bias": 0.00807210925039592, + "top_bins": { + "0.9": { + "n": 384, + "errors": 12, + "error_rate": 0.03125 + }, + "0.95": { + "n": 315, + "errors": 8, + "error_rate": 0.025396825396825397 + }, + "0.99": { + "n": 79, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9725609756097561, + "confidence_cutoff": 0.9441193658749547 + }, + "0.8": { + "coverage": 0.8003048780487805, + "accuracy": 0.9047619047619048, + "confidence_cutoff": 0.693693819569072 + } + }, + "score_mae": 0.27196301946029916, + "ranked_probability_score": 0.0837200362891968 + }, + "sources": { + "composition_holdout": { + "n": 96, + "nll": 0.29615290237046005, + "acc": 0.875, + "ece": 0.06938096946882913, + "brier": 0.1808256349283982, + "mean_conf": 0.9443809694688293, + "confident_error_rate": 0.041666666666666664, + "coverage_at_0_9": 0.8333333333333334, + "accuracy_at_0_9": 0.95, + "coverage_at_5pct_error": 0.8645833333333334, + "coverage_at_1pct_error": 0.6666666666666666, + "aurc": 0.021756148279610994, + "error_rate_at_0_9": 0.05, + "confidence_bias": 0.06938096946882932, + "top_bins": { + "0.9": { + "n": 80, + "errors": 4, + "error_rate": 0.05 + }, + "0.95": { + "n": 73, + "errors": 3, + "error_rate": 0.0410958904109589 + }, + "0.99": { + "n": 2, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.980048039700624 + }, + "0.8": { + "coverage": 0.8020833333333334, + "accuracy": 0.948051948051948, + "confidence_cutoff": 0.9379020298350527 + } + } + }, + "contrastive": { + "n": 80, + "nll": 0.2412487469898486, + "acc": 0.875, + "ece": 0.054181213136020756, + "brier": 0.15115638127662953, + "mean_conf": 0.9258459913436543, + "confident_error_rate": 0.0125, + "coverage_at_0_9": 0.8125, + "accuracy_at_0_9": 0.9846153846153847, + "coverage_at_5pct_error": 0.875, + "coverage_at_1pct_error": 0.7875, + "aurc": 0.01404271679837261, + "error_rate_at_0_9": 0.015384615384615385, + "confidence_bias": 0.050845991343654307, + "top_bins": { + "0.9": { + "n": 65, + "errors": 1, + "error_rate": 0.015384615384615385 + }, + "0.95": { + "n": 62, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 26, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9886860005278774 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.984375, + "confidence_cutoff": 0.9353245409640992 + } + }, + "score_mae": 0.27196301946029916, + "ranked_probability_score": 0.0837200362891968 + }, + "emotion": { + "n": 80, + "nll": 1.0001023668251066, + "acc": 0.6625, + "ece": 0.14881697863795876, + "brier": 0.4744803648425345, + "mean_conf": 0.591235292513326, + "confident_error_rate": 0.0, + "coverage_at_0_9": 0.05, + "accuracy_at_0_9": 1.0, + "coverage_at_5pct_error": 0.0625, + "coverage_at_1pct_error": 0.0625, + "aurc": 0.18702686566035498, + "error_rate_at_0_9": 0.0, + "confidence_bias": -0.07126470748667402, + "top_bins": { + "0.9": { + "n": 4, + "errors": 0, + "error_rate": 0.0 + }, + "0.95": { + "n": 2, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 0, + "errors": 0, + "error_rate": null + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.8, + "confidence_cutoff": 0.5874489256154185 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.71875, + "confidence_cutoff": 0.3855885748389788 + } + } + }, + "mmlu": { + "n": 80, + "nll": 0.777539674387988, + "acc": 0.7, + "ece": 0.13070084664599038, + "brier": 0.40568344535429135, + "mean_conf": 0.6636013764250539, + "confident_error_rate": 0.0, + "coverage_at_0_9": 0.175, + "accuracy_at_0_9": 1.0, + "coverage_at_5pct_error": 0.2625, + "coverage_at_1pct_error": 0.225, + "aurc": 0.12322939721812473, + "error_rate_at_0_9": 0.0, + "confidence_bias": -0.036398623574946054, + "top_bins": { + "0.9": { + "n": 14, + "errors": 0, + "error_rate": 0.0 + }, + "0.95": { + "n": 7, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 1, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9, + "confidence_cutoff": 0.6761349246582344 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.796875, + "confidence_cutoff": 0.4466846683613359 + } + } + }, + "paws": { + "n": 80, + "nll": 0.512348189940992, + "acc": 0.8125, + "ece": 0.10522416975834781, + "brier": 0.2914852770101592, + "mean_conf": 0.9091572173256038, + "confident_error_rate": 0.0875, + "coverage_at_0_9": 0.75, + "accuracy_at_0_9": 0.8833333333333333, + "coverage_at_5pct_error": 0.2625, + "coverage_at_1pct_error": 0.2, + "aurc": 0.09484078957534384, + "error_rate_at_0_9": 0.11666666666666667, + "confidence_bias": 0.09665721732560384, + "top_bins": { + "0.9": { + "n": 60, + "errors": 7, + "error_rate": 0.11666666666666667 + }, + "0.95": { + "n": 51, + "errors": 5, + "error_rate": 0.09803921568627451 + }, + "0.99": { + "n": 4, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.875, + "confidence_cutoff": 0.974944024208843 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.875, + "confidence_cutoff": 0.8613523629912301 + } + } + }, + "qnli": { + "n": 80, + "nll": 0.20826182445785202, + "acc": 0.9125, + "ece": 0.04839798205004245, + "brier": 0.12797478277700258, + "mean_conf": 0.9090029243294062, + "confident_error_rate": 0.0, + "coverage_at_0_9": 0.7, + "accuracy_at_0_9": 1.0, + "coverage_at_5pct_error": 0.8875, + "coverage_at_1pct_error": 0.7, + "aurc": 0.014269660330833212, + "error_rate_at_0_9": 0.0, + "confidence_bias": -0.0034970756705937633, + "top_bins": { + "0.9": { + "n": 56, + "errors": 0, + "error_rate": 0.0 + }, + "0.95": { + "n": 45, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 10, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9595371830584098 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.96875, + "confidence_cutoff": 0.8535858138327058 + } + } + }, + "sciq": { + "n": 80, + "nll": 0.04197126388379959, + "acc": 1.0, + "ece": 0.03774145345553605, + "brier": 0.010730106398852924, + "mean_conf": 0.962258546544464, + "confident_error_rate": 0.0, + "coverage_at_0_9": 0.9, + "accuracy_at_0_9": 1.0, + "coverage_at_5pct_error": 1.0, + "coverage_at_1pct_error": 1.0, + "aurc": 0.0, + "error_rate_at_0_9": 0.0, + "confidence_bias": -0.037741453455536034, + "top_bins": { + "0.9": { + "n": 72, + "errors": 0, + "error_rate": 0.0 + }, + "0.95": { + "n": 69, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 36, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9883715126062644 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 1.0, + "confidence_cutoff": 0.9713861020562242 + } + } + }, + "tweet_offensive": { + "n": 80, + "nll": 0.3552842015730585, + "acc": 0.85, + "ece": 0.056220131039951354, + "brier": 0.22283962974506544, + "mean_conf": 0.8343327840091443, + "confident_error_rate": 0.0, + "coverage_at_0_9": 0.4125, + "accuracy_at_0_9": 1.0, + "coverage_at_5pct_error": 0.6, + "coverage_at_1pct_error": 0.525, + "aurc": 0.045801042430234855, + "error_rate_at_0_9": 0.0, + "confidence_bias": -0.015667215990855632, + "top_bins": { + "0.9": { + "n": 33, + "errors": 0, + "error_rate": 0.0 + }, + "0.95": { + "n": 6, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 0, + "errors": 0, + "error_rate": null + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.8768513333328267 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.890625, + "confidence_cutoff": 0.7493184208869934 + } + } + } + }, + "types": { + "choice": { + "n": 336, + "nll": 0.5178563780812967, + "acc": 0.8125, + "ece": 0.04292515250441144, + "brier": 0.26378206631184675, + "mean_conf": 0.797702947344152, + "confident_error_rate": 0.011904761904761904, + "coverage_at_0_9": 0.5059523809523809, + "accuracy_at_0_9": 0.9764705882352941, + "coverage_at_5pct_error": 0.6130952380952381, + "coverage_at_1pct_error": 0.40773809523809523, + "aurc": 0.04890439545080765, + "error_rate_at_0_9": 0.023529411764705882, + "confidence_bias": -0.014797052655847964, + "top_bins": { + "0.9": { + "n": 170, + "errors": 4, + "error_rate": 0.023529411764705882 + }, + "0.95": { + "n": 151, + "errors": 3, + "error_rate": 0.019867549668874173 + }, + "0.99": { + "n": 39, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9761904761904762, + "confidence_cutoff": 0.9209057199846711 + }, + "0.8": { + "coverage": 0.8005952380952381, + "accuracy": 0.8996282527881041, + "confidence_cutoff": 0.5765734433053208 + } + } + }, + "noul": { + "n": 280, + "nll": 0.3088148767031123, + "acc": 0.8785714285714286, + "ece": 0.02716426081477808, + "brier": 0.18354251467515595, + "mean_conf": 0.8993028632705061, + "confident_error_rate": 0.025, + "coverage_at_0_9": 0.675, + "accuracy_at_0_9": 0.9629629629629629, + "coverage_at_5pct_error": 0.7428571428571429, + "coverage_at_1pct_error": 0.24642857142857144, + "aurc": 0.04037503899248962, + "error_rate_at_0_9": 0.037037037037037035, + "confidence_bias": 0.020731434699077567, + "top_bins": { + "0.9": { + "n": 189, + "errors": 7, + "error_rate": 0.037037037037037035 + }, + "0.95": { + "n": 142, + "errors": 5, + "error_rate": 0.035211267605633804 + }, + "0.99": { + "n": 35, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 0.9642857142857143, + "confidence_cutoff": 0.951430429315972 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.9330357142857143, + "confidence_cutoff": 0.8169286372799969 + } + } + }, + "score": { + "n": 40, + "nll": 0.47258178900171605, + "acc": 0.75, + "ece": 0.11341449184915596, + "brier": 0.3021145388916218, + "mean_conf": 0.8615577911220736, + "confident_error_rate": 0.025, + "coverage_at_0_9": 0.625, + "accuracy_at_0_9": 0.96, + "coverage_at_5pct_error": 0.65, + "coverage_at_1pct_error": 0.575, + "aurc": 0.060906503666048904, + "error_rate_at_0_9": 0.04, + "confidence_bias": 0.11155779112207365, + "top_bins": { + "0.9": { + "n": 25, + "errors": 1, + "error_rate": 0.04 + }, + "0.95": { + "n": 22, + "errors": 0, + "error_rate": 0.0 + }, + "0.99": { + "n": 5, + "errors": 0, + "error_rate": 0.0 + } + }, + "selective": { + "0.5": { + "coverage": 0.5, + "accuracy": 1.0, + "confidence_cutoff": 0.9618942423403596 + }, + "0.8": { + "coverage": 0.8, + "accuracy": 0.84375, + "confidence_cutoff": 0.7283845543861389 + } + }, + "score_mae": 0.27196301946029916, + "ranked_probability_score": 0.0837200362891968 + } + } + } + }, + "base_revision": "1001bb4d826a52d1f399e183466143f4da7b741b", + "test_used_for_selection": false +}