{ "selected_epoch": 3, "epochs": [ { "epoch": 1, "development": { "raw": { "questions": 14912, "accuracy": 0.8959898068669528, "mean_class_recall": 0.8086847041598828, "nll": 0.2676199905412643, "brier": 0.14581168647550916, "ece15": 0.024724176262894793, "temperature": 1.0, "coverage_at_confidence_0_9": 0.773806330472103, "error_rate_at_confidence_0_9": 0.0240055464078343, "ordinal_expected_level_mae": 0.00949861612167232, "tasks": { "agentdojo_tool_outputs:agentdojo_boundary_action": { "questions": 222, "accuracy": 0.9819819819819819, "mean_class_recall": 0.9670930232558139, "nll": 0.06952124509948641, "brier": 0.033360936223710204, "ece15": 0.01963763286720348, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9774774774774775, "error_rate_at_confidence_0_9": 0.013824884792626729, "ordinal_expected_level_mae": null }, "agentdojo_tool_outputs:agentdojo_embedded_injection": { "questions": 222, "accuracy": 0.9864864864864865, "mean_class_recall": 0.97, "nll": 0.06796588080667874, "brier": 0.02442578405456454, "ece15": 0.013534851962772593, "temperature": 1.0, "coverage_at_confidence_0_9": 0.972972972972973, "error_rate_at_confidence_0_9": 0.009259259259259259, "ordinal_expected_level_mae": null }, "byzfl_mnist:fl_attack_generated_update": { "questions": 1400, "accuracy": 0.9742857142857143, "mean_class_recall": 0.9504464285714286, "nll": 0.07440929350732509, "brier": 0.03861493679124129, "ece15": 0.01061565077999396, "temperature": 1.0, "coverage_at_confidence_0_9": 0.925, "error_rate_at_confidence_0_9": 0.008494208494208495, "ordinal_expected_level_mae": null }, "ciciot2023:ciciot_attack": { "questions": 5000, "accuracy": 0.8334, "mean_class_recall": 0.7313333333333334, "nll": 0.4357438451410003, "brier": 0.2420765591030422, "ece15": 0.05574236954650583, "temperature": 1.0, "coverage_at_confidence_0_9": 0.644, "error_rate_at_confidence_0_9": 0.06677018633540373, "ordinal_expected_level_mae": null }, "injecagent:injecagent_requested_operation_v13": { "questions": 124, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.004390672226438891, "brier": 0.0015490359962901127, "ece15": 0.003915493134373699, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9919354838709677, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_known_logon_action_v16": { "questions": 328, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.004780661486616687, "brier": 0.0026501860965206722, "ece15": 0.0038321931288341792, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9908536585365854, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_observable_policy": { "questions": 328, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 2.8436047490948972e-05, "brier": 2.213459113159646e-07, "ece15": 2.8380504783511674e-05, "temperature": 1.0, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "ton_iot_network:ton_attack": { "questions": 2500, "accuracy": 0.9832, "mean_class_recall": 0.9856, "nll": 0.0716936772308886, "brier": 0.032718644101254024, "ece15": 0.01640596644430267, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9948, "error_rate_at_confidence_0_9": 0.01648572577402493, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_evidence_level": { "questions": 797, "accuracy": 0.9937264742785445, "mean_class_recall": 0.9934913409721132, "nll": 0.0153080496936535, "brier": 0.009627399097334195, "ece15": 0.004599932218083785, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9912170639899623, "error_rate_at_confidence_0_9": 0.0037974683544303796, "ordinal_expected_level_mae": 0.00949861612167232 }, "twins_streamlet:twins_policy_action": { "questions": 797, "accuracy": 0.9924717691342535, "mean_class_recall": 0.9923419156847569, "nll": 0.017883068917921716, "brier": 0.011294880553062277, "ece15": 0.0038184765498114437, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9811794228356336, "error_rate_at_confidence_0_9": 0.0012787723785166241, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_quorum": { "questions": 797, "accuracy": 0.9962358845671268, "mean_class_recall": 0.995834315034425, "nll": 0.011957269820791244, "brier": 0.007393224542639707, "ece15": 0.00670405372457154, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9836888331242158, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_visible_conflict": { "questions": 797, "accuracy": 0.9974905897114178, "mean_class_recall": 0.9969442527413541, "nll": 0.010437285979484219, "brier": 0.0038211519865233618, "ece15": 0.0009184477173700703, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9962358845671268, "error_rate_at_confidence_0_9": 0.0012594458438287153, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_message_manipulation": { "questions": 800, "accuracy": 0.63, "mean_class_recall": 0.6354549163606691, "nll": 0.5876157448193948, "brier": 0.4142276501377162, "ece15": 0.031917999077751745, "temperature": 1.0, "coverage_at_confidence_0_9": 0.10375, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_supported_attack_type_v16": { "questions": 800, "accuracy": 0.59875, "mean_class_recall": 0.4115791153670771, "nll": 1.2269806519983115, "brier": 0.571523062943736, "ece15": 0.08060690424281568, "temperature": 1.0, "coverage_at_confidence_0_9": 0.11875, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null } }, "macro_task_accuracy": 0.9262877786032521, "macro_task_balanced_accuracy": 0.9021513315229265 }, "recalibrated": { "questions": 14912, "accuracy": 0.8959898068669528, "mean_class_recall": 0.8086847041598828, "nll": 0.2562444405938866, "brier": 0.1441282781781607, "ece15": 0.016168446150980925, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.7631437768240343, "error_rate_at_confidence_0_9": 0.020913884007029877, "ordinal_expected_level_mae": 0.010086743500138846, "tasks": { "agentdojo_tool_outputs:agentdojo_boundary_action": { "questions": 222, "accuracy": 0.9819819819819819, "mean_class_recall": 0.9670930232558139, "nll": 0.062064251368360995, "brier": 0.03274641091592082, "ece15": 0.018974706155035728, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.9774774774774775, "error_rate_at_confidence_0_9": 0.013824884792626729, "ordinal_expected_level_mae": null }, "agentdojo_tool_outputs:agentdojo_embedded_injection": { "questions": 222, "accuracy": 0.9864864864864865, "mean_class_recall": 0.97, "nll": 0.06063186595056717, "brier": 0.02487828836064738, "ece15": 0.013603842355616328, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.972972972972973, "error_rate_at_confidence_0_9": 0.009259259259259259, "ordinal_expected_level_mae": null }, "byzfl_mnist:fl_attack_generated_update": { "questions": 1400, "accuracy": 0.9742857142857143, "mean_class_recall": 0.9504464285714286, "nll": 0.08099031193469938, "brier": 0.03952213251707266, "ece15": 0.021719508126191233, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.9007142857142857, "error_rate_at_confidence_0_9": 0.007137192704203013, "ordinal_expected_level_mae": null }, "ciciot2023:ciciot_attack": { "questions": 5000, "accuracy": 0.8334, "mean_class_recall": 0.7313333333333334, "nll": 0.4055190039113824, "brier": 0.23828084117194667, "ece15": 0.047374338949895636, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.6276, "error_rate_at_confidence_0_9": 0.058954748247291265, "ordinal_expected_level_mae": null }, "injecagent:injecagent_requested_operation_v13": { "questions": 124, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.0058641097080354134, "brier": 0.002007000102514143, "ece15": 0.0052393541355612895, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.9838709677419355, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_known_logon_action_v16": { "questions": 328, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.005681917468602561, "brier": 0.002881636759121932, "ece15": 0.004649618743350316, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.9908536585365854, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_observable_policy": { "questions": 328, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 7.967277164193898e-05, "brier": 1.0603152088487418e-06, "ece15": 7.940563040823767e-05, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "ton_iot_network:ton_attack": { "questions": 2500, "accuracy": 0.9832, "mean_class_recall": 0.9856, "nll": 0.06283986398882055, "brier": 0.031969827800453, "ece15": 0.015381335792818867, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.992, "error_rate_at_confidence_0_9": 0.014516129032258065, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_evidence_level": { "questions": 797, "accuracy": 0.9937264742785445, "mean_class_recall": 0.9934913409721132, "nll": 0.014979010411112437, "brier": 0.009232024594375448, "ece15": 0.0059751389669031076, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.986198243412798, "error_rate_at_confidence_0_9": 0.001272264631043257, "ordinal_expected_level_mae": 0.010086743500138846 }, "twins_streamlet:twins_policy_action": { "questions": 797, "accuracy": 0.9924717691342535, "mean_class_recall": 0.9923419156847569, "nll": 0.017946444333002753, "brier": 0.011016825969457151, "ece15": 0.005758410373299677, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.9799247176913425, "error_rate_at_confidence_0_9": 0.0012804097311139564, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_quorum": { "questions": 797, "accuracy": 0.9962358845671268, "mean_class_recall": 0.995834315034425, "nll": 0.012786451464898273, "brier": 0.00742639513463676, "ece15": 0.007511975455200797, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.9824341279799247, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_visible_conflict": { "questions": 797, "accuracy": 0.9974905897114178, "mean_class_recall": 0.9969442527413541, "nll": 0.010010236373401123, "brier": 0.003879830332877274, "ece15": 0.0006442096172233073, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.9962358845671268, "error_rate_at_confidence_0_9": 0.0012594458438287153, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_message_manipulation": { "questions": 800, "accuracy": 0.63, "mean_class_recall": 0.6354549163606691, "nll": 0.5899608657261348, "brier": 0.41478182050444184, "ece15": 0.02797725458606776, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.095, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_supported_attack_type_v16": { "questions": 800, "accuracy": 0.59875, "mean_class_recall": 0.4115791153670771, "nll": 1.22100156554001, "brier": 0.5645239942686594, "ece15": 0.03015245360912201, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.09125, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null } }, "macro_task_accuracy": 0.9262877786032521, "macro_task_balanced_accuracy": 0.9021513315229265 } }, "test": { "raw": { "questions": 21634, "accuracy": 0.8587870943884626, "mean_class_recall": 0.8070634668409765, "nll": 0.38882363258516517, "brier": 0.19821716542849313, "ece15": 0.010114739193927335, "temperature": 1.0, "coverage_at_confidence_0_9": 0.6294259036701488, "error_rate_at_confidence_0_9": 0.016596900932657706, "ordinal_expected_level_mae": 0.010228305213524986, "tasks": { "agentdojo_tool_outputs:agentdojo_boundary_action": { "questions": 426, "accuracy": 0.9953051643192489, "mean_class_recall": 0.9895833333333333, "nll": 0.025162438436180455, "brier": 0.012711490628916015, "ece15": 0.009157764610888798, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9694835680751174, "error_rate_at_confidence_0_9": 0.002421307506053269, "ordinal_expected_level_mae": null }, "agentdojo_tool_outputs:agentdojo_embedded_injection": { "questions": 426, "accuracy": 0.9835680751173709, "mean_class_recall": 0.9635416666666667, "nll": 0.039729487219800165, "brier": 0.024078530771277282, "ece15": 0.008718119493403046, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9765258215962441, "error_rate_at_confidence_0_9": 0.007211538461538462, "ordinal_expected_level_mae": null }, "byzfl_mnist:fl_attack_generated_update": { "questions": 800, "accuracy": 0.93875, "mean_class_recall": 0.8984375, "nll": 0.19453941595728141, "brier": 0.09689065974959639, "ece15": 0.027271759841056658, "temperature": 1.0, "coverage_at_confidence_0_9": 0.87875, "error_rate_at_confidence_0_9": 0.034139402560455195, "ordinal_expected_level_mae": null }, "ciciot2023:ciciot_attack": { "questions": 6000, "accuracy": 0.9085, "mean_class_recall": 0.8498888888888889, "nll": 0.30026072378768554, "brier": 0.1436264364242417, "ece15": 0.037464827151113794, "temperature": 1.0, "coverage_at_confidence_0_9": 0.6986666666666667, "error_rate_at_confidence_0_9": 0.029818702290076337, "ordinal_expected_level_mae": null }, "injecagent:injecagent_requested_operation_v13": { "questions": 124, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.0009360610136724553, "brier": 5.531806653087891e-05, "ece15": 0.0009217275317422124, "temperature": 1.0, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_known_logon_action_v16": { "questions": 540, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.0029619998489780613, "brier": 0.0010650062308243538, "ece15": 0.0026318040347967494, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9925925925925926, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_observable_policy": { "questions": 540, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 4.009888988278539e-05, "brier": 8.791167963328194e-07, "ece15": 3.98768557623178e-05, "temperature": 1.0, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "ton_iot_network:ton_attack": { "questions": 3500, "accuracy": 0.9882857142857143, "mean_class_recall": 0.9807619047619047, "nll": 0.0658259406128984, "brier": 0.02104448067811733, "ece15": 0.0069241929022703305, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9845714285714285, "error_rate_at_confidence_0_9": 0.00783517121300058, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_evidence_level": { "questions": 659, "accuracy": 0.9908952959028832, "mean_class_recall": 0.9899995024628091, "nll": 0.022081750206618136, "brier": 0.01158884274796682, "ece15": 0.006288646106945627, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9954476479514416, "error_rate_at_confidence_0_9": 0.004573170731707317, "ordinal_expected_level_mae": 0.010228305213524986 }, "twins_streamlet:twins_policy_action": { "questions": 659, "accuracy": 0.9939301972685888, "mean_class_recall": 0.9932998324958123, "nll": 0.017255345447197507, "brier": 0.008938281239516815, "ece15": 0.004558085120676935, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9893778452200304, "error_rate_at_confidence_0_9": 0.003067484662576687, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_quorum": { "questions": 659, "accuracy": 0.9969650986342944, "mean_class_recall": 0.9966867772750125, "nll": 0.005807395652175951, "brier": 0.0034083401253427324, "ece15": 0.0031822580740001574, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9893778452200304, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_visible_conflict": { "questions": 659, "accuracy": 0.9969650986342944, "mean_class_recall": 0.9949748743718593, "nll": 0.009270762829542105, "brier": 0.00610585714141352, "ece15": 0.0051900114831071945, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9878603945371776, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_message_manipulation": { "questions": 3321, "accuracy": 0.6546221017765733, "mean_class_recall": 0.6604367829458773, "nll": 0.6040832398786956, "brier": 0.4230681455327693, "ece15": 0.012144611565586284, "temperature": 1.0, "coverage_at_confidence_0_9": 0.08220415537488708, "error_rate_at_confidence_0_9": 0.014652014652014652, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_supported_attack_type_v16": { "questions": 3321, "accuracy": 0.6248118036735922, "mean_class_recall": 0.3856953745435135, "nll": 1.250473514719575, "brier": 0.5523161094538271, "ece15": 0.04328702573140643, "temperature": 1.0, "coverage_at_confidence_0_9": 0.1093044263775971, "error_rate_at_confidence_0_9": 0.10192837465564739, "ordinal_expected_level_mae": null } }, "macro_task_accuracy": 0.9337570392580401, "macro_task_balanced_accuracy": 0.9073790312675484 }, "recalibrated": { "questions": 21634, "accuracy": 0.8587870943884626, "mean_class_recall": 0.8070634668409765, "nll": 0.3819588765605003, "brier": 0.1988268469084605, "ece15": 0.019530087212558746, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.6154663954885828, "error_rate_at_confidence_0_9": 0.014344723995493804, "ordinal_expected_level_mae": 0.01072047331602824, "tasks": { "agentdojo_tool_outputs:agentdojo_boundary_action": { "questions": 426, "accuracy": 0.9953051643192489, "mean_class_recall": 0.9895833333333333, "nll": 0.028102839471687808, "brier": 0.013352375860260531, "ece15": 0.011924724016462706, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.9624413145539906, "error_rate_at_confidence_0_9": 0.0024390243902439024, "ordinal_expected_level_mae": null }, "agentdojo_tool_outputs:agentdojo_embedded_injection": { "questions": 426, "accuracy": 0.9835680751173709, "mean_class_recall": 0.9635416666666667, "nll": 0.03830754803404818, "brier": 0.02344796092548543, "ece15": 0.009297528928361507, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.9694835680751174, "error_rate_at_confidence_0_9": 0.004842615012106538, "ordinal_expected_level_mae": null }, "byzfl_mnist:fl_attack_generated_update": { "questions": 800, "accuracy": 0.93875, "mean_class_recall": 0.8984375, "nll": 0.18853618602333774, "brier": 0.09702707273078204, "ece15": 0.01936307032837748, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.84125, "error_rate_at_confidence_0_9": 0.03417533432392273, "ordinal_expected_level_mae": null }, "ciciot2023:ciciot_attack": { "questions": 6000, "accuracy": 0.9085, "mean_class_recall": 0.8498888888888889, "nll": 0.2843389317505148, "brier": 0.14586901269531594, "ece15": 0.04356789248344108, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.6861666666666667, "error_rate_at_confidence_0_9": 0.028175856205975226, "ordinal_expected_level_mae": null }, "injecagent:injecagent_requested_operation_v13": { "questions": 124, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.0018322563352568268, "brier": 0.0001319762188887517, "ece15": 0.0017975613028342768, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_known_logon_action_v16": { "questions": 540, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.004015391153194959, "brier": 0.0013677195263880856, "ece15": 0.0035859188244034786, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.9888888888888889, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_observable_policy": { "questions": 540, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 9.895528751872936e-05, "brier": 3.024028962806258e-06, "ece15": 9.818509110354407e-05, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "ton_iot_network:ton_attack": { "questions": 3500, "accuracy": 0.9882857142857143, "mean_class_recall": 0.9807619047619047, "nll": 0.059841713174448447, "brier": 0.021127495151853498, "ece15": 0.005242220550012023, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.976, "error_rate_at_confidence_0_9": 0.007318501170960188, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_evidence_level": { "questions": 659, "accuracy": 0.9908952959028832, "mean_class_recall": 0.9899995024628091, "nll": 0.020306967381801818, "brier": 0.011159005956953514, "ece15": 0.005337269869592108, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.9908952959028832, "error_rate_at_confidence_0_9": 0.0030627871362940277, "ordinal_expected_level_mae": 0.01072047331602824 }, "twins_streamlet:twins_policy_action": { "questions": 659, "accuracy": 0.9939301972685888, "mean_class_recall": 0.9932998324958123, "nll": 0.016710459187929856, "brier": 0.008886576198762068, "ece15": 0.0032207054419096, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.9893778452200304, "error_rate_at_confidence_0_9": 0.003067484662576687, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_quorum": { "questions": 659, "accuracy": 0.9969650986342944, "mean_class_recall": 0.9966867772750125, "nll": 0.006802759775612477, "brier": 0.003478846491952278, "ece15": 0.004168012801242965, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.9893778452200304, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_visible_conflict": { "questions": 659, "accuracy": 0.9969650986342944, "mean_class_recall": 0.9949748743718593, "nll": 0.009405080434222151, "brier": 0.006029858296029803, "ece15": 0.004873038962556261, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.9863429438543247, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_message_manipulation": { "questions": 3321, "accuracy": 0.6546221017765733, "mean_class_recall": 0.6604367829458773, "nll": 0.6050451348525548, "brier": 0.4236393366516397, "ece15": 0.022216352122328073, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.0671484492622704, "error_rate_at_confidence_0_9": 0.008968609865470852, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_supported_attack_type_v16": { "questions": 3321, "accuracy": 0.6248118036735922, "mean_class_recall": 0.3856953745435135, "nll": 1.2411381163756228, "brier": 0.5515874786124301, "ece15": 0.032556668330635724, "temperature": 1.1759291426711453, "coverage_at_confidence_0_9": 0.07768744354110207, "error_rate_at_confidence_0_9": 0.06976744186046512, "ordinal_expected_level_mae": null } }, "macro_task_accuracy": 0.9337570392580401, "macro_task_balanced_accuracy": 0.9073790312675484 } }, "general": { "decision-v7": { "raw_clean": { "n": 1200, "nll": 0.5075688394241022, "acc": 0.8666666666666667, "ece": 0.07962904500464703, "brier": 0.2102660845468813, "mean_conf": 0.9440260985257297, "confident_error_rate": 0.0575, "coverage_at_0_9": 0.8266666666666667, "accuracy_at_0_9": 0.9304435483870968, "coverage_at_5pct_error": 0.7891666666666667, "coverage_at_1pct_error": 0.6058333333333333, "aurc": 0.028250531410146766, "error_rate_at_0_9": 0.06955645161290322, "confidence_bias": 0.07735943185906302, "top_bins": { "0.9": { "n": 992, "errors": 69, "error_rate": 0.06955645161290322 }, "0.95": { "n": 931, "errors": 44, "error_rate": 0.047261009667024706 }, "0.99": { "n": 853, "errors": 25, "error_rate": 0.029308323563892145 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9916666666666667, "confidence_cutoff": 0.9999431212544798 }, "0.8": { "coverage": 0.8, "accuracy": 0.9458333333333333, "confidence_cutoff": 0.9297460863402224 } }, "score_mae": 0.4607497712522395, "ranked_probability_score": 0.08554771406167339 }, "as_shipped_clean": { "n": 1200, "nll": 0.454544651852342, "acc": 0.8666666666666667, "ece": 0.06910317787787193, "brier": 0.20468221280246773, "mean_conf": 0.9344856098346994, "confident_error_rate": 0.041666666666666664, "coverage_at_0_9": 0.795, "accuracy_at_0_9": 0.9475890985324947, "coverage_at_5pct_error": 0.7866666666666666, "coverage_at_1pct_error": 0.6041666666666666, "aurc": 0.02824430039449196, "error_rate_at_0_9": 0.05241090146750524, "confidence_bias": 0.06781894316803272, "top_bins": { "0.9": { "n": 954, "errors": 50, "error_rate": 0.05241090146750524 }, "0.95": { "n": 901, "errors": 37, "error_rate": 0.041065482796892344 }, "0.99": { "n": 820, "errors": 16, "error_rate": 0.01951219512195122 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9916666666666667, "confidence_cutoff": 0.9997511018421836 }, "0.8": { "coverage": 0.8, "accuracy": 0.9458333333333333, "confidence_cutoff": 0.894833765474474 } }, "score_mae": 0.46716677242673305, "ranked_probability_score": 0.08353281985680432 }, "sources": { "agnews": { "n": 240, "nll": 0.2969571491022943, "acc": 0.9166666666666666, "ece": 0.05669702215129781, "brier": 0.13610080496920665, "mean_conf": 0.9733636888179642, "confident_error_rate": 0.04583333333333333, "coverage_at_0_9": 0.9291666666666667, "accuracy_at_0_9": 0.9506726457399103, "coverage_at_5pct_error": 0.9291666666666667, "coverage_at_1pct_error": 0.7333333333333333, "aurc": 0.011459961623908478, "error_rate_at_0_9": 0.04932735426008968, "confidence_bias": 0.05669702215129757, "top_bins": { "0.9": { "n": 223, "errors": 11, "error_rate": 0.04932735426008968 }, "0.95": { "n": 216, "errors": 10, "error_rate": 0.046296296296296294 }, "0.99": { "n": 197, "errors": 4, "error_rate": 0.02030456852791878 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.999603972973965 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.9927218344770156 } } }, "amazon": { "n": 80, "nll": 1.1950196702821447, "acc": 0.6125, "ece": 0.13013344623625947, "brier": 0.5588490719296325, "mean_conf": 0.7426334462362595, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.2625, "accuracy_at_0_9": 0.7619047619047619, "coverage_at_5pct_error": 0.075, "coverage_at_1pct_error": 0.075, "aurc": 0.2609215557981968, "error_rate_at_0_9": 0.23809523809523808, "confidence_bias": 0.13013344623625944, "top_bins": { "0.9": { "n": 21, "errors": 5, "error_rate": 0.23809523809523808 }, "0.95": { "n": 13, "errors": 2, "error_rate": 0.15384615384615385 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.775, "confidence_cutoff": 0.7941759769280257 }, "0.8": { "coverage": 0.8, "accuracy": 0.65625, "confidence_cutoff": 0.5659829933652708 } }, "score_mae": 0.4892758257511954, "ranked_probability_score": 0.08858888411388968 }, "banking77": { "n": 80, "nll": 0.5758902612345038, "acc": 0.85, "ece": 0.08719569564847485, "brier": 0.2407008008106888, "mean_conf": 0.898073693405655, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 0.95, "coverage_at_5pct_error": 0.7625, "coverage_at_1pct_error": 0.625, "aurc": 0.03238808988486375, "error_rate_at_0_9": 0.05, "confidence_bias": 0.04807369340565504, "top_bins": { "0.9": { "n": 60, "errors": 3, "error_rate": 0.05 }, "0.95": { "n": 55, "errors": 2, "error_rate": 0.03636363636363636 }, "0.99": { "n": 47, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9967902498867545 }, "0.8": { "coverage": 0.8, "accuracy": 0.90625, "confidence_cutoff": 0.8441430830009494 } } }, "boolq": { "n": 80, "nll": 0.5929837671372773, "acc": 0.875, "ece": 0.12193170422191033, "brier": 0.2235795473979305, "mean_conf": 0.9525887301568842, "confident_error_rate": 0.0875, "coverage_at_0_9": 0.85, "accuracy_at_0_9": 0.8970588235294118, "coverage_at_5pct_error": 0.6875, "coverage_at_1pct_error": 0.1, "aurc": 0.05282386578594207, "error_rate_at_0_9": 0.10294117647058823, "confidence_bias": 0.07758873015688417, "top_bins": { "0.9": { "n": 68, "errors": 7, "error_rate": 0.10294117647058823 }, "0.95": { "n": 67, "errors": 7, "error_rate": 0.1044776119402985 }, "0.99": { "n": 61, "errors": 4, "error_rate": 0.06557377049180328 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.999911295185737 }, "0.8": { "coverage": 0.8, "accuracy": 0.921875, "confidence_cutoff": 0.9794008835333886 } } }, "contrastive": { "n": 160, "nll": 0.05996336403593442, "acc": 0.96875, "ece": 0.02417385494863131, "brier": 0.04256566388357948, "mean_conf": 0.9929238549486312, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.98125, "accuracy_at_0_9": 0.9872611464968153, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.975, "aurc": 0.0005908980274374722, "error_rate_at_0_9": 0.012738853503184714, "confidence_bias": 0.024173854948631224, "top_bins": { "0.9": { "n": 157, "errors": 2, "error_rate": 0.012738853503184714 }, "0.95": { "n": 155, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 148, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.999992580361535 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9997815066400773 } } }, "dbpedia14": { "n": 80, "nll": 0.018296334655618373, "acc": 0.9875, "ece": 0.010929643397143786, "brier": 0.013547082606835407, "mean_conf": 0.9948529231687339, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9875, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.9875, "aurc": 0.00015625, "error_rate_at_0_9": 0.0, "confidence_bias": 0.007352923168733816, "top_bins": { "0.9": { "n": 79, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 77, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 77, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999968083760991 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.999980508233233 } } }, "imdb": { "n": 80, "nll": 0.06650112598274324, "acc": 0.9625, "ece": 0.021127312041626652, "brier": 0.047311917135916516, "mean_conf": 0.9836273120416268, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.9625, "accuracy_at_0_9": 0.987012987012987, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.95, "aurc": 0.0012680561790530144, "error_rate_at_0_9": 0.012987012987012988, "confidence_bias": 0.021127312041626767, "top_bins": { "0.9": { "n": 77, "errors": 1, "error_rate": 0.012987012987012988 }, "0.95": { "n": 75, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 71, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999868991306584 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.999740450053584 } } }, "mnli": { "n": 80, "nll": 0.4800709042620615, "acc": 0.9125, "ece": 0.07753013500835038, "brier": 0.16332522643400862, "mean_conf": 0.9812306735501618, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.925, "accuracy_at_0_9": 0.9324324324324325, "coverage_at_5pct_error": 0.8875, "coverage_at_1pct_error": 0.6375, "aurc": 0.016122981848020253, "error_rate_at_0_9": 0.06756756756756757, "confidence_bias": 0.06873067355016182, "top_bins": { "0.9": { "n": 74, "errors": 5, "error_rate": 0.06756756756756757 }, "0.95": { "n": 74, "errors": 5, "error_rate": 0.06756756756756757 }, "0.99": { "n": 71, "errors": 3, "error_rate": 0.04225352112676056 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.999991586558766 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.9998147211094783 } } }, "sst5": { "n": 80, "nll": 1.2448974820413372, "acc": 0.6125, "ece": 0.22505814089878196, "brier": 0.5919475867934173, "mean_conf": 0.8141449665678422, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.2875, "accuracy_at_0_9": 0.782608695652174, "coverage_at_5pct_error": 0.0875, "coverage_at_1pct_error": 0.0875, "aurc": 0.30028061666521133, "error_rate_at_0_9": 0.21739130434782608, "confidence_bias": 0.20164496656784214, "top_bins": { "0.9": { "n": 23, "errors": 5, "error_rate": 0.21739130434782608 }, "0.95": { "n": 8, "errors": 1, "error_rate": 0.125 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.855339285968297 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.7133087914054365 } }, "score_mae": 0.4884492482076522, "ranked_probability_score": 0.08971730097335581 }, "trec": { "n": 80, "nll": 0.28480951015184036, "acc": 0.9625, "ece": 0.030856363792743223, "brier": 0.058704890234289875, "mean_conf": 0.9903723975747134, "confident_error_rate": 0.025, "coverage_at_0_9": 0.975, "accuracy_at_0_9": 0.9743589743589743, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.325, "aurc": 0.015881097479045377, "error_rate_at_0_9": 0.02564102564102564, "confidence_bias": 0.02787239757471338, "top_bins": { "0.9": { "n": 78, "errors": 2, "error_rate": 0.02564102564102564 }, "0.95": { "n": 78, "errors": 2, "error_rate": 0.02564102564102564 }, "0.99": { "n": 77, "errors": 2, "error_rate": 0.025974025974025976 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9999910962505757 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.9999026419356177 } } }, "yelp": { "n": 160, "nll": 0.6744512733294261, "acc": 0.76875, "ece": 0.11010641434949675, "brier": 0.33941666300975865, "mean_conf": 0.8769106142337293, "confident_error_rate": 0.05625, "coverage_at_0_9": 0.5875, "accuracy_at_0_9": 0.9042553191489362, "coverage_at_5pct_error": 0.425, "coverage_at_1pct_error": 0.16875, "aurc": 0.08692888800847512, "error_rate_at_0_9": 0.09574468085106383, "confidence_bias": 0.10816061423372925, "top_bins": { "0.9": { "n": 94, "errors": 9, "error_rate": 0.09574468085106383 }, "0.95": { "n": 83, "errors": 8, "error_rate": 0.0963855421686747 }, "0.99": { "n": 67, "errors": 3, "error_rate": 0.04477611940298507 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.9744081429606993 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.7335309795017884 } }, "score_mae": 0.4237752433213515, "ranked_probability_score": 0.07229227448316741 } }, "types": { "choice": { "n": 480, "nll": 0.3107149818460764, "acc": 0.93125, "ece": 0.04989938310601072, "brier": 0.11702630095906619, "mean_conf": 0.9725961663793784, "confident_error_rate": 0.03333333333333333, "coverage_at_0_9": 0.925, "accuracy_at_0_9": 0.963963963963964, "coverage_at_5pct_error": 0.95, "coverage_at_1pct_error": 0.8333333333333334, "aurc": 0.011065724252754237, "error_rate_at_0_9": 0.036036036036036036, "confidence_bias": 0.04134616637937838, "top_bins": { "0.9": { "n": 444, "errors": 16, "error_rate": 0.036036036036036036 }, "0.95": { "n": 436, "errors": 15, "error_rate": 0.034403669724770644 }, "0.99": { "n": 415, "errors": 7, "error_rate": 0.016867469879518072 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9958333333333333, "confidence_cutoff": 0.9999830700690301 }, "0.8": { "coverage": 0.8, "accuracy": 0.9895833333333334, "confidence_cutoff": 0.9987196152255894 } } }, "noul": { "n": 480, "nll": 0.2523830232779272, "acc": 0.925, "ece": 0.04746414517655202, "brier": 0.115343937114107, "mean_conf": 0.9724641451765519, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.91875, "accuracy_at_0_9": 0.9591836734693877, "coverage_at_5pct_error": 0.94375, "coverage_at_1pct_error": 0.7041666666666667, "aurc": 0.015342193433417054, "error_rate_at_0_9": 0.04081632653061224, "confidence_bias": 0.04746414517655184, "top_bins": { "0.9": { "n": 441, "errors": 18, "error_rate": 0.04081632653061224 }, "0.95": { "n": 427, "errors": 14, "error_rate": 0.03278688524590164 }, "0.99": { "n": 395, "errors": 8, "error_rate": 0.020253164556962026 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9958333333333333, "confidence_cutoff": 0.9998511569646503 }, "0.8": { "coverage": 0.8, "accuracy": 0.9791666666666666, "confidence_cutoff": 0.9945634588215609 } } }, "score": { "n": 240, "nll": 1.146527249013703, "acc": 0.6208333333333333, "ece": 0.16277510247546034, "brier": 0.5586705878659922, "mean_conf": 0.7823074260616364, "confident_error_rate": 0.06666666666666667, "coverage_at_0_9": 0.2875, "accuracy_at_0_9": 0.7681159420289855, "coverage_at_5pct_error": 0.016666666666666666, "coverage_at_1pct_error": 0.016666666666666666, "aurc": 0.2783513871168549, "error_rate_at_0_9": 0.2318840579710145, "confidence_bias": 0.16147409272830304, "top_bins": { "0.9": { "n": 69, "errors": 16, "error_rate": 0.2318840579710145 }, "0.95": { "n": 38, "errors": 8, "error_rate": 0.21052631578947367 }, "0.99": { "n": 10, "errors": 1, "error_rate": 0.1 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7166666666666667, "confidence_cutoff": 0.8132439678266241 }, "0.8": { "coverage": 0.8, "accuracy": 0.6614583333333334, "confidence_cutoff": 0.6191827053278941 } }, "score_mae": 0.46716677242673305, "ranked_probability_score": 0.08353281985680432 } } }, "transfer-v4": { "raw_clean": { "n": 656, "nll": 0.7060605455091327, "acc": 0.8384146341463414, "ece": 0.10225794465378485, "brier": 0.26913580782629415, "mean_conf": 0.936023112192006, "confident_error_rate": 0.08689024390243902, "coverage_at_0_9": 0.8246951219512195, "accuracy_at_0_9": 0.8946395563770795, "coverage_at_5pct_error": 0.5975609756097561, "coverage_at_1pct_error": 0.38109756097560976, "aurc": 0.04670337348873893, "error_rate_at_0_9": 0.10536044362292052, "confidence_bias": 0.09760847804566453, "top_bins": { "0.9": { "n": 541, "errors": 57, "error_rate": 0.10536044362292052 }, "0.95": { "n": 508, "errors": 46, "error_rate": 0.09055118110236221 }, "0.99": { "n": 449, "errors": 28, "error_rate": 0.062360801781737196 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9695121951219512, "confidence_cutoff": 0.9996513990871982 }, "0.8": { "coverage": 0.8003048780487805, "accuracy": 0.900952380952381, "confidence_cutoff": 0.9260639080978317 } }, "score_mae": 0.28732928759135146, "ranked_probability_score": 0.12507059546399885 }, "as_shipped_clean": { "n": 656, "nll": 0.6270891070988179, "acc": 0.8384146341463414, "ece": 0.09035981505437185, "brier": 0.26355975613405186, "mean_conf": 0.9253405484079532, "confident_error_rate": 0.07469512195121951, "coverage_at_0_9": 0.7896341463414634, "accuracy_at_0_9": 0.9054054054054054, "coverage_at_5pct_error": 0.5960365853658537, "coverage_at_1pct_error": 0.3795731707317073, "aurc": 0.046713600417941994, "error_rate_at_0_9": 0.0945945945945946, "confidence_bias": 0.08692591426161178, "top_bins": { "0.9": { "n": 518, "errors": 49, "error_rate": 0.0945945945945946 }, "0.95": { "n": 494, "errors": 44, "error_rate": 0.08906882591093117 }, "0.99": { "n": 413, "errors": 23, "error_rate": 0.05569007263922518 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9695121951219512, "confidence_cutoff": 0.9988538098904463 }, "0.8": { "coverage": 0.8003048780487805, "accuracy": 0.900952380952381, "confidence_cutoff": 0.8926384237014877 } }, "score_mae": 0.28661481608517747, "ranked_probability_score": 0.1219924564403267 }, "sources": { "composition_holdout": { "n": 96, "nll": 0.7716759343811087, "acc": 0.8854166666666666, "ece": 0.13161705342878371, "brier": 0.2418445379901801, "mean_conf": 0.9798888855966136, "confident_error_rate": 0.11458333333333333, "coverage_at_0_9": 0.9479166666666666, "accuracy_at_0_9": 0.8791208791208791, "coverage_at_5pct_error": 0.6875, "coverage_at_1pct_error": 0.6666666666666666, "aurc": 0.02787241712770423, "error_rate_at_0_9": 0.12087912087912088, "confidence_bias": 0.09447221892994695, "top_bins": { "0.9": { "n": 91, "errors": 11, "error_rate": 0.12087912087912088 }, "0.95": { "n": 90, "errors": 11, "error_rate": 0.12222222222222222 }, "0.99": { "n": 86, "errors": 11, "error_rate": 0.12790697674418605 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.999944022537299 }, "0.8": { "coverage": 0.8020833333333334, "accuracy": 0.948051948051948, "confidence_cutoff": 0.9994106055377326 } } }, "contrastive": { "n": 80, "nll": 0.5084316515078089, "acc": 0.85, "ece": 0.11914860470823861, "brier": 0.24157018604381048, "mean_conf": 0.9691486047082385, "confident_error_rate": 0.0875, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.9027777777777778, "coverage_at_5pct_error": 0.85, "coverage_at_1pct_error": 0.8, "aurc": 0.018225744933534453, "error_rate_at_0_9": 0.09722222222222222, "confidence_bias": 0.11914860470823851, "top_bins": { "0.9": { "n": 72, "errors": 7, "error_rate": 0.09722222222222222 }, "0.95": { "n": 72, "errors": 7, "error_rate": 0.09722222222222222 }, "0.99": { "n": 68, "errors": 3, "error_rate": 0.04411764705882353 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999840553223158 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9992294841723544 } }, "score_mae": 0.28661481608517747, "ranked_probability_score": 0.1219924564403267 }, "emotion": { "n": 80, "nll": 1.1842140096136198, "acc": 0.6625, "ece": 0.16344433654532525, "brier": 0.49695454986128207, "mean_conf": 0.7770470897943043, "confident_error_rate": 0.075, "coverage_at_0_9": 0.4375, "accuracy_at_0_9": 0.8285714285714286, "coverage_at_5pct_error": 0.075, "coverage_at_1pct_error": 0.075, "aurc": 0.18204891186299516, "error_rate_at_0_9": 0.17142857142857143, "confidence_bias": 0.11454708979430428, "top_bins": { "0.9": { "n": 35, "errors": 6, "error_rate": 0.17142857142857143 }, "0.95": { "n": 30, "errors": 4, "error_rate": 0.13333333333333333 }, "0.99": { "n": 12, "errors": 1, "error_rate": 0.08333333333333333 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8455177052439261 }, "0.8": { "coverage": 0.8, "accuracy": 0.734375, "confidence_cutoff": 0.5895563704278984 } } }, "mmlu": { "n": 80, "nll": 0.8811102729379066, "acc": 0.7125, "ece": 0.1128354245761854, "brier": 0.4333401015309152, "mean_conf": 0.8253354245761854, "confident_error_rate": 0.05, "coverage_at_0_9": 0.475, "accuracy_at_0_9": 0.8947368421052632, "coverage_at_5pct_error": 0.3625, "coverage_at_1pct_error": 0.3125, "aurc": 0.11284549915448887, "error_rate_at_0_9": 0.10526315789473684, "confidence_bias": 0.11283542457618534, "top_bins": { "0.9": { "n": 38, "errors": 4, "error_rate": 0.10526315789473684 }, "0.95": { "n": 32, "errors": 2, "error_rate": 0.0625 }, "0.99": { "n": 19, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.8934639334502172 }, "0.8": { "coverage": 0.8, "accuracy": 0.796875, "confidence_cutoff": 0.6529787827885164 } } }, "paws": { "n": 80, "nll": 0.9149710000217235, "acc": 0.8, "ece": 0.15742064884121132, "brier": 0.3163214505434944, "mean_conf": 0.9481862302428528, "confident_error_rate": 0.125, "coverage_at_0_9": 0.8625, "accuracy_at_0_9": 0.855072463768116, "coverage_at_5pct_error": 0.275, "coverage_at_1pct_error": 0.175, "aurc": 0.09027125896066751, "error_rate_at_0_9": 0.14492753623188406, "confidence_bias": 0.1481862302428527, "top_bins": { "0.9": { "n": 69, "errors": 10, "error_rate": 0.14492753623188406 }, "0.95": { "n": 69, "errors": 10, "error_rate": 0.14492753623188406 }, "0.99": { "n": 61, "errors": 6, "error_rate": 0.09836065573770492 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.9997018073599228 }, "0.8": { "coverage": 0.8, "accuracy": 0.890625, "confidence_cutoff": 0.9816976299326602 } } }, "qnli": { "n": 80, "nll": 0.3010750024235448, "acc": 0.925, "ece": 0.08174310999100654, "brier": 0.13992055357407857, "mean_conf": 0.9736296740922775, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.925, "accuracy_at_0_9": 0.9324324324324325, "coverage_at_5pct_error": 0.8625, "coverage_at_1pct_error": 0.625, "aurc": 0.014993418347854443, "error_rate_at_0_9": 0.06756756756756757, "confidence_bias": 0.048629674092277475, "top_bins": { "0.9": { "n": 74, "errors": 5, "error_rate": 0.06756756756756757 }, "0.95": { "n": 71, "errors": 4, "error_rate": 0.056338028169014086 }, "0.99": { "n": 64, "errors": 2, "error_rate": 0.03125 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9996265539820621 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.9930251162586794 } } }, "sciq": { "n": 80, "nll": 0.007314621146129305, "acc": 1.0, "ece": 0.006816864281393114, "brier": 0.0015733066903292278, "mean_conf": 0.993183135718607, "confident_error_rate": 0.0, "coverage_at_0_9": 0.975, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0068168642813930536, "top_bins": { "0.9": { "n": 78, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 77, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 72, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999608059905641 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9995954309046333 } } }, "tweet_offensive": { "n": 80, "nll": 0.4190029993022442, "acc": 0.8625, "ece": 0.08458362708432104, "brier": 0.2412964064670991, "mean_conf": 0.9253956750968154, "confident_error_rate": 0.075, "coverage_at_0_9": 0.7625, "accuracy_at_0_9": 0.9016393442622951, "coverage_at_5pct_error": 0.575, "coverage_at_1pct_error": 0.5, "aurc": 0.04870527959080802, "error_rate_at_0_9": 0.09836065573770492, "confidence_bias": 0.06289567509681537, "top_bins": { "0.9": { "n": 61, "errors": 6, "error_rate": 0.09836065573770492 }, "0.95": { "n": 53, "errors": 6, "error_rate": 0.11320754716981132 }, "0.99": { "n": 31, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9833703279193756 }, "0.8": { "coverage": 0.8, "accuracy": 0.890625, "confidence_cutoff": 0.891081054907394 } } } }, "types": { "choice": { "n": 336, "nll": 0.7139642916559491, "acc": 0.8184523809523809, "ece": 0.08621197602515834, "brier": 0.29097176277874826, "mean_conf": 0.8979600744773888, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.7202380952380952, "accuracy_at_0_9": 0.9132231404958677, "coverage_at_5pct_error": 0.5208333333333334, "coverage_at_1pct_error": 0.3958333333333333, "aurc": 0.05304090653277752, "error_rate_at_0_9": 0.08677685950413223, "confidence_bias": 0.07950769352500786, "top_bins": { "0.9": { "n": 242, "errors": 21, "error_rate": 0.08677685950413223 }, "0.95": { "n": 229, "errors": 17, "error_rate": 0.07423580786026202 }, "0.99": { "n": 189, "errors": 12, "error_rate": 0.06349206349206349 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9702380952380952, "confidence_cutoff": 0.9970506958588666 }, "0.8": { "coverage": 0.8005952380952381, "accuracy": 0.895910780669145, "confidence_cutoff": 0.7834280359430882 } } }, "noul": { "n": 280, "nll": 0.4671583132891337, "acc": 0.8821428571428571, "ece": 0.07465765160771948, "brier": 0.1992966887782935, "mean_conf": 0.9563447099149558, "confident_error_rate": 0.075, "coverage_at_0_9": 0.8714285714285714, "accuracy_at_0_9": 0.9139344262295082, "coverage_at_5pct_error": 0.7642857142857142, "coverage_at_1pct_error": 0.26785714285714285, "aurc": 0.03774979623265823, "error_rate_at_0_9": 0.0860655737704918, "confidence_bias": 0.07420185277209868, "top_bins": { "0.9": { "n": 244, "errors": 21, "error_rate": 0.0860655737704918 }, "0.95": { "n": 233, "errors": 20, "error_rate": 0.08583690987124463 }, "0.99": { "n": 196, "errors": 8, "error_rate": 0.04081632653061224 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9642857142857143, "confidence_cutoff": 0.9991586798493963 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.964939495122677 } } }, "score": { "n": 40, "nll": 1.0168531134867067, "acc": 0.7, "ece": 0.23830739887567784, "brier": 0.48314037180891034, "mean_conf": 0.9383073988756777, "confident_error_rate": 0.175, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 0.78125, "coverage_at_5pct_error": 0.65, "coverage_at_1pct_error": 0.6, "aurc": 0.07895709505044536, "error_rate_at_0_9": 0.21875, "confidence_bias": 0.23830739887567776, "top_bins": { "0.9": { "n": 32, "errors": 7, "error_rate": 0.21875 }, "0.95": { "n": 32, "errors": 7, "error_rate": 0.21875 }, "0.99": { "n": 28, "errors": 3, "error_rate": 0.10714285714285714 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9997651676938517 }, "0.8": { "coverage": 0.8, "accuracy": 0.78125, "confidence_cutoff": 0.9590969197346936 } }, "score_mae": 0.28661481608517747, "ranked_probability_score": 0.1219924564403267 } } } }, "temperature": 1.1759291426711453 }, { "epoch": 2, "development": { "raw": { "questions": 14912, "accuracy": 0.9046405579399142, "mean_class_recall": 0.8351706914084861, "nll": 0.2609479185046837, "brier": 0.13871271598030968, "ece15": 0.02912533581539033, "temperature": 1.0, "coverage_at_confidence_0_9": 0.811762339055794, "error_rate_at_confidence_0_9": 0.030318050392399834, "ordinal_expected_level_mae": 0.007142014228908907, "tasks": { "agentdojo_tool_outputs:agentdojo_boundary_action": { "questions": 222, "accuracy": 0.9774774774774775, "mean_class_recall": 0.9570930232558139, "nll": 0.0857490162426134, "brier": 0.03780111867446018, "ece15": 0.020326742033581136, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9864864864864865, "error_rate_at_confidence_0_9": 0.0136986301369863, "ordinal_expected_level_mae": null }, "agentdojo_tool_outputs:agentdojo_embedded_injection": { "questions": 222, "accuracy": 0.9819819819819819, "mean_class_recall": 0.96, "nll": 0.08583505640070498, "brier": 0.03170617674969949, "ece15": 0.018778574788420183, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9819819819819819, "error_rate_at_confidence_0_9": 0.009174311926605505, "ordinal_expected_level_mae": null }, "byzfl_mnist:fl_attack_generated_update": { "questions": 1400, "accuracy": 0.9828571428571429, "mean_class_recall": 0.9678571428571429, "nll": 0.05038447430212686, "brier": 0.026644462415551518, "ece15": 0.006643072262975827, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9757142857142858, "error_rate_at_confidence_0_9": 0.008784773060029283, "ordinal_expected_level_mae": null }, "ciciot2023:ciciot_attack": { "questions": 5000, "accuracy": 0.839, "mean_class_recall": 0.7132000000000001, "nll": 0.4567278707638309, "brier": 0.24289094068797595, "ece15": 0.07089882316707959, "temperature": 1.0, "coverage_at_confidence_0_9": 0.7078, "error_rate_at_confidence_0_9": 0.08194405199208817, "ordinal_expected_level_mae": null }, "injecagent:injecagent_requested_operation_v13": { "questions": 124, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.0030323687345377927, "brier": 0.0010147751519150774, "ece15": 0.0027304101493378944, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9919354838709677, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_known_logon_action_v16": { "questions": 328, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.0013590718776517526, "brier": 0.0005853758373684321, "ece15": 0.001174090997847941, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9969512195121951, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_observable_policy": { "questions": 328, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 2.706087239227311e-05, "brier": 4.4631636950527135e-07, "ece15": 2.6948652833984242e-05, "temperature": 1.0, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "ton_iot_network:ton_attack": { "questions": 2500, "accuracy": 0.9788, "mean_class_recall": 0.98, "nll": 0.08755100200107269, "brier": 0.038158588944742826, "ece15": 0.017829224144769444, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9896, "error_rate_at_confidence_0_9": 0.016976556184316895, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_evidence_level": { "questions": 797, "accuracy": 0.9949811794228356, "mean_class_recall": 0.9949406163344321, "nll": 0.020100883811983803, "brier": 0.009532705405460478, "ece15": 0.004664870689987746, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9962358845671268, "error_rate_at_confidence_0_9": 0.003778337531486146, "ordinal_expected_level_mae": 0.007142014228908907 }, "twins_streamlet:twins_policy_action": { "questions": 797, "accuracy": 0.9887076537013801, "mean_class_recall": 0.9889237691623501, "nll": 0.029531627570397425, "brier": 0.01682083664455817, "ece15": 0.008194444119065863, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9887076537013801, "error_rate_at_confidence_0_9": 0.0038071065989847717, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_quorum": { "questions": 797, "accuracy": 0.9937264742785445, "mean_class_recall": 0.9922118380062306, "nll": 0.019942020828876864, "brier": 0.011733672902765547, "ece15": 0.006519660642330038, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9912170639899623, "error_rate_at_confidence_0_9": 0.0037974683544303796, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_visible_conflict": { "questions": 797, "accuracy": 0.9974905897114178, "mean_class_recall": 0.998236331569665, "nll": 0.016769983022509982, "brier": 0.0049992664597086935, "ece15": 0.0021278163552494573, "temperature": 1.0, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.002509410288582183, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_message_manipulation": { "questions": 800, "accuracy": 0.6875, "mean_class_recall": 0.6966034271725826, "nll": 0.5142628800857739, "brier": 0.36155138388149327, "ece15": 0.02205690591199928, "temperature": 1.0, "coverage_at_confidence_0_9": 0.22125, "error_rate_at_confidence_0_9": 0.011299435028248588, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_supported_attack_type_v16": { "questions": 800, "accuracy": 0.67375, "mean_class_recall": 0.5089660282717738, "nll": 0.998813782244008, "brier": 0.4775012041931279, "ece15": 0.03873448735113525, "temperature": 1.0, "coverage_at_confidence_0_9": 0.20625, "error_rate_at_confidence_0_9": 0.030303030303030304, "ordinal_expected_level_mae": null } }, "macro_task_accuracy": 0.9354480356736271, "macro_task_balanced_accuracy": 0.9112880126164279 }, "recalibrated": { "questions": 14912, "accuracy": 0.9046405579399142, "mean_class_recall": 0.8351706914084861, "nll": 0.24022147566124724, "brier": 0.13631204830326463, "ece15": 0.023735839802210217, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.790973712446352, "error_rate_at_confidence_0_9": 0.02467147096227215, "ordinal_expected_level_mae": 0.007897347093607133, "tasks": { "agentdojo_tool_outputs:agentdojo_boundary_action": { "questions": 222, "accuracy": 0.9774774774774775, "mean_class_recall": 0.9570930232558139, "nll": 0.06980345492279078, "brier": 0.036072899559119326, "ece15": 0.020348852979949178, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.9819819819819819, "error_rate_at_confidence_0_9": 0.013761467889908258, "ordinal_expected_level_mae": null }, "agentdojo_tool_outputs:agentdojo_embedded_injection": { "questions": 222, "accuracy": 0.9819819819819819, "mean_class_recall": 0.96, "nll": 0.0697534508038578, "brier": 0.030632585867415095, "ece15": 0.018874774396367578, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.9819819819819819, "error_rate_at_confidence_0_9": 0.009174311926605505, "ordinal_expected_level_mae": null }, "byzfl_mnist:fl_attack_generated_update": { "questions": 1400, "accuracy": 0.9828571428571429, "mean_class_recall": 0.9678571428571429, "nll": 0.05022700385733144, "brier": 0.026160310719648357, "ece15": 0.005133274295293359, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.9642857142857143, "error_rate_at_confidence_0_9": 0.007407407407407408, "ordinal_expected_level_mae": null }, "ciciot2023:ciciot_attack": { "questions": 5000, "accuracy": 0.839, "mean_class_recall": 0.7132000000000001, "nll": 0.4007512736870225, "brier": 0.23542102872463042, "ece15": 0.05680924935578224, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.6622, "error_rate_at_confidence_0_9": 0.0664451827242525, "ordinal_expected_level_mae": null }, "injecagent:injecagent_requested_operation_v13": { "questions": 124, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.00473204279698464, "brier": 0.0015928649780981342, "ece15": 0.004243332776407401, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.9919354838709677, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_known_logon_action_v16": { "questions": 328, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.0022032517878559357, "brier": 0.0007796053221569698, "ece15": 0.0019494525692642349, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.9969512195121951, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_observable_policy": { "questions": 328, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 9.041263403421138e-05, "brier": 3.609861838766288e-06, "ece15": 8.949527479169929e-05, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "ton_iot_network:ton_attack": { "questions": 2500, "accuracy": 0.9788, "mean_class_recall": 0.98, "nll": 0.07409278691965634, "brier": 0.03695008653192076, "ece15": 0.013363996653905837, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.984, "error_rate_at_confidence_0_9": 0.016666666666666666, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_evidence_level": { "questions": 797, "accuracy": 0.9949811794228356, "mean_class_recall": 0.9949406163344321, "nll": 0.017338830453877865, "brier": 0.00923947768144349, "ece15": 0.0037098432118081266, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.9899623588456713, "error_rate_at_confidence_0_9": 0.0038022813688212928, "ordinal_expected_level_mae": 0.007897347093607133 }, "twins_streamlet:twins_policy_action": { "questions": 797, "accuracy": 0.9887076537013801, "mean_class_recall": 0.9889237691623501, "nll": 0.025810322060464137, "brier": 0.015590356305548642, "ece15": 0.0063347054801968505, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.9824341279799247, "error_rate_at_confidence_0_9": 0.002554278416347382, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_quorum": { "questions": 797, "accuracy": 0.9937264742785445, "mean_class_recall": 0.9922118380062306, "nll": 0.017984967092368, "brier": 0.011131458808574956, "ece15": 0.005638113127633191, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.9912170639899623, "error_rate_at_confidence_0_9": 0.0037974683544303796, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_visible_conflict": { "questions": 797, "accuracy": 0.9974905897114178, "mean_class_recall": 0.998236331569665, "nll": 0.013810181529074162, "brier": 0.004981534821676031, "ece15": 0.0021482520070447817, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.998745294855709, "error_rate_at_confidence_0_9": 0.002512562814070352, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_message_manipulation": { "questions": 800, "accuracy": 0.6875, "mean_class_recall": 0.6966034271725826, "nll": 0.5181534061242398, "brier": 0.36342749885940506, "ece15": 0.019537157326473344, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.205, "error_rate_at_confidence_0_9": 0.012195121951219513, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_supported_attack_type_v16": { "questions": 800, "accuracy": 0.67375, "mean_class_recall": 0.5089660282717738, "nll": 1.020378053391093, "brier": 0.48493001129627705, "ece15": 0.07743873861233927, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.1725, "error_rate_at_confidence_0_9": 0.021739130434782608, "ordinal_expected_level_mae": null } }, "macro_task_accuracy": 0.9354480356736271, "macro_task_balanced_accuracy": 0.9112880126164279 } }, "test": { "raw": { "questions": 21634, "accuracy": 0.8712212258482019, "mean_class_recall": 0.8343950403877007, "nll": 0.36437325393469283, "brier": 0.1826183728339598, "ece15": 0.016790325586147077, "temperature": 1.0, "coverage_at_confidence_0_9": 0.6702412868632708, "error_rate_at_confidence_0_9": 0.01903448275862069, "ordinal_expected_level_mae": 0.005114573364023923, "tasks": { "agentdojo_tool_outputs:agentdojo_boundary_action": { "questions": 426, "accuracy": 0.9953051643192489, "mean_class_recall": 0.9895833333333333, "nll": 0.020419671491798054, "brier": 0.008500766418781964, "ece15": 0.004676929181256299, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9765258215962441, "error_rate_at_confidence_0_9": 0.002403846153846154, "ordinal_expected_level_mae": null }, "agentdojo_tool_outputs:agentdojo_embedded_injection": { "questions": 426, "accuracy": 0.9835680751173709, "mean_class_recall": 0.9635416666666667, "nll": 0.03064858581307839, "brier": 0.020119931286614087, "ece15": 0.009967851623295965, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9788732394366197, "error_rate_at_confidence_0_9": 0.004796163069544364, "ordinal_expected_level_mae": null }, "byzfl_mnist:fl_attack_generated_update": { "questions": 800, "accuracy": 0.9525, "mean_class_recall": 0.91640625, "nll": 0.17731219866650544, "brier": 0.08433190754386494, "ece15": 0.03454275208230118, "temperature": 1.0, "coverage_at_confidence_0_9": 0.95, "error_rate_at_confidence_0_9": 0.03289473684210526, "ordinal_expected_level_mae": null }, "ciciot2023:ciciot_attack": { "questions": 6000, "accuracy": 0.8995, "mean_class_recall": 0.8221111111111111, "nll": 0.30893788807797395, "brier": 0.15122747877892262, "ece15": 0.028859158451860423, "temperature": 1.0, "coverage_at_confidence_0_9": 0.737, "error_rate_at_confidence_0_9": 0.03640886476707372, "ordinal_expected_level_mae": null }, "injecagent:injecagent_requested_operation_v13": { "questions": 124, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.00019733561912440662, "brier": 1.9375590740059706e-06, "ece15": 0.0001968485399507404, "temperature": 1.0, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_known_logon_action_v16": { "questions": 540, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.0017246613457991484, "brier": 0.0007621008009601763, "ece15": 0.001458779457079616, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9981481481481481, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_observable_policy": { "questions": 540, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 1.1611401866187557e-06, "brier": 5.747739403769056e-11, "ece15": 1.1611258165194832e-06, "temperature": 1.0, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "ton_iot_network:ton_attack": { "questions": 3500, "accuracy": 0.9894285714285714, "mean_class_recall": 0.9826666666666667, "nll": 0.059799522199125266, "brier": 0.02016997152334037, "ece15": 0.00639039716386258, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9728571428571429, "error_rate_at_confidence_0_9": 0.0076358296622613805, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_evidence_level": { "questions": 659, "accuracy": 0.9984825493171472, "mean_class_recall": 0.9983249581239532, "nll": 0.0075671027444564045, "brier": 0.0038579171936140126, "ece15": 0.0029747801235328448, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9939301972685888, "error_rate_at_confidence_0_9": 0.0015267175572519084, "ordinal_expected_level_mae": 0.005114573364023923 }, "twins_streamlet:twins_policy_action": { "questions": 659, "accuracy": 0.9969650986342944, "mean_class_recall": 0.9966499162479062, "nll": 0.007401075027874373, "brier": 0.004494834346902429, "ece15": 0.0034418284412443194, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9939301972685888, "error_rate_at_confidence_0_9": 0.0015267175572519084, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_quorum": { "questions": 659, "accuracy": 0.9984825493171472, "mean_class_recall": 0.9978632478632479, "nll": 0.0026045927984607647, "brier": 0.00133772985585407, "ece15": 0.002039732462895639, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9954476479514416, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_visible_conflict": { "questions": 659, "accuracy": 0.9969650986342944, "mean_class_recall": 0.9978260869565218, "nll": 0.007434124878991116, "brier": 0.004449797406116125, "ece15": 0.002637941445288803, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9969650986342944, "error_rate_at_confidence_0_9": 0.0015220700152207, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_message_manipulation": { "questions": 3321, "accuracy": 0.6838301716350497, "mean_class_recall": 0.6880928586224566, "nll": 0.5412193961251557, "brier": 0.3797960822714433, "ece15": 0.028068617701296383, "temperature": 1.0, "coverage_at_confidence_0_9": 0.19451972297500753, "error_rate_at_confidence_0_9": 0.01393188854489164, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_supported_attack_type_v16": { "questions": 3321, "accuracy": 0.6859379704908161, "mean_class_recall": 0.4840299509564737, "nll": 1.1567283681183516, "brier": 0.4884417807744329, "ece15": 0.038297669894539775, "temperature": 1.0, "coverage_at_confidence_0_9": 0.18307738632941886, "error_rate_at_confidence_0_9": 0.0805921052631579, "ordinal_expected_level_mae": null } }, "macro_task_accuracy": 0.9414975177781386, "macro_task_balanced_accuracy": 0.9169354318963097 }, "recalibrated": { "questions": 21634, "accuracy": 0.8712212258482019, "mean_class_recall": 0.8343950403877007, "nll": 0.35395590855273523, "brier": 0.18501029425464544, "ece15": 0.030017031519889922, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.6506887307016733, "error_rate_at_confidence_0_9": 0.016054557078923067, "ordinal_expected_level_mae": 0.006011043764813851, "tasks": { "agentdojo_tool_outputs:agentdojo_boundary_action": { "questions": 426, "accuracy": 0.9953051643192489, "mean_class_recall": 0.9895833333333333, "nll": 0.023835787531761698, "brier": 0.009460218069430947, "ece15": 0.00990292779030044, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.9671361502347418, "error_rate_at_confidence_0_9": 0.0024271844660194173, "ordinal_expected_level_mae": null }, "agentdojo_tool_outputs:agentdojo_embedded_injection": { "questions": 426, "accuracy": 0.9835680751173709, "mean_class_recall": 0.9635416666666667, "nll": 0.02937486871225543, "brier": 0.01905460565156044, "ece15": 0.011296185892511, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.9694835680751174, "error_rate_at_confidence_0_9": 0.002421307506053269, "ordinal_expected_level_mae": null }, "byzfl_mnist:fl_attack_generated_update": { "questions": 800, "accuracy": 0.9525, "mean_class_recall": 0.91640625, "nll": 0.15527362489298477, "brier": 0.08190930430000586, "ece15": 0.02574101560783749, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.92625, "error_rate_at_confidence_0_9": 0.029689608636977057, "ordinal_expected_level_mae": null }, "ciciot2023:ciciot_attack": { "questions": 6000, "accuracy": 0.8995, "mean_class_recall": 0.8221111111111111, "nll": 0.2798275436837799, "brier": 0.15134523246061166, "ece15": 0.029238705666976703, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.7075, "error_rate_at_confidence_0_9": 0.0303886925795053, "ordinal_expected_level_mae": null }, "injecagent:injecagent_requested_operation_v13": { "questions": 124, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.0008001907223348217, "brier": 1.793746254505973e-05, "ece15": 0.0007956440534692383, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_known_logon_action_v16": { "questions": 540, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.0027723209709698475, "brier": 0.0009529345145823382, "ece15": 0.0024453488801810066, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.9925925925925926, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_observable_policy": { "questions": 540, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 1.8070166383123626e-05, "brier": 5.357685907682576e-09, "ece15": 1.806882660338892e-05, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "ton_iot_network:ton_attack": { "questions": 3500, "accuracy": 0.9894285714285714, "mean_class_recall": 0.9826666666666667, "nll": 0.05655522961511102, "brier": 0.021374549663926688, "ece15": 0.009425764157233412, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.95, "error_rate_at_confidence_0_9": 0.0069172932330827065, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_evidence_level": { "questions": 659, "accuracy": 0.9984825493171472, "mean_class_recall": 0.9983249581239532, "nll": 0.007520014659349455, "brier": 0.003991311156523451, "ece15": 0.00377357794140357, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.992412746585736, "error_rate_at_confidence_0_9": 0.0015290519877675841, "ordinal_expected_level_mae": 0.006011043764813851 }, "twins_streamlet:twins_policy_action": { "questions": 659, "accuracy": 0.9969650986342944, "mean_class_recall": 0.9966499162479062, "nll": 0.008102634890372476, "brier": 0.004356473026911708, "ece15": 0.004867738330078791, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.9908952959028832, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_quorum": { "questions": 659, "accuracy": 0.9984825493171472, "mean_class_recall": 0.9978632478632479, "nll": 0.0038576227479580737, "brier": 0.0014748958385226618, "ece15": 0.003282715492202558, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.9939301972685888, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_visible_conflict": { "questions": 659, "accuracy": 0.9969650986342944, "mean_class_recall": 0.9978260869565218, "nll": 0.006788751578147029, "brier": 0.004171759480970426, "ece15": 0.003177496987382077, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.9939301972685888, "error_rate_at_confidence_0_9": 0.0015267175572519084, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_message_manipulation": { "questions": 3321, "accuracy": 0.6838301716350497, "mean_class_recall": 0.6880928586224566, "nll": 0.5429719234394278, "brier": 0.3809937714731844, "ece15": 0.02889956013524373, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.17765733212887685, "error_rate_at_confidence_0_9": 0.010169491525423728, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_supported_attack_type_v16": { "questions": 3321, "accuracy": 0.6859379704908161, "mean_class_recall": 0.4840299509564737, "nll": 1.1477144959529286, "brier": 0.5019380206266735, "ece15": 0.11401240417222197, "temperature": 1.2873604198505975, "coverage_at_confidence_0_9": 0.16079494128274616, "error_rate_at_confidence_0_9": 0.07865168539325842, "ordinal_expected_level_mae": null } }, "macro_task_accuracy": 0.9414975177781386, "macro_task_balanced_accuracy": 0.9169354318963097 } }, "general": { "decision-v7": { "raw_clean": { "n": 1200, "nll": 0.5237394417093835, "acc": 0.865, "ece": 0.08203125833030098, "brier": 0.21310411533941315, "mean_conf": 0.9455011120677191, "confident_error_rate": 0.06, "coverage_at_0_9": 0.8308333333333333, "accuracy_at_0_9": 0.9277833500501504, "coverage_at_5pct_error": 0.7791666666666667, "coverage_at_1pct_error": 0.6158333333333333, "aurc": 0.028219153802824883, "error_rate_at_0_9": 0.07221664994984955, "confidence_bias": 0.08050111206771915, "top_bins": { "0.9": { "n": 997, "errors": 72, "error_rate": 0.07221664994984955 }, "0.95": { "n": 937, "errors": 47, "error_rate": 0.05016008537886873 }, "0.99": { "n": 861, "errors": 27, "error_rate": 0.0313588850174216 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9933333333333333, "confidence_cutoff": 0.9999626357556778 }, "0.8": { "coverage": 0.8, "accuracy": 0.9458333333333333, "confidence_cutoff": 0.9342652506942892 } }, "score_mae": 0.4647882331825802, "ranked_probability_score": 0.08637141204655961 }, "as_shipped_clean": { "n": 1200, "nll": 0.44174931033716425, "acc": 0.865, "ece": 0.0702397073006123, "brier": 0.20431799318642196, "mean_conf": 0.930435938959295, "confident_error_rate": 0.04, "coverage_at_0_9": 0.7858333333333334, "accuracy_at_0_9": 0.9490986214209968, "coverage_at_5pct_error": 0.7783333333333333, "coverage_at_1pct_error": 0.6125, "aurc": 0.028152549316370398, "error_rate_at_0_9": 0.05090137857900318, "confidence_bias": 0.065435938959295, "top_bins": { "0.9": { "n": 943, "errors": 48, "error_rate": 0.05090137857900318 }, "0.95": { "n": 893, "errors": 35, "error_rate": 0.03919372900335946 }, "0.99": { "n": 803, "errors": 15, "error_rate": 0.0186799501867995 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9916666666666667, "confidence_cutoff": 0.9996192297027195 }, "0.8": { "coverage": 0.8, "accuracy": 0.9447916666666667, "confidence_cutoff": 0.8818964099133315 } }, "score_mae": 0.47431418086986715, "ranked_probability_score": 0.08319761613591657 }, "sources": { "agnews": { "n": 240, "nll": 0.2796999395688286, "acc": 0.9125, "ece": 0.05892681778016237, "brier": 0.1350754514620047, "mean_conf": 0.9714268177801624, "confident_error_rate": 0.04583333333333333, "coverage_at_0_9": 0.9208333333333333, "accuracy_at_0_9": 0.9502262443438914, "coverage_at_5pct_error": 0.9333333333333333, "coverage_at_1pct_error": 0.75, "aurc": 0.011330835078134182, "error_rate_at_0_9": 0.049773755656108594, "confidence_bias": 0.0589268177801624, "top_bins": { "0.9": { "n": 221, "errors": 11, "error_rate": 0.049773755656108594 }, "0.95": { "n": 214, "errors": 9, "error_rate": 0.04205607476635514 }, "0.99": { "n": 189, "errors": 3, "error_rate": 0.015873015873015872 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9994019817408221 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.9890606274392002 } } }, "amazon": { "n": 80, "nll": 1.1725112640711228, "acc": 0.6, "ece": 0.14900986362097515, "brier": 0.5611978409181134, "mean_conf": 0.7279331238481536, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.2125, "accuracy_at_0_9": 0.8235294117647058, "coverage_at_5pct_error": 0.075, "coverage_at_1pct_error": 0.075, "aurc": 0.268674359040746, "error_rate_at_0_9": 0.17647058823529413, "confidence_bias": 0.12793312384815358, "top_bins": { "0.9": { "n": 17, "errors": 3, "error_rate": 0.17647058823529413 }, "0.95": { "n": 12, "errors": 2, "error_rate": 0.16666666666666666 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.725, "confidence_cutoff": 0.7565342654626949 }, "0.8": { "coverage": 0.8, "accuracy": 0.640625, "confidence_cutoff": 0.5366925117472928 } }, "score_mae": 0.5012771753189752, "ranked_probability_score": 0.0890286871906647 }, "banking77": { "n": 80, "nll": 0.5995652320511431, "acc": 0.85, "ece": 0.08817712605098262, "brier": 0.24932641415691287, "mean_conf": 0.8916570456057876, "confident_error_rate": 0.05, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 0.9333333333333333, "coverage_at_5pct_error": 0.6875, "coverage_at_1pct_error": 0.6125, "aurc": 0.03525784971074972, "error_rate_at_0_9": 0.06666666666666667, "confidence_bias": 0.041657045605787646, "top_bins": { "0.9": { "n": 60, "errors": 4, "error_rate": 0.06666666666666667 }, "0.95": { "n": 55, "errors": 2, "error_rate": 0.03636363636363636 }, "0.99": { "n": 46, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9933430562867247 }, "0.8": { "coverage": 0.8, "accuracy": 0.90625, "confidence_cutoff": 0.8362446757691954 } } }, "boolq": { "n": 80, "nll": 0.5712250447804231, "acc": 0.875, "ece": 0.11692367054964475, "brier": 0.22310665871912266, "mean_conf": 0.948749077509772, "confident_error_rate": 0.0875, "coverage_at_0_9": 0.85, "accuracy_at_0_9": 0.8970588235294118, "coverage_at_5pct_error": 0.6875, "coverage_at_1pct_error": 0.1, "aurc": 0.05278244760094704, "error_rate_at_0_9": 0.10294117647058823, "confidence_bias": 0.073749077509772, "top_bins": { "0.9": { "n": 68, "errors": 7, "error_rate": 0.10294117647058823 }, "0.95": { "n": 66, "errors": 6, "error_rate": 0.09090909090909091 }, "0.99": { "n": 60, "errors": 4, "error_rate": 0.06666666666666667 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9998615919766226 }, "0.8": { "coverage": 0.8, "accuracy": 0.921875, "confidence_cutoff": 0.979302462267494 } } }, "contrastive": { "n": 160, "nll": 0.06554428006012027, "acc": 0.96875, "ece": 0.024615155693287222, "brier": 0.04485687620328682, "mean_conf": 0.9933651556932872, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.98125, "accuracy_at_0_9": 0.9872611464968153, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.975, "aurc": 0.000671805023667049, "error_rate_at_0_9": 0.012738853503184714, "confidence_bias": 0.024615155693287205, "top_bins": { "0.9": { "n": 157, "errors": 2, "error_rate": 0.012738853503184714 }, "0.95": { "n": 154, "errors": 1, "error_rate": 0.006493506493506494 }, "0.99": { "n": 148, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999882930499685 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9997675703852573 } } }, "dbpedia14": { "n": 80, "nll": 0.01926796153789536, "acc": 0.9875, "ece": 0.011278233290686156, "brier": 0.014217088541729535, "mean_conf": 0.9949340220579689, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9875, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.9875, "aurc": 0.00015625, "error_rate_at_0_9": 0.0, "confidence_bias": 0.0074340220579688365, "top_bins": { "0.9": { "n": 79, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 77, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 76, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999950740905034 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9999745354541041 } } }, "imdb": { "n": 80, "nll": 0.06446545489140978, "acc": 0.9625, "ece": 0.025460341159037583, "brier": 0.045829990154110004, "mean_conf": 0.9836424817249945, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9375, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.95, "aurc": 0.0012680561790530144, "error_rate_at_0_9": 0.0, "confidence_bias": 0.021142481724994444, "top_bins": { "0.9": { "n": 75, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 75, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 71, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999788743849367 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9996364092230372 } } }, "mnli": { "n": 80, "nll": 0.46179093448986874, "acc": 0.9125, "ece": 0.07967308878997346, "brier": 0.16221252791471635, "mean_conf": 0.985103225410972, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.925, "accuracy_at_0_9": 0.9324324324324325, "coverage_at_5pct_error": 0.8875, "coverage_at_1pct_error": 0.65, "aurc": 0.015873702018260973, "error_rate_at_0_9": 0.06756756756756757, "confidence_bias": 0.07260322541097197, "top_bins": { "0.9": { "n": 74, "errors": 5, "error_rate": 0.06756756756756757 }, "0.95": { "n": 74, "errors": 5, "error_rate": 0.06756756756756757 }, "0.99": { "n": 71, "errors": 3, "error_rate": 0.04225352112676056 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.999987939363811 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.9997555402093751 } } }, "sst5": { "n": 80, "nll": 1.1930808545317393, "acc": 0.6125, "ece": 0.23848723097529123, "brier": 0.5810003072599018, "mean_conf": 0.7987986774228062, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.2625, "accuracy_at_0_9": 0.7619047619047619, "coverage_at_5pct_error": 0.0875, "coverage_at_1pct_error": 0.0875, "aurc": 0.2998766100005863, "error_rate_at_0_9": 0.23809523809523808, "confidence_bias": 0.18629867742280615, "top_bins": { "0.9": { "n": 21, "errors": 5, "error_rate": 0.23809523809523808 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.625, "confidence_cutoff": 0.8363874325149153 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.6881083881339022 } }, "score_mae": 0.4905379704011906, "ranked_probability_score": 0.08831634129641223 }, "trec": { "n": 80, "nll": 0.268396899685072, "acc": 0.9625, "ece": 0.02818577556782262, "brier": 0.058300946365123306, "mean_conf": 0.9906857755678227, "confident_error_rate": 0.025, "coverage_at_0_9": 0.9875, "accuracy_at_0_9": 0.9746835443037974, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.325, "aurc": 0.015881097479045377, "error_rate_at_0_9": 0.02531645569620253, "confidence_bias": 0.028185775567822646, "top_bins": { "0.9": { "n": 79, "errors": 2, "error_rate": 0.02531645569620253 }, "0.95": { "n": 78, "errors": 2, "error_rate": 0.02564102564102564 }, "0.99": { "n": 77, "errors": 2, "error_rate": 0.025974025974025976 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9999834449407883 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.9998382341930544 } } }, "yelp": { "n": 160, "nll": 0.6528738150960317, "acc": 0.76875, "ece": 0.10032078827768093, "brier": 0.3373190084870059, "mean_conf": 0.867012445257043, "confident_error_rate": 0.05625, "coverage_at_0_9": 0.575, "accuracy_at_0_9": 0.9021739130434783, "coverage_at_5pct_error": 0.41875, "coverage_at_1pct_error": 0.18125, "aurc": 0.08667681796341256, "error_rate_at_0_9": 0.09782608695652174, "confidence_bias": 0.09826244525704297, "top_bins": { "0.9": { "n": 92, "errors": 9, "error_rate": 0.09782608695652174 }, "0.95": { "n": 83, "errors": 8, "error_rate": 0.0963855421686747 }, "0.99": { "n": 63, "errors": 3, "error_rate": 0.047619047619047616 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.9656164925493048 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.7039733738891296 } }, "score_mae": 0.4311273968894357, "ranked_probability_score": 0.07224781992067277 } }, "types": { "choice": { "n": 480, "nll": 0.3037822176153947, "acc": 0.93125, "ece": 0.050099480347752155, "brier": 0.11775456449644288, "mean_conf": 0.9718507476012318, "confident_error_rate": 0.035416666666666666, "coverage_at_0_9": 0.9270833333333334, "accuracy_at_0_9": 0.9617977528089887, "coverage_at_5pct_error": 0.94375, "coverage_at_1pct_error": 0.725, "aurc": 0.010673395731944459, "error_rate_at_0_9": 0.038202247191011236, "confidence_bias": 0.04060074760123178, "top_bins": { "0.9": { "n": 445, "errors": 17, "error_rate": 0.038202247191011236 }, "0.95": { "n": 435, "errors": 15, "error_rate": 0.034482758620689655 }, "0.99": { "n": 408, "errors": 6, "error_rate": 0.014705882352941176 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9958333333333333, "confidence_cutoff": 0.9999737433376136 }, "0.8": { "coverage": 0.8, "accuracy": 0.9895833333333334, "confidence_cutoff": 0.9984835061671959 } } }, "noul": { "n": 480, "nll": 0.24523830247044484, "acc": 0.9229166666666667, "ece": 0.04796953512867931, "brier": 0.11605074761778626, "mean_conf": 0.970886201795346, "confident_error_rate": 0.035416666666666666, "coverage_at_0_9": 0.9104166666666667, "accuracy_at_0_9": 0.9610983981693364, "coverage_at_5pct_error": 0.94375, "coverage_at_1pct_error": 0.71875, "aurc": 0.01472209854520942, "error_rate_at_0_9": 0.038901601830663615, "confidence_bias": 0.0479695351286793, "top_bins": { "0.9": { "n": 437, "errors": 17, "error_rate": 0.038901601830663615 }, "0.95": { "n": 424, "errors": 13, "error_rate": 0.030660377358490566 }, "0.99": { "n": 389, "errors": 8, "error_rate": 0.02056555269922879 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9958333333333333, "confidence_cutoff": 0.9997753303513292 }, "0.8": { "coverage": 0.8, "accuracy": 0.9791666666666666, "confidence_cutoff": 0.9930519257681523 } } }, "score": { "n": 240, "nll": 1.110705511514142, "acc": 0.6166666666666667, "ece": 0.16184498803254416, "brier": 0.5539793417036516, "mean_conf": 0.7667057960033195, "confident_error_rate": 0.058333333333333334, "coverage_at_0_9": 0.25416666666666665, "accuracy_at_0_9": 0.7704918032786885, "coverage_at_5pct_error": 0.016666666666666666, "coverage_at_1pct_error": 0.016666666666666666, "aurc": 0.28057303182426463, "error_rate_at_0_9": 0.22950819672131148, "confidence_bias": 0.15003912933665275, "top_bins": { "0.9": { "n": 61, "errors": 14, "error_rate": 0.22950819672131148 }, "0.95": { "n": 34, "errors": 7, "error_rate": 0.20588235294117646 }, "0.99": { "n": 6, "errors": 1, "error_rate": 0.16666666666666666 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6916666666666667, "confidence_cutoff": 0.8026983558729256 }, "0.8": { "coverage": 0.8, "accuracy": 0.6666666666666666, "confidence_cutoff": 0.6054379586272015 } }, "score_mae": 0.47431418086986715, "ranked_probability_score": 0.08319761613591657 } } }, "transfer-v4": { "raw_clean": { "n": 656, "nll": 0.7491303449341485, "acc": 0.836890243902439, "ece": 0.10484780065311013, "brier": 0.2706697225616099, "mean_conf": 0.9369188798045006, "confident_error_rate": 0.08689024390243902, "coverage_at_0_9": 0.8277439024390244, "accuracy_at_0_9": 0.8950276243093923, "coverage_at_5pct_error": 0.5701219512195121, "coverage_at_1pct_error": 0.3567073170731707, "aurc": 0.048474736987579994, "error_rate_at_0_9": 0.10497237569060773, "confidence_bias": 0.10002863590206157, "top_bins": { "0.9": { "n": 543, "errors": 57, "error_rate": 0.10497237569060773 }, "0.95": { "n": 512, "errors": 48, "error_rate": 0.09375 }, "0.99": { "n": 455, "errors": 30, "error_rate": 0.06593406593406594 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9634146341463414, "confidence_cutoff": 0.9997756567114997 }, "0.8": { "coverage": 0.8003048780487805, "accuracy": 0.8990476190476191, "confidence_cutoff": 0.9312810361300458 } }, "score_mae": 0.2771201607516616, "ranked_probability_score": 0.12002374674647091 }, "as_shipped_clean": { "n": 656, "nll": 0.6222520301084513, "acc": 0.836890243902439, "ece": 0.09048309685573616, "brier": 0.2622492394102412, "mean_conf": 0.9203846154028757, "confident_error_rate": 0.07317073170731707, "coverage_at_0_9": 0.7820121951219512, "accuracy_at_0_9": 0.9064327485380117, "coverage_at_5pct_error": 0.5670731707317073, "coverage_at_1pct_error": 0.3551829268292683, "aurc": 0.0484684761006514, "error_rate_at_0_9": 0.0935672514619883, "confidence_bias": 0.0834943715004367, "top_bins": { "0.9": { "n": 513, "errors": 48, "error_rate": 0.0935672514619883 }, "0.95": { "n": 477, "errors": 37, "error_rate": 0.07756813417190776 }, "0.99": { "n": 405, "errors": 23, "error_rate": 0.056790123456790124 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9634146341463414, "confidence_cutoff": 0.9985297137731126 }, "0.8": { "coverage": 0.8003048780487805, "accuracy": 0.900952380952381, "confidence_cutoff": 0.8749540187432164 } }, "score_mae": 0.2768980570793622, "ranked_probability_score": 0.11577841556824224 }, "sources": { "composition_holdout": { "n": 96, "nll": 0.8314301766731748, "acc": 0.8645833333333334, "ece": 0.12259156340261834, "brier": 0.25260603356221595, "mean_conf": 0.9783762896019588, "confident_error_rate": 0.11458333333333333, "coverage_at_0_9": 0.9479166666666666, "accuracy_at_0_9": 0.8791208791208791, "coverage_at_5pct_error": 0.7291666666666666, "coverage_at_1pct_error": 0.6666666666666666, "aurc": 0.029361211962366768, "error_rate_at_0_9": 0.12087912087912088, "confidence_bias": 0.1137929562686254, "top_bins": { "0.9": { "n": 91, "errors": 11, "error_rate": 0.12087912087912088 }, "0.95": { "n": 90, "errors": 11, "error_rate": 0.12222222222222222 }, "0.99": { "n": 86, "errors": 11, "error_rate": 0.12790697674418605 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.99991410687893 }, "0.8": { "coverage": 0.8020833333333334, "accuracy": 0.948051948051948, "confidence_cutoff": 0.9986476483509216 } } }, "contrastive": { "n": 80, "nll": 0.46595606789147936, "acc": 0.85, "ece": 0.11588755121828759, "brier": 0.22911401994889413, "mean_conf": 0.9658875512182877, "confident_error_rate": 0.0875, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.9027777777777778, "coverage_at_5pct_error": 0.85, "coverage_at_1pct_error": 0.8, "aurc": 0.017400442212878862, "error_rate_at_0_9": 0.09722222222222222, "confidence_bias": 0.11588755121828775, "top_bins": { "0.9": { "n": 72, "errors": 7, "error_rate": 0.09722222222222222 }, "0.95": { "n": 70, "errors": 5, "error_rate": 0.07142857142857142 }, "0.99": { "n": 68, "errors": 3, "error_rate": 0.04411764705882353 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999823513518187 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9993279314433398 } }, "score_mae": 0.2768980570793622, "ranked_probability_score": 0.11577841556824224 }, "emotion": { "n": 80, "nll": 1.1680421729742996, "acc": 0.6625, "ece": 0.15215129647550107, "brier": 0.49482559482198313, "mean_conf": 0.767505586427657, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.425, "accuracy_at_0_9": 0.8529411764705882, "coverage_at_5pct_error": 0.075, "coverage_at_1pct_error": 0.075, "aurc": 0.18480906195062094, "error_rate_at_0_9": 0.14705882352941177, "confidence_bias": 0.105005586427657, "top_bins": { "0.9": { "n": 34, "errors": 5, "error_rate": 0.14705882352941177 }, "0.95": { "n": 26, "errors": 3, "error_rate": 0.11538461538461539 }, "0.99": { "n": 8, "errors": 1, "error_rate": 0.125 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8315422519453946 }, "0.8": { "coverage": 0.8, "accuracy": 0.734375, "confidence_cutoff": 0.5608119629701582 } } }, "mmlu": { "n": 80, "nll": 0.8706892848634225, "acc": 0.725, "ece": 0.14396914879993838, "brier": 0.42715412327538493, "mean_conf": 0.8115029447218166, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.475, "accuracy_at_0_9": 0.868421052631579, "coverage_at_5pct_error": 0.375, "coverage_at_1pct_error": 0.3375, "aurc": 0.11356864595903862, "error_rate_at_0_9": 0.13157894736842105, "confidence_bias": 0.08650294472181663, "top_bins": { "0.9": { "n": 38, "errors": 5, "error_rate": 0.13157894736842105 }, "0.95": { "n": 28, "errors": 1, "error_rate": 0.03571428571428571 }, "0.99": { "n": 18, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.8706443942318668 }, "0.8": { "coverage": 0.8, "accuracy": 0.796875, "confidence_cutoff": 0.5685435872401945 } } }, "paws": { "n": 80, "nll": 0.8953763773747229, "acc": 0.8, "ece": 0.16414062885679848, "brier": 0.31529713556607303, "mean_conf": 0.947798906827078, "confident_error_rate": 0.125, "coverage_at_0_9": 0.8625, "accuracy_at_0_9": 0.855072463768116, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.15, "aurc": 0.09544667963478717, "error_rate_at_0_9": 0.14492753623188406, "confidence_bias": 0.14779890682707797, "top_bins": { "0.9": { "n": 69, "errors": 10, "error_rate": 0.14492753623188406 }, "0.95": { "n": 69, "errors": 10, "error_rate": 0.14492753623188406 }, "0.99": { "n": 60, "errors": 6, "error_rate": 0.1 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.875, "confidence_cutoff": 0.9995333943902521 }, "0.8": { "coverage": 0.8, "accuracy": 0.890625, "confidence_cutoff": 0.9804900927788639 } } }, "qnli": { "n": 80, "nll": 0.2866833743812309, "acc": 0.925, "ece": 0.07720126969420893, "brier": 0.1353782597318704, "mean_conf": 0.969044048844119, "confident_error_rate": 0.05, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.9444444444444444, "coverage_at_5pct_error": 0.8625, "coverage_at_1pct_error": 0.6125, "aurc": 0.01474092888988553, "error_rate_at_0_9": 0.05555555555555555, "confidence_bias": 0.044044048844118944, "top_bins": { "0.9": { "n": 72, "errors": 4, "error_rate": 0.05555555555555555 }, "0.95": { "n": 69, "errors": 3, "error_rate": 0.043478260869565216 }, "0.99": { "n": 64, "errors": 2, "error_rate": 0.03125 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9994500997969659 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.9906893719466282 } } }, "sciq": { "n": 80, "nll": 0.007797247790721174, "acc": 1.0, "ece": 0.007304169605922202, "brier": 0.00159066044034253, "mean_conf": 0.9926958303940776, "confident_error_rate": 0.0, "coverage_at_0_9": 0.975, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.007304169605922395, "top_bins": { "0.9": { "n": 78, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 76, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 71, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999411463988364 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.999496345294104 } } }, "tweet_offensive": { "n": 80, "nll": 0.4102059096056144, "acc": 0.8625, "ece": 0.082947906119192, "brier": 0.24395672910477048, "mean_conf": 0.9186674303481943, "confident_error_rate": 0.075, "coverage_at_0_9": 0.7375, "accuracy_at_0_9": 0.8983050847457628, "coverage_at_5pct_error": 0.575, "coverage_at_1pct_error": 0.5125, "aurc": 0.05023960594359948, "error_rate_at_0_9": 0.1016949152542373, "confidence_bias": 0.05616743034819427, "top_bins": { "0.9": { "n": 59, "errors": 6, "error_rate": 0.1016949152542373 }, "0.95": { "n": 49, "errors": 4, "error_rate": 0.08163265306122448 }, "0.99": { "n": 30, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9765148863831499 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.8834322098576286 } } } }, "types": { "choice": { "n": 336, "nll": 0.724820218484822, "acc": 0.8154761904761905, "ece": 0.09541834159449251, "brier": 0.29207086162199275, "mean_conf": 0.8918466450156909, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.7172619047619048, "accuracy_at_0_9": 0.9128630705394191, "coverage_at_5pct_error": 0.48214285714285715, "coverage_at_1pct_error": 0.36904761904761907, "aurc": 0.05557065380684749, "error_rate_at_0_9": 0.08713692946058091, "confidence_bias": 0.07637045453950042, "top_bins": { "0.9": { "n": 241, "errors": 21, "error_rate": 0.08713692946058091 }, "0.95": { "n": 220, "errors": 15, "error_rate": 0.06818181818181818 }, "0.99": { "n": 183, "errors": 12, "error_rate": 0.06557377049180328 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9464285714285714, "confidence_cutoff": 0.9967131547386102 }, "0.8": { "coverage": 0.8005952380952381, "accuracy": 0.8921933085501859, "confidence_cutoff": 0.7705104187347486 } } }, "noul": { "n": 280, "nll": 0.4549349162962509, "acc": 0.8821428571428571, "ece": 0.08452238489264785, "brier": 0.19846632133861988, "mean_conf": 0.9530010972612158, "confident_error_rate": 0.07142857142857142, "coverage_at_0_9": 0.8571428571428571, "accuracy_at_0_9": 0.9166666666666666, "coverage_at_5pct_error": 0.7607142857142857, "coverage_at_1pct_error": 0.2571428571428571, "aurc": 0.03904590143532812, "error_rate_at_0_9": 0.08333333333333333, "confidence_bias": 0.07085824011835873, "top_bins": { "0.9": { "n": 240, "errors": 20, "error_rate": 0.08333333333333333 }, "0.95": { "n": 227, "errors": 17, "error_rate": 0.07488986784140969 }, "0.99": { "n": 194, "errors": 8, "error_rate": 0.041237113402061855 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9571428571428572, "confidence_cutoff": 0.9989211416259292 }, "0.8": { "coverage": 0.8, "accuracy": 0.9330357142857143, "confidence_cutoff": 0.95900292163327 } } }, "score": { "n": 40, "nll": 0.9318990444323394, "acc": 0.7, "ece": 0.2317881936468471, "brier": 0.45822803933287715, "mean_conf": 0.9317881936468471, "confident_error_rate": 0.175, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 0.78125, "coverage_at_5pct_error": 0.65, "coverage_at_1pct_error": 0.6, "aurc": 0.07545101908306909, "error_rate_at_0_9": 0.21875, "confidence_bias": 0.2317881936468471, "top_bins": { "0.9": { "n": 32, "errors": 7, "error_rate": 0.21875 }, "0.95": { "n": 30, "errors": 5, "error_rate": 0.16666666666666666 }, "0.99": { "n": 28, "errors": 3, "error_rate": 0.10714285714285714 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9997762256879271 }, "0.8": { "coverage": 0.8, "accuracy": 0.78125, "confidence_cutoff": 0.9360940664483365 } }, "score_mae": 0.2768980570793622, "ranked_probability_score": 0.11577841556824224 } } } }, "temperature": 1.2873604198505975 }, { "epoch": 3, "development": { "raw": { "questions": 14912, "accuracy": 0.9075912017167382, "mean_class_recall": 0.8447857522159936, "nll": 0.24807737680415345, "brier": 0.12998082994334342, "ece15": 0.027028147820359012, "temperature": 1.0, "coverage_at_confidence_0_9": 0.8103540772532188, "error_rate_at_confidence_0_9": 0.023502151605428664, "ordinal_expected_level_mae": 0.00481932257817722, "tasks": { "agentdojo_tool_outputs:agentdojo_boundary_action": { "questions": 222, "accuracy": 0.9774774774774775, "mean_class_recall": 0.9570930232558139, "nll": 0.10407641066616782, "brier": 0.039347876403718875, "ece15": 0.021432863016197557, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9864864864864865, "error_rate_at_confidence_0_9": 0.0136986301369863, "ordinal_expected_level_mae": null }, "agentdojo_tool_outputs:agentdojo_embedded_injection": { "questions": 222, "accuracy": 0.9819819819819819, "mean_class_recall": 0.96, "nll": 0.10023555924012383, "brier": 0.031563944714978254, "ece15": 0.015911253108198964, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9864864864864865, "error_rate_at_confidence_0_9": 0.0136986301369863, "ordinal_expected_level_mae": null }, "byzfl_mnist:fl_attack_generated_update": { "questions": 1400, "accuracy": 0.9885714285714285, "mean_class_recall": 0.9821428571428572, "nll": 0.0442702602822601, "brier": 0.02139513973607548, "ece15": 0.007710400104216533, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9842857142857143, "error_rate_at_confidence_0_9": 0.008708272859216255, "ordinal_expected_level_mae": null }, "ciciot2023:ciciot_attack": { "questions": 5000, "accuracy": 0.8418, "mean_class_recall": 0.7537333333333334, "nll": 0.4173917752941542, "brier": 0.22451704505631584, "ece15": 0.06284084146027137, "temperature": 1.0, "coverage_at_confidence_0_9": 0.6928, "error_rate_at_confidence_0_9": 0.06293302540415704, "ordinal_expected_level_mae": null }, "injecagent:injecagent_requested_operation_v13": { "questions": 124, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.0014205610073774013, "brier": 0.0002211290260091384, "ece15": 0.0013613576932818498, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9919354838709677, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_known_logon_action_v16": { "questions": 328, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.000610657881540054, "brier": 8.099314527924365e-05, "ece15": 0.0005891478461821947, "temperature": 1.0, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_observable_policy": { "questions": 328, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 1.5480302068279906e-07, "brier": 4.4217021643060835e-12, "ece15": 1.548019148098009e-07, "temperature": 1.0, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "ton_iot_network:ton_attack": { "questions": 2500, "accuracy": 0.9792, "mean_class_recall": 0.9818666666666667, "nll": 0.11218220058914397, "brier": 0.03724045269615108, "ece15": 0.01842697861877182, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9928, "error_rate_at_confidence_0_9": 0.015310233682514102, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_evidence_level": { "questions": 797, "accuracy": 0.9962358845671268, "mean_class_recall": 0.9963898916967509, "nll": 0.012217254972570101, "brier": 0.006818829719414822, "ece15": 0.004133398699556409, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9949811794228356, "error_rate_at_confidence_0_9": 0.0012610340479192938, "ordinal_expected_level_mae": 0.00481932257817722 }, "twins_streamlet:twins_policy_action": { "questions": 797, "accuracy": 0.9949811794228356, "mean_class_recall": 0.9951865222623345, "nll": 0.013078119892699127, "brier": 0.007721112955228842, "ece15": 0.003423998250058524, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9899623588456713, "error_rate_at_confidence_0_9": 0.0012674271229404308, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_quorum": { "questions": 797, "accuracy": 0.9949811794228356, "mean_class_recall": 0.994276682635671, "nll": 0.01027058602627885, "brier": 0.007209131739335666, "ece15": 0.004201163796495765, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9912170639899623, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_visible_conflict": { "questions": 797, "accuracy": 0.9974905897114178, "mean_class_recall": 0.998236331569665, "nll": 0.017287182530912663, "brier": 0.004953169208636321, "ece15": 0.0025266183132433223, "temperature": 1.0, "coverage_at_confidence_0_9": 0.998745294855709, "error_rate_at_confidence_0_9": 0.002512562814070352, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_message_manipulation": { "questions": 800, "accuracy": 0.69, "mean_class_recall": 0.7116483068135455, "nll": 0.5092641106435474, "brier": 0.3569714849542106, "ece15": 0.03650030376694629, "temperature": 1.0, "coverage_at_confidence_0_9": 0.2425, "error_rate_at_confidence_0_9": 0.010309278350515464, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_supported_attack_type_v16": { "questions": 800, "accuracy": 0.68875, "mean_class_recall": 0.529392069171139, "nll": 0.96833512912194, "brier": 0.46247412161427887, "ece15": 0.02731423542286039, "temperature": 1.0, "coverage_at_confidence_0_9": 0.22625, "error_rate_at_confidence_0_9": 0.022099447513812154, "ordinal_expected_level_mae": null } }, "macro_task_accuracy": 0.9379621229396502, "macro_task_balanced_accuracy": 0.9185689774676985 }, "recalibrated": { "questions": 14912, "accuracy": 0.9075912017167382, "mean_class_recall": 0.8447857522159936, "nll": 0.2269545932235271, "brier": 0.12792407717450321, "ece15": 0.01858051585491712, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.7862124463519313, "error_rate_at_confidence_0_9": 0.018679631525076765, "ordinal_expected_level_mae": 0.005329911013904147, "tasks": { "agentdojo_tool_outputs:agentdojo_boundary_action": { "questions": 222, "accuracy": 0.9774774774774775, "mean_class_recall": 0.9570930232558139, "nll": 0.08092683597443519, "brier": 0.037546344393533104, "ece15": 0.021320738677048364, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.9819819819819819, "error_rate_at_confidence_0_9": 0.013761467889908258, "ordinal_expected_level_mae": null }, "agentdojo_tool_outputs:agentdojo_embedded_injection": { "questions": 222, "accuracy": 0.9819819819819819, "mean_class_recall": 0.96, "nll": 0.07809919245507242, "brier": 0.030623326048867618, "ece15": 0.015019163882298042, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.9819819819819819, "error_rate_at_confidence_0_9": 0.009174311926605505, "ordinal_expected_level_mae": null }, "byzfl_mnist:fl_attack_generated_update": { "questions": 1400, "accuracy": 0.9885714285714285, "mean_class_recall": 0.9821428571428572, "nll": 0.04094699270351949, "brier": 0.021018406686723334, "ece15": 0.0038434318338614677, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.9678571428571429, "error_rate_at_confidence_0_9": 0.006642066420664207, "ordinal_expected_level_mae": null }, "ciciot2023:ciciot_attack": { "questions": 5000, "accuracy": 0.8418, "mean_class_recall": 0.7537333333333334, "nll": 0.36679030702727994, "brier": 0.21757064336558796, "ece15": 0.043357678681868704, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.6394, "error_rate_at_confidence_0_9": 0.05004691898654989, "ordinal_expected_level_mae": null }, "injecagent:injecagent_requested_operation_v13": { "questions": 124, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.0028701157894541487, "brier": 0.0006216354903482641, "ece15": 0.0026971564840139535, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.9919354838709677, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_known_logon_action_v16": { "questions": 328, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.001405432942745028, "brier": 0.00025083300955486444, "ece15": 0.0013358908781109478, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.9939024390243902, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_observable_policy": { "questions": 328, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 4.6272880953518815e-06, "brier": 9.796066038293659e-10, "ece15": 4.627043141135246e-06, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "ton_iot_network:ton_attack": { "questions": 2500, "accuracy": 0.9792, "mean_class_recall": 0.9818666666666667, "nll": 0.0877031817842779, "brier": 0.03599036114227489, "ece15": 0.01538869285344686, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.988, "error_rate_at_confidence_0_9": 0.014574898785425101, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_evidence_level": { "questions": 797, "accuracy": 0.9962358845671268, "mean_class_recall": 0.9963898916967509, "nll": 0.010603240403009088, "brier": 0.006373682001028304, "ece15": 0.004218405852177047, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.9937264742785445, "error_rate_at_confidence_0_9": 0.0012626262626262627, "ordinal_expected_level_mae": 0.005329911013904147 }, "twins_streamlet:twins_policy_action": { "questions": 797, "accuracy": 0.9949811794228356, "mean_class_recall": 0.9951865222623345, "nll": 0.01230072195659841, "brier": 0.007337903591834537, "ece15": 0.0031241457310850957, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.986198243412798, "error_rate_at_confidence_0_9": 0.001272264631043257, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_quorum": { "questions": 797, "accuracy": 0.9949811794228356, "mean_class_recall": 0.994276682635671, "nll": 0.009584146250019401, "brier": 0.00659904677807738, "ece15": 0.005028933100804552, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.9899623588456713, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_visible_conflict": { "questions": 797, "accuracy": 0.9974905897114178, "mean_class_recall": 0.998236331569665, "nll": 0.013437428285294059, "brier": 0.004858731838764845, "ece15": 0.0022785888551786117, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.998745294855709, "error_rate_at_confidence_0_9": 0.002512562814070352, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_message_manipulation": { "questions": 800, "accuracy": 0.69, "mean_class_recall": 0.7116483068135455, "nll": 0.5102124840713312, "brier": 0.35753050242002155, "ece15": 0.03811996421246104, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.22125, "error_rate_at_confidence_0_9": 0.011299435028248588, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_supported_attack_type_v16": { "questions": 800, "accuracy": 0.68875, "mean_class_recall": 0.529392069171139, "nll": 0.9911460027249259, "brier": 0.4737144267281807, "ece15": 0.09647043512079573, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.18625, "error_rate_at_confidence_0_9": 0.020134228187919462, "ordinal_expected_level_mae": null } }, "macro_task_accuracy": 0.9379621229396502, "macro_task_balanced_accuracy": 0.9185689774676985 } }, "test": { "raw": { "questions": 21634, "accuracy": 0.8801423684940372, "mean_class_recall": 0.8422636415920437, "nll": 0.35756736839135106, "brier": 0.17329397786588344, "ece15": 0.018370944185424988, "temperature": 1.0, "coverage_at_confidence_0_9": 0.681750947582509, "error_rate_at_confidence_0_9": 0.017289307749677945, "ordinal_expected_level_mae": 0.004830545729071066, "tasks": { "agentdojo_tool_outputs:agentdojo_boundary_action": { "questions": 426, "accuracy": 0.9953051643192489, "mean_class_recall": 0.9895833333333333, "nll": 0.019578322598972707, "brier": 0.008470972915844083, "ece15": 0.0032192125141472924, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9882629107981221, "error_rate_at_confidence_0_9": 0.0023752969121140144, "ordinal_expected_level_mae": null }, "agentdojo_tool_outputs:agentdojo_embedded_injection": { "questions": 426, "accuracy": 0.9835680751173709, "mean_class_recall": 0.9635416666666667, "nll": 0.03323309985013501, "brier": 0.020700086530374747, "ece15": 0.012858446963474107, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9835680751173709, "error_rate_at_confidence_0_9": 0.007159904534606206, "ordinal_expected_level_mae": null }, "byzfl_mnist:fl_attack_generated_update": { "questions": 800, "accuracy": 0.95875, "mean_class_recall": 0.9296875, "nll": 0.14798192410675134, "brier": 0.07208296294789789, "ece15": 0.031237402444916955, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9575, "error_rate_at_confidence_0_9": 0.02741514360313316, "ordinal_expected_level_mae": null }, "ciciot2023:ciciot_attack": { "questions": 6000, "accuracy": 0.9171666666666667, "mean_class_recall": 0.8741111111111111, "nll": 0.2872065840367128, "brier": 0.1291518236859529, "ece15": 0.02997403903668212, "temperature": 1.0, "coverage_at_confidence_0_9": 0.7491666666666666, "error_rate_at_confidence_0_9": 0.03181312569521691, "ordinal_expected_level_mae": null }, "injecagent:injecagent_requested_operation_v13": { "questions": 124, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 6.416181236130915e-05, "brier": 3.664856836177276e-07, "ece15": 6.406994017019252e-05, "temperature": 1.0, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_known_logon_action_v16": { "questions": 540, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.0003909349271697467, "brier": 5.411140895974333e-05, "ece15": 0.000376397356322764, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9981481481481481, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_observable_policy": { "questions": 540, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 1.9301873747749096e-07, "brier": 4.461551704214049e-12, "ece15": 1.9301762319156735e-07, "temperature": 1.0, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "ton_iot_network:ton_attack": { "questions": 3500, "accuracy": 0.988, "mean_class_recall": 0.9820952380952381, "nll": 0.052315917826989855, "brier": 0.021002166306533596, "ece15": 0.005742982405630235, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9757142857142858, "error_rate_at_confidence_0_9": 0.007320644216691069, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_evidence_level": { "questions": 659, "accuracy": 0.9954476479514416, "mean_class_recall": 0.9949748743718594, "nll": 0.006489090778116727, "brier": 0.004985928434486637, "ece15": 0.0035010905751582675, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9954476479514416, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": 0.004830545729071066 }, "twins_streamlet:twins_policy_action": { "questions": 659, "accuracy": 0.9984825493171472, "mean_class_recall": 0.9983249581239532, "nll": 0.007759166929265847, "brier": 0.0031279766576844436, "ece15": 0.0017302646891685911, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9969650986342944, "error_rate_at_confidence_0_9": 0.0015220700152207, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_quorum": { "questions": 659, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.0012916521858483854, "brier": 0.0006719742504699832, "ece15": 0.0010543087689309648, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9969650986342944, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_visible_conflict": { "questions": 659, "accuracy": 0.9984825493171472, "mean_class_recall": 0.9989130434782609, "nll": 0.0030258654215359524, "brier": 0.0020473738981638956, "ece15": 0.0002839890142556098, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9954476479514416, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_message_manipulation": { "questions": 3321, "accuracy": 0.7034025895814514, "mean_class_recall": 0.7106693655944849, "nll": 0.5271449745997652, "brier": 0.3679898300956757, "ece15": 0.03254066478455375, "temperature": 1.0, "coverage_at_confidence_0_9": 0.21499548328816623, "error_rate_at_confidence_0_9": 0.011204481792717087, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_supported_attack_type_v16": { "questions": 3321, "accuracy": 0.692261367058115, "mean_class_recall": 0.49711800080440777, "nll": 1.181957709780574, "brier": 0.48216427161657194, "ece15": 0.04703079874679226, "temperature": 1.0, "coverage_at_confidence_0_9": 0.20776874435411022, "error_rate_at_confidence_0_9": 0.07681159420289856, "ordinal_expected_level_mae": null } }, "macro_task_accuracy": 0.9450619006663279, "macro_task_balanced_accuracy": 0.9242156493985226 }, "recalibrated": { "questions": 21634, "accuracy": 0.8801423684940372, "mean_class_recall": 0.8422636415920437, "nll": 0.3441350952603644, "brier": 0.1767857045083663, "ece15": 0.035627089616313175, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.6567440140519553, "error_rate_at_confidence_0_9": 0.014358108108108109, "ordinal_expected_level_mae": 0.004973955616734103, "tasks": { "agentdojo_tool_outputs:agentdojo_boundary_action": { "questions": 426, "accuracy": 0.9953051643192489, "mean_class_recall": 0.9895833333333333, "nll": 0.02085470758455793, "brier": 0.009033562612123906, "ece15": 0.005062121140742335, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.9741784037558685, "error_rate_at_confidence_0_9": 0.0024096385542168677, "ordinal_expected_level_mae": null }, "agentdojo_tool_outputs:agentdojo_embedded_injection": { "questions": 426, "accuracy": 0.9835680751173709, "mean_class_recall": 0.9635416666666667, "nll": 0.029576905474440387, "brier": 0.019371229609764937, "ece15": 0.0128149515297424, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.9741784037558685, "error_rate_at_confidence_0_9": 0.004819277108433735, "ordinal_expected_level_mae": null }, "byzfl_mnist:fl_attack_generated_update": { "questions": 800, "accuracy": 0.95875, "mean_class_recall": 0.9296875, "nll": 0.1252495255860491, "brier": 0.06891977027707019, "ece15": 0.02206119521855819, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.93375, "error_rate_at_confidence_0_9": 0.0214190093708166, "ordinal_expected_level_mae": null }, "ciciot2023:ciciot_attack": { "questions": 6000, "accuracy": 0.9171666666666667, "mean_class_recall": 0.8741111111111111, "nll": 0.258923302286342, "brier": 0.1323925339560934, "ece15": 0.040786104751424794, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.6995, "error_rate_at_confidence_0_9": 0.027162258756254467, "ordinal_expected_level_mae": null }, "injecagent:injecagent_requested_operation_v13": { "questions": 124, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.00036120828507376734, "brier": 6.118819643947097e-06, "ece15": 0.0003596634947089362, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_known_logon_action_v16": { "questions": 540, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.0010259839180316675, "brier": 0.00015355821904600842, "ece15": 0.0009831313211751498, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.9981481481481481, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_observable_policy": { "questions": 540, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 5.4752266593721125e-06, "brier": 1.1349090161477055e-09, "ece15": 5.474942881804694e-06, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "ton_iot_network:ton_attack": { "questions": 3500, "accuracy": 0.988, "mean_class_recall": 0.9820952380952381, "nll": 0.0471904212042291, "brier": 0.02136284177729358, "ece15": 0.004930332206950378, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.9597142857142857, "error_rate_at_confidence_0_9": 0.006251860672819291, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_evidence_level": { "questions": 659, "accuracy": 0.9954476479514416, "mean_class_recall": 0.9949748743718594, "nll": 0.005989405513587896, "brier": 0.004363630530423605, "ece15": 0.003686325313576514, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.9954476479514416, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": 0.004973955616734103 }, "twins_streamlet:twins_policy_action": { "questions": 659, "accuracy": 0.9984825493171472, "mean_class_recall": 0.9983249581239532, "nll": 0.0069631222649601515, "brier": 0.003204452557959292, "ece15": 0.001531985329986691, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.9969650986342944, "error_rate_at_confidence_0_9": 0.0015220700152207, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_quorum": { "questions": 659, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.0018952623564185236, "brier": 0.0008114028234666237, "ece15": 0.0016108880631044219, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.9954476479514416, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_visible_conflict": { "questions": 659, "accuracy": 0.9984825493171472, "mean_class_recall": 0.9989130434782609, "nll": 0.0033168872089549845, "brier": 0.0020980871703588605, "ece15": 0.0012518453771527926, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.9954476479514416, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_message_manipulation": { "questions": 3321, "accuracy": 0.7034025895814514, "mean_class_recall": 0.7106693655944849, "nll": 0.5278136514287293, "brier": 0.3689683794121852, "ece15": 0.03240031037986953, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.1969286359530262, "error_rate_at_confidence_0_9": 0.009174311926605505, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_supported_attack_type_v16": { "questions": 3321, "accuracy": 0.692261367058115, "mean_class_recall": 0.49711800080440777, "nll": 1.1560335557868109, "brier": 0.498611302076152, "ece15": 0.12617627958797675, "temperature": 1.334968103387446, "coverage_at_confidence_0_9": 0.17856067449563384, "error_rate_at_confidence_0_9": 0.07251264755480608, "ordinal_expected_level_mae": null } }, "macro_task_accuracy": 0.9450619006663279, "macro_task_balanced_accuracy": 0.9242156493985226 } }, "general": { "decision-v7": { "raw_clean": { "n": 1200, "nll": 0.5734957671127382, "acc": 0.8658333333333333, "ece": 0.08617509900940432, "brier": 0.21792288610472857, "mean_conf": 0.9509841477415141, "confident_error_rate": 0.0675, "coverage_at_0_9": 0.8525, "accuracy_at_0_9": 0.9208211143695014, "coverage_at_5pct_error": 0.7833333333333333, "coverage_at_1pct_error": 0.6166666666666667, "aurc": 0.0281999905676559, "error_rate_at_0_9": 0.07917888563049853, "confidence_bias": 0.08515081440818073, "top_bins": { "0.9": { "n": 1023, "errors": 81, "error_rate": 0.07917888563049853 }, "0.95": { "n": 962, "errors": 53, "error_rate": 0.0550935550935551 }, "0.99": { "n": 885, "errors": 34, "error_rate": 0.0384180790960452 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9933333333333333, "confidence_cutoff": 0.9999894749108045 }, "0.8": { "coverage": 0.8, "accuracy": 0.9447916666666667, "confidence_cutoff": 0.9524106736120255 } }, "score_mae": 0.4625639513458072, "ranked_probability_score": 0.08810095245487483 }, "as_shipped_clean": { "n": 1200, "nll": 0.46540017253913407, "acc": 0.8658333333333333, "ece": 0.07412868458003202, "brier": 0.20783721757683873, "mean_conf": 0.9351787338923708, "confident_error_rate": 0.041666666666666664, "coverage_at_0_9": 0.7983333333333333, "accuracy_at_0_9": 0.9478079331941545, "coverage_at_5pct_error": 0.7783333333333333, "coverage_at_1pct_error": 0.6141666666666666, "aurc": 0.02814862771926311, "error_rate_at_0_9": 0.05219206680584551, "confidence_bias": 0.06934540055903748, "top_bins": { "0.9": { "n": 958, "errors": 50, "error_rate": 0.05219206680584551 }, "0.95": { "n": 912, "errors": 40, "error_rate": 0.043859649122807015 }, "0.99": { "n": 822, "errors": 18, "error_rate": 0.021897810218978103 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9933333333333333, "confidence_cutoff": 0.9998006494793819 }, "0.8": { "coverage": 0.8, "accuracy": 0.9458333333333333, "confidence_cutoff": 0.8959586216727804 } }, "score_mae": 0.47318044838075235, "ranked_probability_score": 0.08431774694216215 }, "sources": { "agnews": { "n": 240, "nll": 0.2925312694754093, "acc": 0.9166666666666666, "ece": 0.05790914903059574, "brier": 0.1357836869201196, "mean_conf": 0.9733704974572437, "confident_error_rate": 0.04583333333333333, "coverage_at_0_9": 0.9291666666666667, "accuracy_at_0_9": 0.9506726457399103, "coverage_at_5pct_error": 0.9333333333333333, "coverage_at_1pct_error": 0.7583333333333333, "aurc": 0.011635752911899793, "error_rate_at_0_9": 0.04932735426008968, "confidence_bias": 0.05670383079057706, "top_bins": { "0.9": { "n": 223, "errors": 11, "error_rate": 0.04932735426008968 }, "0.95": { "n": 217, "errors": 9, "error_rate": 0.041474654377880185 }, "0.99": { "n": 197, "errors": 4, "error_rate": 0.02030456852791878 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9996851444568575 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.9921275226980688 } } }, "amazon": { "n": 80, "nll": 1.2207591140196519, "acc": 0.6125, "ece": 0.1615943676102411, "brier": 0.5707631816876242, "mean_conf": 0.7431377036571911, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.2375, "accuracy_at_0_9": 0.8421052631578947, "coverage_at_5pct_error": 0.075, "coverage_at_1pct_error": 0.075, "aurc": 0.272315413996461, "error_rate_at_0_9": 0.15789473684210525, "confidence_bias": 0.13063770365719107, "top_bins": { "0.9": { "n": 19, "errors": 3, "error_rate": 0.15789473684210525 }, "0.95": { "n": 14, "errors": 2, "error_rate": 0.14285714285714285 }, "0.99": { "n": 3, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.7847517191083221 }, "0.8": { "coverage": 0.8, "accuracy": 0.640625, "confidence_cutoff": 0.5534749021853737 } }, "score_mae": 0.5042438581009662, "ranked_probability_score": 0.09073105215000309 }, "banking77": { "n": 80, "nll": 0.6278787803563556, "acc": 0.8375, "ece": 0.1237375681167378, "brier": 0.25438988606734264, "mean_conf": 0.8981608539632144, "confident_error_rate": 0.05, "coverage_at_0_9": 0.7625, "accuracy_at_0_9": 0.9344262295081968, "coverage_at_5pct_error": 0.6875, "coverage_at_1pct_error": 0.6125, "aurc": 0.035783745285874746, "error_rate_at_0_9": 0.06557377049180328, "confidence_bias": 0.06066085396321441, "top_bins": { "0.9": { "n": 61, "errors": 4, "error_rate": 0.06557377049180328 }, "0.95": { "n": 57, "errors": 4, "error_rate": 0.07017543859649122 }, "0.99": { "n": 48, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9966194161989207 }, "0.8": { "coverage": 0.8, "accuracy": 0.90625, "confidence_cutoff": 0.8503200856915479 } } }, "boolq": { "n": 80, "nll": 0.6125951836430432, "acc": 0.875, "ece": 0.11756712414134643, "brier": 0.22505069986723222, "mean_conf": 0.9514740522819943, "confident_error_rate": 0.0875, "coverage_at_0_9": 0.85, "accuracy_at_0_9": 0.8970588235294118, "coverage_at_5pct_error": 0.6875, "coverage_at_1pct_error": 0.1125, "aurc": 0.0511510509173725, "error_rate_at_0_9": 0.10294117647058823, "confidence_bias": 0.07647405228199433, "top_bins": { "0.9": { "n": 68, "errors": 7, "error_rate": 0.10294117647058823 }, "0.95": { "n": 67, "errors": 7, "error_rate": 0.1044776119402985 }, "0.99": { "n": 61, "errors": 4, "error_rate": 0.06557377049180328 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9999426387743003 }, "0.8": { "coverage": 0.8, "accuracy": 0.921875, "confidence_cutoff": 0.9819850746275051 } } }, "contrastive": { "n": 160, "nll": 0.07962468888836484, "acc": 0.96875, "ece": 0.026633113578874344, "brier": 0.05007247893674288, "mean_conf": 0.9953831135788744, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.98125, "accuracy_at_0_9": 0.9872611464968153, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.96875, "aurc": 0.000752718799433766, "error_rate_at_0_9": 0.012738853503184714, "confidence_bias": 0.026633113578874434, "top_bins": { "0.9": { "n": 157, "errors": 2, "error_rate": 0.012738853503184714 }, "0.95": { "n": 156, "errors": 2, "error_rate": 0.01282051282051282 }, "0.99": { "n": 148, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999957907736378 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9999070137203582 } } }, "dbpedia14": { "n": 80, "nll": 0.02165084849307894, "acc": 0.9875, "ece": 0.01152608625473029, "brier": 0.016104080808397182, "mean_conf": 0.9959462330871315, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9875, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.9875, "aurc": 0.00015625, "error_rate_at_0_9": 0.0, "confidence_bias": 0.008446233087131438, "top_bins": { "0.9": { "n": 79, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 78, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 77, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999981627832084 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9999896312065648 } } }, "imdb": { "n": 80, "nll": 0.06521915089368877, "acc": 0.9625, "ece": 0.02850319056358286, "brier": 0.04639252879503235, "mean_conf": 0.985401588169408, "confident_error_rate": 0.0, "coverage_at_0_9": 0.95, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.95, "aurc": 0.0012680561790530144, "error_rate_at_0_9": 0.0, "confidence_bias": 0.022901588169408016, "top_bins": { "0.9": { "n": 76, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 75, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 71, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999903501374642 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9998069111372814 } } }, "mnli": { "n": 80, "nll": 0.5051164719750952, "acc": 0.9125, "ece": 0.08269743141349266, "brier": 0.16426794867102618, "mean_conf": 0.9883215126078937, "confident_error_rate": 0.075, "coverage_at_0_9": 0.95, "accuracy_at_0_9": 0.9210526315789473, "coverage_at_5pct_error": 0.8875, "coverage_at_1pct_error": 0.65, "aurc": 0.015551107945666897, "error_rate_at_0_9": 0.07894736842105263, "confidence_bias": 0.07582151260789372, "top_bins": { "0.9": { "n": 76, "errors": 6, "error_rate": 0.07894736842105263 }, "0.95": { "n": 74, "errors": 5, "error_rate": 0.06756756756756757 }, "0.99": { "n": 72, "errors": 4, "error_rate": 0.05555555555555555 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999963968506167 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.9998914656545862 } } }, "sst5": { "n": 80, "nll": 1.2493438944839201, "acc": 0.6125, "ece": 0.24936946322063214, "brier": 0.5917259991138234, "mean_conf": 0.8126564640723977, "confident_error_rate": 0.075, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.75, "coverage_at_5pct_error": 0.075, "coverage_at_1pct_error": 0.075, "aurc": 0.3025049973121495, "error_rate_at_0_9": 0.25, "confidence_bias": 0.20015646407239762, "top_bins": { "0.9": { "n": 24, "errors": 6, "error_rate": 0.25 }, "0.95": { "n": 10, "errors": 1, "error_rate": 0.1 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.625, "confidence_cutoff": 0.8548749588336553 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.7024572592103087 } }, "score_mae": 0.48716165684142193, "ranked_probability_score": 0.08942157579848162 }, "trec": { "n": 80, "nll": 0.2837433451436081, "acc": 0.9625, "ece": 0.028515907037196753, "brier": 0.057680311047061936, "mean_conf": 0.9910159070371967, "confident_error_rate": 0.025, "coverage_at_0_9": 0.9875, "accuracy_at_0_9": 0.9746835443037974, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.275, "aurc": 0.01757155053741791, "error_rate_at_0_9": 0.02531645569620253, "confidence_bias": 0.02851590703719664, "top_bins": { "0.9": { "n": 79, "errors": 2, "error_rate": 0.02531645569620253 }, "0.95": { "n": 78, "errors": 2, "error_rate": 0.02564102564102564 }, "0.99": { "n": 77, "errors": 2, "error_rate": 0.025974025974025976 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9999924765263039 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.9999100942077365 } } }, "yelp": { "n": 160, "nll": 0.6789263064378058, "acc": 0.76875, "ece": 0.10810206862028397, "brier": 0.34184380448059815, "mean_conf": 0.8753444869898281, "confident_error_rate": 0.05625, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.90625, "coverage_at_5pct_error": 0.41875, "coverage_at_1pct_error": 0.1875, "aurc": 0.08672761133002724, "error_rate_at_0_9": 0.09375, "confidence_bias": 0.1065944869898281, "top_bins": { "0.9": { "n": 96, "errors": 9, "error_rate": 0.09375 }, "0.95": { "n": 86, "errors": 8, "error_rate": 0.09302325581395349 }, "0.99": { "n": 68, "errors": 4, "error_rate": 0.058823529411764705 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.971684991366676 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.7267623543739319 } }, "score_mae": 0.428135830199869, "ranked_probability_score": 0.07280061287800174 } }, "types": { "choice": { "n": 480, "nll": 0.3225147096064765, "acc": 0.9291666666666667, "ece": 0.05269727298309242, "brier": 0.11939209773780447, "mean_conf": 0.9740635473132214, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.9333333333333333, "accuracy_at_0_9": 0.9598214285714286, "coverage_at_5pct_error": 0.9479166666666666, "coverage_at_1pct_error": 0.7354166666666667, "aurc": 0.010523805656224608, "error_rate_at_0_9": 0.04017857142857143, "confidence_bias": 0.04489688064655473, "top_bins": { "0.9": { "n": 448, "errors": 18, "error_rate": 0.04017857142857143 }, "0.95": { "n": 439, "errors": 17, "error_rate": 0.0387243735763098 }, "0.99": { "n": 416, "errors": 8, "error_rate": 0.019230769230769232 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9958333333333333, "confidence_cutoff": 0.999989087291256 }, "0.8": { "coverage": 0.8, "accuracy": 0.9895833333333334, "confidence_cutoff": 0.9991522595300467 } } }, "noul": { "n": 480, "nll": 0.2630675588624098, "acc": 0.925, "ece": 0.048426932319500904, "brier": 0.11894188754077414, "mean_conf": 0.9734269323195011, "confident_error_rate": 0.035416666666666666, "coverage_at_0_9": 0.91875, "accuracy_at_0_9": 0.9614512471655329, "coverage_at_5pct_error": 0.9416666666666667, "coverage_at_1pct_error": 0.71875, "aurc": 0.014292966145900442, "error_rate_at_0_9": 0.03854875283446712, "confidence_bias": 0.048426932319501015, "top_bins": { "0.9": { "n": 441, "errors": 17, "error_rate": 0.03854875283446712 }, "0.95": { "n": 431, "errors": 15, "error_rate": 0.03480278422273782 }, "0.99": { "n": 397, "errors": 9, "error_rate": 0.022670025188916875 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9958333333333333, "confidence_cutoff": 0.9999067506240331 }, "0.8": { "coverage": 0.8, "accuracy": 0.9791666666666666, "confidence_cutoff": 0.9950087776054062 } } }, "score": { "n": 240, "nll": 1.1558363257578976, "acc": 0.6208333333333333, "ece": 0.1746159580142597, "brier": 0.5625181173270365, "mean_conf": 0.7809127101964097, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.2875, "accuracy_at_0_9": 0.782608695652174, "coverage_at_5pct_error": 0.016666666666666666, "coverage_at_1pct_error": 0.016666666666666666, "aurc": 0.28163895271092954, "error_rate_at_0_9": 0.21739130434782608, "confidence_bias": 0.16007937686307638, "top_bins": { "0.9": { "n": 69, "errors": 15, "error_rate": 0.21739130434782608 }, "0.95": { "n": 42, "errors": 8, "error_rate": 0.19047619047619047 }, "0.99": { "n": 9, "errors": 1, "error_rate": 0.1111111111111111 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6916666666666667, "confidence_cutoff": 0.8195696673851967 }, "0.8": { "coverage": 0.8, "accuracy": 0.6614583333333334, "confidence_cutoff": 0.6215591950077792 } }, "score_mae": 0.47318044838075235, "ranked_probability_score": 0.08431774694216215 } } }, "transfer-v4": { "raw_clean": { "n": 656, "nll": 0.840653344789253, "acc": 0.8384146341463414, "ece": 0.11570395215040587, "brier": 0.2772648298322645, "mean_conf": 0.9433821481191502, "confident_error_rate": 0.0899390243902439, "coverage_at_0_9": 0.8429878048780488, "accuracy_at_0_9": 0.8933092224231465, "coverage_at_5pct_error": 0.5777439024390244, "coverage_at_1pct_error": 0.3551829268292683, "aurc": 0.0494438083179723, "error_rate_at_0_9": 0.10669077757685352, "confidence_bias": 0.10496751397280879, "top_bins": { "0.9": { "n": 553, "errors": 59, "error_rate": 0.10669077757685352 }, "0.95": { "n": 527, "errors": 54, "error_rate": 0.10246679316888045 }, "0.99": { "n": 472, "errors": 34, "error_rate": 0.07203389830508475 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9634146341463414, "confidence_cutoff": 0.9999382193883597 }, "0.8": { "coverage": 0.8003048780487805, "accuracy": 0.900952380952381, "confidence_cutoff": 0.951274711387887 } }, "score_mae": 0.2762329771748361, "ranked_probability_score": 0.12228321967451691 }, "as_shipped_clean": { "n": 656, "nll": 0.6710659810048166, "acc": 0.8384146341463414, "ece": 0.09231046175305074, "brier": 0.26721979560494846, "mean_conf": 0.9260829686930101, "confident_error_rate": 0.07926829268292683, "coverage_at_0_9": 0.7987804878048781, "accuracy_at_0_9": 0.9007633587786259, "coverage_at_5pct_error": 0.5746951219512195, "coverage_at_1pct_error": 0.3551829268292683, "aurc": 0.049407167334717506, "error_rate_at_0_9": 0.09923664122137404, "confidence_bias": 0.08766833454666867, "top_bins": { "0.9": { "n": 524, "errors": 52, "error_rate": 0.09923664122137404 }, "0.95": { "n": 493, "errors": 42, "error_rate": 0.08519269776876268 }, "0.99": { "n": 422, "errors": 23, "error_rate": 0.054502369668246446 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9573170731707317, "confidence_cutoff": 0.9992490894147191 }, "0.8": { "coverage": 0.8003048780487805, "accuracy": 0.8990476190476191, "confidence_cutoff": 0.8992331757610149 } }, "score_mae": 0.27589619803682164, "ranked_probability_score": 0.11777542530670457 }, "sources": { "composition_holdout": { "n": 96, "nll": 0.946778069631199, "acc": 0.8645833333333334, "ece": 0.14288844371789414, "brier": 0.2646184933261653, "mean_conf": 0.9787006948112301, "confident_error_rate": 0.11458333333333333, "coverage_at_0_9": 0.9479166666666666, "accuracy_at_0_9": 0.8791208791208791, "coverage_at_5pct_error": 0.6875, "coverage_at_1pct_error": 0.65625, "aurc": 0.032783504799663264, "error_rate_at_0_9": 0.12087912087912088, "confidence_bias": 0.11411736147789675, "top_bins": { "0.9": { "n": 91, "errors": 11, "error_rate": 0.12087912087912088 }, "0.95": { "n": 90, "errors": 11, "error_rate": 0.12222222222222222 }, "0.99": { "n": 89, "errors": 11, "error_rate": 0.12359550561797752 } }, "selective": { "0.5": { "coverage": 0.5104166666666666, "accuracy": 1.0, "confidence_cutoff": 0.9999696869828117 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8974358974358975, "confidence_cutoff": 0.9992490894147191 } } }, "contrastive": { "n": 80, "nll": 0.5115728584657595, "acc": 0.875, "ece": 0.13006292045346943, "brier": 0.23406192561698297, "mean_conf": 0.9705133073135578, "confident_error_rate": 0.0875, "coverage_at_0_9": 0.9125, "accuracy_at_0_9": 0.9041095890410958, "coverage_at_5pct_error": 0.85, "coverage_at_1pct_error": 0.8, "aurc": 0.0169297143647776, "error_rate_at_0_9": 0.0958904109589041, "confidence_bias": 0.09551330731355778, "top_bins": { "0.9": { "n": 73, "errors": 7, "error_rate": 0.0958904109589041 }, "0.95": { "n": 71, "errors": 6, "error_rate": 0.08450704225352113 }, "0.99": { "n": 68, "errors": 3, "error_rate": 0.04411764705882353 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999941701311645 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9997393381382442 } }, "score_mae": 0.27589619803682164, "ranked_probability_score": 0.11777542530670457 }, "emotion": { "n": 80, "nll": 1.2094004500158717, "acc": 0.675, "ece": 0.19812590622433582, "brier": 0.4998620899742045, "mean_conf": 0.7808203087114324, "confident_error_rate": 0.075, "coverage_at_0_9": 0.4375, "accuracy_at_0_9": 0.8285714285714286, "coverage_at_5pct_error": 0.075, "coverage_at_1pct_error": 0.075, "aurc": 0.1832973887490438, "error_rate_at_0_9": 0.17142857142857143, "confidence_bias": 0.10582030871143233, "top_bins": { "0.9": { "n": 35, "errors": 6, "error_rate": 0.17142857142857143 }, "0.95": { "n": 30, "errors": 4, "error_rate": 0.13333333333333333 }, "0.99": { "n": 12, "errors": 1, "error_rate": 0.08333333333333333 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.8500919215367448 }, "0.8": { "coverage": 0.8, "accuracy": 0.734375, "confidence_cutoff": 0.5761070047579089 } } }, "mmlu": { "n": 80, "nll": 0.9027907855123264, "acc": 0.6875, "ece": 0.14251964152159943, "brier": 0.4317941195669751, "mean_conf": 0.8238540468148077, "confident_error_rate": 0.075, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.85, "coverage_at_5pct_error": 0.375, "coverage_at_1pct_error": 0.325, "aurc": 0.11589530494156297, "error_rate_at_0_9": 0.15, "confidence_bias": 0.13635404681480767, "top_bins": { "0.9": { "n": 40, "errors": 6, "error_rate": 0.15 }, "0.95": { "n": 30, "errors": 1, "error_rate": 0.03333333333333333 }, "0.99": { "n": 19, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.85, "confidence_cutoff": 0.9017783630155174 }, "0.8": { "coverage": 0.8, "accuracy": 0.8125, "confidence_cutoff": 0.613701094142181 } } }, "paws": { "n": 80, "nll": 0.9771542282927816, "acc": 0.8125, "ece": 0.1651648000763565, "brier": 0.31695123775026923, "mean_conf": 0.9524627544936155, "confident_error_rate": 0.125, "coverage_at_0_9": 0.8625, "accuracy_at_0_9": 0.855072463768116, "coverage_at_5pct_error": 0.15, "coverage_at_1pct_error": 0.15, "aurc": 0.0997884943666906, "error_rate_at_0_9": 0.14492753623188406, "confidence_bias": 0.1399627544936155, "top_bins": { "0.9": { "n": 69, "errors": 10, "error_rate": 0.14492753623188406 }, "0.95": { "n": 69, "errors": 10, "error_rate": 0.14492753623188406 }, "0.99": { "n": 61, "errors": 6, "error_rate": 0.09836065573770492 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.875, "confidence_cutoff": 0.9997721781585782 }, "0.8": { "coverage": 0.8, "accuracy": 0.890625, "confidence_cutoff": 0.9846682768581441 } } }, "qnli": { "n": 80, "nll": 0.32010583310414403, "acc": 0.925, "ece": 0.06578826678776872, "brier": 0.13874220003757964, "mean_conf": 0.9719094562930305, "confident_error_rate": 0.05, "coverage_at_0_9": 0.925, "accuracy_at_0_9": 0.9459459459459459, "coverage_at_5pct_error": 0.8625, "coverage_at_1pct_error": 0.6, "aurc": 0.015384772492900507, "error_rate_at_0_9": 0.05405405405405406, "confidence_bias": 0.04690945629303045, "top_bins": { "0.9": { "n": 74, "errors": 4, "error_rate": 0.05405405405405406 }, "0.95": { "n": 72, "errors": 4, "error_rate": 0.05555555555555555 }, "0.99": { "n": 66, "errors": 2, "error_rate": 0.030303030303030304 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9997714445133897 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.9937610824296935 } } }, "sciq": { "n": 80, "nll": 0.00696809639784113, "acc": 1.0, "ece": 0.006546558428673463, "brier": 0.0013901896311441294, "mean_conf": 0.9934534415713265, "confident_error_rate": 0.0, "coverage_at_0_9": 0.975, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.006546558428673466, "top_bins": { "0.9": { "n": 78, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 76, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 72, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999699221173486 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9997295605534583 } } }, "tweet_offensive": { "n": 80, "nll": 0.4386151088933339, "acc": 0.8625, "ece": 0.10954228653253775, "brier": 0.2508583693920234, "mean_conf": 0.926426194311437, "confident_error_rate": 0.1, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 0.875, "coverage_at_5pct_error": 0.575, "coverage_at_1pct_error": 0.4875, "aurc": 0.05180455375068149, "error_rate_at_0_9": 0.125, "confidence_bias": 0.06392619431143698, "top_bins": { "0.9": { "n": 64, "errors": 8, "error_rate": 0.125 }, "0.95": { "n": 55, "errors": 6, "error_rate": 0.10909090909090909 }, "0.99": { "n": 35, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9842379652550892 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.902661270668069 } } } }, "types": { "choice": { "n": 336, "nll": 0.7750697655913044, "acc": 0.8095238095238095, "ece": 0.09777001761032751, "brier": 0.2977591883723148, "mean_conf": 0.8982306263978672, "confident_error_rate": 0.06845238095238096, "coverage_at_0_9": 0.7261904761904762, "accuracy_at_0_9": 0.9057377049180327, "coverage_at_5pct_error": 0.4583333333333333, "coverage_at_1pct_error": 0.3630952380952381, "aurc": 0.05730883047817376, "error_rate_at_0_9": 0.0942622950819672, "confidence_bias": 0.08870681687405768, "top_bins": { "0.9": { "n": 244, "errors": 23, "error_rate": 0.0942622950819672 }, "0.95": { "n": 226, "errors": 16, "error_rate": 0.07079646017699115 }, "0.99": { "n": 192, "errors": 12, "error_rate": 0.0625 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9464285714285714, "confidence_cutoff": 0.9979451630958165 }, "0.8": { "coverage": 0.8005952380952381, "accuracy": 0.8921933085501859, "confidence_cutoff": 0.805212947551472 } } }, "noul": { "n": 280, "nll": 0.4959650021746835, "acc": 0.8857142857142857, "ece": 0.08836865713253901, "brier": 0.20187194492263405, "mean_conf": 0.9573703050826868, "confident_error_rate": 0.07857142857142857, "coverage_at_0_9": 0.8821428571428571, "accuracy_at_0_9": 0.9109311740890689, "coverage_at_5pct_error": 0.7535714285714286, "coverage_at_1pct_error": 0.25, "aurc": 0.039651831256480245, "error_rate_at_0_9": 0.08906882591093117, "confidence_bias": 0.07165601936840116, "top_bins": { "0.9": { "n": 247, "errors": 22, "error_rate": 0.08906882591093117 }, "0.95": { "n": 236, "errors": 20, "error_rate": 0.0847457627118644 }, "0.99": { "n": 202, "errors": 8, "error_rate": 0.039603960396039604 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9571428571428572, "confidence_cutoff": 0.9995423463157226 }, "0.8": { "coverage": 0.8, "accuracy": 0.9330357142857143, "confidence_cutoff": 0.9697351853044945 } } }, "score": { "n": 40, "nll": 1.0231410422892537, "acc": 0.75, "ece": 0.26013051552429656, "brier": 0.46812385113527205, "mean_conf": 0.9410312892444732, "confident_error_rate": 0.175, "coverage_at_0_9": 0.825, "accuracy_at_0_9": 0.7878787878787878, "coverage_at_5pct_error": 0.65, "coverage_at_1pct_error": 0.6, "aurc": 0.07355999344204343, "error_rate_at_0_9": 0.21212121212121213, "confidence_bias": 0.1910312892444732, "top_bins": { "0.9": { "n": 33, "errors": 7, "error_rate": 0.21212121212121213 }, "0.95": { "n": 31, "errors": 6, "error_rate": 0.1935483870967742 }, "0.99": { "n": 28, "errors": 3, "error_rate": 0.10714285714285714 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9999132212992297 }, "0.8": { "coverage": 0.8, "accuracy": 0.78125, "confidence_cutoff": 0.9461605303081507 } }, "score_mae": 0.27589619803682164, "ranked_probability_score": 0.11777542530670457 } } } }, "temperature": 1.334968103387446 } ], "kev_4b_security": { "test": { "raw": { "questions": 21634, "accuracy": 0.5048996949246556, "mean_class_recall": 0.5647195198307282, "nll": 1.2328084492499014, "brier": 0.7147920651499965, "ece15": 0.3370024824919156, "temperature": 1.0, "coverage_at_confidence_0_9": 0.44924655634649163, "error_rate_at_confidence_0_9": 0.34736083959255065, "ordinal_expected_level_mae": 0.2811370953407191, "tasks": { "agentdojo_tool_outputs:agentdojo_boundary_action": { "questions": 426, "accuracy": 0.5422535211267606, "mean_class_recall": 0.7045454545454546, "nll": 1.2120829748020316, "brier": 0.6804576908439421, "ece15": 0.2992356768561654, "temperature": 1.0, "coverage_at_confidence_0_9": 0.5187793427230047, "error_rate_at_confidence_0_9": 0.3936651583710407, "ordinal_expected_level_mae": null }, "agentdojo_tool_outputs:agentdojo_embedded_injection": { "questions": 426, "accuracy": 0.7300469483568075, "mean_class_recall": 0.8257575757575757, "nll": 0.6169123027998439, "brier": 0.3918536224381035, "ece15": 0.1421670820222778, "temperature": 1.0, "coverage_at_confidence_0_9": 0.57981220657277, "error_rate_at_confidence_0_9": 0.15384615384615385, "ordinal_expected_level_mae": null }, "byzfl_mnist:fl_attack_generated_update": { "questions": 800, "accuracy": 0.79875, "mean_class_recall": 0.49921875, "nll": 0.5895657949467273, "brier": 0.35429205012331033, "ece15": 0.1133528731473528, "temperature": 1.0, "coverage_at_confidence_0_9": 0.2175, "error_rate_at_confidence_0_9": 0.4942528735632184, "ordinal_expected_level_mae": null }, "ciciot2023:ciciot_attack": { "questions": 6000, "accuracy": 0.2628333333333333, "mean_class_recall": 0.5085555555555555, "nll": 1.2759496387732032, "brier": 0.9560488943280168, "ece15": 0.5505303178793823, "temperature": 1.0, "coverage_at_confidence_0_9": 0.12266666666666666, "error_rate_at_confidence_0_9": 0.453804347826087, "ordinal_expected_level_mae": null }, "injecagent:injecagent_requested_operation_v13": { "questions": 124, "accuracy": 0.6290322580645161, "mean_class_recall": 0.6714285714285715, "nll": 1.0817458611434574, "brier": 0.6236238714203711, "ece15": 0.30916032986760006, "temperature": 1.0, "coverage_at_confidence_0_9": 0.75, "error_rate_at_confidence_0_9": 0.34408602150537637, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_known_logon_action_v16": { "questions": 540, "accuracy": 0.7981481481481482, "mean_class_recall": 0.7528344671201813, "nll": 0.4871733371313632, "brier": 0.29625578551796805, "ece15": 0.15242774183988111, "temperature": 1.0, "coverage_at_confidence_0_9": 0.7962962962962963, "error_rate_at_confidence_0_9": 0.08837209302325581, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_observable_policy": { "questions": 540, "accuracy": 0.9944444444444445, "mean_class_recall": 0.9886363636363636, "nll": 0.01641532065146085, "brier": 0.007136931607478901, "ece15": 0.013212748100349914, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9777777777777777, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "ton_iot_network:ton_attack": { "questions": 3500, "accuracy": 0.5065714285714286, "mean_class_recall": 0.6710476190476191, "nll": 1.1359565067657027, "brier": 0.6406729439019092, "ece15": 0.23932739622085006, "temperature": 1.0, "coverage_at_confidence_0_9": 0.38971428571428574, "error_rate_at_confidence_0_9": 0.3878299120234604, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_evidence_level": { "questions": 659, "accuracy": 0.8118361153262519, "mean_class_recall": 0.794717762064859, "nll": 0.47493573588084764, "brier": 0.2794985686136285, "ece15": 0.05201558678606951, "temperature": 1.0, "coverage_at_confidence_0_9": 0.4931714719271624, "error_rate_at_confidence_0_9": 0.04923076923076923, "ordinal_expected_level_mae": 0.2811370953407191 }, "twins_streamlet:twins_policy_action": { "questions": 659, "accuracy": 0.7177541729893778, "mean_class_recall": 0.6923577773226661, "nll": 0.5581275002335633, "brier": 0.3443282053616745, "ece15": 0.09062810464195983, "temperature": 1.0, "coverage_at_confidence_0_9": 0.44613050075872535, "error_rate_at_confidence_0_9": 0.04421768707482993, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_quorum": { "questions": 659, "accuracy": 0.7814871016691958, "mean_class_recall": 0.7019105077928607, "nll": 0.4980710054489271, "brier": 0.31257573538065736, "ece15": 0.09857831652926007, "temperature": 1.0, "coverage_at_confidence_0_9": 0.590288315629742, "error_rate_at_confidence_0_9": 0.09511568123393316, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_visible_conflict": { "questions": 659, "accuracy": 0.9711684370257967, "mean_class_recall": 0.9693685820406379, "nll": 0.10259254119815524, "brier": 0.05091487551434137, "ece15": 0.030811660924900536, "temperature": 1.0, "coverage_at_confidence_0_9": 0.8679817905918058, "error_rate_at_confidence_0_9": 0.006993006993006993, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_message_manipulation": { "questions": 3321, "accuracy": 0.47756699789220114, "mean_class_recall": 0.5, "nll": 1.2951885811608232, "brier": 0.8663028363232907, "ece15": 0.42951687019840074, "temperature": 1.0, "coverage_at_confidence_0_9": 0.7633242999096658, "error_rate_at_confidence_0_9": 0.504930966469428, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_supported_attack_type_v16": { "questions": 3321, "accuracy": 0.48087925323697683, "mean_class_recall": 0.12233932942660039, "nll": 2.41017951541039, "brier": 0.8961545633575424, "ece15": 0.3893600279043549, "temperature": 1.0, "coverage_at_confidence_0_9": 0.5453176753989762, "error_rate_at_confidence_0_9": 0.4870237437879624, "ordinal_expected_level_mae": null } }, "macro_task_accuracy": 0.6787694400132313, "macro_task_balanced_accuracy": 0.671622736838496 }, "recalibrated": { "questions": 21634, "accuracy": 0.5048996949246556, "mean_class_recall": 0.5647195198307282, "nll": 1.2328084492499014, "brier": 0.7147920651499965, "ece15": 0.3370024824919156, "temperature": 1.0, "coverage_at_confidence_0_9": 0.44924655634649163, "error_rate_at_confidence_0_9": 0.34736083959255065, "ordinal_expected_level_mae": 0.2811370953407191, "tasks": { "agentdojo_tool_outputs:agentdojo_boundary_action": { "questions": 426, "accuracy": 0.5422535211267606, "mean_class_recall": 0.7045454545454546, "nll": 1.2120829748020316, "brier": 0.6804576908439421, "ece15": 0.2992356768561654, "temperature": 1.0, "coverage_at_confidence_0_9": 0.5187793427230047, "error_rate_at_confidence_0_9": 0.3936651583710407, "ordinal_expected_level_mae": null }, "agentdojo_tool_outputs:agentdojo_embedded_injection": { "questions": 426, "accuracy": 0.7300469483568075, "mean_class_recall": 0.8257575757575757, "nll": 0.6169123027998439, "brier": 0.3918536224381035, "ece15": 0.1421670820222778, "temperature": 1.0, "coverage_at_confidence_0_9": 0.57981220657277, "error_rate_at_confidence_0_9": 0.15384615384615385, "ordinal_expected_level_mae": null }, "byzfl_mnist:fl_attack_generated_update": { "questions": 800, "accuracy": 0.79875, "mean_class_recall": 0.49921875, "nll": 0.5895657949467273, "brier": 0.35429205012331033, "ece15": 0.1133528731473528, "temperature": 1.0, "coverage_at_confidence_0_9": 0.2175, "error_rate_at_confidence_0_9": 0.4942528735632184, "ordinal_expected_level_mae": null }, "ciciot2023:ciciot_attack": { "questions": 6000, "accuracy": 0.2628333333333333, "mean_class_recall": 0.5085555555555555, "nll": 1.2759496387732032, "brier": 0.9560488943280168, "ece15": 0.5505303178793823, "temperature": 1.0, "coverage_at_confidence_0_9": 0.12266666666666666, "error_rate_at_confidence_0_9": 0.453804347826087, "ordinal_expected_level_mae": null }, "injecagent:injecagent_requested_operation_v13": { "questions": 124, "accuracy": 0.6290322580645161, "mean_class_recall": 0.6714285714285715, "nll": 1.0817458611434574, "brier": 0.6236238714203711, "ece15": 0.30916032986760006, "temperature": 1.0, "coverage_at_confidence_0_9": 0.75, "error_rate_at_confidence_0_9": 0.34408602150537637, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_known_logon_action_v16": { "questions": 540, "accuracy": 0.7981481481481482, "mean_class_recall": 0.7528344671201813, "nll": 0.4871733371313632, "brier": 0.29625578551796805, "ece15": 0.15242774183988111, "temperature": 1.0, "coverage_at_confidence_0_9": 0.7962962962962963, "error_rate_at_confidence_0_9": 0.08837209302325581, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_observable_policy": { "questions": 540, "accuracy": 0.9944444444444445, "mean_class_recall": 0.9886363636363636, "nll": 0.01641532065146085, "brier": 0.007136931607478901, "ece15": 0.013212748100349914, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9777777777777777, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "ton_iot_network:ton_attack": { "questions": 3500, "accuracy": 0.5065714285714286, "mean_class_recall": 0.6710476190476191, "nll": 1.1359565067657027, "brier": 0.6406729439019092, "ece15": 0.23932739622085006, "temperature": 1.0, "coverage_at_confidence_0_9": 0.38971428571428574, "error_rate_at_confidence_0_9": 0.3878299120234604, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_evidence_level": { "questions": 659, "accuracy": 0.8118361153262519, "mean_class_recall": 0.794717762064859, "nll": 0.47493573588084764, "brier": 0.2794985686136285, "ece15": 0.05201558678606951, "temperature": 1.0, "coverage_at_confidence_0_9": 0.4931714719271624, "error_rate_at_confidence_0_9": 0.04923076923076923, "ordinal_expected_level_mae": 0.2811370953407191 }, "twins_streamlet:twins_policy_action": { "questions": 659, "accuracy": 0.7177541729893778, "mean_class_recall": 0.6923577773226661, "nll": 0.5581275002335633, "brier": 0.3443282053616745, "ece15": 0.09062810464195983, "temperature": 1.0, "coverage_at_confidence_0_9": 0.44613050075872535, "error_rate_at_confidence_0_9": 0.04421768707482993, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_quorum": { "questions": 659, "accuracy": 0.7814871016691958, "mean_class_recall": 0.7019105077928607, "nll": 0.4980710054489271, "brier": 0.31257573538065736, "ece15": 0.09857831652926007, "temperature": 1.0, "coverage_at_confidence_0_9": 0.590288315629742, "error_rate_at_confidence_0_9": 0.09511568123393316, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_visible_conflict": { "questions": 659, "accuracy": 0.9711684370257967, "mean_class_recall": 0.9693685820406379, "nll": 0.10259254119815524, "brier": 0.05091487551434137, "ece15": 0.030811660924900536, "temperature": 1.0, "coverage_at_confidence_0_9": 0.8679817905918058, "error_rate_at_confidence_0_9": 0.006993006993006993, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_message_manipulation": { "questions": 3321, "accuracy": 0.47756699789220114, "mean_class_recall": 0.5, "nll": 1.2951885811608232, "brier": 0.8663028363232907, "ece15": 0.42951687019840074, "temperature": 1.0, "coverage_at_confidence_0_9": 0.7633242999096658, "error_rate_at_confidence_0_9": 0.504930966469428, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_supported_attack_type_v16": { "questions": 3321, "accuracy": 0.48087925323697683, "mean_class_recall": 0.12233932942660039, "nll": 2.41017951541039, "brier": 0.8961545633575424, "ece15": 0.3893600279043549, "temperature": 1.0, "coverage_at_confidence_0_9": 0.5453176753989762, "error_rate_at_confidence_0_9": 0.4870237437879624, "ordinal_expected_level_mae": null } }, "macro_task_accuracy": 0.6787694400132313, "macro_task_balanced_accuracy": 0.671622736838496 } }, "test_used_for_selection": false }, "kev_4b_general": { "decision-v7": { "raw_clean": { "n": 1200, "nll": 0.46776271359373983, "acc": 0.8708333333333333, "ece": 0.07324011844924935, "brier": 0.2068727227531146, "mean_conf": 0.9391164907697294, "confident_error_rate": 0.04833333333333333, "coverage_at_0_9": 0.8166666666666667, "accuracy_at_0_9": 0.9408163265306122, "coverage_at_5pct_error": 0.7791666666666667, "coverage_at_1pct_error": 0.6133333333333333, "aurc": 0.02837551275061399, "error_rate_at_0_9": 0.05918367346938776, "confidence_bias": 0.06828315743639601, "top_bins": { "0.9": { "n": 980, "errors": 58, "error_rate": 0.05918367346938776 }, "0.95": { "n": 915, "errors": 39, "error_rate": 0.04262295081967213 }, "0.99": { "n": 829, "errors": 21, "error_rate": 0.025331724969843185 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9916666666666667, "confidence_cutoff": 0.999598385219589 }, "0.8": { "coverage": 0.8, "accuracy": 0.94375, "confidence_cutoff": 0.9220693757732149 } }, "score_mae": 0.4635506535819828, "ranked_probability_score": 0.08574066842436183 }, "as_shipped_clean": { "n": 1200, "nll": 0.35957326104404586, "acc": 0.8708333333333333, "ece": 0.020190532051177417, "brier": 0.18914513739887015, "mean_conf": 0.8656556446393923, "confident_error_rate": 0.013333333333333334, "coverage_at_0_9": 0.6583333333333333, "accuracy_at_0_9": 0.979746835443038, "coverage_at_5pct_error": 0.7883333333333333, "coverage_at_1pct_error": 0.585, "aurc": 0.02903042619721546, "error_rate_at_0_9": 0.020253164556962026, "confidence_bias": -0.005177688693941085, "top_bins": { "0.9": { "n": 790, "errors": 16, "error_rate": 0.020253164556962026 }, "0.95": { "n": 697, "errors": 6, "error_rate": 0.00860832137733142 }, "0.99": { "n": 320, "errors": 2, "error_rate": 0.00625 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9916666666666667, "confidence_cutoff": 0.9690438255888087 }, "0.8": { "coverage": 0.8, "accuracy": 0.946875, "confidence_cutoff": 0.7097948924008157 } }, "score_mae": 0.496047231607107, "ranked_probability_score": 0.07900573734715889 }, "sources": { "agnews": { "n": 240, "nll": 0.2188339468959484, "acc": 0.9208333333333333, "ece": 0.03458717155438394, "brier": 0.11967225736034369, "mean_conf": 0.9210081021968138, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.7875, "accuracy_at_0_9": 0.9841269841269841, "coverage_at_5pct_error": 0.9291666666666667, "coverage_at_1pct_error": 0.7208333333333333, "aurc": 0.010595808181650849, "error_rate_at_0_9": 0.015873015873015872, "confidence_bias": 0.00017476886348055665, "top_bins": { "0.9": { "n": 189, "errors": 3, "error_rate": 0.015873015873015872 }, "0.95": { "n": 151, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 8, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.962887554254757 }, "0.8": { "coverage": 0.8, "accuracy": 0.9791666666666666, "confidence_cutoff": 0.8982351380482543 } } }, "amazon": { "n": 80, "nll": 1.019521275107054, "acc": 0.6125, "ece": 0.06021888255181947, "brier": 0.5421117796889148, "mean_conf": 0.5878388840884237, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0375, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.0625, "coverage_at_1pct_error": 0.0625, "aurc": 0.26205124362469656, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02466111591157638, "top_bins": { "0.9": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 1, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.75, "confidence_cutoff": 0.5617743497724099 }, "0.8": { "coverage": 0.8, "accuracy": 0.640625, "confidence_cutoff": 0.44733349967779523 } }, "score_mae": 0.5169737287164902, "ranked_probability_score": 0.08574565040577714 }, "banking77": { "n": 80, "nll": 0.6434976771895786, "acc": 0.875, "ece": 0.16139951556317184, "brier": 0.23632755413887385, "mean_conf": 0.7261215293796409, "confident_error_rate": 0.0, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.75, "coverage_at_1pct_error": 0.675, "aurc": 0.025355014288770995, "error_rate_at_0_9": 0.0, "confidence_bias": -0.14887847062035908, "top_bins": { "0.9": { "n": 32, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 24, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.853940262270331 }, "0.8": { "coverage": 0.8, "accuracy": 0.921875, "confidence_cutoff": 0.4421809473945611 } } }, "boolq": { "n": 80, "nll": 0.3692803969173892, "acc": 0.875, "ece": 0.0748529395052494, "brier": 0.2143912889240845, "mean_conf": 0.9094612721753246, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 0.9166666666666666, "coverage_at_5pct_error": 0.6625, "coverage_at_1pct_error": 0.075, "aurc": 0.059468554539862016, "error_rate_at_0_9": 0.08333333333333333, "confidence_bias": 0.034461272175324575, "top_bins": { "0.9": { "n": 60, "errors": 5, "error_rate": 0.08333333333333333 }, "0.95": { "n": 50, "errors": 1, "error_rate": 0.02 }, "0.99": { "n": 26, "errors": 1, "error_rate": 0.038461538461538464 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.9805121330496657 }, "0.8": { "coverage": 0.8, "accuracy": 0.890625, "confidence_cutoff": 0.8395732170279804 } } }, "contrastive": { "n": 160, "nll": 0.06053811563664827, "acc": 0.96875, "ece": 0.03645679005941199, "brier": 0.032513686428614616, "mean_conf": 0.9645709125518511, "confident_error_rate": 0.0, "coverage_at_0_9": 0.90625, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.975, "aurc": 0.0006312206080826334, "error_rate_at_0_9": 0.0, "confidence_bias": -0.004179087448148877, "top_bins": { "0.9": { "n": 145, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 139, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 83, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9906152293058859 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9714574090050602 } } }, "dbpedia14": { "n": 80, "nll": 0.031074258317027002, "acc": 1.0, "ece": 0.02597689933514287, "brier": 0.010683464293698132, "mean_conf": 0.9740231006648571, "confident_error_rate": 0.0, "coverage_at_0_9": 0.95, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02597689933514291, "top_bins": { "0.9": { "n": 76, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 75, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 57, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9936484074458076 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9825304423953378 } } }, "imdb": { "n": 80, "nll": 0.0720718883732508, "acc": 0.95, "ece": 0.044527070446157675, "brier": 0.0446787843047757, "mean_conf": 0.9624677353009415, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.9375, "aurc": 0.0017470077113648063, "error_rate_at_0_9": 0.0, "confidence_bias": 0.012467735300941585, "top_bins": { "0.9": { "n": 72, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 70, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 50, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9920587271734113 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.979826985587298 } } }, "mnli": { "n": 80, "nll": 0.27465339009465656, "acc": 0.9125, "ece": 0.09412857766900207, "brier": 0.15288319760619568, "mean_conf": 0.94639379824153, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.875, "accuracy_at_0_9": 0.9571428571428572, "coverage_at_5pct_error": 0.875, "coverage_at_1pct_error": 0.675, "aurc": 0.01561228714610165, "error_rate_at_0_9": 0.04285714285714286, "confidence_bias": 0.03389379824153005, "top_bins": { "0.9": { "n": 70, "errors": 3, "error_rate": 0.04285714285714286 }, "0.95": { "n": 67, "errors": 2, "error_rate": 0.029850746268656716 }, "0.99": { "n": 45, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9920616734287709 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.9661281848866967 } } }, "sst5": { "n": 80, "nll": 0.9636808958022751, "acc": 0.6125, "ece": 0.09386529578800333, "brier": 0.5310525330540982, "mean_conf": 0.6617158029994963, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1625, "coverage_at_1pct_error": 0.1625, "aurc": 0.26405329260422244, "error_rate_at_0_9": null, "confidence_bias": 0.04921580299949624, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.65, "confidence_cutoff": 0.6682213711318762 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.5940210248093071 } }, "score_mae": 0.5227774115005795, "ranked_probability_score": 0.08371156976927716 }, "trec": { "n": 80, "nll": 0.16488315000308515, "acc": 0.9625, "ece": 0.02170728565832692, "brier": 0.06057653876714909, "mean_conf": 0.9707615704179906, "confident_error_rate": 0.025, "coverage_at_0_9": 0.9625, "accuracy_at_0_9": 0.974025974025974, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.2625, "aurc": 0.018315788693627898, "error_rate_at_0_9": 0.025974025974025976, "confidence_bias": 0.008261570417990582, "top_bins": { "0.9": { "n": 77, "errors": 2, "error_rate": 0.025974025974025976 }, "0.95": { "n": 73, "errors": 2, "error_rate": 0.0273972602739726 }, "0.99": { "n": 47, "errors": 1, "error_rate": 0.02127659574468085 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.990957410950485 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.9755548475921001 } } }, "yelp": { "n": 160, "nll": 0.5386789559476151, "acc": 0.78125, "ece": 0.06183692449171378, "brier": 0.3102138876335011, "mean_conf": 0.7769424223142675, "confident_error_rate": 0.01875, "coverage_at_0_9": 0.4125, "accuracy_at_0_9": 0.9545454545454546, "coverage_at_5pct_error": 0.4375, "coverage_at_1pct_error": 0.14375, "aurc": 0.0856358031592544, "error_rate_at_0_9": 0.045454545454545456, "confidence_bias": -0.004307577685732533, "top_bins": { "0.9": { "n": 66, "errors": 3, "error_rate": 0.045454545454545456 }, "0.95": { "n": 47, "errors": 1, "error_rate": 0.02127659574468085 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9125, "confidence_cutoff": 0.8216206137745724 }, "0.8": { "coverage": 0.8, "accuracy": 0.8515625, "confidence_cutoff": 0.577455875568763 } }, "score_mae": 0.44839055460425137, "ranked_probability_score": 0.0675599918664224 } }, "types": { "choice": { "n": 480, "nll": 0.2482065801683337, "acc": 0.9375, "ece": 0.03877078391929752, "brier": 0.10960480604725589, "mean_conf": 0.9179948652716078, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.8145833333333333, "accuracy_at_0_9": 0.9846547314578005, "coverage_at_5pct_error": 0.9583333333333334, "coverage_at_1pct_error": 0.69375, "aurc": 0.01188927247802621, "error_rate_at_0_9": 0.015345268542199489, "confidence_bias": -0.019505134728392215, "top_bins": { "0.9": { "n": 391, "errors": 6, "error_rate": 0.015345268542199489 }, "0.95": { "n": 361, "errors": 4, "error_rate": 0.0110803324099723 }, "0.99": { "n": 202, "errors": 1, "error_rate": 0.0049504950495049506 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9958333333333333, "confidence_cutoff": 0.9867396184306336 }, "0.8": { "coverage": 0.8, "accuracy": 0.9869791666666666, "confidence_cutoff": 0.9197849614177179 } } }, "noul": { "n": 480, "nll": 0.18011051231286856, "acc": 0.9270833333333334, "ece": 0.014056851970226658, "brier": 0.10328956268313709, "mean_conf": 0.9306022039720875, "confident_error_rate": 0.01875, "coverage_at_0_9": 0.8166666666666667, "accuracy_at_0_9": 0.9770408163265306, "coverage_at_5pct_error": 0.9416666666666667, "coverage_at_1pct_error": 0.7041666666666667, "aurc": 0.015597964078219255, "error_rate_at_0_9": 0.02295918367346939, "confidence_bias": 0.0035188706387541124, "top_bins": { "0.9": { "n": 392, "errors": 9, "error_rate": 0.02295918367346939 }, "0.95": { "n": 334, "errors": 2, "error_rate": 0.005988023952095809 }, "0.99": { "n": 118, "errors": 1, "error_rate": 0.00847457627118644 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9916666666666667, "confidence_cutoff": 0.9755983279973393 }, "0.8": { "coverage": 0.8, "accuracy": 0.9765625, "confidence_cutoff": 0.9172471488342538 } } }, "score": { "n": 240, "nll": 0.9412321202578248, "acc": 0.625, "ece": 0.048867889504322116, "brier": 0.519936949533565, "mean_conf": 0.6310840847095708, "confident_error_rate": 0.004166666666666667, "coverage_at_0_9": 0.029166666666666667, "accuracy_at_0_9": 0.8571428571428571, "coverage_at_5pct_error": 0.016666666666666666, "coverage_at_1pct_error": 0.016666666666666666, "aurc": 0.2774440601733873, "error_rate_at_0_9": 0.14285714285714285, "confidence_bias": 0.006084084709570781, "top_bins": { "0.9": { "n": 7, "errors": 1, "error_rate": 0.14285714285714285 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7166666666666667, "confidence_cutoff": 0.6329725841576003 }, "0.8": { "coverage": 0.8, "accuracy": 0.6614583333333334, "confidence_cutoff": 0.4922131178494099 } }, "score_mae": 0.496047231607107, "ranked_probability_score": 0.07900573734715889 } } }, "transfer-v4": { "raw_clean": { "n": 656, "nll": 0.5640777093137941, "acc": 0.836890243902439, "ece": 0.10195634226245451, "brier": 0.25517441623108567, "mean_conf": 0.9301396868913653, "confident_error_rate": 0.06707317073170732, "coverage_at_0_9": 0.8094512195121951, "accuracy_at_0_9": 0.9171374764595104, "coverage_at_5pct_error": 0.6829268292682927, "coverage_at_1pct_error": 0.2820121951219512, "aurc": 0.043201879169005745, "error_rate_at_0_9": 0.08286252354048965, "confidence_bias": 0.09324944298892623, "top_bins": { "0.9": { "n": 531, "errors": 44, "error_rate": 0.08286252354048965 }, "0.95": { "n": 487, "errors": 33, "error_rate": 0.06776180698151951 }, "0.99": { "n": 406, "errors": 14, "error_rate": 0.034482758620689655 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9725609756097561, "confidence_cutoff": 0.9980492531927935 }, "0.8": { "coverage": 0.8003048780487805, "accuracy": 0.92, "confidence_cutoff": 0.9074193401309704 } }, "score_mae": 0.24004523406424622, "ranked_probability_score": 0.09767451376793074 }, "as_shipped_clean": { "n": 656, "nll": 0.42587070132965826, "acc": 0.836890243902439, "ece": 0.0314949650736429, "brier": 0.23187082308763576, "mean_conf": 0.844962353152835, "confident_error_rate": 0.018292682926829267, "coverage_at_0_9": 0.5853658536585366, "accuracy_at_0_9": 0.96875, "coverage_at_5pct_error": 0.6570121951219512, "coverage_at_1pct_error": 0.2530487804878049, "aurc": 0.04507835320830427, "error_rate_at_0_9": 0.03125, "confidence_bias": 0.00807210925039592, "top_bins": { "0.9": { "n": 384, "errors": 12, "error_rate": 0.03125 }, "0.95": { "n": 315, "errors": 8, "error_rate": 0.025396825396825397 }, "0.99": { "n": 79, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9725609756097561, "confidence_cutoff": 0.9441193658749547 }, "0.8": { "coverage": 0.8003048780487805, "accuracy": 0.9047619047619048, "confidence_cutoff": 0.693693819569072 } }, "score_mae": 0.27196301946029916, "ranked_probability_score": 0.0837200362891968 }, "sources": { "composition_holdout": { "n": 96, "nll": 0.29615290237046005, "acc": 0.875, "ece": 0.06938096946882913, "brier": 0.1808256349283982, "mean_conf": 0.9443809694688293, "confident_error_rate": 0.041666666666666664, "coverage_at_0_9": 0.8333333333333334, "accuracy_at_0_9": 0.95, "coverage_at_5pct_error": 0.8645833333333334, "coverage_at_1pct_error": 0.6666666666666666, "aurc": 0.021756148279610994, "error_rate_at_0_9": 0.05, "confidence_bias": 0.06938096946882932, "top_bins": { "0.9": { "n": 80, "errors": 4, "error_rate": 0.05 }, "0.95": { "n": 73, "errors": 3, "error_rate": 0.0410958904109589 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.980048039700624 }, "0.8": { "coverage": 0.8020833333333334, "accuracy": 0.948051948051948, "confidence_cutoff": 0.9379020298350527 } } }, "contrastive": { "n": 80, "nll": 0.2412487469898486, "acc": 0.875, "ece": 0.054181213136020756, "brier": 0.15115638127662953, "mean_conf": 0.9258459913436543, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.8125, "accuracy_at_0_9": 0.9846153846153847, "coverage_at_5pct_error": 0.875, "coverage_at_1pct_error": 0.7875, "aurc": 0.01404271679837261, "error_rate_at_0_9": 0.015384615384615385, "confidence_bias": 0.050845991343654307, "top_bins": { "0.9": { "n": 65, "errors": 1, "error_rate": 0.015384615384615385 }, "0.95": { "n": 62, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 26, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9886860005278774 }, "0.8": { "coverage": 0.8, "accuracy": 0.984375, "confidence_cutoff": 0.9353245409640992 } }, "score_mae": 0.27196301946029916, "ranked_probability_score": 0.0837200362891968 }, "emotion": { "n": 80, "nll": 1.0001023668251066, "acc": 0.6625, "ece": 0.14881697863795876, "brier": 0.4744803648425345, "mean_conf": 0.591235292513326, "confident_error_rate": 0.0, "coverage_at_0_9": 0.05, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.0625, "coverage_at_1pct_error": 0.0625, "aurc": 0.18702686566035498, "error_rate_at_0_9": 0.0, "confidence_bias": -0.07126470748667402, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.5874489256154185 }, "0.8": { "coverage": 0.8, "accuracy": 0.71875, "confidence_cutoff": 0.3855885748389788 } } }, "mmlu": { "n": 80, "nll": 0.777539674387988, "acc": 0.7, "ece": 0.13070084664599038, "brier": 0.40568344535429135, "mean_conf": 0.6636013764250539, "confident_error_rate": 0.0, "coverage_at_0_9": 0.175, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.2625, "coverage_at_1pct_error": 0.225, "aurc": 0.12322939721812473, "error_rate_at_0_9": 0.0, "confidence_bias": -0.036398623574946054, "top_bins": { "0.9": { "n": 14, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.6761349246582344 }, "0.8": { "coverage": 0.8, "accuracy": 0.796875, "confidence_cutoff": 0.4466846683613359 } } }, "paws": { "n": 80, "nll": 0.512348189940992, "acc": 0.8125, "ece": 0.10522416975834781, "brier": 0.2914852770101592, "mean_conf": 0.9091572173256038, "confident_error_rate": 0.0875, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 0.8833333333333333, "coverage_at_5pct_error": 0.2625, "coverage_at_1pct_error": 0.2, "aurc": 0.09484078957534384, "error_rate_at_0_9": 0.11666666666666667, "confidence_bias": 0.09665721732560384, "top_bins": { "0.9": { "n": 60, "errors": 7, "error_rate": 0.11666666666666667 }, "0.95": { "n": 51, "errors": 5, "error_rate": 0.09803921568627451 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.875, "confidence_cutoff": 0.974944024208843 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.8613523629912301 } } }, "qnli": { "n": 80, "nll": 0.20826182445785202, "acc": 0.9125, "ece": 0.04839798205004245, "brier": 0.12797478277700258, "mean_conf": 0.9090029243294062, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8875, "coverage_at_1pct_error": 0.7, "aurc": 0.014269660330833212, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0034970756705937633, "top_bins": { "0.9": { "n": 56, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 45, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9595371830584098 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.8535858138327058 } } }, "sciq": { "n": 80, "nll": 0.04197126388379959, "acc": 1.0, "ece": 0.03774145345553605, "brier": 0.010730106398852924, "mean_conf": 0.962258546544464, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.037741453455536034, "top_bins": { "0.9": { "n": 72, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 69, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 36, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9883715126062644 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.9713861020562242 } } }, "tweet_offensive": { "n": 80, "nll": 0.3552842015730585, "acc": 0.85, "ece": 0.056220131039951354, "brier": 0.22283962974506544, "mean_conf": 0.8343327840091443, "confident_error_rate": 0.0, "coverage_at_0_9": 0.4125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.525, "aurc": 0.045801042430234855, "error_rate_at_0_9": 0.0, "confidence_bias": -0.015667215990855632, "top_bins": { "0.9": { "n": 33, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.8768513333328267 }, "0.8": { "coverage": 0.8, "accuracy": 0.890625, "confidence_cutoff": 0.7493184208869934 } } } }, "types": { "choice": { "n": 336, "nll": 0.5178563780812967, "acc": 0.8125, "ece": 0.04292515250441144, "brier": 0.26378206631184675, "mean_conf": 0.797702947344152, "confident_error_rate": 0.011904761904761904, "coverage_at_0_9": 0.5059523809523809, "accuracy_at_0_9": 0.9764705882352941, "coverage_at_5pct_error": 0.6130952380952381, "coverage_at_1pct_error": 0.40773809523809523, "aurc": 0.04890439545080765, "error_rate_at_0_9": 0.023529411764705882, "confidence_bias": -0.014797052655847964, "top_bins": { "0.9": { "n": 170, "errors": 4, "error_rate": 0.023529411764705882 }, "0.95": { "n": 151, "errors": 3, "error_rate": 0.019867549668874173 }, "0.99": { "n": 39, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9761904761904762, "confidence_cutoff": 0.9209057199846711 }, "0.8": { "coverage": 0.8005952380952381, "accuracy": 0.8996282527881041, "confidence_cutoff": 0.5765734433053208 } } }, "noul": { "n": 280, "nll": 0.3088148767031123, "acc": 0.8785714285714286, "ece": 0.02716426081477808, "brier": 0.18354251467515595, "mean_conf": 0.8993028632705061, "confident_error_rate": 0.025, "coverage_at_0_9": 0.675, "accuracy_at_0_9": 0.9629629629629629, "coverage_at_5pct_error": 0.7428571428571429, "coverage_at_1pct_error": 0.24642857142857144, "aurc": 0.04037503899248962, "error_rate_at_0_9": 0.037037037037037035, "confidence_bias": 0.020731434699077567, "top_bins": { "0.9": { "n": 189, "errors": 7, "error_rate": 0.037037037037037035 }, "0.95": { "n": 142, "errors": 5, "error_rate": 0.035211267605633804 }, "0.99": { "n": 35, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9642857142857143, "confidence_cutoff": 0.951430429315972 }, "0.8": { "coverage": 0.8, "accuracy": 0.9330357142857143, "confidence_cutoff": 0.8169286372799969 } } }, "score": { "n": 40, "nll": 0.47258178900171605, "acc": 0.75, "ece": 0.11341449184915596, "brier": 0.3021145388916218, "mean_conf": 0.8615577911220736, "confident_error_rate": 0.025, "coverage_at_0_9": 0.625, "accuracy_at_0_9": 0.96, "coverage_at_5pct_error": 0.65, "coverage_at_1pct_error": 0.575, "aurc": 0.060906503666048904, "error_rate_at_0_9": 0.04, "confidence_bias": 0.11155779112207365, "top_bins": { "0.9": { "n": 25, "errors": 1, "error_rate": 0.04 }, "0.95": { "n": 22, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.9618942423403596 }, "0.8": { "coverage": 0.8, "accuracy": 0.84375, "confidence_cutoff": 0.7283845543861389 } }, "score_mae": 0.27196301946029916, "ranked_probability_score": 0.0837200362891968 } } } }, "base_revision": "1001bb4d826a52d1f399e183466143f4da7b741b", "test_used_for_selection": false }