| { |
| "selected_epoch": 3, |
| "epochs": [ |
| { |
| "epoch": 1, |
| "development": { |
| "raw": { |
| "questions": 14912, |
| "accuracy": 0.8959898068669528, |
| "mean_class_recall": 0.8086847041598828, |
| "nll": 0.2676199905412643, |
| "brier": 0.14581168647550916, |
| "ece15": 0.024724176262894793, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.773806330472103, |
| "error_rate_at_confidence_0_9": 0.0240055464078343, |
| "ordinal_expected_level_mae": 0.00949861612167232, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 222, |
| "accuracy": 0.9819819819819819, |
| "mean_class_recall": 0.9670930232558139, |
| "nll": 0.06952124509948641, |
| "brier": 0.033360936223710204, |
| "ece15": 0.01963763286720348, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9774774774774775, |
| "error_rate_at_confidence_0_9": 0.013824884792626729, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 222, |
| "accuracy": 0.9864864864864865, |
| "mean_class_recall": 0.97, |
| "nll": 0.06796588080667874, |
| "brier": 0.02442578405456454, |
| "ece15": 0.013534851962772593, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.972972972972973, |
| "error_rate_at_confidence_0_9": 0.009259259259259259, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 1400, |
| "accuracy": 0.9742857142857143, |
| "mean_class_recall": 0.9504464285714286, |
| "nll": 0.07440929350732509, |
| "brier": 0.03861493679124129, |
| "ece15": 0.01061565077999396, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.925, |
| "error_rate_at_confidence_0_9": 0.008494208494208495, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 5000, |
| "accuracy": 0.8334, |
| "mean_class_recall": 0.7313333333333334, |
| "nll": 0.4357438451410003, |
| "brier": 0.2420765591030422, |
| "ece15": 0.05574236954650583, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.644, |
| "error_rate_at_confidence_0_9": 0.06677018633540373, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.004390672226438891, |
| "brier": 0.0015490359962901127, |
| "ece15": 0.003915493134373699, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9919354838709677, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.004780661486616687, |
| "brier": 0.0026501860965206722, |
| "ece15": 0.0038321931288341792, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9908536585365854, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 2.8436047490948972e-05, |
| "brier": 2.213459113159646e-07, |
| "ece15": 2.8380504783511674e-05, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 2500, |
| "accuracy": 0.9832, |
| "mean_class_recall": 0.9856, |
| "nll": 0.0716936772308886, |
| "brier": 0.032718644101254024, |
| "ece15": 0.01640596644430267, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9948, |
| "error_rate_at_confidence_0_9": 0.01648572577402493, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 797, |
| "accuracy": 0.9937264742785445, |
| "mean_class_recall": 0.9934913409721132, |
| "nll": 0.0153080496936535, |
| "brier": 0.009627399097334195, |
| "ece15": 0.004599932218083785, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9912170639899623, |
| "error_rate_at_confidence_0_9": 0.0037974683544303796, |
| "ordinal_expected_level_mae": 0.00949861612167232 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 797, |
| "accuracy": 0.9924717691342535, |
| "mean_class_recall": 0.9923419156847569, |
| "nll": 0.017883068917921716, |
| "brier": 0.011294880553062277, |
| "ece15": 0.0038184765498114437, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9811794228356336, |
| "error_rate_at_confidence_0_9": 0.0012787723785166241, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 797, |
| "accuracy": 0.9962358845671268, |
| "mean_class_recall": 0.995834315034425, |
| "nll": 0.011957269820791244, |
| "brier": 0.007393224542639707, |
| "ece15": 0.00670405372457154, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9836888331242158, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 797, |
| "accuracy": 0.9974905897114178, |
| "mean_class_recall": 0.9969442527413541, |
| "nll": 0.010437285979484219, |
| "brier": 0.0038211519865233618, |
| "ece15": 0.0009184477173700703, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9962358845671268, |
| "error_rate_at_confidence_0_9": 0.0012594458438287153, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 800, |
| "accuracy": 0.63, |
| "mean_class_recall": 0.6354549163606691, |
| "nll": 0.5876157448193948, |
| "brier": 0.4142276501377162, |
| "ece15": 0.031917999077751745, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.10375, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 800, |
| "accuracy": 0.59875, |
| "mean_class_recall": 0.4115791153670771, |
| "nll": 1.2269806519983115, |
| "brier": 0.571523062943736, |
| "ece15": 0.08060690424281568, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.11875, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9262877786032521, |
| "macro_task_balanced_accuracy": 0.9021513315229265 |
| }, |
| "recalibrated": { |
| "questions": 14912, |
| "accuracy": 0.8959898068669528, |
| "mean_class_recall": 0.8086847041598828, |
| "nll": 0.2562444405938866, |
| "brier": 0.1441282781781607, |
| "ece15": 0.016168446150980925, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.7631437768240343, |
| "error_rate_at_confidence_0_9": 0.020913884007029877, |
| "ordinal_expected_level_mae": 0.010086743500138846, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 222, |
| "accuracy": 0.9819819819819819, |
| "mean_class_recall": 0.9670930232558139, |
| "nll": 0.062064251368360995, |
| "brier": 0.03274641091592082, |
| "ece15": 0.018974706155035728, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.9774774774774775, |
| "error_rate_at_confidence_0_9": 0.013824884792626729, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 222, |
| "accuracy": 0.9864864864864865, |
| "mean_class_recall": 0.97, |
| "nll": 0.06063186595056717, |
| "brier": 0.02487828836064738, |
| "ece15": 0.013603842355616328, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.972972972972973, |
| "error_rate_at_confidence_0_9": 0.009259259259259259, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 1400, |
| "accuracy": 0.9742857142857143, |
| "mean_class_recall": 0.9504464285714286, |
| "nll": 0.08099031193469938, |
| "brier": 0.03952213251707266, |
| "ece15": 0.021719508126191233, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.9007142857142857, |
| "error_rate_at_confidence_0_9": 0.007137192704203013, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 5000, |
| "accuracy": 0.8334, |
| "mean_class_recall": 0.7313333333333334, |
| "nll": 0.4055190039113824, |
| "brier": 0.23828084117194667, |
| "ece15": 0.047374338949895636, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.6276, |
| "error_rate_at_confidence_0_9": 0.058954748247291265, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0058641097080354134, |
| "brier": 0.002007000102514143, |
| "ece15": 0.0052393541355612895, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.9838709677419355, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.005681917468602561, |
| "brier": 0.002881636759121932, |
| "ece15": 0.004649618743350316, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.9908536585365854, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 7.967277164193898e-05, |
| "brier": 1.0603152088487418e-06, |
| "ece15": 7.940563040823767e-05, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 2500, |
| "accuracy": 0.9832, |
| "mean_class_recall": 0.9856, |
| "nll": 0.06283986398882055, |
| "brier": 0.031969827800453, |
| "ece15": 0.015381335792818867, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.992, |
| "error_rate_at_confidence_0_9": 0.014516129032258065, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 797, |
| "accuracy": 0.9937264742785445, |
| "mean_class_recall": 0.9934913409721132, |
| "nll": 0.014979010411112437, |
| "brier": 0.009232024594375448, |
| "ece15": 0.0059751389669031076, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.986198243412798, |
| "error_rate_at_confidence_0_9": 0.001272264631043257, |
| "ordinal_expected_level_mae": 0.010086743500138846 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 797, |
| "accuracy": 0.9924717691342535, |
| "mean_class_recall": 0.9923419156847569, |
| "nll": 0.017946444333002753, |
| "brier": 0.011016825969457151, |
| "ece15": 0.005758410373299677, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.9799247176913425, |
| "error_rate_at_confidence_0_9": 0.0012804097311139564, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 797, |
| "accuracy": 0.9962358845671268, |
| "mean_class_recall": 0.995834315034425, |
| "nll": 0.012786451464898273, |
| "brier": 0.00742639513463676, |
| "ece15": 0.007511975455200797, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.9824341279799247, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 797, |
| "accuracy": 0.9974905897114178, |
| "mean_class_recall": 0.9969442527413541, |
| "nll": 0.010010236373401123, |
| "brier": 0.003879830332877274, |
| "ece15": 0.0006442096172233073, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.9962358845671268, |
| "error_rate_at_confidence_0_9": 0.0012594458438287153, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 800, |
| "accuracy": 0.63, |
| "mean_class_recall": 0.6354549163606691, |
| "nll": 0.5899608657261348, |
| "brier": 0.41478182050444184, |
| "ece15": 0.02797725458606776, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.095, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 800, |
| "accuracy": 0.59875, |
| "mean_class_recall": 0.4115791153670771, |
| "nll": 1.22100156554001, |
| "brier": 0.5645239942686594, |
| "ece15": 0.03015245360912201, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.09125, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9262877786032521, |
| "macro_task_balanced_accuracy": 0.9021513315229265 |
| } |
| }, |
| "test": { |
| "raw": { |
| "questions": 21634, |
| "accuracy": 0.8587870943884626, |
| "mean_class_recall": 0.8070634668409765, |
| "nll": 0.38882363258516517, |
| "brier": 0.19821716542849313, |
| "ece15": 0.010114739193927335, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.6294259036701488, |
| "error_rate_at_confidence_0_9": 0.016596900932657706, |
| "ordinal_expected_level_mae": 0.010228305213524986, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 426, |
| "accuracy": 0.9953051643192489, |
| "mean_class_recall": 0.9895833333333333, |
| "nll": 0.025162438436180455, |
| "brier": 0.012711490628916015, |
| "ece15": 0.009157764610888798, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9694835680751174, |
| "error_rate_at_confidence_0_9": 0.002421307506053269, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 426, |
| "accuracy": 0.9835680751173709, |
| "mean_class_recall": 0.9635416666666667, |
| "nll": 0.039729487219800165, |
| "brier": 0.024078530771277282, |
| "ece15": 0.008718119493403046, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9765258215962441, |
| "error_rate_at_confidence_0_9": 0.007211538461538462, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 800, |
| "accuracy": 0.93875, |
| "mean_class_recall": 0.8984375, |
| "nll": 0.19453941595728141, |
| "brier": 0.09689065974959639, |
| "ece15": 0.027271759841056658, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.87875, |
| "error_rate_at_confidence_0_9": 0.034139402560455195, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 6000, |
| "accuracy": 0.9085, |
| "mean_class_recall": 0.8498888888888889, |
| "nll": 0.30026072378768554, |
| "brier": 0.1436264364242417, |
| "ece15": 0.037464827151113794, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.6986666666666667, |
| "error_rate_at_confidence_0_9": 0.029818702290076337, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0009360610136724553, |
| "brier": 5.531806653087891e-05, |
| "ece15": 0.0009217275317422124, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 540, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0029619998489780613, |
| "brier": 0.0010650062308243538, |
| "ece15": 0.0026318040347967494, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9925925925925926, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 540, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 4.009888988278539e-05, |
| "brier": 8.791167963328194e-07, |
| "ece15": 3.98768557623178e-05, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 3500, |
| "accuracy": 0.9882857142857143, |
| "mean_class_recall": 0.9807619047619047, |
| "nll": 0.0658259406128984, |
| "brier": 0.02104448067811733, |
| "ece15": 0.0069241929022703305, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9845714285714285, |
| "error_rate_at_confidence_0_9": 0.00783517121300058, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 659, |
| "accuracy": 0.9908952959028832, |
| "mean_class_recall": 0.9899995024628091, |
| "nll": 0.022081750206618136, |
| "brier": 0.01158884274796682, |
| "ece15": 0.006288646106945627, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9954476479514416, |
| "error_rate_at_confidence_0_9": 0.004573170731707317, |
| "ordinal_expected_level_mae": 0.010228305213524986 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 659, |
| "accuracy": 0.9939301972685888, |
| "mean_class_recall": 0.9932998324958123, |
| "nll": 0.017255345447197507, |
| "brier": 0.008938281239516815, |
| "ece15": 0.004558085120676935, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9893778452200304, |
| "error_rate_at_confidence_0_9": 0.003067484662576687, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 659, |
| "accuracy": 0.9969650986342944, |
| "mean_class_recall": 0.9966867772750125, |
| "nll": 0.005807395652175951, |
| "brier": 0.0034083401253427324, |
| "ece15": 0.0031822580740001574, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9893778452200304, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 659, |
| "accuracy": 0.9969650986342944, |
| "mean_class_recall": 0.9949748743718593, |
| "nll": 0.009270762829542105, |
| "brier": 0.00610585714141352, |
| "ece15": 0.0051900114831071945, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9878603945371776, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 3321, |
| "accuracy": 0.6546221017765733, |
| "mean_class_recall": 0.6604367829458773, |
| "nll": 0.6040832398786956, |
| "brier": 0.4230681455327693, |
| "ece15": 0.012144611565586284, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.08220415537488708, |
| "error_rate_at_confidence_0_9": 0.014652014652014652, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 3321, |
| "accuracy": 0.6248118036735922, |
| "mean_class_recall": 0.3856953745435135, |
| "nll": 1.250473514719575, |
| "brier": 0.5523161094538271, |
| "ece15": 0.04328702573140643, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.1093044263775971, |
| "error_rate_at_confidence_0_9": 0.10192837465564739, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9337570392580401, |
| "macro_task_balanced_accuracy": 0.9073790312675484 |
| }, |
| "recalibrated": { |
| "questions": 21634, |
| "accuracy": 0.8587870943884626, |
| "mean_class_recall": 0.8070634668409765, |
| "nll": 0.3819588765605003, |
| "brier": 0.1988268469084605, |
| "ece15": 0.019530087212558746, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.6154663954885828, |
| "error_rate_at_confidence_0_9": 0.014344723995493804, |
| "ordinal_expected_level_mae": 0.01072047331602824, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 426, |
| "accuracy": 0.9953051643192489, |
| "mean_class_recall": 0.9895833333333333, |
| "nll": 0.028102839471687808, |
| "brier": 0.013352375860260531, |
| "ece15": 0.011924724016462706, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.9624413145539906, |
| "error_rate_at_confidence_0_9": 0.0024390243902439024, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 426, |
| "accuracy": 0.9835680751173709, |
| "mean_class_recall": 0.9635416666666667, |
| "nll": 0.03830754803404818, |
| "brier": 0.02344796092548543, |
| "ece15": 0.009297528928361507, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.9694835680751174, |
| "error_rate_at_confidence_0_9": 0.004842615012106538, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 800, |
| "accuracy": 0.93875, |
| "mean_class_recall": 0.8984375, |
| "nll": 0.18853618602333774, |
| "brier": 0.09702707273078204, |
| "ece15": 0.01936307032837748, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.84125, |
| "error_rate_at_confidence_0_9": 0.03417533432392273, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 6000, |
| "accuracy": 0.9085, |
| "mean_class_recall": 0.8498888888888889, |
| "nll": 0.2843389317505148, |
| "brier": 0.14586901269531594, |
| "ece15": 0.04356789248344108, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.6861666666666667, |
| "error_rate_at_confidence_0_9": 0.028175856205975226, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0018322563352568268, |
| "brier": 0.0001319762188887517, |
| "ece15": 0.0017975613028342768, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 540, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.004015391153194959, |
| "brier": 0.0013677195263880856, |
| "ece15": 0.0035859188244034786, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.9888888888888889, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 540, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 9.895528751872936e-05, |
| "brier": 3.024028962806258e-06, |
| "ece15": 9.818509110354407e-05, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 3500, |
| "accuracy": 0.9882857142857143, |
| "mean_class_recall": 0.9807619047619047, |
| "nll": 0.059841713174448447, |
| "brier": 0.021127495151853498, |
| "ece15": 0.005242220550012023, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.976, |
| "error_rate_at_confidence_0_9": 0.007318501170960188, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 659, |
| "accuracy": 0.9908952959028832, |
| "mean_class_recall": 0.9899995024628091, |
| "nll": 0.020306967381801818, |
| "brier": 0.011159005956953514, |
| "ece15": 0.005337269869592108, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.9908952959028832, |
| "error_rate_at_confidence_0_9": 0.0030627871362940277, |
| "ordinal_expected_level_mae": 0.01072047331602824 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 659, |
| "accuracy": 0.9939301972685888, |
| "mean_class_recall": 0.9932998324958123, |
| "nll": 0.016710459187929856, |
| "brier": 0.008886576198762068, |
| "ece15": 0.0032207054419096, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.9893778452200304, |
| "error_rate_at_confidence_0_9": 0.003067484662576687, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 659, |
| "accuracy": 0.9969650986342944, |
| "mean_class_recall": 0.9966867772750125, |
| "nll": 0.006802759775612477, |
| "brier": 0.003478846491952278, |
| "ece15": 0.004168012801242965, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.9893778452200304, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 659, |
| "accuracy": 0.9969650986342944, |
| "mean_class_recall": 0.9949748743718593, |
| "nll": 0.009405080434222151, |
| "brier": 0.006029858296029803, |
| "ece15": 0.004873038962556261, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.9863429438543247, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 3321, |
| "accuracy": 0.6546221017765733, |
| "mean_class_recall": 0.6604367829458773, |
| "nll": 0.6050451348525548, |
| "brier": 0.4236393366516397, |
| "ece15": 0.022216352122328073, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.0671484492622704, |
| "error_rate_at_confidence_0_9": 0.008968609865470852, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 3321, |
| "accuracy": 0.6248118036735922, |
| "mean_class_recall": 0.3856953745435135, |
| "nll": 1.2411381163756228, |
| "brier": 0.5515874786124301, |
| "ece15": 0.032556668330635724, |
| "temperature": 1.1759291426711453, |
| "coverage_at_confidence_0_9": 0.07768744354110207, |
| "error_rate_at_confidence_0_9": 0.06976744186046512, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9337570392580401, |
| "macro_task_balanced_accuracy": 0.9073790312675484 |
| } |
| }, |
| "general": { |
| "decision-v7": { |
| "raw_clean": { |
| "n": 1200, |
| "nll": 0.5075688394241022, |
| "acc": 0.8666666666666667, |
| "ece": 0.07962904500464703, |
| "brier": 0.2102660845468813, |
| "mean_conf": 0.9440260985257297, |
| "confident_error_rate": 0.0575, |
| "coverage_at_0_9": 0.8266666666666667, |
| "accuracy_at_0_9": 0.9304435483870968, |
| "coverage_at_5pct_error": 0.7891666666666667, |
| "coverage_at_1pct_error": 0.6058333333333333, |
| "aurc": 0.028250531410146766, |
| "error_rate_at_0_9": 0.06955645161290322, |
| "confidence_bias": 0.07735943185906302, |
| "top_bins": { |
| "0.9": { |
| "n": 992, |
| "errors": 69, |
| "error_rate": 0.06955645161290322 |
| }, |
| "0.95": { |
| "n": 931, |
| "errors": 44, |
| "error_rate": 0.047261009667024706 |
| }, |
| "0.99": { |
| "n": 853, |
| "errors": 25, |
| "error_rate": 0.029308323563892145 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9916666666666667, |
| "confidence_cutoff": 0.9999431212544798 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9458333333333333, |
| "confidence_cutoff": 0.9297460863402224 |
| } |
| }, |
| "score_mae": 0.4607497712522395, |
| "ranked_probability_score": 0.08554771406167339 |
| }, |
| "as_shipped_clean": { |
| "n": 1200, |
| "nll": 0.454544651852342, |
| "acc": 0.8666666666666667, |
| "ece": 0.06910317787787193, |
| "brier": 0.20468221280246773, |
| "mean_conf": 0.9344856098346994, |
| "confident_error_rate": 0.041666666666666664, |
| "coverage_at_0_9": 0.795, |
| "accuracy_at_0_9": 0.9475890985324947, |
| "coverage_at_5pct_error": 0.7866666666666666, |
| "coverage_at_1pct_error": 0.6041666666666666, |
| "aurc": 0.02824430039449196, |
| "error_rate_at_0_9": 0.05241090146750524, |
| "confidence_bias": 0.06781894316803272, |
| "top_bins": { |
| "0.9": { |
| "n": 954, |
| "errors": 50, |
| "error_rate": 0.05241090146750524 |
| }, |
| "0.95": { |
| "n": 901, |
| "errors": 37, |
| "error_rate": 0.041065482796892344 |
| }, |
| "0.99": { |
| "n": 820, |
| "errors": 16, |
| "error_rate": 0.01951219512195122 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9916666666666667, |
| "confidence_cutoff": 0.9997511018421836 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9458333333333333, |
| "confidence_cutoff": 0.894833765474474 |
| } |
| }, |
| "score_mae": 0.46716677242673305, |
| "ranked_probability_score": 0.08353281985680432 |
| }, |
| "sources": { |
| "agnews": { |
| "n": 240, |
| "nll": 0.2969571491022943, |
| "acc": 0.9166666666666666, |
| "ece": 0.05669702215129781, |
| "brier": 0.13610080496920665, |
| "mean_conf": 0.9733636888179642, |
| "confident_error_rate": 0.04583333333333333, |
| "coverage_at_0_9": 0.9291666666666667, |
| "accuracy_at_0_9": 0.9506726457399103, |
| "coverage_at_5pct_error": 0.9291666666666667, |
| "coverage_at_1pct_error": 0.7333333333333333, |
| "aurc": 0.011459961623908478, |
| "error_rate_at_0_9": 0.04932735426008968, |
| "confidence_bias": 0.05669702215129757, |
| "top_bins": { |
| "0.9": { |
| "n": 223, |
| "errors": 11, |
| "error_rate": 0.04932735426008968 |
| }, |
| "0.95": { |
| "n": 216, |
| "errors": 10, |
| "error_rate": 0.046296296296296294 |
| }, |
| "0.99": { |
| "n": 197, |
| "errors": 4, |
| "error_rate": 0.02030456852791878 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.999603972973965 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9927218344770156 |
| } |
| } |
| }, |
| "amazon": { |
| "n": 80, |
| "nll": 1.1950196702821447, |
| "acc": 0.6125, |
| "ece": 0.13013344623625947, |
| "brier": 0.5588490719296325, |
| "mean_conf": 0.7426334462362595, |
| "confident_error_rate": 0.0625, |
| "coverage_at_0_9": 0.2625, |
| "accuracy_at_0_9": 0.7619047619047619, |
| "coverage_at_5pct_error": 0.075, |
| "coverage_at_1pct_error": 0.075, |
| "aurc": 0.2609215557981968, |
| "error_rate_at_0_9": 0.23809523809523808, |
| "confidence_bias": 0.13013344623625944, |
| "top_bins": { |
| "0.9": { |
| "n": 21, |
| "errors": 5, |
| "error_rate": 0.23809523809523808 |
| }, |
| "0.95": { |
| "n": 13, |
| "errors": 2, |
| "error_rate": 0.15384615384615385 |
| }, |
| "0.99": { |
| "n": 4, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.775, |
| "confidence_cutoff": 0.7941759769280257 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.65625, |
| "confidence_cutoff": 0.5659829933652708 |
| } |
| }, |
| "score_mae": 0.4892758257511954, |
| "ranked_probability_score": 0.08858888411388968 |
| }, |
| "banking77": { |
| "n": 80, |
| "nll": 0.5758902612345038, |
| "acc": 0.85, |
| "ece": 0.08719569564847485, |
| "brier": 0.2407008008106888, |
| "mean_conf": 0.898073693405655, |
| "confident_error_rate": 0.0375, |
| "coverage_at_0_9": 0.75, |
| "accuracy_at_0_9": 0.95, |
| "coverage_at_5pct_error": 0.7625, |
| "coverage_at_1pct_error": 0.625, |
| "aurc": 0.03238808988486375, |
| "error_rate_at_0_9": 0.05, |
| "confidence_bias": 0.04807369340565504, |
| "top_bins": { |
| "0.9": { |
| "n": 60, |
| "errors": 3, |
| "error_rate": 0.05 |
| }, |
| "0.95": { |
| "n": 55, |
| "errors": 2, |
| "error_rate": 0.03636363636363636 |
| }, |
| "0.99": { |
| "n": 47, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9967902498867545 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.90625, |
| "confidence_cutoff": 0.8441430830009494 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 80, |
| "nll": 0.5929837671372773, |
| "acc": 0.875, |
| "ece": 0.12193170422191033, |
| "brier": 0.2235795473979305, |
| "mean_conf": 0.9525887301568842, |
| "confident_error_rate": 0.0875, |
| "coverage_at_0_9": 0.85, |
| "accuracy_at_0_9": 0.8970588235294118, |
| "coverage_at_5pct_error": 0.6875, |
| "coverage_at_1pct_error": 0.1, |
| "aurc": 0.05282386578594207, |
| "error_rate_at_0_9": 0.10294117647058823, |
| "confidence_bias": 0.07758873015688417, |
| "top_bins": { |
| "0.9": { |
| "n": 68, |
| "errors": 7, |
| "error_rate": 0.10294117647058823 |
| }, |
| "0.95": { |
| "n": 67, |
| "errors": 7, |
| "error_rate": 0.1044776119402985 |
| }, |
| "0.99": { |
| "n": 61, |
| "errors": 4, |
| "error_rate": 0.06557377049180328 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.999911295185737 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.921875, |
| "confidence_cutoff": 0.9794008835333886 |
| } |
| } |
| }, |
| "contrastive": { |
| "n": 160, |
| "nll": 0.05996336403593442, |
| "acc": 0.96875, |
| "ece": 0.02417385494863131, |
| "brier": 0.04256566388357948, |
| "mean_conf": 0.9929238549486312, |
| "confident_error_rate": 0.0125, |
| "coverage_at_0_9": 0.98125, |
| "accuracy_at_0_9": 0.9872611464968153, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.975, |
| "aurc": 0.0005908980274374722, |
| "error_rate_at_0_9": 0.012738853503184714, |
| "confidence_bias": 0.024173854948631224, |
| "top_bins": { |
| "0.9": { |
| "n": 157, |
| "errors": 2, |
| "error_rate": 0.012738853503184714 |
| }, |
| "0.95": { |
| "n": 155, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 148, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.999992580361535 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9997815066400773 |
| } |
| } |
| }, |
| "dbpedia14": { |
| "n": 80, |
| "nll": 0.018296334655618373, |
| "acc": 0.9875, |
| "ece": 0.010929643397143786, |
| "brier": 0.013547082606835407, |
| "mean_conf": 0.9948529231687339, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9875, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.9875, |
| "aurc": 0.00015625, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.007352923168733816, |
| "top_bins": { |
| "0.9": { |
| "n": 79, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 77, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 77, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999968083760991 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.999980508233233 |
| } |
| } |
| }, |
| "imdb": { |
| "n": 80, |
| "nll": 0.06650112598274324, |
| "acc": 0.9625, |
| "ece": 0.021127312041626652, |
| "brier": 0.047311917135916516, |
| "mean_conf": 0.9836273120416268, |
| "confident_error_rate": 0.0125, |
| "coverage_at_0_9": 0.9625, |
| "accuracy_at_0_9": 0.987012987012987, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.95, |
| "aurc": 0.0012680561790530144, |
| "error_rate_at_0_9": 0.012987012987012988, |
| "confidence_bias": 0.021127312041626767, |
| "top_bins": { |
| "0.9": { |
| "n": 77, |
| "errors": 1, |
| "error_rate": 0.012987012987012988 |
| }, |
| "0.95": { |
| "n": 75, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 71, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999868991306584 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.999740450053584 |
| } |
| } |
| }, |
| "mnli": { |
| "n": 80, |
| "nll": 0.4800709042620615, |
| "acc": 0.9125, |
| "ece": 0.07753013500835038, |
| "brier": 0.16332522643400862, |
| "mean_conf": 0.9812306735501618, |
| "confident_error_rate": 0.0625, |
| "coverage_at_0_9": 0.925, |
| "accuracy_at_0_9": 0.9324324324324325, |
| "coverage_at_5pct_error": 0.8875, |
| "coverage_at_1pct_error": 0.6375, |
| "aurc": 0.016122981848020253, |
| "error_rate_at_0_9": 0.06756756756756757, |
| "confidence_bias": 0.06873067355016182, |
| "top_bins": { |
| "0.9": { |
| "n": 74, |
| "errors": 5, |
| "error_rate": 0.06756756756756757 |
| }, |
| "0.95": { |
| "n": 74, |
| "errors": 5, |
| "error_rate": 0.06756756756756757 |
| }, |
| "0.99": { |
| "n": 71, |
| "errors": 3, |
| "error_rate": 0.04225352112676056 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.999991586558766 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.96875, |
| "confidence_cutoff": 0.9998147211094783 |
| } |
| } |
| }, |
| "sst5": { |
| "n": 80, |
| "nll": 1.2448974820413372, |
| "acc": 0.6125, |
| "ece": 0.22505814089878196, |
| "brier": 0.5919475867934173, |
| "mean_conf": 0.8141449665678422, |
| "confident_error_rate": 0.0625, |
| "coverage_at_0_9": 0.2875, |
| "accuracy_at_0_9": 0.782608695652174, |
| "coverage_at_5pct_error": 0.0875, |
| "coverage_at_1pct_error": 0.0875, |
| "aurc": 0.30028061666521133, |
| "error_rate_at_0_9": 0.21739130434782608, |
| "confidence_bias": 0.20164496656784214, |
| "top_bins": { |
| "0.9": { |
| "n": 23, |
| "errors": 5, |
| "error_rate": 0.21739130434782608 |
| }, |
| "0.95": { |
| "n": 8, |
| "errors": 1, |
| "error_rate": 0.125 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.6, |
| "confidence_cutoff": 0.855339285968297 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.625, |
| "confidence_cutoff": 0.7133087914054365 |
| } |
| }, |
| "score_mae": 0.4884492482076522, |
| "ranked_probability_score": 0.08971730097335581 |
| }, |
| "trec": { |
| "n": 80, |
| "nll": 0.28480951015184036, |
| "acc": 0.9625, |
| "ece": 0.030856363792743223, |
| "brier": 0.058704890234289875, |
| "mean_conf": 0.9903723975747134, |
| "confident_error_rate": 0.025, |
| "coverage_at_0_9": 0.975, |
| "accuracy_at_0_9": 0.9743589743589743, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.325, |
| "aurc": 0.015881097479045377, |
| "error_rate_at_0_9": 0.02564102564102564, |
| "confidence_bias": 0.02787239757471338, |
| "top_bins": { |
| "0.9": { |
| "n": 78, |
| "errors": 2, |
| "error_rate": 0.02564102564102564 |
| }, |
| "0.95": { |
| "n": 78, |
| "errors": 2, |
| "error_rate": 0.02564102564102564 |
| }, |
| "0.99": { |
| "n": 77, |
| "errors": 2, |
| "error_rate": 0.025974025974025976 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.9999910962505757 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9999026419356177 |
| } |
| } |
| }, |
| "yelp": { |
| "n": 160, |
| "nll": 0.6744512733294261, |
| "acc": 0.76875, |
| "ece": 0.11010641434949675, |
| "brier": 0.33941666300975865, |
| "mean_conf": 0.8769106142337293, |
| "confident_error_rate": 0.05625, |
| "coverage_at_0_9": 0.5875, |
| "accuracy_at_0_9": 0.9042553191489362, |
| "coverage_at_5pct_error": 0.425, |
| "coverage_at_1pct_error": 0.16875, |
| "aurc": 0.08692888800847512, |
| "error_rate_at_0_9": 0.09574468085106383, |
| "confidence_bias": 0.10816061423372925, |
| "top_bins": { |
| "0.9": { |
| "n": 94, |
| "errors": 9, |
| "error_rate": 0.09574468085106383 |
| }, |
| "0.95": { |
| "n": 83, |
| "errors": 8, |
| "error_rate": 0.0963855421686747 |
| }, |
| "0.99": { |
| "n": 67, |
| "errors": 3, |
| "error_rate": 0.04477611940298507 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.925, |
| "confidence_cutoff": 0.9744081429606993 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.84375, |
| "confidence_cutoff": 0.7335309795017884 |
| } |
| }, |
| "score_mae": 0.4237752433213515, |
| "ranked_probability_score": 0.07229227448316741 |
| } |
| }, |
| "types": { |
| "choice": { |
| "n": 480, |
| "nll": 0.3107149818460764, |
| "acc": 0.93125, |
| "ece": 0.04989938310601072, |
| "brier": 0.11702630095906619, |
| "mean_conf": 0.9725961663793784, |
| "confident_error_rate": 0.03333333333333333, |
| "coverage_at_0_9": 0.925, |
| "accuracy_at_0_9": 0.963963963963964, |
| "coverage_at_5pct_error": 0.95, |
| "coverage_at_1pct_error": 0.8333333333333334, |
| "aurc": 0.011065724252754237, |
| "error_rate_at_0_9": 0.036036036036036036, |
| "confidence_bias": 0.04134616637937838, |
| "top_bins": { |
| "0.9": { |
| "n": 444, |
| "errors": 16, |
| "error_rate": 0.036036036036036036 |
| }, |
| "0.95": { |
| "n": 436, |
| "errors": 15, |
| "error_rate": 0.034403669724770644 |
| }, |
| "0.99": { |
| "n": 415, |
| "errors": 7, |
| "error_rate": 0.016867469879518072 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9958333333333333, |
| "confidence_cutoff": 0.9999830700690301 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9895833333333334, |
| "confidence_cutoff": 0.9987196152255894 |
| } |
| } |
| }, |
| "noul": { |
| "n": 480, |
| "nll": 0.2523830232779272, |
| "acc": 0.925, |
| "ece": 0.04746414517655202, |
| "brier": 0.115343937114107, |
| "mean_conf": 0.9724641451765519, |
| "confident_error_rate": 0.0375, |
| "coverage_at_0_9": 0.91875, |
| "accuracy_at_0_9": 0.9591836734693877, |
| "coverage_at_5pct_error": 0.94375, |
| "coverage_at_1pct_error": 0.7041666666666667, |
| "aurc": 0.015342193433417054, |
| "error_rate_at_0_9": 0.04081632653061224, |
| "confidence_bias": 0.04746414517655184, |
| "top_bins": { |
| "0.9": { |
| "n": 441, |
| "errors": 18, |
| "error_rate": 0.04081632653061224 |
| }, |
| "0.95": { |
| "n": 427, |
| "errors": 14, |
| "error_rate": 0.03278688524590164 |
| }, |
| "0.99": { |
| "n": 395, |
| "errors": 8, |
| "error_rate": 0.020253164556962026 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9958333333333333, |
| "confidence_cutoff": 0.9998511569646503 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9791666666666666, |
| "confidence_cutoff": 0.9945634588215609 |
| } |
| } |
| }, |
| "score": { |
| "n": 240, |
| "nll": 1.146527249013703, |
| "acc": 0.6208333333333333, |
| "ece": 0.16277510247546034, |
| "brier": 0.5586705878659922, |
| "mean_conf": 0.7823074260616364, |
| "confident_error_rate": 0.06666666666666667, |
| "coverage_at_0_9": 0.2875, |
| "accuracy_at_0_9": 0.7681159420289855, |
| "coverage_at_5pct_error": 0.016666666666666666, |
| "coverage_at_1pct_error": 0.016666666666666666, |
| "aurc": 0.2783513871168549, |
| "error_rate_at_0_9": 0.2318840579710145, |
| "confidence_bias": 0.16147409272830304, |
| "top_bins": { |
| "0.9": { |
| "n": 69, |
| "errors": 16, |
| "error_rate": 0.2318840579710145 |
| }, |
| "0.95": { |
| "n": 38, |
| "errors": 8, |
| "error_rate": 0.21052631578947367 |
| }, |
| "0.99": { |
| "n": 10, |
| "errors": 1, |
| "error_rate": 0.1 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.7166666666666667, |
| "confidence_cutoff": 0.8132439678266241 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.6614583333333334, |
| "confidence_cutoff": 0.6191827053278941 |
| } |
| }, |
| "score_mae": 0.46716677242673305, |
| "ranked_probability_score": 0.08353281985680432 |
| } |
| } |
| }, |
| "transfer-v4": { |
| "raw_clean": { |
| "n": 656, |
| "nll": 0.7060605455091327, |
| "acc": 0.8384146341463414, |
| "ece": 0.10225794465378485, |
| "brier": 0.26913580782629415, |
| "mean_conf": 0.936023112192006, |
| "confident_error_rate": 0.08689024390243902, |
| "coverage_at_0_9": 0.8246951219512195, |
| "accuracy_at_0_9": 0.8946395563770795, |
| "coverage_at_5pct_error": 0.5975609756097561, |
| "coverage_at_1pct_error": 0.38109756097560976, |
| "aurc": 0.04670337348873893, |
| "error_rate_at_0_9": 0.10536044362292052, |
| "confidence_bias": 0.09760847804566453, |
| "top_bins": { |
| "0.9": { |
| "n": 541, |
| "errors": 57, |
| "error_rate": 0.10536044362292052 |
| }, |
| "0.95": { |
| "n": 508, |
| "errors": 46, |
| "error_rate": 0.09055118110236221 |
| }, |
| "0.99": { |
| "n": 449, |
| "errors": 28, |
| "error_rate": 0.062360801781737196 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9695121951219512, |
| "confidence_cutoff": 0.9996513990871982 |
| }, |
| "0.8": { |
| "coverage": 0.8003048780487805, |
| "accuracy": 0.900952380952381, |
| "confidence_cutoff": 0.9260639080978317 |
| } |
| }, |
| "score_mae": 0.28732928759135146, |
| "ranked_probability_score": 0.12507059546399885 |
| }, |
| "as_shipped_clean": { |
| "n": 656, |
| "nll": 0.6270891070988179, |
| "acc": 0.8384146341463414, |
| "ece": 0.09035981505437185, |
| "brier": 0.26355975613405186, |
| "mean_conf": 0.9253405484079532, |
| "confident_error_rate": 0.07469512195121951, |
| "coverage_at_0_9": 0.7896341463414634, |
| "accuracy_at_0_9": 0.9054054054054054, |
| "coverage_at_5pct_error": 0.5960365853658537, |
| "coverage_at_1pct_error": 0.3795731707317073, |
| "aurc": 0.046713600417941994, |
| "error_rate_at_0_9": 0.0945945945945946, |
| "confidence_bias": 0.08692591426161178, |
| "top_bins": { |
| "0.9": { |
| "n": 518, |
| "errors": 49, |
| "error_rate": 0.0945945945945946 |
| }, |
| "0.95": { |
| "n": 494, |
| "errors": 44, |
| "error_rate": 0.08906882591093117 |
| }, |
| "0.99": { |
| "n": 413, |
| "errors": 23, |
| "error_rate": 0.05569007263922518 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9695121951219512, |
| "confidence_cutoff": 0.9988538098904463 |
| }, |
| "0.8": { |
| "coverage": 0.8003048780487805, |
| "accuracy": 0.900952380952381, |
| "confidence_cutoff": 0.8926384237014877 |
| } |
| }, |
| "score_mae": 0.28661481608517747, |
| "ranked_probability_score": 0.1219924564403267 |
| }, |
| "sources": { |
| "composition_holdout": { |
| "n": 96, |
| "nll": 0.7716759343811087, |
| "acc": 0.8854166666666666, |
| "ece": 0.13161705342878371, |
| "brier": 0.2418445379901801, |
| "mean_conf": 0.9798888855966136, |
| "confident_error_rate": 0.11458333333333333, |
| "coverage_at_0_9": 0.9479166666666666, |
| "accuracy_at_0_9": 0.8791208791208791, |
| "coverage_at_5pct_error": 0.6875, |
| "coverage_at_1pct_error": 0.6666666666666666, |
| "aurc": 0.02787241712770423, |
| "error_rate_at_0_9": 0.12087912087912088, |
| "confidence_bias": 0.09447221892994695, |
| "top_bins": { |
| "0.9": { |
| "n": 91, |
| "errors": 11, |
| "error_rate": 0.12087912087912088 |
| }, |
| "0.95": { |
| "n": 90, |
| "errors": 11, |
| "error_rate": 0.12222222222222222 |
| }, |
| "0.99": { |
| "n": 86, |
| "errors": 11, |
| "error_rate": 0.12790697674418605 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.999944022537299 |
| }, |
| "0.8": { |
| "coverage": 0.8020833333333334, |
| "accuracy": 0.948051948051948, |
| "confidence_cutoff": 0.9994106055377326 |
| } |
| } |
| }, |
| "contrastive": { |
| "n": 80, |
| "nll": 0.5084316515078089, |
| "acc": 0.85, |
| "ece": 0.11914860470823861, |
| "brier": 0.24157018604381048, |
| "mean_conf": 0.9691486047082385, |
| "confident_error_rate": 0.0875, |
| "coverage_at_0_9": 0.9, |
| "accuracy_at_0_9": 0.9027777777777778, |
| "coverage_at_5pct_error": 0.85, |
| "coverage_at_1pct_error": 0.8, |
| "aurc": 0.018225744933534453, |
| "error_rate_at_0_9": 0.09722222222222222, |
| "confidence_bias": 0.11914860470823851, |
| "top_bins": { |
| "0.9": { |
| "n": 72, |
| "errors": 7, |
| "error_rate": 0.09722222222222222 |
| }, |
| "0.95": { |
| "n": 72, |
| "errors": 7, |
| "error_rate": 0.09722222222222222 |
| }, |
| "0.99": { |
| "n": 68, |
| "errors": 3, |
| "error_rate": 0.04411764705882353 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999840553223158 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9992294841723544 |
| } |
| }, |
| "score_mae": 0.28661481608517747, |
| "ranked_probability_score": 0.1219924564403267 |
| }, |
| "emotion": { |
| "n": 80, |
| "nll": 1.1842140096136198, |
| "acc": 0.6625, |
| "ece": 0.16344433654532525, |
| "brier": 0.49695454986128207, |
| "mean_conf": 0.7770470897943043, |
| "confident_error_rate": 0.075, |
| "coverage_at_0_9": 0.4375, |
| "accuracy_at_0_9": 0.8285714285714286, |
| "coverage_at_5pct_error": 0.075, |
| "coverage_at_1pct_error": 0.075, |
| "aurc": 0.18204891186299516, |
| "error_rate_at_0_9": 0.17142857142857143, |
| "confidence_bias": 0.11454708979430428, |
| "top_bins": { |
| "0.9": { |
| "n": 35, |
| "errors": 6, |
| "error_rate": 0.17142857142857143 |
| }, |
| "0.95": { |
| "n": 30, |
| "errors": 4, |
| "error_rate": 0.13333333333333333 |
| }, |
| "0.99": { |
| "n": 12, |
| "errors": 1, |
| "error_rate": 0.08333333333333333 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.8, |
| "confidence_cutoff": 0.8455177052439261 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.734375, |
| "confidence_cutoff": 0.5895563704278984 |
| } |
| } |
| }, |
| "mmlu": { |
| "n": 80, |
| "nll": 0.8811102729379066, |
| "acc": 0.7125, |
| "ece": 0.1128354245761854, |
| "brier": 0.4333401015309152, |
| "mean_conf": 0.8253354245761854, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.475, |
| "accuracy_at_0_9": 0.8947368421052632, |
| "coverage_at_5pct_error": 0.3625, |
| "coverage_at_1pct_error": 0.3125, |
| "aurc": 0.11284549915448887, |
| "error_rate_at_0_9": 0.10526315789473684, |
| "confidence_bias": 0.11283542457618534, |
| "top_bins": { |
| "0.9": { |
| "n": 38, |
| "errors": 4, |
| "error_rate": 0.10526315789473684 |
| }, |
| "0.95": { |
| "n": 32, |
| "errors": 2, |
| "error_rate": 0.0625 |
| }, |
| "0.99": { |
| "n": 19, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9, |
| "confidence_cutoff": 0.8934639334502172 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.796875, |
| "confidence_cutoff": 0.6529787827885164 |
| } |
| } |
| }, |
| "paws": { |
| "n": 80, |
| "nll": 0.9149710000217235, |
| "acc": 0.8, |
| "ece": 0.15742064884121132, |
| "brier": 0.3163214505434944, |
| "mean_conf": 0.9481862302428528, |
| "confident_error_rate": 0.125, |
| "coverage_at_0_9": 0.8625, |
| "accuracy_at_0_9": 0.855072463768116, |
| "coverage_at_5pct_error": 0.275, |
| "coverage_at_1pct_error": 0.175, |
| "aurc": 0.09027125896066751, |
| "error_rate_at_0_9": 0.14492753623188406, |
| "confidence_bias": 0.1481862302428527, |
| "top_bins": { |
| "0.9": { |
| "n": 69, |
| "errors": 10, |
| "error_rate": 0.14492753623188406 |
| }, |
| "0.95": { |
| "n": 69, |
| "errors": 10, |
| "error_rate": 0.14492753623188406 |
| }, |
| "0.99": { |
| "n": 61, |
| "errors": 6, |
| "error_rate": 0.09836065573770492 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9, |
| "confidence_cutoff": 0.9997018073599228 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.890625, |
| "confidence_cutoff": 0.9816976299326602 |
| } |
| } |
| }, |
| "qnli": { |
| "n": 80, |
| "nll": 0.3010750024235448, |
| "acc": 0.925, |
| "ece": 0.08174310999100654, |
| "brier": 0.13992055357407857, |
| "mean_conf": 0.9736296740922775, |
| "confident_error_rate": 0.0625, |
| "coverage_at_0_9": 0.925, |
| "accuracy_at_0_9": 0.9324324324324325, |
| "coverage_at_5pct_error": 0.8625, |
| "coverage_at_1pct_error": 0.625, |
| "aurc": 0.014993418347854443, |
| "error_rate_at_0_9": 0.06756756756756757, |
| "confidence_bias": 0.048629674092277475, |
| "top_bins": { |
| "0.9": { |
| "n": 74, |
| "errors": 5, |
| "error_rate": 0.06756756756756757 |
| }, |
| "0.95": { |
| "n": 71, |
| "errors": 4, |
| "error_rate": 0.056338028169014086 |
| }, |
| "0.99": { |
| "n": 64, |
| "errors": 2, |
| "error_rate": 0.03125 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9996265539820621 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.96875, |
| "confidence_cutoff": 0.9930251162586794 |
| } |
| } |
| }, |
| "sciq": { |
| "n": 80, |
| "nll": 0.007314621146129305, |
| "acc": 1.0, |
| "ece": 0.006816864281393114, |
| "brier": 0.0015733066903292278, |
| "mean_conf": 0.993183135718607, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.975, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0068168642813930536, |
| "top_bins": { |
| "0.9": { |
| "n": 78, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 77, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 72, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999608059905641 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9995954309046333 |
| } |
| } |
| }, |
| "tweet_offensive": { |
| "n": 80, |
| "nll": 0.4190029993022442, |
| "acc": 0.8625, |
| "ece": 0.08458362708432104, |
| "brier": 0.2412964064670991, |
| "mean_conf": 0.9253956750968154, |
| "confident_error_rate": 0.075, |
| "coverage_at_0_9": 0.7625, |
| "accuracy_at_0_9": 0.9016393442622951, |
| "coverage_at_5pct_error": 0.575, |
| "coverage_at_1pct_error": 0.5, |
| "aurc": 0.04870527959080802, |
| "error_rate_at_0_9": 0.09836065573770492, |
| "confidence_bias": 0.06289567509681537, |
| "top_bins": { |
| "0.9": { |
| "n": 61, |
| "errors": 6, |
| "error_rate": 0.09836065573770492 |
| }, |
| "0.95": { |
| "n": 53, |
| "errors": 6, |
| "error_rate": 0.11320754716981132 |
| }, |
| "0.99": { |
| "n": 31, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9833703279193756 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.890625, |
| "confidence_cutoff": 0.891081054907394 |
| } |
| } |
| } |
| }, |
| "types": { |
| "choice": { |
| "n": 336, |
| "nll": 0.7139642916559491, |
| "acc": 0.8184523809523809, |
| "ece": 0.08621197602515834, |
| "brier": 0.29097176277874826, |
| "mean_conf": 0.8979600744773888, |
| "confident_error_rate": 0.0625, |
| "coverage_at_0_9": 0.7202380952380952, |
| "accuracy_at_0_9": 0.9132231404958677, |
| "coverage_at_5pct_error": 0.5208333333333334, |
| "coverage_at_1pct_error": 0.3958333333333333, |
| "aurc": 0.05304090653277752, |
| "error_rate_at_0_9": 0.08677685950413223, |
| "confidence_bias": 0.07950769352500786, |
| "top_bins": { |
| "0.9": { |
| "n": 242, |
| "errors": 21, |
| "error_rate": 0.08677685950413223 |
| }, |
| "0.95": { |
| "n": 229, |
| "errors": 17, |
| "error_rate": 0.07423580786026202 |
| }, |
| "0.99": { |
| "n": 189, |
| "errors": 12, |
| "error_rate": 0.06349206349206349 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9702380952380952, |
| "confidence_cutoff": 0.9970506958588666 |
| }, |
| "0.8": { |
| "coverage": 0.8005952380952381, |
| "accuracy": 0.895910780669145, |
| "confidence_cutoff": 0.7834280359430882 |
| } |
| } |
| }, |
| "noul": { |
| "n": 280, |
| "nll": 0.4671583132891337, |
| "acc": 0.8821428571428571, |
| "ece": 0.07465765160771948, |
| "brier": 0.1992966887782935, |
| "mean_conf": 0.9563447099149558, |
| "confident_error_rate": 0.075, |
| "coverage_at_0_9": 0.8714285714285714, |
| "accuracy_at_0_9": 0.9139344262295082, |
| "coverage_at_5pct_error": 0.7642857142857142, |
| "coverage_at_1pct_error": 0.26785714285714285, |
| "aurc": 0.03774979623265823, |
| "error_rate_at_0_9": 0.0860655737704918, |
| "confidence_bias": 0.07420185277209868, |
| "top_bins": { |
| "0.9": { |
| "n": 244, |
| "errors": 21, |
| "error_rate": 0.0860655737704918 |
| }, |
| "0.95": { |
| "n": 233, |
| "errors": 20, |
| "error_rate": 0.08583690987124463 |
| }, |
| "0.99": { |
| "n": 196, |
| "errors": 8, |
| "error_rate": 0.04081632653061224 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9642857142857143, |
| "confidence_cutoff": 0.9991586798493963 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9375, |
| "confidence_cutoff": 0.964939495122677 |
| } |
| } |
| }, |
| "score": { |
| "n": 40, |
| "nll": 1.0168531134867067, |
| "acc": 0.7, |
| "ece": 0.23830739887567784, |
| "brier": 0.48314037180891034, |
| "mean_conf": 0.9383073988756777, |
| "confident_error_rate": 0.175, |
| "coverage_at_0_9": 0.8, |
| "accuracy_at_0_9": 0.78125, |
| "coverage_at_5pct_error": 0.65, |
| "coverage_at_1pct_error": 0.6, |
| "aurc": 0.07895709505044536, |
| "error_rate_at_0_9": 0.21875, |
| "confidence_bias": 0.23830739887567776, |
| "top_bins": { |
| "0.9": { |
| "n": 32, |
| "errors": 7, |
| "error_rate": 0.21875 |
| }, |
| "0.95": { |
| "n": 32, |
| "errors": 7, |
| "error_rate": 0.21875 |
| }, |
| "0.99": { |
| "n": 28, |
| "errors": 3, |
| "error_rate": 0.10714285714285714 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9997651676938517 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.78125, |
| "confidence_cutoff": 0.9590969197346936 |
| } |
| }, |
| "score_mae": 0.28661481608517747, |
| "ranked_probability_score": 0.1219924564403267 |
| } |
| } |
| } |
| }, |
| "temperature": 1.1759291426711453 |
| }, |
| { |
| "epoch": 2, |
| "development": { |
| "raw": { |
| "questions": 14912, |
| "accuracy": 0.9046405579399142, |
| "mean_class_recall": 0.8351706914084861, |
| "nll": 0.2609479185046837, |
| "brier": 0.13871271598030968, |
| "ece15": 0.02912533581539033, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.811762339055794, |
| "error_rate_at_confidence_0_9": 0.030318050392399834, |
| "ordinal_expected_level_mae": 0.007142014228908907, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 222, |
| "accuracy": 0.9774774774774775, |
| "mean_class_recall": 0.9570930232558139, |
| "nll": 0.0857490162426134, |
| "brier": 0.03780111867446018, |
| "ece15": 0.020326742033581136, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9864864864864865, |
| "error_rate_at_confidence_0_9": 0.0136986301369863, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 222, |
| "accuracy": 0.9819819819819819, |
| "mean_class_recall": 0.96, |
| "nll": 0.08583505640070498, |
| "brier": 0.03170617674969949, |
| "ece15": 0.018778574788420183, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9819819819819819, |
| "error_rate_at_confidence_0_9": 0.009174311926605505, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 1400, |
| "accuracy": 0.9828571428571429, |
| "mean_class_recall": 0.9678571428571429, |
| "nll": 0.05038447430212686, |
| "brier": 0.026644462415551518, |
| "ece15": 0.006643072262975827, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9757142857142858, |
| "error_rate_at_confidence_0_9": 0.008784773060029283, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 5000, |
| "accuracy": 0.839, |
| "mean_class_recall": 0.7132000000000001, |
| "nll": 0.4567278707638309, |
| "brier": 0.24289094068797595, |
| "ece15": 0.07089882316707959, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.7078, |
| "error_rate_at_confidence_0_9": 0.08194405199208817, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0030323687345377927, |
| "brier": 0.0010147751519150774, |
| "ece15": 0.0027304101493378944, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9919354838709677, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0013590718776517526, |
| "brier": 0.0005853758373684321, |
| "ece15": 0.001174090997847941, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9969512195121951, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 2.706087239227311e-05, |
| "brier": 4.4631636950527135e-07, |
| "ece15": 2.6948652833984242e-05, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 2500, |
| "accuracy": 0.9788, |
| "mean_class_recall": 0.98, |
| "nll": 0.08755100200107269, |
| "brier": 0.038158588944742826, |
| "ece15": 0.017829224144769444, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9896, |
| "error_rate_at_confidence_0_9": 0.016976556184316895, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 797, |
| "accuracy": 0.9949811794228356, |
| "mean_class_recall": 0.9949406163344321, |
| "nll": 0.020100883811983803, |
| "brier": 0.009532705405460478, |
| "ece15": 0.004664870689987746, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9962358845671268, |
| "error_rate_at_confidence_0_9": 0.003778337531486146, |
| "ordinal_expected_level_mae": 0.007142014228908907 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 797, |
| "accuracy": 0.9887076537013801, |
| "mean_class_recall": 0.9889237691623501, |
| "nll": 0.029531627570397425, |
| "brier": 0.01682083664455817, |
| "ece15": 0.008194444119065863, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9887076537013801, |
| "error_rate_at_confidence_0_9": 0.0038071065989847717, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 797, |
| "accuracy": 0.9937264742785445, |
| "mean_class_recall": 0.9922118380062306, |
| "nll": 0.019942020828876864, |
| "brier": 0.011733672902765547, |
| "ece15": 0.006519660642330038, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9912170639899623, |
| "error_rate_at_confidence_0_9": 0.0037974683544303796, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 797, |
| "accuracy": 0.9974905897114178, |
| "mean_class_recall": 0.998236331569665, |
| "nll": 0.016769983022509982, |
| "brier": 0.0049992664597086935, |
| "ece15": 0.0021278163552494573, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.002509410288582183, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 800, |
| "accuracy": 0.6875, |
| "mean_class_recall": 0.6966034271725826, |
| "nll": 0.5142628800857739, |
| "brier": 0.36155138388149327, |
| "ece15": 0.02205690591199928, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.22125, |
| "error_rate_at_confidence_0_9": 0.011299435028248588, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 800, |
| "accuracy": 0.67375, |
| "mean_class_recall": 0.5089660282717738, |
| "nll": 0.998813782244008, |
| "brier": 0.4775012041931279, |
| "ece15": 0.03873448735113525, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.20625, |
| "error_rate_at_confidence_0_9": 0.030303030303030304, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9354480356736271, |
| "macro_task_balanced_accuracy": 0.9112880126164279 |
| }, |
| "recalibrated": { |
| "questions": 14912, |
| "accuracy": 0.9046405579399142, |
| "mean_class_recall": 0.8351706914084861, |
| "nll": 0.24022147566124724, |
| "brier": 0.13631204830326463, |
| "ece15": 0.023735839802210217, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.790973712446352, |
| "error_rate_at_confidence_0_9": 0.02467147096227215, |
| "ordinal_expected_level_mae": 0.007897347093607133, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 222, |
| "accuracy": 0.9774774774774775, |
| "mean_class_recall": 0.9570930232558139, |
| "nll": 0.06980345492279078, |
| "brier": 0.036072899559119326, |
| "ece15": 0.020348852979949178, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.9819819819819819, |
| "error_rate_at_confidence_0_9": 0.013761467889908258, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 222, |
| "accuracy": 0.9819819819819819, |
| "mean_class_recall": 0.96, |
| "nll": 0.0697534508038578, |
| "brier": 0.030632585867415095, |
| "ece15": 0.018874774396367578, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.9819819819819819, |
| "error_rate_at_confidence_0_9": 0.009174311926605505, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 1400, |
| "accuracy": 0.9828571428571429, |
| "mean_class_recall": 0.9678571428571429, |
| "nll": 0.05022700385733144, |
| "brier": 0.026160310719648357, |
| "ece15": 0.005133274295293359, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.9642857142857143, |
| "error_rate_at_confidence_0_9": 0.007407407407407408, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 5000, |
| "accuracy": 0.839, |
| "mean_class_recall": 0.7132000000000001, |
| "nll": 0.4007512736870225, |
| "brier": 0.23542102872463042, |
| "ece15": 0.05680924935578224, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.6622, |
| "error_rate_at_confidence_0_9": 0.0664451827242525, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.00473204279698464, |
| "brier": 0.0015928649780981342, |
| "ece15": 0.004243332776407401, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.9919354838709677, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0022032517878559357, |
| "brier": 0.0007796053221569698, |
| "ece15": 0.0019494525692642349, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.9969512195121951, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 9.041263403421138e-05, |
| "brier": 3.609861838766288e-06, |
| "ece15": 8.949527479169929e-05, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 2500, |
| "accuracy": 0.9788, |
| "mean_class_recall": 0.98, |
| "nll": 0.07409278691965634, |
| "brier": 0.03695008653192076, |
| "ece15": 0.013363996653905837, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.984, |
| "error_rate_at_confidence_0_9": 0.016666666666666666, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 797, |
| "accuracy": 0.9949811794228356, |
| "mean_class_recall": 0.9949406163344321, |
| "nll": 0.017338830453877865, |
| "brier": 0.00923947768144349, |
| "ece15": 0.0037098432118081266, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.9899623588456713, |
| "error_rate_at_confidence_0_9": 0.0038022813688212928, |
| "ordinal_expected_level_mae": 0.007897347093607133 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 797, |
| "accuracy": 0.9887076537013801, |
| "mean_class_recall": 0.9889237691623501, |
| "nll": 0.025810322060464137, |
| "brier": 0.015590356305548642, |
| "ece15": 0.0063347054801968505, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.9824341279799247, |
| "error_rate_at_confidence_0_9": 0.002554278416347382, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 797, |
| "accuracy": 0.9937264742785445, |
| "mean_class_recall": 0.9922118380062306, |
| "nll": 0.017984967092368, |
| "brier": 0.011131458808574956, |
| "ece15": 0.005638113127633191, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.9912170639899623, |
| "error_rate_at_confidence_0_9": 0.0037974683544303796, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 797, |
| "accuracy": 0.9974905897114178, |
| "mean_class_recall": 0.998236331569665, |
| "nll": 0.013810181529074162, |
| "brier": 0.004981534821676031, |
| "ece15": 0.0021482520070447817, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.998745294855709, |
| "error_rate_at_confidence_0_9": 0.002512562814070352, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 800, |
| "accuracy": 0.6875, |
| "mean_class_recall": 0.6966034271725826, |
| "nll": 0.5181534061242398, |
| "brier": 0.36342749885940506, |
| "ece15": 0.019537157326473344, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.205, |
| "error_rate_at_confidence_0_9": 0.012195121951219513, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 800, |
| "accuracy": 0.67375, |
| "mean_class_recall": 0.5089660282717738, |
| "nll": 1.020378053391093, |
| "brier": 0.48493001129627705, |
| "ece15": 0.07743873861233927, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.1725, |
| "error_rate_at_confidence_0_9": 0.021739130434782608, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9354480356736271, |
| "macro_task_balanced_accuracy": 0.9112880126164279 |
| } |
| }, |
| "test": { |
| "raw": { |
| "questions": 21634, |
| "accuracy": 0.8712212258482019, |
| "mean_class_recall": 0.8343950403877007, |
| "nll": 0.36437325393469283, |
| "brier": 0.1826183728339598, |
| "ece15": 0.016790325586147077, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.6702412868632708, |
| "error_rate_at_confidence_0_9": 0.01903448275862069, |
| "ordinal_expected_level_mae": 0.005114573364023923, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 426, |
| "accuracy": 0.9953051643192489, |
| "mean_class_recall": 0.9895833333333333, |
| "nll": 0.020419671491798054, |
| "brier": 0.008500766418781964, |
| "ece15": 0.004676929181256299, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9765258215962441, |
| "error_rate_at_confidence_0_9": 0.002403846153846154, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 426, |
| "accuracy": 0.9835680751173709, |
| "mean_class_recall": 0.9635416666666667, |
| "nll": 0.03064858581307839, |
| "brier": 0.020119931286614087, |
| "ece15": 0.009967851623295965, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9788732394366197, |
| "error_rate_at_confidence_0_9": 0.004796163069544364, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 800, |
| "accuracy": 0.9525, |
| "mean_class_recall": 0.91640625, |
| "nll": 0.17731219866650544, |
| "brier": 0.08433190754386494, |
| "ece15": 0.03454275208230118, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.95, |
| "error_rate_at_confidence_0_9": 0.03289473684210526, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 6000, |
| "accuracy": 0.8995, |
| "mean_class_recall": 0.8221111111111111, |
| "nll": 0.30893788807797395, |
| "brier": 0.15122747877892262, |
| "ece15": 0.028859158451860423, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.737, |
| "error_rate_at_confidence_0_9": 0.03640886476707372, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.00019733561912440662, |
| "brier": 1.9375590740059706e-06, |
| "ece15": 0.0001968485399507404, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 540, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0017246613457991484, |
| "brier": 0.0007621008009601763, |
| "ece15": 0.001458779457079616, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9981481481481481, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 540, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 1.1611401866187557e-06, |
| "brier": 5.747739403769056e-11, |
| "ece15": 1.1611258165194832e-06, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 3500, |
| "accuracy": 0.9894285714285714, |
| "mean_class_recall": 0.9826666666666667, |
| "nll": 0.059799522199125266, |
| "brier": 0.02016997152334037, |
| "ece15": 0.00639039716386258, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9728571428571429, |
| "error_rate_at_confidence_0_9": 0.0076358296622613805, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 659, |
| "accuracy": 0.9984825493171472, |
| "mean_class_recall": 0.9983249581239532, |
| "nll": 0.0075671027444564045, |
| "brier": 0.0038579171936140126, |
| "ece15": 0.0029747801235328448, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9939301972685888, |
| "error_rate_at_confidence_0_9": 0.0015267175572519084, |
| "ordinal_expected_level_mae": 0.005114573364023923 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 659, |
| "accuracy": 0.9969650986342944, |
| "mean_class_recall": 0.9966499162479062, |
| "nll": 0.007401075027874373, |
| "brier": 0.004494834346902429, |
| "ece15": 0.0034418284412443194, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9939301972685888, |
| "error_rate_at_confidence_0_9": 0.0015267175572519084, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 659, |
| "accuracy": 0.9984825493171472, |
| "mean_class_recall": 0.9978632478632479, |
| "nll": 0.0026045927984607647, |
| "brier": 0.00133772985585407, |
| "ece15": 0.002039732462895639, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9954476479514416, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 659, |
| "accuracy": 0.9969650986342944, |
| "mean_class_recall": 0.9978260869565218, |
| "nll": 0.007434124878991116, |
| "brier": 0.004449797406116125, |
| "ece15": 0.002637941445288803, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9969650986342944, |
| "error_rate_at_confidence_0_9": 0.0015220700152207, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 3321, |
| "accuracy": 0.6838301716350497, |
| "mean_class_recall": 0.6880928586224566, |
| "nll": 0.5412193961251557, |
| "brier": 0.3797960822714433, |
| "ece15": 0.028068617701296383, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.19451972297500753, |
| "error_rate_at_confidence_0_9": 0.01393188854489164, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 3321, |
| "accuracy": 0.6859379704908161, |
| "mean_class_recall": 0.4840299509564737, |
| "nll": 1.1567283681183516, |
| "brier": 0.4884417807744329, |
| "ece15": 0.038297669894539775, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.18307738632941886, |
| "error_rate_at_confidence_0_9": 0.0805921052631579, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9414975177781386, |
| "macro_task_balanced_accuracy": 0.9169354318963097 |
| }, |
| "recalibrated": { |
| "questions": 21634, |
| "accuracy": 0.8712212258482019, |
| "mean_class_recall": 0.8343950403877007, |
| "nll": 0.35395590855273523, |
| "brier": 0.18501029425464544, |
| "ece15": 0.030017031519889922, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.6506887307016733, |
| "error_rate_at_confidence_0_9": 0.016054557078923067, |
| "ordinal_expected_level_mae": 0.006011043764813851, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 426, |
| "accuracy": 0.9953051643192489, |
| "mean_class_recall": 0.9895833333333333, |
| "nll": 0.023835787531761698, |
| "brier": 0.009460218069430947, |
| "ece15": 0.00990292779030044, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.9671361502347418, |
| "error_rate_at_confidence_0_9": 0.0024271844660194173, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 426, |
| "accuracy": 0.9835680751173709, |
| "mean_class_recall": 0.9635416666666667, |
| "nll": 0.02937486871225543, |
| "brier": 0.01905460565156044, |
| "ece15": 0.011296185892511, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.9694835680751174, |
| "error_rate_at_confidence_0_9": 0.002421307506053269, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 800, |
| "accuracy": 0.9525, |
| "mean_class_recall": 0.91640625, |
| "nll": 0.15527362489298477, |
| "brier": 0.08190930430000586, |
| "ece15": 0.02574101560783749, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.92625, |
| "error_rate_at_confidence_0_9": 0.029689608636977057, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 6000, |
| "accuracy": 0.8995, |
| "mean_class_recall": 0.8221111111111111, |
| "nll": 0.2798275436837799, |
| "brier": 0.15134523246061166, |
| "ece15": 0.029238705666976703, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.7075, |
| "error_rate_at_confidence_0_9": 0.0303886925795053, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0008001907223348217, |
| "brier": 1.793746254505973e-05, |
| "ece15": 0.0007956440534692383, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 540, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0027723209709698475, |
| "brier": 0.0009529345145823382, |
| "ece15": 0.0024453488801810066, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.9925925925925926, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 540, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 1.8070166383123626e-05, |
| "brier": 5.357685907682576e-09, |
| "ece15": 1.806882660338892e-05, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 3500, |
| "accuracy": 0.9894285714285714, |
| "mean_class_recall": 0.9826666666666667, |
| "nll": 0.05655522961511102, |
| "brier": 0.021374549663926688, |
| "ece15": 0.009425764157233412, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.95, |
| "error_rate_at_confidence_0_9": 0.0069172932330827065, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 659, |
| "accuracy": 0.9984825493171472, |
| "mean_class_recall": 0.9983249581239532, |
| "nll": 0.007520014659349455, |
| "brier": 0.003991311156523451, |
| "ece15": 0.00377357794140357, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.992412746585736, |
| "error_rate_at_confidence_0_9": 0.0015290519877675841, |
| "ordinal_expected_level_mae": 0.006011043764813851 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 659, |
| "accuracy": 0.9969650986342944, |
| "mean_class_recall": 0.9966499162479062, |
| "nll": 0.008102634890372476, |
| "brier": 0.004356473026911708, |
| "ece15": 0.004867738330078791, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.9908952959028832, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 659, |
| "accuracy": 0.9984825493171472, |
| "mean_class_recall": 0.9978632478632479, |
| "nll": 0.0038576227479580737, |
| "brier": 0.0014748958385226618, |
| "ece15": 0.003282715492202558, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.9939301972685888, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 659, |
| "accuracy": 0.9969650986342944, |
| "mean_class_recall": 0.9978260869565218, |
| "nll": 0.006788751578147029, |
| "brier": 0.004171759480970426, |
| "ece15": 0.003177496987382077, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.9939301972685888, |
| "error_rate_at_confidence_0_9": 0.0015267175572519084, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 3321, |
| "accuracy": 0.6838301716350497, |
| "mean_class_recall": 0.6880928586224566, |
| "nll": 0.5429719234394278, |
| "brier": 0.3809937714731844, |
| "ece15": 0.02889956013524373, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.17765733212887685, |
| "error_rate_at_confidence_0_9": 0.010169491525423728, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 3321, |
| "accuracy": 0.6859379704908161, |
| "mean_class_recall": 0.4840299509564737, |
| "nll": 1.1477144959529286, |
| "brier": 0.5019380206266735, |
| "ece15": 0.11401240417222197, |
| "temperature": 1.2873604198505975, |
| "coverage_at_confidence_0_9": 0.16079494128274616, |
| "error_rate_at_confidence_0_9": 0.07865168539325842, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9414975177781386, |
| "macro_task_balanced_accuracy": 0.9169354318963097 |
| } |
| }, |
| "general": { |
| "decision-v7": { |
| "raw_clean": { |
| "n": 1200, |
| "nll": 0.5237394417093835, |
| "acc": 0.865, |
| "ece": 0.08203125833030098, |
| "brier": 0.21310411533941315, |
| "mean_conf": 0.9455011120677191, |
| "confident_error_rate": 0.06, |
| "coverage_at_0_9": 0.8308333333333333, |
| "accuracy_at_0_9": 0.9277833500501504, |
| "coverage_at_5pct_error": 0.7791666666666667, |
| "coverage_at_1pct_error": 0.6158333333333333, |
| "aurc": 0.028219153802824883, |
| "error_rate_at_0_9": 0.07221664994984955, |
| "confidence_bias": 0.08050111206771915, |
| "top_bins": { |
| "0.9": { |
| "n": 997, |
| "errors": 72, |
| "error_rate": 0.07221664994984955 |
| }, |
| "0.95": { |
| "n": 937, |
| "errors": 47, |
| "error_rate": 0.05016008537886873 |
| }, |
| "0.99": { |
| "n": 861, |
| "errors": 27, |
| "error_rate": 0.0313588850174216 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9933333333333333, |
| "confidence_cutoff": 0.9999626357556778 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9458333333333333, |
| "confidence_cutoff": 0.9342652506942892 |
| } |
| }, |
| "score_mae": 0.4647882331825802, |
| "ranked_probability_score": 0.08637141204655961 |
| }, |
| "as_shipped_clean": { |
| "n": 1200, |
| "nll": 0.44174931033716425, |
| "acc": 0.865, |
| "ece": 0.0702397073006123, |
| "brier": 0.20431799318642196, |
| "mean_conf": 0.930435938959295, |
| "confident_error_rate": 0.04, |
| "coverage_at_0_9": 0.7858333333333334, |
| "accuracy_at_0_9": 0.9490986214209968, |
| "coverage_at_5pct_error": 0.7783333333333333, |
| "coverage_at_1pct_error": 0.6125, |
| "aurc": 0.028152549316370398, |
| "error_rate_at_0_9": 0.05090137857900318, |
| "confidence_bias": 0.065435938959295, |
| "top_bins": { |
| "0.9": { |
| "n": 943, |
| "errors": 48, |
| "error_rate": 0.05090137857900318 |
| }, |
| "0.95": { |
| "n": 893, |
| "errors": 35, |
| "error_rate": 0.03919372900335946 |
| }, |
| "0.99": { |
| "n": 803, |
| "errors": 15, |
| "error_rate": 0.0186799501867995 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9916666666666667, |
| "confidence_cutoff": 0.9996192297027195 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9447916666666667, |
| "confidence_cutoff": 0.8818964099133315 |
| } |
| }, |
| "score_mae": 0.47431418086986715, |
| "ranked_probability_score": 0.08319761613591657 |
| }, |
| "sources": { |
| "agnews": { |
| "n": 240, |
| "nll": 0.2796999395688286, |
| "acc": 0.9125, |
| "ece": 0.05892681778016237, |
| "brier": 0.1350754514620047, |
| "mean_conf": 0.9714268177801624, |
| "confident_error_rate": 0.04583333333333333, |
| "coverage_at_0_9": 0.9208333333333333, |
| "accuracy_at_0_9": 0.9502262443438914, |
| "coverage_at_5pct_error": 0.9333333333333333, |
| "coverage_at_1pct_error": 0.75, |
| "aurc": 0.011330835078134182, |
| "error_rate_at_0_9": 0.049773755656108594, |
| "confidence_bias": 0.0589268177801624, |
| "top_bins": { |
| "0.9": { |
| "n": 221, |
| "errors": 11, |
| "error_rate": 0.049773755656108594 |
| }, |
| "0.95": { |
| "n": 214, |
| "errors": 9, |
| "error_rate": 0.04205607476635514 |
| }, |
| "0.99": { |
| "n": 189, |
| "errors": 3, |
| "error_rate": 0.015873015873015872 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9994019817408221 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9890606274392002 |
| } |
| } |
| }, |
| "amazon": { |
| "n": 80, |
| "nll": 1.1725112640711228, |
| "acc": 0.6, |
| "ece": 0.14900986362097515, |
| "brier": 0.5611978409181134, |
| "mean_conf": 0.7279331238481536, |
| "confident_error_rate": 0.0375, |
| "coverage_at_0_9": 0.2125, |
| "accuracy_at_0_9": 0.8235294117647058, |
| "coverage_at_5pct_error": 0.075, |
| "coverage_at_1pct_error": 0.075, |
| "aurc": 0.268674359040746, |
| "error_rate_at_0_9": 0.17647058823529413, |
| "confidence_bias": 0.12793312384815358, |
| "top_bins": { |
| "0.9": { |
| "n": 17, |
| "errors": 3, |
| "error_rate": 0.17647058823529413 |
| }, |
| "0.95": { |
| "n": 12, |
| "errors": 2, |
| "error_rate": 0.16666666666666666 |
| }, |
| "0.99": { |
| "n": 2, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.725, |
| "confidence_cutoff": 0.7565342654626949 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.640625, |
| "confidence_cutoff": 0.5366925117472928 |
| } |
| }, |
| "score_mae": 0.5012771753189752, |
| "ranked_probability_score": 0.0890286871906647 |
| }, |
| "banking77": { |
| "n": 80, |
| "nll": 0.5995652320511431, |
| "acc": 0.85, |
| "ece": 0.08817712605098262, |
| "brier": 0.24932641415691287, |
| "mean_conf": 0.8916570456057876, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.75, |
| "accuracy_at_0_9": 0.9333333333333333, |
| "coverage_at_5pct_error": 0.6875, |
| "coverage_at_1pct_error": 0.6125, |
| "aurc": 0.03525784971074972, |
| "error_rate_at_0_9": 0.06666666666666667, |
| "confidence_bias": 0.041657045605787646, |
| "top_bins": { |
| "0.9": { |
| "n": 60, |
| "errors": 4, |
| "error_rate": 0.06666666666666667 |
| }, |
| "0.95": { |
| "n": 55, |
| "errors": 2, |
| "error_rate": 0.03636363636363636 |
| }, |
| "0.99": { |
| "n": 46, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9933430562867247 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.90625, |
| "confidence_cutoff": 0.8362446757691954 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 80, |
| "nll": 0.5712250447804231, |
| "acc": 0.875, |
| "ece": 0.11692367054964475, |
| "brier": 0.22310665871912266, |
| "mean_conf": 0.948749077509772, |
| "confident_error_rate": 0.0875, |
| "coverage_at_0_9": 0.85, |
| "accuracy_at_0_9": 0.8970588235294118, |
| "coverage_at_5pct_error": 0.6875, |
| "coverage_at_1pct_error": 0.1, |
| "aurc": 0.05278244760094704, |
| "error_rate_at_0_9": 0.10294117647058823, |
| "confidence_bias": 0.073749077509772, |
| "top_bins": { |
| "0.9": { |
| "n": 68, |
| "errors": 7, |
| "error_rate": 0.10294117647058823 |
| }, |
| "0.95": { |
| "n": 66, |
| "errors": 6, |
| "error_rate": 0.09090909090909091 |
| }, |
| "0.99": { |
| "n": 60, |
| "errors": 4, |
| "error_rate": 0.06666666666666667 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.9998615919766226 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.921875, |
| "confidence_cutoff": 0.979302462267494 |
| } |
| } |
| }, |
| "contrastive": { |
| "n": 160, |
| "nll": 0.06554428006012027, |
| "acc": 0.96875, |
| "ece": 0.024615155693287222, |
| "brier": 0.04485687620328682, |
| "mean_conf": 0.9933651556932872, |
| "confident_error_rate": 0.0125, |
| "coverage_at_0_9": 0.98125, |
| "accuracy_at_0_9": 0.9872611464968153, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.975, |
| "aurc": 0.000671805023667049, |
| "error_rate_at_0_9": 0.012738853503184714, |
| "confidence_bias": 0.024615155693287205, |
| "top_bins": { |
| "0.9": { |
| "n": 157, |
| "errors": 2, |
| "error_rate": 0.012738853503184714 |
| }, |
| "0.95": { |
| "n": 154, |
| "errors": 1, |
| "error_rate": 0.006493506493506494 |
| }, |
| "0.99": { |
| "n": 148, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999882930499685 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9997675703852573 |
| } |
| } |
| }, |
| "dbpedia14": { |
| "n": 80, |
| "nll": 0.01926796153789536, |
| "acc": 0.9875, |
| "ece": 0.011278233290686156, |
| "brier": 0.014217088541729535, |
| "mean_conf": 0.9949340220579689, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9875, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.9875, |
| "aurc": 0.00015625, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.0074340220579688365, |
| "top_bins": { |
| "0.9": { |
| "n": 79, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 77, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 76, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999950740905034 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999745354541041 |
| } |
| } |
| }, |
| "imdb": { |
| "n": 80, |
| "nll": 0.06446545489140978, |
| "acc": 0.9625, |
| "ece": 0.025460341159037583, |
| "brier": 0.045829990154110004, |
| "mean_conf": 0.9836424817249945, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9375, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.95, |
| "aurc": 0.0012680561790530144, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.021142481724994444, |
| "top_bins": { |
| "0.9": { |
| "n": 75, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 75, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 71, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999788743849367 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9996364092230372 |
| } |
| } |
| }, |
| "mnli": { |
| "n": 80, |
| "nll": 0.46179093448986874, |
| "acc": 0.9125, |
| "ece": 0.07967308878997346, |
| "brier": 0.16221252791471635, |
| "mean_conf": 0.985103225410972, |
| "confident_error_rate": 0.0625, |
| "coverage_at_0_9": 0.925, |
| "accuracy_at_0_9": 0.9324324324324325, |
| "coverage_at_5pct_error": 0.8875, |
| "coverage_at_1pct_error": 0.65, |
| "aurc": 0.015873702018260973, |
| "error_rate_at_0_9": 0.06756756756756757, |
| "confidence_bias": 0.07260322541097197, |
| "top_bins": { |
| "0.9": { |
| "n": 74, |
| "errors": 5, |
| "error_rate": 0.06756756756756757 |
| }, |
| "0.95": { |
| "n": 74, |
| "errors": 5, |
| "error_rate": 0.06756756756756757 |
| }, |
| "0.99": { |
| "n": 71, |
| "errors": 3, |
| "error_rate": 0.04225352112676056 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.999987939363811 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.96875, |
| "confidence_cutoff": 0.9997555402093751 |
| } |
| } |
| }, |
| "sst5": { |
| "n": 80, |
| "nll": 1.1930808545317393, |
| "acc": 0.6125, |
| "ece": 0.23848723097529123, |
| "brier": 0.5810003072599018, |
| "mean_conf": 0.7987986774228062, |
| "confident_error_rate": 0.0625, |
| "coverage_at_0_9": 0.2625, |
| "accuracy_at_0_9": 0.7619047619047619, |
| "coverage_at_5pct_error": 0.0875, |
| "coverage_at_1pct_error": 0.0875, |
| "aurc": 0.2998766100005863, |
| "error_rate_at_0_9": 0.23809523809523808, |
| "confidence_bias": 0.18629867742280615, |
| "top_bins": { |
| "0.9": { |
| "n": 21, |
| "errors": 5, |
| "error_rate": 0.23809523809523808 |
| }, |
| "0.95": { |
| "n": 5, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.625, |
| "confidence_cutoff": 0.8363874325149153 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.625, |
| "confidence_cutoff": 0.6881083881339022 |
| } |
| }, |
| "score_mae": 0.4905379704011906, |
| "ranked_probability_score": 0.08831634129641223 |
| }, |
| "trec": { |
| "n": 80, |
| "nll": 0.268396899685072, |
| "acc": 0.9625, |
| "ece": 0.02818577556782262, |
| "brier": 0.058300946365123306, |
| "mean_conf": 0.9906857755678227, |
| "confident_error_rate": 0.025, |
| "coverage_at_0_9": 0.9875, |
| "accuracy_at_0_9": 0.9746835443037974, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.325, |
| "aurc": 0.015881097479045377, |
| "error_rate_at_0_9": 0.02531645569620253, |
| "confidence_bias": 0.028185775567822646, |
| "top_bins": { |
| "0.9": { |
| "n": 79, |
| "errors": 2, |
| "error_rate": 0.02531645569620253 |
| }, |
| "0.95": { |
| "n": 78, |
| "errors": 2, |
| "error_rate": 0.02564102564102564 |
| }, |
| "0.99": { |
| "n": 77, |
| "errors": 2, |
| "error_rate": 0.025974025974025976 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.9999834449407883 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9998382341930544 |
| } |
| } |
| }, |
| "yelp": { |
| "n": 160, |
| "nll": 0.6528738150960317, |
| "acc": 0.76875, |
| "ece": 0.10032078827768093, |
| "brier": 0.3373190084870059, |
| "mean_conf": 0.867012445257043, |
| "confident_error_rate": 0.05625, |
| "coverage_at_0_9": 0.575, |
| "accuracy_at_0_9": 0.9021739130434783, |
| "coverage_at_5pct_error": 0.41875, |
| "coverage_at_1pct_error": 0.18125, |
| "aurc": 0.08667681796341256, |
| "error_rate_at_0_9": 0.09782608695652174, |
| "confidence_bias": 0.09826244525704297, |
| "top_bins": { |
| "0.9": { |
| "n": 92, |
| "errors": 9, |
| "error_rate": 0.09782608695652174 |
| }, |
| "0.95": { |
| "n": 83, |
| "errors": 8, |
| "error_rate": 0.0963855421686747 |
| }, |
| "0.99": { |
| "n": 63, |
| "errors": 3, |
| "error_rate": 0.047619047619047616 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.925, |
| "confidence_cutoff": 0.9656164925493048 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.84375, |
| "confidence_cutoff": 0.7039733738891296 |
| } |
| }, |
| "score_mae": 0.4311273968894357, |
| "ranked_probability_score": 0.07224781992067277 |
| } |
| }, |
| "types": { |
| "choice": { |
| "n": 480, |
| "nll": 0.3037822176153947, |
| "acc": 0.93125, |
| "ece": 0.050099480347752155, |
| "brier": 0.11775456449644288, |
| "mean_conf": 0.9718507476012318, |
| "confident_error_rate": 0.035416666666666666, |
| "coverage_at_0_9": 0.9270833333333334, |
| "accuracy_at_0_9": 0.9617977528089887, |
| "coverage_at_5pct_error": 0.94375, |
| "coverage_at_1pct_error": 0.725, |
| "aurc": 0.010673395731944459, |
| "error_rate_at_0_9": 0.038202247191011236, |
| "confidence_bias": 0.04060074760123178, |
| "top_bins": { |
| "0.9": { |
| "n": 445, |
| "errors": 17, |
| "error_rate": 0.038202247191011236 |
| }, |
| "0.95": { |
| "n": 435, |
| "errors": 15, |
| "error_rate": 0.034482758620689655 |
| }, |
| "0.99": { |
| "n": 408, |
| "errors": 6, |
| "error_rate": 0.014705882352941176 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9958333333333333, |
| "confidence_cutoff": 0.9999737433376136 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9895833333333334, |
| "confidence_cutoff": 0.9984835061671959 |
| } |
| } |
| }, |
| "noul": { |
| "n": 480, |
| "nll": 0.24523830247044484, |
| "acc": 0.9229166666666667, |
| "ece": 0.04796953512867931, |
| "brier": 0.11605074761778626, |
| "mean_conf": 0.970886201795346, |
| "confident_error_rate": 0.035416666666666666, |
| "coverage_at_0_9": 0.9104166666666667, |
| "accuracy_at_0_9": 0.9610983981693364, |
| "coverage_at_5pct_error": 0.94375, |
| "coverage_at_1pct_error": 0.71875, |
| "aurc": 0.01472209854520942, |
| "error_rate_at_0_9": 0.038901601830663615, |
| "confidence_bias": 0.0479695351286793, |
| "top_bins": { |
| "0.9": { |
| "n": 437, |
| "errors": 17, |
| "error_rate": 0.038901601830663615 |
| }, |
| "0.95": { |
| "n": 424, |
| "errors": 13, |
| "error_rate": 0.030660377358490566 |
| }, |
| "0.99": { |
| "n": 389, |
| "errors": 8, |
| "error_rate": 0.02056555269922879 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9958333333333333, |
| "confidence_cutoff": 0.9997753303513292 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9791666666666666, |
| "confidence_cutoff": 0.9930519257681523 |
| } |
| } |
| }, |
| "score": { |
| "n": 240, |
| "nll": 1.110705511514142, |
| "acc": 0.6166666666666667, |
| "ece": 0.16184498803254416, |
| "brier": 0.5539793417036516, |
| "mean_conf": 0.7667057960033195, |
| "confident_error_rate": 0.058333333333333334, |
| "coverage_at_0_9": 0.25416666666666665, |
| "accuracy_at_0_9": 0.7704918032786885, |
| "coverage_at_5pct_error": 0.016666666666666666, |
| "coverage_at_1pct_error": 0.016666666666666666, |
| "aurc": 0.28057303182426463, |
| "error_rate_at_0_9": 0.22950819672131148, |
| "confidence_bias": 0.15003912933665275, |
| "top_bins": { |
| "0.9": { |
| "n": 61, |
| "errors": 14, |
| "error_rate": 0.22950819672131148 |
| }, |
| "0.95": { |
| "n": 34, |
| "errors": 7, |
| "error_rate": 0.20588235294117646 |
| }, |
| "0.99": { |
| "n": 6, |
| "errors": 1, |
| "error_rate": 0.16666666666666666 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.6916666666666667, |
| "confidence_cutoff": 0.8026983558729256 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.6666666666666666, |
| "confidence_cutoff": 0.6054379586272015 |
| } |
| }, |
| "score_mae": 0.47431418086986715, |
| "ranked_probability_score": 0.08319761613591657 |
| } |
| } |
| }, |
| "transfer-v4": { |
| "raw_clean": { |
| "n": 656, |
| "nll": 0.7491303449341485, |
| "acc": 0.836890243902439, |
| "ece": 0.10484780065311013, |
| "brier": 0.2706697225616099, |
| "mean_conf": 0.9369188798045006, |
| "confident_error_rate": 0.08689024390243902, |
| "coverage_at_0_9": 0.8277439024390244, |
| "accuracy_at_0_9": 0.8950276243093923, |
| "coverage_at_5pct_error": 0.5701219512195121, |
| "coverage_at_1pct_error": 0.3567073170731707, |
| "aurc": 0.048474736987579994, |
| "error_rate_at_0_9": 0.10497237569060773, |
| "confidence_bias": 0.10002863590206157, |
| "top_bins": { |
| "0.9": { |
| "n": 543, |
| "errors": 57, |
| "error_rate": 0.10497237569060773 |
| }, |
| "0.95": { |
| "n": 512, |
| "errors": 48, |
| "error_rate": 0.09375 |
| }, |
| "0.99": { |
| "n": 455, |
| "errors": 30, |
| "error_rate": 0.06593406593406594 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9634146341463414, |
| "confidence_cutoff": 0.9997756567114997 |
| }, |
| "0.8": { |
| "coverage": 0.8003048780487805, |
| "accuracy": 0.8990476190476191, |
| "confidence_cutoff": 0.9312810361300458 |
| } |
| }, |
| "score_mae": 0.2771201607516616, |
| "ranked_probability_score": 0.12002374674647091 |
| }, |
| "as_shipped_clean": { |
| "n": 656, |
| "nll": 0.6222520301084513, |
| "acc": 0.836890243902439, |
| "ece": 0.09048309685573616, |
| "brier": 0.2622492394102412, |
| "mean_conf": 0.9203846154028757, |
| "confident_error_rate": 0.07317073170731707, |
| "coverage_at_0_9": 0.7820121951219512, |
| "accuracy_at_0_9": 0.9064327485380117, |
| "coverage_at_5pct_error": 0.5670731707317073, |
| "coverage_at_1pct_error": 0.3551829268292683, |
| "aurc": 0.0484684761006514, |
| "error_rate_at_0_9": 0.0935672514619883, |
| "confidence_bias": 0.0834943715004367, |
| "top_bins": { |
| "0.9": { |
| "n": 513, |
| "errors": 48, |
| "error_rate": 0.0935672514619883 |
| }, |
| "0.95": { |
| "n": 477, |
| "errors": 37, |
| "error_rate": 0.07756813417190776 |
| }, |
| "0.99": { |
| "n": 405, |
| "errors": 23, |
| "error_rate": 0.056790123456790124 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9634146341463414, |
| "confidence_cutoff": 0.9985297137731126 |
| }, |
| "0.8": { |
| "coverage": 0.8003048780487805, |
| "accuracy": 0.900952380952381, |
| "confidence_cutoff": 0.8749540187432164 |
| } |
| }, |
| "score_mae": 0.2768980570793622, |
| "ranked_probability_score": 0.11577841556824224 |
| }, |
| "sources": { |
| "composition_holdout": { |
| "n": 96, |
| "nll": 0.8314301766731748, |
| "acc": 0.8645833333333334, |
| "ece": 0.12259156340261834, |
| "brier": 0.25260603356221595, |
| "mean_conf": 0.9783762896019588, |
| "confident_error_rate": 0.11458333333333333, |
| "coverage_at_0_9": 0.9479166666666666, |
| "accuracy_at_0_9": 0.8791208791208791, |
| "coverage_at_5pct_error": 0.7291666666666666, |
| "coverage_at_1pct_error": 0.6666666666666666, |
| "aurc": 0.029361211962366768, |
| "error_rate_at_0_9": 0.12087912087912088, |
| "confidence_bias": 0.1137929562686254, |
| "top_bins": { |
| "0.9": { |
| "n": 91, |
| "errors": 11, |
| "error_rate": 0.12087912087912088 |
| }, |
| "0.95": { |
| "n": 90, |
| "errors": 11, |
| "error_rate": 0.12222222222222222 |
| }, |
| "0.99": { |
| "n": 86, |
| "errors": 11, |
| "error_rate": 0.12790697674418605 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.99991410687893 |
| }, |
| "0.8": { |
| "coverage": 0.8020833333333334, |
| "accuracy": 0.948051948051948, |
| "confidence_cutoff": 0.9986476483509216 |
| } |
| } |
| }, |
| "contrastive": { |
| "n": 80, |
| "nll": 0.46595606789147936, |
| "acc": 0.85, |
| "ece": 0.11588755121828759, |
| "brier": 0.22911401994889413, |
| "mean_conf": 0.9658875512182877, |
| "confident_error_rate": 0.0875, |
| "coverage_at_0_9": 0.9, |
| "accuracy_at_0_9": 0.9027777777777778, |
| "coverage_at_5pct_error": 0.85, |
| "coverage_at_1pct_error": 0.8, |
| "aurc": 0.017400442212878862, |
| "error_rate_at_0_9": 0.09722222222222222, |
| "confidence_bias": 0.11588755121828775, |
| "top_bins": { |
| "0.9": { |
| "n": 72, |
| "errors": 7, |
| "error_rate": 0.09722222222222222 |
| }, |
| "0.95": { |
| "n": 70, |
| "errors": 5, |
| "error_rate": 0.07142857142857142 |
| }, |
| "0.99": { |
| "n": 68, |
| "errors": 3, |
| "error_rate": 0.04411764705882353 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999823513518187 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9993279314433398 |
| } |
| }, |
| "score_mae": 0.2768980570793622, |
| "ranked_probability_score": 0.11577841556824224 |
| }, |
| "emotion": { |
| "n": 80, |
| "nll": 1.1680421729742996, |
| "acc": 0.6625, |
| "ece": 0.15215129647550107, |
| "brier": 0.49482559482198313, |
| "mean_conf": 0.767505586427657, |
| "confident_error_rate": 0.0625, |
| "coverage_at_0_9": 0.425, |
| "accuracy_at_0_9": 0.8529411764705882, |
| "coverage_at_5pct_error": 0.075, |
| "coverage_at_1pct_error": 0.075, |
| "aurc": 0.18480906195062094, |
| "error_rate_at_0_9": 0.14705882352941177, |
| "confidence_bias": 0.105005586427657, |
| "top_bins": { |
| "0.9": { |
| "n": 34, |
| "errors": 5, |
| "error_rate": 0.14705882352941177 |
| }, |
| "0.95": { |
| "n": 26, |
| "errors": 3, |
| "error_rate": 0.11538461538461539 |
| }, |
| "0.99": { |
| "n": 8, |
| "errors": 1, |
| "error_rate": 0.125 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.8, |
| "confidence_cutoff": 0.8315422519453946 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.734375, |
| "confidence_cutoff": 0.5608119629701582 |
| } |
| } |
| }, |
| "mmlu": { |
| "n": 80, |
| "nll": 0.8706892848634225, |
| "acc": 0.725, |
| "ece": 0.14396914879993838, |
| "brier": 0.42715412327538493, |
| "mean_conf": 0.8115029447218166, |
| "confident_error_rate": 0.0625, |
| "coverage_at_0_9": 0.475, |
| "accuracy_at_0_9": 0.868421052631579, |
| "coverage_at_5pct_error": 0.375, |
| "coverage_at_1pct_error": 0.3375, |
| "aurc": 0.11356864595903862, |
| "error_rate_at_0_9": 0.13157894736842105, |
| "confidence_bias": 0.08650294472181663, |
| "top_bins": { |
| "0.9": { |
| "n": 38, |
| "errors": 5, |
| "error_rate": 0.13157894736842105 |
| }, |
| "0.95": { |
| "n": 28, |
| "errors": 1, |
| "error_rate": 0.03571428571428571 |
| }, |
| "0.99": { |
| "n": 18, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.85, |
| "confidence_cutoff": 0.8706443942318668 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.796875, |
| "confidence_cutoff": 0.5685435872401945 |
| } |
| } |
| }, |
| "paws": { |
| "n": 80, |
| "nll": 0.8953763773747229, |
| "acc": 0.8, |
| "ece": 0.16414062885679848, |
| "brier": 0.31529713556607303, |
| "mean_conf": 0.947798906827078, |
| "confident_error_rate": 0.125, |
| "coverage_at_0_9": 0.8625, |
| "accuracy_at_0_9": 0.855072463768116, |
| "coverage_at_5pct_error": 0.3, |
| "coverage_at_1pct_error": 0.15, |
| "aurc": 0.09544667963478717, |
| "error_rate_at_0_9": 0.14492753623188406, |
| "confidence_bias": 0.14779890682707797, |
| "top_bins": { |
| "0.9": { |
| "n": 69, |
| "errors": 10, |
| "error_rate": 0.14492753623188406 |
| }, |
| "0.95": { |
| "n": 69, |
| "errors": 10, |
| "error_rate": 0.14492753623188406 |
| }, |
| "0.99": { |
| "n": 60, |
| "errors": 6, |
| "error_rate": 0.1 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.875, |
| "confidence_cutoff": 0.9995333943902521 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.890625, |
| "confidence_cutoff": 0.9804900927788639 |
| } |
| } |
| }, |
| "qnli": { |
| "n": 80, |
| "nll": 0.2866833743812309, |
| "acc": 0.925, |
| "ece": 0.07720126969420893, |
| "brier": 0.1353782597318704, |
| "mean_conf": 0.969044048844119, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.9, |
| "accuracy_at_0_9": 0.9444444444444444, |
| "coverage_at_5pct_error": 0.8625, |
| "coverage_at_1pct_error": 0.6125, |
| "aurc": 0.01474092888988553, |
| "error_rate_at_0_9": 0.05555555555555555, |
| "confidence_bias": 0.044044048844118944, |
| "top_bins": { |
| "0.9": { |
| "n": 72, |
| "errors": 4, |
| "error_rate": 0.05555555555555555 |
| }, |
| "0.95": { |
| "n": 69, |
| "errors": 3, |
| "error_rate": 0.043478260869565216 |
| }, |
| "0.99": { |
| "n": 64, |
| "errors": 2, |
| "error_rate": 0.03125 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9994500997969659 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.96875, |
| "confidence_cutoff": 0.9906893719466282 |
| } |
| } |
| }, |
| "sciq": { |
| "n": 80, |
| "nll": 0.007797247790721174, |
| "acc": 1.0, |
| "ece": 0.007304169605922202, |
| "brier": 0.00159066044034253, |
| "mean_conf": 0.9926958303940776, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.975, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.007304169605922395, |
| "top_bins": { |
| "0.9": { |
| "n": 78, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 76, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 71, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999411463988364 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.999496345294104 |
| } |
| } |
| }, |
| "tweet_offensive": { |
| "n": 80, |
| "nll": 0.4102059096056144, |
| "acc": 0.8625, |
| "ece": 0.082947906119192, |
| "brier": 0.24395672910477048, |
| "mean_conf": 0.9186674303481943, |
| "confident_error_rate": 0.075, |
| "coverage_at_0_9": 0.7375, |
| "accuracy_at_0_9": 0.8983050847457628, |
| "coverage_at_5pct_error": 0.575, |
| "coverage_at_1pct_error": 0.5125, |
| "aurc": 0.05023960594359948, |
| "error_rate_at_0_9": 0.1016949152542373, |
| "confidence_bias": 0.05616743034819427, |
| "top_bins": { |
| "0.9": { |
| "n": 59, |
| "errors": 6, |
| "error_rate": 0.1016949152542373 |
| }, |
| "0.95": { |
| "n": 49, |
| "errors": 4, |
| "error_rate": 0.08163265306122448 |
| }, |
| "0.99": { |
| "n": 30, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9765148863831499 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.875, |
| "confidence_cutoff": 0.8834322098576286 |
| } |
| } |
| } |
| }, |
| "types": { |
| "choice": { |
| "n": 336, |
| "nll": 0.724820218484822, |
| "acc": 0.8154761904761905, |
| "ece": 0.09541834159449251, |
| "brier": 0.29207086162199275, |
| "mean_conf": 0.8918466450156909, |
| "confident_error_rate": 0.0625, |
| "coverage_at_0_9": 0.7172619047619048, |
| "accuracy_at_0_9": 0.9128630705394191, |
| "coverage_at_5pct_error": 0.48214285714285715, |
| "coverage_at_1pct_error": 0.36904761904761907, |
| "aurc": 0.05557065380684749, |
| "error_rate_at_0_9": 0.08713692946058091, |
| "confidence_bias": 0.07637045453950042, |
| "top_bins": { |
| "0.9": { |
| "n": 241, |
| "errors": 21, |
| "error_rate": 0.08713692946058091 |
| }, |
| "0.95": { |
| "n": 220, |
| "errors": 15, |
| "error_rate": 0.06818181818181818 |
| }, |
| "0.99": { |
| "n": 183, |
| "errors": 12, |
| "error_rate": 0.06557377049180328 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9464285714285714, |
| "confidence_cutoff": 0.9967131547386102 |
| }, |
| "0.8": { |
| "coverage": 0.8005952380952381, |
| "accuracy": 0.8921933085501859, |
| "confidence_cutoff": 0.7705104187347486 |
| } |
| } |
| }, |
| "noul": { |
| "n": 280, |
| "nll": 0.4549349162962509, |
| "acc": 0.8821428571428571, |
| "ece": 0.08452238489264785, |
| "brier": 0.19846632133861988, |
| "mean_conf": 0.9530010972612158, |
| "confident_error_rate": 0.07142857142857142, |
| "coverage_at_0_9": 0.8571428571428571, |
| "accuracy_at_0_9": 0.9166666666666666, |
| "coverage_at_5pct_error": 0.7607142857142857, |
| "coverage_at_1pct_error": 0.2571428571428571, |
| "aurc": 0.03904590143532812, |
| "error_rate_at_0_9": 0.08333333333333333, |
| "confidence_bias": 0.07085824011835873, |
| "top_bins": { |
| "0.9": { |
| "n": 240, |
| "errors": 20, |
| "error_rate": 0.08333333333333333 |
| }, |
| "0.95": { |
| "n": 227, |
| "errors": 17, |
| "error_rate": 0.07488986784140969 |
| }, |
| "0.99": { |
| "n": 194, |
| "errors": 8, |
| "error_rate": 0.041237113402061855 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9571428571428572, |
| "confidence_cutoff": 0.9989211416259292 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9330357142857143, |
| "confidence_cutoff": 0.95900292163327 |
| } |
| } |
| }, |
| "score": { |
| "n": 40, |
| "nll": 0.9318990444323394, |
| "acc": 0.7, |
| "ece": 0.2317881936468471, |
| "brier": 0.45822803933287715, |
| "mean_conf": 0.9317881936468471, |
| "confident_error_rate": 0.175, |
| "coverage_at_0_9": 0.8, |
| "accuracy_at_0_9": 0.78125, |
| "coverage_at_5pct_error": 0.65, |
| "coverage_at_1pct_error": 0.6, |
| "aurc": 0.07545101908306909, |
| "error_rate_at_0_9": 0.21875, |
| "confidence_bias": 0.2317881936468471, |
| "top_bins": { |
| "0.9": { |
| "n": 32, |
| "errors": 7, |
| "error_rate": 0.21875 |
| }, |
| "0.95": { |
| "n": 30, |
| "errors": 5, |
| "error_rate": 0.16666666666666666 |
| }, |
| "0.99": { |
| "n": 28, |
| "errors": 3, |
| "error_rate": 0.10714285714285714 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9997762256879271 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.78125, |
| "confidence_cutoff": 0.9360940664483365 |
| } |
| }, |
| "score_mae": 0.2768980570793622, |
| "ranked_probability_score": 0.11577841556824224 |
| } |
| } |
| } |
| }, |
| "temperature": 1.2873604198505975 |
| }, |
| { |
| "epoch": 3, |
| "development": { |
| "raw": { |
| "questions": 14912, |
| "accuracy": 0.9075912017167382, |
| "mean_class_recall": 0.8447857522159936, |
| "nll": 0.24807737680415345, |
| "brier": 0.12998082994334342, |
| "ece15": 0.027028147820359012, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.8103540772532188, |
| "error_rate_at_confidence_0_9": 0.023502151605428664, |
| "ordinal_expected_level_mae": 0.00481932257817722, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 222, |
| "accuracy": 0.9774774774774775, |
| "mean_class_recall": 0.9570930232558139, |
| "nll": 0.10407641066616782, |
| "brier": 0.039347876403718875, |
| "ece15": 0.021432863016197557, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9864864864864865, |
| "error_rate_at_confidence_0_9": 0.0136986301369863, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 222, |
| "accuracy": 0.9819819819819819, |
| "mean_class_recall": 0.96, |
| "nll": 0.10023555924012383, |
| "brier": 0.031563944714978254, |
| "ece15": 0.015911253108198964, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9864864864864865, |
| "error_rate_at_confidence_0_9": 0.0136986301369863, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 1400, |
| "accuracy": 0.9885714285714285, |
| "mean_class_recall": 0.9821428571428572, |
| "nll": 0.0442702602822601, |
| "brier": 0.02139513973607548, |
| "ece15": 0.007710400104216533, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9842857142857143, |
| "error_rate_at_confidence_0_9": 0.008708272859216255, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 5000, |
| "accuracy": 0.8418, |
| "mean_class_recall": 0.7537333333333334, |
| "nll": 0.4173917752941542, |
| "brier": 0.22451704505631584, |
| "ece15": 0.06284084146027137, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.6928, |
| "error_rate_at_confidence_0_9": 0.06293302540415704, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0014205610073774013, |
| "brier": 0.0002211290260091384, |
| "ece15": 0.0013613576932818498, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9919354838709677, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.000610657881540054, |
| "brier": 8.099314527924365e-05, |
| "ece15": 0.0005891478461821947, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 1.5480302068279906e-07, |
| "brier": 4.4217021643060835e-12, |
| "ece15": 1.548019148098009e-07, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 2500, |
| "accuracy": 0.9792, |
| "mean_class_recall": 0.9818666666666667, |
| "nll": 0.11218220058914397, |
| "brier": 0.03724045269615108, |
| "ece15": 0.01842697861877182, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9928, |
| "error_rate_at_confidence_0_9": 0.015310233682514102, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 797, |
| "accuracy": 0.9962358845671268, |
| "mean_class_recall": 0.9963898916967509, |
| "nll": 0.012217254972570101, |
| "brier": 0.006818829719414822, |
| "ece15": 0.004133398699556409, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9949811794228356, |
| "error_rate_at_confidence_0_9": 0.0012610340479192938, |
| "ordinal_expected_level_mae": 0.00481932257817722 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 797, |
| "accuracy": 0.9949811794228356, |
| "mean_class_recall": 0.9951865222623345, |
| "nll": 0.013078119892699127, |
| "brier": 0.007721112955228842, |
| "ece15": 0.003423998250058524, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9899623588456713, |
| "error_rate_at_confidence_0_9": 0.0012674271229404308, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 797, |
| "accuracy": 0.9949811794228356, |
| "mean_class_recall": 0.994276682635671, |
| "nll": 0.01027058602627885, |
| "brier": 0.007209131739335666, |
| "ece15": 0.004201163796495765, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9912170639899623, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 797, |
| "accuracy": 0.9974905897114178, |
| "mean_class_recall": 0.998236331569665, |
| "nll": 0.017287182530912663, |
| "brier": 0.004953169208636321, |
| "ece15": 0.0025266183132433223, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.998745294855709, |
| "error_rate_at_confidence_0_9": 0.002512562814070352, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 800, |
| "accuracy": 0.69, |
| "mean_class_recall": 0.7116483068135455, |
| "nll": 0.5092641106435474, |
| "brier": 0.3569714849542106, |
| "ece15": 0.03650030376694629, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.2425, |
| "error_rate_at_confidence_0_9": 0.010309278350515464, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 800, |
| "accuracy": 0.68875, |
| "mean_class_recall": 0.529392069171139, |
| "nll": 0.96833512912194, |
| "brier": 0.46247412161427887, |
| "ece15": 0.02731423542286039, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.22625, |
| "error_rate_at_confidence_0_9": 0.022099447513812154, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9379621229396502, |
| "macro_task_balanced_accuracy": 0.9185689774676985 |
| }, |
| "recalibrated": { |
| "questions": 14912, |
| "accuracy": 0.9075912017167382, |
| "mean_class_recall": 0.8447857522159936, |
| "nll": 0.2269545932235271, |
| "brier": 0.12792407717450321, |
| "ece15": 0.01858051585491712, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.7862124463519313, |
| "error_rate_at_confidence_0_9": 0.018679631525076765, |
| "ordinal_expected_level_mae": 0.005329911013904147, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 222, |
| "accuracy": 0.9774774774774775, |
| "mean_class_recall": 0.9570930232558139, |
| "nll": 0.08092683597443519, |
| "brier": 0.037546344393533104, |
| "ece15": 0.021320738677048364, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9819819819819819, |
| "error_rate_at_confidence_0_9": 0.013761467889908258, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 222, |
| "accuracy": 0.9819819819819819, |
| "mean_class_recall": 0.96, |
| "nll": 0.07809919245507242, |
| "brier": 0.030623326048867618, |
| "ece15": 0.015019163882298042, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9819819819819819, |
| "error_rate_at_confidence_0_9": 0.009174311926605505, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 1400, |
| "accuracy": 0.9885714285714285, |
| "mean_class_recall": 0.9821428571428572, |
| "nll": 0.04094699270351949, |
| "brier": 0.021018406686723334, |
| "ece15": 0.0038434318338614677, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9678571428571429, |
| "error_rate_at_confidence_0_9": 0.006642066420664207, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 5000, |
| "accuracy": 0.8418, |
| "mean_class_recall": 0.7537333333333334, |
| "nll": 0.36679030702727994, |
| "brier": 0.21757064336558796, |
| "ece15": 0.043357678681868704, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.6394, |
| "error_rate_at_confidence_0_9": 0.05004691898654989, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0028701157894541487, |
| "brier": 0.0006216354903482641, |
| "ece15": 0.0026971564840139535, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9919354838709677, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.001405432942745028, |
| "brier": 0.00025083300955486444, |
| "ece15": 0.0013358908781109478, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9939024390243902, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 4.6272880953518815e-06, |
| "brier": 9.796066038293659e-10, |
| "ece15": 4.627043141135246e-06, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 2500, |
| "accuracy": 0.9792, |
| "mean_class_recall": 0.9818666666666667, |
| "nll": 0.0877031817842779, |
| "brier": 0.03599036114227489, |
| "ece15": 0.01538869285344686, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.988, |
| "error_rate_at_confidence_0_9": 0.014574898785425101, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 797, |
| "accuracy": 0.9962358845671268, |
| "mean_class_recall": 0.9963898916967509, |
| "nll": 0.010603240403009088, |
| "brier": 0.006373682001028304, |
| "ece15": 0.004218405852177047, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9937264742785445, |
| "error_rate_at_confidence_0_9": 0.0012626262626262627, |
| "ordinal_expected_level_mae": 0.005329911013904147 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 797, |
| "accuracy": 0.9949811794228356, |
| "mean_class_recall": 0.9951865222623345, |
| "nll": 0.01230072195659841, |
| "brier": 0.007337903591834537, |
| "ece15": 0.0031241457310850957, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.986198243412798, |
| "error_rate_at_confidence_0_9": 0.001272264631043257, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 797, |
| "accuracy": 0.9949811794228356, |
| "mean_class_recall": 0.994276682635671, |
| "nll": 0.009584146250019401, |
| "brier": 0.00659904677807738, |
| "ece15": 0.005028933100804552, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9899623588456713, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 797, |
| "accuracy": 0.9974905897114178, |
| "mean_class_recall": 0.998236331569665, |
| "nll": 0.013437428285294059, |
| "brier": 0.004858731838764845, |
| "ece15": 0.0022785888551786117, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.998745294855709, |
| "error_rate_at_confidence_0_9": 0.002512562814070352, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 800, |
| "accuracy": 0.69, |
| "mean_class_recall": 0.7116483068135455, |
| "nll": 0.5102124840713312, |
| "brier": 0.35753050242002155, |
| "ece15": 0.03811996421246104, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.22125, |
| "error_rate_at_confidence_0_9": 0.011299435028248588, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 800, |
| "accuracy": 0.68875, |
| "mean_class_recall": 0.529392069171139, |
| "nll": 0.9911460027249259, |
| "brier": 0.4737144267281807, |
| "ece15": 0.09647043512079573, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.18625, |
| "error_rate_at_confidence_0_9": 0.020134228187919462, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9379621229396502, |
| "macro_task_balanced_accuracy": 0.9185689774676985 |
| } |
| }, |
| "test": { |
| "raw": { |
| "questions": 21634, |
| "accuracy": 0.8801423684940372, |
| "mean_class_recall": 0.8422636415920437, |
| "nll": 0.35756736839135106, |
| "brier": 0.17329397786588344, |
| "ece15": 0.018370944185424988, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.681750947582509, |
| "error_rate_at_confidence_0_9": 0.017289307749677945, |
| "ordinal_expected_level_mae": 0.004830545729071066, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 426, |
| "accuracy": 0.9953051643192489, |
| "mean_class_recall": 0.9895833333333333, |
| "nll": 0.019578322598972707, |
| "brier": 0.008470972915844083, |
| "ece15": 0.0032192125141472924, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9882629107981221, |
| "error_rate_at_confidence_0_9": 0.0023752969121140144, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 426, |
| "accuracy": 0.9835680751173709, |
| "mean_class_recall": 0.9635416666666667, |
| "nll": 0.03323309985013501, |
| "brier": 0.020700086530374747, |
| "ece15": 0.012858446963474107, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9835680751173709, |
| "error_rate_at_confidence_0_9": 0.007159904534606206, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 800, |
| "accuracy": 0.95875, |
| "mean_class_recall": 0.9296875, |
| "nll": 0.14798192410675134, |
| "brier": 0.07208296294789789, |
| "ece15": 0.031237402444916955, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9575, |
| "error_rate_at_confidence_0_9": 0.02741514360313316, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 6000, |
| "accuracy": 0.9171666666666667, |
| "mean_class_recall": 0.8741111111111111, |
| "nll": 0.2872065840367128, |
| "brier": 0.1291518236859529, |
| "ece15": 0.02997403903668212, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.7491666666666666, |
| "error_rate_at_confidence_0_9": 0.03181312569521691, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 6.416181236130915e-05, |
| "brier": 3.664856836177276e-07, |
| "ece15": 6.406994017019252e-05, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 540, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0003909349271697467, |
| "brier": 5.411140895974333e-05, |
| "ece15": 0.000376397356322764, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9981481481481481, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 540, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 1.9301873747749096e-07, |
| "brier": 4.461551704214049e-12, |
| "ece15": 1.9301762319156735e-07, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 3500, |
| "accuracy": 0.988, |
| "mean_class_recall": 0.9820952380952381, |
| "nll": 0.052315917826989855, |
| "brier": 0.021002166306533596, |
| "ece15": 0.005742982405630235, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9757142857142858, |
| "error_rate_at_confidence_0_9": 0.007320644216691069, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 659, |
| "accuracy": 0.9954476479514416, |
| "mean_class_recall": 0.9949748743718594, |
| "nll": 0.006489090778116727, |
| "brier": 0.004985928434486637, |
| "ece15": 0.0035010905751582675, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9954476479514416, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": 0.004830545729071066 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 659, |
| "accuracy": 0.9984825493171472, |
| "mean_class_recall": 0.9983249581239532, |
| "nll": 0.007759166929265847, |
| "brier": 0.0031279766576844436, |
| "ece15": 0.0017302646891685911, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9969650986342944, |
| "error_rate_at_confidence_0_9": 0.0015220700152207, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 659, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0012916521858483854, |
| "brier": 0.0006719742504699832, |
| "ece15": 0.0010543087689309648, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9969650986342944, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 659, |
| "accuracy": 0.9984825493171472, |
| "mean_class_recall": 0.9989130434782609, |
| "nll": 0.0030258654215359524, |
| "brier": 0.0020473738981638956, |
| "ece15": 0.0002839890142556098, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9954476479514416, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 3321, |
| "accuracy": 0.7034025895814514, |
| "mean_class_recall": 0.7106693655944849, |
| "nll": 0.5271449745997652, |
| "brier": 0.3679898300956757, |
| "ece15": 0.03254066478455375, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.21499548328816623, |
| "error_rate_at_confidence_0_9": 0.011204481792717087, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 3321, |
| "accuracy": 0.692261367058115, |
| "mean_class_recall": 0.49711800080440777, |
| "nll": 1.181957709780574, |
| "brier": 0.48216427161657194, |
| "ece15": 0.04703079874679226, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.20776874435411022, |
| "error_rate_at_confidence_0_9": 0.07681159420289856, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9450619006663279, |
| "macro_task_balanced_accuracy": 0.9242156493985226 |
| }, |
| "recalibrated": { |
| "questions": 21634, |
| "accuracy": 0.8801423684940372, |
| "mean_class_recall": 0.8422636415920437, |
| "nll": 0.3441350952603644, |
| "brier": 0.1767857045083663, |
| "ece15": 0.035627089616313175, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.6567440140519553, |
| "error_rate_at_confidence_0_9": 0.014358108108108109, |
| "ordinal_expected_level_mae": 0.004973955616734103, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 426, |
| "accuracy": 0.9953051643192489, |
| "mean_class_recall": 0.9895833333333333, |
| "nll": 0.02085470758455793, |
| "brier": 0.009033562612123906, |
| "ece15": 0.005062121140742335, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9741784037558685, |
| "error_rate_at_confidence_0_9": 0.0024096385542168677, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 426, |
| "accuracy": 0.9835680751173709, |
| "mean_class_recall": 0.9635416666666667, |
| "nll": 0.029576905474440387, |
| "brier": 0.019371229609764937, |
| "ece15": 0.0128149515297424, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9741784037558685, |
| "error_rate_at_confidence_0_9": 0.004819277108433735, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 800, |
| "accuracy": 0.95875, |
| "mean_class_recall": 0.9296875, |
| "nll": 0.1252495255860491, |
| "brier": 0.06891977027707019, |
| "ece15": 0.02206119521855819, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.93375, |
| "error_rate_at_confidence_0_9": 0.0214190093708166, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 6000, |
| "accuracy": 0.9171666666666667, |
| "mean_class_recall": 0.8741111111111111, |
| "nll": 0.258923302286342, |
| "brier": 0.1323925339560934, |
| "ece15": 0.040786104751424794, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.6995, |
| "error_rate_at_confidence_0_9": 0.027162258756254467, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.00036120828507376734, |
| "brier": 6.118819643947097e-06, |
| "ece15": 0.0003596634947089362, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 540, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0010259839180316675, |
| "brier": 0.00015355821904600842, |
| "ece15": 0.0009831313211751498, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9981481481481481, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 540, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 5.4752266593721125e-06, |
| "brier": 1.1349090161477055e-09, |
| "ece15": 5.474942881804694e-06, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 3500, |
| "accuracy": 0.988, |
| "mean_class_recall": 0.9820952380952381, |
| "nll": 0.0471904212042291, |
| "brier": 0.02136284177729358, |
| "ece15": 0.004930332206950378, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9597142857142857, |
| "error_rate_at_confidence_0_9": 0.006251860672819291, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 659, |
| "accuracy": 0.9954476479514416, |
| "mean_class_recall": 0.9949748743718594, |
| "nll": 0.005989405513587896, |
| "brier": 0.004363630530423605, |
| "ece15": 0.003686325313576514, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9954476479514416, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": 0.004973955616734103 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 659, |
| "accuracy": 0.9984825493171472, |
| "mean_class_recall": 0.9983249581239532, |
| "nll": 0.0069631222649601515, |
| "brier": 0.003204452557959292, |
| "ece15": 0.001531985329986691, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9969650986342944, |
| "error_rate_at_confidence_0_9": 0.0015220700152207, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 659, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0018952623564185236, |
| "brier": 0.0008114028234666237, |
| "ece15": 0.0016108880631044219, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9954476479514416, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 659, |
| "accuracy": 0.9984825493171472, |
| "mean_class_recall": 0.9989130434782609, |
| "nll": 0.0033168872089549845, |
| "brier": 0.0020980871703588605, |
| "ece15": 0.0012518453771527926, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9954476479514416, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 3321, |
| "accuracy": 0.7034025895814514, |
| "mean_class_recall": 0.7106693655944849, |
| "nll": 0.5278136514287293, |
| "brier": 0.3689683794121852, |
| "ece15": 0.03240031037986953, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.1969286359530262, |
| "error_rate_at_confidence_0_9": 0.009174311926605505, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 3321, |
| "accuracy": 0.692261367058115, |
| "mean_class_recall": 0.49711800080440777, |
| "nll": 1.1560335557868109, |
| "brier": 0.498611302076152, |
| "ece15": 0.12617627958797675, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.17856067449563384, |
| "error_rate_at_confidence_0_9": 0.07251264755480608, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9450619006663279, |
| "macro_task_balanced_accuracy": 0.9242156493985226 |
| } |
| }, |
| "general": { |
| "decision-v7": { |
| "raw_clean": { |
| "n": 1200, |
| "nll": 0.5734957671127382, |
| "acc": 0.8658333333333333, |
| "ece": 0.08617509900940432, |
| "brier": 0.21792288610472857, |
| "mean_conf": 0.9509841477415141, |
| "confident_error_rate": 0.0675, |
| "coverage_at_0_9": 0.8525, |
| "accuracy_at_0_9": 0.9208211143695014, |
| "coverage_at_5pct_error": 0.7833333333333333, |
| "coverage_at_1pct_error": 0.6166666666666667, |
| "aurc": 0.0281999905676559, |
| "error_rate_at_0_9": 0.07917888563049853, |
| "confidence_bias": 0.08515081440818073, |
| "top_bins": { |
| "0.9": { |
| "n": 1023, |
| "errors": 81, |
| "error_rate": 0.07917888563049853 |
| }, |
| "0.95": { |
| "n": 962, |
| "errors": 53, |
| "error_rate": 0.0550935550935551 |
| }, |
| "0.99": { |
| "n": 885, |
| "errors": 34, |
| "error_rate": 0.0384180790960452 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9933333333333333, |
| "confidence_cutoff": 0.9999894749108045 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9447916666666667, |
| "confidence_cutoff": 0.9524106736120255 |
| } |
| }, |
| "score_mae": 0.4625639513458072, |
| "ranked_probability_score": 0.08810095245487483 |
| }, |
| "as_shipped_clean": { |
| "n": 1200, |
| "nll": 0.46540017253913407, |
| "acc": 0.8658333333333333, |
| "ece": 0.07412868458003202, |
| "brier": 0.20783721757683873, |
| "mean_conf": 0.9351787338923708, |
| "confident_error_rate": 0.041666666666666664, |
| "coverage_at_0_9": 0.7983333333333333, |
| "accuracy_at_0_9": 0.9478079331941545, |
| "coverage_at_5pct_error": 0.7783333333333333, |
| "coverage_at_1pct_error": 0.6141666666666666, |
| "aurc": 0.02814862771926311, |
| "error_rate_at_0_9": 0.05219206680584551, |
| "confidence_bias": 0.06934540055903748, |
| "top_bins": { |
| "0.9": { |
| "n": 958, |
| "errors": 50, |
| "error_rate": 0.05219206680584551 |
| }, |
| "0.95": { |
| "n": 912, |
| "errors": 40, |
| "error_rate": 0.043859649122807015 |
| }, |
| "0.99": { |
| "n": 822, |
| "errors": 18, |
| "error_rate": 0.021897810218978103 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9933333333333333, |
| "confidence_cutoff": 0.9998006494793819 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9458333333333333, |
| "confidence_cutoff": 0.8959586216727804 |
| } |
| }, |
| "score_mae": 0.47318044838075235, |
| "ranked_probability_score": 0.08431774694216215 |
| }, |
| "sources": { |
| "agnews": { |
| "n": 240, |
| "nll": 0.2925312694754093, |
| "acc": 0.9166666666666666, |
| "ece": 0.05790914903059574, |
| "brier": 0.1357836869201196, |
| "mean_conf": 0.9733704974572437, |
| "confident_error_rate": 0.04583333333333333, |
| "coverage_at_0_9": 0.9291666666666667, |
| "accuracy_at_0_9": 0.9506726457399103, |
| "coverage_at_5pct_error": 0.9333333333333333, |
| "coverage_at_1pct_error": 0.7583333333333333, |
| "aurc": 0.011635752911899793, |
| "error_rate_at_0_9": 0.04932735426008968, |
| "confidence_bias": 0.05670383079057706, |
| "top_bins": { |
| "0.9": { |
| "n": 223, |
| "errors": 11, |
| "error_rate": 0.04932735426008968 |
| }, |
| "0.95": { |
| "n": 217, |
| "errors": 9, |
| "error_rate": 0.041474654377880185 |
| }, |
| "0.99": { |
| "n": 197, |
| "errors": 4, |
| "error_rate": 0.02030456852791878 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9996851444568575 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9921275226980688 |
| } |
| } |
| }, |
| "amazon": { |
| "n": 80, |
| "nll": 1.2207591140196519, |
| "acc": 0.6125, |
| "ece": 0.1615943676102411, |
| "brier": 0.5707631816876242, |
| "mean_conf": 0.7431377036571911, |
| "confident_error_rate": 0.0375, |
| "coverage_at_0_9": 0.2375, |
| "accuracy_at_0_9": 0.8421052631578947, |
| "coverage_at_5pct_error": 0.075, |
| "coverage_at_1pct_error": 0.075, |
| "aurc": 0.272315413996461, |
| "error_rate_at_0_9": 0.15789473684210525, |
| "confidence_bias": 0.13063770365719107, |
| "top_bins": { |
| "0.9": { |
| "n": 19, |
| "errors": 3, |
| "error_rate": 0.15789473684210525 |
| }, |
| "0.95": { |
| "n": 14, |
| "errors": 2, |
| "error_rate": 0.14285714285714285 |
| }, |
| "0.99": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.7, |
| "confidence_cutoff": 0.7847517191083221 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.640625, |
| "confidence_cutoff": 0.5534749021853737 |
| } |
| }, |
| "score_mae": 0.5042438581009662, |
| "ranked_probability_score": 0.09073105215000309 |
| }, |
| "banking77": { |
| "n": 80, |
| "nll": 0.6278787803563556, |
| "acc": 0.8375, |
| "ece": 0.1237375681167378, |
| "brier": 0.25438988606734264, |
| "mean_conf": 0.8981608539632144, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.7625, |
| "accuracy_at_0_9": 0.9344262295081968, |
| "coverage_at_5pct_error": 0.6875, |
| "coverage_at_1pct_error": 0.6125, |
| "aurc": 0.035783745285874746, |
| "error_rate_at_0_9": 0.06557377049180328, |
| "confidence_bias": 0.06066085396321441, |
| "top_bins": { |
| "0.9": { |
| "n": 61, |
| "errors": 4, |
| "error_rate": 0.06557377049180328 |
| }, |
| "0.95": { |
| "n": 57, |
| "errors": 4, |
| "error_rate": 0.07017543859649122 |
| }, |
| "0.99": { |
| "n": 48, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9966194161989207 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.90625, |
| "confidence_cutoff": 0.8503200856915479 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 80, |
| "nll": 0.6125951836430432, |
| "acc": 0.875, |
| "ece": 0.11756712414134643, |
| "brier": 0.22505069986723222, |
| "mean_conf": 0.9514740522819943, |
| "confident_error_rate": 0.0875, |
| "coverage_at_0_9": 0.85, |
| "accuracy_at_0_9": 0.8970588235294118, |
| "coverage_at_5pct_error": 0.6875, |
| "coverage_at_1pct_error": 0.1125, |
| "aurc": 0.0511510509173725, |
| "error_rate_at_0_9": 0.10294117647058823, |
| "confidence_bias": 0.07647405228199433, |
| "top_bins": { |
| "0.9": { |
| "n": 68, |
| "errors": 7, |
| "error_rate": 0.10294117647058823 |
| }, |
| "0.95": { |
| "n": 67, |
| "errors": 7, |
| "error_rate": 0.1044776119402985 |
| }, |
| "0.99": { |
| "n": 61, |
| "errors": 4, |
| "error_rate": 0.06557377049180328 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.9999426387743003 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.921875, |
| "confidence_cutoff": 0.9819850746275051 |
| } |
| } |
| }, |
| "contrastive": { |
| "n": 160, |
| "nll": 0.07962468888836484, |
| "acc": 0.96875, |
| "ece": 0.026633113578874344, |
| "brier": 0.05007247893674288, |
| "mean_conf": 0.9953831135788744, |
| "confident_error_rate": 0.0125, |
| "coverage_at_0_9": 0.98125, |
| "accuracy_at_0_9": 0.9872611464968153, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.96875, |
| "aurc": 0.000752718799433766, |
| "error_rate_at_0_9": 0.012738853503184714, |
| "confidence_bias": 0.026633113578874434, |
| "top_bins": { |
| "0.9": { |
| "n": 157, |
| "errors": 2, |
| "error_rate": 0.012738853503184714 |
| }, |
| "0.95": { |
| "n": 156, |
| "errors": 2, |
| "error_rate": 0.01282051282051282 |
| }, |
| "0.99": { |
| "n": 148, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999957907736378 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999070137203582 |
| } |
| } |
| }, |
| "dbpedia14": { |
| "n": 80, |
| "nll": 0.02165084849307894, |
| "acc": 0.9875, |
| "ece": 0.01152608625473029, |
| "brier": 0.016104080808397182, |
| "mean_conf": 0.9959462330871315, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9875, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.9875, |
| "aurc": 0.00015625, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.008446233087131438, |
| "top_bins": { |
| "0.9": { |
| "n": 79, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 78, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 77, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999981627832084 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999896312065648 |
| } |
| } |
| }, |
| "imdb": { |
| "n": 80, |
| "nll": 0.06521915089368877, |
| "acc": 0.9625, |
| "ece": 0.02850319056358286, |
| "brier": 0.04639252879503235, |
| "mean_conf": 0.985401588169408, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.95, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.95, |
| "aurc": 0.0012680561790530144, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.022901588169408016, |
| "top_bins": { |
| "0.9": { |
| "n": 76, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 75, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 71, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999903501374642 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9998069111372814 |
| } |
| } |
| }, |
| "mnli": { |
| "n": 80, |
| "nll": 0.5051164719750952, |
| "acc": 0.9125, |
| "ece": 0.08269743141349266, |
| "brier": 0.16426794867102618, |
| "mean_conf": 0.9883215126078937, |
| "confident_error_rate": 0.075, |
| "coverage_at_0_9": 0.95, |
| "accuracy_at_0_9": 0.9210526315789473, |
| "coverage_at_5pct_error": 0.8875, |
| "coverage_at_1pct_error": 0.65, |
| "aurc": 0.015551107945666897, |
| "error_rate_at_0_9": 0.07894736842105263, |
| "confidence_bias": 0.07582151260789372, |
| "top_bins": { |
| "0.9": { |
| "n": 76, |
| "errors": 6, |
| "error_rate": 0.07894736842105263 |
| }, |
| "0.95": { |
| "n": 74, |
| "errors": 5, |
| "error_rate": 0.06756756756756757 |
| }, |
| "0.99": { |
| "n": 72, |
| "errors": 4, |
| "error_rate": 0.05555555555555555 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999963968506167 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.96875, |
| "confidence_cutoff": 0.9998914656545862 |
| } |
| } |
| }, |
| "sst5": { |
| "n": 80, |
| "nll": 1.2493438944839201, |
| "acc": 0.6125, |
| "ece": 0.24936946322063214, |
| "brier": 0.5917259991138234, |
| "mean_conf": 0.8126564640723977, |
| "confident_error_rate": 0.075, |
| "coverage_at_0_9": 0.3, |
| "accuracy_at_0_9": 0.75, |
| "coverage_at_5pct_error": 0.075, |
| "coverage_at_1pct_error": 0.075, |
| "aurc": 0.3025049973121495, |
| "error_rate_at_0_9": 0.25, |
| "confidence_bias": 0.20015646407239762, |
| "top_bins": { |
| "0.9": { |
| "n": 24, |
| "errors": 6, |
| "error_rate": 0.25 |
| }, |
| "0.95": { |
| "n": 10, |
| "errors": 1, |
| "error_rate": 0.1 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.625, |
| "confidence_cutoff": 0.8548749588336553 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.625, |
| "confidence_cutoff": 0.7024572592103087 |
| } |
| }, |
| "score_mae": 0.48716165684142193, |
| "ranked_probability_score": 0.08942157579848162 |
| }, |
| "trec": { |
| "n": 80, |
| "nll": 0.2837433451436081, |
| "acc": 0.9625, |
| "ece": 0.028515907037196753, |
| "brier": 0.057680311047061936, |
| "mean_conf": 0.9910159070371967, |
| "confident_error_rate": 0.025, |
| "coverage_at_0_9": 0.9875, |
| "accuracy_at_0_9": 0.9746835443037974, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.275, |
| "aurc": 0.01757155053741791, |
| "error_rate_at_0_9": 0.02531645569620253, |
| "confidence_bias": 0.02851590703719664, |
| "top_bins": { |
| "0.9": { |
| "n": 79, |
| "errors": 2, |
| "error_rate": 0.02531645569620253 |
| }, |
| "0.95": { |
| "n": 78, |
| "errors": 2, |
| "error_rate": 0.02564102564102564 |
| }, |
| "0.99": { |
| "n": 77, |
| "errors": 2, |
| "error_rate": 0.025974025974025976 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.9999924765263039 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9999100942077365 |
| } |
| } |
| }, |
| "yelp": { |
| "n": 160, |
| "nll": 0.6789263064378058, |
| "acc": 0.76875, |
| "ece": 0.10810206862028397, |
| "brier": 0.34184380448059815, |
| "mean_conf": 0.8753444869898281, |
| "confident_error_rate": 0.05625, |
| "coverage_at_0_9": 0.6, |
| "accuracy_at_0_9": 0.90625, |
| "coverage_at_5pct_error": 0.41875, |
| "coverage_at_1pct_error": 0.1875, |
| "aurc": 0.08672761133002724, |
| "error_rate_at_0_9": 0.09375, |
| "confidence_bias": 0.1065944869898281, |
| "top_bins": { |
| "0.9": { |
| "n": 96, |
| "errors": 9, |
| "error_rate": 0.09375 |
| }, |
| "0.95": { |
| "n": 86, |
| "errors": 8, |
| "error_rate": 0.09302325581395349 |
| }, |
| "0.99": { |
| "n": 68, |
| "errors": 4, |
| "error_rate": 0.058823529411764705 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.925, |
| "confidence_cutoff": 0.971684991366676 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.84375, |
| "confidence_cutoff": 0.7267623543739319 |
| } |
| }, |
| "score_mae": 0.428135830199869, |
| "ranked_probability_score": 0.07280061287800174 |
| } |
| }, |
| "types": { |
| "choice": { |
| "n": 480, |
| "nll": 0.3225147096064765, |
| "acc": 0.9291666666666667, |
| "ece": 0.05269727298309242, |
| "brier": 0.11939209773780447, |
| "mean_conf": 0.9740635473132214, |
| "confident_error_rate": 0.0375, |
| "coverage_at_0_9": 0.9333333333333333, |
| "accuracy_at_0_9": 0.9598214285714286, |
| "coverage_at_5pct_error": 0.9479166666666666, |
| "coverage_at_1pct_error": 0.7354166666666667, |
| "aurc": 0.010523805656224608, |
| "error_rate_at_0_9": 0.04017857142857143, |
| "confidence_bias": 0.04489688064655473, |
| "top_bins": { |
| "0.9": { |
| "n": 448, |
| "errors": 18, |
| "error_rate": 0.04017857142857143 |
| }, |
| "0.95": { |
| "n": 439, |
| "errors": 17, |
| "error_rate": 0.0387243735763098 |
| }, |
| "0.99": { |
| "n": 416, |
| "errors": 8, |
| "error_rate": 0.019230769230769232 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9958333333333333, |
| "confidence_cutoff": 0.999989087291256 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9895833333333334, |
| "confidence_cutoff": 0.9991522595300467 |
| } |
| } |
| }, |
| "noul": { |
| "n": 480, |
| "nll": 0.2630675588624098, |
| "acc": 0.925, |
| "ece": 0.048426932319500904, |
| "brier": 0.11894188754077414, |
| "mean_conf": 0.9734269323195011, |
| "confident_error_rate": 0.035416666666666666, |
| "coverage_at_0_9": 0.91875, |
| "accuracy_at_0_9": 0.9614512471655329, |
| "coverage_at_5pct_error": 0.9416666666666667, |
| "coverage_at_1pct_error": 0.71875, |
| "aurc": 0.014292966145900442, |
| "error_rate_at_0_9": 0.03854875283446712, |
| "confidence_bias": 0.048426932319501015, |
| "top_bins": { |
| "0.9": { |
| "n": 441, |
| "errors": 17, |
| "error_rate": 0.03854875283446712 |
| }, |
| "0.95": { |
| "n": 431, |
| "errors": 15, |
| "error_rate": 0.03480278422273782 |
| }, |
| "0.99": { |
| "n": 397, |
| "errors": 9, |
| "error_rate": 0.022670025188916875 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9958333333333333, |
| "confidence_cutoff": 0.9999067506240331 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9791666666666666, |
| "confidence_cutoff": 0.9950087776054062 |
| } |
| } |
| }, |
| "score": { |
| "n": 240, |
| "nll": 1.1558363257578976, |
| "acc": 0.6208333333333333, |
| "ece": 0.1746159580142597, |
| "brier": 0.5625181173270365, |
| "mean_conf": 0.7809127101964097, |
| "confident_error_rate": 0.0625, |
| "coverage_at_0_9": 0.2875, |
| "accuracy_at_0_9": 0.782608695652174, |
| "coverage_at_5pct_error": 0.016666666666666666, |
| "coverage_at_1pct_error": 0.016666666666666666, |
| "aurc": 0.28163895271092954, |
| "error_rate_at_0_9": 0.21739130434782608, |
| "confidence_bias": 0.16007937686307638, |
| "top_bins": { |
| "0.9": { |
| "n": 69, |
| "errors": 15, |
| "error_rate": 0.21739130434782608 |
| }, |
| "0.95": { |
| "n": 42, |
| "errors": 8, |
| "error_rate": 0.19047619047619047 |
| }, |
| "0.99": { |
| "n": 9, |
| "errors": 1, |
| "error_rate": 0.1111111111111111 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.6916666666666667, |
| "confidence_cutoff": 0.8195696673851967 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.6614583333333334, |
| "confidence_cutoff": 0.6215591950077792 |
| } |
| }, |
| "score_mae": 0.47318044838075235, |
| "ranked_probability_score": 0.08431774694216215 |
| } |
| } |
| }, |
| "transfer-v4": { |
| "raw_clean": { |
| "n": 656, |
| "nll": 0.840653344789253, |
| "acc": 0.8384146341463414, |
| "ece": 0.11570395215040587, |
| "brier": 0.2772648298322645, |
| "mean_conf": 0.9433821481191502, |
| "confident_error_rate": 0.0899390243902439, |
| "coverage_at_0_9": 0.8429878048780488, |
| "accuracy_at_0_9": 0.8933092224231465, |
| "coverage_at_5pct_error": 0.5777439024390244, |
| "coverage_at_1pct_error": 0.3551829268292683, |
| "aurc": 0.0494438083179723, |
| "error_rate_at_0_9": 0.10669077757685352, |
| "confidence_bias": 0.10496751397280879, |
| "top_bins": { |
| "0.9": { |
| "n": 553, |
| "errors": 59, |
| "error_rate": 0.10669077757685352 |
| }, |
| "0.95": { |
| "n": 527, |
| "errors": 54, |
| "error_rate": 0.10246679316888045 |
| }, |
| "0.99": { |
| "n": 472, |
| "errors": 34, |
| "error_rate": 0.07203389830508475 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9634146341463414, |
| "confidence_cutoff": 0.9999382193883597 |
| }, |
| "0.8": { |
| "coverage": 0.8003048780487805, |
| "accuracy": 0.900952380952381, |
| "confidence_cutoff": 0.951274711387887 |
| } |
| }, |
| "score_mae": 0.2762329771748361, |
| "ranked_probability_score": 0.12228321967451691 |
| }, |
| "as_shipped_clean": { |
| "n": 656, |
| "nll": 0.6710659810048166, |
| "acc": 0.8384146341463414, |
| "ece": 0.09231046175305074, |
| "brier": 0.26721979560494846, |
| "mean_conf": 0.9260829686930101, |
| "confident_error_rate": 0.07926829268292683, |
| "coverage_at_0_9": 0.7987804878048781, |
| "accuracy_at_0_9": 0.9007633587786259, |
| "coverage_at_5pct_error": 0.5746951219512195, |
| "coverage_at_1pct_error": 0.3551829268292683, |
| "aurc": 0.049407167334717506, |
| "error_rate_at_0_9": 0.09923664122137404, |
| "confidence_bias": 0.08766833454666867, |
| "top_bins": { |
| "0.9": { |
| "n": 524, |
| "errors": 52, |
| "error_rate": 0.09923664122137404 |
| }, |
| "0.95": { |
| "n": 493, |
| "errors": 42, |
| "error_rate": 0.08519269776876268 |
| }, |
| "0.99": { |
| "n": 422, |
| "errors": 23, |
| "error_rate": 0.054502369668246446 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9573170731707317, |
| "confidence_cutoff": 0.9992490894147191 |
| }, |
| "0.8": { |
| "coverage": 0.8003048780487805, |
| "accuracy": 0.8990476190476191, |
| "confidence_cutoff": 0.8992331757610149 |
| } |
| }, |
| "score_mae": 0.27589619803682164, |
| "ranked_probability_score": 0.11777542530670457 |
| }, |
| "sources": { |
| "composition_holdout": { |
| "n": 96, |
| "nll": 0.946778069631199, |
| "acc": 0.8645833333333334, |
| "ece": 0.14288844371789414, |
| "brier": 0.2646184933261653, |
| "mean_conf": 0.9787006948112301, |
| "confident_error_rate": 0.11458333333333333, |
| "coverage_at_0_9": 0.9479166666666666, |
| "accuracy_at_0_9": 0.8791208791208791, |
| "coverage_at_5pct_error": 0.6875, |
| "coverage_at_1pct_error": 0.65625, |
| "aurc": 0.032783504799663264, |
| "error_rate_at_0_9": 0.12087912087912088, |
| "confidence_bias": 0.11411736147789675, |
| "top_bins": { |
| "0.9": { |
| "n": 91, |
| "errors": 11, |
| "error_rate": 0.12087912087912088 |
| }, |
| "0.95": { |
| "n": 90, |
| "errors": 11, |
| "error_rate": 0.12222222222222222 |
| }, |
| "0.99": { |
| "n": 89, |
| "errors": 11, |
| "error_rate": 0.12359550561797752 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5104166666666666, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999696869828117 |
| }, |
| "0.8": { |
| "coverage": 0.8125, |
| "accuracy": 0.8974358974358975, |
| "confidence_cutoff": 0.9992490894147191 |
| } |
| } |
| }, |
| "contrastive": { |
| "n": 80, |
| "nll": 0.5115728584657595, |
| "acc": 0.875, |
| "ece": 0.13006292045346943, |
| "brier": 0.23406192561698297, |
| "mean_conf": 0.9705133073135578, |
| "confident_error_rate": 0.0875, |
| "coverage_at_0_9": 0.9125, |
| "accuracy_at_0_9": 0.9041095890410958, |
| "coverage_at_5pct_error": 0.85, |
| "coverage_at_1pct_error": 0.8, |
| "aurc": 0.0169297143647776, |
| "error_rate_at_0_9": 0.0958904109589041, |
| "confidence_bias": 0.09551330731355778, |
| "top_bins": { |
| "0.9": { |
| "n": 73, |
| "errors": 7, |
| "error_rate": 0.0958904109589041 |
| }, |
| "0.95": { |
| "n": 71, |
| "errors": 6, |
| "error_rate": 0.08450704225352113 |
| }, |
| "0.99": { |
| "n": 68, |
| "errors": 3, |
| "error_rate": 0.04411764705882353 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999941701311645 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9997393381382442 |
| } |
| }, |
| "score_mae": 0.27589619803682164, |
| "ranked_probability_score": 0.11777542530670457 |
| }, |
| "emotion": { |
| "n": 80, |
| "nll": 1.2094004500158717, |
| "acc": 0.675, |
| "ece": 0.19812590622433582, |
| "brier": 0.4998620899742045, |
| "mean_conf": 0.7808203087114324, |
| "confident_error_rate": 0.075, |
| "coverage_at_0_9": 0.4375, |
| "accuracy_at_0_9": 0.8285714285714286, |
| "coverage_at_5pct_error": 0.075, |
| "coverage_at_1pct_error": 0.075, |
| "aurc": 0.1832973887490438, |
| "error_rate_at_0_9": 0.17142857142857143, |
| "confidence_bias": 0.10582030871143233, |
| "top_bins": { |
| "0.9": { |
| "n": 35, |
| "errors": 6, |
| "error_rate": 0.17142857142857143 |
| }, |
| "0.95": { |
| "n": 30, |
| "errors": 4, |
| "error_rate": 0.13333333333333333 |
| }, |
| "0.99": { |
| "n": 12, |
| "errors": 1, |
| "error_rate": 0.08333333333333333 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.8, |
| "confidence_cutoff": 0.8500919215367448 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.734375, |
| "confidence_cutoff": 0.5761070047579089 |
| } |
| } |
| }, |
| "mmlu": { |
| "n": 80, |
| "nll": 0.9027907855123264, |
| "acc": 0.6875, |
| "ece": 0.14251964152159943, |
| "brier": 0.4317941195669751, |
| "mean_conf": 0.8238540468148077, |
| "confident_error_rate": 0.075, |
| "coverage_at_0_9": 0.5, |
| "accuracy_at_0_9": 0.85, |
| "coverage_at_5pct_error": 0.375, |
| "coverage_at_1pct_error": 0.325, |
| "aurc": 0.11589530494156297, |
| "error_rate_at_0_9": 0.15, |
| "confidence_bias": 0.13635404681480767, |
| "top_bins": { |
| "0.9": { |
| "n": 40, |
| "errors": 6, |
| "error_rate": 0.15 |
| }, |
| "0.95": { |
| "n": 30, |
| "errors": 1, |
| "error_rate": 0.03333333333333333 |
| }, |
| "0.99": { |
| "n": 19, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.85, |
| "confidence_cutoff": 0.9017783630155174 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.8125, |
| "confidence_cutoff": 0.613701094142181 |
| } |
| } |
| }, |
| "paws": { |
| "n": 80, |
| "nll": 0.9771542282927816, |
| "acc": 0.8125, |
| "ece": 0.1651648000763565, |
| "brier": 0.31695123775026923, |
| "mean_conf": 0.9524627544936155, |
| "confident_error_rate": 0.125, |
| "coverage_at_0_9": 0.8625, |
| "accuracy_at_0_9": 0.855072463768116, |
| "coverage_at_5pct_error": 0.15, |
| "coverage_at_1pct_error": 0.15, |
| "aurc": 0.0997884943666906, |
| "error_rate_at_0_9": 0.14492753623188406, |
| "confidence_bias": 0.1399627544936155, |
| "top_bins": { |
| "0.9": { |
| "n": 69, |
| "errors": 10, |
| "error_rate": 0.14492753623188406 |
| }, |
| "0.95": { |
| "n": 69, |
| "errors": 10, |
| "error_rate": 0.14492753623188406 |
| }, |
| "0.99": { |
| "n": 61, |
| "errors": 6, |
| "error_rate": 0.09836065573770492 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.875, |
| "confidence_cutoff": 0.9997721781585782 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.890625, |
| "confidence_cutoff": 0.9846682768581441 |
| } |
| } |
| }, |
| "qnli": { |
| "n": 80, |
| "nll": 0.32010583310414403, |
| "acc": 0.925, |
| "ece": 0.06578826678776872, |
| "brier": 0.13874220003757964, |
| "mean_conf": 0.9719094562930305, |
| "confident_error_rate": 0.05, |
| "coverage_at_0_9": 0.925, |
| "accuracy_at_0_9": 0.9459459459459459, |
| "coverage_at_5pct_error": 0.8625, |
| "coverage_at_1pct_error": 0.6, |
| "aurc": 0.015384772492900507, |
| "error_rate_at_0_9": 0.05405405405405406, |
| "confidence_bias": 0.04690945629303045, |
| "top_bins": { |
| "0.9": { |
| "n": 74, |
| "errors": 4, |
| "error_rate": 0.05405405405405406 |
| }, |
| "0.95": { |
| "n": 72, |
| "errors": 4, |
| "error_rate": 0.05555555555555555 |
| }, |
| "0.99": { |
| "n": 66, |
| "errors": 2, |
| "error_rate": 0.030303030303030304 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9997714445133897 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.96875, |
| "confidence_cutoff": 0.9937610824296935 |
| } |
| } |
| }, |
| "sciq": { |
| "n": 80, |
| "nll": 0.00696809639784113, |
| "acc": 1.0, |
| "ece": 0.006546558428673463, |
| "brier": 0.0013901896311441294, |
| "mean_conf": 0.9934534415713265, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.975, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.006546558428673466, |
| "top_bins": { |
| "0.9": { |
| "n": 78, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 76, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 72, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999699221173486 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9997295605534583 |
| } |
| } |
| }, |
| "tweet_offensive": { |
| "n": 80, |
| "nll": 0.4386151088933339, |
| "acc": 0.8625, |
| "ece": 0.10954228653253775, |
| "brier": 0.2508583693920234, |
| "mean_conf": 0.926426194311437, |
| "confident_error_rate": 0.1, |
| "coverage_at_0_9": 0.8, |
| "accuracy_at_0_9": 0.875, |
| "coverage_at_5pct_error": 0.575, |
| "coverage_at_1pct_error": 0.4875, |
| "aurc": 0.05180455375068149, |
| "error_rate_at_0_9": 0.125, |
| "confidence_bias": 0.06392619431143698, |
| "top_bins": { |
| "0.9": { |
| "n": 64, |
| "errors": 8, |
| "error_rate": 0.125 |
| }, |
| "0.95": { |
| "n": 55, |
| "errors": 6, |
| "error_rate": 0.10909090909090909 |
| }, |
| "0.99": { |
| "n": 35, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.9842379652550892 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.875, |
| "confidence_cutoff": 0.902661270668069 |
| } |
| } |
| } |
| }, |
| "types": { |
| "choice": { |
| "n": 336, |
| "nll": 0.7750697655913044, |
| "acc": 0.8095238095238095, |
| "ece": 0.09777001761032751, |
| "brier": 0.2977591883723148, |
| "mean_conf": 0.8982306263978672, |
| "confident_error_rate": 0.06845238095238096, |
| "coverage_at_0_9": 0.7261904761904762, |
| "accuracy_at_0_9": 0.9057377049180327, |
| "coverage_at_5pct_error": 0.4583333333333333, |
| "coverage_at_1pct_error": 0.3630952380952381, |
| "aurc": 0.05730883047817376, |
| "error_rate_at_0_9": 0.0942622950819672, |
| "confidence_bias": 0.08870681687405768, |
| "top_bins": { |
| "0.9": { |
| "n": 244, |
| "errors": 23, |
| "error_rate": 0.0942622950819672 |
| }, |
| "0.95": { |
| "n": 226, |
| "errors": 16, |
| "error_rate": 0.07079646017699115 |
| }, |
| "0.99": { |
| "n": 192, |
| "errors": 12, |
| "error_rate": 0.0625 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9464285714285714, |
| "confidence_cutoff": 0.9979451630958165 |
| }, |
| "0.8": { |
| "coverage": 0.8005952380952381, |
| "accuracy": 0.8921933085501859, |
| "confidence_cutoff": 0.805212947551472 |
| } |
| } |
| }, |
| "noul": { |
| "n": 280, |
| "nll": 0.4959650021746835, |
| "acc": 0.8857142857142857, |
| "ece": 0.08836865713253901, |
| "brier": 0.20187194492263405, |
| "mean_conf": 0.9573703050826868, |
| "confident_error_rate": 0.07857142857142857, |
| "coverage_at_0_9": 0.8821428571428571, |
| "accuracy_at_0_9": 0.9109311740890689, |
| "coverage_at_5pct_error": 0.7535714285714286, |
| "coverage_at_1pct_error": 0.25, |
| "aurc": 0.039651831256480245, |
| "error_rate_at_0_9": 0.08906882591093117, |
| "confidence_bias": 0.07165601936840116, |
| "top_bins": { |
| "0.9": { |
| "n": 247, |
| "errors": 22, |
| "error_rate": 0.08906882591093117 |
| }, |
| "0.95": { |
| "n": 236, |
| "errors": 20, |
| "error_rate": 0.0847457627118644 |
| }, |
| "0.99": { |
| "n": 202, |
| "errors": 8, |
| "error_rate": 0.039603960396039604 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9571428571428572, |
| "confidence_cutoff": 0.9995423463157226 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9330357142857143, |
| "confidence_cutoff": 0.9697351853044945 |
| } |
| } |
| }, |
| "score": { |
| "n": 40, |
| "nll": 1.0231410422892537, |
| "acc": 0.75, |
| "ece": 0.26013051552429656, |
| "brier": 0.46812385113527205, |
| "mean_conf": 0.9410312892444732, |
| "confident_error_rate": 0.175, |
| "coverage_at_0_9": 0.825, |
| "accuracy_at_0_9": 0.7878787878787878, |
| "coverage_at_5pct_error": 0.65, |
| "coverage_at_1pct_error": 0.6, |
| "aurc": 0.07355999344204343, |
| "error_rate_at_0_9": 0.21212121212121213, |
| "confidence_bias": 0.1910312892444732, |
| "top_bins": { |
| "0.9": { |
| "n": 33, |
| "errors": 7, |
| "error_rate": 0.21212121212121213 |
| }, |
| "0.95": { |
| "n": 31, |
| "errors": 6, |
| "error_rate": 0.1935483870967742 |
| }, |
| "0.99": { |
| "n": 28, |
| "errors": 3, |
| "error_rate": 0.10714285714285714 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9999132212992297 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.78125, |
| "confidence_cutoff": 0.9461605303081507 |
| } |
| }, |
| "score_mae": 0.27589619803682164, |
| "ranked_probability_score": 0.11777542530670457 |
| } |
| } |
| } |
| }, |
| "temperature": 1.334968103387446 |
| } |
| ], |
| "kev_4b_security": { |
| "test": { |
| "raw": { |
| "questions": 21634, |
| "accuracy": 0.5048996949246556, |
| "mean_class_recall": 0.5647195198307282, |
| "nll": 1.2328084492499014, |
| "brier": 0.7147920651499965, |
| "ece15": 0.3370024824919156, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.44924655634649163, |
| "error_rate_at_confidence_0_9": 0.34736083959255065, |
| "ordinal_expected_level_mae": 0.2811370953407191, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 426, |
| "accuracy": 0.5422535211267606, |
| "mean_class_recall": 0.7045454545454546, |
| "nll": 1.2120829748020316, |
| "brier": 0.6804576908439421, |
| "ece15": 0.2992356768561654, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.5187793427230047, |
| "error_rate_at_confidence_0_9": 0.3936651583710407, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 426, |
| "accuracy": 0.7300469483568075, |
| "mean_class_recall": 0.8257575757575757, |
| "nll": 0.6169123027998439, |
| "brier": 0.3918536224381035, |
| "ece15": 0.1421670820222778, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.57981220657277, |
| "error_rate_at_confidence_0_9": 0.15384615384615385, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 800, |
| "accuracy": 0.79875, |
| "mean_class_recall": 0.49921875, |
| "nll": 0.5895657949467273, |
| "brier": 0.35429205012331033, |
| "ece15": 0.1133528731473528, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.2175, |
| "error_rate_at_confidence_0_9": 0.4942528735632184, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 6000, |
| "accuracy": 0.2628333333333333, |
| "mean_class_recall": 0.5085555555555555, |
| "nll": 1.2759496387732032, |
| "brier": 0.9560488943280168, |
| "ece15": 0.5505303178793823, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.12266666666666666, |
| "error_rate_at_confidence_0_9": 0.453804347826087, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 0.6290322580645161, |
| "mean_class_recall": 0.6714285714285715, |
| "nll": 1.0817458611434574, |
| "brier": 0.6236238714203711, |
| "ece15": 0.30916032986760006, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.75, |
| "error_rate_at_confidence_0_9": 0.34408602150537637, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 540, |
| "accuracy": 0.7981481481481482, |
| "mean_class_recall": 0.7528344671201813, |
| "nll": 0.4871733371313632, |
| "brier": 0.29625578551796805, |
| "ece15": 0.15242774183988111, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.7962962962962963, |
| "error_rate_at_confidence_0_9": 0.08837209302325581, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 540, |
| "accuracy": 0.9944444444444445, |
| "mean_class_recall": 0.9886363636363636, |
| "nll": 0.01641532065146085, |
| "brier": 0.007136931607478901, |
| "ece15": 0.013212748100349914, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9777777777777777, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 3500, |
| "accuracy": 0.5065714285714286, |
| "mean_class_recall": 0.6710476190476191, |
| "nll": 1.1359565067657027, |
| "brier": 0.6406729439019092, |
| "ece15": 0.23932739622085006, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.38971428571428574, |
| "error_rate_at_confidence_0_9": 0.3878299120234604, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 659, |
| "accuracy": 0.8118361153262519, |
| "mean_class_recall": 0.794717762064859, |
| "nll": 0.47493573588084764, |
| "brier": 0.2794985686136285, |
| "ece15": 0.05201558678606951, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.4931714719271624, |
| "error_rate_at_confidence_0_9": 0.04923076923076923, |
| "ordinal_expected_level_mae": 0.2811370953407191 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 659, |
| "accuracy": 0.7177541729893778, |
| "mean_class_recall": 0.6923577773226661, |
| "nll": 0.5581275002335633, |
| "brier": 0.3443282053616745, |
| "ece15": 0.09062810464195983, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.44613050075872535, |
| "error_rate_at_confidence_0_9": 0.04421768707482993, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 659, |
| "accuracy": 0.7814871016691958, |
| "mean_class_recall": 0.7019105077928607, |
| "nll": 0.4980710054489271, |
| "brier": 0.31257573538065736, |
| "ece15": 0.09857831652926007, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.590288315629742, |
| "error_rate_at_confidence_0_9": 0.09511568123393316, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 659, |
| "accuracy": 0.9711684370257967, |
| "mean_class_recall": 0.9693685820406379, |
| "nll": 0.10259254119815524, |
| "brier": 0.05091487551434137, |
| "ece15": 0.030811660924900536, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.8679817905918058, |
| "error_rate_at_confidence_0_9": 0.006993006993006993, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 3321, |
| "accuracy": 0.47756699789220114, |
| "mean_class_recall": 0.5, |
| "nll": 1.2951885811608232, |
| "brier": 0.8663028363232907, |
| "ece15": 0.42951687019840074, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.7633242999096658, |
| "error_rate_at_confidence_0_9": 0.504930966469428, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 3321, |
| "accuracy": 0.48087925323697683, |
| "mean_class_recall": 0.12233932942660039, |
| "nll": 2.41017951541039, |
| "brier": 0.8961545633575424, |
| "ece15": 0.3893600279043549, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.5453176753989762, |
| "error_rate_at_confidence_0_9": 0.4870237437879624, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.6787694400132313, |
| "macro_task_balanced_accuracy": 0.671622736838496 |
| }, |
| "recalibrated": { |
| "questions": 21634, |
| "accuracy": 0.5048996949246556, |
| "mean_class_recall": 0.5647195198307282, |
| "nll": 1.2328084492499014, |
| "brier": 0.7147920651499965, |
| "ece15": 0.3370024824919156, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.44924655634649163, |
| "error_rate_at_confidence_0_9": 0.34736083959255065, |
| "ordinal_expected_level_mae": 0.2811370953407191, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 426, |
| "accuracy": 0.5422535211267606, |
| "mean_class_recall": 0.7045454545454546, |
| "nll": 1.2120829748020316, |
| "brier": 0.6804576908439421, |
| "ece15": 0.2992356768561654, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.5187793427230047, |
| "error_rate_at_confidence_0_9": 0.3936651583710407, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 426, |
| "accuracy": 0.7300469483568075, |
| "mean_class_recall": 0.8257575757575757, |
| "nll": 0.6169123027998439, |
| "brier": 0.3918536224381035, |
| "ece15": 0.1421670820222778, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.57981220657277, |
| "error_rate_at_confidence_0_9": 0.15384615384615385, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 800, |
| "accuracy": 0.79875, |
| "mean_class_recall": 0.49921875, |
| "nll": 0.5895657949467273, |
| "brier": 0.35429205012331033, |
| "ece15": 0.1133528731473528, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.2175, |
| "error_rate_at_confidence_0_9": 0.4942528735632184, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 6000, |
| "accuracy": 0.2628333333333333, |
| "mean_class_recall": 0.5085555555555555, |
| "nll": 1.2759496387732032, |
| "brier": 0.9560488943280168, |
| "ece15": 0.5505303178793823, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.12266666666666666, |
| "error_rate_at_confidence_0_9": 0.453804347826087, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 0.6290322580645161, |
| "mean_class_recall": 0.6714285714285715, |
| "nll": 1.0817458611434574, |
| "brier": 0.6236238714203711, |
| "ece15": 0.30916032986760006, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.75, |
| "error_rate_at_confidence_0_9": 0.34408602150537637, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 540, |
| "accuracy": 0.7981481481481482, |
| "mean_class_recall": 0.7528344671201813, |
| "nll": 0.4871733371313632, |
| "brier": 0.29625578551796805, |
| "ece15": 0.15242774183988111, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.7962962962962963, |
| "error_rate_at_confidence_0_9": 0.08837209302325581, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 540, |
| "accuracy": 0.9944444444444445, |
| "mean_class_recall": 0.9886363636363636, |
| "nll": 0.01641532065146085, |
| "brier": 0.007136931607478901, |
| "ece15": 0.013212748100349914, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9777777777777777, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 3500, |
| "accuracy": 0.5065714285714286, |
| "mean_class_recall": 0.6710476190476191, |
| "nll": 1.1359565067657027, |
| "brier": 0.6406729439019092, |
| "ece15": 0.23932739622085006, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.38971428571428574, |
| "error_rate_at_confidence_0_9": 0.3878299120234604, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 659, |
| "accuracy": 0.8118361153262519, |
| "mean_class_recall": 0.794717762064859, |
| "nll": 0.47493573588084764, |
| "brier": 0.2794985686136285, |
| "ece15": 0.05201558678606951, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.4931714719271624, |
| "error_rate_at_confidence_0_9": 0.04923076923076923, |
| "ordinal_expected_level_mae": 0.2811370953407191 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 659, |
| "accuracy": 0.7177541729893778, |
| "mean_class_recall": 0.6923577773226661, |
| "nll": 0.5581275002335633, |
| "brier": 0.3443282053616745, |
| "ece15": 0.09062810464195983, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.44613050075872535, |
| "error_rate_at_confidence_0_9": 0.04421768707482993, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 659, |
| "accuracy": 0.7814871016691958, |
| "mean_class_recall": 0.7019105077928607, |
| "nll": 0.4980710054489271, |
| "brier": 0.31257573538065736, |
| "ece15": 0.09857831652926007, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.590288315629742, |
| "error_rate_at_confidence_0_9": 0.09511568123393316, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 659, |
| "accuracy": 0.9711684370257967, |
| "mean_class_recall": 0.9693685820406379, |
| "nll": 0.10259254119815524, |
| "brier": 0.05091487551434137, |
| "ece15": 0.030811660924900536, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.8679817905918058, |
| "error_rate_at_confidence_0_9": 0.006993006993006993, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 3321, |
| "accuracy": 0.47756699789220114, |
| "mean_class_recall": 0.5, |
| "nll": 1.2951885811608232, |
| "brier": 0.8663028363232907, |
| "ece15": 0.42951687019840074, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.7633242999096658, |
| "error_rate_at_confidence_0_9": 0.504930966469428, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 3321, |
| "accuracy": 0.48087925323697683, |
| "mean_class_recall": 0.12233932942660039, |
| "nll": 2.41017951541039, |
| "brier": 0.8961545633575424, |
| "ece15": 0.3893600279043549, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.5453176753989762, |
| "error_rate_at_confidence_0_9": 0.4870237437879624, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.6787694400132313, |
| "macro_task_balanced_accuracy": 0.671622736838496 |
| } |
| }, |
| "test_used_for_selection": false |
| }, |
| "kev_4b_general": { |
| "decision-v7": { |
| "raw_clean": { |
| "n": 1200, |
| "nll": 0.46776271359373983, |
| "acc": 0.8708333333333333, |
| "ece": 0.07324011844924935, |
| "brier": 0.2068727227531146, |
| "mean_conf": 0.9391164907697294, |
| "confident_error_rate": 0.04833333333333333, |
| "coverage_at_0_9": 0.8166666666666667, |
| "accuracy_at_0_9": 0.9408163265306122, |
| "coverage_at_5pct_error": 0.7791666666666667, |
| "coverage_at_1pct_error": 0.6133333333333333, |
| "aurc": 0.02837551275061399, |
| "error_rate_at_0_9": 0.05918367346938776, |
| "confidence_bias": 0.06828315743639601, |
| "top_bins": { |
| "0.9": { |
| "n": 980, |
| "errors": 58, |
| "error_rate": 0.05918367346938776 |
| }, |
| "0.95": { |
| "n": 915, |
| "errors": 39, |
| "error_rate": 0.04262295081967213 |
| }, |
| "0.99": { |
| "n": 829, |
| "errors": 21, |
| "error_rate": 0.025331724969843185 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9916666666666667, |
| "confidence_cutoff": 0.999598385219589 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.94375, |
| "confidence_cutoff": 0.9220693757732149 |
| } |
| }, |
| "score_mae": 0.4635506535819828, |
| "ranked_probability_score": 0.08574066842436183 |
| }, |
| "as_shipped_clean": { |
| "n": 1200, |
| "nll": 0.35957326104404586, |
| "acc": 0.8708333333333333, |
| "ece": 0.020190532051177417, |
| "brier": 0.18914513739887015, |
| "mean_conf": 0.8656556446393923, |
| "confident_error_rate": 0.013333333333333334, |
| "coverage_at_0_9": 0.6583333333333333, |
| "accuracy_at_0_9": 0.979746835443038, |
| "coverage_at_5pct_error": 0.7883333333333333, |
| "coverage_at_1pct_error": 0.585, |
| "aurc": 0.02903042619721546, |
| "error_rate_at_0_9": 0.020253164556962026, |
| "confidence_bias": -0.005177688693941085, |
| "top_bins": { |
| "0.9": { |
| "n": 790, |
| "errors": 16, |
| "error_rate": 0.020253164556962026 |
| }, |
| "0.95": { |
| "n": 697, |
| "errors": 6, |
| "error_rate": 0.00860832137733142 |
| }, |
| "0.99": { |
| "n": 320, |
| "errors": 2, |
| "error_rate": 0.00625 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9916666666666667, |
| "confidence_cutoff": 0.9690438255888087 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.946875, |
| "confidence_cutoff": 0.7097948924008157 |
| } |
| }, |
| "score_mae": 0.496047231607107, |
| "ranked_probability_score": 0.07900573734715889 |
| }, |
| "sources": { |
| "agnews": { |
| "n": 240, |
| "nll": 0.2188339468959484, |
| "acc": 0.9208333333333333, |
| "ece": 0.03458717155438394, |
| "brier": 0.11967225736034369, |
| "mean_conf": 0.9210081021968138, |
| "confident_error_rate": 0.0125, |
| "coverage_at_0_9": 0.7875, |
| "accuracy_at_0_9": 0.9841269841269841, |
| "coverage_at_5pct_error": 0.9291666666666667, |
| "coverage_at_1pct_error": 0.7208333333333333, |
| "aurc": 0.010595808181650849, |
| "error_rate_at_0_9": 0.015873015873015872, |
| "confidence_bias": 0.00017476886348055665, |
| "top_bins": { |
| "0.9": { |
| "n": 189, |
| "errors": 3, |
| "error_rate": 0.015873015873015872 |
| }, |
| "0.95": { |
| "n": 151, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 8, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.962887554254757 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9791666666666666, |
| "confidence_cutoff": 0.8982351380482543 |
| } |
| } |
| }, |
| "amazon": { |
| "n": 80, |
| "nll": 1.019521275107054, |
| "acc": 0.6125, |
| "ece": 0.06021888255181947, |
| "brier": 0.5421117796889148, |
| "mean_conf": 0.5878388840884237, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0375, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.0625, |
| "coverage_at_1pct_error": 0.0625, |
| "aurc": 0.26205124362469656, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.02466111591157638, |
| "top_bins": { |
| "0.9": { |
| "n": 3, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.75, |
| "confidence_cutoff": 0.5617743497724099 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.640625, |
| "confidence_cutoff": 0.44733349967779523 |
| } |
| }, |
| "score_mae": 0.5169737287164902, |
| "ranked_probability_score": 0.08574565040577714 |
| }, |
| "banking77": { |
| "n": 80, |
| "nll": 0.6434976771895786, |
| "acc": 0.875, |
| "ece": 0.16139951556317184, |
| "brier": 0.23632755413887385, |
| "mean_conf": 0.7261215293796409, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.4, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.75, |
| "coverage_at_1pct_error": 0.675, |
| "aurc": 0.025355014288770995, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.14887847062035908, |
| "top_bins": { |
| "0.9": { |
| "n": 32, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 24, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 2, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.853940262270331 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.921875, |
| "confidence_cutoff": 0.4421809473945611 |
| } |
| } |
| }, |
| "boolq": { |
| "n": 80, |
| "nll": 0.3692803969173892, |
| "acc": 0.875, |
| "ece": 0.0748529395052494, |
| "brier": 0.2143912889240845, |
| "mean_conf": 0.9094612721753246, |
| "confident_error_rate": 0.0625, |
| "coverage_at_0_9": 0.75, |
| "accuracy_at_0_9": 0.9166666666666666, |
| "coverage_at_5pct_error": 0.6625, |
| "coverage_at_1pct_error": 0.075, |
| "aurc": 0.059468554539862016, |
| "error_rate_at_0_9": 0.08333333333333333, |
| "confidence_bias": 0.034461272175324575, |
| "top_bins": { |
| "0.9": { |
| "n": 60, |
| "errors": 5, |
| "error_rate": 0.08333333333333333 |
| }, |
| "0.95": { |
| "n": 50, |
| "errors": 1, |
| "error_rate": 0.02 |
| }, |
| "0.99": { |
| "n": 26, |
| "errors": 1, |
| "error_rate": 0.038461538461538464 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.9805121330496657 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.890625, |
| "confidence_cutoff": 0.8395732170279804 |
| } |
| } |
| }, |
| "contrastive": { |
| "n": 160, |
| "nll": 0.06053811563664827, |
| "acc": 0.96875, |
| "ece": 0.03645679005941199, |
| "brier": 0.032513686428614616, |
| "mean_conf": 0.9645709125518511, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.90625, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.975, |
| "aurc": 0.0006312206080826334, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.004179087448148877, |
| "top_bins": { |
| "0.9": { |
| "n": 145, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 139, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 83, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9906152293058859 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9714574090050602 |
| } |
| } |
| }, |
| "dbpedia14": { |
| "n": 80, |
| "nll": 0.031074258317027002, |
| "acc": 1.0, |
| "ece": 0.02597689933514287, |
| "brier": 0.010683464293698132, |
| "mean_conf": 0.9740231006648571, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.95, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.02597689933514291, |
| "top_bins": { |
| "0.9": { |
| "n": 76, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 75, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 57, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9936484074458076 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9825304423953378 |
| } |
| } |
| }, |
| "imdb": { |
| "n": 80, |
| "nll": 0.0720718883732508, |
| "acc": 0.95, |
| "ece": 0.044527070446157675, |
| "brier": 0.0446787843047757, |
| "mean_conf": 0.9624677353009415, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.9375, |
| "aurc": 0.0017470077113648063, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": 0.012467735300941585, |
| "top_bins": { |
| "0.9": { |
| "n": 72, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 70, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 50, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9920587271734113 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.979826985587298 |
| } |
| } |
| }, |
| "mnli": { |
| "n": 80, |
| "nll": 0.27465339009465656, |
| "acc": 0.9125, |
| "ece": 0.09412857766900207, |
| "brier": 0.15288319760619568, |
| "mean_conf": 0.94639379824153, |
| "confident_error_rate": 0.0375, |
| "coverage_at_0_9": 0.875, |
| "accuracy_at_0_9": 0.9571428571428572, |
| "coverage_at_5pct_error": 0.875, |
| "coverage_at_1pct_error": 0.675, |
| "aurc": 0.01561228714610165, |
| "error_rate_at_0_9": 0.04285714285714286, |
| "confidence_bias": 0.03389379824153005, |
| "top_bins": { |
| "0.9": { |
| "n": 70, |
| "errors": 3, |
| "error_rate": 0.04285714285714286 |
| }, |
| "0.95": { |
| "n": 67, |
| "errors": 2, |
| "error_rate": 0.029850746268656716 |
| }, |
| "0.99": { |
| "n": 45, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9920616734287709 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.96875, |
| "confidence_cutoff": 0.9661281848866967 |
| } |
| } |
| }, |
| "sst5": { |
| "n": 80, |
| "nll": 0.9636808958022751, |
| "acc": 0.6125, |
| "ece": 0.09386529578800333, |
| "brier": 0.5310525330540982, |
| "mean_conf": 0.6617158029994963, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.0, |
| "accuracy_at_0_9": null, |
| "coverage_at_5pct_error": 0.1625, |
| "coverage_at_1pct_error": 0.1625, |
| "aurc": 0.26405329260422244, |
| "error_rate_at_0_9": null, |
| "confidence_bias": 0.04921580299949624, |
| "top_bins": { |
| "0.9": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.95": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.65, |
| "confidence_cutoff": 0.6682213711318762 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.625, |
| "confidence_cutoff": 0.5940210248093071 |
| } |
| }, |
| "score_mae": 0.5227774115005795, |
| "ranked_probability_score": 0.08371156976927716 |
| }, |
| "trec": { |
| "n": 80, |
| "nll": 0.16488315000308515, |
| "acc": 0.9625, |
| "ece": 0.02170728565832692, |
| "brier": 0.06057653876714909, |
| "mean_conf": 0.9707615704179906, |
| "confident_error_rate": 0.025, |
| "coverage_at_0_9": 0.9625, |
| "accuracy_at_0_9": 0.974025974025974, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 0.2625, |
| "aurc": 0.018315788693627898, |
| "error_rate_at_0_9": 0.025974025974025976, |
| "confidence_bias": 0.008261570417990582, |
| "top_bins": { |
| "0.9": { |
| "n": 77, |
| "errors": 2, |
| "error_rate": 0.025974025974025976 |
| }, |
| "0.95": { |
| "n": 73, |
| "errors": 2, |
| "error_rate": 0.0273972602739726 |
| }, |
| "0.99": { |
| "n": 47, |
| "errors": 1, |
| "error_rate": 0.02127659574468085 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.975, |
| "confidence_cutoff": 0.990957410950485 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9755548475921001 |
| } |
| } |
| }, |
| "yelp": { |
| "n": 160, |
| "nll": 0.5386789559476151, |
| "acc": 0.78125, |
| "ece": 0.06183692449171378, |
| "brier": 0.3102138876335011, |
| "mean_conf": 0.7769424223142675, |
| "confident_error_rate": 0.01875, |
| "coverage_at_0_9": 0.4125, |
| "accuracy_at_0_9": 0.9545454545454546, |
| "coverage_at_5pct_error": 0.4375, |
| "coverage_at_1pct_error": 0.14375, |
| "aurc": 0.0856358031592544, |
| "error_rate_at_0_9": 0.045454545454545456, |
| "confidence_bias": -0.004307577685732533, |
| "top_bins": { |
| "0.9": { |
| "n": 66, |
| "errors": 3, |
| "error_rate": 0.045454545454545456 |
| }, |
| "0.95": { |
| "n": 47, |
| "errors": 1, |
| "error_rate": 0.02127659574468085 |
| }, |
| "0.99": { |
| "n": 2, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9125, |
| "confidence_cutoff": 0.8216206137745724 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.8515625, |
| "confidence_cutoff": 0.577455875568763 |
| } |
| }, |
| "score_mae": 0.44839055460425137, |
| "ranked_probability_score": 0.0675599918664224 |
| } |
| }, |
| "types": { |
| "choice": { |
| "n": 480, |
| "nll": 0.2482065801683337, |
| "acc": 0.9375, |
| "ece": 0.03877078391929752, |
| "brier": 0.10960480604725589, |
| "mean_conf": 0.9179948652716078, |
| "confident_error_rate": 0.0125, |
| "coverage_at_0_9": 0.8145833333333333, |
| "accuracy_at_0_9": 0.9846547314578005, |
| "coverage_at_5pct_error": 0.9583333333333334, |
| "coverage_at_1pct_error": 0.69375, |
| "aurc": 0.01188927247802621, |
| "error_rate_at_0_9": 0.015345268542199489, |
| "confidence_bias": -0.019505134728392215, |
| "top_bins": { |
| "0.9": { |
| "n": 391, |
| "errors": 6, |
| "error_rate": 0.015345268542199489 |
| }, |
| "0.95": { |
| "n": 361, |
| "errors": 4, |
| "error_rate": 0.0110803324099723 |
| }, |
| "0.99": { |
| "n": 202, |
| "errors": 1, |
| "error_rate": 0.0049504950495049506 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9958333333333333, |
| "confidence_cutoff": 0.9867396184306336 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9869791666666666, |
| "confidence_cutoff": 0.9197849614177179 |
| } |
| } |
| }, |
| "noul": { |
| "n": 480, |
| "nll": 0.18011051231286856, |
| "acc": 0.9270833333333334, |
| "ece": 0.014056851970226658, |
| "brier": 0.10328956268313709, |
| "mean_conf": 0.9306022039720875, |
| "confident_error_rate": 0.01875, |
| "coverage_at_0_9": 0.8166666666666667, |
| "accuracy_at_0_9": 0.9770408163265306, |
| "coverage_at_5pct_error": 0.9416666666666667, |
| "coverage_at_1pct_error": 0.7041666666666667, |
| "aurc": 0.015597964078219255, |
| "error_rate_at_0_9": 0.02295918367346939, |
| "confidence_bias": 0.0035188706387541124, |
| "top_bins": { |
| "0.9": { |
| "n": 392, |
| "errors": 9, |
| "error_rate": 0.02295918367346939 |
| }, |
| "0.95": { |
| "n": 334, |
| "errors": 2, |
| "error_rate": 0.005988023952095809 |
| }, |
| "0.99": { |
| "n": 118, |
| "errors": 1, |
| "error_rate": 0.00847457627118644 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9916666666666667, |
| "confidence_cutoff": 0.9755983279973393 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9765625, |
| "confidence_cutoff": 0.9172471488342538 |
| } |
| } |
| }, |
| "score": { |
| "n": 240, |
| "nll": 0.9412321202578248, |
| "acc": 0.625, |
| "ece": 0.048867889504322116, |
| "brier": 0.519936949533565, |
| "mean_conf": 0.6310840847095708, |
| "confident_error_rate": 0.004166666666666667, |
| "coverage_at_0_9": 0.029166666666666667, |
| "accuracy_at_0_9": 0.8571428571428571, |
| "coverage_at_5pct_error": 0.016666666666666666, |
| "coverage_at_1pct_error": 0.016666666666666666, |
| "aurc": 0.2774440601733873, |
| "error_rate_at_0_9": 0.14285714285714285, |
| "confidence_bias": 0.006084084709570781, |
| "top_bins": { |
| "0.9": { |
| "n": 7, |
| "errors": 1, |
| "error_rate": 0.14285714285714285 |
| }, |
| "0.95": { |
| "n": 2, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.7166666666666667, |
| "confidence_cutoff": 0.6329725841576003 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.6614583333333334, |
| "confidence_cutoff": 0.4922131178494099 |
| } |
| }, |
| "score_mae": 0.496047231607107, |
| "ranked_probability_score": 0.07900573734715889 |
| } |
| } |
| }, |
| "transfer-v4": { |
| "raw_clean": { |
| "n": 656, |
| "nll": 0.5640777093137941, |
| "acc": 0.836890243902439, |
| "ece": 0.10195634226245451, |
| "brier": 0.25517441623108567, |
| "mean_conf": 0.9301396868913653, |
| "confident_error_rate": 0.06707317073170732, |
| "coverage_at_0_9": 0.8094512195121951, |
| "accuracy_at_0_9": 0.9171374764595104, |
| "coverage_at_5pct_error": 0.6829268292682927, |
| "coverage_at_1pct_error": 0.2820121951219512, |
| "aurc": 0.043201879169005745, |
| "error_rate_at_0_9": 0.08286252354048965, |
| "confidence_bias": 0.09324944298892623, |
| "top_bins": { |
| "0.9": { |
| "n": 531, |
| "errors": 44, |
| "error_rate": 0.08286252354048965 |
| }, |
| "0.95": { |
| "n": 487, |
| "errors": 33, |
| "error_rate": 0.06776180698151951 |
| }, |
| "0.99": { |
| "n": 406, |
| "errors": 14, |
| "error_rate": 0.034482758620689655 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9725609756097561, |
| "confidence_cutoff": 0.9980492531927935 |
| }, |
| "0.8": { |
| "coverage": 0.8003048780487805, |
| "accuracy": 0.92, |
| "confidence_cutoff": 0.9074193401309704 |
| } |
| }, |
| "score_mae": 0.24004523406424622, |
| "ranked_probability_score": 0.09767451376793074 |
| }, |
| "as_shipped_clean": { |
| "n": 656, |
| "nll": 0.42587070132965826, |
| "acc": 0.836890243902439, |
| "ece": 0.0314949650736429, |
| "brier": 0.23187082308763576, |
| "mean_conf": 0.844962353152835, |
| "confident_error_rate": 0.018292682926829267, |
| "coverage_at_0_9": 0.5853658536585366, |
| "accuracy_at_0_9": 0.96875, |
| "coverage_at_5pct_error": 0.6570121951219512, |
| "coverage_at_1pct_error": 0.2530487804878049, |
| "aurc": 0.04507835320830427, |
| "error_rate_at_0_9": 0.03125, |
| "confidence_bias": 0.00807210925039592, |
| "top_bins": { |
| "0.9": { |
| "n": 384, |
| "errors": 12, |
| "error_rate": 0.03125 |
| }, |
| "0.95": { |
| "n": 315, |
| "errors": 8, |
| "error_rate": 0.025396825396825397 |
| }, |
| "0.99": { |
| "n": 79, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9725609756097561, |
| "confidence_cutoff": 0.9441193658749547 |
| }, |
| "0.8": { |
| "coverage": 0.8003048780487805, |
| "accuracy": 0.9047619047619048, |
| "confidence_cutoff": 0.693693819569072 |
| } |
| }, |
| "score_mae": 0.27196301946029916, |
| "ranked_probability_score": 0.0837200362891968 |
| }, |
| "sources": { |
| "composition_holdout": { |
| "n": 96, |
| "nll": 0.29615290237046005, |
| "acc": 0.875, |
| "ece": 0.06938096946882913, |
| "brier": 0.1808256349283982, |
| "mean_conf": 0.9443809694688293, |
| "confident_error_rate": 0.041666666666666664, |
| "coverage_at_0_9": 0.8333333333333334, |
| "accuracy_at_0_9": 0.95, |
| "coverage_at_5pct_error": 0.8645833333333334, |
| "coverage_at_1pct_error": 0.6666666666666666, |
| "aurc": 0.021756148279610994, |
| "error_rate_at_0_9": 0.05, |
| "confidence_bias": 0.06938096946882932, |
| "top_bins": { |
| "0.9": { |
| "n": 80, |
| "errors": 4, |
| "error_rate": 0.05 |
| }, |
| "0.95": { |
| "n": 73, |
| "errors": 3, |
| "error_rate": 0.0410958904109589 |
| }, |
| "0.99": { |
| "n": 2, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.980048039700624 |
| }, |
| "0.8": { |
| "coverage": 0.8020833333333334, |
| "accuracy": 0.948051948051948, |
| "confidence_cutoff": 0.9379020298350527 |
| } |
| } |
| }, |
| "contrastive": { |
| "n": 80, |
| "nll": 0.2412487469898486, |
| "acc": 0.875, |
| "ece": 0.054181213136020756, |
| "brier": 0.15115638127662953, |
| "mean_conf": 0.9258459913436543, |
| "confident_error_rate": 0.0125, |
| "coverage_at_0_9": 0.8125, |
| "accuracy_at_0_9": 0.9846153846153847, |
| "coverage_at_5pct_error": 0.875, |
| "coverage_at_1pct_error": 0.7875, |
| "aurc": 0.01404271679837261, |
| "error_rate_at_0_9": 0.015384615384615385, |
| "confidence_bias": 0.050845991343654307, |
| "top_bins": { |
| "0.9": { |
| "n": 65, |
| "errors": 1, |
| "error_rate": 0.015384615384615385 |
| }, |
| "0.95": { |
| "n": 62, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 26, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9886860005278774 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.984375, |
| "confidence_cutoff": 0.9353245409640992 |
| } |
| }, |
| "score_mae": 0.27196301946029916, |
| "ranked_probability_score": 0.0837200362891968 |
| }, |
| "emotion": { |
| "n": 80, |
| "nll": 1.0001023668251066, |
| "acc": 0.6625, |
| "ece": 0.14881697863795876, |
| "brier": 0.4744803648425345, |
| "mean_conf": 0.591235292513326, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.05, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.0625, |
| "coverage_at_1pct_error": 0.0625, |
| "aurc": 0.18702686566035498, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.07126470748667402, |
| "top_bins": { |
| "0.9": { |
| "n": 4, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 2, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.8, |
| "confidence_cutoff": 0.5874489256154185 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.71875, |
| "confidence_cutoff": 0.3855885748389788 |
| } |
| } |
| }, |
| "mmlu": { |
| "n": 80, |
| "nll": 0.777539674387988, |
| "acc": 0.7, |
| "ece": 0.13070084664599038, |
| "brier": 0.40568344535429135, |
| "mean_conf": 0.6636013764250539, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.175, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.2625, |
| "coverage_at_1pct_error": 0.225, |
| "aurc": 0.12322939721812473, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.036398623574946054, |
| "top_bins": { |
| "0.9": { |
| "n": 14, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 7, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 1, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9, |
| "confidence_cutoff": 0.6761349246582344 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.796875, |
| "confidence_cutoff": 0.4466846683613359 |
| } |
| } |
| }, |
| "paws": { |
| "n": 80, |
| "nll": 0.512348189940992, |
| "acc": 0.8125, |
| "ece": 0.10522416975834781, |
| "brier": 0.2914852770101592, |
| "mean_conf": 0.9091572173256038, |
| "confident_error_rate": 0.0875, |
| "coverage_at_0_9": 0.75, |
| "accuracy_at_0_9": 0.8833333333333333, |
| "coverage_at_5pct_error": 0.2625, |
| "coverage_at_1pct_error": 0.2, |
| "aurc": 0.09484078957534384, |
| "error_rate_at_0_9": 0.11666666666666667, |
| "confidence_bias": 0.09665721732560384, |
| "top_bins": { |
| "0.9": { |
| "n": 60, |
| "errors": 7, |
| "error_rate": 0.11666666666666667 |
| }, |
| "0.95": { |
| "n": 51, |
| "errors": 5, |
| "error_rate": 0.09803921568627451 |
| }, |
| "0.99": { |
| "n": 4, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.875, |
| "confidence_cutoff": 0.974944024208843 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.875, |
| "confidence_cutoff": 0.8613523629912301 |
| } |
| } |
| }, |
| "qnli": { |
| "n": 80, |
| "nll": 0.20826182445785202, |
| "acc": 0.9125, |
| "ece": 0.04839798205004245, |
| "brier": 0.12797478277700258, |
| "mean_conf": 0.9090029243294062, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.7, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.8875, |
| "coverage_at_1pct_error": 0.7, |
| "aurc": 0.014269660330833212, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.0034970756705937633, |
| "top_bins": { |
| "0.9": { |
| "n": 56, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 45, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 10, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9595371830584098 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.96875, |
| "confidence_cutoff": 0.8535858138327058 |
| } |
| } |
| }, |
| "sciq": { |
| "n": 80, |
| "nll": 0.04197126388379959, |
| "acc": 1.0, |
| "ece": 0.03774145345553605, |
| "brier": 0.010730106398852924, |
| "mean_conf": 0.962258546544464, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.9, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 1.0, |
| "coverage_at_1pct_error": 1.0, |
| "aurc": 0.0, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.037741453455536034, |
| "top_bins": { |
| "0.9": { |
| "n": 72, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 69, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 36, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9883715126062644 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9713861020562242 |
| } |
| } |
| }, |
| "tweet_offensive": { |
| "n": 80, |
| "nll": 0.3552842015730585, |
| "acc": 0.85, |
| "ece": 0.056220131039951354, |
| "brier": 0.22283962974506544, |
| "mean_conf": 0.8343327840091443, |
| "confident_error_rate": 0.0, |
| "coverage_at_0_9": 0.4125, |
| "accuracy_at_0_9": 1.0, |
| "coverage_at_5pct_error": 0.6, |
| "coverage_at_1pct_error": 0.525, |
| "aurc": 0.045801042430234855, |
| "error_rate_at_0_9": 0.0, |
| "confidence_bias": -0.015667215990855632, |
| "top_bins": { |
| "0.9": { |
| "n": 33, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.95": { |
| "n": 6, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 0, |
| "errors": 0, |
| "error_rate": null |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.8768513333328267 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.890625, |
| "confidence_cutoff": 0.7493184208869934 |
| } |
| } |
| } |
| }, |
| "types": { |
| "choice": { |
| "n": 336, |
| "nll": 0.5178563780812967, |
| "acc": 0.8125, |
| "ece": 0.04292515250441144, |
| "brier": 0.26378206631184675, |
| "mean_conf": 0.797702947344152, |
| "confident_error_rate": 0.011904761904761904, |
| "coverage_at_0_9": 0.5059523809523809, |
| "accuracy_at_0_9": 0.9764705882352941, |
| "coverage_at_5pct_error": 0.6130952380952381, |
| "coverage_at_1pct_error": 0.40773809523809523, |
| "aurc": 0.04890439545080765, |
| "error_rate_at_0_9": 0.023529411764705882, |
| "confidence_bias": -0.014797052655847964, |
| "top_bins": { |
| "0.9": { |
| "n": 170, |
| "errors": 4, |
| "error_rate": 0.023529411764705882 |
| }, |
| "0.95": { |
| "n": 151, |
| "errors": 3, |
| "error_rate": 0.019867549668874173 |
| }, |
| "0.99": { |
| "n": 39, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9761904761904762, |
| "confidence_cutoff": 0.9209057199846711 |
| }, |
| "0.8": { |
| "coverage": 0.8005952380952381, |
| "accuracy": 0.8996282527881041, |
| "confidence_cutoff": 0.5765734433053208 |
| } |
| } |
| }, |
| "noul": { |
| "n": 280, |
| "nll": 0.3088148767031123, |
| "acc": 0.8785714285714286, |
| "ece": 0.02716426081477808, |
| "brier": 0.18354251467515595, |
| "mean_conf": 0.8993028632705061, |
| "confident_error_rate": 0.025, |
| "coverage_at_0_9": 0.675, |
| "accuracy_at_0_9": 0.9629629629629629, |
| "coverage_at_5pct_error": 0.7428571428571429, |
| "coverage_at_1pct_error": 0.24642857142857144, |
| "aurc": 0.04037503899248962, |
| "error_rate_at_0_9": 0.037037037037037035, |
| "confidence_bias": 0.020731434699077567, |
| "top_bins": { |
| "0.9": { |
| "n": 189, |
| "errors": 7, |
| "error_rate": 0.037037037037037035 |
| }, |
| "0.95": { |
| "n": 142, |
| "errors": 5, |
| "error_rate": 0.035211267605633804 |
| }, |
| "0.99": { |
| "n": 35, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 0.9642857142857143, |
| "confidence_cutoff": 0.951430429315972 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.9330357142857143, |
| "confidence_cutoff": 0.8169286372799969 |
| } |
| } |
| }, |
| "score": { |
| "n": 40, |
| "nll": 0.47258178900171605, |
| "acc": 0.75, |
| "ece": 0.11341449184915596, |
| "brier": 0.3021145388916218, |
| "mean_conf": 0.8615577911220736, |
| "confident_error_rate": 0.025, |
| "coverage_at_0_9": 0.625, |
| "accuracy_at_0_9": 0.96, |
| "coverage_at_5pct_error": 0.65, |
| "coverage_at_1pct_error": 0.575, |
| "aurc": 0.060906503666048904, |
| "error_rate_at_0_9": 0.04, |
| "confidence_bias": 0.11155779112207365, |
| "top_bins": { |
| "0.9": { |
| "n": 25, |
| "errors": 1, |
| "error_rate": 0.04 |
| }, |
| "0.95": { |
| "n": 22, |
| "errors": 0, |
| "error_rate": 0.0 |
| }, |
| "0.99": { |
| "n": 5, |
| "errors": 0, |
| "error_rate": 0.0 |
| } |
| }, |
| "selective": { |
| "0.5": { |
| "coverage": 0.5, |
| "accuracy": 1.0, |
| "confidence_cutoff": 0.9618942423403596 |
| }, |
| "0.8": { |
| "coverage": 0.8, |
| "accuracy": 0.84375, |
| "confidence_cutoff": 0.7283845543861389 |
| } |
| }, |
| "score_mae": 0.27196301946029916, |
| "ranked_probability_score": 0.0837200362891968 |
| } |
| } |
| } |
| }, |
| "base_revision": "1001bb4d826a52d1f399e183466143f4da7b741b", |
| "test_used_for_selection": false |
| } |
|
|
