| { |
| "epoch": 3, |
| "temperature": 1.334968103387446, |
| "development": { |
| "raw": { |
| "questions": 14912, |
| "accuracy": 0.9075912017167382, |
| "mean_class_recall": 0.8447857522159936, |
| "nll": 0.24807737680415345, |
| "brier": 0.12998082994334342, |
| "ece15": 0.027028147820359012, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.8103540772532188, |
| "error_rate_at_confidence_0_9": 0.023502151605428664, |
| "ordinal_expected_level_mae": 0.00481932257817722, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 222, |
| "accuracy": 0.9774774774774775, |
| "mean_class_recall": 0.9570930232558139, |
| "nll": 0.10407641066616782, |
| "brier": 0.039347876403718875, |
| "ece15": 0.021432863016197557, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9864864864864865, |
| "error_rate_at_confidence_0_9": 0.0136986301369863, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 222, |
| "accuracy": 0.9819819819819819, |
| "mean_class_recall": 0.96, |
| "nll": 0.10023555924012383, |
| "brier": 0.031563944714978254, |
| "ece15": 0.015911253108198964, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9864864864864865, |
| "error_rate_at_confidence_0_9": 0.0136986301369863, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 1400, |
| "accuracy": 0.9885714285714285, |
| "mean_class_recall": 0.9821428571428572, |
| "nll": 0.0442702602822601, |
| "brier": 0.02139513973607548, |
| "ece15": 0.007710400104216533, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9842857142857143, |
| "error_rate_at_confidence_0_9": 0.008708272859216255, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 5000, |
| "accuracy": 0.8418, |
| "mean_class_recall": 0.7537333333333334, |
| "nll": 0.4173917752941542, |
| "brier": 0.22451704505631584, |
| "ece15": 0.06284084146027137, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.6928, |
| "error_rate_at_confidence_0_9": 0.06293302540415704, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0014205610073774013, |
| "brier": 0.0002211290260091384, |
| "ece15": 0.0013613576932818498, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9919354838709677, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.000610657881540054, |
| "brier": 8.099314527924365e-05, |
| "ece15": 0.0005891478461821947, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 1.5480302068279906e-07, |
| "brier": 4.4217021643060835e-12, |
| "ece15": 1.548019148098009e-07, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 2500, |
| "accuracy": 0.9792, |
| "mean_class_recall": 0.9818666666666667, |
| "nll": 0.11218220058914397, |
| "brier": 0.03724045269615108, |
| "ece15": 0.01842697861877182, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9928, |
| "error_rate_at_confidence_0_9": 0.015310233682514102, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 797, |
| "accuracy": 0.9962358845671268, |
| "mean_class_recall": 0.9963898916967509, |
| "nll": 0.012217254972570101, |
| "brier": 0.006818829719414822, |
| "ece15": 0.004133398699556409, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9949811794228356, |
| "error_rate_at_confidence_0_9": 0.0012610340479192938, |
| "ordinal_expected_level_mae": 0.00481932257817722 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 797, |
| "accuracy": 0.9949811794228356, |
| "mean_class_recall": 0.9951865222623345, |
| "nll": 0.013078119892699127, |
| "brier": 0.007721112955228842, |
| "ece15": 0.003423998250058524, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9899623588456713, |
| "error_rate_at_confidence_0_9": 0.0012674271229404308, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 797, |
| "accuracy": 0.9949811794228356, |
| "mean_class_recall": 0.994276682635671, |
| "nll": 0.01027058602627885, |
| "brier": 0.007209131739335666, |
| "ece15": 0.004201163796495765, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9912170639899623, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 797, |
| "accuracy": 0.9974905897114178, |
| "mean_class_recall": 0.998236331569665, |
| "nll": 0.017287182530912663, |
| "brier": 0.004953169208636321, |
| "ece15": 0.0025266183132433223, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.998745294855709, |
| "error_rate_at_confidence_0_9": 0.002512562814070352, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 800, |
| "accuracy": 0.69, |
| "mean_class_recall": 0.7116483068135455, |
| "nll": 0.5092641106435474, |
| "brier": 0.3569714849542106, |
| "ece15": 0.03650030376694629, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.2425, |
| "error_rate_at_confidence_0_9": 0.010309278350515464, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 800, |
| "accuracy": 0.68875, |
| "mean_class_recall": 0.529392069171139, |
| "nll": 0.96833512912194, |
| "brier": 0.46247412161427887, |
| "ece15": 0.02731423542286039, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.22625, |
| "error_rate_at_confidence_0_9": 0.022099447513812154, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9379621229396502, |
| "macro_task_balanced_accuracy": 0.9185689774676985 |
| }, |
| "recalibrated": { |
| "questions": 14912, |
| "accuracy": 0.9075912017167382, |
| "mean_class_recall": 0.8447857522159936, |
| "nll": 0.2269545932235271, |
| "brier": 0.12792407717450321, |
| "ece15": 0.01858051585491712, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.7862124463519313, |
| "error_rate_at_confidence_0_9": 0.018679631525076765, |
| "ordinal_expected_level_mae": 0.005329911013904147, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 222, |
| "accuracy": 0.9774774774774775, |
| "mean_class_recall": 0.9570930232558139, |
| "nll": 0.08092683597443519, |
| "brier": 0.037546344393533104, |
| "ece15": 0.021320738677048364, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9819819819819819, |
| "error_rate_at_confidence_0_9": 0.013761467889908258, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 222, |
| "accuracy": 0.9819819819819819, |
| "mean_class_recall": 0.96, |
| "nll": 0.07809919245507242, |
| "brier": 0.030623326048867618, |
| "ece15": 0.015019163882298042, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9819819819819819, |
| "error_rate_at_confidence_0_9": 0.009174311926605505, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 1400, |
| "accuracy": 0.9885714285714285, |
| "mean_class_recall": 0.9821428571428572, |
| "nll": 0.04094699270351949, |
| "brier": 0.021018406686723334, |
| "ece15": 0.0038434318338614677, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9678571428571429, |
| "error_rate_at_confidence_0_9": 0.006642066420664207, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 5000, |
| "accuracy": 0.8418, |
| "mean_class_recall": 0.7537333333333334, |
| "nll": 0.36679030702727994, |
| "brier": 0.21757064336558796, |
| "ece15": 0.043357678681868704, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.6394, |
| "error_rate_at_confidence_0_9": 0.05004691898654989, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0028701157894541487, |
| "brier": 0.0006216354903482641, |
| "ece15": 0.0026971564840139535, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9919354838709677, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.001405432942745028, |
| "brier": 0.00025083300955486444, |
| "ece15": 0.0013358908781109478, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9939024390243902, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 4.6272880953518815e-06, |
| "brier": 9.796066038293659e-10, |
| "ece15": 4.627043141135246e-06, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 2500, |
| "accuracy": 0.9792, |
| "mean_class_recall": 0.9818666666666667, |
| "nll": 0.0877031817842779, |
| "brier": 0.03599036114227489, |
| "ece15": 0.01538869285344686, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.988, |
| "error_rate_at_confidence_0_9": 0.014574898785425101, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 797, |
| "accuracy": 0.9962358845671268, |
| "mean_class_recall": 0.9963898916967509, |
| "nll": 0.010603240403009088, |
| "brier": 0.006373682001028304, |
| "ece15": 0.004218405852177047, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9937264742785445, |
| "error_rate_at_confidence_0_9": 0.0012626262626262627, |
| "ordinal_expected_level_mae": 0.005329911013904147 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 797, |
| "accuracy": 0.9949811794228356, |
| "mean_class_recall": 0.9951865222623345, |
| "nll": 0.01230072195659841, |
| "brier": 0.007337903591834537, |
| "ece15": 0.0031241457310850957, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.986198243412798, |
| "error_rate_at_confidence_0_9": 0.001272264631043257, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 797, |
| "accuracy": 0.9949811794228356, |
| "mean_class_recall": 0.994276682635671, |
| "nll": 0.009584146250019401, |
| "brier": 0.00659904677807738, |
| "ece15": 0.005028933100804552, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9899623588456713, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 797, |
| "accuracy": 0.9974905897114178, |
| "mean_class_recall": 0.998236331569665, |
| "nll": 0.013437428285294059, |
| "brier": 0.004858731838764845, |
| "ece15": 0.0022785888551786117, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.998745294855709, |
| "error_rate_at_confidence_0_9": 0.002512562814070352, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 800, |
| "accuracy": 0.69, |
| "mean_class_recall": 0.7116483068135455, |
| "nll": 0.5102124840713312, |
| "brier": 0.35753050242002155, |
| "ece15": 0.03811996421246104, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.22125, |
| "error_rate_at_confidence_0_9": 0.011299435028248588, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 800, |
| "accuracy": 0.68875, |
| "mean_class_recall": 0.529392069171139, |
| "nll": 0.9911460027249259, |
| "brier": 0.4737144267281807, |
| "ece15": 0.09647043512079573, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.18625, |
| "error_rate_at_confidence_0_9": 0.020134228187919462, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9379621229396502, |
| "macro_task_balanced_accuracy": 0.9185689774676985 |
| } |
| }, |
| "calibration": { |
| "raw": { |
| "questions": 14821, |
| "accuracy": 0.9313811483705553, |
| "mean_class_recall": 0.8460223158089928, |
| "nll": 0.18480685074016417, |
| "brier": 0.09978353062869037, |
| "ece15": 0.010514966894040724, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.814789825247959, |
| "error_rate_at_confidence_0_9": 0.013580655846306725, |
| "ordinal_expected_level_mae": 0.001461981275023823, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 325, |
| "accuracy": 0.96, |
| "mean_class_recall": 0.9216916119306557, |
| "nll": 0.12908652765840894, |
| "brier": 0.06258807450225369, |
| "ece15": 0.033510331142289206, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9815384615384616, |
| "error_rate_at_confidence_0_9": 0.0219435736677116, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 328, |
| "accuracy": 0.9695121951219512, |
| "mean_class_recall": 0.9372206852521814, |
| "nll": 0.16127769578392517, |
| "brier": 0.05858994554032181, |
| "ece15": 0.030459345972402876, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9939024390243902, |
| "error_rate_at_confidence_0_9": 0.027607361963190184, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 2400, |
| "accuracy": 0.9404166666666667, |
| "mean_class_recall": 0.8877604166666666, |
| "nll": 0.19653998710185536, |
| "brier": 0.09947525932448194, |
| "ece15": 0.04423318187692424, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9479166666666666, |
| "error_rate_at_confidence_0_9": 0.03824175824175824, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 5000, |
| "accuracy": 0.9292, |
| "mean_class_recall": 0.8968, |
| "nll": 0.18887834668200343, |
| "brier": 0.10636649585712103, |
| "ece15": 0.019887550017025992, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.7252, |
| "error_rate_at_confidence_0_9": 0.013513513513513514, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 0.9919354838709677, |
| "mean_class_recall": 0.9928571428571429, |
| "nll": 0.021390831595053342, |
| "brier": 0.015445766007169246, |
| "ece15": 0.011074830403285848, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9838709677419355, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 235, |
| "accuracy": 0.9957446808510638, |
| "mean_class_recall": 0.9949494949494949, |
| "nll": 0.015420468691595452, |
| "brier": 0.008033852620508352, |
| "ece15": 0.003625196423068977, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.00425531914893617, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 235, |
| "accuracy": 0.9957446808510638, |
| "mean_class_recall": 0.9905660377358491, |
| "nll": 0.018077073969230188, |
| "brier": 0.00826909442413202, |
| "ece15": 0.004194003764167214, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.00425531914893617, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 2000, |
| "accuracy": 0.9985, |
| "mean_class_recall": 0.9976666666666667, |
| "nll": 0.010567088503443185, |
| "brier": 0.003178972042914288, |
| "ece15": 0.0009240536217447044, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.999, |
| "error_rate_at_confidence_0_9": 0.0015015015015015015, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 641, |
| "accuracy": 0.9984399375975039, |
| "mean_class_recall": 0.9985507246376812, |
| "nll": 0.003039872100282549, |
| "brier": 0.0022297539963869384, |
| "ece15": 0.0014593852165271412, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9984399375975039, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": 0.001461981275023823 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 641, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0004967972153911127, |
| "brier": 9.688544623270663e-05, |
| "ece15": 0.00046955068951652005, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9984399375975039, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 641, |
| "accuracy": 0.9968798751950078, |
| "mean_class_recall": 0.9961240310077519, |
| "nll": 0.016173152134660414, |
| "brier": 0.007502235795954162, |
| "ece15": 0.0049434253477994355, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9937597503900156, |
| "error_rate_at_confidence_0_9": 0.0031397174254317113, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 641, |
| "accuracy": 0.9984399375975039, |
| "mean_class_recall": 0.9969512195121951, |
| "nll": 0.004162755288573824, |
| "brier": 0.0026938896990115683, |
| "ece15": 0.0014825770107887316, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0015600624024961, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 805, |
| "accuracy": 0.7031055900621118, |
| "mean_class_recall": 0.7239978843175034, |
| "nll": 0.49879740669601635, |
| "brier": 0.351098943795658, |
| "ece15": 0.02568931078110801, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.23850931677018633, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 805, |
| "accuracy": 0.6919254658385093, |
| "mean_class_recall": 0.5331098265895953, |
| "nll": 0.9684343228937802, |
| "brier": 0.45465071518796674, |
| "ece15": 0.01705305962599151, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.2360248447204969, |
| "error_rate_at_confidence_0_9": 0.021052631578947368, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9407031795465964, |
| "macro_task_balanced_accuracy": 0.9191604101516703 |
| }, |
| "recalibrated": { |
| "questions": 14821, |
| "accuracy": 0.9313811483705553, |
| "mean_class_recall": 0.8460223158089928, |
| "nll": 0.17961943925788992, |
| "brier": 0.10111838724890135, |
| "ece15": 0.017864195303344165, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.7859118817893529, |
| "error_rate_at_confidence_0_9": 0.010645604395604396, |
| "ordinal_expected_level_mae": 0.001742299523022841, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 325, |
| "accuracy": 0.96, |
| "mean_class_recall": 0.9216916119306557, |
| "nll": 0.10308600797920058, |
| "brier": 0.05761479429147346, |
| "ece15": 0.0294057816164861, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.963076923076923, |
| "error_rate_at_confidence_0_9": 0.012779552715654952, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 328, |
| "accuracy": 0.9695121951219512, |
| "mean_class_recall": 0.9372206852521814, |
| "nll": 0.12289366290953901, |
| "brier": 0.057082591730227304, |
| "ece15": 0.029897091535857363, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9939024390243902, |
| "error_rate_at_confidence_0_9": 0.027607361963190184, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 2400, |
| "accuracy": 0.9404166666666667, |
| "mean_class_recall": 0.8877604166666666, |
| "nll": 0.16174786934019353, |
| "brier": 0.09375828073422167, |
| "ece15": 0.03645209485430555, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.92875, |
| "error_rate_at_confidence_0_9": 0.03095558546433378, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 5000, |
| "accuracy": 0.9292, |
| "mean_class_recall": 0.8968, |
| "nll": 0.19125358889505403, |
| "brier": 0.11141554439383701, |
| "ece15": 0.03505488361604012, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.6616, |
| "error_rate_at_confidence_0_9": 0.010278113663845224, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 0.9919354838709677, |
| "mean_class_recall": 0.9928571428571429, |
| "nll": 0.019190864186870816, |
| "brier": 0.01397734899726431, |
| "ece15": 0.011351501004051424, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9838709677419355, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 235, |
| "accuracy": 0.9957446808510638, |
| "mean_class_recall": 0.9949494949494949, |
| "nll": 0.01285197836061796, |
| "brier": 0.007512756902037326, |
| "ece15": 0.004004760561973723, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.00425531914893617, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 235, |
| "accuracy": 0.9957446808510638, |
| "mean_class_recall": 0.9905660377358491, |
| "nll": 0.01368047084450566, |
| "brier": 0.007839165259612221, |
| "ece15": 0.004073337312958388, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.00425531914893617, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 2000, |
| "accuracy": 0.9985, |
| "mean_class_recall": 0.9976666666666667, |
| "nll": 0.010650627249048795, |
| "brier": 0.003323170074906094, |
| "ece15": 0.002118687755931114, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9975, |
| "error_rate_at_confidence_0_9": 0.0015037593984962407, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 641, |
| "accuracy": 0.9984399375975039, |
| "mean_class_recall": 0.9985507246376812, |
| "nll": 0.0028802196260483376, |
| "brier": 0.001935795675809169, |
| "ece15": 0.0017273023271884604, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9968798751950078, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": 0.001742299523022841 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 641, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0012409177190170738, |
| "brier": 0.00021235107519573934, |
| "ece15": 0.0011788827327451372, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9984399375975039, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 641, |
| "accuracy": 0.9968798751950078, |
| "mean_class_recall": 0.9961240310077519, |
| "nll": 0.013816362060687534, |
| "brier": 0.007467944863319216, |
| "ece15": 0.004893228894077442, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9921996879875195, |
| "error_rate_at_confidence_0_9": 0.0031446540880503146, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 641, |
| "accuracy": 0.9984399375975039, |
| "mean_class_recall": 0.9969512195121951, |
| "nll": 0.0034185049808860444, |
| "brier": 0.0023816170036107054, |
| "ece15": 0.0015601193471088582, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9984399375975039, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 805, |
| "accuracy": 0.7031055900621118, |
| "mean_class_recall": 0.7239978843175034, |
| "nll": 0.503140821446039, |
| "brier": 0.35269849107227197, |
| "ece15": 0.030978365187072362, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.21366459627329193, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 805, |
| "accuracy": 0.6919254658385093, |
| "mean_class_recall": 0.5331098265895953, |
| "nll": 0.9878648410254747, |
| "brier": 0.46649698606417656, |
| "ece15": 0.09086134384209663, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.19627329192546583, |
| "error_rate_at_confidence_0_9": 0.006329113924050633, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9407031795465964, |
| "macro_task_balanced_accuracy": 0.9191604101516703 |
| } |
| } |
| } |
|
|
