| { |
| "epoch": 3, |
| "temperature": 1.334968103387446, |
| "test": { |
| "raw": { |
| "questions": 21634, |
| "accuracy": 0.8801423684940372, |
| "mean_class_recall": 0.8422636415920437, |
| "nll": 0.35756736839135106, |
| "brier": 0.17329397786588344, |
| "ece15": 0.018370944185424988, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.681750947582509, |
| "error_rate_at_confidence_0_9": 0.017289307749677945, |
| "ordinal_expected_level_mae": 0.004830545729071066, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 426, |
| "accuracy": 0.9953051643192489, |
| "mean_class_recall": 0.9895833333333333, |
| "nll": 0.019578322598972707, |
| "brier": 0.008470972915844083, |
| "ece15": 0.0032192125141472924, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9882629107981221, |
| "error_rate_at_confidence_0_9": 0.0023752969121140144, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 426, |
| "accuracy": 0.9835680751173709, |
| "mean_class_recall": 0.9635416666666667, |
| "nll": 0.03323309985013501, |
| "brier": 0.020700086530374747, |
| "ece15": 0.012858446963474107, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9835680751173709, |
| "error_rate_at_confidence_0_9": 0.007159904534606206, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 800, |
| "accuracy": 0.95875, |
| "mean_class_recall": 0.9296875, |
| "nll": 0.14798192410675134, |
| "brier": 0.07208296294789789, |
| "ece15": 0.031237402444916955, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9575, |
| "error_rate_at_confidence_0_9": 0.02741514360313316, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 6000, |
| "accuracy": 0.9171666666666667, |
| "mean_class_recall": 0.8741111111111111, |
| "nll": 0.2872065840367128, |
| "brier": 0.1291518236859529, |
| "ece15": 0.02997403903668212, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.7491666666666666, |
| "error_rate_at_confidence_0_9": 0.03181312569521691, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 6.416181236130915e-05, |
| "brier": 3.664856836177276e-07, |
| "ece15": 6.406994017019252e-05, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 540, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0003909349271697467, |
| "brier": 5.411140895974333e-05, |
| "ece15": 0.000376397356322764, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9981481481481481, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 540, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 1.9301873747749096e-07, |
| "brier": 4.461551704214049e-12, |
| "ece15": 1.9301762319156735e-07, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 3500, |
| "accuracy": 0.988, |
| "mean_class_recall": 0.9820952380952381, |
| "nll": 0.052315917826989855, |
| "brier": 0.021002166306533596, |
| "ece15": 0.005742982405630235, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9757142857142858, |
| "error_rate_at_confidence_0_9": 0.007320644216691069, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 659, |
| "accuracy": 0.9954476479514416, |
| "mean_class_recall": 0.9949748743718594, |
| "nll": 0.006489090778116727, |
| "brier": 0.004985928434486637, |
| "ece15": 0.0035010905751582675, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9954476479514416, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": 0.004830545729071066 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 659, |
| "accuracy": 0.9984825493171472, |
| "mean_class_recall": 0.9983249581239532, |
| "nll": 0.007759166929265847, |
| "brier": 0.0031279766576844436, |
| "ece15": 0.0017302646891685911, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9969650986342944, |
| "error_rate_at_confidence_0_9": 0.0015220700152207, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 659, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0012916521858483854, |
| "brier": 0.0006719742504699832, |
| "ece15": 0.0010543087689309648, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9969650986342944, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 659, |
| "accuracy": 0.9984825493171472, |
| "mean_class_recall": 0.9989130434782609, |
| "nll": 0.0030258654215359524, |
| "brier": 0.0020473738981638956, |
| "ece15": 0.0002839890142556098, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9954476479514416, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 3321, |
| "accuracy": 0.7034025895814514, |
| "mean_class_recall": 0.7106693655944849, |
| "nll": 0.5271449745997652, |
| "brier": 0.3679898300956757, |
| "ece15": 0.03254066478455375, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.21499548328816623, |
| "error_rate_at_confidence_0_9": 0.011204481792717087, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 3321, |
| "accuracy": 0.692261367058115, |
| "mean_class_recall": 0.49711800080440777, |
| "nll": 1.181957709780574, |
| "brier": 0.48216427161657194, |
| "ece15": 0.04703079874679226, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.20776874435411022, |
| "error_rate_at_confidence_0_9": 0.07681159420289856, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9450619006663279, |
| "macro_task_balanced_accuracy": 0.9242156493985226 |
| }, |
| "recalibrated": { |
| "questions": 21634, |
| "accuracy": 0.8801423684940372, |
| "mean_class_recall": 0.8422636415920437, |
| "nll": 0.3441350952603644, |
| "brier": 0.1767857045083663, |
| "ece15": 0.035627089616313175, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.6567440140519553, |
| "error_rate_at_confidence_0_9": 0.014358108108108109, |
| "ordinal_expected_level_mae": 0.004973955616734103, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 426, |
| "accuracy": 0.9953051643192489, |
| "mean_class_recall": 0.9895833333333333, |
| "nll": 0.02085470758455793, |
| "brier": 0.009033562612123906, |
| "ece15": 0.005062121140742335, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9741784037558685, |
| "error_rate_at_confidence_0_9": 0.0024096385542168677, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 426, |
| "accuracy": 0.9835680751173709, |
| "mean_class_recall": 0.9635416666666667, |
| "nll": 0.029576905474440387, |
| "brier": 0.019371229609764937, |
| "ece15": 0.0128149515297424, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9741784037558685, |
| "error_rate_at_confidence_0_9": 0.004819277108433735, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 800, |
| "accuracy": 0.95875, |
| "mean_class_recall": 0.9296875, |
| "nll": 0.1252495255860491, |
| "brier": 0.06891977027707019, |
| "ece15": 0.02206119521855819, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.93375, |
| "error_rate_at_confidence_0_9": 0.0214190093708166, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 6000, |
| "accuracy": 0.9171666666666667, |
| "mean_class_recall": 0.8741111111111111, |
| "nll": 0.258923302286342, |
| "brier": 0.1323925339560934, |
| "ece15": 0.040786104751424794, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.6995, |
| "error_rate_at_confidence_0_9": 0.027162258756254467, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.00036120828507376734, |
| "brier": 6.118819643947097e-06, |
| "ece15": 0.0003596634947089362, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 540, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0010259839180316675, |
| "brier": 0.00015355821904600842, |
| "ece15": 0.0009831313211751498, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9981481481481481, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 540, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 5.4752266593721125e-06, |
| "brier": 1.1349090161477055e-09, |
| "ece15": 5.474942881804694e-06, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 3500, |
| "accuracy": 0.988, |
| "mean_class_recall": 0.9820952380952381, |
| "nll": 0.0471904212042291, |
| "brier": 0.02136284177729358, |
| "ece15": 0.004930332206950378, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9597142857142857, |
| "error_rate_at_confidence_0_9": 0.006251860672819291, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 659, |
| "accuracy": 0.9954476479514416, |
| "mean_class_recall": 0.9949748743718594, |
| "nll": 0.005989405513587896, |
| "brier": 0.004363630530423605, |
| "ece15": 0.003686325313576514, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9954476479514416, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": 0.004973955616734103 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 659, |
| "accuracy": 0.9984825493171472, |
| "mean_class_recall": 0.9983249581239532, |
| "nll": 0.0069631222649601515, |
| "brier": 0.003204452557959292, |
| "ece15": 0.001531985329986691, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9969650986342944, |
| "error_rate_at_confidence_0_9": 0.0015220700152207, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 659, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0018952623564185236, |
| "brier": 0.0008114028234666237, |
| "ece15": 0.0016108880631044219, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9954476479514416, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 659, |
| "accuracy": 0.9984825493171472, |
| "mean_class_recall": 0.9989130434782609, |
| "nll": 0.0033168872089549845, |
| "brier": 0.0020980871703588605, |
| "ece15": 0.0012518453771527926, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.9954476479514416, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 3321, |
| "accuracy": 0.7034025895814514, |
| "mean_class_recall": 0.7106693655944849, |
| "nll": 0.5278136514287293, |
| "brier": 0.3689683794121852, |
| "ece15": 0.03240031037986953, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.1969286359530262, |
| "error_rate_at_confidence_0_9": 0.009174311926605505, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 3321, |
| "accuracy": 0.692261367058115, |
| "mean_class_recall": 0.49711800080440777, |
| "nll": 1.1560335557868109, |
| "brier": 0.498611302076152, |
| "ece15": 0.12617627958797675, |
| "temperature": 1.334968103387446, |
| "coverage_at_confidence_0_9": 0.17856067449563384, |
| "error_rate_at_confidence_0_9": 0.07251264755480608, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9450619006663279, |
| "macro_task_balanced_accuracy": 0.9242156493985226 |
| } |
| }, |
| "test_used_for_selection": false |
| } |
|
|
