{ "selected_epoch": 3, "candidates": [ { "epoch": 1, "metrics": { "questions": 14912, "accuracy": 0.8959898068669528, "mean_class_recall": 0.8086847041598828, "nll": 0.2676199905412643, "brier": 0.14581168647550916, "ece15": 0.024724176262894793, "temperature": 1.0, "coverage_at_confidence_0_9": 0.773806330472103, "error_rate_at_confidence_0_9": 0.0240055464078343, "ordinal_expected_level_mae": 0.00949861612167232, "tasks": { "agentdojo_tool_outputs:agentdojo_boundary_action": { "questions": 222, "accuracy": 0.9819819819819819, "mean_class_recall": 0.9670930232558139, "nll": 0.06952124509948641, "brier": 0.033360936223710204, "ece15": 0.01963763286720348, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9774774774774775, "error_rate_at_confidence_0_9": 0.013824884792626729, "ordinal_expected_level_mae": null }, "agentdojo_tool_outputs:agentdojo_embedded_injection": { "questions": 222, "accuracy": 0.9864864864864865, "mean_class_recall": 0.97, "nll": 0.06796588080667874, "brier": 0.02442578405456454, "ece15": 0.013534851962772593, "temperature": 1.0, "coverage_at_confidence_0_9": 0.972972972972973, "error_rate_at_confidence_0_9": 0.009259259259259259, "ordinal_expected_level_mae": null }, "byzfl_mnist:fl_attack_generated_update": { "questions": 1400, "accuracy": 0.9742857142857143, "mean_class_recall": 0.9504464285714286, "nll": 0.07440929350732509, "brier": 0.03861493679124129, "ece15": 0.01061565077999396, "temperature": 1.0, "coverage_at_confidence_0_9": 0.925, "error_rate_at_confidence_0_9": 0.008494208494208495, "ordinal_expected_level_mae": null }, "ciciot2023:ciciot_attack": { "questions": 5000, "accuracy": 0.8334, "mean_class_recall": 0.7313333333333334, "nll": 0.4357438451410003, "brier": 0.2420765591030422, "ece15": 0.05574236954650583, "temperature": 1.0, "coverage_at_confidence_0_9": 0.644, "error_rate_at_confidence_0_9": 0.06677018633540373, "ordinal_expected_level_mae": null }, "injecagent:injecagent_requested_operation_v13": { "questions": 124, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.004390672226438891, "brier": 0.0015490359962901127, "ece15": 0.003915493134373699, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9919354838709677, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_known_logon_action_v16": { "questions": 328, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.004780661486616687, "brier": 0.0026501860965206722, "ece15": 0.0038321931288341792, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9908536585365854, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_observable_policy": { "questions": 328, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 2.8436047490948972e-05, "brier": 2.213459113159646e-07, "ece15": 2.8380504783511674e-05, "temperature": 1.0, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "ton_iot_network:ton_attack": { "questions": 2500, "accuracy": 0.9832, "mean_class_recall": 0.9856, "nll": 0.0716936772308886, "brier": 0.032718644101254024, "ece15": 0.01640596644430267, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9948, "error_rate_at_confidence_0_9": 0.01648572577402493, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_evidence_level": { "questions": 797, "accuracy": 0.9937264742785445, "mean_class_recall": 0.9934913409721132, "nll": 0.0153080496936535, "brier": 0.009627399097334195, "ece15": 0.004599932218083785, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9912170639899623, "error_rate_at_confidence_0_9": 0.0037974683544303796, "ordinal_expected_level_mae": 0.00949861612167232 }, "twins_streamlet:twins_policy_action": { "questions": 797, "accuracy": 0.9924717691342535, "mean_class_recall": 0.9923419156847569, "nll": 0.017883068917921716, "brier": 0.011294880553062277, "ece15": 0.0038184765498114437, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9811794228356336, "error_rate_at_confidence_0_9": 0.0012787723785166241, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_quorum": { "questions": 797, "accuracy": 0.9962358845671268, "mean_class_recall": 0.995834315034425, "nll": 0.011957269820791244, "brier": 0.007393224542639707, "ece15": 0.00670405372457154, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9836888331242158, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_visible_conflict": { "questions": 797, "accuracy": 0.9974905897114178, "mean_class_recall": 0.9969442527413541, "nll": 0.010437285979484219, "brier": 0.0038211519865233618, "ece15": 0.0009184477173700703, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9962358845671268, "error_rate_at_confidence_0_9": 0.0012594458438287153, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_message_manipulation": { "questions": 800, "accuracy": 0.63, "mean_class_recall": 0.6354549163606691, "nll": 0.5876157448193948, "brier": 0.4142276501377162, "ece15": 0.031917999077751745, "temperature": 1.0, "coverage_at_confidence_0_9": 0.10375, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_supported_attack_type_v16": { "questions": 800, "accuracy": 0.59875, "mean_class_recall": 0.4115791153670771, "nll": 1.2269806519983115, "brier": 0.571523062943736, "ece15": 0.08060690424281568, "temperature": 1.0, "coverage_at_confidence_0_9": 0.11875, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null } }, "macro_task_accuracy": 0.9262877786032521, "macro_task_balanced_accuracy": 0.9021513315229265, "macro_task_nll": 0.18562255591253446 }, "checkpoint_files": { "chat_template.jinja": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", "tokenizer.json": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523", "adapter_model.safetensors": "793c6d72e5c0254c28884384a8bc3ca633fd4de9511834f710b5ee2dc95a942f", "tokenizer_config.json": "353c083805d863719e54973335a38e252a92e1ac64d32b83bb0b333cc1644ef2", "head.pt": "75a1b2c4362ce9eaccad18d0531fea66ab895aa924b73492af78fe7abf997986", "README.md": "fdbf6f672a47dca9ef2ad668b395cccbdf4a09c88d16bf8a90ecf4f7445f31ce", "adapter_config.json": "94aad580ce54a1dfcec0803beaba2c471e17b790f10efe7a481125c81fe1dd02" } }, { "epoch": 2, "metrics": { "questions": 14912, "accuracy": 0.9046405579399142, "mean_class_recall": 0.8351706914084861, "nll": 0.2609479185046837, "brier": 0.13871271598030968, "ece15": 0.02912533581539033, "temperature": 1.0, "coverage_at_confidence_0_9": 0.811762339055794, "error_rate_at_confidence_0_9": 0.030318050392399834, "ordinal_expected_level_mae": 0.007142014228908907, "tasks": { "agentdojo_tool_outputs:agentdojo_boundary_action": { "questions": 222, "accuracy": 0.9774774774774775, "mean_class_recall": 0.9570930232558139, "nll": 0.0857490162426134, "brier": 0.03780111867446018, "ece15": 0.020326742033581136, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9864864864864865, "error_rate_at_confidence_0_9": 0.0136986301369863, "ordinal_expected_level_mae": null }, "agentdojo_tool_outputs:agentdojo_embedded_injection": { "questions": 222, "accuracy": 0.9819819819819819, "mean_class_recall": 0.96, "nll": 0.08583505640070498, "brier": 0.03170617674969949, "ece15": 0.018778574788420183, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9819819819819819, "error_rate_at_confidence_0_9": 0.009174311926605505, "ordinal_expected_level_mae": null }, "byzfl_mnist:fl_attack_generated_update": { "questions": 1400, "accuracy": 0.9828571428571429, "mean_class_recall": 0.9678571428571429, "nll": 0.05038447430212686, "brier": 0.026644462415551518, "ece15": 0.006643072262975827, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9757142857142858, "error_rate_at_confidence_0_9": 0.008784773060029283, "ordinal_expected_level_mae": null }, "ciciot2023:ciciot_attack": { "questions": 5000, "accuracy": 0.839, "mean_class_recall": 0.7132000000000001, "nll": 0.4567278707638309, "brier": 0.24289094068797595, "ece15": 0.07089882316707959, "temperature": 1.0, "coverage_at_confidence_0_9": 0.7078, "error_rate_at_confidence_0_9": 0.08194405199208817, "ordinal_expected_level_mae": null }, "injecagent:injecagent_requested_operation_v13": { "questions": 124, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.0030323687345377927, "brier": 0.0010147751519150774, "ece15": 0.0027304101493378944, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9919354838709677, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_known_logon_action_v16": { "questions": 328, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.0013590718776517526, "brier": 0.0005853758373684321, "ece15": 0.001174090997847941, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9969512195121951, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_observable_policy": { "questions": 328, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 2.706087239227311e-05, "brier": 4.4631636950527135e-07, "ece15": 2.6948652833984242e-05, "temperature": 1.0, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "ton_iot_network:ton_attack": { "questions": 2500, "accuracy": 0.9788, "mean_class_recall": 0.98, "nll": 0.08755100200107269, "brier": 0.038158588944742826, "ece15": 0.017829224144769444, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9896, "error_rate_at_confidence_0_9": 0.016976556184316895, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_evidence_level": { "questions": 797, "accuracy": 0.9949811794228356, "mean_class_recall": 0.9949406163344321, "nll": 0.020100883811983803, "brier": 0.009532705405460478, "ece15": 0.004664870689987746, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9962358845671268, "error_rate_at_confidence_0_9": 0.003778337531486146, "ordinal_expected_level_mae": 0.007142014228908907 }, "twins_streamlet:twins_policy_action": { "questions": 797, "accuracy": 0.9887076537013801, "mean_class_recall": 0.9889237691623501, "nll": 0.029531627570397425, "brier": 0.01682083664455817, "ece15": 0.008194444119065863, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9887076537013801, "error_rate_at_confidence_0_9": 0.0038071065989847717, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_quorum": { "questions": 797, "accuracy": 0.9937264742785445, "mean_class_recall": 0.9922118380062306, "nll": 0.019942020828876864, "brier": 0.011733672902765547, "ece15": 0.006519660642330038, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9912170639899623, "error_rate_at_confidence_0_9": 0.0037974683544303796, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_visible_conflict": { "questions": 797, "accuracy": 0.9974905897114178, "mean_class_recall": 0.998236331569665, "nll": 0.016769983022509982, "brier": 0.0049992664597086935, "ece15": 0.0021278163552494573, "temperature": 1.0, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.002509410288582183, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_message_manipulation": { "questions": 800, "accuracy": 0.6875, "mean_class_recall": 0.6966034271725826, "nll": 0.5142628800857739, "brier": 0.36155138388149327, "ece15": 0.02205690591199928, "temperature": 1.0, "coverage_at_confidence_0_9": 0.22125, "error_rate_at_confidence_0_9": 0.011299435028248588, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_supported_attack_type_v16": { "questions": 800, "accuracy": 0.67375, "mean_class_recall": 0.5089660282717738, "nll": 0.998813782244008, "brier": 0.4775012041931279, "ece15": 0.03873448735113525, "temperature": 1.0, "coverage_at_confidence_0_9": 0.20625, "error_rate_at_confidence_0_9": 0.030303030303030304, "ordinal_expected_level_mae": null } }, "macro_task_accuracy": 0.9354480356736271, "macro_task_balanced_accuracy": 0.9112880126164279, "macro_task_nll": 0.16929193562560574 }, "checkpoint_files": { "chat_template.jinja": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", "tokenizer.json": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523", "adapter_model.safetensors": "f01d67ab57ea35f6f96dcf615f54147213977022560741db20fb7eab6eb462bd", "tokenizer_config.json": "353c083805d863719e54973335a38e252a92e1ac64d32b83bb0b333cc1644ef2", "head.pt": "460a08af9f39d347ecd48b74049d8a6e6140f1293160a4a5807a201b92ecc3c1", "README.md": "fdbf6f672a47dca9ef2ad668b395cccbdf4a09c88d16bf8a90ecf4f7445f31ce", "adapter_config.json": "94aad580ce54a1dfcec0803beaba2c471e17b790f10efe7a481125c81fe1dd02" } }, { "epoch": 3, "metrics": { "questions": 14912, "accuracy": 0.9075912017167382, "mean_class_recall": 0.8447857522159936, "nll": 0.24807737680415345, "brier": 0.12998082994334342, "ece15": 0.027028147820359012, "temperature": 1.0, "coverage_at_confidence_0_9": 0.8103540772532188, "error_rate_at_confidence_0_9": 0.023502151605428664, "ordinal_expected_level_mae": 0.00481932257817722, "tasks": { "agentdojo_tool_outputs:agentdojo_boundary_action": { "questions": 222, "accuracy": 0.9774774774774775, "mean_class_recall": 0.9570930232558139, "nll": 0.10407641066616782, "brier": 0.039347876403718875, "ece15": 0.021432863016197557, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9864864864864865, "error_rate_at_confidence_0_9": 0.0136986301369863, "ordinal_expected_level_mae": null }, "agentdojo_tool_outputs:agentdojo_embedded_injection": { "questions": 222, "accuracy": 0.9819819819819819, "mean_class_recall": 0.96, "nll": 0.10023555924012383, "brier": 0.031563944714978254, "ece15": 0.015911253108198964, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9864864864864865, "error_rate_at_confidence_0_9": 0.0136986301369863, "ordinal_expected_level_mae": null }, "byzfl_mnist:fl_attack_generated_update": { "questions": 1400, "accuracy": 0.9885714285714285, "mean_class_recall": 0.9821428571428572, "nll": 0.0442702602822601, "brier": 0.02139513973607548, "ece15": 0.007710400104216533, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9842857142857143, "error_rate_at_confidence_0_9": 0.008708272859216255, "ordinal_expected_level_mae": null }, "ciciot2023:ciciot_attack": { "questions": 5000, "accuracy": 0.8418, "mean_class_recall": 0.7537333333333334, "nll": 0.4173917752941542, "brier": 0.22451704505631584, "ece15": 0.06284084146027137, "temperature": 1.0, "coverage_at_confidence_0_9": 0.6928, "error_rate_at_confidence_0_9": 0.06293302540415704, "ordinal_expected_level_mae": null }, "injecagent:injecagent_requested_operation_v13": { "questions": 124, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.0014205610073774013, "brier": 0.0002211290260091384, "ece15": 0.0013613576932818498, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9919354838709677, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_known_logon_action_v16": { "questions": 328, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 0.000610657881540054, "brier": 8.099314527924365e-05, "ece15": 0.0005891478461821947, "temperature": 1.0, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "lanl_auth_policy:lanl_observable_policy": { "questions": 328, "accuracy": 1.0, "mean_class_recall": 1.0, "nll": 1.5480302068279906e-07, "brier": 4.4217021643060835e-12, "ece15": 1.548019148098009e-07, "temperature": 1.0, "coverage_at_confidence_0_9": 1.0, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "ton_iot_network:ton_attack": { "questions": 2500, "accuracy": 0.9792, "mean_class_recall": 0.9818666666666667, "nll": 0.11218220058914397, "brier": 0.03724045269615108, "ece15": 0.01842697861877182, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9928, "error_rate_at_confidence_0_9": 0.015310233682514102, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_evidence_level": { "questions": 797, "accuracy": 0.9962358845671268, "mean_class_recall": 0.9963898916967509, "nll": 0.012217254972570101, "brier": 0.006818829719414822, "ece15": 0.004133398699556409, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9949811794228356, "error_rate_at_confidence_0_9": 0.0012610340479192938, "ordinal_expected_level_mae": 0.00481932257817722 }, "twins_streamlet:twins_policy_action": { "questions": 797, "accuracy": 0.9949811794228356, "mean_class_recall": 0.9951865222623345, "nll": 0.013078119892699127, "brier": 0.007721112955228842, "ece15": 0.003423998250058524, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9899623588456713, "error_rate_at_confidence_0_9": 0.0012674271229404308, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_quorum": { "questions": 797, "accuracy": 0.9949811794228356, "mean_class_recall": 0.994276682635671, "nll": 0.01027058602627885, "brier": 0.007209131739335666, "ece15": 0.004201163796495765, "temperature": 1.0, "coverage_at_confidence_0_9": 0.9912170639899623, "error_rate_at_confidence_0_9": 0.0, "ordinal_expected_level_mae": null }, "twins_streamlet:twins_visible_conflict": { "questions": 797, "accuracy": 0.9974905897114178, "mean_class_recall": 0.998236331569665, "nll": 0.017287182530912663, "brier": 0.004953169208636321, "ece15": 0.0025266183132433223, "temperature": 1.0, "coverage_at_confidence_0_9": 0.998745294855709, "error_rate_at_confidence_0_9": 0.002512562814070352, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_message_manipulation": { "questions": 800, "accuracy": 0.69, "mean_class_recall": 0.7116483068135455, "nll": 0.5092641106435474, "brier": 0.3569714849542106, "ece15": 0.03650030376694629, "temperature": 1.0, "coverage_at_confidence_0_9": 0.2425, "error_rate_at_confidence_0_9": 0.010309278350515464, "ordinal_expected_level_mae": null }, "veremi_nextgen:veremi_supported_attack_type_v16": { "questions": 800, "accuracy": 0.68875, "mean_class_recall": 0.529392069171139, "nll": 0.96833512912194, "brier": 0.46247412161427887, "ece15": 0.02731423542286039, "temperature": 1.0, "coverage_at_confidence_0_9": 0.22625, "error_rate_at_confidence_0_9": 0.022099447513812154, "ordinal_expected_level_mae": null } }, "macro_task_accuracy": 0.9379621229396502, "macro_task_balanced_accuracy": 0.9185689774676985, "macro_task_nll": 0.16504571163940973 }, "checkpoint_files": { "chat_template.jinja": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", "tokenizer.json": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523", "adapter_model.safetensors": "0087e92e59b987fea721124d1a3f61b1aca26b3ac01b974032abb60799e7b175", "tokenizer_config.json": "353c083805d863719e54973335a38e252a92e1ac64d32b83bb0b333cc1644ef2", "head.pt": "f3c477c0419bd8dbf64d530d7f2576e084f65cdee3b351e34a7804e67b3979cf", "README.md": "fdbf6f672a47dca9ef2ad668b395cccbdf4a09c88d16bf8a90ecf4f7445f31ce", "adapter_config.json": "94aad580ce54a1dfcec0803beaba2c471e17b790f10efe7a481125c81fe1dd02" } } ], "rule": "maximum development macro-task semantic-class balanced accuracy; tie macro-task raw NLL, then earlier epoch", "test_used_for_selection": false }