File size: 15,015 Bytes
caf14ad | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 | {
"epoch": 3,
"temperature": 1.334968103387446,
"test": {
"raw": {
"questions": 21634,
"accuracy": 0.8801423684940372,
"mean_class_recall": 0.8422636415920437,
"nll": 0.35756736839135106,
"brier": 0.17329397786588344,
"ece15": 0.018370944185424988,
"temperature": 1.0,
"coverage_at_confidence_0_9": 0.681750947582509,
"error_rate_at_confidence_0_9": 0.017289307749677945,
"ordinal_expected_level_mae": 0.004830545729071066,
"tasks": {
"agentdojo_tool_outputs:agentdojo_boundary_action": {
"questions": 426,
"accuracy": 0.9953051643192489,
"mean_class_recall": 0.9895833333333333,
"nll": 0.019578322598972707,
"brier": 0.008470972915844083,
"ece15": 0.0032192125141472924,
"temperature": 1.0,
"coverage_at_confidence_0_9": 0.9882629107981221,
"error_rate_at_confidence_0_9": 0.0023752969121140144,
"ordinal_expected_level_mae": null
},
"agentdojo_tool_outputs:agentdojo_embedded_injection": {
"questions": 426,
"accuracy": 0.9835680751173709,
"mean_class_recall": 0.9635416666666667,
"nll": 0.03323309985013501,
"brier": 0.020700086530374747,
"ece15": 0.012858446963474107,
"temperature": 1.0,
"coverage_at_confidence_0_9": 0.9835680751173709,
"error_rate_at_confidence_0_9": 0.007159904534606206,
"ordinal_expected_level_mae": null
},
"byzfl_mnist:fl_attack_generated_update": {
"questions": 800,
"accuracy": 0.95875,
"mean_class_recall": 0.9296875,
"nll": 0.14798192410675134,
"brier": 0.07208296294789789,
"ece15": 0.031237402444916955,
"temperature": 1.0,
"coverage_at_confidence_0_9": 0.9575,
"error_rate_at_confidence_0_9": 0.02741514360313316,
"ordinal_expected_level_mae": null
},
"ciciot2023:ciciot_attack": {
"questions": 6000,
"accuracy": 0.9171666666666667,
"mean_class_recall": 0.8741111111111111,
"nll": 0.2872065840367128,
"brier": 0.1291518236859529,
"ece15": 0.02997403903668212,
"temperature": 1.0,
"coverage_at_confidence_0_9": 0.7491666666666666,
"error_rate_at_confidence_0_9": 0.03181312569521691,
"ordinal_expected_level_mae": null
},
"injecagent:injecagent_requested_operation_v13": {
"questions": 124,
"accuracy": 1.0,
"mean_class_recall": 1.0,
"nll": 6.416181236130915e-05,
"brier": 3.664856836177276e-07,
"ece15": 6.406994017019252e-05,
"temperature": 1.0,
"coverage_at_confidence_0_9": 1.0,
"error_rate_at_confidence_0_9": 0.0,
"ordinal_expected_level_mae": null
},
"lanl_auth_policy:lanl_known_logon_action_v16": {
"questions": 540,
"accuracy": 1.0,
"mean_class_recall": 1.0,
"nll": 0.0003909349271697467,
"brier": 5.411140895974333e-05,
"ece15": 0.000376397356322764,
"temperature": 1.0,
"coverage_at_confidence_0_9": 0.9981481481481481,
"error_rate_at_confidence_0_9": 0.0,
"ordinal_expected_level_mae": null
},
"lanl_auth_policy:lanl_observable_policy": {
"questions": 540,
"accuracy": 1.0,
"mean_class_recall": 1.0,
"nll": 1.9301873747749096e-07,
"brier": 4.461551704214049e-12,
"ece15": 1.9301762319156735e-07,
"temperature": 1.0,
"coverage_at_confidence_0_9": 1.0,
"error_rate_at_confidence_0_9": 0.0,
"ordinal_expected_level_mae": null
},
"ton_iot_network:ton_attack": {
"questions": 3500,
"accuracy": 0.988,
"mean_class_recall": 0.9820952380952381,
"nll": 0.052315917826989855,
"brier": 0.021002166306533596,
"ece15": 0.005742982405630235,
"temperature": 1.0,
"coverage_at_confidence_0_9": 0.9757142857142858,
"error_rate_at_confidence_0_9": 0.007320644216691069,
"ordinal_expected_level_mae": null
},
"twins_streamlet:twins_evidence_level": {
"questions": 659,
"accuracy": 0.9954476479514416,
"mean_class_recall": 0.9949748743718594,
"nll": 0.006489090778116727,
"brier": 0.004985928434486637,
"ece15": 0.0035010905751582675,
"temperature": 1.0,
"coverage_at_confidence_0_9": 0.9954476479514416,
"error_rate_at_confidence_0_9": 0.0,
"ordinal_expected_level_mae": 0.004830545729071066
},
"twins_streamlet:twins_policy_action": {
"questions": 659,
"accuracy": 0.9984825493171472,
"mean_class_recall": 0.9983249581239532,
"nll": 0.007759166929265847,
"brier": 0.0031279766576844436,
"ece15": 0.0017302646891685911,
"temperature": 1.0,
"coverage_at_confidence_0_9": 0.9969650986342944,
"error_rate_at_confidence_0_9": 0.0015220700152207,
"ordinal_expected_level_mae": null
},
"twins_streamlet:twins_quorum": {
"questions": 659,
"accuracy": 1.0,
"mean_class_recall": 1.0,
"nll": 0.0012916521858483854,
"brier": 0.0006719742504699832,
"ece15": 0.0010543087689309648,
"temperature": 1.0,
"coverage_at_confidence_0_9": 0.9969650986342944,
"error_rate_at_confidence_0_9": 0.0,
"ordinal_expected_level_mae": null
},
"twins_streamlet:twins_visible_conflict": {
"questions": 659,
"accuracy": 0.9984825493171472,
"mean_class_recall": 0.9989130434782609,
"nll": 0.0030258654215359524,
"brier": 0.0020473738981638956,
"ece15": 0.0002839890142556098,
"temperature": 1.0,
"coverage_at_confidence_0_9": 0.9954476479514416,
"error_rate_at_confidence_0_9": 0.0,
"ordinal_expected_level_mae": null
},
"veremi_nextgen:veremi_message_manipulation": {
"questions": 3321,
"accuracy": 0.7034025895814514,
"mean_class_recall": 0.7106693655944849,
"nll": 0.5271449745997652,
"brier": 0.3679898300956757,
"ece15": 0.03254066478455375,
"temperature": 1.0,
"coverage_at_confidence_0_9": 0.21499548328816623,
"error_rate_at_confidence_0_9": 0.011204481792717087,
"ordinal_expected_level_mae": null
},
"veremi_nextgen:veremi_supported_attack_type_v16": {
"questions": 3321,
"accuracy": 0.692261367058115,
"mean_class_recall": 0.49711800080440777,
"nll": 1.181957709780574,
"brier": 0.48216427161657194,
"ece15": 0.04703079874679226,
"temperature": 1.0,
"coverage_at_confidence_0_9": 0.20776874435411022,
"error_rate_at_confidence_0_9": 0.07681159420289856,
"ordinal_expected_level_mae": null
}
},
"macro_task_accuracy": 0.9450619006663279,
"macro_task_balanced_accuracy": 0.9242156493985226
},
"recalibrated": {
"questions": 21634,
"accuracy": 0.8801423684940372,
"mean_class_recall": 0.8422636415920437,
"nll": 0.3441350952603644,
"brier": 0.1767857045083663,
"ece15": 0.035627089616313175,
"temperature": 1.334968103387446,
"coverage_at_confidence_0_9": 0.6567440140519553,
"error_rate_at_confidence_0_9": 0.014358108108108109,
"ordinal_expected_level_mae": 0.004973955616734103,
"tasks": {
"agentdojo_tool_outputs:agentdojo_boundary_action": {
"questions": 426,
"accuracy": 0.9953051643192489,
"mean_class_recall": 0.9895833333333333,
"nll": 0.02085470758455793,
"brier": 0.009033562612123906,
"ece15": 0.005062121140742335,
"temperature": 1.334968103387446,
"coverage_at_confidence_0_9": 0.9741784037558685,
"error_rate_at_confidence_0_9": 0.0024096385542168677,
"ordinal_expected_level_mae": null
},
"agentdojo_tool_outputs:agentdojo_embedded_injection": {
"questions": 426,
"accuracy": 0.9835680751173709,
"mean_class_recall": 0.9635416666666667,
"nll": 0.029576905474440387,
"brier": 0.019371229609764937,
"ece15": 0.0128149515297424,
"temperature": 1.334968103387446,
"coverage_at_confidence_0_9": 0.9741784037558685,
"error_rate_at_confidence_0_9": 0.004819277108433735,
"ordinal_expected_level_mae": null
},
"byzfl_mnist:fl_attack_generated_update": {
"questions": 800,
"accuracy": 0.95875,
"mean_class_recall": 0.9296875,
"nll": 0.1252495255860491,
"brier": 0.06891977027707019,
"ece15": 0.02206119521855819,
"temperature": 1.334968103387446,
"coverage_at_confidence_0_9": 0.93375,
"error_rate_at_confidence_0_9": 0.0214190093708166,
"ordinal_expected_level_mae": null
},
"ciciot2023:ciciot_attack": {
"questions": 6000,
"accuracy": 0.9171666666666667,
"mean_class_recall": 0.8741111111111111,
"nll": 0.258923302286342,
"brier": 0.1323925339560934,
"ece15": 0.040786104751424794,
"temperature": 1.334968103387446,
"coverage_at_confidence_0_9": 0.6995,
"error_rate_at_confidence_0_9": 0.027162258756254467,
"ordinal_expected_level_mae": null
},
"injecagent:injecagent_requested_operation_v13": {
"questions": 124,
"accuracy": 1.0,
"mean_class_recall": 1.0,
"nll": 0.00036120828507376734,
"brier": 6.118819643947097e-06,
"ece15": 0.0003596634947089362,
"temperature": 1.334968103387446,
"coverage_at_confidence_0_9": 1.0,
"error_rate_at_confidence_0_9": 0.0,
"ordinal_expected_level_mae": null
},
"lanl_auth_policy:lanl_known_logon_action_v16": {
"questions": 540,
"accuracy": 1.0,
"mean_class_recall": 1.0,
"nll": 0.0010259839180316675,
"brier": 0.00015355821904600842,
"ece15": 0.0009831313211751498,
"temperature": 1.334968103387446,
"coverage_at_confidence_0_9": 0.9981481481481481,
"error_rate_at_confidence_0_9": 0.0,
"ordinal_expected_level_mae": null
},
"lanl_auth_policy:lanl_observable_policy": {
"questions": 540,
"accuracy": 1.0,
"mean_class_recall": 1.0,
"nll": 5.4752266593721125e-06,
"brier": 1.1349090161477055e-09,
"ece15": 5.474942881804694e-06,
"temperature": 1.334968103387446,
"coverage_at_confidence_0_9": 1.0,
"error_rate_at_confidence_0_9": 0.0,
"ordinal_expected_level_mae": null
},
"ton_iot_network:ton_attack": {
"questions": 3500,
"accuracy": 0.988,
"mean_class_recall": 0.9820952380952381,
"nll": 0.0471904212042291,
"brier": 0.02136284177729358,
"ece15": 0.004930332206950378,
"temperature": 1.334968103387446,
"coverage_at_confidence_0_9": 0.9597142857142857,
"error_rate_at_confidence_0_9": 0.006251860672819291,
"ordinal_expected_level_mae": null
},
"twins_streamlet:twins_evidence_level": {
"questions": 659,
"accuracy": 0.9954476479514416,
"mean_class_recall": 0.9949748743718594,
"nll": 0.005989405513587896,
"brier": 0.004363630530423605,
"ece15": 0.003686325313576514,
"temperature": 1.334968103387446,
"coverage_at_confidence_0_9": 0.9954476479514416,
"error_rate_at_confidence_0_9": 0.0,
"ordinal_expected_level_mae": 0.004973955616734103
},
"twins_streamlet:twins_policy_action": {
"questions": 659,
"accuracy": 0.9984825493171472,
"mean_class_recall": 0.9983249581239532,
"nll": 0.0069631222649601515,
"brier": 0.003204452557959292,
"ece15": 0.001531985329986691,
"temperature": 1.334968103387446,
"coverage_at_confidence_0_9": 0.9969650986342944,
"error_rate_at_confidence_0_9": 0.0015220700152207,
"ordinal_expected_level_mae": null
},
"twins_streamlet:twins_quorum": {
"questions": 659,
"accuracy": 1.0,
"mean_class_recall": 1.0,
"nll": 0.0018952623564185236,
"brier": 0.0008114028234666237,
"ece15": 0.0016108880631044219,
"temperature": 1.334968103387446,
"coverage_at_confidence_0_9": 0.9954476479514416,
"error_rate_at_confidence_0_9": 0.0,
"ordinal_expected_level_mae": null
},
"twins_streamlet:twins_visible_conflict": {
"questions": 659,
"accuracy": 0.9984825493171472,
"mean_class_recall": 0.9989130434782609,
"nll": 0.0033168872089549845,
"brier": 0.0020980871703588605,
"ece15": 0.0012518453771527926,
"temperature": 1.334968103387446,
"coverage_at_confidence_0_9": 0.9954476479514416,
"error_rate_at_confidence_0_9": 0.0,
"ordinal_expected_level_mae": null
},
"veremi_nextgen:veremi_message_manipulation": {
"questions": 3321,
"accuracy": 0.7034025895814514,
"mean_class_recall": 0.7106693655944849,
"nll": 0.5278136514287293,
"brier": 0.3689683794121852,
"ece15": 0.03240031037986953,
"temperature": 1.334968103387446,
"coverage_at_confidence_0_9": 0.1969286359530262,
"error_rate_at_confidence_0_9": 0.009174311926605505,
"ordinal_expected_level_mae": null
},
"veremi_nextgen:veremi_supported_attack_type_v16": {
"questions": 3321,
"accuracy": 0.692261367058115,
"mean_class_recall": 0.49711800080440777,
"nll": 1.1560335557868109,
"brier": 0.498611302076152,
"ece15": 0.12617627958797675,
"temperature": 1.334968103387446,
"coverage_at_confidence_0_9": 0.17856067449563384,
"error_rate_at_confidence_0_9": 0.07251264755480608,
"ordinal_expected_level_mae": null
}
},
"macro_task_accuracy": 0.9450619006663279,
"macro_task_balanced_accuracy": 0.9242156493985226
}
},
"test_used_for_selection": false
}
|