| { |
| "selected_epoch": 3, |
| "candidates": [ |
| { |
| "epoch": 1, |
| "metrics": { |
| "questions": 14912, |
| "accuracy": 0.8959898068669528, |
| "mean_class_recall": 0.8086847041598828, |
| "nll": 0.2676199905412643, |
| "brier": 0.14581168647550916, |
| "ece15": 0.024724176262894793, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.773806330472103, |
| "error_rate_at_confidence_0_9": 0.0240055464078343, |
| "ordinal_expected_level_mae": 0.00949861612167232, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 222, |
| "accuracy": 0.9819819819819819, |
| "mean_class_recall": 0.9670930232558139, |
| "nll": 0.06952124509948641, |
| "brier": 0.033360936223710204, |
| "ece15": 0.01963763286720348, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9774774774774775, |
| "error_rate_at_confidence_0_9": 0.013824884792626729, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 222, |
| "accuracy": 0.9864864864864865, |
| "mean_class_recall": 0.97, |
| "nll": 0.06796588080667874, |
| "brier": 0.02442578405456454, |
| "ece15": 0.013534851962772593, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.972972972972973, |
| "error_rate_at_confidence_0_9": 0.009259259259259259, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 1400, |
| "accuracy": 0.9742857142857143, |
| "mean_class_recall": 0.9504464285714286, |
| "nll": 0.07440929350732509, |
| "brier": 0.03861493679124129, |
| "ece15": 0.01061565077999396, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.925, |
| "error_rate_at_confidence_0_9": 0.008494208494208495, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 5000, |
| "accuracy": 0.8334, |
| "mean_class_recall": 0.7313333333333334, |
| "nll": 0.4357438451410003, |
| "brier": 0.2420765591030422, |
| "ece15": 0.05574236954650583, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.644, |
| "error_rate_at_confidence_0_9": 0.06677018633540373, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.004390672226438891, |
| "brier": 0.0015490359962901127, |
| "ece15": 0.003915493134373699, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9919354838709677, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.004780661486616687, |
| "brier": 0.0026501860965206722, |
| "ece15": 0.0038321931288341792, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9908536585365854, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 2.8436047490948972e-05, |
| "brier": 2.213459113159646e-07, |
| "ece15": 2.8380504783511674e-05, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 2500, |
| "accuracy": 0.9832, |
| "mean_class_recall": 0.9856, |
| "nll": 0.0716936772308886, |
| "brier": 0.032718644101254024, |
| "ece15": 0.01640596644430267, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9948, |
| "error_rate_at_confidence_0_9": 0.01648572577402493, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 797, |
| "accuracy": 0.9937264742785445, |
| "mean_class_recall": 0.9934913409721132, |
| "nll": 0.0153080496936535, |
| "brier": 0.009627399097334195, |
| "ece15": 0.004599932218083785, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9912170639899623, |
| "error_rate_at_confidence_0_9": 0.0037974683544303796, |
| "ordinal_expected_level_mae": 0.00949861612167232 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 797, |
| "accuracy": 0.9924717691342535, |
| "mean_class_recall": 0.9923419156847569, |
| "nll": 0.017883068917921716, |
| "brier": 0.011294880553062277, |
| "ece15": 0.0038184765498114437, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9811794228356336, |
| "error_rate_at_confidence_0_9": 0.0012787723785166241, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 797, |
| "accuracy": 0.9962358845671268, |
| "mean_class_recall": 0.995834315034425, |
| "nll": 0.011957269820791244, |
| "brier": 0.007393224542639707, |
| "ece15": 0.00670405372457154, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9836888331242158, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 797, |
| "accuracy": 0.9974905897114178, |
| "mean_class_recall": 0.9969442527413541, |
| "nll": 0.010437285979484219, |
| "brier": 0.0038211519865233618, |
| "ece15": 0.0009184477173700703, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9962358845671268, |
| "error_rate_at_confidence_0_9": 0.0012594458438287153, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 800, |
| "accuracy": 0.63, |
| "mean_class_recall": 0.6354549163606691, |
| "nll": 0.5876157448193948, |
| "brier": 0.4142276501377162, |
| "ece15": 0.031917999077751745, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.10375, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 800, |
| "accuracy": 0.59875, |
| "mean_class_recall": 0.4115791153670771, |
| "nll": 1.2269806519983115, |
| "brier": 0.571523062943736, |
| "ece15": 0.08060690424281568, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.11875, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9262877786032521, |
| "macro_task_balanced_accuracy": 0.9021513315229265, |
| "macro_task_nll": 0.18562255591253446 |
| }, |
| "checkpoint_files": { |
| "chat_template.jinja": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", |
| "tokenizer.json": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523", |
| "adapter_model.safetensors": "793c6d72e5c0254c28884384a8bc3ca633fd4de9511834f710b5ee2dc95a942f", |
| "tokenizer_config.json": "353c083805d863719e54973335a38e252a92e1ac64d32b83bb0b333cc1644ef2", |
| "head.pt": "75a1b2c4362ce9eaccad18d0531fea66ab895aa924b73492af78fe7abf997986", |
| "README.md": "fdbf6f672a47dca9ef2ad668b395cccbdf4a09c88d16bf8a90ecf4f7445f31ce", |
| "adapter_config.json": "94aad580ce54a1dfcec0803beaba2c471e17b790f10efe7a481125c81fe1dd02" |
| } |
| }, |
| { |
| "epoch": 2, |
| "metrics": { |
| "questions": 14912, |
| "accuracy": 0.9046405579399142, |
| "mean_class_recall": 0.8351706914084861, |
| "nll": 0.2609479185046837, |
| "brier": 0.13871271598030968, |
| "ece15": 0.02912533581539033, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.811762339055794, |
| "error_rate_at_confidence_0_9": 0.030318050392399834, |
| "ordinal_expected_level_mae": 0.007142014228908907, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 222, |
| "accuracy": 0.9774774774774775, |
| "mean_class_recall": 0.9570930232558139, |
| "nll": 0.0857490162426134, |
| "brier": 0.03780111867446018, |
| "ece15": 0.020326742033581136, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9864864864864865, |
| "error_rate_at_confidence_0_9": 0.0136986301369863, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 222, |
| "accuracy": 0.9819819819819819, |
| "mean_class_recall": 0.96, |
| "nll": 0.08583505640070498, |
| "brier": 0.03170617674969949, |
| "ece15": 0.018778574788420183, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9819819819819819, |
| "error_rate_at_confidence_0_9": 0.009174311926605505, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 1400, |
| "accuracy": 0.9828571428571429, |
| "mean_class_recall": 0.9678571428571429, |
| "nll": 0.05038447430212686, |
| "brier": 0.026644462415551518, |
| "ece15": 0.006643072262975827, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9757142857142858, |
| "error_rate_at_confidence_0_9": 0.008784773060029283, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 5000, |
| "accuracy": 0.839, |
| "mean_class_recall": 0.7132000000000001, |
| "nll": 0.4567278707638309, |
| "brier": 0.24289094068797595, |
| "ece15": 0.07089882316707959, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.7078, |
| "error_rate_at_confidence_0_9": 0.08194405199208817, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0030323687345377927, |
| "brier": 0.0010147751519150774, |
| "ece15": 0.0027304101493378944, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9919354838709677, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0013590718776517526, |
| "brier": 0.0005853758373684321, |
| "ece15": 0.001174090997847941, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9969512195121951, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 2.706087239227311e-05, |
| "brier": 4.4631636950527135e-07, |
| "ece15": 2.6948652833984242e-05, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 2500, |
| "accuracy": 0.9788, |
| "mean_class_recall": 0.98, |
| "nll": 0.08755100200107269, |
| "brier": 0.038158588944742826, |
| "ece15": 0.017829224144769444, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9896, |
| "error_rate_at_confidence_0_9": 0.016976556184316895, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 797, |
| "accuracy": 0.9949811794228356, |
| "mean_class_recall": 0.9949406163344321, |
| "nll": 0.020100883811983803, |
| "brier": 0.009532705405460478, |
| "ece15": 0.004664870689987746, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9962358845671268, |
| "error_rate_at_confidence_0_9": 0.003778337531486146, |
| "ordinal_expected_level_mae": 0.007142014228908907 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 797, |
| "accuracy": 0.9887076537013801, |
| "mean_class_recall": 0.9889237691623501, |
| "nll": 0.029531627570397425, |
| "brier": 0.01682083664455817, |
| "ece15": 0.008194444119065863, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9887076537013801, |
| "error_rate_at_confidence_0_9": 0.0038071065989847717, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 797, |
| "accuracy": 0.9937264742785445, |
| "mean_class_recall": 0.9922118380062306, |
| "nll": 0.019942020828876864, |
| "brier": 0.011733672902765547, |
| "ece15": 0.006519660642330038, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9912170639899623, |
| "error_rate_at_confidence_0_9": 0.0037974683544303796, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 797, |
| "accuracy": 0.9974905897114178, |
| "mean_class_recall": 0.998236331569665, |
| "nll": 0.016769983022509982, |
| "brier": 0.0049992664597086935, |
| "ece15": 0.0021278163552494573, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.002509410288582183, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 800, |
| "accuracy": 0.6875, |
| "mean_class_recall": 0.6966034271725826, |
| "nll": 0.5142628800857739, |
| "brier": 0.36155138388149327, |
| "ece15": 0.02205690591199928, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.22125, |
| "error_rate_at_confidence_0_9": 0.011299435028248588, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 800, |
| "accuracy": 0.67375, |
| "mean_class_recall": 0.5089660282717738, |
| "nll": 0.998813782244008, |
| "brier": 0.4775012041931279, |
| "ece15": 0.03873448735113525, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.20625, |
| "error_rate_at_confidence_0_9": 0.030303030303030304, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9354480356736271, |
| "macro_task_balanced_accuracy": 0.9112880126164279, |
| "macro_task_nll": 0.16929193562560574 |
| }, |
| "checkpoint_files": { |
| "chat_template.jinja": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", |
| "tokenizer.json": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523", |
| "adapter_model.safetensors": "f01d67ab57ea35f6f96dcf615f54147213977022560741db20fb7eab6eb462bd", |
| "tokenizer_config.json": "353c083805d863719e54973335a38e252a92e1ac64d32b83bb0b333cc1644ef2", |
| "head.pt": "460a08af9f39d347ecd48b74049d8a6e6140f1293160a4a5807a201b92ecc3c1", |
| "README.md": "fdbf6f672a47dca9ef2ad668b395cccbdf4a09c88d16bf8a90ecf4f7445f31ce", |
| "adapter_config.json": "94aad580ce54a1dfcec0803beaba2c471e17b790f10efe7a481125c81fe1dd02" |
| } |
| }, |
| { |
| "epoch": 3, |
| "metrics": { |
| "questions": 14912, |
| "accuracy": 0.9075912017167382, |
| "mean_class_recall": 0.8447857522159936, |
| "nll": 0.24807737680415345, |
| "brier": 0.12998082994334342, |
| "ece15": 0.027028147820359012, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.8103540772532188, |
| "error_rate_at_confidence_0_9": 0.023502151605428664, |
| "ordinal_expected_level_mae": 0.00481932257817722, |
| "tasks": { |
| "agentdojo_tool_outputs:agentdojo_boundary_action": { |
| "questions": 222, |
| "accuracy": 0.9774774774774775, |
| "mean_class_recall": 0.9570930232558139, |
| "nll": 0.10407641066616782, |
| "brier": 0.039347876403718875, |
| "ece15": 0.021432863016197557, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9864864864864865, |
| "error_rate_at_confidence_0_9": 0.0136986301369863, |
| "ordinal_expected_level_mae": null |
| }, |
| "agentdojo_tool_outputs:agentdojo_embedded_injection": { |
| "questions": 222, |
| "accuracy": 0.9819819819819819, |
| "mean_class_recall": 0.96, |
| "nll": 0.10023555924012383, |
| "brier": 0.031563944714978254, |
| "ece15": 0.015911253108198964, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9864864864864865, |
| "error_rate_at_confidence_0_9": 0.0136986301369863, |
| "ordinal_expected_level_mae": null |
| }, |
| "byzfl_mnist:fl_attack_generated_update": { |
| "questions": 1400, |
| "accuracy": 0.9885714285714285, |
| "mean_class_recall": 0.9821428571428572, |
| "nll": 0.0442702602822601, |
| "brier": 0.02139513973607548, |
| "ece15": 0.007710400104216533, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9842857142857143, |
| "error_rate_at_confidence_0_9": 0.008708272859216255, |
| "ordinal_expected_level_mae": null |
| }, |
| "ciciot2023:ciciot_attack": { |
| "questions": 5000, |
| "accuracy": 0.8418, |
| "mean_class_recall": 0.7537333333333334, |
| "nll": 0.4173917752941542, |
| "brier": 0.22451704505631584, |
| "ece15": 0.06284084146027137, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.6928, |
| "error_rate_at_confidence_0_9": 0.06293302540415704, |
| "ordinal_expected_level_mae": null |
| }, |
| "injecagent:injecagent_requested_operation_v13": { |
| "questions": 124, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.0014205610073774013, |
| "brier": 0.0002211290260091384, |
| "ece15": 0.0013613576932818498, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9919354838709677, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_known_logon_action_v16": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 0.000610657881540054, |
| "brier": 8.099314527924365e-05, |
| "ece15": 0.0005891478461821947, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "lanl_auth_policy:lanl_observable_policy": { |
| "questions": 328, |
| "accuracy": 1.0, |
| "mean_class_recall": 1.0, |
| "nll": 1.5480302068279906e-07, |
| "brier": 4.4217021643060835e-12, |
| "ece15": 1.548019148098009e-07, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 1.0, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "ton_iot_network:ton_attack": { |
| "questions": 2500, |
| "accuracy": 0.9792, |
| "mean_class_recall": 0.9818666666666667, |
| "nll": 0.11218220058914397, |
| "brier": 0.03724045269615108, |
| "ece15": 0.01842697861877182, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9928, |
| "error_rate_at_confidence_0_9": 0.015310233682514102, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_evidence_level": { |
| "questions": 797, |
| "accuracy": 0.9962358845671268, |
| "mean_class_recall": 0.9963898916967509, |
| "nll": 0.012217254972570101, |
| "brier": 0.006818829719414822, |
| "ece15": 0.004133398699556409, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9949811794228356, |
| "error_rate_at_confidence_0_9": 0.0012610340479192938, |
| "ordinal_expected_level_mae": 0.00481932257817722 |
| }, |
| "twins_streamlet:twins_policy_action": { |
| "questions": 797, |
| "accuracy": 0.9949811794228356, |
| "mean_class_recall": 0.9951865222623345, |
| "nll": 0.013078119892699127, |
| "brier": 0.007721112955228842, |
| "ece15": 0.003423998250058524, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9899623588456713, |
| "error_rate_at_confidence_0_9": 0.0012674271229404308, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_quorum": { |
| "questions": 797, |
| "accuracy": 0.9949811794228356, |
| "mean_class_recall": 0.994276682635671, |
| "nll": 0.01027058602627885, |
| "brier": 0.007209131739335666, |
| "ece15": 0.004201163796495765, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.9912170639899623, |
| "error_rate_at_confidence_0_9": 0.0, |
| "ordinal_expected_level_mae": null |
| }, |
| "twins_streamlet:twins_visible_conflict": { |
| "questions": 797, |
| "accuracy": 0.9974905897114178, |
| "mean_class_recall": 0.998236331569665, |
| "nll": 0.017287182530912663, |
| "brier": 0.004953169208636321, |
| "ece15": 0.0025266183132433223, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.998745294855709, |
| "error_rate_at_confidence_0_9": 0.002512562814070352, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_message_manipulation": { |
| "questions": 800, |
| "accuracy": 0.69, |
| "mean_class_recall": 0.7116483068135455, |
| "nll": 0.5092641106435474, |
| "brier": 0.3569714849542106, |
| "ece15": 0.03650030376694629, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.2425, |
| "error_rate_at_confidence_0_9": 0.010309278350515464, |
| "ordinal_expected_level_mae": null |
| }, |
| "veremi_nextgen:veremi_supported_attack_type_v16": { |
| "questions": 800, |
| "accuracy": 0.68875, |
| "mean_class_recall": 0.529392069171139, |
| "nll": 0.96833512912194, |
| "brier": 0.46247412161427887, |
| "ece15": 0.02731423542286039, |
| "temperature": 1.0, |
| "coverage_at_confidence_0_9": 0.22625, |
| "error_rate_at_confidence_0_9": 0.022099447513812154, |
| "ordinal_expected_level_mae": null |
| } |
| }, |
| "macro_task_accuracy": 0.9379621229396502, |
| "macro_task_balanced_accuracy": 0.9185689774676985, |
| "macro_task_nll": 0.16504571163940973 |
| }, |
| "checkpoint_files": { |
| "chat_template.jinja": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715", |
| "tokenizer.json": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523", |
| "adapter_model.safetensors": "0087e92e59b987fea721124d1a3f61b1aca26b3ac01b974032abb60799e7b175", |
| "tokenizer_config.json": "353c083805d863719e54973335a38e252a92e1ac64d32b83bb0b333cc1644ef2", |
| "head.pt": "f3c477c0419bd8dbf64d530d7f2576e084f65cdee3b351e34a7804e67b3979cf", |
| "README.md": "fdbf6f672a47dca9ef2ad668b395cccbdf4a09c88d16bf8a90ecf4f7445f31ce", |
| "adapter_config.json": "94aad580ce54a1dfcec0803beaba2c471e17b790f10efe7a481125c81fe1dd02" |
| } |
| } |
| ], |
| "rule": "maximum development macro-task semantic-class balanced accuracy; tie macro-task raw NLL, then earlier epoch", |
| "test_used_for_selection": false |
| } |
|
|
