{
  "epoch": 3,
  "temperature": 1.334968103387446,
  "test": {
    "raw": {
      "questions": 21634,
      "accuracy": 0.8801423684940372,
      "mean_class_recall": 0.8422636415920437,
      "nll": 0.35756736839135106,
      "brier": 0.17329397786588344,
      "ece15": 0.018370944185424988,
      "temperature": 1.0,
      "coverage_at_confidence_0_9": 0.681750947582509,
      "error_rate_at_confidence_0_9": 0.017289307749677945,
      "ordinal_expected_level_mae": 0.004830545729071066,
      "tasks": {
        "agentdojo_tool_outputs:agentdojo_boundary_action": {
          "questions": 426,
          "accuracy": 0.9953051643192489,
          "mean_class_recall": 0.9895833333333333,
          "nll": 0.019578322598972707,
          "brier": 0.008470972915844083,
          "ece15": 0.0032192125141472924,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9882629107981221,
          "error_rate_at_confidence_0_9": 0.0023752969121140144,
          "ordinal_expected_level_mae": null
        },
        "agentdojo_tool_outputs:agentdojo_embedded_injection": {
          "questions": 426,
          "accuracy": 0.9835680751173709,
          "mean_class_recall": 0.9635416666666667,
          "nll": 0.03323309985013501,
          "brier": 0.020700086530374747,
          "ece15": 0.012858446963474107,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9835680751173709,
          "error_rate_at_confidence_0_9": 0.007159904534606206,
          "ordinal_expected_level_mae": null
        },
        "byzfl_mnist:fl_attack_generated_update": {
          "questions": 800,
          "accuracy": 0.95875,
          "mean_class_recall": 0.9296875,
          "nll": 0.14798192410675134,
          "brier": 0.07208296294789789,
          "ece15": 0.031237402444916955,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9575,
          "error_rate_at_confidence_0_9": 0.02741514360313316,
          "ordinal_expected_level_mae": null
        },
        "ciciot2023:ciciot_attack": {
          "questions": 6000,
          "accuracy": 0.9171666666666667,
          "mean_class_recall": 0.8741111111111111,
          "nll": 0.2872065840367128,
          "brier": 0.1291518236859529,
          "ece15": 0.02997403903668212,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.7491666666666666,
          "error_rate_at_confidence_0_9": 0.03181312569521691,
          "ordinal_expected_level_mae": null
        },
        "injecagent:injecagent_requested_operation_v13": {
          "questions": 124,
          "accuracy": 1.0,
          "mean_class_recall": 1.0,
          "nll": 6.416181236130915e-05,
          "brier": 3.664856836177276e-07,
          "ece15": 6.406994017019252e-05,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 1.0,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "lanl_auth_policy:lanl_known_logon_action_v16": {
          "questions": 540,
          "accuracy": 1.0,
          "mean_class_recall": 1.0,
          "nll": 0.0003909349271697467,
          "brier": 5.411140895974333e-05,
          "ece15": 0.000376397356322764,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9981481481481481,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "lanl_auth_policy:lanl_observable_policy": {
          "questions": 540,
          "accuracy": 1.0,
          "mean_class_recall": 1.0,
          "nll": 1.9301873747749096e-07,
          "brier": 4.461551704214049e-12,
          "ece15": 1.9301762319156735e-07,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 1.0,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "ton_iot_network:ton_attack": {
          "questions": 3500,
          "accuracy": 0.988,
          "mean_class_recall": 0.9820952380952381,
          "nll": 0.052315917826989855,
          "brier": 0.021002166306533596,
          "ece15": 0.005742982405630235,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9757142857142858,
          "error_rate_at_confidence_0_9": 0.007320644216691069,
          "ordinal_expected_level_mae": null
        },
        "twins_streamlet:twins_evidence_level": {
          "questions": 659,
          "accuracy": 0.9954476479514416,
          "mean_class_recall": 0.9949748743718594,
          "nll": 0.006489090778116727,
          "brier": 0.004985928434486637,
          "ece15": 0.0035010905751582675,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9954476479514416,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": 0.004830545729071066
        },
        "twins_streamlet:twins_policy_action": {
          "questions": 659,
          "accuracy": 0.9984825493171472,
          "mean_class_recall": 0.9983249581239532,
          "nll": 0.007759166929265847,
          "brier": 0.0031279766576844436,
          "ece15": 0.0017302646891685911,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9969650986342944,
          "error_rate_at_confidence_0_9": 0.0015220700152207,
          "ordinal_expected_level_mae": null
        },
        "twins_streamlet:twins_quorum": {
          "questions": 659,
          "accuracy": 1.0,
          "mean_class_recall": 1.0,
          "nll": 0.0012916521858483854,
          "brier": 0.0006719742504699832,
          "ece15": 0.0010543087689309648,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9969650986342944,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "twins_streamlet:twins_visible_conflict": {
          "questions": 659,
          "accuracy": 0.9984825493171472,
          "mean_class_recall": 0.9989130434782609,
          "nll": 0.0030258654215359524,
          "brier": 0.0020473738981638956,
          "ece15": 0.0002839890142556098,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9954476479514416,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "veremi_nextgen:veremi_message_manipulation": {
          "questions": 3321,
          "accuracy": 0.7034025895814514,
          "mean_class_recall": 0.7106693655944849,
          "nll": 0.5271449745997652,
          "brier": 0.3679898300956757,
          "ece15": 0.03254066478455375,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.21499548328816623,
          "error_rate_at_confidence_0_9": 0.011204481792717087,
          "ordinal_expected_level_mae": null
        },
        "veremi_nextgen:veremi_supported_attack_type_v16": {
          "questions": 3321,
          "accuracy": 0.692261367058115,
          "mean_class_recall": 0.49711800080440777,
          "nll": 1.181957709780574,
          "brier": 0.48216427161657194,
          "ece15": 0.04703079874679226,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.20776874435411022,
          "error_rate_at_confidence_0_9": 0.07681159420289856,
          "ordinal_expected_level_mae": null
        }
      },
      "macro_task_accuracy": 0.9450619006663279,
      "macro_task_balanced_accuracy": 0.9242156493985226
    },
    "recalibrated": {
      "questions": 21634,
      "accuracy": 0.8801423684940372,
      "mean_class_recall": 0.8422636415920437,
      "nll": 0.3441350952603644,
      "brier": 0.1767857045083663,
      "ece15": 0.035627089616313175,
      "temperature": 1.334968103387446,
      "coverage_at_confidence_0_9": 0.6567440140519553,
      "error_rate_at_confidence_0_9": 0.014358108108108109,
      "ordinal_expected_level_mae": 0.004973955616734103,
      "tasks": {
        "agentdojo_tool_outputs:agentdojo_boundary_action": {
          "questions": 426,
          "accuracy": 0.9953051643192489,
          "mean_class_recall": 0.9895833333333333,
          "nll": 0.02085470758455793,
          "brier": 0.009033562612123906,
          "ece15": 0.005062121140742335,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9741784037558685,
          "error_rate_at_confidence_0_9": 0.0024096385542168677,
          "ordinal_expected_level_mae": null
        },
        "agentdojo_tool_outputs:agentdojo_embedded_injection": {
          "questions": 426,
          "accuracy": 0.9835680751173709,
          "mean_class_recall": 0.9635416666666667,
          "nll": 0.029576905474440387,
          "brier": 0.019371229609764937,
          "ece15": 0.0128149515297424,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9741784037558685,
          "error_rate_at_confidence_0_9": 0.004819277108433735,
          "ordinal_expected_level_mae": null
        },
        "byzfl_mnist:fl_attack_generated_update": {
          "questions": 800,
          "accuracy": 0.95875,
          "mean_class_recall": 0.9296875,
          "nll": 0.1252495255860491,
          "brier": 0.06891977027707019,
          "ece15": 0.02206119521855819,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.93375,
          "error_rate_at_confidence_0_9": 0.0214190093708166,
          "ordinal_expected_level_mae": null
        },
        "ciciot2023:ciciot_attack": {
          "questions": 6000,
          "accuracy": 0.9171666666666667,
          "mean_class_recall": 0.8741111111111111,
          "nll": 0.258923302286342,
          "brier": 0.1323925339560934,
          "ece15": 0.040786104751424794,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.6995,
          "error_rate_at_confidence_0_9": 0.027162258756254467,
          "ordinal_expected_level_mae": null
        },
        "injecagent:injecagent_requested_operation_v13": {
          "questions": 124,
          "accuracy": 1.0,
          "mean_class_recall": 1.0,
          "nll": 0.00036120828507376734,
          "brier": 6.118819643947097e-06,
          "ece15": 0.0003596634947089362,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 1.0,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "lanl_auth_policy:lanl_known_logon_action_v16": {
          "questions": 540,
          "accuracy": 1.0,
          "mean_class_recall": 1.0,
          "nll": 0.0010259839180316675,
          "brier": 0.00015355821904600842,
          "ece15": 0.0009831313211751498,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9981481481481481,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "lanl_auth_policy:lanl_observable_policy": {
          "questions": 540,
          "accuracy": 1.0,
          "mean_class_recall": 1.0,
          "nll": 5.4752266593721125e-06,
          "brier": 1.1349090161477055e-09,
          "ece15": 5.474942881804694e-06,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 1.0,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "ton_iot_network:ton_attack": {
          "questions": 3500,
          "accuracy": 0.988,
          "mean_class_recall": 0.9820952380952381,
          "nll": 0.0471904212042291,
          "brier": 0.02136284177729358,
          "ece15": 0.004930332206950378,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9597142857142857,
          "error_rate_at_confidence_0_9": 0.006251860672819291,
          "ordinal_expected_level_mae": null
        },
        "twins_streamlet:twins_evidence_level": {
          "questions": 659,
          "accuracy": 0.9954476479514416,
          "mean_class_recall": 0.9949748743718594,
          "nll": 0.005989405513587896,
          "brier": 0.004363630530423605,
          "ece15": 0.003686325313576514,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9954476479514416,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": 0.004973955616734103
        },
        "twins_streamlet:twins_policy_action": {
          "questions": 659,
          "accuracy": 0.9984825493171472,
          "mean_class_recall": 0.9983249581239532,
          "nll": 0.0069631222649601515,
          "brier": 0.003204452557959292,
          "ece15": 0.001531985329986691,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9969650986342944,
          "error_rate_at_confidence_0_9": 0.0015220700152207,
          "ordinal_expected_level_mae": null
        },
        "twins_streamlet:twins_quorum": {
          "questions": 659,
          "accuracy": 1.0,
          "mean_class_recall": 1.0,
          "nll": 0.0018952623564185236,
          "brier": 0.0008114028234666237,
          "ece15": 0.0016108880631044219,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9954476479514416,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "twins_streamlet:twins_visible_conflict": {
          "questions": 659,
          "accuracy": 0.9984825493171472,
          "mean_class_recall": 0.9989130434782609,
          "nll": 0.0033168872089549845,
          "brier": 0.0020980871703588605,
          "ece15": 0.0012518453771527926,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9954476479514416,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "veremi_nextgen:veremi_message_manipulation": {
          "questions": 3321,
          "accuracy": 0.7034025895814514,
          "mean_class_recall": 0.7106693655944849,
          "nll": 0.5278136514287293,
          "brier": 0.3689683794121852,
          "ece15": 0.03240031037986953,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.1969286359530262,
          "error_rate_at_confidence_0_9": 0.009174311926605505,
          "ordinal_expected_level_mae": null
        },
        "veremi_nextgen:veremi_supported_attack_type_v16": {
          "questions": 3321,
          "accuracy": 0.692261367058115,
          "mean_class_recall": 0.49711800080440777,
          "nll": 1.1560335557868109,
          "brier": 0.498611302076152,
          "ece15": 0.12617627958797675,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.17856067449563384,
          "error_rate_at_confidence_0_9": 0.07251264755480608,
          "ordinal_expected_level_mae": null
        }
      },
      "macro_task_accuracy": 0.9450619006663279,
      "macro_task_balanced_accuracy": 0.9242156493985226
    }
  },
  "test_used_for_selection": false
}