{
  "epoch": 3,
  "temperature": 1.334968103387446,
  "development": {
    "raw": {
      "questions": 14912,
      "accuracy": 0.9075912017167382,
      "mean_class_recall": 0.8447857522159936,
      "nll": 0.24807737680415345,
      "brier": 0.12998082994334342,
      "ece15": 0.027028147820359012,
      "temperature": 1.0,
      "coverage_at_confidence_0_9": 0.8103540772532188,
      "error_rate_at_confidence_0_9": 0.023502151605428664,
      "ordinal_expected_level_mae": 0.00481932257817722,
      "tasks": {
        "agentdojo_tool_outputs:agentdojo_boundary_action": {
          "questions": 222,
          "accuracy": 0.9774774774774775,
          "mean_class_recall": 0.9570930232558139,
          "nll": 0.10407641066616782,
          "brier": 0.039347876403718875,
          "ece15": 0.021432863016197557,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9864864864864865,
          "error_rate_at_confidence_0_9": 0.0136986301369863,
          "ordinal_expected_level_mae": null
        },
        "agentdojo_tool_outputs:agentdojo_embedded_injection": {
          "questions": 222,
          "accuracy": 0.9819819819819819,
          "mean_class_recall": 0.96,
          "nll": 0.10023555924012383,
          "brier": 0.031563944714978254,
          "ece15": 0.015911253108198964,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9864864864864865,
          "error_rate_at_confidence_0_9": 0.0136986301369863,
          "ordinal_expected_level_mae": null
        },
        "byzfl_mnist:fl_attack_generated_update": {
          "questions": 1400,
          "accuracy": 0.9885714285714285,
          "mean_class_recall": 0.9821428571428572,
          "nll": 0.0442702602822601,
          "brier": 0.02139513973607548,
          "ece15": 0.007710400104216533,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9842857142857143,
          "error_rate_at_confidence_0_9": 0.008708272859216255,
          "ordinal_expected_level_mae": null
        },
        "ciciot2023:ciciot_attack": {
          "questions": 5000,
          "accuracy": 0.8418,
          "mean_class_recall": 0.7537333333333334,
          "nll": 0.4173917752941542,
          "brier": 0.22451704505631584,
          "ece15": 0.06284084146027137,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.6928,
          "error_rate_at_confidence_0_9": 0.06293302540415704,
          "ordinal_expected_level_mae": null
        },
        "injecagent:injecagent_requested_operation_v13": {
          "questions": 124,
          "accuracy": 1.0,
          "mean_class_recall": 1.0,
          "nll": 0.0014205610073774013,
          "brier": 0.0002211290260091384,
          "ece15": 0.0013613576932818498,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9919354838709677,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "lanl_auth_policy:lanl_known_logon_action_v16": {
          "questions": 328,
          "accuracy": 1.0,
          "mean_class_recall": 1.0,
          "nll": 0.000610657881540054,
          "brier": 8.099314527924365e-05,
          "ece15": 0.0005891478461821947,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 1.0,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "lanl_auth_policy:lanl_observable_policy": {
          "questions": 328,
          "accuracy": 1.0,
          "mean_class_recall": 1.0,
          "nll": 1.5480302068279906e-07,
          "brier": 4.4217021643060835e-12,
          "ece15": 1.548019148098009e-07,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 1.0,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "ton_iot_network:ton_attack": {
          "questions": 2500,
          "accuracy": 0.9792,
          "mean_class_recall": 0.9818666666666667,
          "nll": 0.11218220058914397,
          "brier": 0.03724045269615108,
          "ece15": 0.01842697861877182,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9928,
          "error_rate_at_confidence_0_9": 0.015310233682514102,
          "ordinal_expected_level_mae": null
        },
        "twins_streamlet:twins_evidence_level": {
          "questions": 797,
          "accuracy": 0.9962358845671268,
          "mean_class_recall": 0.9963898916967509,
          "nll": 0.012217254972570101,
          "brier": 0.006818829719414822,
          "ece15": 0.004133398699556409,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9949811794228356,
          "error_rate_at_confidence_0_9": 0.0012610340479192938,
          "ordinal_expected_level_mae": 0.00481932257817722
        },
        "twins_streamlet:twins_policy_action": {
          "questions": 797,
          "accuracy": 0.9949811794228356,
          "mean_class_recall": 0.9951865222623345,
          "nll": 0.013078119892699127,
          "brier": 0.007721112955228842,
          "ece15": 0.003423998250058524,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9899623588456713,
          "error_rate_at_confidence_0_9": 0.0012674271229404308,
          "ordinal_expected_level_mae": null
        },
        "twins_streamlet:twins_quorum": {
          "questions": 797,
          "accuracy": 0.9949811794228356,
          "mean_class_recall": 0.994276682635671,
          "nll": 0.01027058602627885,
          "brier": 0.007209131739335666,
          "ece15": 0.004201163796495765,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9912170639899623,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "twins_streamlet:twins_visible_conflict": {
          "questions": 797,
          "accuracy": 0.9974905897114178,
          "mean_class_recall": 0.998236331569665,
          "nll": 0.017287182530912663,
          "brier": 0.004953169208636321,
          "ece15": 0.0025266183132433223,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.998745294855709,
          "error_rate_at_confidence_0_9": 0.002512562814070352,
          "ordinal_expected_level_mae": null
        },
        "veremi_nextgen:veremi_message_manipulation": {
          "questions": 800,
          "accuracy": 0.69,
          "mean_class_recall": 0.7116483068135455,
          "nll": 0.5092641106435474,
          "brier": 0.3569714849542106,
          "ece15": 0.03650030376694629,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.2425,
          "error_rate_at_confidence_0_9": 0.010309278350515464,
          "ordinal_expected_level_mae": null
        },
        "veremi_nextgen:veremi_supported_attack_type_v16": {
          "questions": 800,
          "accuracy": 0.68875,
          "mean_class_recall": 0.529392069171139,
          "nll": 0.96833512912194,
          "brier": 0.46247412161427887,
          "ece15": 0.02731423542286039,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.22625,
          "error_rate_at_confidence_0_9": 0.022099447513812154,
          "ordinal_expected_level_mae": null
        }
      },
      "macro_task_accuracy": 0.9379621229396502,
      "macro_task_balanced_accuracy": 0.9185689774676985
    },
    "recalibrated": {
      "questions": 14912,
      "accuracy": 0.9075912017167382,
      "mean_class_recall": 0.8447857522159936,
      "nll": 0.2269545932235271,
      "brier": 0.12792407717450321,
      "ece15": 0.01858051585491712,
      "temperature": 1.334968103387446,
      "coverage_at_confidence_0_9": 0.7862124463519313,
      "error_rate_at_confidence_0_9": 0.018679631525076765,
      "ordinal_expected_level_mae": 0.005329911013904147,
      "tasks": {
        "agentdojo_tool_outputs:agentdojo_boundary_action": {
          "questions": 222,
          "accuracy": 0.9774774774774775,
          "mean_class_recall": 0.9570930232558139,
          "nll": 0.08092683597443519,
          "brier": 0.037546344393533104,
          "ece15": 0.021320738677048364,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9819819819819819,
          "error_rate_at_confidence_0_9": 0.013761467889908258,
          "ordinal_expected_level_mae": null
        },
        "agentdojo_tool_outputs:agentdojo_embedded_injection": {
          "questions": 222,
          "accuracy": 0.9819819819819819,
          "mean_class_recall": 0.96,
          "nll": 0.07809919245507242,
          "brier": 0.030623326048867618,
          "ece15": 0.015019163882298042,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9819819819819819,
          "error_rate_at_confidence_0_9": 0.009174311926605505,
          "ordinal_expected_level_mae": null
        },
        "byzfl_mnist:fl_attack_generated_update": {
          "questions": 1400,
          "accuracy": 0.9885714285714285,
          "mean_class_recall": 0.9821428571428572,
          "nll": 0.04094699270351949,
          "brier": 0.021018406686723334,
          "ece15": 0.0038434318338614677,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9678571428571429,
          "error_rate_at_confidence_0_9": 0.006642066420664207,
          "ordinal_expected_level_mae": null
        },
        "ciciot2023:ciciot_attack": {
          "questions": 5000,
          "accuracy": 0.8418,
          "mean_class_recall": 0.7537333333333334,
          "nll": 0.36679030702727994,
          "brier": 0.21757064336558796,
          "ece15": 0.043357678681868704,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.6394,
          "error_rate_at_confidence_0_9": 0.05004691898654989,
          "ordinal_expected_level_mae": null
        },
        "injecagent:injecagent_requested_operation_v13": {
          "questions": 124,
          "accuracy": 1.0,
          "mean_class_recall": 1.0,
          "nll": 0.0028701157894541487,
          "brier": 0.0006216354903482641,
          "ece15": 0.0026971564840139535,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9919354838709677,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "lanl_auth_policy:lanl_known_logon_action_v16": {
          "questions": 328,
          "accuracy": 1.0,
          "mean_class_recall": 1.0,
          "nll": 0.001405432942745028,
          "brier": 0.00025083300955486444,
          "ece15": 0.0013358908781109478,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9939024390243902,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "lanl_auth_policy:lanl_observable_policy": {
          "questions": 328,
          "accuracy": 1.0,
          "mean_class_recall": 1.0,
          "nll": 4.6272880953518815e-06,
          "brier": 9.796066038293659e-10,
          "ece15": 4.627043141135246e-06,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 1.0,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "ton_iot_network:ton_attack": {
          "questions": 2500,
          "accuracy": 0.9792,
          "mean_class_recall": 0.9818666666666667,
          "nll": 0.0877031817842779,
          "brier": 0.03599036114227489,
          "ece15": 0.01538869285344686,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.988,
          "error_rate_at_confidence_0_9": 0.014574898785425101,
          "ordinal_expected_level_mae": null
        },
        "twins_streamlet:twins_evidence_level": {
          "questions": 797,
          "accuracy": 0.9962358845671268,
          "mean_class_recall": 0.9963898916967509,
          "nll": 0.010603240403009088,
          "brier": 0.006373682001028304,
          "ece15": 0.004218405852177047,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9937264742785445,
          "error_rate_at_confidence_0_9": 0.0012626262626262627,
          "ordinal_expected_level_mae": 0.005329911013904147
        },
        "twins_streamlet:twins_policy_action": {
          "questions": 797,
          "accuracy": 0.9949811794228356,
          "mean_class_recall": 0.9951865222623345,
          "nll": 0.01230072195659841,
          "brier": 0.007337903591834537,
          "ece15": 0.0031241457310850957,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.986198243412798,
          "error_rate_at_confidence_0_9": 0.001272264631043257,
          "ordinal_expected_level_mae": null
        },
        "twins_streamlet:twins_quorum": {
          "questions": 797,
          "accuracy": 0.9949811794228356,
          "mean_class_recall": 0.994276682635671,
          "nll": 0.009584146250019401,
          "brier": 0.00659904677807738,
          "ece15": 0.005028933100804552,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9899623588456713,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "twins_streamlet:twins_visible_conflict": {
          "questions": 797,
          "accuracy": 0.9974905897114178,
          "mean_class_recall": 0.998236331569665,
          "nll": 0.013437428285294059,
          "brier": 0.004858731838764845,
          "ece15": 0.0022785888551786117,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.998745294855709,
          "error_rate_at_confidence_0_9": 0.002512562814070352,
          "ordinal_expected_level_mae": null
        },
        "veremi_nextgen:veremi_message_manipulation": {
          "questions": 800,
          "accuracy": 0.69,
          "mean_class_recall": 0.7116483068135455,
          "nll": 0.5102124840713312,
          "brier": 0.35753050242002155,
          "ece15": 0.03811996421246104,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.22125,
          "error_rate_at_confidence_0_9": 0.011299435028248588,
          "ordinal_expected_level_mae": null
        },
        "veremi_nextgen:veremi_supported_attack_type_v16": {
          "questions": 800,
          "accuracy": 0.68875,
          "mean_class_recall": 0.529392069171139,
          "nll": 0.9911460027249259,
          "brier": 0.4737144267281807,
          "ece15": 0.09647043512079573,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.18625,
          "error_rate_at_confidence_0_9": 0.020134228187919462,
          "ordinal_expected_level_mae": null
        }
      },
      "macro_task_accuracy": 0.9379621229396502,
      "macro_task_balanced_accuracy": 0.9185689774676985
    }
  },
  "calibration": {
    "raw": {
      "questions": 14821,
      "accuracy": 0.9313811483705553,
      "mean_class_recall": 0.8460223158089928,
      "nll": 0.18480685074016417,
      "brier": 0.09978353062869037,
      "ece15": 0.010514966894040724,
      "temperature": 1.0,
      "coverage_at_confidence_0_9": 0.814789825247959,
      "error_rate_at_confidence_0_9": 0.013580655846306725,
      "ordinal_expected_level_mae": 0.001461981275023823,
      "tasks": {
        "agentdojo_tool_outputs:agentdojo_boundary_action": {
          "questions": 325,
          "accuracy": 0.96,
          "mean_class_recall": 0.9216916119306557,
          "nll": 0.12908652765840894,
          "brier": 0.06258807450225369,
          "ece15": 0.033510331142289206,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9815384615384616,
          "error_rate_at_confidence_0_9": 0.0219435736677116,
          "ordinal_expected_level_mae": null
        },
        "agentdojo_tool_outputs:agentdojo_embedded_injection": {
          "questions": 328,
          "accuracy": 0.9695121951219512,
          "mean_class_recall": 0.9372206852521814,
          "nll": 0.16127769578392517,
          "brier": 0.05858994554032181,
          "ece15": 0.030459345972402876,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9939024390243902,
          "error_rate_at_confidence_0_9": 0.027607361963190184,
          "ordinal_expected_level_mae": null
        },
        "byzfl_mnist:fl_attack_generated_update": {
          "questions": 2400,
          "accuracy": 0.9404166666666667,
          "mean_class_recall": 0.8877604166666666,
          "nll": 0.19653998710185536,
          "brier": 0.09947525932448194,
          "ece15": 0.04423318187692424,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9479166666666666,
          "error_rate_at_confidence_0_9": 0.03824175824175824,
          "ordinal_expected_level_mae": null
        },
        "ciciot2023:ciciot_attack": {
          "questions": 5000,
          "accuracy": 0.9292,
          "mean_class_recall": 0.8968,
          "nll": 0.18887834668200343,
          "brier": 0.10636649585712103,
          "ece15": 0.019887550017025992,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.7252,
          "error_rate_at_confidence_0_9": 0.013513513513513514,
          "ordinal_expected_level_mae": null
        },
        "injecagent:injecagent_requested_operation_v13": {
          "questions": 124,
          "accuracy": 0.9919354838709677,
          "mean_class_recall": 0.9928571428571429,
          "nll": 0.021390831595053342,
          "brier": 0.015445766007169246,
          "ece15": 0.011074830403285848,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9838709677419355,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "lanl_auth_policy:lanl_known_logon_action_v16": {
          "questions": 235,
          "accuracy": 0.9957446808510638,
          "mean_class_recall": 0.9949494949494949,
          "nll": 0.015420468691595452,
          "brier": 0.008033852620508352,
          "ece15": 0.003625196423068977,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 1.0,
          "error_rate_at_confidence_0_9": 0.00425531914893617,
          "ordinal_expected_level_mae": null
        },
        "lanl_auth_policy:lanl_observable_policy": {
          "questions": 235,
          "accuracy": 0.9957446808510638,
          "mean_class_recall": 0.9905660377358491,
          "nll": 0.018077073969230188,
          "brier": 0.00826909442413202,
          "ece15": 0.004194003764167214,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 1.0,
          "error_rate_at_confidence_0_9": 0.00425531914893617,
          "ordinal_expected_level_mae": null
        },
        "ton_iot_network:ton_attack": {
          "questions": 2000,
          "accuracy": 0.9985,
          "mean_class_recall": 0.9976666666666667,
          "nll": 0.010567088503443185,
          "brier": 0.003178972042914288,
          "ece15": 0.0009240536217447044,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.999,
          "error_rate_at_confidence_0_9": 0.0015015015015015015,
          "ordinal_expected_level_mae": null
        },
        "twins_streamlet:twins_evidence_level": {
          "questions": 641,
          "accuracy": 0.9984399375975039,
          "mean_class_recall": 0.9985507246376812,
          "nll": 0.003039872100282549,
          "brier": 0.0022297539963869384,
          "ece15": 0.0014593852165271412,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9984399375975039,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": 0.001461981275023823
        },
        "twins_streamlet:twins_policy_action": {
          "questions": 641,
          "accuracy": 1.0,
          "mean_class_recall": 1.0,
          "nll": 0.0004967972153911127,
          "brier": 9.688544623270663e-05,
          "ece15": 0.00046955068951652005,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9984399375975039,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "twins_streamlet:twins_quorum": {
          "questions": 641,
          "accuracy": 0.9968798751950078,
          "mean_class_recall": 0.9961240310077519,
          "nll": 0.016173152134660414,
          "brier": 0.007502235795954162,
          "ece15": 0.0049434253477994355,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.9937597503900156,
          "error_rate_at_confidence_0_9": 0.0031397174254317113,
          "ordinal_expected_level_mae": null
        },
        "twins_streamlet:twins_visible_conflict": {
          "questions": 641,
          "accuracy": 0.9984399375975039,
          "mean_class_recall": 0.9969512195121951,
          "nll": 0.004162755288573824,
          "brier": 0.0026938896990115683,
          "ece15": 0.0014825770107887316,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 1.0,
          "error_rate_at_confidence_0_9": 0.0015600624024961,
          "ordinal_expected_level_mae": null
        },
        "veremi_nextgen:veremi_message_manipulation": {
          "questions": 805,
          "accuracy": 0.7031055900621118,
          "mean_class_recall": 0.7239978843175034,
          "nll": 0.49879740669601635,
          "brier": 0.351098943795658,
          "ece15": 0.02568931078110801,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.23850931677018633,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "veremi_nextgen:veremi_supported_attack_type_v16": {
          "questions": 805,
          "accuracy": 0.6919254658385093,
          "mean_class_recall": 0.5331098265895953,
          "nll": 0.9684343228937802,
          "brier": 0.45465071518796674,
          "ece15": 0.01705305962599151,
          "temperature": 1.0,
          "coverage_at_confidence_0_9": 0.2360248447204969,
          "error_rate_at_confidence_0_9": 0.021052631578947368,
          "ordinal_expected_level_mae": null
        }
      },
      "macro_task_accuracy": 0.9407031795465964,
      "macro_task_balanced_accuracy": 0.9191604101516703
    },
    "recalibrated": {
      "questions": 14821,
      "accuracy": 0.9313811483705553,
      "mean_class_recall": 0.8460223158089928,
      "nll": 0.17961943925788992,
      "brier": 0.10111838724890135,
      "ece15": 0.017864195303344165,
      "temperature": 1.334968103387446,
      "coverage_at_confidence_0_9": 0.7859118817893529,
      "error_rate_at_confidence_0_9": 0.010645604395604396,
      "ordinal_expected_level_mae": 0.001742299523022841,
      "tasks": {
        "agentdojo_tool_outputs:agentdojo_boundary_action": {
          "questions": 325,
          "accuracy": 0.96,
          "mean_class_recall": 0.9216916119306557,
          "nll": 0.10308600797920058,
          "brier": 0.05761479429147346,
          "ece15": 0.0294057816164861,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.963076923076923,
          "error_rate_at_confidence_0_9": 0.012779552715654952,
          "ordinal_expected_level_mae": null
        },
        "agentdojo_tool_outputs:agentdojo_embedded_injection": {
          "questions": 328,
          "accuracy": 0.9695121951219512,
          "mean_class_recall": 0.9372206852521814,
          "nll": 0.12289366290953901,
          "brier": 0.057082591730227304,
          "ece15": 0.029897091535857363,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9939024390243902,
          "error_rate_at_confidence_0_9": 0.027607361963190184,
          "ordinal_expected_level_mae": null
        },
        "byzfl_mnist:fl_attack_generated_update": {
          "questions": 2400,
          "accuracy": 0.9404166666666667,
          "mean_class_recall": 0.8877604166666666,
          "nll": 0.16174786934019353,
          "brier": 0.09375828073422167,
          "ece15": 0.03645209485430555,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.92875,
          "error_rate_at_confidence_0_9": 0.03095558546433378,
          "ordinal_expected_level_mae": null
        },
        "ciciot2023:ciciot_attack": {
          "questions": 5000,
          "accuracy": 0.9292,
          "mean_class_recall": 0.8968,
          "nll": 0.19125358889505403,
          "brier": 0.11141554439383701,
          "ece15": 0.03505488361604012,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.6616,
          "error_rate_at_confidence_0_9": 0.010278113663845224,
          "ordinal_expected_level_mae": null
        },
        "injecagent:injecagent_requested_operation_v13": {
          "questions": 124,
          "accuracy": 0.9919354838709677,
          "mean_class_recall": 0.9928571428571429,
          "nll": 0.019190864186870816,
          "brier": 0.01397734899726431,
          "ece15": 0.011351501004051424,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9838709677419355,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "lanl_auth_policy:lanl_known_logon_action_v16": {
          "questions": 235,
          "accuracy": 0.9957446808510638,
          "mean_class_recall": 0.9949494949494949,
          "nll": 0.01285197836061796,
          "brier": 0.007512756902037326,
          "ece15": 0.004004760561973723,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 1.0,
          "error_rate_at_confidence_0_9": 0.00425531914893617,
          "ordinal_expected_level_mae": null
        },
        "lanl_auth_policy:lanl_observable_policy": {
          "questions": 235,
          "accuracy": 0.9957446808510638,
          "mean_class_recall": 0.9905660377358491,
          "nll": 0.01368047084450566,
          "brier": 0.007839165259612221,
          "ece15": 0.004073337312958388,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 1.0,
          "error_rate_at_confidence_0_9": 0.00425531914893617,
          "ordinal_expected_level_mae": null
        },
        "ton_iot_network:ton_attack": {
          "questions": 2000,
          "accuracy": 0.9985,
          "mean_class_recall": 0.9976666666666667,
          "nll": 0.010650627249048795,
          "brier": 0.003323170074906094,
          "ece15": 0.002118687755931114,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9975,
          "error_rate_at_confidence_0_9": 0.0015037593984962407,
          "ordinal_expected_level_mae": null
        },
        "twins_streamlet:twins_evidence_level": {
          "questions": 641,
          "accuracy": 0.9984399375975039,
          "mean_class_recall": 0.9985507246376812,
          "nll": 0.0028802196260483376,
          "brier": 0.001935795675809169,
          "ece15": 0.0017273023271884604,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9968798751950078,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": 0.001742299523022841
        },
        "twins_streamlet:twins_policy_action": {
          "questions": 641,
          "accuracy": 1.0,
          "mean_class_recall": 1.0,
          "nll": 0.0012409177190170738,
          "brier": 0.00021235107519573934,
          "ece15": 0.0011788827327451372,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9984399375975039,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "twins_streamlet:twins_quorum": {
          "questions": 641,
          "accuracy": 0.9968798751950078,
          "mean_class_recall": 0.9961240310077519,
          "nll": 0.013816362060687534,
          "brier": 0.007467944863319216,
          "ece15": 0.004893228894077442,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9921996879875195,
          "error_rate_at_confidence_0_9": 0.0031446540880503146,
          "ordinal_expected_level_mae": null
        },
        "twins_streamlet:twins_visible_conflict": {
          "questions": 641,
          "accuracy": 0.9984399375975039,
          "mean_class_recall": 0.9969512195121951,
          "nll": 0.0034185049808860444,
          "brier": 0.0023816170036107054,
          "ece15": 0.0015601193471088582,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.9984399375975039,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "veremi_nextgen:veremi_message_manipulation": {
          "questions": 805,
          "accuracy": 0.7031055900621118,
          "mean_class_recall": 0.7239978843175034,
          "nll": 0.503140821446039,
          "brier": 0.35269849107227197,
          "ece15": 0.030978365187072362,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.21366459627329193,
          "error_rate_at_confidence_0_9": 0.0,
          "ordinal_expected_level_mae": null
        },
        "veremi_nextgen:veremi_supported_attack_type_v16": {
          "questions": 805,
          "accuracy": 0.6919254658385093,
          "mean_class_recall": 0.5331098265895953,
          "nll": 0.9878648410254747,
          "brier": 0.46649698606417656,
          "ece15": 0.09086134384209663,
          "temperature": 1.334968103387446,
          "coverage_at_confidence_0_9": 0.19627329192546583,
          "error_rate_at_confidence_0_9": 0.006329113924050633,
          "ordinal_expected_level_mae": null
        }
      },
      "macro_task_accuracy": 0.9407031795465964,
      "macro_task_balanced_accuracy": 0.9191604101516703
    }
  }
}