eval/validation.json

{
 "model": "Wolf Defender Jev",
 "split": "internal validation set (in distribution)",
 "bal_acc": 0.9584183592204583,
 "macro_f1": 0.9564921501262407,
 "agent_bal_acc": 0.9952958803302826,
 "hn_fpr": 0.008452291510142707,
 "hn_n": 5324,
 "multiling_acc": 0.9366352941176471,
 "acc_by_lang": {
  "ar": 0.927,
  "de": 0.9533,
  "es": 0.9489,
  "fr": 0.9556,
  "hi": 0.9165,
  "it": 0.9498,
  "ja": 0.9364,
  "ko": 0.769,
  "nl": 0.9312,
  "pl": 0.98,
  "pt": 0.9739,
  "ru": 0.9709,
  "sv": 0.9769,
  "th": 0.8395,
  "tr": 0.9652,
  "vi": 0.9812,
  "zh": 0.9475
 },
 "ece": 0.02290203936867429,
 "per_concept": {
  "action.alignment": {
   "n": 2460,
   "acc": 1.0,
   "bal_acc": 1.0,
   "macro_f1": 1.0
  },
  "action.consent_required": {
   "n": 1902,
   "acc": 1.0,
   "bal_acc": 1.0,
   "macro_f1": 1.0
  },
  "action.intent_source": {
   "n": 1866,
   "acc": 1.0,
   "bal_acc": 1.0,
   "macro_f1": 1.0
  },
  "action.reversibility": {
   "n": 1056,
   "acc": 1.0,
   "bal_acc": 1.0,
   "macro_f1": 1.0
  },
  "attack.family": {
   "n": 1081,
   "acc": 0.9389,
   "bal_acc": 0.8841,
   "macro_f1": 0.8584
  },
  "attack.vector": {
   "n": 2040,
   "acc": 0.977,
   "bal_acc": 0.9561,
   "macro_f1": 0.9673
  },
  "content.role": {
   "n": 2093,
   "acc": 0.9947,
   "bal_acc": 0.9958,
   "macro_f1": 0.9963
  },
  "exfiltration.channel": {
   "n": 228,
   "acc": 1.0,
   "bal_acc": 1.0,
   "macro_f1": 1.0
  },
  "exfiltration.presence": {
   "n": 699,
   "acc": 0.9814,
   "bal_acc": 0.9794,
   "macro_f1": 0.9716
  },
  "harmful.request": {
   "n": 6490,
   "acc": 0.8305,
   "bal_acc": 0.8363,
   "macro_f1": 0.8304
  },
  "injection.indirect_presence": {
   "n": 2205,
   "acc": 0.9946,
   "bal_acc": 0.9955,
   "macro_f1": 0.9954
  },
  "injection.presence": {
   "n": 6063,
   "acc": 0.9779,
   "bal_acc": 0.9724,
   "macro_f1": 0.9768
  },
  "instruction.handling": {
   "n": 2109,
   "acc": 0.9227,
   "bal_acc": 0.5,
   "macro_f1": 0.4799
  },
  "intent.attack_vs_discussion": {
   "n": 734,
   "acc": 1.0,
   "bal_acc": 1.0,
   "macro_f1": 1.0
  },
  "jailbreak.presence": {
   "n": 980,
   "acc": 0.9918,
   "bal_acc": 0.9974,
   "macro_f1": 0.9882
  },
  "manipulation.any": {
   "n": 2117,
   "acc": 0.9953,
   "bal_acc": 0.9846,
   "macro_f1": 0.9893
  },
  "policy.compliance": {
   "n": 1897,
   "acc": 0.9937,
   "bal_acc": 0.9947,
   "macro_f1": 0.9864
  },
  "policy.decision": {
   "n": 301,
   "acc": 1.0,
   "bal_acc": 1.0,
   "macro_f1": 1.0
  },
  "prompt_leak.attempt": {
   "n": 1786,
   "acc": 1.0,
   "bal_acc": 1.0,
   "macro_f1": 1.0
  },
  "risk.severity": {
   "n": 2377,
   "acc": 0.8439,
   "bal_acc": 0.8077,
   "macro_f1": 0.8084
  },
  "secret.kind": {
   "n": 153,
   "acc": 1.0,
   "bal_acc": 1.0,
   "macro_f1": 1.0
  },
  "secret.presence": {
   "n": 912,
   "acc": 0.989,
   "bal_acc": 0.9831,
   "macro_f1": 0.987
  },
  "sensitive.category": {
   "n": 2046,
   "acc": 0.9878,
   "bal_acc": 0.9904,
   "macro_f1": 0.9904
  },
  "sensitive.presence": {
   "n": 2721,
   "acc": 0.9996,
   "bal_acc": 0.9997,
   "macro_f1": 0.9996
  },
  "task_drift": {
   "n": 1056,
   "acc": 1.0,
   "bal_acc": 1.0,
   "macro_f1": 1.0
  },
  "tool_metadata.poisoning": {
   "n": 167,
   "acc": 1.0,
   "bal_acc": 1.0,
   "macro_f1": 1.0
  },
  "trust_boundary.crossing": {
   "n": 1063,
   "acc": 1.0,
   "bal_acc": 1.0,
   "macro_f1": 1.0
  }
 },
 "score": 0.9682238015893038,
 "score_components": {
  "bal_acc": 0.9584,
  "macro_f1": 0.9565,
  "agent_bal_acc": 0.9953,
  "hn_tnr": 0.9915,
  "multiling_acc": 0.9366,
  "calib": 0.9771
 },
 "acc": 0.9591992099090573,
 "n": 48602
}