{
 "variant": "v4_vocab32k-fp16",
 "backend": "torch",
 "size_bytes": 307846551,
 "validation_all": {
  "unweighted": {
   "rows": 4914,
   "weighted": false,
   "complexity": {
    "rows": 4850,
    "mae": 0.07579664213927752,
    "spearman": 0.7329635072519433,
    "qwk_rounded": 0.7519040607655082,
    "confusion_true_by_pred": [
     [
      145.0,
      157.0,
      6.0,
      0.0,
      0.0
     ],
     [
      39.0,
      2574.0,
      263.0,
      2.0,
      0.0
     ],
     [
      4.0,
      406.0,
      880.0,
      55.0,
      0.0
     ],
     [
      1.0,
      10.0,
      76.0,
      224.0,
      0.0
     ],
     [
      0.0,
      0.0,
      1.0,
      3.0,
      4.0
     ]
    ],
    "support": {
     "trivial": 308,
     "low": 2878,
     "medium": 1345,
     "high": 311,
     "expert": 8
    }
   },
   "task": {
    "accuracy": 0.8398453398453398,
    "macro_f1": 0.7496592483362028,
    "macro_f1_present": 0.7496592483362028,
    "per_class": {
     "information_seeking": {
      "f1": 0.8687520154788778,
      "support": 1528
     },
     "reasoning": {
      "f1": 0.5802469135802469,
      "support": 92
     },
     "planning": {
      "f1": 0.7132867132867133,
      "support": 81
     },
     "editing": {
      "f1": 0.8702290076335878,
      "support": 193
     },
     "summarization": {
      "f1": 0.8605577689243028,
      "support": 128
     },
     "extraction": {
      "f1": 0.7982832618025751,
      "support": 229
     },
     "classification": {
      "f1": 0.8901098901098901,
      "support": 89
     },
     "translation": {
      "f1": 0.9103448275862069,
      "support": 73
     },
     "coding_debugging": {
      "f1": 0.8940397350993378,
      "support": 453
     },
     "math": {
      "f1": 0.8767676767676768,
      "support": 236
     },
     "data_analysis": {
      "f1": 0.7058823529411765,
      "support": 22
     },
     "writing": {
      "f1": 0.9045936395759717,
      "support": 991
     },
     "role_playing": {
      "f1": 0.8771929824561403,
      "support": 83
     },
     "advice_seeking": {
      "f1": 0.7138263665594855,
      "support": 157
     },
     "brainstorming": {
      "f1": 0.6024096385542169,
      "support": 134
     },
     "completion": {
      "f1": 0.764378478664193,
      "support": 267
     },
     "action_execution": {
      "f1": 0.5263157894736842,
      "support": 11
     },
     "chat": {
      "f1": 0.5786163522012578,
      "support": 91
     },
     "other": {
      "f1": 0.3076923076923077,
      "support": 56
     }
    }
   },
   "needs_tools": {
    "f1": 0.6178489702517163,
    "auroc": 0.9481470840480274,
    "positives": 250
   }
  },
  "weighted": {
   "rows": 4914,
   "weighted": true,
   "complexity": {
    "rows": 4850,
    "mae": 0.0734992234917565,
    "spearman": 0.6931422117949425,
    "qwk_rounded": 0.7000283091495797,
    "confusion_true_by_pred": [
     [
      138.9,
      154.5,
      5.0,
      0.0,
      0.0
     ],
     [
      41.0,
      2767.2,
      283.8,
      2.2,
      0.0
     ],
     [
      4.1,
      408.0,
      884.3,
      54.9,
      0.0
     ],
     [
      0.4,
      4.1,
      32.3,
      94.6,
      0.0
     ],
     [
      0.0,
      0.0,
      0.1,
      0.4,
      0.6
     ]
    ],
    "support": {
     "trivial": 308,
     "low": 2878,
     "medium": 1345,
     "high": 311,
     "expert": 8
    }
   },
   "task": {
    "accuracy": 0.8417469271956327,
    "macro_f1": 0.7361416383741431,
    "macro_f1_present": 0.7361416383741431,
    "per_class": {
     "information_seeking": {
      "f1": 0.8743961299148704,
      "support": 1528
     },
     "reasoning": {
      "f1": 0.5876168321012015,
      "support": 92
     },
     "planning": {
      "f1": 0.6923419444052038,
      "support": 81
     },
     "editing": {
      "f1": 0.8750917385680073,
      "support": 193
     },
     "summarization": {
      "f1": 0.8617685118085122,
      "support": 128
     },
     "extraction": {
      "f1": 0.8024884010513994,
      "support": 229
     },
     "classification": {
      "f1": 0.8889433148815616,
      "support": 89
     },
     "translation": {
      "f1": 0.9102897416990978,
      "support": 73
     },
     "coding_debugging": {
      "f1": 0.8951849640199118,
      "support": 453
     },
     "math": {
      "f1": 0.8720808285568271,
      "support": 236
     },
     "data_analysis": {
      "f1": 0.6036172496413325,
      "support": 22
     },
     "writing": {
      "f1": 0.8998060042441275,
      "support": 991
     },
     "role_playing": {
      "f1": 0.856782491891807,
      "support": 83
     },
     "advice_seeking": {
      "f1": 0.7267918392567481,
      "support": 157
     },
     "brainstorming": {
      "f1": 0.6040013806205294,
      "support": 134
     },
     "completion": {
      "f1": 0.7735029017960995,
      "support": 267
     },
     "action_execution": {
      "f1": 0.4050722792506393,
      "support": 11
     },
     "chat": {
      "f1": 0.5820792184544086,
      "support": 91
     },
     "other": {
      "f1": 0.27483535694643446,
      "support": 56
     }
    }
   },
   "needs_tools": {
    "f1": 0.6202334654791641,
    "auroc": 0.9483244764486874,
    "positives": 250
   }
  }
 },
 "validation_four_sources": {
  "unweighted": {
   "rows": 3130,
   "weighted": false,
   "complexity": {
    "rows": 3101,
    "mae": 0.0732659558825476,
    "spearman": 0.7434607091466564,
    "qwk_rounded": 0.7590494324005435,
    "confusion_true_by_pred": [
     [
      100.0,
      97.0,
      4.0,
      0.0,
      0.0
     ],
     [
      25.0,
      1653.0,
      160.0,
      0.0,
      0.0
     ],
     [
      3.0,
      251.0,
      580.0,
      38.0,
      0.0
     ],
     [
      1.0,
      4.0,
      46.0,
      138.0,
      0.0
     ],
     [
      0.0,
      0.0,
      0.0,
      0.0,
      1.0
     ]
    ],
    "support": {
     "trivial": 201,
     "low": 1838,
     "medium": 872,
     "high": 189,
     "expert": 1
    }
   },
   "task": {
    "accuracy": 0.8488817891373802,
    "macro_f1": 0.7513423725525952,
    "macro_f1_present": 0.7513423725525952,
    "per_class": {
     "information_seeking": {
      "f1": 0.8823257960313798,
      "support": 1068
     },
     "reasoning": {
      "f1": 0.5977011494252874,
      "support": 48
     },
     "planning": {
      "f1": 0.651685393258427,
      "support": 53
     },
     "editing": {
      "f1": 0.8817204301075269,
      "support": 139
     },
     "summarization": {
      "f1": 0.8631578947368421,
      "support": 98
     },
     "extraction": {
      "f1": 0.819672131147541,
      "support": 180
     },
     "classification": {
      "f1": 0.875,
      "support": 47
     },
     "translation": {
      "f1": 0.9166666666666666,
      "support": 61
     },
     "coding_debugging": {
      "f1": 0.9020408163265307,
      "support": 246
     },
     "math": {
      "f1": 0.7415730337078652,
      "support": 41
     },
     "data_analysis": {
      "f1": 0.6666666666666666,
      "support": 5
     },
     "writing": {
      "f1": 0.9049964813511612,
      "support": 705
     },
     "role_playing": {
      "f1": 0.9152542372881356,
      "support": 30
     },
     "advice_seeking": {
      "f1": 0.6901408450704225,
      "support": 75
     },
     "brainstorming": {
      "f1": 0.5660377358490566,
      "support": 83
     },
     "completion": {
      "f1": 0.7818181818181819,
      "support": 162
     },
     "action_execution": {
      "f1": 0.625,
      "support": 8
     },
     "chat": {
      "f1": 0.6190476190476191,
      "support": 47
     },
     "other": {
      "f1": 0.375,
      "support": 34
     }
    }
   },
   "needs_tools": {
    "f1": 0.5808823529411765,
    "auroc": 0.960451698314529,
    "positives": 153
   }
  },
  "weighted": {
   "rows": 3130,
   "weighted": true,
   "complexity": {
    "rows": 3101,
    "mae": 0.07136375761254943,
    "spearman": 0.7050611798721679,
    "qwk_rounded": 0.7112237182045522,
    "confusion_true_by_pred": [
     [
      95.6,
      96.6,
      3.5,
      0.0,
      0.0
     ],
     [
      26.1,
      1790.5,
      174.8,
      0.0,
      0.0
     ],
     [
      3.1,
      254.4,
      589.0,
      37.6,
      0.0
     ],
     [
      0.4,
      1.6,
      19.7,
      58.1,
      0.0
     ],
     [
      0.0,
      0.0,
      0.0,
      0.0,
      0.1
     ]
    ],
    "support": {
     "trivial": 201,
     "low": 1838,
     "medium": 872,
     "high": 189,
     "expert": 1
    }
   },
   "task": {
    "accuracy": 0.8510913634686621,
    "macro_f1": 0.7336388186360535,
    "macro_f1_present": 0.7336388186360535,
    "per_class": {
     "information_seeking": {
      "f1": 0.8863005064157066,
      "support": 1068
     },
     "reasoning": {
      "f1": 0.6007655619943084,
      "support": 48
     },
     "planning": {
      "f1": 0.6322924335863169,
      "support": 53
     },
     "editing": {
      "f1": 0.8872579498056196,
      "support": 139
     },
     "summarization": {
      "f1": 0.8655264865753218,
      "support": 98
     },
     "extraction": {
      "f1": 0.8217981441939354,
      "support": 180
     },
     "classification": {
      "f1": 0.8744675021779321,
      "support": 47
     },
     "translation": {
      "f1": 0.9165540456122309,
      "support": 61
     },
     "coding_debugging": {
      "f1": 0.9014211381493499,
      "support": 246
     },
     "math": {
      "f1": 0.7206863251222526,
      "support": 41
     },
     "data_analysis": {
      "f1": 0.5512676749948888,
      "support": 5
     },
     "writing": {
      "f1": 0.9022412453550238,
      "support": 705
     },
     "role_playing": {
      "f1": 0.9018366831956603,
      "support": 30
     },
     "advice_seeking": {
      "f1": 0.7058665577471023,
      "support": 75
     },
     "brainstorming": {
      "f1": 0.5672344732772423,
      "support": 83
     },
     "completion": {
      "f1": 0.7921253431420334,
      "support": 162
     },
     "action_execution": {
      "f1": 0.46184553221044683,
      "support": 8
     },
     "chat": {
      "f1": 0.6134643357166086,
      "support": 47
     },
     "other": {
      "f1": 0.3361856148130364,
      "support": 34
     }
    }
   },
   "needs_tools": {
    "f1": 0.5784521352010515,
    "auroc": 0.9630510085880657,
    "positives": 153
   }
  }
 },
 "agreement_with_reference": {
  "max_abs_logit_difference": 16.74053192138672,
  "mean_abs_logit_difference": 0.7664107084274292,
  "same_task": 0.9171754171754172,
  "same_needs_tools": 0.9816849816849816,
  "same_complexity_tier": 0.9092389092389093,
  "same_policy_tier": 0.914936914936915
 },
 "agreement_with_parent": {
  "max_abs_logit_difference": 0.9007568359375,
  "mean_abs_logit_difference": 0.0033105933107435703,
  "same_task": 0.9995929995929996,
  "same_needs_tools": 1.0,
  "same_complexity_tier": 0.9993894993894994,
  "same_policy_tier": 0.9993894993894994
 },
 "latency_ms_batch1": {
  "p50": 10.002875002101064,
  "p95": 38.70008300145855,
  "mean": 14.027030840043153,
  "prompts": 200,
  "device": "mps"
 }
}