{
 "variant": "v7_small_l6-fp16",
 "backend": "torch",
 "size_bytes": 84514110,
 "validation_all": {
  "unweighted": {
   "rows": 4914,
   "weighted": false,
   "complexity": {
    "rows": 4850,
    "mae": 0.09123653288428316,
    "spearman": 0.6593453452413092,
    "qwk_rounded": 0.6490167052545179,
    "confusion_true_by_pred": [
     [
      92.0,
      198.0,
      18.0,
      0.0,
      0.0
     ],
     [
      48.0,
      2451.0,
      377.0,
      2.0,
      0.0
     ],
     [
      5.0,
      463.0,
      842.0,
      35.0,
      0.0
     ],
     [
      0.0,
      22.0,
      125.0,
      164.0,
      0.0
     ],
     [
      0.0,
      1.0,
      4.0,
      3.0,
      0.0
     ]
    ],
    "support": {
     "trivial": 308,
     "low": 2878,
     "medium": 1345,
     "high": 311,
     "expert": 8
    }
   },
   "task": {
    "accuracy": 0.7816442816442817,
    "macro_f1": 0.6623087453835308,
    "macro_f1_present": 0.6623087453835308,
    "per_class": {
     "information_seeking": {
      "f1": 0.8244226510597912,
      "support": 1528
     },
     "reasoning": {
      "f1": 0.5149700598802395,
      "support": 92
     },
     "planning": {
      "f1": 0.6474820143884892,
      "support": 81
     },
     "editing": {
      "f1": 0.7917737789203085,
      "support": 193
     },
     "summarization": {
      "f1": 0.7649402390438247,
      "support": 128
     },
     "extraction": {
      "f1": 0.7352297592997812,
      "support": 229
     },
     "classification": {
      "f1": 0.7932960893854749,
      "support": 89
     },
     "translation": {
      "f1": 0.9041095890410958,
      "support": 73
     },
     "coding_debugging": {
      "f1": 0.8478260869565217,
      "support": 453
     },
     "math": {
      "f1": 0.8530020703933747,
      "support": 236
     },
     "data_analysis": {
      "f1": 0.5882352941176471,
      "support": 22
     },
     "writing": {
      "f1": 0.8550148957298908,
      "support": 991
     },
     "role_playing": {
      "f1": 0.7544910179640718,
      "support": 83
     },
     "advice_seeking": {
      "f1": 0.6099290780141844,
      "support": 157
     },
     "brainstorming": {
      "f1": 0.5281385281385281,
      "support": 134
     },
     "completion": {
      "f1": 0.6311787072243346,
      "support": 267
     },
     "action_execution": {
      "f1": 0.25,
      "support": 11
     },
     "chat": {
      "f1": 0.41025641025641024,
      "support": 91
     },
     "other": {
      "f1": 0.27956989247311825,
      "support": 56
     }
    }
   },
   "needs_tools": {
    "f1": 0.4841075794621027,
    "auroc": 0.9042765866209262,
    "positives": 250
   }
  },
  "weighted": {
   "rows": 4914,
   "weighted": true,
   "complexity": {
    "rows": 4850,
    "mae": 0.08725421027348824,
    "spearman": 0.6160875172388919,
    "qwk_rounded": 0.596045096988439,
    "confusion_true_by_pred": [
     [
      86.2,
      195.8,
      16.3,
      0.0,
      0.0
     ],
     [
      51.3,
      2634.9,
      406.4,
      1.7,
      0.0
     ],
     [
      5.2,
      467.1,
      845.1,
      34.0,
      0.0
     ],
     [
      0.0,
      9.3,
      52.7,
      69.4,
      0.0
     ],
     [
      0.0,
      0.1,
      0.5,
      0.4,
      0.0
     ]
    ],
    "support": {
     "trivial": 308,
     "low": 2878,
     "medium": 1345,
     "high": 311,
     "expert": 8
    }
   },
   "task": {
    "accuracy": 0.7848718360585144,
    "macro_f1": 0.6535855109314881,
    "macro_f1_present": 0.6535855109314881,
    "per_class": {
     "information_seeking": {
      "f1": 0.832598914469181,
      "support": 1528
     },
     "reasoning": {
      "f1": 0.5297883989506079,
      "support": 92
     },
     "planning": {
      "f1": 0.6323352636377239,
      "support": 81
     },
     "editing": {
      "f1": 0.7972908310414097,
      "support": 193
     },
     "summarization": {
      "f1": 0.771281034042985,
      "support": 128
     },
     "extraction": {
      "f1": 0.739535897050014,
      "support": 229
     },
     "classification": {
      "f1": 0.7906490364919273,
      "support": 89
     },
     "translation": {
      "f1": 0.9039649545580544,
      "support": 73
     },
     "coding_debugging": {
      "f1": 0.850302929061513,
      "support": 453
     },
     "math": {
      "f1": 0.8457726621104891,
      "support": 236
     },
     "data_analysis": {
      "f1": 0.5085499923284106,
      "support": 22
     },
     "writing": {
      "f1": 0.8495558798328425,
      "support": 991
     },
     "role_playing": {
      "f1": 0.722487088007609,
      "support": 83
     },
     "advice_seeking": {
      "f1": 0.6228240602260838,
      "support": 157
     },
     "brainstorming": {
      "f1": 0.5321998740461651,
      "support": 134
     },
     "completion": {
      "f1": 0.6401729708095073,
      "support": 267
     },
     "action_execution": {
      "f1": 0.17852667118303162,
      "support": 11
     },
     "chat": {
      "f1": 0.41431580139190316,
      "support": 91
     },
     "other": {
      "f1": 0.2559724484588159,
      "support": 56
     }
    }
   },
   "needs_tools": {
    "f1": 0.4833044372624333,
    "auroc": 0.9023762363440311,
    "positives": 250
   }
  }
 },
 "validation_four_sources": {
  "unweighted": {
   "rows": 3130,
   "weighted": false,
   "complexity": {
    "rows": 3101,
    "mae": 0.08777755751143883,
    "spearman": 0.6784958198543671,
    "qwk_rounded": 0.6711515464869717,
    "confusion_true_by_pred": [
     [
      63.0,
      129.0,
      9.0,
      0.0,
      0.0
     ],
     [
      34.0,
      1567.0,
      235.0,
      2.0,
      0.0
     ],
     [
      2.0,
      275.0,
      566.0,
      29.0,
      0.0
     ],
     [
      0.0,
      8.0,
      81.0,
      100.0,
      0.0
     ],
     [
      0.0,
      0.0,
      0.0,
      1.0,
      0.0
     ]
    ],
    "support": {
     "trivial": 201,
     "low": 1838,
     "medium": 872,
     "high": 189,
     "expert": 1
    }
   },
   "task": {
    "accuracy": 0.7955271565495208,
    "macro_f1": 0.6418206886744169,
    "macro_f1_present": 0.6418206886744169,
    "per_class": {
     "information_seeking": {
      "f1": 0.8394724874943156,
      "support": 1068
     },
     "reasoning": {
      "f1": 0.4943820224719101,
      "support": 48
     },
     "planning": {
      "f1": 0.6067415730337079,
      "support": 53
     },
     "editing": {
      "f1": 0.8,
      "support": 139
     },
     "summarization": {
      "f1": 0.78125,
      "support": 98
     },
     "extraction": {
      "f1": 0.7688022284122563,
      "support": 180
     },
     "classification": {
      "f1": 0.7578947368421053,
      "support": 47
     },
     "translation": {
      "f1": 0.907563025210084,
      "support": 61
     },
     "coding_debugging": {
      "f1": 0.8577235772357723,
      "support": 246
     },
     "math": {
      "f1": 0.7058823529411765,
      "support": 41
     },
     "data_analysis": {
      "f1": 0.18181818181818182,
      "support": 5
     },
     "writing": {
      "f1": 0.8631284916201117,
      "support": 705
     },
     "role_playing": {
      "f1": 0.7419354838709677,
      "support": 30
     },
     "advice_seeking": {
      "f1": 0.576,
      "support": 75
     },
     "brainstorming": {
      "f1": 0.5285714285714286,
      "support": 83
     },
     "completion": {
      "f1": 0.6923076923076923,
      "support": 162
     },
     "action_execution": {
      "f1": 0.3076923076923077,
      "support": 8
     },
     "chat": {
      "f1": 0.4444444444444444,
      "support": 47
     },
     "other": {
      "f1": 0.3389830508474576,
      "support": 34
     }
    }
   },
   "needs_tools": {
    "f1": 0.43824701195219123,
    "auroc": 0.9143597647322281,
    "positives": 153
   }
  },
  "weighted": {
   "rows": 3130,
   "weighted": true,
   "complexity": {
    "rows": 3101,
    "mae": 0.08461350760287671,
    "spearman": 0.636775261257106,
    "qwk_rounded": 0.61930401674141,
    "confusion_true_by_pred": [
     [
      59.1,
      128.4,
      8.1,
      0.0,
      0.0
     ],
     [
      36.9,
      1696.7,
      256.0,
      1.7,
      0.0
     ],
     [
      2.1,
      279.5,
      574.5,
      28.0,
      0.0
     ],
     [
      0.0,
      3.3,
      34.1,
      42.3,
      0.0
     ],
     [
      0.0,
      0.0,
      0.0,
      0.1,
      0.0
     ]
    ],
    "support": {
     "trivial": 201,
     "low": 1838,
     "medium": 872,
     "high": 189,
     "expert": 1
    }
   },
   "task": {
    "accuracy": 0.8006841010532753,
    "macro_f1": 0.6323733904733824,
    "macro_f1_present": 0.6323733904733824,
    "per_class": {
     "information_seeking": {
      "f1": 0.8480068752176662,
      "support": 1068
     },
     "reasoning": {
      "f1": 0.5097422442799173,
      "support": 48
     },
     "planning": {
      "f1": 0.5885036465886375,
      "support": 53
     },
     "editing": {
      "f1": 0.8047164777623818,
      "support": 139
     },
     "summarization": {
      "f1": 0.7884833764231882,
      "support": 98
     },
     "extraction": {
      "f1": 0.7743659734172523,
      "support": 180
     },
     "classification": {
      "f1": 0.755260299322683,
      "support": 47
     },
     "translation": {
      "f1": 0.9069849377410021,
      "support": 61
     },
     "coding_debugging": {
      "f1": 0.8598117983995479,
      "support": 246
     },
     "math": {
      "f1": 0.7029567273827195,
      "support": 41
     },
     "data_analysis": {
      "f1": 0.13601164014439981,
      "support": 5
     },
     "writing": {
      "f1": 0.8599838608390206,
      "support": 705
     },
     "role_playing": {
      "f1": 0.699100469772375,
      "support": 30
     },
     "advice_seeking": {
      "f1": 0.586485551349588,
      "support": 75
     },
     "brainstorming": {
      "f1": 0.5322990203969951,
      "support": 83
     },
     "completion": {
      "f1": 0.7009731628974076,
      "support": 162
     },
     "action_execution": {
      "f1": 0.20741165161792083,
      "support": 8
     },
     "chat": {
      "f1": 0.44317185671144815,
      "support": 47
     },
     "other": {
      "f1": 0.3108248487301116,
      "support": 34
     }
    }
   },
   "needs_tools": {
    "f1": 0.4231775842460709,
    "auroc": 0.9139293957758868,
    "positives": 153
   }
  }
 },
 "agreement_with_reference": {
  "max_abs_logit_difference": 19.687698364257812,
  "mean_abs_logit_difference": 2.079862594604492,
  "same_task": 0.8133903133903134,
  "same_needs_tools": 0.967032967032967,
  "same_complexity_tier": 0.7964997964997965,
  "same_policy_tier": 0.8085063085063086
 },
 "agreement_with_parent": {
  "max_abs_logit_difference": 0.0420689582824707,
  "mean_abs_logit_difference": 0.004016549792140722,
  "same_task": 0.9991859991859992,
  "same_needs_tools": 0.9997964997964998,
  "same_complexity_tier": 0.9971509971509972,
  "same_policy_tier": 0.9991859991859992
 },
 "latency_ms_batch1": {
  "p50": 3.0535839978256263,
  "p95": 8.006165997358039,
  "mean": 4.419368929666234,
  "prompts": 200,
  "device": "mps"
 }
}