{
 "variant": "v7_small_l6-mlx-8bit",
 "backend": "mlx",
 "size_bytes": 62125397,
 "validation_all": {
  "unweighted": {
   "rows": 4914,
   "weighted": false,
   "complexity": {
    "rows": 4850,
    "mae": 0.09117008546431041,
    "spearman": 0.6591213539927813,
    "qwk_rounded": 0.6471972183137378,
    "confusion_true_by_pred": [
     [
      97.0,
      194.0,
      17.0,
      0.0,
      0.0
     ],
     [
      58.0,
      2464.0,
      354.0,
      2.0,
      0.0
     ],
     [
      5.0,
      487.0,
      820.0,
      33.0,
      0.0
     ],
     [
      0.0,
      23.0,
      125.0,
      163.0,
      0.0
     ],
     [
      0.0,
      1.0,
      5.0,
      2.0,
      0.0
     ]
    ],
    "support": {
     "trivial": 308,
     "low": 2878,
     "medium": 1345,
     "high": 311,
     "expert": 8
    }
   },
   "task": {
    "accuracy": 0.7822547822547823,
    "macro_f1": 0.6628007985114895,
    "macro_f1_present": 0.6628007985114895,
    "per_class": {
     "information_seeking": {
      "f1": 0.8253868013893274,
      "support": 1528
     },
     "reasoning": {
      "f1": 0.5058823529411764,
      "support": 92
     },
     "planning": {
      "f1": 0.6666666666666666,
      "support": 81
     },
     "editing": {
      "f1": 0.7938144329896907,
      "support": 193
     },
     "summarization": {
      "f1": 0.768,
      "support": 128
     },
     "extraction": {
      "f1": 0.7363834422657952,
      "support": 229
     },
     "classification": {
      "f1": 0.7845303867403315,
      "support": 89
     },
     "translation": {
      "f1": 0.8979591836734694,
      "support": 73
     },
     "coding_debugging": {
      "f1": 0.8474945533769063,
      "support": 453
     },
     "math": {
      "f1": 0.8565488565488566,
      "support": 236
     },
     "data_analysis": {
      "f1": 0.6,
      "support": 22
     },
     "writing": {
      "f1": 0.8551518168242906,
      "support": 991
     },
     "role_playing": {
      "f1": 0.7544910179640718,
      "support": 83
     },
     "advice_seeking": {
      "f1": 0.604982206405694,
      "support": 157
     },
     "brainstorming": {
      "f1": 0.5327510917030568,
      "support": 134
     },
     "completion": {
      "f1": 0.630188679245283,
      "support": 267
     },
     "action_execution": {
      "f1": 0.25,
      "support": 11
     },
     "chat": {
      "f1": 0.41025641025641024,
      "support": 91
     },
     "other": {
      "f1": 0.2727272727272727,
      "support": 56
     }
    }
   },
   "needs_tools": {
    "f1": 0.4963855421686747,
    "auroc": 0.9044644082332761,
    "positives": 250
   }
  },
  "weighted": {
   "rows": 4914,
   "weighted": true,
   "complexity": {
    "rows": 4850,
    "mae": 0.08705916285165423,
    "spearman": 0.6158774632764406,
    "qwk_rounded": 0.594693223073677,
    "confusion_true_by_pred": [
     [
      91.1,
      192.0,
      15.2,
      0.0,
      0.0
     ],
     [
      61.9,
      2649.2,
      381.4,
      1.7,
      0.0
     ],
     [
      5.2,
      491.5,
      822.7,
      31.9,
      0.0
     ],
     [
      0.0,
      9.7,
      52.7,
      69.0,
      0.0
     ],
     [
      0.0,
      0.1,
      0.7,
      0.3,
      0.0
     ]
    ],
    "support": {
     "trivial": 308,
     "low": 2878,
     "medium": 1345,
     "high": 311,
     "expert": 8
    }
   },
   "task": {
    "accuracy": 0.7852253583254882,
    "macro_f1": 0.654148796360693,
    "macro_f1_present": 0.654148796360693,
    "per_class": {
     "information_seeking": {
      "f1": 0.8333494423886427,
      "support": 1528
     },
     "reasoning": {
      "f1": 0.5184660006109744,
      "support": 92
     },
     "planning": {
      "f1": 0.6457943382576198,
      "support": 81
     },
     "editing": {
      "f1": 0.7993242277530591,
      "support": 193
     },
     "summarization": {
      "f1": 0.7743852582192624,
      "support": 128
     },
     "extraction": {
      "f1": 0.7406019477788531,
      "support": 229
     },
     "classification": {
      "f1": 0.781658295992325,
      "support": 89
     },
     "translation": {
      "f1": 0.8981919704304817,
      "support": 73
     },
     "coding_debugging": {
      "f1": 0.850027407542987,
      "support": 453
     },
     "math": {
      "f1": 0.849948001056337,
      "support": 236
     },
     "data_analysis": {
      "f1": 0.5252115206113127,
      "support": 22
     },
     "writing": {
      "f1": 0.8492298605681178,
      "support": 991
     },
     "role_playing": {
      "f1": 0.722487088007609,
      "support": 83
     },
     "advice_seeking": {
      "f1": 0.6177654053304441,
      "support": 157
     },
     "brainstorming": {
      "f1": 0.5366834803332317,
      "support": 134
     },
     "completion": {
      "f1": 0.6390743945365348,
      "support": 267
     },
     "action_execution": {
      "f1": 0.17852667118303162,
      "support": 11
     },
     "chat": {
      "f1": 0.4148694891583502,
      "support": 91
     },
     "other": {
      "f1": 0.25323233109399224,
      "support": 56
     }
    }
   },
   "needs_tools": {
    "f1": 0.49273464955738516,
    "auroc": 0.9023776046979766,
    "positives": 250
   }
  }
 },
 "validation_four_sources": {
  "unweighted": {
   "rows": 3130,
   "weighted": false,
   "complexity": {
    "rows": 3101,
    "mae": 0.08768109154870378,
    "spearman": 0.679063401419824,
    "qwk_rounded": 0.6720474415326875,
    "confusion_true_by_pred": [
     [
      68.0,
      124.0,
      9.0,
      0.0,
      0.0
     ],
     [
      43.0,
      1574.0,
      219.0,
      2.0,
      0.0
     ],
     [
      2.0,
      288.0,
      555.0,
      27.0,
      0.0
     ],
     [
      0.0,
      8.0,
      82.0,
      99.0,
      0.0
     ],
     [
      0.0,
      0.0,
      0.0,
      1.0,
      0.0
     ]
    ],
    "support": {
     "trivial": 201,
     "low": 1838,
     "medium": 872,
     "high": 189,
     "expert": 1
    }
   },
   "task": {
    "accuracy": 0.797444089456869,
    "macro_f1": 0.6437112427191728,
    "macro_f1_present": 0.6437112427191728,
    "per_class": {
     "information_seeking": {
      "f1": 0.8419618528610354,
      "support": 1068
     },
     "reasoning": {
      "f1": 0.4943820224719101,
      "support": 48
     },
     "planning": {
      "f1": 0.6373626373626373,
      "support": 53
     },
     "editing": {
      "f1": 0.8028673835125448,
      "support": 139
     },
     "summarization": {
      "f1": 0.7853403141361257,
      "support": 98
     },
     "extraction": {
      "f1": 0.7700831024930748,
      "support": 180
     },
     "classification": {
      "f1": 0.7422680412371134,
      "support": 47
     },
     "translation": {
      "f1": 0.907563025210084,
      "support": 61
     },
     "coding_debugging": {
      "f1": 0.8577235772357723,
      "support": 246
     },
     "math": {
      "f1": 0.7142857142857143,
      "support": 41
     },
     "data_analysis": {
      "f1": 0.18181818181818182,
      "support": 5
     },
     "writing": {
      "f1": 0.8625525946704067,
      "support": 705
     },
     "role_playing": {
      "f1": 0.7419354838709677,
      "support": 30
     },
     "advice_seeking": {
      "f1": 0.576,
      "support": 75
     },
     "brainstorming": {
      "f1": 0.5428571428571428,
      "support": 83
     },
     "completion": {
      "f1": 0.6979472140762464,
      "support": 162
     },
     "action_execution": {
      "f1": 0.3076923076923077,
      "support": 8
     },
     "chat": {
      "f1": 0.4444444444444444,
      "support": 47
     },
     "other": {
      "f1": 0.32142857142857145,
      "support": 34
     }
    }
   },
   "needs_tools": {
    "f1": 0.4505928853754941,
    "auroc": 0.9142851183693721,
    "positives": 153
   }
  },
  "weighted": {
   "rows": 3130,
   "weighted": true,
   "complexity": {
    "rows": 3101,
    "mae": 0.08441043102181192,
    "spearman": 0.6373176375399857,
    "qwk_rounded": 0.6201069884726371,
    "confusion_true_by_pred": [
     [
      64.0,
      123.5,
      8.1,
      0.0,
      0.0
     ],
     [
      46.5,
      1704.6,
      238.5,
      1.7,
      0.0
     ],
     [
      2.1,
      292.9,
      563.2,
      25.9,
      0.0
     ],
     [
      0.0,
      3.3,
      34.6,
      41.9,
      0.0
     ],
     [
      0.0,
      0.0,
      0.0,
      0.1,
      0.0
     ]
    ],
    "support": {
     "trivial": 201,
     "low": 1838,
     "medium": 872,
     "high": 189,
     "expert": 1
    }
   },
   "task": {
    "accuracy": 0.8022331516771223,
    "macro_f1": 0.6339116489790322,
    "macro_f1_present": 0.6339116489790322,
    "per_class": {
     "information_seeking": {
      "f1": 0.8502006170050622,
      "support": 1068
     },
     "reasoning": {
      "f1": 0.5065343776674673,
      "support": 48
     },
     "planning": {
      "f1": 0.6103781252883665,
      "support": 53
     },
     "editing": {
      "f1": 0.8075727610466713,
      "support": 139
     },
     "summarization": {
      "f1": 0.7926531319470876,
      "support": 98
     },
     "extraction": {
      "f1": 0.7755297266811971,
      "support": 180
     },
     "classification": {
      "f1": 0.7393803668231865,
      "support": 47
     },
     "translation": {
      "f1": 0.9069849377410021,
      "support": 61
     },
     "coding_debugging": {
      "f1": 0.8598117983995479,
      "support": 246
     },
     "math": {
      "f1": 0.7134046788865379,
      "support": 41
     },
     "data_analysis": {
      "f1": 0.13601164014439981,
      "support": 5
     },
     "writing": {
      "f1": 0.8587568124350751,
      "support": 705
     },
     "role_playing": {
      "f1": 0.699100469772375,
      "support": 30
     },
     "advice_seeking": {
      "f1": 0.586485551349588,
      "support": 75
     },
     "brainstorming": {
      "f1": 0.5462773432289784,
      "support": 83
     },
     "completion": {
      "f1": 0.7063641824585476,
      "support": 162
     },
     "action_execution": {
      "f1": 0.20741165161792083,
      "support": 8
     },
     "chat": {
      "f1": 0.4433824978398469,
      "support": 47
     },
     "other": {
      "f1": 0.2980806602687547,
      "support": 34
     }
    }
   },
   "needs_tools": {
    "f1": 0.4340467042871764,
    "auroc": 0.9137223950124054,
    "positives": 153
   }
  }
 },
 "agreement_with_reference": {
  "max_abs_logit_difference": 19.72652244567871,
  "mean_abs_logit_difference": 2.084580898284912,
  "same_task": 0.812983312983313,
  "same_needs_tools": 0.967032967032967,
  "same_complexity_tier": 0.7932437932437932,
  "same_policy_tier": 0.807895807895808
 },
 "agreement_with_parent": {
  "max_abs_logit_difference": 1.166677474975586,
  "mean_abs_logit_difference": 0.07745294272899628,
  "same_task": 0.9920634920634921,
  "same_needs_tools": 0.9985754985754985,
  "same_complexity_tier": 0.9861619861619861,
  "same_policy_tier": 0.9881969881969882
 },
 "latency_ms_batch1": {
  "p50": 1.1119580012746155,
  "p95": 4.60683400160633,
  "mean": 1.5198578301715315,
  "prompts": 200,
  "device": "cpu"
 }
}