{
 "variant": "v7_small_vocab32k-fp16",
 "backend": "torch",
 "size_bytes": 145890531,
 "validation_all": {
  "unweighted": {
   "rows": 4914,
   "weighted": false,
   "complexity": {
    "rows": 4850,
    "mae": 0.07900026063624116,
    "spearman": 0.7222024284562206,
    "qwk_rounded": 0.7317020389269229,
    "confusion_true_by_pred": [
     [
      132.0,
      167.0,
      8.0,
      1.0,
      0.0
     ],
     [
      36.0,
      2560.0,
      280.0,
      2.0,
      0.0
     ],
     [
      0.0,
      428.0,
      878.0,
      39.0,
      0.0
     ],
     [
      1.0,
      9.0,
      92.0,
      207.0,
      2.0
     ],
     [
      0.0,
      0.0,
      4.0,
      4.0,
      0.0
     ]
    ],
    "support": {
     "trivial": 308,
     "low": 2878,
     "medium": 1345,
     "high": 311,
     "expert": 8
    }
   },
   "task": {
    "accuracy": 0.8341473341473341,
    "macro_f1": 0.7464171717856395,
    "macro_f1_present": 0.7464171717856395,
    "per_class": {
     "information_seeking": {
      "f1": 0.8650208400128246,
      "support": 1528
     },
     "reasoning": {
      "f1": 0.6219512195121951,
      "support": 92
     },
     "planning": {
      "f1": 0.7368421052631579,
      "support": 81
     },
     "editing": {
      "f1": 0.8564102564102564,
      "support": 193
     },
     "summarization": {
      "f1": 0.8387096774193549,
      "support": 128
     },
     "extraction": {
      "f1": 0.8053691275167785,
      "support": 229
     },
     "classification": {
      "f1": 0.8715083798882681,
      "support": 89
     },
     "translation": {
      "f1": 0.9178082191780822,
      "support": 73
     },
     "coding_debugging": {
      "f1": 0.8962472406181016,
      "support": 453
     },
     "math": {
      "f1": 0.8731808731808732,
      "support": 236
     },
     "data_analysis": {
      "f1": 0.6956521739130435,
      "support": 22
     },
     "writing": {
      "f1": 0.8909811694747275,
      "support": 991
     },
     "role_playing": {
      "f1": 0.8670520231213873,
      "support": 83
     },
     "advice_seeking": {
      "f1": 0.6666666666666666,
      "support": 157
     },
     "brainstorming": {
      "f1": 0.6260162601626016,
      "support": 134
     },
     "completion": {
      "f1": 0.7283018867924528,
      "support": 267
     },
     "action_execution": {
      "f1": 0.47058823529411764,
      "support": 11
     },
     "chat": {
      "f1": 0.5882352941176471,
      "support": 91
     },
     "other": {
      "f1": 0.36538461538461536,
      "support": 56
     }
    }
   },
   "needs_tools": {
    "f1": 0.6086956521739131,
    "auroc": 0.9491402229845626,
    "positives": 250
   }
  },
  "weighted": {
   "rows": 4914,
   "weighted": true,
   "complexity": {
    "rows": 4850,
    "mae": 0.07629462325742778,
    "spearman": 0.6815255381752715,
    "qwk_rounded": 0.6807493662146209,
    "confusion_true_by_pred": [
     [
      125.3,
      165.3,
      6.8,
      1.0,
      0.0
     ],
     [
      38.2,
      2752.4,
      301.6,
      2.1,
      0.0
     ],
     [
      0.0,
      431.5,
      880.8,
      39.0,
      0.0
     ],
     [
      0.4,
      3.7,
      38.9,
      87.5,
      0.9
     ],
     [
      0.0,
      0.0,
      0.5,
      0.6,
      0.0
     ]
    ],
    "support": {
     "trivial": 308,
     "low": 2878,
     "medium": 1345,
     "high": 311,
     "expert": 8
    }
   },
   "task": {
    "accuracy": 0.8360364312760598,
    "macro_f1": 0.7367827601549547,
    "macro_f1_present": 0.7367827601549547,
    "per_class": {
     "information_seeking": {
      "f1": 0.8704398782926369,
      "support": 1528
     },
     "reasoning": {
      "f1": 0.6265303864755468,
      "support": 92
     },
     "planning": {
      "f1": 0.7323813968125432,
      "support": 81
     },
     "editing": {
      "f1": 0.8589192475917232,
      "support": 193
     },
     "summarization": {
      "f1": 0.8399826842288818,
      "support": 128
     },
     "extraction": {
      "f1": 0.8087667015041471,
      "support": 229
     },
     "classification": {
      "f1": 0.8712522560555723,
      "support": 89
     },
     "translation": {
      "f1": 0.9176235948947219,
      "support": 73
     },
     "coding_debugging": {
      "f1": 0.8986539511949297,
      "support": 453
     },
     "math": {
      "f1": 0.8668086489406949,
      "support": 236
     },
     "data_analysis": {
      "f1": 0.6085351755921948,
      "support": 22
     },
     "writing": {
      "f1": 0.8868448174873386,
      "support": 991
     },
     "role_playing": {
      "f1": 0.8516799847804152,
      "support": 83
     },
     "advice_seeking": {
      "f1": 0.6774566607237414,
      "support": 157
     },
     "brainstorming": {
      "f1": 0.6249814980063544,
      "support": 134
     },
     "completion": {
      "f1": 0.7381312484477714,
      "support": 267
     },
     "action_execution": {
      "f1": 0.4003685794935799,
      "support": 11
     },
     "chat": {
      "f1": 0.5906997369070713,
      "support": 91
     },
     "other": {
      "f1": 0.3288159955142728,
      "support": 56
     }
    }
   },
   "needs_tools": {
    "f1": 0.6089291837619497,
    "auroc": 0.9492480466932003,
    "positives": 250
   }
  }
 },
 "validation_four_sources": {
  "unweighted": {
   "rows": 3130,
   "weighted": false,
   "complexity": {
    "rows": 3101,
    "mae": 0.07550227157841265,
    "spearman": 0.7334132762168679,
    "qwk_rounded": 0.7530434836036132,
    "confusion_true_by_pred": [
     [
      91.0,
      104.0,
      5.0,
      1.0,
      0.0
     ],
     [
      23.0,
      1654.0,
      161.0,
      0.0,
      0.0
     ],
     [
      0.0,
      252.0,
      589.0,
      31.0,
      0.0
     ],
     [
      1.0,
      3.0,
      52.0,
      133.0,
      0.0
     ],
     [
      0.0,
      0.0,
      0.0,
      1.0,
      0.0
     ]
    ],
    "support": {
     "trivial": 201,
     "low": 1838,
     "medium": 872,
     "high": 189,
     "expert": 1
    }
   },
   "task": {
    "accuracy": 0.8431309904153355,
    "macro_f1": 0.7298313231096484,
    "macro_f1_present": 0.7298313231096484,
    "per_class": {
     "information_seeking": {
      "f1": 0.8771283939254487,
      "support": 1068
     },
     "reasoning": {
      "f1": 0.6067415730337079,
      "support": 48
     },
     "planning": {
      "f1": 0.723404255319149,
      "support": 53
     },
     "editing": {
      "f1": 0.8825622775800712,
      "support": 139
     },
     "summarization": {
      "f1": 0.8393782383419689,
      "support": 98
     },
     "extraction": {
      "f1": 0.8101983002832861,
      "support": 180
     },
     "classification": {
      "f1": 0.8387096774193549,
      "support": 47
     },
     "translation": {
      "f1": 0.9333333333333333,
      "support": 61
     },
     "coding_debugging": {
      "f1": 0.9109730848861284,
      "support": 246
     },
     "math": {
      "f1": 0.7126436781609196,
      "support": 41
     },
     "data_analysis": {
      "f1": 0.4,
      "support": 5
     },
     "writing": {
      "f1": 0.896551724137931,
      "support": 705
     },
     "role_playing": {
      "f1": 0.8771929824561403,
      "support": 30
     },
     "advice_seeking": {
      "f1": 0.6433566433566433,
      "support": 75
     },
     "brainstorming": {
      "f1": 0.6103896103896104,
      "support": 83
     },
     "completion": {
      "f1": 0.7515527950310559,
      "support": 162
     },
     "action_execution": {
      "f1": 0.5714285714285714,
      "support": 8
     },
     "chat": {
      "f1": 0.575,
      "support": 47
     },
     "other": {
      "f1": 0.40625,
      "support": 34
     }
    }
   },
   "needs_tools": {
    "f1": 0.562962962962963,
    "auroc": 0.9559904803932546,
    "positives": 153
   }
  },
  "weighted": {
   "rows": 3130,
   "weighted": true,
   "complexity": {
    "rows": 3101,
    "mae": 0.0735351988821101,
    "spearman": 0.6949124415343185,
    "qwk_rounded": 0.7050546771905661,
    "confusion_true_by_pred": [
     [
      86.7,
      103.6,
      4.3,
      1.0,
      0.0
     ],
     [
      24.6,
      1791.6,
      175.1,
      0.0,
      0.0
     ],
     [
      0.0,
      256.6,
      596.6,
      30.9,
      0.0
     ],
     [
      0.4,
      1.2,
      22.2,
      56.0,
      0.0
     ],
     [
      0.0,
      0.0,
      0.0,
      0.1,
      0.0
     ]
    ],
    "support": {
     "trivial": 201,
     "low": 1838,
     "medium": 872,
     "high": 189,
     "expert": 1
    }
   },
   "task": {
    "accuracy": 0.8464363313815629,
    "macro_f1": 0.7191940497229352,
    "macro_f1_present": 0.7191940497229352,
    "per_class": {
     "information_seeking": {
      "f1": 0.8819119808363137,
      "support": 1068
     },
     "reasoning": {
      "f1": 0.6098229973539746,
      "support": 48
     },
     "planning": {
      "f1": 0.7289926121381023,
      "support": 53
     },
     "editing": {
      "f1": 0.8867976739704146,
      "support": 139
     },
     "summarization": {
      "f1": 0.840773699604592,
      "support": 98
     },
     "extraction": {
      "f1": 0.813574237492829,
      "support": 180
     },
     "classification": {
      "f1": 0.8377569961461717,
      "support": 47
     },
     "translation": {
      "f1": 0.9330111119858632,
      "support": 61
     },
     "coding_debugging": {
      "f1": 0.91387223972592,
      "support": 246
     },
     "math": {
      "f1": 0.6898563786378996,
      "support": 41
     },
     "data_analysis": {
      "f1": 0.315320475519074,
      "support": 5
     },
     "writing": {
      "f1": 0.8951125829152632,
      "support": 705
     },
     "role_playing": {
      "f1": 0.8753643299117938,
      "support": 30
     },
     "advice_seeking": {
      "f1": 0.6556990776879805,
      "support": 75
     },
     "brainstorming": {
      "f1": 0.6089745389657842,
      "support": 83
     },
     "completion": {
      "f1": 0.7600811270560087,
      "support": 162
     },
     "action_execution": {
      "f1": 0.47169812452618104,
      "support": 8
     },
     "chat": {
      "f1": 0.5755933848778737,
      "support": 47
     },
     "other": {
      "f1": 0.3704733753837275,
      "support": 34
     }
    }
   },
   "needs_tools": {
    "f1": 0.5546134214881696,
    "auroc": 0.9544811115467666,
    "positives": 153
   }
  }
 },
 "agreement_with_reference": {
  "max_abs_logit_difference": 15.826028823852539,
  "mean_abs_logit_difference": 1.3709417581558228,
  "same_task": 0.8764753764753764,
  "same_needs_tools": 0.9735449735449735,
  "same_complexity_tier": 0.8650793650793651,
  "same_policy_tier": 0.8715913715913716
 },
 "agreement_with_parent": {
  "max_abs_logit_difference": 0.052532970905303955,
  "mean_abs_logit_difference": 0.003039550269022584,
  "same_task": 1.0,
  "same_needs_tools": 1.0,
  "same_complexity_tier": 0.9987789987789988,
  "same_policy_tier": 0.9991859991859992
 },
 "latency_ms_batch1": {
  "p50": 9.136207998380996,
  "p95": 25.018457999976818,
  "mean": 13.463028115475026,
  "prompts": 200,
  "device": "mps"
 }
}