| { |
| "variant": "v7_small_l6-mlx-8bit", |
| "backend": "mlx", |
| "size_bytes": 62125397, |
| "validation_all": { |
| "unweighted": { |
| "rows": 4914, |
| "weighted": false, |
| "complexity": { |
| "rows": 4850, |
| "mae": 0.09117008546431041, |
| "spearman": 0.6591213539927813, |
| "qwk_rounded": 0.6471972183137378, |
| "confusion_true_by_pred": [ |
| [ |
| 97.0, |
| 194.0, |
| 17.0, |
| 0.0, |
| 0.0 |
| ], |
| [ |
| 58.0, |
| 2464.0, |
| 354.0, |
| 2.0, |
| 0.0 |
| ], |
| [ |
| 5.0, |
| 487.0, |
| 820.0, |
| 33.0, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 23.0, |
| 125.0, |
| 163.0, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 1.0, |
| 5.0, |
| 2.0, |
| 0.0 |
| ] |
| ], |
| "support": { |
| "trivial": 308, |
| "low": 2878, |
| "medium": 1345, |
| "high": 311, |
| "expert": 8 |
| } |
| }, |
| "task": { |
| "accuracy": 0.7822547822547823, |
| "macro_f1": 0.6628007985114895, |
| "macro_f1_present": 0.6628007985114895, |
| "per_class": { |
| "information_seeking": { |
| "f1": 0.8253868013893274, |
| "support": 1528 |
| }, |
| "reasoning": { |
| "f1": 0.5058823529411764, |
| "support": 92 |
| }, |
| "planning": { |
| "f1": 0.6666666666666666, |
| "support": 81 |
| }, |
| "editing": { |
| "f1": 0.7938144329896907, |
| "support": 193 |
| }, |
| "summarization": { |
| "f1": 0.768, |
| "support": 128 |
| }, |
| "extraction": { |
| "f1": 0.7363834422657952, |
| "support": 229 |
| }, |
| "classification": { |
| "f1": 0.7845303867403315, |
| "support": 89 |
| }, |
| "translation": { |
| "f1": 0.8979591836734694, |
| "support": 73 |
| }, |
| "coding_debugging": { |
| "f1": 0.8474945533769063, |
| "support": 453 |
| }, |
| "math": { |
| "f1": 0.8565488565488566, |
| "support": 236 |
| }, |
| "data_analysis": { |
| "f1": 0.6, |
| "support": 22 |
| }, |
| "writing": { |
| "f1": 0.8551518168242906, |
| "support": 991 |
| }, |
| "role_playing": { |
| "f1": 0.7544910179640718, |
| "support": 83 |
| }, |
| "advice_seeking": { |
| "f1": 0.604982206405694, |
| "support": 157 |
| }, |
| "brainstorming": { |
| "f1": 0.5327510917030568, |
| "support": 134 |
| }, |
| "completion": { |
| "f1": 0.630188679245283, |
| "support": 267 |
| }, |
| "action_execution": { |
| "f1": 0.25, |
| "support": 11 |
| }, |
| "chat": { |
| "f1": 0.41025641025641024, |
| "support": 91 |
| }, |
| "other": { |
| "f1": 0.2727272727272727, |
| "support": 56 |
| } |
| } |
| }, |
| "needs_tools": { |
| "f1": 0.4963855421686747, |
| "auroc": 0.9044644082332761, |
| "positives": 250 |
| } |
| }, |
| "weighted": { |
| "rows": 4914, |
| "weighted": true, |
| "complexity": { |
| "rows": 4850, |
| "mae": 0.08705916285165423, |
| "spearman": 0.6158774632764406, |
| "qwk_rounded": 0.594693223073677, |
| "confusion_true_by_pred": [ |
| [ |
| 91.1, |
| 192.0, |
| 15.2, |
| 0.0, |
| 0.0 |
| ], |
| [ |
| 61.9, |
| 2649.2, |
| 381.4, |
| 1.7, |
| 0.0 |
| ], |
| [ |
| 5.2, |
| 491.5, |
| 822.7, |
| 31.9, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 9.7, |
| 52.7, |
| 69.0, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 0.1, |
| 0.7, |
| 0.3, |
| 0.0 |
| ] |
| ], |
| "support": { |
| "trivial": 308, |
| "low": 2878, |
| "medium": 1345, |
| "high": 311, |
| "expert": 8 |
| } |
| }, |
| "task": { |
| "accuracy": 0.7852253583254882, |
| "macro_f1": 0.654148796360693, |
| "macro_f1_present": 0.654148796360693, |
| "per_class": { |
| "information_seeking": { |
| "f1": 0.8333494423886427, |
| "support": 1528 |
| }, |
| "reasoning": { |
| "f1": 0.5184660006109744, |
| "support": 92 |
| }, |
| "planning": { |
| "f1": 0.6457943382576198, |
| "support": 81 |
| }, |
| "editing": { |
| "f1": 0.7993242277530591, |
| "support": 193 |
| }, |
| "summarization": { |
| "f1": 0.7743852582192624, |
| "support": 128 |
| }, |
| "extraction": { |
| "f1": 0.7406019477788531, |
| "support": 229 |
| }, |
| "classification": { |
| "f1": 0.781658295992325, |
| "support": 89 |
| }, |
| "translation": { |
| "f1": 0.8981919704304817, |
| "support": 73 |
| }, |
| "coding_debugging": { |
| "f1": 0.850027407542987, |
| "support": 453 |
| }, |
| "math": { |
| "f1": 0.849948001056337, |
| "support": 236 |
| }, |
| "data_analysis": { |
| "f1": 0.5252115206113127, |
| "support": 22 |
| }, |
| "writing": { |
| "f1": 0.8492298605681178, |
| "support": 991 |
| }, |
| "role_playing": { |
| "f1": 0.722487088007609, |
| "support": 83 |
| }, |
| "advice_seeking": { |
| "f1": 0.6177654053304441, |
| "support": 157 |
| }, |
| "brainstorming": { |
| "f1": 0.5366834803332317, |
| "support": 134 |
| }, |
| "completion": { |
| "f1": 0.6390743945365348, |
| "support": 267 |
| }, |
| "action_execution": { |
| "f1": 0.17852667118303162, |
| "support": 11 |
| }, |
| "chat": { |
| "f1": 0.4148694891583502, |
| "support": 91 |
| }, |
| "other": { |
| "f1": 0.25323233109399224, |
| "support": 56 |
| } |
| } |
| }, |
| "needs_tools": { |
| "f1": 0.49273464955738516, |
| "auroc": 0.9023776046979766, |
| "positives": 250 |
| } |
| } |
| }, |
| "validation_four_sources": { |
| "unweighted": { |
| "rows": 3130, |
| "weighted": false, |
| "complexity": { |
| "rows": 3101, |
| "mae": 0.08768109154870378, |
| "spearman": 0.679063401419824, |
| "qwk_rounded": 0.6720474415326875, |
| "confusion_true_by_pred": [ |
| [ |
| 68.0, |
| 124.0, |
| 9.0, |
| 0.0, |
| 0.0 |
| ], |
| [ |
| 43.0, |
| 1574.0, |
| 219.0, |
| 2.0, |
| 0.0 |
| ], |
| [ |
| 2.0, |
| 288.0, |
| 555.0, |
| 27.0, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 8.0, |
| 82.0, |
| 99.0, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 0.0, |
| 0.0, |
| 1.0, |
| 0.0 |
| ] |
| ], |
| "support": { |
| "trivial": 201, |
| "low": 1838, |
| "medium": 872, |
| "high": 189, |
| "expert": 1 |
| } |
| }, |
| "task": { |
| "accuracy": 0.797444089456869, |
| "macro_f1": 0.6437112427191728, |
| "macro_f1_present": 0.6437112427191728, |
| "per_class": { |
| "information_seeking": { |
| "f1": 0.8419618528610354, |
| "support": 1068 |
| }, |
| "reasoning": { |
| "f1": 0.4943820224719101, |
| "support": 48 |
| }, |
| "planning": { |
| "f1": 0.6373626373626373, |
| "support": 53 |
| }, |
| "editing": { |
| "f1": 0.8028673835125448, |
| "support": 139 |
| }, |
| "summarization": { |
| "f1": 0.7853403141361257, |
| "support": 98 |
| }, |
| "extraction": { |
| "f1": 0.7700831024930748, |
| "support": 180 |
| }, |
| "classification": { |
| "f1": 0.7422680412371134, |
| "support": 47 |
| }, |
| "translation": { |
| "f1": 0.907563025210084, |
| "support": 61 |
| }, |
| "coding_debugging": { |
| "f1": 0.8577235772357723, |
| "support": 246 |
| }, |
| "math": { |
| "f1": 0.7142857142857143, |
| "support": 41 |
| }, |
| "data_analysis": { |
| "f1": 0.18181818181818182, |
| "support": 5 |
| }, |
| "writing": { |
| "f1": 0.8625525946704067, |
| "support": 705 |
| }, |
| "role_playing": { |
| "f1": 0.7419354838709677, |
| "support": 30 |
| }, |
| "advice_seeking": { |
| "f1": 0.576, |
| "support": 75 |
| }, |
| "brainstorming": { |
| "f1": 0.5428571428571428, |
| "support": 83 |
| }, |
| "completion": { |
| "f1": 0.6979472140762464, |
| "support": 162 |
| }, |
| "action_execution": { |
| "f1": 0.3076923076923077, |
| "support": 8 |
| }, |
| "chat": { |
| "f1": 0.4444444444444444, |
| "support": 47 |
| }, |
| "other": { |
| "f1": 0.32142857142857145, |
| "support": 34 |
| } |
| } |
| }, |
| "needs_tools": { |
| "f1": 0.4505928853754941, |
| "auroc": 0.9142851183693721, |
| "positives": 153 |
| } |
| }, |
| "weighted": { |
| "rows": 3130, |
| "weighted": true, |
| "complexity": { |
| "rows": 3101, |
| "mae": 0.08441043102181192, |
| "spearman": 0.6373176375399857, |
| "qwk_rounded": 0.6201069884726371, |
| "confusion_true_by_pred": [ |
| [ |
| 64.0, |
| 123.5, |
| 8.1, |
| 0.0, |
| 0.0 |
| ], |
| [ |
| 46.5, |
| 1704.6, |
| 238.5, |
| 1.7, |
| 0.0 |
| ], |
| [ |
| 2.1, |
| 292.9, |
| 563.2, |
| 25.9, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 3.3, |
| 34.6, |
| 41.9, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.1, |
| 0.0 |
| ] |
| ], |
| "support": { |
| "trivial": 201, |
| "low": 1838, |
| "medium": 872, |
| "high": 189, |
| "expert": 1 |
| } |
| }, |
| "task": { |
| "accuracy": 0.8022331516771223, |
| "macro_f1": 0.6339116489790322, |
| "macro_f1_present": 0.6339116489790322, |
| "per_class": { |
| "information_seeking": { |
| "f1": 0.8502006170050622, |
| "support": 1068 |
| }, |
| "reasoning": { |
| "f1": 0.5065343776674673, |
| "support": 48 |
| }, |
| "planning": { |
| "f1": 0.6103781252883665, |
| "support": 53 |
| }, |
| "editing": { |
| "f1": 0.8075727610466713, |
| "support": 139 |
| }, |
| "summarization": { |
| "f1": 0.7926531319470876, |
| "support": 98 |
| }, |
| "extraction": { |
| "f1": 0.7755297266811971, |
| "support": 180 |
| }, |
| "classification": { |
| "f1": 0.7393803668231865, |
| "support": 47 |
| }, |
| "translation": { |
| "f1": 0.9069849377410021, |
| "support": 61 |
| }, |
| "coding_debugging": { |
| "f1": 0.8598117983995479, |
| "support": 246 |
| }, |
| "math": { |
| "f1": 0.7134046788865379, |
| "support": 41 |
| }, |
| "data_analysis": { |
| "f1": 0.13601164014439981, |
| "support": 5 |
| }, |
| "writing": { |
| "f1": 0.8587568124350751, |
| "support": 705 |
| }, |
| "role_playing": { |
| "f1": 0.699100469772375, |
| "support": 30 |
| }, |
| "advice_seeking": { |
| "f1": 0.586485551349588, |
| "support": 75 |
| }, |
| "brainstorming": { |
| "f1": 0.5462773432289784, |
| "support": 83 |
| }, |
| "completion": { |
| "f1": 0.7063641824585476, |
| "support": 162 |
| }, |
| "action_execution": { |
| "f1": 0.20741165161792083, |
| "support": 8 |
| }, |
| "chat": { |
| "f1": 0.4433824978398469, |
| "support": 47 |
| }, |
| "other": { |
| "f1": 0.2980806602687547, |
| "support": 34 |
| } |
| } |
| }, |
| "needs_tools": { |
| "f1": 0.4340467042871764, |
| "auroc": 0.9137223950124054, |
| "positives": 153 |
| } |
| } |
| }, |
| "agreement_with_reference": { |
| "max_abs_logit_difference": 19.72652244567871, |
| "mean_abs_logit_difference": 2.084580898284912, |
| "same_task": 0.812983312983313, |
| "same_needs_tools": 0.967032967032967, |
| "same_complexity_tier": 0.7932437932437932, |
| "same_policy_tier": 0.807895807895808 |
| }, |
| "agreement_with_parent": { |
| "max_abs_logit_difference": 1.166677474975586, |
| "mean_abs_logit_difference": 0.07745294272899628, |
| "same_task": 0.9920634920634921, |
| "same_needs_tools": 0.9985754985754985, |
| "same_complexity_tier": 0.9861619861619861, |
| "same_policy_tier": 0.9881969881969882 |
| }, |
| "latency_ms_batch1": { |
| "p50": 1.1119580012746155, |
| "p95": 4.60683400160633, |
| "mean": 1.5198578301715315, |
| "prompts": 200, |
| "device": "cpu" |
| } |
| } |
