| { |
| "variant": "v7_small_l6-fp16", |
| "backend": "torch", |
| "size_bytes": 84514110, |
| "validation_all": { |
| "unweighted": { |
| "rows": 4914, |
| "weighted": false, |
| "complexity": { |
| "rows": 4850, |
| "mae": 0.09123653288428316, |
| "spearman": 0.6593453452413092, |
| "qwk_rounded": 0.6490167052545179, |
| "confusion_true_by_pred": [ |
| [ |
| 92.0, |
| 198.0, |
| 18.0, |
| 0.0, |
| 0.0 |
| ], |
| [ |
| 48.0, |
| 2451.0, |
| 377.0, |
| 2.0, |
| 0.0 |
| ], |
| [ |
| 5.0, |
| 463.0, |
| 842.0, |
| 35.0, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 22.0, |
| 125.0, |
| 164.0, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 1.0, |
| 4.0, |
| 3.0, |
| 0.0 |
| ] |
| ], |
| "support": { |
| "trivial": 308, |
| "low": 2878, |
| "medium": 1345, |
| "high": 311, |
| "expert": 8 |
| } |
| }, |
| "task": { |
| "accuracy": 0.7816442816442817, |
| "macro_f1": 0.6623087453835308, |
| "macro_f1_present": 0.6623087453835308, |
| "per_class": { |
| "information_seeking": { |
| "f1": 0.8244226510597912, |
| "support": 1528 |
| }, |
| "reasoning": { |
| "f1": 0.5149700598802395, |
| "support": 92 |
| }, |
| "planning": { |
| "f1": 0.6474820143884892, |
| "support": 81 |
| }, |
| "editing": { |
| "f1": 0.7917737789203085, |
| "support": 193 |
| }, |
| "summarization": { |
| "f1": 0.7649402390438247, |
| "support": 128 |
| }, |
| "extraction": { |
| "f1": 0.7352297592997812, |
| "support": 229 |
| }, |
| "classification": { |
| "f1": 0.7932960893854749, |
| "support": 89 |
| }, |
| "translation": { |
| "f1": 0.9041095890410958, |
| "support": 73 |
| }, |
| "coding_debugging": { |
| "f1": 0.8478260869565217, |
| "support": 453 |
| }, |
| "math": { |
| "f1": 0.8530020703933747, |
| "support": 236 |
| }, |
| "data_analysis": { |
| "f1": 0.5882352941176471, |
| "support": 22 |
| }, |
| "writing": { |
| "f1": 0.8550148957298908, |
| "support": 991 |
| }, |
| "role_playing": { |
| "f1": 0.7544910179640718, |
| "support": 83 |
| }, |
| "advice_seeking": { |
| "f1": 0.6099290780141844, |
| "support": 157 |
| }, |
| "brainstorming": { |
| "f1": 0.5281385281385281, |
| "support": 134 |
| }, |
| "completion": { |
| "f1": 0.6311787072243346, |
| "support": 267 |
| }, |
| "action_execution": { |
| "f1": 0.25, |
| "support": 11 |
| }, |
| "chat": { |
| "f1": 0.41025641025641024, |
| "support": 91 |
| }, |
| "other": { |
| "f1": 0.27956989247311825, |
| "support": 56 |
| } |
| } |
| }, |
| "needs_tools": { |
| "f1": 0.4841075794621027, |
| "auroc": 0.9042765866209262, |
| "positives": 250 |
| } |
| }, |
| "weighted": { |
| "rows": 4914, |
| "weighted": true, |
| "complexity": { |
| "rows": 4850, |
| "mae": 0.08725421027348824, |
| "spearman": 0.6160875172388919, |
| "qwk_rounded": 0.596045096988439, |
| "confusion_true_by_pred": [ |
| [ |
| 86.2, |
| 195.8, |
| 16.3, |
| 0.0, |
| 0.0 |
| ], |
| [ |
| 51.3, |
| 2634.9, |
| 406.4, |
| 1.7, |
| 0.0 |
| ], |
| [ |
| 5.2, |
| 467.1, |
| 845.1, |
| 34.0, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 9.3, |
| 52.7, |
| 69.4, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 0.1, |
| 0.5, |
| 0.4, |
| 0.0 |
| ] |
| ], |
| "support": { |
| "trivial": 308, |
| "low": 2878, |
| "medium": 1345, |
| "high": 311, |
| "expert": 8 |
| } |
| }, |
| "task": { |
| "accuracy": 0.7848718360585144, |
| "macro_f1": 0.6535855109314881, |
| "macro_f1_present": 0.6535855109314881, |
| "per_class": { |
| "information_seeking": { |
| "f1": 0.832598914469181, |
| "support": 1528 |
| }, |
| "reasoning": { |
| "f1": 0.5297883989506079, |
| "support": 92 |
| }, |
| "planning": { |
| "f1": 0.6323352636377239, |
| "support": 81 |
| }, |
| "editing": { |
| "f1": 0.7972908310414097, |
| "support": 193 |
| }, |
| "summarization": { |
| "f1": 0.771281034042985, |
| "support": 128 |
| }, |
| "extraction": { |
| "f1": 0.739535897050014, |
| "support": 229 |
| }, |
| "classification": { |
| "f1": 0.7906490364919273, |
| "support": 89 |
| }, |
| "translation": { |
| "f1": 0.9039649545580544, |
| "support": 73 |
| }, |
| "coding_debugging": { |
| "f1": 0.850302929061513, |
| "support": 453 |
| }, |
| "math": { |
| "f1": 0.8457726621104891, |
| "support": 236 |
| }, |
| "data_analysis": { |
| "f1": 0.5085499923284106, |
| "support": 22 |
| }, |
| "writing": { |
| "f1": 0.8495558798328425, |
| "support": 991 |
| }, |
| "role_playing": { |
| "f1": 0.722487088007609, |
| "support": 83 |
| }, |
| "advice_seeking": { |
| "f1": 0.6228240602260838, |
| "support": 157 |
| }, |
| "brainstorming": { |
| "f1": 0.5321998740461651, |
| "support": 134 |
| }, |
| "completion": { |
| "f1": 0.6401729708095073, |
| "support": 267 |
| }, |
| "action_execution": { |
| "f1": 0.17852667118303162, |
| "support": 11 |
| }, |
| "chat": { |
| "f1": 0.41431580139190316, |
| "support": 91 |
| }, |
| "other": { |
| "f1": 0.2559724484588159, |
| "support": 56 |
| } |
| } |
| }, |
| "needs_tools": { |
| "f1": 0.4833044372624333, |
| "auroc": 0.9023762363440311, |
| "positives": 250 |
| } |
| } |
| }, |
| "validation_four_sources": { |
| "unweighted": { |
| "rows": 3130, |
| "weighted": false, |
| "complexity": { |
| "rows": 3101, |
| "mae": 0.08777755751143883, |
| "spearman": 0.6784958198543671, |
| "qwk_rounded": 0.6711515464869717, |
| "confusion_true_by_pred": [ |
| [ |
| 63.0, |
| 129.0, |
| 9.0, |
| 0.0, |
| 0.0 |
| ], |
| [ |
| 34.0, |
| 1567.0, |
| 235.0, |
| 2.0, |
| 0.0 |
| ], |
| [ |
| 2.0, |
| 275.0, |
| 566.0, |
| 29.0, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 8.0, |
| 81.0, |
| 100.0, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 0.0, |
| 0.0, |
| 1.0, |
| 0.0 |
| ] |
| ], |
| "support": { |
| "trivial": 201, |
| "low": 1838, |
| "medium": 872, |
| "high": 189, |
| "expert": 1 |
| } |
| }, |
| "task": { |
| "accuracy": 0.7955271565495208, |
| "macro_f1": 0.6418206886744169, |
| "macro_f1_present": 0.6418206886744169, |
| "per_class": { |
| "information_seeking": { |
| "f1": 0.8394724874943156, |
| "support": 1068 |
| }, |
| "reasoning": { |
| "f1": 0.4943820224719101, |
| "support": 48 |
| }, |
| "planning": { |
| "f1": 0.6067415730337079, |
| "support": 53 |
| }, |
| "editing": { |
| "f1": 0.8, |
| "support": 139 |
| }, |
| "summarization": { |
| "f1": 0.78125, |
| "support": 98 |
| }, |
| "extraction": { |
| "f1": 0.7688022284122563, |
| "support": 180 |
| }, |
| "classification": { |
| "f1": 0.7578947368421053, |
| "support": 47 |
| }, |
| "translation": { |
| "f1": 0.907563025210084, |
| "support": 61 |
| }, |
| "coding_debugging": { |
| "f1": 0.8577235772357723, |
| "support": 246 |
| }, |
| "math": { |
| "f1": 0.7058823529411765, |
| "support": 41 |
| }, |
| "data_analysis": { |
| "f1": 0.18181818181818182, |
| "support": 5 |
| }, |
| "writing": { |
| "f1": 0.8631284916201117, |
| "support": 705 |
| }, |
| "role_playing": { |
| "f1": 0.7419354838709677, |
| "support": 30 |
| }, |
| "advice_seeking": { |
| "f1": 0.576, |
| "support": 75 |
| }, |
| "brainstorming": { |
| "f1": 0.5285714285714286, |
| "support": 83 |
| }, |
| "completion": { |
| "f1": 0.6923076923076923, |
| "support": 162 |
| }, |
| "action_execution": { |
| "f1": 0.3076923076923077, |
| "support": 8 |
| }, |
| "chat": { |
| "f1": 0.4444444444444444, |
| "support": 47 |
| }, |
| "other": { |
| "f1": 0.3389830508474576, |
| "support": 34 |
| } |
| } |
| }, |
| "needs_tools": { |
| "f1": 0.43824701195219123, |
| "auroc": 0.9143597647322281, |
| "positives": 153 |
| } |
| }, |
| "weighted": { |
| "rows": 3130, |
| "weighted": true, |
| "complexity": { |
| "rows": 3101, |
| "mae": 0.08461350760287671, |
| "spearman": 0.636775261257106, |
| "qwk_rounded": 0.61930401674141, |
| "confusion_true_by_pred": [ |
| [ |
| 59.1, |
| 128.4, |
| 8.1, |
| 0.0, |
| 0.0 |
| ], |
| [ |
| 36.9, |
| 1696.7, |
| 256.0, |
| 1.7, |
| 0.0 |
| ], |
| [ |
| 2.1, |
| 279.5, |
| 574.5, |
| 28.0, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 3.3, |
| 34.1, |
| 42.3, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.1, |
| 0.0 |
| ] |
| ], |
| "support": { |
| "trivial": 201, |
| "low": 1838, |
| "medium": 872, |
| "high": 189, |
| "expert": 1 |
| } |
| }, |
| "task": { |
| "accuracy": 0.8006841010532753, |
| "macro_f1": 0.6323733904733824, |
| "macro_f1_present": 0.6323733904733824, |
| "per_class": { |
| "information_seeking": { |
| "f1": 0.8480068752176662, |
| "support": 1068 |
| }, |
| "reasoning": { |
| "f1": 0.5097422442799173, |
| "support": 48 |
| }, |
| "planning": { |
| "f1": 0.5885036465886375, |
| "support": 53 |
| }, |
| "editing": { |
| "f1": 0.8047164777623818, |
| "support": 139 |
| }, |
| "summarization": { |
| "f1": 0.7884833764231882, |
| "support": 98 |
| }, |
| "extraction": { |
| "f1": 0.7743659734172523, |
| "support": 180 |
| }, |
| "classification": { |
| "f1": 0.755260299322683, |
| "support": 47 |
| }, |
| "translation": { |
| "f1": 0.9069849377410021, |
| "support": 61 |
| }, |
| "coding_debugging": { |
| "f1": 0.8598117983995479, |
| "support": 246 |
| }, |
| "math": { |
| "f1": 0.7029567273827195, |
| "support": 41 |
| }, |
| "data_analysis": { |
| "f1": 0.13601164014439981, |
| "support": 5 |
| }, |
| "writing": { |
| "f1": 0.8599838608390206, |
| "support": 705 |
| }, |
| "role_playing": { |
| "f1": 0.699100469772375, |
| "support": 30 |
| }, |
| "advice_seeking": { |
| "f1": 0.586485551349588, |
| "support": 75 |
| }, |
| "brainstorming": { |
| "f1": 0.5322990203969951, |
| "support": 83 |
| }, |
| "completion": { |
| "f1": 0.7009731628974076, |
| "support": 162 |
| }, |
| "action_execution": { |
| "f1": 0.20741165161792083, |
| "support": 8 |
| }, |
| "chat": { |
| "f1": 0.44317185671144815, |
| "support": 47 |
| }, |
| "other": { |
| "f1": 0.3108248487301116, |
| "support": 34 |
| } |
| } |
| }, |
| "needs_tools": { |
| "f1": 0.4231775842460709, |
| "auroc": 0.9139293957758868, |
| "positives": 153 |
| } |
| } |
| }, |
| "agreement_with_reference": { |
| "max_abs_logit_difference": 19.687698364257812, |
| "mean_abs_logit_difference": 2.079862594604492, |
| "same_task": 0.8133903133903134, |
| "same_needs_tools": 0.967032967032967, |
| "same_complexity_tier": 0.7964997964997965, |
| "same_policy_tier": 0.8085063085063086 |
| }, |
| "agreement_with_parent": { |
| "max_abs_logit_difference": 0.0420689582824707, |
| "mean_abs_logit_difference": 0.004016549792140722, |
| "same_task": 0.9991859991859992, |
| "same_needs_tools": 0.9997964997964998, |
| "same_complexity_tier": 0.9971509971509972, |
| "same_policy_tier": 0.9991859991859992 |
| }, |
| "latency_ms_batch1": { |
| "p50": 3.0535839978256263, |
| "p95": 8.006165997358039, |
| "mean": 4.419368929666234, |
| "prompts": 200, |
| "device": "mps" |
| } |
| } |
