| { |
| "variant": "v7_small_vocab32k-fp16", |
| "backend": "torch", |
| "size_bytes": 145890531, |
| "validation_all": { |
| "unweighted": { |
| "rows": 4914, |
| "weighted": false, |
| "complexity": { |
| "rows": 4850, |
| "mae": 0.07900026063624116, |
| "spearman": 0.7222024284562206, |
| "qwk_rounded": 0.7317020389269229, |
| "confusion_true_by_pred": [ |
| [ |
| 132.0, |
| 167.0, |
| 8.0, |
| 1.0, |
| 0.0 |
| ], |
| [ |
| 36.0, |
| 2560.0, |
| 280.0, |
| 2.0, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 428.0, |
| 878.0, |
| 39.0, |
| 0.0 |
| ], |
| [ |
| 1.0, |
| 9.0, |
| 92.0, |
| 207.0, |
| 2.0 |
| ], |
| [ |
| 0.0, |
| 0.0, |
| 4.0, |
| 4.0, |
| 0.0 |
| ] |
| ], |
| "support": { |
| "trivial": 308, |
| "low": 2878, |
| "medium": 1345, |
| "high": 311, |
| "expert": 8 |
| } |
| }, |
| "task": { |
| "accuracy": 0.8341473341473341, |
| "macro_f1": 0.7464171717856395, |
| "macro_f1_present": 0.7464171717856395, |
| "per_class": { |
| "information_seeking": { |
| "f1": 0.8650208400128246, |
| "support": 1528 |
| }, |
| "reasoning": { |
| "f1": 0.6219512195121951, |
| "support": 92 |
| }, |
| "planning": { |
| "f1": 0.7368421052631579, |
| "support": 81 |
| }, |
| "editing": { |
| "f1": 0.8564102564102564, |
| "support": 193 |
| }, |
| "summarization": { |
| "f1": 0.8387096774193549, |
| "support": 128 |
| }, |
| "extraction": { |
| "f1": 0.8053691275167785, |
| "support": 229 |
| }, |
| "classification": { |
| "f1": 0.8715083798882681, |
| "support": 89 |
| }, |
| "translation": { |
| "f1": 0.9178082191780822, |
| "support": 73 |
| }, |
| "coding_debugging": { |
| "f1": 0.8962472406181016, |
| "support": 453 |
| }, |
| "math": { |
| "f1": 0.8731808731808732, |
| "support": 236 |
| }, |
| "data_analysis": { |
| "f1": 0.6956521739130435, |
| "support": 22 |
| }, |
| "writing": { |
| "f1": 0.8909811694747275, |
| "support": 991 |
| }, |
| "role_playing": { |
| "f1": 0.8670520231213873, |
| "support": 83 |
| }, |
| "advice_seeking": { |
| "f1": 0.6666666666666666, |
| "support": 157 |
| }, |
| "brainstorming": { |
| "f1": 0.6260162601626016, |
| "support": 134 |
| }, |
| "completion": { |
| "f1": 0.7283018867924528, |
| "support": 267 |
| }, |
| "action_execution": { |
| "f1": 0.47058823529411764, |
| "support": 11 |
| }, |
| "chat": { |
| "f1": 0.5882352941176471, |
| "support": 91 |
| }, |
| "other": { |
| "f1": 0.36538461538461536, |
| "support": 56 |
| } |
| } |
| }, |
| "needs_tools": { |
| "f1": 0.6086956521739131, |
| "auroc": 0.9491402229845626, |
| "positives": 250 |
| } |
| }, |
| "weighted": { |
| "rows": 4914, |
| "weighted": true, |
| "complexity": { |
| "rows": 4850, |
| "mae": 0.07629462325742778, |
| "spearman": 0.6815255381752715, |
| "qwk_rounded": 0.6807493662146209, |
| "confusion_true_by_pred": [ |
| [ |
| 125.3, |
| 165.3, |
| 6.8, |
| 1.0, |
| 0.0 |
| ], |
| [ |
| 38.2, |
| 2752.4, |
| 301.6, |
| 2.1, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 431.5, |
| 880.8, |
| 39.0, |
| 0.0 |
| ], |
| [ |
| 0.4, |
| 3.7, |
| 38.9, |
| 87.5, |
| 0.9 |
| ], |
| [ |
| 0.0, |
| 0.0, |
| 0.5, |
| 0.6, |
| 0.0 |
| ] |
| ], |
| "support": { |
| "trivial": 308, |
| "low": 2878, |
| "medium": 1345, |
| "high": 311, |
| "expert": 8 |
| } |
| }, |
| "task": { |
| "accuracy": 0.8360364312760598, |
| "macro_f1": 0.7367827601549547, |
| "macro_f1_present": 0.7367827601549547, |
| "per_class": { |
| "information_seeking": { |
| "f1": 0.8704398782926369, |
| "support": 1528 |
| }, |
| "reasoning": { |
| "f1": 0.6265303864755468, |
| "support": 92 |
| }, |
| "planning": { |
| "f1": 0.7323813968125432, |
| "support": 81 |
| }, |
| "editing": { |
| "f1": 0.8589192475917232, |
| "support": 193 |
| }, |
| "summarization": { |
| "f1": 0.8399826842288818, |
| "support": 128 |
| }, |
| "extraction": { |
| "f1": 0.8087667015041471, |
| "support": 229 |
| }, |
| "classification": { |
| "f1": 0.8712522560555723, |
| "support": 89 |
| }, |
| "translation": { |
| "f1": 0.9176235948947219, |
| "support": 73 |
| }, |
| "coding_debugging": { |
| "f1": 0.8986539511949297, |
| "support": 453 |
| }, |
| "math": { |
| "f1": 0.8668086489406949, |
| "support": 236 |
| }, |
| "data_analysis": { |
| "f1": 0.6085351755921948, |
| "support": 22 |
| }, |
| "writing": { |
| "f1": 0.8868448174873386, |
| "support": 991 |
| }, |
| "role_playing": { |
| "f1": 0.8516799847804152, |
| "support": 83 |
| }, |
| "advice_seeking": { |
| "f1": 0.6774566607237414, |
| "support": 157 |
| }, |
| "brainstorming": { |
| "f1": 0.6249814980063544, |
| "support": 134 |
| }, |
| "completion": { |
| "f1": 0.7381312484477714, |
| "support": 267 |
| }, |
| "action_execution": { |
| "f1": 0.4003685794935799, |
| "support": 11 |
| }, |
| "chat": { |
| "f1": 0.5906997369070713, |
| "support": 91 |
| }, |
| "other": { |
| "f1": 0.3288159955142728, |
| "support": 56 |
| } |
| } |
| }, |
| "needs_tools": { |
| "f1": 0.6089291837619497, |
| "auroc": 0.9492480466932003, |
| "positives": 250 |
| } |
| } |
| }, |
| "validation_four_sources": { |
| "unweighted": { |
| "rows": 3130, |
| "weighted": false, |
| "complexity": { |
| "rows": 3101, |
| "mae": 0.07550227157841265, |
| "spearman": 0.7334132762168679, |
| "qwk_rounded": 0.7530434836036132, |
| "confusion_true_by_pred": [ |
| [ |
| 91.0, |
| 104.0, |
| 5.0, |
| 1.0, |
| 0.0 |
| ], |
| [ |
| 23.0, |
| 1654.0, |
| 161.0, |
| 0.0, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 252.0, |
| 589.0, |
| 31.0, |
| 0.0 |
| ], |
| [ |
| 1.0, |
| 3.0, |
| 52.0, |
| 133.0, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 0.0, |
| 0.0, |
| 1.0, |
| 0.0 |
| ] |
| ], |
| "support": { |
| "trivial": 201, |
| "low": 1838, |
| "medium": 872, |
| "high": 189, |
| "expert": 1 |
| } |
| }, |
| "task": { |
| "accuracy": 0.8431309904153355, |
| "macro_f1": 0.7298313231096484, |
| "macro_f1_present": 0.7298313231096484, |
| "per_class": { |
| "information_seeking": { |
| "f1": 0.8771283939254487, |
| "support": 1068 |
| }, |
| "reasoning": { |
| "f1": 0.6067415730337079, |
| "support": 48 |
| }, |
| "planning": { |
| "f1": 0.723404255319149, |
| "support": 53 |
| }, |
| "editing": { |
| "f1": 0.8825622775800712, |
| "support": 139 |
| }, |
| "summarization": { |
| "f1": 0.8393782383419689, |
| "support": 98 |
| }, |
| "extraction": { |
| "f1": 0.8101983002832861, |
| "support": 180 |
| }, |
| "classification": { |
| "f1": 0.8387096774193549, |
| "support": 47 |
| }, |
| "translation": { |
| "f1": 0.9333333333333333, |
| "support": 61 |
| }, |
| "coding_debugging": { |
| "f1": 0.9109730848861284, |
| "support": 246 |
| }, |
| "math": { |
| "f1": 0.7126436781609196, |
| "support": 41 |
| }, |
| "data_analysis": { |
| "f1": 0.4, |
| "support": 5 |
| }, |
| "writing": { |
| "f1": 0.896551724137931, |
| "support": 705 |
| }, |
| "role_playing": { |
| "f1": 0.8771929824561403, |
| "support": 30 |
| }, |
| "advice_seeking": { |
| "f1": 0.6433566433566433, |
| "support": 75 |
| }, |
| "brainstorming": { |
| "f1": 0.6103896103896104, |
| "support": 83 |
| }, |
| "completion": { |
| "f1": 0.7515527950310559, |
| "support": 162 |
| }, |
| "action_execution": { |
| "f1": 0.5714285714285714, |
| "support": 8 |
| }, |
| "chat": { |
| "f1": 0.575, |
| "support": 47 |
| }, |
| "other": { |
| "f1": 0.40625, |
| "support": 34 |
| } |
| } |
| }, |
| "needs_tools": { |
| "f1": 0.562962962962963, |
| "auroc": 0.9559904803932546, |
| "positives": 153 |
| } |
| }, |
| "weighted": { |
| "rows": 3130, |
| "weighted": true, |
| "complexity": { |
| "rows": 3101, |
| "mae": 0.0735351988821101, |
| "spearman": 0.6949124415343185, |
| "qwk_rounded": 0.7050546771905661, |
| "confusion_true_by_pred": [ |
| [ |
| 86.7, |
| 103.6, |
| 4.3, |
| 1.0, |
| 0.0 |
| ], |
| [ |
| 24.6, |
| 1791.6, |
| 175.1, |
| 0.0, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 256.6, |
| 596.6, |
| 30.9, |
| 0.0 |
| ], |
| [ |
| 0.4, |
| 1.2, |
| 22.2, |
| 56.0, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.1, |
| 0.0 |
| ] |
| ], |
| "support": { |
| "trivial": 201, |
| "low": 1838, |
| "medium": 872, |
| "high": 189, |
| "expert": 1 |
| } |
| }, |
| "task": { |
| "accuracy": 0.8464363313815629, |
| "macro_f1": 0.7191940497229352, |
| "macro_f1_present": 0.7191940497229352, |
| "per_class": { |
| "information_seeking": { |
| "f1": 0.8819119808363137, |
| "support": 1068 |
| }, |
| "reasoning": { |
| "f1": 0.6098229973539746, |
| "support": 48 |
| }, |
| "planning": { |
| "f1": 0.7289926121381023, |
| "support": 53 |
| }, |
| "editing": { |
| "f1": 0.8867976739704146, |
| "support": 139 |
| }, |
| "summarization": { |
| "f1": 0.840773699604592, |
| "support": 98 |
| }, |
| "extraction": { |
| "f1": 0.813574237492829, |
| "support": 180 |
| }, |
| "classification": { |
| "f1": 0.8377569961461717, |
| "support": 47 |
| }, |
| "translation": { |
| "f1": 0.9330111119858632, |
| "support": 61 |
| }, |
| "coding_debugging": { |
| "f1": 0.91387223972592, |
| "support": 246 |
| }, |
| "math": { |
| "f1": 0.6898563786378996, |
| "support": 41 |
| }, |
| "data_analysis": { |
| "f1": 0.315320475519074, |
| "support": 5 |
| }, |
| "writing": { |
| "f1": 0.8951125829152632, |
| "support": 705 |
| }, |
| "role_playing": { |
| "f1": 0.8753643299117938, |
| "support": 30 |
| }, |
| "advice_seeking": { |
| "f1": 0.6556990776879805, |
| "support": 75 |
| }, |
| "brainstorming": { |
| "f1": 0.6089745389657842, |
| "support": 83 |
| }, |
| "completion": { |
| "f1": 0.7600811270560087, |
| "support": 162 |
| }, |
| "action_execution": { |
| "f1": 0.47169812452618104, |
| "support": 8 |
| }, |
| "chat": { |
| "f1": 0.5755933848778737, |
| "support": 47 |
| }, |
| "other": { |
| "f1": 0.3704733753837275, |
| "support": 34 |
| } |
| } |
| }, |
| "needs_tools": { |
| "f1": 0.5546134214881696, |
| "auroc": 0.9544811115467666, |
| "positives": 153 |
| } |
| } |
| }, |
| "agreement_with_reference": { |
| "max_abs_logit_difference": 15.826028823852539, |
| "mean_abs_logit_difference": 1.3709417581558228, |
| "same_task": 0.8764753764753764, |
| "same_needs_tools": 0.9735449735449735, |
| "same_complexity_tier": 0.8650793650793651, |
| "same_policy_tier": 0.8715913715913716 |
| }, |
| "agreement_with_parent": { |
| "max_abs_logit_difference": 0.052532970905303955, |
| "mean_abs_logit_difference": 0.003039550269022584, |
| "same_task": 1.0, |
| "same_needs_tools": 1.0, |
| "same_complexity_tier": 0.9987789987789988, |
| "same_policy_tier": 0.9991859991859992 |
| }, |
| "latency_ms_batch1": { |
| "p50": 9.136207998380996, |
| "p95": 25.018457999976818, |
| "mean": 13.463028115475026, |
| "prompts": 200, |
| "device": "mps" |
| } |
| } |
