| { |
| "variant": "v4_vocab32k-fp16", |
| "backend": "torch", |
| "size_bytes": 307846551, |
| "validation_all": { |
| "unweighted": { |
| "rows": 4914, |
| "weighted": false, |
| "complexity": { |
| "rows": 4850, |
| "mae": 0.07579664213927752, |
| "spearman": 0.7329635072519433, |
| "qwk_rounded": 0.7519040607655082, |
| "confusion_true_by_pred": [ |
| [ |
| 145.0, |
| 157.0, |
| 6.0, |
| 0.0, |
| 0.0 |
| ], |
| [ |
| 39.0, |
| 2574.0, |
| 263.0, |
| 2.0, |
| 0.0 |
| ], |
| [ |
| 4.0, |
| 406.0, |
| 880.0, |
| 55.0, |
| 0.0 |
| ], |
| [ |
| 1.0, |
| 10.0, |
| 76.0, |
| 224.0, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 0.0, |
| 1.0, |
| 3.0, |
| 4.0 |
| ] |
| ], |
| "support": { |
| "trivial": 308, |
| "low": 2878, |
| "medium": 1345, |
| "high": 311, |
| "expert": 8 |
| } |
| }, |
| "task": { |
| "accuracy": 0.8398453398453398, |
| "macro_f1": 0.7496592483362028, |
| "macro_f1_present": 0.7496592483362028, |
| "per_class": { |
| "information_seeking": { |
| "f1": 0.8687520154788778, |
| "support": 1528 |
| }, |
| "reasoning": { |
| "f1": 0.5802469135802469, |
| "support": 92 |
| }, |
| "planning": { |
| "f1": 0.7132867132867133, |
| "support": 81 |
| }, |
| "editing": { |
| "f1": 0.8702290076335878, |
| "support": 193 |
| }, |
| "summarization": { |
| "f1": 0.8605577689243028, |
| "support": 128 |
| }, |
| "extraction": { |
| "f1": 0.7982832618025751, |
| "support": 229 |
| }, |
| "classification": { |
| "f1": 0.8901098901098901, |
| "support": 89 |
| }, |
| "translation": { |
| "f1": 0.9103448275862069, |
| "support": 73 |
| }, |
| "coding_debugging": { |
| "f1": 0.8940397350993378, |
| "support": 453 |
| }, |
| "math": { |
| "f1": 0.8767676767676768, |
| "support": 236 |
| }, |
| "data_analysis": { |
| "f1": 0.7058823529411765, |
| "support": 22 |
| }, |
| "writing": { |
| "f1": 0.9045936395759717, |
| "support": 991 |
| }, |
| "role_playing": { |
| "f1": 0.8771929824561403, |
| "support": 83 |
| }, |
| "advice_seeking": { |
| "f1": 0.7138263665594855, |
| "support": 157 |
| }, |
| "brainstorming": { |
| "f1": 0.6024096385542169, |
| "support": 134 |
| }, |
| "completion": { |
| "f1": 0.764378478664193, |
| "support": 267 |
| }, |
| "action_execution": { |
| "f1": 0.5263157894736842, |
| "support": 11 |
| }, |
| "chat": { |
| "f1": 0.5786163522012578, |
| "support": 91 |
| }, |
| "other": { |
| "f1": 0.3076923076923077, |
| "support": 56 |
| } |
| } |
| }, |
| "needs_tools": { |
| "f1": 0.6178489702517163, |
| "auroc": 0.9481470840480274, |
| "positives": 250 |
| } |
| }, |
| "weighted": { |
| "rows": 4914, |
| "weighted": true, |
| "complexity": { |
| "rows": 4850, |
| "mae": 0.0734992234917565, |
| "spearman": 0.6931422117949425, |
| "qwk_rounded": 0.7000283091495797, |
| "confusion_true_by_pred": [ |
| [ |
| 138.9, |
| 154.5, |
| 5.0, |
| 0.0, |
| 0.0 |
| ], |
| [ |
| 41.0, |
| 2767.2, |
| 283.8, |
| 2.2, |
| 0.0 |
| ], |
| [ |
| 4.1, |
| 408.0, |
| 884.3, |
| 54.9, |
| 0.0 |
| ], |
| [ |
| 0.4, |
| 4.1, |
| 32.3, |
| 94.6, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 0.0, |
| 0.1, |
| 0.4, |
| 0.6 |
| ] |
| ], |
| "support": { |
| "trivial": 308, |
| "low": 2878, |
| "medium": 1345, |
| "high": 311, |
| "expert": 8 |
| } |
| }, |
| "task": { |
| "accuracy": 0.8417469271956327, |
| "macro_f1": 0.7361416383741431, |
| "macro_f1_present": 0.7361416383741431, |
| "per_class": { |
| "information_seeking": { |
| "f1": 0.8743961299148704, |
| "support": 1528 |
| }, |
| "reasoning": { |
| "f1": 0.5876168321012015, |
| "support": 92 |
| }, |
| "planning": { |
| "f1": 0.6923419444052038, |
| "support": 81 |
| }, |
| "editing": { |
| "f1": 0.8750917385680073, |
| "support": 193 |
| }, |
| "summarization": { |
| "f1": 0.8617685118085122, |
| "support": 128 |
| }, |
| "extraction": { |
| "f1": 0.8024884010513994, |
| "support": 229 |
| }, |
| "classification": { |
| "f1": 0.8889433148815616, |
| "support": 89 |
| }, |
| "translation": { |
| "f1": 0.9102897416990978, |
| "support": 73 |
| }, |
| "coding_debugging": { |
| "f1": 0.8951849640199118, |
| "support": 453 |
| }, |
| "math": { |
| "f1": 0.8720808285568271, |
| "support": 236 |
| }, |
| "data_analysis": { |
| "f1": 0.6036172496413325, |
| "support": 22 |
| }, |
| "writing": { |
| "f1": 0.8998060042441275, |
| "support": 991 |
| }, |
| "role_playing": { |
| "f1": 0.856782491891807, |
| "support": 83 |
| }, |
| "advice_seeking": { |
| "f1": 0.7267918392567481, |
| "support": 157 |
| }, |
| "brainstorming": { |
| "f1": 0.6040013806205294, |
| "support": 134 |
| }, |
| "completion": { |
| "f1": 0.7735029017960995, |
| "support": 267 |
| }, |
| "action_execution": { |
| "f1": 0.4050722792506393, |
| "support": 11 |
| }, |
| "chat": { |
| "f1": 0.5820792184544086, |
| "support": 91 |
| }, |
| "other": { |
| "f1": 0.27483535694643446, |
| "support": 56 |
| } |
| } |
| }, |
| "needs_tools": { |
| "f1": 0.6202334654791641, |
| "auroc": 0.9483244764486874, |
| "positives": 250 |
| } |
| } |
| }, |
| "validation_four_sources": { |
| "unweighted": { |
| "rows": 3130, |
| "weighted": false, |
| "complexity": { |
| "rows": 3101, |
| "mae": 0.0732659558825476, |
| "spearman": 0.7434607091466564, |
| "qwk_rounded": 0.7590494324005435, |
| "confusion_true_by_pred": [ |
| [ |
| 100.0, |
| 97.0, |
| 4.0, |
| 0.0, |
| 0.0 |
| ], |
| [ |
| 25.0, |
| 1653.0, |
| 160.0, |
| 0.0, |
| 0.0 |
| ], |
| [ |
| 3.0, |
| 251.0, |
| 580.0, |
| 38.0, |
| 0.0 |
| ], |
| [ |
| 1.0, |
| 4.0, |
| 46.0, |
| 138.0, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 1.0 |
| ] |
| ], |
| "support": { |
| "trivial": 201, |
| "low": 1838, |
| "medium": 872, |
| "high": 189, |
| "expert": 1 |
| } |
| }, |
| "task": { |
| "accuracy": 0.8488817891373802, |
| "macro_f1": 0.7513423725525952, |
| "macro_f1_present": 0.7513423725525952, |
| "per_class": { |
| "information_seeking": { |
| "f1": 0.8823257960313798, |
| "support": 1068 |
| }, |
| "reasoning": { |
| "f1": 0.5977011494252874, |
| "support": 48 |
| }, |
| "planning": { |
| "f1": 0.651685393258427, |
| "support": 53 |
| }, |
| "editing": { |
| "f1": 0.8817204301075269, |
| "support": 139 |
| }, |
| "summarization": { |
| "f1": 0.8631578947368421, |
| "support": 98 |
| }, |
| "extraction": { |
| "f1": 0.819672131147541, |
| "support": 180 |
| }, |
| "classification": { |
| "f1": 0.875, |
| "support": 47 |
| }, |
| "translation": { |
| "f1": 0.9166666666666666, |
| "support": 61 |
| }, |
| "coding_debugging": { |
| "f1": 0.9020408163265307, |
| "support": 246 |
| }, |
| "math": { |
| "f1": 0.7415730337078652, |
| "support": 41 |
| }, |
| "data_analysis": { |
| "f1": 0.6666666666666666, |
| "support": 5 |
| }, |
| "writing": { |
| "f1": 0.9049964813511612, |
| "support": 705 |
| }, |
| "role_playing": { |
| "f1": 0.9152542372881356, |
| "support": 30 |
| }, |
| "advice_seeking": { |
| "f1": 0.6901408450704225, |
| "support": 75 |
| }, |
| "brainstorming": { |
| "f1": 0.5660377358490566, |
| "support": 83 |
| }, |
| "completion": { |
| "f1": 0.7818181818181819, |
| "support": 162 |
| }, |
| "action_execution": { |
| "f1": 0.625, |
| "support": 8 |
| }, |
| "chat": { |
| "f1": 0.6190476190476191, |
| "support": 47 |
| }, |
| "other": { |
| "f1": 0.375, |
| "support": 34 |
| } |
| } |
| }, |
| "needs_tools": { |
| "f1": 0.5808823529411765, |
| "auroc": 0.960451698314529, |
| "positives": 153 |
| } |
| }, |
| "weighted": { |
| "rows": 3130, |
| "weighted": true, |
| "complexity": { |
| "rows": 3101, |
| "mae": 0.07136375761254943, |
| "spearman": 0.7050611798721679, |
| "qwk_rounded": 0.7112237182045522, |
| "confusion_true_by_pred": [ |
| [ |
| 95.6, |
| 96.6, |
| 3.5, |
| 0.0, |
| 0.0 |
| ], |
| [ |
| 26.1, |
| 1790.5, |
| 174.8, |
| 0.0, |
| 0.0 |
| ], |
| [ |
| 3.1, |
| 254.4, |
| 589.0, |
| 37.6, |
| 0.0 |
| ], |
| [ |
| 0.4, |
| 1.6, |
| 19.7, |
| 58.1, |
| 0.0 |
| ], |
| [ |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.1 |
| ] |
| ], |
| "support": { |
| "trivial": 201, |
| "low": 1838, |
| "medium": 872, |
| "high": 189, |
| "expert": 1 |
| } |
| }, |
| "task": { |
| "accuracy": 0.8510913634686621, |
| "macro_f1": 0.7336388186360535, |
| "macro_f1_present": 0.7336388186360535, |
| "per_class": { |
| "information_seeking": { |
| "f1": 0.8863005064157066, |
| "support": 1068 |
| }, |
| "reasoning": { |
| "f1": 0.6007655619943084, |
| "support": 48 |
| }, |
| "planning": { |
| "f1": 0.6322924335863169, |
| "support": 53 |
| }, |
| "editing": { |
| "f1": 0.8872579498056196, |
| "support": 139 |
| }, |
| "summarization": { |
| "f1": 0.8655264865753218, |
| "support": 98 |
| }, |
| "extraction": { |
| "f1": 0.8217981441939354, |
| "support": 180 |
| }, |
| "classification": { |
| "f1": 0.8744675021779321, |
| "support": 47 |
| }, |
| "translation": { |
| "f1": 0.9165540456122309, |
| "support": 61 |
| }, |
| "coding_debugging": { |
| "f1": 0.9014211381493499, |
| "support": 246 |
| }, |
| "math": { |
| "f1": 0.7206863251222526, |
| "support": 41 |
| }, |
| "data_analysis": { |
| "f1": 0.5512676749948888, |
| "support": 5 |
| }, |
| "writing": { |
| "f1": 0.9022412453550238, |
| "support": 705 |
| }, |
| "role_playing": { |
| "f1": 0.9018366831956603, |
| "support": 30 |
| }, |
| "advice_seeking": { |
| "f1": 0.7058665577471023, |
| "support": 75 |
| }, |
| "brainstorming": { |
| "f1": 0.5672344732772423, |
| "support": 83 |
| }, |
| "completion": { |
| "f1": 0.7921253431420334, |
| "support": 162 |
| }, |
| "action_execution": { |
| "f1": 0.46184553221044683, |
| "support": 8 |
| }, |
| "chat": { |
| "f1": 0.6134643357166086, |
| "support": 47 |
| }, |
| "other": { |
| "f1": 0.3361856148130364, |
| "support": 34 |
| } |
| } |
| }, |
| "needs_tools": { |
| "f1": 0.5784521352010515, |
| "auroc": 0.9630510085880657, |
| "positives": 153 |
| } |
| } |
| }, |
| "agreement_with_reference": { |
| "max_abs_logit_difference": 16.74053192138672, |
| "mean_abs_logit_difference": 0.7664107084274292, |
| "same_task": 0.9171754171754172, |
| "same_needs_tools": 0.9816849816849816, |
| "same_complexity_tier": 0.9092389092389093, |
| "same_policy_tier": 0.914936914936915 |
| }, |
| "agreement_with_parent": { |
| "max_abs_logit_difference": 0.9007568359375, |
| "mean_abs_logit_difference": 0.0033105933107435703, |
| "same_task": 0.9995929995929996, |
| "same_needs_tools": 1.0, |
| "same_complexity_tier": 0.9993894993894994, |
| "same_policy_tier": 0.9993894993894994 |
| }, |
| "latency_ms_batch1": { |
| "p50": 10.002875002101064, |
| "p95": 38.70008300145855, |
| "mean": 14.027030840043153, |
| "prompts": 200, |
| "device": "mps" |
| } |
| } |
