| { |
| "model": "Glint-Router-1M", |
| "author": "Glint-Research", |
| "parameters": 985826, |
| "baseline": "SupraLabs/Supra-Router-51M (51,786,240 params)", |
| "training": { |
| "params": 985826, |
| "train_rows": 87604, |
| "val_rows": 7617, |
| "steps": 14479, |
| "minutes": 15.0, |
| "domain": 0.9464356045687279, |
| "route": 0.939608769856899, |
| "complexity": 0.7923066824209006, |
| "code": 0.991466456610214, |
| "math": 0.978075357752396, |
| "reasoning": 0.8949717736641722, |
| "long_output": 0.9516870158855192, |
| "ece_raw": 0.05564934242829622, |
| "ece_calibrated": 0.02029288645098859, |
| "hardware": "RTX 3060 Ti", |
| "rounds": 1, |
| "wall_clock_minutes": 15.0, |
| "corpus_rows": 95221 |
| }, |
| "neutral_holdout": { |
| "val_rows": 3418, |
| "slice_rows": { |
| "aqua_rat": 254, |
| "dolly": 2500, |
| "humaneval": 164, |
| "mbpp": 500 |
| }, |
| "glint": { |
| "route_accuracy": 0.9441193680514921, |
| "code_accuracy": 0.9751316559391457, |
| "math_accuracy": 0.9526038619075483, |
| "complexity_exact": 0.46284376828554713, |
| "complexity_within_1": 0.9590403744880047, |
| "ms_per_prompt": 0.05992146021082586, |
| "parse_failures": 0 |
| }, |
| "glint_aqua_rat": { |
| "route_accuracy": 0.9094488188976378, |
| "code_accuracy": 0.9960629921259843, |
| "math_accuracy": 0.889763779527559, |
| "complexity_exact": 0.1141732283464567, |
| "complexity_within_1": 0.8070866141732284, |
| "ms_per_prompt": 0.05992146021082586, |
| "parse_failures": 0 |
| }, |
| "glint_dolly": { |
| "route_accuracy": 0.9424, |
| "code_accuracy": 0.9956, |
| "math_accuracy": 0.9612, |
| "complexity_exact": 0.5688, |
| "complexity_within_1": 0.9944, |
| "ms_per_prompt": 0.05992146021082586, |
| "parse_failures": 0 |
| }, |
| "glint_humaneval": { |
| "route_accuracy": 0.8719512195121951, |
| "code_accuracy": 0.5670731707317073, |
| "math_accuracy": 0.9024390243902439, |
| "complexity_exact": 0.4451219512195122, |
| "complexity_within_1": 0.9817073170731707, |
| "ms_per_prompt": 0.05992146021082586, |
| "parse_failures": 0 |
| }, |
| "glint_mbpp": { |
| "route_accuracy": 0.994, |
| "code_accuracy": 0.996, |
| "math_accuracy": 0.958, |
| "complexity_exact": 0.116, |
| "complexity_within_1": 0.852, |
| "ms_per_prompt": 0.05992146021082586, |
| "parse_failures": 0 |
| }, |
| "supra": { |
| "route_accuracy": 0.903744880046811, |
| "code_accuracy": 0.9643066120538326, |
| "math_accuracy": 0.9207138677589234, |
| "complexity_exact": 0.3905792861322411, |
| "complexity_within_1": 0.9163253364540667, |
| "ms_per_prompt": 14.334112498536923, |
| "parse_failures": 4 |
| }, |
| "supra_aqua_rat": { |
| "route_accuracy": 0.9133858267716536, |
| "code_accuracy": 1.0, |
| "math_accuracy": 0.9094488188976378, |
| "complexity_exact": 0.0, |
| "complexity_within_1": 0.09448818897637795, |
| "ms_per_prompt": 14.334112498536923, |
| "parse_failures": 4 |
| }, |
| "supra_dolly": { |
| "route_accuracy": 0.8896, |
| "code_accuracy": 0.996, |
| "math_accuracy": 0.9428, |
| "complexity_exact": 0.482, |
| "complexity_within_1": 0.9776, |
| "ms_per_prompt": 14.334112498536923, |
| "parse_failures": 4 |
| }, |
| "supra_humaneval": { |
| "route_accuracy": 0.8292682926829268, |
| "code_accuracy": 0.676829268292683, |
| "math_accuracy": 0.8414634146341463, |
| "complexity_exact": 0.16463414634146342, |
| "complexity_within_1": 1.0, |
| "ms_per_prompt": 14.334112498536923, |
| "parse_failures": 4 |
| }, |
| "supra_mbpp": { |
| "route_accuracy": 0.994, |
| "code_accuracy": 0.882, |
| "math_accuracy": 0.842, |
| "complexity_exact": 0.206, |
| "complexity_within_1": 1.0, |
| "ms_per_prompt": 14.334112498536923, |
| "parse_failures": 4 |
| } |
| }, |
| "in_corpus_val": { |
| "val_rows": 7617, |
| "glint": { |
| "route_accuracy": 0.939608769856899, |
| "code_accuracy": 0.991466456610214, |
| "math_accuracy": 0.978075357752396, |
| "complexity_exact": 0.7923066824209006, |
| "complexity_within_1": 0.9883156098201391, |
| "ms_per_prompt": 0.0748591950898926, |
| "parse_failures": 0 |
| }, |
| "glint_trap": { |
| "route_accuracy": 1.0, |
| "code_accuracy": 1.0, |
| "math_accuracy": 1.0, |
| "complexity_exact": 1.0, |
| "complexity_within_1": 1.0, |
| "parse_failures": 0 |
| }, |
| "supra": { |
| "route_accuracy": 0.8575554680320336, |
| "code_accuracy": 0.9761060785085992, |
| "math_accuracy": 0.9400026257056584, |
| "complexity_exact": 0.4629119075751608, |
| "complexity_within_1": 0.923723250623605, |
| "ms_per_prompt": 14.768008985689887, |
| "parse_failures": 1 |
| }, |
| "supra_trap": { |
| "route_accuracy": 0.908695652173913, |
| "code_accuracy": 0.9869565217391304, |
| "math_accuracy": 1.0, |
| "complexity_exact": 0.33043478260869563, |
| "complexity_within_1": 0.8391304347826087, |
| "parse_failures": 1 |
| }, |
| "note": "labels from the glint pipeline. glint is in-distribution here, supra is not. read the holdout numbers for the fair comparison." |
| }, |
| "supra_own_rows": { |
| "val_rows": 91, |
| "glint": { |
| "route_accuracy": 0.8901098901098901, |
| "code_accuracy": 0.9010989010989011, |
| "math_accuracy": 0.8131868131868132, |
| "complexity_exact": 0.5054945054945055 |
| }, |
| "supra": { |
| "route_accuracy": 0.967032967032967, |
| "code_accuracy": 1.0, |
| "math_accuracy": 0.978021978021978, |
| "complexity_exact": 0.8571428571428571 |
| }, |
| "note": "supra's own 992-row dataset, held out of glint training. these rows are supra's TRAINING data, so supra is scoring a memorised set. published because leaving it out would be dishonest." |
| } |
| } |
