{ "meta": { "runs_total": 67, "final_runs": 26, "curves_runs": 20, "ksweep_runs": 21, "baseline_runs": 3, "baseline_records_excluded_total": 3, "records_excluded_total": 271, "exclusions_per_run": { "final-gemma-q4-dflash-f16": { "total": 1473, "kept": 1455, "excluded": 18 }, "final-gemma-q4-dflash-q4": { "total": 1473, "kept": 1455, "excluded": 18 }, "final-gemma-q4-dflash-q8": { "total": 1473, "kept": 1455, "excluded": 18 }, "final-gemma-q4-mtp": { "total": 1473, "kept": 1455, "excluded": 18 }, "final-gemma-q4-solo": { "total": 1473, "kept": 1455, "excluded": 18 }, "final-gemma-q5-dflash-f16": { "total": 1474, "kept": 1449, "excluded": 25 }, "final-gemma-q5-mtp": { "total": 1474, "kept": 1449, "excluded": 25 }, "final-gemma-q5-solo": { "total": 1474, "kept": 1449, "excluded": 25 }, "final-gemma-q8-dflash-f16": { "total": 1472, "kept": 1452, "excluded": 20 }, "final-gemma-q8-mtp": { "total": 1472, "kept": 1452, "excluded": 20 }, "final-gemma-q8-solo": { "total": 1472, "kept": 1452, "excluded": 20 }, "final-qwen-q4-dflash": { "total": 1474, "kept": 1474, "excluded": 0 }, "final-qwen-q4-dspark-p0": { "total": 1474, "kept": 1474, "excluded": 0 }, "final-qwen-q4-dspark-p2": { "total": 1474, "kept": 1474, "excluded": 0 }, "final-qwen-q4-dspark-p4": { "total": 1474, "kept": 1474, "excluded": 0 }, "final-qwen-q4-dspark-p6": { "total": 1474, "kept": 1474, "excluded": 0 }, "final-qwen-q4-eagle3": { "total": 1474, "kept": 1474, "excluded": 0 }, "final-qwen-q4-solo": { "total": 1474, "kept": 1474, "excluded": 0 }, "final-qwen-q4-vanilla17b": { "total": 1474, "kept": 1474, "excluded": 0 }, "final-qwen-q5-dflash": { "total": 1474, "kept": 1474, "excluded": 0 }, "final-qwen-q5-dspark-p0": { "total": 1474, "kept": 1474, "excluded": 0 }, "final-qwen-q5-eagle3": { "total": 1474, "kept": 1474, "excluded": 0 }, "final-qwen-q5-solo": { "total": 1474, "kept": 1474, "excluded": 0 }, "final-qwen-q8-dspark-p0": { "total": 1474, "kept": 1474, "excluded": 0 }, "final-qwen-q8-eagle3": { "total": 1474, "kept": 1474, "excluded": 0 }, "final-qwen-q8-solo": { "total": 1474, "kept": 1474, "excluded": 0 } }, "log_mapping": { "curves-gemma-q4-dflash-f16": { "log_lines": 175, "n_matched": 175, "max_alpha_diff": 4.999999999999449e-05 }, "curves-gemma-q4-dflash-q4": { "log_lines": 175, "n_matched": 175, "max_alpha_diff": 4.999999999999449e-05 }, "curves-gemma-q4-dflash-q8": { "log_lines": 175, "n_matched": 175, "max_alpha_diff": 4.999999999999449e-05 }, "curves-gemma-q4-mtp": { "log_lines": 175, "n_matched": 175, "max_alpha_diff": 4.999999999999449e-05 }, "curves-gemma-q5-dflash-f16": { "log_lines": 174, "n_matched": 174, "max_alpha_diff": 5.0000000000050004e-05 }, "curves-gemma-q5-mtp": { "log_lines": 174, "n_matched": 174, "max_alpha_diff": 4.999999999999449e-05 }, "curves-gemma-q8-dflash-f16": { "log_lines": 174, "n_matched": 174, "max_alpha_diff": 5.0000000000050004e-05 }, "curves-gemma-q8-mtp": { "log_lines": 174, "n_matched": 174, "max_alpha_diff": 4.999999999999449e-05 }, "curves-qwen-q4-dflash": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.0000000000000565e-05 }, "curves-qwen-q4-dspark-p0": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 4.999999999999449e-05 }, "curves-qwen-q4-dspark-p2": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 4.999999999999449e-05 }, "curves-qwen-q4-dspark-p4": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.0000000000105516e-05 }, "curves-qwen-q4-dspark-p6": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.0000000000105516e-05 }, "curves-qwen-q4-eagle3": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.0000000000050004e-05 }, "curves-qwen-q4-vanilla17b": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 4.999999999999449e-05 }, "curves-qwen-q5-dflash": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.0000000000000565e-05 }, "curves-qwen-q5-dspark-p0": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 4.999999999999449e-05 }, "curves-qwen-q5-eagle3": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.000000000000837e-05 }, "curves-qwen-q8-dspark-p0": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.000000000002225e-05 }, "curves-qwen-q8-eagle3": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.0000000000050004e-05 }, "ksweep-gemma-q4-dflash-f16-k10": { "log_lines": 175, "n_matched": 175, "max_alpha_diff": 5.0000000000050004e-05 }, "ksweep-gemma-q4-dflash-f16-k5": { "log_lines": 175, "n_matched": 175, "max_alpha_diff": 4.999999999999449e-05 }, "ksweep-gemma-q4-dflash-f16-k7": { "log_lines": 175, "n_matched": 175, "max_alpha_diff": 5.0000000000050004e-05 }, "ksweep-gemma-q4-mtp-k10": { "log_lines": 175, "n_matched": 175, "max_alpha_diff": 5.0000000000105516e-05 }, "ksweep-gemma-q4-mtp-k5": { "log_lines": 175, "n_matched": 175, "max_alpha_diff": 5.0000000000050004e-05 }, "ksweep-gemma-q4-mtp-k7": { "log_lines": 175, "n_matched": 175, "max_alpha_diff": 5.0000000000050004e-05 }, "ksweep-qwen-q4-dflash-k10": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.000000000000143e-05 }, "ksweep-qwen-q4-dflash-k5": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 4.99999999999997e-05 }, "ksweep-qwen-q4-dflash-k7": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.000000000000143e-05 }, "ksweep-qwen-q4-dspark-p0-k10": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.0000000000105516e-05 }, "ksweep-qwen-q4-dspark-p0-k5": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.0000000000050004e-05 }, "ksweep-qwen-q4-dspark-p0-k7": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.0000000000105516e-05 }, "ksweep-qwen-q4-eagle3-k10": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.000000000002225e-05 }, "ksweep-qwen-q4-eagle3-k5": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.0000000000050004e-05 }, "ksweep-qwen-q4-eagle3-k7": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.000000000002225e-05 }, "ksweep-qwen-q4-vanilla17b-k10": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.000000000002225e-05 }, "ksweep-qwen-q4-vanilla17b-k5": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.0000000000050004e-05 }, "ksweep-qwen-q4-vanilla17b-k7": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.0000000000105516e-05 }, "ksweep-qwen-q8-eagle3-k10": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.000000000002225e-05 }, "ksweep-qwen-q8-eagle3-k5": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 4.999999999999449e-05 }, "ksweep-qwen-q8-eagle3-k7": { "log_lines": 178, "n_matched": 178, "max_alpha_diff": 5.000000000002225e-05 } }, "csv_files": { "curves-gemma-q4-dflash-f16": { "csv": "curves/acc_by_pos_curves-gemma-q4-dflash-f16.csv", "k": 3, "n_matched": 175 }, "curves-gemma-q4-dflash-q4": { "csv": "curves/acc_by_pos_curves-gemma-q4-dflash-q4.csv", "k": 3, "n_matched": 175 }, "curves-gemma-q4-dflash-q8": { "csv": "curves/acc_by_pos_curves-gemma-q4-dflash-q8.csv", "k": 3, "n_matched": 175 }, "curves-gemma-q4-mtp": { "csv": "curves/acc_by_pos_curves-gemma-q4-mtp.csv", "k": 3, "n_matched": 175 }, "curves-gemma-q5-dflash-f16": { "csv": "curves/acc_by_pos_curves-gemma-q5-dflash-f16.csv", "k": 3, "n_matched": 174 }, "curves-gemma-q5-mtp": { "csv": "curves/acc_by_pos_curves-gemma-q5-mtp.csv", "k": 3, "n_matched": 174 }, "curves-gemma-q8-dflash-f16": { "csv": "curves/acc_by_pos_curves-gemma-q8-dflash-f16.csv", "k": 3, "n_matched": 174 }, "curves-gemma-q8-mtp": { "csv": "curves/acc_by_pos_curves-gemma-q8-mtp.csv", "k": 3, "n_matched": 174 }, "curves-qwen-q4-dflash": { "csv": "curves/acc_by_pos_curves-qwen-q4-dflash.csv", "k": 3, "n_matched": 178 }, "curves-qwen-q4-dspark-p0": { "csv": "curves/acc_by_pos_curves-qwen-q4-dspark-p0.csv", "k": 3, "n_matched": 178 }, "curves-qwen-q4-dspark-p2": { "csv": "curves/acc_by_pos_curves-qwen-q4-dspark-p2.csv", "k": 3, "n_matched": 178 }, "curves-qwen-q4-dspark-p4": { "csv": "curves/acc_by_pos_curves-qwen-q4-dspark-p4.csv", "k": 3, "n_matched": 178 }, "curves-qwen-q4-dspark-p6": { "csv": "curves/acc_by_pos_curves-qwen-q4-dspark-p6.csv", "k": 3, "n_matched": 178 }, "curves-qwen-q4-eagle3": { "csv": "curves/acc_by_pos_curves-qwen-q4-eagle3.csv", "k": 3, "n_matched": 178 }, "curves-qwen-q4-vanilla17b": { "csv": "curves/acc_by_pos_curves-qwen-q4-vanilla17b.csv", "k": 3, "n_matched": 178 }, "curves-qwen-q5-dflash": { "csv": "curves/acc_by_pos_curves-qwen-q5-dflash.csv", "k": 3, "n_matched": 178 }, "curves-qwen-q5-dspark-p0": { "csv": "curves/acc_by_pos_curves-qwen-q5-dspark-p0.csv", "k": 3, "n_matched": 178 }, "curves-qwen-q5-eagle3": { "csv": "curves/acc_by_pos_curves-qwen-q5-eagle3.csv", "k": 3, "n_matched": 178 }, "curves-qwen-q8-dspark-p0": { "csv": "curves/acc_by_pos_curves-qwen-q8-dspark-p0.csv", "k": 3, "n_matched": 178 }, "curves-qwen-q8-eagle3": { "csv": "curves/acc_by_pos_curves-qwen-q8-eagle3.csv", "k": 3, "n_matched": 178 }, "ksweep-gemma-q4-dflash-f16-k10": { "csv": "curves/acc_by_pos_ksweep-gemma-q4-dflash-f16-k10.csv", "k": 10, "n_matched": 175 }, "ksweep-gemma-q4-dflash-f16-k5": { "csv": "curves/acc_by_pos_ksweep-gemma-q4-dflash-f16-k5.csv", "k": 5, "n_matched": 175 }, "ksweep-gemma-q4-dflash-f16-k7": { "csv": "curves/acc_by_pos_ksweep-gemma-q4-dflash-f16-k7.csv", "k": 7, "n_matched": 175 }, "ksweep-gemma-q4-mtp-k10": { "csv": "curves/acc_by_pos_ksweep-gemma-q4-mtp-k10.csv", "k": 10, "n_matched": 175 }, "ksweep-gemma-q4-mtp-k5": { "csv": "curves/acc_by_pos_ksweep-gemma-q4-mtp-k5.csv", "k": 5, "n_matched": 175 }, "ksweep-gemma-q4-mtp-k7": { "csv": "curves/acc_by_pos_ksweep-gemma-q4-mtp-k7.csv", "k": 7, "n_matched": 175 }, "ksweep-qwen-q4-dflash-k10": { "csv": "curves/acc_by_pos_ksweep-qwen-q4-dflash-k10.csv", "k": 10, "n_matched": 178 }, "ksweep-qwen-q4-dflash-k5": { "csv": "curves/acc_by_pos_ksweep-qwen-q4-dflash-k5.csv", "k": 5, "n_matched": 178 }, "ksweep-qwen-q4-dflash-k7": { "csv": "curves/acc_by_pos_ksweep-qwen-q4-dflash-k7.csv", "k": 7, "n_matched": 178 }, "ksweep-qwen-q4-dspark-p0-k10": { "csv": "curves/acc_by_pos_ksweep-qwen-q4-dspark-p0-k10.csv", "k": 10, "n_matched": 178 }, "ksweep-qwen-q4-dspark-p0-k5": { "csv": "curves/acc_by_pos_ksweep-qwen-q4-dspark-p0-k5.csv", "k": 5, "n_matched": 178 }, "ksweep-qwen-q4-dspark-p0-k7": { "csv": "curves/acc_by_pos_ksweep-qwen-q4-dspark-p0-k7.csv", "k": 7, "n_matched": 178 }, "ksweep-qwen-q4-eagle3-k10": { "csv": "curves/acc_by_pos_ksweep-qwen-q4-eagle3-k10.csv", "k": 10, "n_matched": 178 }, "ksweep-qwen-q4-eagle3-k5": { "csv": "curves/acc_by_pos_ksweep-qwen-q4-eagle3-k5.csv", "k": 5, "n_matched": 178 }, "ksweep-qwen-q4-eagle3-k7": { "csv": "curves/acc_by_pos_ksweep-qwen-q4-eagle3-k7.csv", "k": 7, "n_matched": 178 }, "ksweep-qwen-q4-vanilla17b-k10": { "csv": "curves/acc_by_pos_ksweep-qwen-q4-vanilla17b-k10.csv", "k": 10, "n_matched": 178 }, "ksweep-qwen-q4-vanilla17b-k5": { "csv": "curves/acc_by_pos_ksweep-qwen-q4-vanilla17b-k5.csv", "k": 5, "n_matched": 178 }, "ksweep-qwen-q4-vanilla17b-k7": { "csv": "curves/acc_by_pos_ksweep-qwen-q4-vanilla17b-k7.csv", "k": 7, "n_matched": 178 }, "ksweep-qwen-q8-eagle3-k10": { "csv": "curves/acc_by_pos_ksweep-qwen-q8-eagle3-k10.csv", "k": 10, "n_matched": 178 }, "ksweep-qwen-q8-eagle3-k5": { "csv": "curves/acc_by_pos_ksweep-qwen-q8-eagle3-k5.csv", "k": 5, "n_matched": 178 }, "ksweep-qwen-q8-eagle3-k7": { "csv": "curves/acc_by_pos_ksweep-qwen-q8-eagle3-k7.csv", "k": 7, "n_matched": 178 } }, "method": "sentinel exclusion: tok_per_s >= 1e5 or predicted_ms <= 0; log<->record by order of appearance; OLS alpha_be per paper #32" }, "final": { "final-gemma-q4-dflash-f16": { "run": "final-gemma-q4-dflash-f16", "target": "gemma-q4", "family": "gemma", "quant": "q4", "drafter": "dflash-f16", "k": 3, "ctx": 16384, "n": 1455, "tok_per_s": { "mean": 72.2039910652921, "median": 75.462, "p95": 88.8179 }, "alpha": { "mean": 0.6566116838487973, "median": 0.7033 }, "tau_mean": 156.13264604810996, "ttft_ms": { "mean": 178.2487972508591, "median": 91.0, "p95": 558.1899999999999 }, "vram_max_mib": 10766, "power_max_w": 157.97, "duration_s": 29.1, "errors": 1, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 80.84256511627908, "median": 81.82149999999999, "p95": 88.83569999999999 }, "alpha": { "mean": 0.7758225581395349, "median": 0.7851 }, "tau_mean": 174.85581395348837, "speedup": { "mean": 2.33136130709954, "median": 2.3507745326695653, "agg": 2.332358114519116, "n_match": 430 } }, "code": { "n": 555, "tok_per_s": { "mean": 76.32085045045045, "median": 76.781, "p95": 90.40169999999999 }, "alpha": { "mean": 0.718401981981982, "median": 0.722 }, "tau_mean": 167.1873873873874, "speedup": { "mean": 2.2174172083676087, "median": 2.2194261119081777, "agg": 2.2019038435887417, "n_match": 555 } }, "chat": { "n": 470, "tok_per_s": { "mean": 59.4392170212766, "median": 58.281499999999994, "p95": 83.13255 }, "alpha": { "mean": 0.4745812765957446, "median": 0.46094999999999997 }, "tau_mean": 125.94893617021276, "speedup": { "mean": 1.7100611460880542, "median": 1.6710309129443266, "agg": 1.7148582549407616, "n_match": 470 } } }, "speedup": { "mean": 2.0872031968097664, "median": 2.185492473628067, "agg": 2.0831298984585085, "n_match": 1455 } }, "final-gemma-q4-dflash-q4": { "run": "final-gemma-q4-dflash-q4", "target": "gemma-q4", "family": "gemma", "quant": "q4", "drafter": "dflash-q4", "k": 3, "ctx": 16384, "n": 1455, "tok_per_s": { "mean": 78.35855463917525, "median": 81.919, "p95": 97.1592 }, "alpha": { "mean": 0.64725470790378, "median": 0.6964 }, "tau_mean": 155.11340206185568, "ttft_ms": { "mean": 177.08604810996565, "median": 90.0, "p95": 555.7299999999999 }, "vram_max_mib": 9730, "power_max_w": 164.05, "duration_s": 22.82, "errors": 1, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 88.11804186046513, "median": 88.844, "p95": 97.2851 }, "alpha": { "mean": 0.7701983720930232, "median": 0.7807 }, "tau_mean": 174.37441860465117, "speedup": { "mean": 2.541166057205258, "median": 2.5700507801313335, "agg": 2.5422601283516837, "n_match": 430 } }, "code": { "n": 555, "tok_per_s": { "mean": 82.80627207207208, "median": 83.517, "p95": 99.07239999999999 }, "alpha": { "mean": 0.708456036036036, "median": 0.716 }, "tau_mean": 166.16936936936938, "speedup": { "mean": 2.405611106727141, "median": 2.420483478758404, "agg": 2.389012277413301, "n_match": 555 } }, "chat": { "n": 470, "tok_per_s": { "mean": 64.17757021276596, "median": 62.519999999999996, "p95": 90.13965 }, "alpha": { "mean": 0.4625046808510639, "median": 0.4477 }, "tau_mean": 124.43617021276596, "speedup": { "mean": 1.8461266568689085, "median": 1.808100534428661, "agg": 1.8515626816215813, "n_match": 470 } } }, "speedup": { "mean": 2.264945084233822, "median": 2.378694381292627, "agg": 2.2606928725207895, "n_match": 1455 } }, "final-gemma-q4-dflash-q8": { "run": "final-gemma-q4-dflash-q8", "target": "gemma-q4", "family": "gemma", "quant": "q4", "drafter": "dflash-q8", "k": 3, "ctx": 16384, "n": 1455, "tok_per_s": { "mean": 76.57275051546392, "median": 80.034, "p95": 94.3417 }, "alpha": { "mean": 0.6557245360824743, "median": 0.7033 }, "tau_mean": 156.05223367697596, "ttft_ms": { "mean": 177.27353951890035, "median": 90.2, "p95": 552.8999999999999 }, "vram_max_mib": 10043, "power_max_w": 161.39, "duration_s": 24.26, "errors": 1, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 85.73577674418605, "median": 86.5975, "p95": 94.36255 }, "alpha": { "mean": 0.7747402325581394, "median": 0.7851 }, "tau_mean": 174.77209302325582, "speedup": { "mean": 2.472454081682774, "median": 2.490608393466564, "agg": 2.4735303030808318, "n_match": 430 } }, "code": { "n": 555, "tok_per_s": { "mean": 80.98567207207208, "median": 81.217, "p95": 96.22209999999998 }, "alpha": { "mean": 0.7176104504504505, "median": 0.719 }, "tau_mean": 167.13693693693693, "speedup": { "mean": 2.3529290187964063, "median": 2.351740524447498, "agg": 2.336486838899742, "n_match": 555 } }, "chat": { "n": 470, "tok_per_s": { "mean": 62.97855319148936, "median": 61.8125, "p95": 88.3801 }, "alpha": { "mean": 0.47375978723404255, "median": 0.45865 }, "tau_mean": 125.83617021276596, "speedup": { "mean": 1.811936448286366, "median": 1.7707875599146599, "agg": 1.8169702973374033, "n_match": 470 } } }, "speedup": { "mean": 2.2134989630585498, "median": 2.3199483722931307, "agg": 2.2091713166066627, "n_match": 1455 } }, "final-gemma-q4-mtp": { "run": "final-gemma-q4-mtp", "target": "gemma-q4", "family": "gemma", "quant": "q4", "drafter": "mtp", "k": 3, "ctx": 16384, "n": 1455, "tok_per_s": { "mean": 82.66167628865979, "median": 85.34, "p95": 97.2279 }, "alpha": { "mean": 0.7388333333333333, "median": 0.7773 }, "tau_mean": 163.71546391752577, "ttft_ms": { "mean": 164.81738831615118, "median": 85.4, "p95": 504.21999999999997 }, "vram_max_mib": 9135, "power_max_w": 164.41, "duration_s": 19.91, "errors": 1, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 89.76367209302326, "median": 90.7055, "p95": 97.2355 }, "alpha": { "mean": 0.8303902325581396, "median": 0.8472 }, "tau_mean": 178.76511627906976, "speedup": { "mean": 2.5891424623179917, "median": 2.6083989781984407, "agg": 2.5897375806181264, "n_match": 430 } }, "code": { "n": 555, "tok_per_s": { "mean": 85.64304684684686, "median": 85.717, "p95": 97.7571 }, "alpha": { "mean": 0.7827306306306306, "median": 0.7841 }, "tau_mean": 172.36756756756756, "speedup": { "mean": 2.4891408054990882, "median": 2.4961926737813673, "agg": 2.470854988063223, "n_match": 555 } }, "chat": { "n": 470, "tok_per_s": { "mean": 72.6435510638298, "median": 71.071, "p95": 95.61234999999998 }, "alpha": { "mean": 0.603232340425532, "median": 0.58805 }, "tau_mean": 139.72978723404256, "speedup": { "mean": 2.0902575697224277, "median": 2.049779416756877, "agg": 2.095811476881123, "n_match": 470 } } }, "speedup": { "mean": 2.3898456794627294, "median": 2.4744594439995424, "agg": 2.3848405994329966, "n_match": 1455 } }, "final-gemma-q4-solo": { "run": "final-gemma-q4-solo", "target": "gemma-q4", "family": "gemma", "quant": "q4", "drafter": "solo", "k": 3, "ctx": 16384, "n": 1455, "tok_per_s": { "mean": 34.661300343642615, "median": 34.829, "p95": 35.0786 }, "alpha": null, "tau_mean": null, "ttft_ms": { "mean": 148.97491408934707, "median": 83.1, "p95": 426.55999999999995 }, "vram_max_mib": 8616, "power_max_w": 164.21, "duration_s": 31.91, "errors": 1, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 34.68004651162791, "median": 34.831500000000005, "p95": 34.930099999999996 }, "alpha": null, "tau_mean": null }, "code": { "n": 555, "tok_per_s": { "mean": 34.40588828828829, "median": 34.654, "p95": 34.964 }, "alpha": null, "tau_mean": null }, "chat": { "n": 470, "tok_per_s": { "mean": 34.945753191489366, "median": 34.9105, "p95": 36.13855 }, "alpha": null, "tau_mean": null } } }, "final-gemma-q5-dflash-f16": { "run": "final-gemma-q5-dflash-f16", "target": "gemma-q5", "family": "gemma", "quant": "q5", "drafter": "dflash-f16", "k": 3, "ctx": 16384, "n": 1449, "tok_per_s": { "mean": 64.50848516218082, "median": 67.763, "p95": 79.3998 }, "alpha": { "mean": 0.659056314699793, "median": 0.709 }, "tau_mean": 156.50103519668738, "ttft_ms": { "mean": 187.51076604554865, "median": 98.2, "p95": 573.2799999999997 }, "vram_max_mib": 11980, "power_max_w": 160.82, "duration_s": 27.93, "errors": 0, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 72.45127209302326, "median": 73.244, "p95": 79.164 }, "alpha": { "mean": 0.7816416279069768, "median": 0.7956 }, "tau_mean": 175.6860465116279, "speedup": { "mean": 2.419694650889923, "median": 2.4436636828827507, "agg": 2.425081750618513, "n_match": 430 } }, "code": { "n": 553, "tok_per_s": { "mean": 68.42736166365282, "median": 68.642, "p95": 81.0476 }, "alpha": { "mean": 0.7238960216998191, "median": 0.725 }, "tau_mean": 167.06690777576853, "speedup": { "mean": 2.301476263202495, "median": 2.306608809450012, "agg": 2.29039382221526, "n_match": 553 } }, "chat": { "n": 466, "tok_per_s": { "mean": 52.528791845493565, "median": 51.403, "p95": 72.68025 }, "alpha": { "mean": 0.4689961373390558, "median": 0.4506 }, "tau_mean": 126.25965665236052, "speedup": { "mean": 1.7535148322519638, "median": 1.7142457329029552, "agg": 1.758238479553374, "n_match": 466 } } }, "speedup": { "mean": 2.1603333231629134, "median": 2.2665265124970806, "agg": 2.159221578966785, "n_match": 1449 } }, "final-gemma-q5-mtp": { "run": "final-gemma-q5-mtp", "target": "gemma-q5", "family": "gemma", "quant": "q5", "drafter": "mtp", "k": 3, "ctx": 16384, "n": 1449, "tok_per_s": { "mean": 75.64226777087647, "median": 78.085, "p95": 88.46039999999999 }, "alpha": { "mean": 0.7446065562456866, "median": 0.7807 }, "tau_mean": 164.80193236714976, "ttft_ms": { "mean": 172.72298136645963, "median": 92.4, "p95": 521.18 }, "vram_max_mib": 10366, "power_max_w": 164.08, "duration_s": 13.39, "errors": 0, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 82.26693488372095, "median": 82.7995, "p95": 88.4153 }, "alpha": { "mean": 0.8372979069767441, "median": 0.84685 }, "tau_mean": 179.5186046511628, "speedup": { "mean": 2.7472747862800895, "median": 2.7623163605961403, "agg": 2.7536306361837446, "n_match": 430 } }, "code": { "n": 553, "tok_per_s": { "mean": 78.36189150090416, "median": 78.361, "p95": 89.0876 }, "alpha": { "mean": 0.7879855334538879, "median": 0.7885 }, "tau_mean": 172.3254972875226, "speedup": { "mean": 2.6363271005548405, "median": 2.642533786032887, "agg": 2.622921413702688, "n_match": 553 } }, "chat": { "n": 466, "tok_per_s": { "mean": 66.30201287553648, "median": 64.481, "p95": 86.9225 }, "alpha": { "mean": 0.6075982832618027, "median": 0.5848 }, "tau_mean": 142.293991416309, "speedup": { "mean": 2.2134025387198037, "median": 2.1624934401359086, "agg": 2.2192543596376737, "n_match": 466 } } }, "speedup": { "mean": 2.5332385284683876, "median": 2.6139016743379364, "agg": 2.5318904395636666, "n_match": 1449 } }, "final-gemma-q5-solo": { "run": "final-gemma-q5-solo", "target": "gemma-q5", "family": "gemma", "quant": "q5", "drafter": "solo", "k": 3, "ctx": 16384, "n": 1449, "tok_per_s": { "mean": 29.875806073153896, "median": 29.952, "p95": 30.119600000000002 }, "alpha": null, "tau_mean": null, "ttft_ms": { "mean": 157.97805383022776, "median": 90.2, "p95": 442.0199999999997 }, "vram_max_mib": 9848, "power_max_w": 164.34, "duration_s": 18.6, "errors": 0, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 29.946332558139538, "median": 29.955, "p95": 30.026 }, "alpha": null, "tau_mean": null }, "code": { "n": 553, "tok_per_s": { "mean": 29.718493670886073, "median": 29.837, "p95": 30.0494 }, "alpha": null, "tau_mean": null }, "chat": { "n": 466, "tok_per_s": { "mean": 29.99740987124464, "median": 29.9895, "p95": 30.8305 }, "alpha": null, "tau_mean": null } } }, "final-gemma-q8-dflash-f16": { "run": "final-gemma-q8-dflash-f16", "target": "gemma-q8", "family": "gemma", "quant": "q8", "drafter": "dflash-f16", "k": 3, "ctx": 8192, "n": 1452, "tok_per_s": { "mean": 45.707994490358125, "median": 47.832499999999996, "p95": 56.6311 }, "alpha": { "mean": 0.6583433195592286, "median": 0.709 }, "tau_mean": 155.7555096418733, "ttft_ms": { "mean": 211.7603305785124, "median": 130.14999999999998, "p95": 580.89 }, "vram_max_mib": 15888, "power_max_w": 149.25, "duration_s": 104.6, "errors": 2, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 51.26042325581395, "median": 51.679500000000004, "p95": 56.893049999999995 }, "alpha": { "mean": 0.7795267441860465, "median": 0.7898000000000001 }, "tau_mean": 175.41162790697675, "speedup": { "mean": 2.5753097406055074, "median": 2.593730133667818, "agg": 2.5714690229558856, "n_match": 430 } }, "code": { "n": 558, "tok_per_s": { "mean": 48.73938888888889, "median": 49.062, "p95": 57.54775 }, "alpha": { "mean": 0.7266213261648745, "median": 0.7322 }, "tau_mean": 166.96236559139786, "speedup": { "mean": 2.4570797043746073, "median": 2.4570426569392354, "agg": 2.4450018311419806, "n_match": 558 } }, "chat": { "n": 464, "tok_per_s": { "mean": 36.91691163793104, "median": 36.471000000000004, "p95": 51.6553 }, "alpha": { "mean": 0.4639295258620689, "median": 0.4534 }, "tau_mean": 124.0625, "speedup": { "mean": 1.8473760999433333, "median": 1.8348887305160884, "agg": 1.851929591497711, "n_match": 464 } } }, "speedup": { "mean": 2.297256318095803, "median": 2.413130461150101, "agg": 2.29293252899669, "n_match": 1452 } }, "final-gemma-q8-mtp": { "run": "final-gemma-q8-mtp", "target": "gemma-q8", "family": "gemma", "quant": "q8", "drafter": "mtp", "k": 3, "ctx": 8192, "n": 1452, "tok_per_s": { "mean": 53.95811019283746, "median": 55.691500000000005, "p95": 63.40005000000001 }, "alpha": { "mean": 0.7454163911845729, "median": 0.7773 }, "tau_mean": 164.02685950413223, "ttft_ms": { "mean": 195.49958677685947, "median": 122.75, "p95": 535.12 }, "vram_max_mib": 14251, "power_max_w": 166.57, "duration_s": 77.81, "errors": 2, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 58.54481627906977, "median": 59.262, "p95": 63.0375 }, "alpha": { "mean": 0.8365709302325581, "median": 0.8472 }, "tau_mean": 179.35813953488372, "speedup": { "mean": 2.9405015463944766, "median": 2.9678438251708847, "agg": 2.936889162326541, "n_match": 430 } }, "code": { "n": 558, "tok_per_s": { "mean": 56.03362544802867, "median": 55.9885, "p95": 63.89855 }, "alpha": { "mean": 0.7901008960573477, "median": 0.7885 }, "tau_mean": 172.20967741935485, "speedup": { "mean": 2.8249630349993744, "median": 2.816732296284015, "agg": 2.8109157695488953, "n_match": 558 } }, "chat": { "n": 464, "tok_per_s": { "mean": 47.211512931034484, "median": 46.5275, "p95": 61.2943 }, "alpha": { "mean": 0.6072043103448276, "median": 0.5926 }, "tau_mean": 139.97844827586206, "speedup": { "mean": 2.3622201722888154, "median": 2.3423838847443745, "agg": 2.368356235046633, "n_match": 464 } } }, "speedup": { "mean": 2.711305233072511, "median": 2.7997846830116755, "agg": 2.706797956109045, "n_match": 1452 } }, "final-gemma-q8-solo": { "run": "final-gemma-q8-solo", "target": "gemma-q8", "family": "gemma", "quant": "q8", "drafter": "solo", "k": 3, "ctx": 8192, "n": 1452, "tok_per_s": { "mean": 19.934295454545452, "median": 19.811500000000002, "p95": 20.451 }, "alpha": null, "tau_mean": null, "ttft_ms": { "mean": 178.38264462809917, "median": 119.8, "p95": 445.27000000000015 }, "vram_max_mib": 13741, "power_max_w": 149.04, "duration_s": 138.21, "errors": 2, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 19.917351162790695, "median": 19.7945, "p95": 20.428 }, "alpha": null, "tau_mean": null }, "code": { "n": 558, "tok_per_s": { "mean": 19.839224014336917, "median": 19.809, "p95": 20.447 }, "alpha": null, "tau_mean": null }, "chat": { "n": 464, "tok_per_s": { "mean": 20.06432974137931, "median": 19.823500000000003, "p95": 20.6116 }, "alpha": null, "tau_mean": null } } }, "final-qwen-q4-dflash": { "run": "final-qwen-q4-dflash", "target": "qwen-q4", "family": "qwen", "quant": "q4", "drafter": "dflash-f16", "k": 3, "ctx": 8192, "n": 1474, "tok_per_s": { "mean": 33.42025169606513, "median": 33.329, "p95": 34.609049999999996 }, "alpha": { "mean": 0.009222184531886024, "median": 0.0081 }, "tau_mean": 6.700135685210312, "ttft_ms": { "mean": 76.68419267299865, "median": 36.3, "p95": 260.24499999999995 }, "vram_max_mib": 9288, "power_max_w": 161.75, "duration_s": 88.06, "errors": 0, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 33.448123255813954, "median": 33.2755, "p95": 34.83115 }, "alpha": { "mean": 0.00873953488372093, "median": 0.0067 }, "tau_mean": 6.341860465116279, "speedup": { "mean": 0.625348238489337, "median": 0.6215996714030199, "agg": 0.6267435724021049, "n_match": 430 } }, "code": { "n": 564, "tok_per_s": { "mean": 33.3616134751773, "median": 33.329, "p95": 34.39455 }, "alpha": { "mean": 0.009304432624113476, "median": 0.0081 }, "tau_mean": 6.792553191489362, "speedup": { "mean": 0.6268145533192508, "median": 0.6247902486921681, "agg": 0.6251225711713553, "n_match": 564 } }, "chat": { "n": 480, "tok_per_s": { "mean": 33.46418333333334, "median": 33.411500000000004, "p95": 34.70445 }, "alpha": { "mean": 0.009557916666666666, "median": 0.0081 }, "tau_mean": 6.9125, "speedup": { "mean": 0.6266434326195484, "median": 0.6238302998776759, "agg": 0.6270445026002082, "n_match": 480 } } }, "speedup": { "mean": 0.6263310707461707, "median": 0.6235980937434453, "agg": 0.626221321249423, "n_match": 1474 } }, "final-qwen-q4-dspark-p0": { "run": "final-qwen-q4-dspark-p0", "target": "qwen-q4", "family": "qwen", "quant": "q4", "drafter": "dspark-p0", "k": 3, "ctx": 8192, "n": 1474, "tok_per_s": { "mean": 87.61712075983718, "median": 86.888, "p95": 109.27405 }, "alpha": { "mean": 0.6163681818181819, "median": 0.6074 }, "tau_mean": 162.40773405698778, "ttft_ms": { "mean": 76.93622795115333, "median": 36.6, "p95": 259.07 }, "vram_max_mib": 9429, "power_max_w": 158.6, "duration_s": 43.65, "errors": 0, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 95.87322790697674, "median": 96.12450000000001, "p95": 111.8299 }, "alpha": { "mean": 0.7061618604651162, "median": 0.709 }, "tau_mean": 171.12558139534883, "speedup": { "mean": 1.792389094534062, "median": 1.8095527871081896, "agg": 1.7964514450207694, "n_match": 430 } }, "code": { "n": 564, "tok_per_s": { "mean": 86.51806382978724, "median": 85.81049999999999, "p95": 101.14075 }, "alpha": { "mean": 0.6055257092198582, "median": 0.5971 }, "tau_mean": 163.10106382978722, "speedup": { "mean": 1.6257053204274554, "median": 1.6120584527896276, "agg": 1.6211564393995967, "n_match": 564 } }, "chat": { "n": 480, "tok_per_s": { "mean": 81.51241666666668, "median": 80.22, "p95": 109.78285 }, "alpha": { "mean": 0.5486679166666667, "median": 0.534 }, "tau_mean": 153.78333333333333, "speedup": { "mean": 1.5262824570915052, "median": 1.4997059758476776, "agg": 1.5273617244852606, "n_match": 480 } } }, "speedup": { "mean": 1.6419543356680153, "median": 1.6271307344012569, "agg": 1.6417503262776312, "n_match": 1474 } }, "final-qwen-q4-dspark-p2": { "run": "final-qwen-q4-dspark-p2", "target": "qwen-q4", "family": "qwen", "quant": "q4", "drafter": "dspark-p2", "k": 3, "ctx": 8192, "n": 1474, "tok_per_s": { "mean": 86.67783649932157, "median": 85.97399999999999, "p95": 108.11829999999999 }, "alpha": { "mean": 0.6207055630936228, "median": 0.6104 }, "tau_mean": 162.35888738127545, "ttft_ms": { "mean": 76.96702849389418, "median": 36.5, "p95": 260.44999999999993 }, "vram_max_mib": 9426, "power_max_w": 158.01, "duration_s": 40.92, "errors": 0, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 94.99026744186047, "median": 95.7945, "p95": 110.67115 }, "alpha": { "mean": 0.7067206976744186, "median": 0.71045 }, "tau_mean": 171.146511627907, "speedup": { "mean": 1.7758914666828969, "median": 1.7920045412645096, "agg": 1.7799067261447827, "n_match": 430 } }, "code": { "n": 564, "tok_per_s": { "mean": 85.59933865248227, "median": 84.9105, "p95": 100.38805 }, "alpha": { "mean": 0.6058659574468085, "median": 0.5971 }, "tau_mean": 163.08510638297872, "speedup": { "mean": 1.6084466080758277, "median": 1.591466231502861, "agg": 1.6039415692176116, "n_match": 564 } }, "chat": { "n": 480, "tok_per_s": { "mean": 80.49851875, "median": 79.1845, "p95": 108.44174999999997 }, "alpha": { "mean": 0.5610868750000001, "median": 0.54575 }, "tau_mean": 153.63333333333333, "speedup": { "mean": 1.507203142793631, "median": 1.4820706392926062, "agg": 1.5083635284584547, "n_match": 480 } } }, "speedup": { "mean": 1.6243247803048544, "median": 1.6104524664676467, "agg": 1.6241502245190282, "n_match": 1474 } }, "final-qwen-q4-dspark-p4": { "run": "final-qwen-q4-dspark-p4", "target": "qwen-q4", "family": "qwen", "quant": "q4", "drafter": "dspark-p4", "k": 3, "ctx": 8192, "n": 1474, "tok_per_s": { "mean": 85.41618995929444, "median": 85.232, "p95": 107.8453 }, "alpha": { "mean": 0.6464877204884668, "median": 0.6293 }, "tau_mean": 160.86227951153325, "ttft_ms": { "mean": 76.99735413839892, "median": 36.6, "p95": 260.0199999999999 }, "vram_max_mib": 9418, "power_max_w": 155.91, "duration_s": 41.37, "errors": 0, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 94.57915581395348, "median": 95.011, "p95": 110.8468 }, "alpha": { "mean": 0.7119279069767442, "median": 0.7169 }, "tau_mean": 170.88372093023256, "speedup": { "mean": 1.7682082029834372, "median": 1.7870420187369391, "agg": 1.7722034069373087, "n_match": 430 } }, "code": { "n": 564, "tok_per_s": { "mean": 84.81273226950354, "median": 84.136, "p95": 99.644 }, "alpha": { "mean": 0.6222294326241136, "median": 0.6157 }, "tau_mean": 162.5904255319149, "speedup": { "mean": 1.5937010350638858, "median": 1.5782483066579922, "agg": 1.589202311927393, "n_match": 564 } }, "chat": { "n": 480, "tok_per_s": { "mean": 77.9167625, "median": 76.28450000000001, "p95": 107.14879999999998 }, "alpha": { "mean": 0.6163677083333333, "median": 0.5869 }, "tau_mean": 149.85416666666666, "speedup": { "mean": 1.458965926898531, "median": 1.4277669171898328, "agg": 1.4599871480313344, "n_match": 480 } } }, "speedup": { "mean": 1.6007330773203559, "median": 1.5970623063583436, "agg": 1.6005097693114878, "n_match": 1474 } }, "final-qwen-q4-dspark-p6": { "run": "final-qwen-q4-dspark-p6", "target": "qwen-q4", "family": "qwen", "quant": "q4", "drafter": "dspark-p6", "k": 3, "ctx": 8192, "n": 1474, "tok_per_s": { "mean": 80.73099660786974, "median": 80.082, "p95": 106.61354999999999 }, "alpha": { "mean": 0.7138444369063772, "median": 0.7036 }, "tau_mean": 154.219131614654, "ttft_ms": { "mean": 77.25006784260516, "median": 36.9, "p95": 260.78999999999985 }, "vram_max_mib": 9417, "power_max_w": 158.04, "duration_s": 45.09, "errors": 0, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 92.6727976744186, "median": 92.857, "p95": 110.70694999999999 }, "alpha": { "mean": 0.7426737209302325, "median": 0.75215 }, "tau_mean": 169.05348837209303, "speedup": { "mean": 1.7325356529294538, "median": 1.7405452449155936, "agg": 1.7364824876644391, "n_match": 430 } }, "code": { "n": 564, "tok_per_s": { "mean": 79.6881755319149, "median": 79.1105, "p95": 95.50435000000002 }, "alpha": { "mean": 0.6840361702127659, "median": 0.6807 }, "tau_mean": 156.5549645390071, "speedup": { "mean": 1.4974874919466474, "median": 1.484013217389872, "agg": 1.493179495575946, "n_match": 564 } }, "chat": { "n": 480, "tok_per_s": { "mean": 71.25844791666665, "median": 68.9305, "p95": 103.77059999999997 }, "alpha": { "mean": 0.7230429166666668, "median": 0.7072499999999999 }, "tau_mean": 138.18541666666667, "speedup": { "mean": 1.3341349356936296, "median": 1.284632294241261, "agg": 1.3352251147112733, "n_match": 480 } } }, "speedup": { "mean": 1.5128616318524535, "median": 1.49748294695411, "agg": 1.5127196473961693, "n_match": 1474 } }, "final-qwen-q4-eagle3": { "run": "final-qwen-q4-eagle3", "target": "qwen-q4", "family": "qwen", "quant": "q4", "drafter": "eagle3", "k": 3, "ctx": 8192, "n": 1474, "tok_per_s": { "mean": 74.37096675712347, "median": 73.1925, "p95": 97.836 }, "alpha": { "mean": 0.44020434192673, "median": 0.4294 }, "tau_mean": 140.5407055630936, "ttft_ms": { "mean": 74.8416553595658, "median": 35.4, "p95": 253.77499999999995 }, "vram_max_mib": 8191, "power_max_w": 159.02, "duration_s": 43.6, "errors": 0, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 74.98914186046511, "median": 74.6525, "p95": 86.56679999999999 }, "alpha": { "mean": 0.4451893023255814, "median": 0.4421 }, "tau_mean": 143.8860465116279, "speedup": { "mean": 1.401978119769266, "median": 1.3938344426680405, "agg": 1.4051300367899346, "n_match": 430 } }, "code": { "n": 564, "tok_per_s": { "mean": 74.86378723404256, "median": 73.24950000000001, "p95": 93.90325 }, "alpha": { "mean": 0.4467317375886525, "median": 0.4294 }, "tau_mean": 144.17021276595744, "speedup": { "mean": 1.4063464266699996, "median": 1.3791478278236826, "agg": 1.4027811693877092, "n_match": 564 } }, "chat": { "n": 480, "tok_per_s": { "mean": 73.23812083333333, "median": 70.8945, "p95": 107.34859999999993 }, "alpha": { "mean": 0.4280689583333333, "median": 0.4029 }, "tau_mean": 133.27916666666667, "speedup": { "mean": 1.3715519431293326, "median": 1.3239886829768142, "agg": 1.372319790143136, "n_match": 480 } } }, "speedup": { "mean": 1.3937414578322551, "median": 1.3727328100680438, "agg": 1.3935468077496915, "n_match": 1474 } }, "final-qwen-q4-solo": { "run": "final-qwen-q4-solo", "target": "qwen-q4", "family": "qwen", "quant": "q4", "drafter": "solo", "k": 3, "ctx": 8192, "n": 1474, "tok_per_s": { "mean": 53.368115332428765, "median": 53.703, "p95": 53.862 }, "alpha": null, "tau_mean": null, "ttft_ms": { "mean": 65.15162822252374, "median": 32.05, "p95": 213.54499999999993 }, "vram_max_mib": 6683, "power_max_w": 158.15, "duration_s": 53.79, "errors": 0, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 53.496337209302325, "median": 53.731, "p95": 53.815200000000004 }, "alpha": null, "tau_mean": null }, "code": { "n": 564, "tok_per_s": { "mean": 53.23322872340426, "median": 53.432500000000005, "p95": 53.8617 }, "alpha": null, "tau_mean": null }, "chat": { "n": 480, "tok_per_s": { "mean": 53.411741666666664, "median": 53.781499999999994, "p95": 53.86905 }, "alpha": null, "tau_mean": null } } }, "final-qwen-q4-vanilla17b": { "run": "final-qwen-q4-vanilla17b", "target": "qwen-q4", "family": "qwen", "quant": "q4", "drafter": "vanilla17b", "k": 3, "ctx": 8192, "n": 1474, "tok_per_s": { "mean": 75.45253731343284, "median": 75.6645, "p95": 87.57839999999999 }, "alpha": { "mean": 0.7251561058344641, "median": 0.7292 }, "tau_mean": 173.27272727272728, "ttft_ms": { "mean": 79.04545454545455, "median": 39.3, "p95": 253.23499999999999 }, "vram_max_mib": 8691, "power_max_w": 164.61, "duration_s": 49.07, "errors": 0, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 79.97231162790698, "median": 80.05449999999999, "p95": 87.7595 }, "alpha": { "mean": 0.7835525581395348, "median": 0.7851 }, "tau_mean": 177.61162790697674, "speedup": { "mean": 1.4951547492953159, "median": 1.4965815573078318, "agg": 1.498503575210803, "n_match": 430 } }, "code": { "n": 564, "tok_per_s": { "mean": 74.41387234042553, "median": 74.43299999999999, "p95": 83.16835 }, "alpha": { "mean": 0.7156375886524823, "median": 0.7149 }, "tau_mean": 173.18971631205673, "speedup": { "mean": 1.398139174151506, "median": 1.394663418381457, "agg": 1.39435076312707, "n_match": 564 } }, "chat": { "n": 480, "tok_per_s": { "mean": 72.62400416666668, "median": 71.2925, "p95": 90.07265 }, "alpha": { "mean": 0.684026875, "median": 0.6576 }, "tau_mean": 169.48333333333332, "speedup": { "mean": 1.359868566571217, "median": 1.327286770288385, "agg": 1.36081260719614, "n_match": 480 } } }, "speedup": { "mean": 1.41397825534099, "median": 1.4146129905513134, "agg": 1.4138130388049253, "n_match": 1474 } }, "final-qwen-q5-dflash": { "run": "final-qwen-q5-dflash", "target": "qwen-q5", "family": "qwen", "quant": "q5", "drafter": "dflash-f16", "k": 3, "ctx": 8192, "n": 1474, "tok_per_s": { "mean": 30.77782089552239, "median": 30.700000000000003, "p95": 31.74435 }, "alpha": { "mean": 0.009008683853459973, "median": 0.0081 }, "tau_mean": 6.550203527815468, "ttft_ms": { "mean": 79.47089552238806, "median": 38.0, "p95": 264.34499999999997 }, "vram_max_mib": 9982, "power_max_w": 162.39, "duration_s": 92.1, "errors": 0, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 30.84132093023256, "median": 30.722, "p95": 32.03855 }, "alpha": { "mean": 0.00911813953488372, "median": 0.0081 }, "tau_mean": 6.611627906976744, "speedup": { "mean": 0.6574222739183073, "median": 0.654970066684825, "agg": 0.6590076471548982, "n_match": 430 } }, "code": { "n": 564, "tok_per_s": { "mean": 30.707031914893616, "median": 30.6725, "p95": 31.5744 }, "alpha": { "mean": 0.00880372340425532, "median": 0.0081 }, "tau_mean": 6.425531914893617, "speedup": { "mean": 0.6573459269265252, "median": 0.6557818202526157, "agg": 0.6561382017041842, "n_match": 564 } }, "chat": { "n": 480, "tok_per_s": { "mean": 30.8041125, "median": 30.741500000000002, "p95": 31.91105 }, "alpha": { "mean": 0.009151458333333333, "median": 0.0081 }, "tau_mean": 6.641666666666667, "speedup": { "mean": 0.6582839316051012, "median": 0.6558193530744056, "agg": 0.6582125891183973, "n_match": 480 } } }, "speedup": { "mean": 0.6576736551844512, "median": 0.6554352371876343, "agg": 0.6576507983816804, "n_match": 1474 } }, "final-qwen-q5-dspark-p0": { "run": "final-qwen-q5-dspark-p0", "target": "qwen-q5", "family": "qwen", "quant": "q5", "drafter": "dspark-p0", "k": 3, "ctx": 8192, "n": 1474, "tok_per_s": { "mean": 80.64056105834464, "median": 80.6435, "p95": 100.58834999999999 }, "alpha": { "mean": 0.6133860922659431, "median": 0.6134 }, "tau_mean": 162.07123473541384, "ttft_ms": { "mean": 79.55481682496607, "median": 37.95, "p95": 266.24499999999995 }, "vram_max_mib": 10130, "power_max_w": 161.21, "duration_s": 43.83, "errors": 0, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 87.93867209302326, "median": 87.176, "p95": 103.6909 }, "alpha": { "mean": 0.6988960465116278, "median": 0.69215 }, "tau_mean": 170.9232558139535, "speedup": { "mean": 1.8745434770684581, "median": 1.8641986168364166, "agg": 1.8790458917452204, "n_match": 430 } }, "code": { "n": 564, "tok_per_s": { "mean": 80.11799290780142, "median": 79.6585, "p95": 94.96770000000001 }, "alpha": { "mean": 0.6089276595744681, "median": 0.6052 }, "tau_mean": 163.13120567375887, "speedup": { "mean": 1.7150924740926523, "median": 1.7078710466918818, "agg": 1.7119360782367408, "n_match": 564 } }, "chat": { "n": 480, "tok_per_s": { "mean": 74.7166875, "median": 73.3435, "p95": 98.82169999999999 }, "alpha": { "mean": 0.5420220833333335, "median": 0.5271 }, "tau_mean": 152.89583333333334, "speedup": { "mean": 1.5971444539851656, "median": 1.5693763901165734, "agg": 1.5965226827984476, "n_match": 480 } } }, "speedup": { "mean": 1.7231989066761009, "median": 1.724499600708512, "agg": 1.7231021501487256, "n_match": 1474 } }, "final-qwen-q5-eagle3": { "run": "final-qwen-q5-eagle3", "target": "qwen-q5", "family": "qwen", "quant": "q5", "drafter": "eagle3", "k": 3, "ctx": 8192, "n": 1474, "tok_per_s": { "mean": 68.33110108548169, "median": 67.4005, "p95": 89.91565 }, "alpha": { "mean": 0.4393555630936229, "median": 0.4281 }, "tau_mean": 140.3697421981004, "ttft_ms": { "mean": 78.1544776119403, "median": 37.1, "p95": 258.635 }, "vram_max_mib": 8902, "power_max_w": 159.27, "duration_s": 46.69, "errors": 0, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 69.03269069767443, "median": 68.809, "p95": 80.3728 }, "alpha": { "mean": 0.44576348837209306, "median": 0.44275 }, "tau_mean": 144.36279069767443, "speedup": { "mean": 1.4714964823037155, "median": 1.4654598663043426, "agg": 1.4750688265382028, "n_match": 430 } }, "code": { "n": 564, "tok_per_s": { "mean": 69.14667907801419, "median": 67.2295, "p95": 86.0588 }, "alpha": { "mean": 0.44972446808510635, "median": 0.4294 }, "tau_mean": 144.2464539007092, "speedup": { "mean": 1.4797767426011872, "median": 1.44102914860788, "agg": 1.4775044944040712, "n_match": 564 } }, "chat": { "n": 480, "tok_per_s": { "mean": 66.74428958333333, "median": 65.0515, "p95": 97.40849999999996 }, "alpha": { "mean": 0.4214316666666667, "median": 0.4017 }, "tau_mean": 132.2375, "speedup": { "mean": 1.4267358444390625, "median": 1.3892576501221157, "agg": 1.42617099114652, "n_match": 480 } } }, "speedup": { "mean": 1.4600887215389535, "median": 1.4391020701542812, "agg": 1.4600774803294798, "n_match": 1474 } }, "final-qwen-q5-solo": { "run": "final-qwen-q5-solo", "target": "qwen-q5", "family": "qwen", "quant": "q5", "drafter": "solo", "k": 3, "ctx": 8192, "n": 1474, "tok_per_s": { "mean": 46.79964043419268, "median": 46.951, "p95": 47.038 }, "alpha": null, "tau_mean": null, "ttft_ms": { "mean": 68.04660786974219, "median": 33.8, "p95": 220.17999999999992 }, "vram_max_mib": 7396, "power_max_w": 164.2, "duration_s": 59.33, "errors": 0, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 46.91275813953488, "median": 46.953, "p95": 47.00555 }, "alpha": null, "tau_mean": null }, "code": { "n": 564, "tok_per_s": { "mean": 46.71516666666667, "median": 46.852, "p95": 47.04 }, "alpha": null, "tau_mean": null }, "chat": { "n": 480, "tok_per_s": { "mean": 46.797562500000005, "median": 46.995, "p95": 47.04205 }, "alpha": null, "tau_mean": null } } }, "final-qwen-q8-dspark-p0": { "run": "final-qwen-q8-dspark-p0", "target": "qwen-q8", "family": "qwen", "quant": "q8", "drafter": "dspark-p0", "k": 3, "ctx": 8192, "n": 1474, "tok_per_s": { "mean": 62.331263907734055, "median": 62.384, "p95": 78.3239 }, "alpha": { "mean": 0.6108656037991859, "median": 0.6111 }, "tau_mean": 162.01085481682497, "ttft_ms": { "mean": 84.01906377204885, "median": 47.9, "p95": 250.26999999999978 }, "vram_max_mib": 12628, "power_max_w": 138.36, "duration_s": 57.82, "errors": 0, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 68.43227209302326, "median": 67.428, "p95": 80.74814999999998 }, "alpha": { "mean": 0.702976511627907, "median": 0.6909000000000001 }, "tau_mean": 171.30232558139534, "speedup": { "mean": 2.112076473512988, "median": 2.0797631263157514, "agg": 2.1156820749266427, "n_match": 430 } }, "code": { "n": 564, "tok_per_s": { "mean": 61.80902836879433, "median": 61.689499999999995, "p95": 71.42160000000001 }, "alpha": { "mean": 0.6026237588652482, "median": 0.6029 }, "tau_mean": 162.93262411347519, "speedup": { "mean": 1.913091765874063, "median": 1.9143401886550064, "agg": 1.9109149731391493, "n_match": 564 } }, "chat": { "n": 480, "tok_per_s": { "mean": 57.47940416666667, "median": 56.7975, "p95": 77.8765 }, "alpha": { "mean": 0.5380337500000001, "median": 0.5271 }, "tau_mean": 152.60416666666666, "speedup": { "mean": 1.777518476338368, "median": 1.7520308139870606, "agg": 1.7770584163502956, "n_match": 480 } } }, "speedup": { "mean": 1.926991525241501, "median": 1.9257494441741478, "agg": 1.927060635628954, "n_match": 1474 } }, "final-qwen-q8-eagle3": { "run": "final-qwen-q8-eagle3", "target": "qwen-q8", "family": "qwen", "quant": "q8", "drafter": "eagle3", "k": 3, "ctx": 8192, "n": 1474, "tok_per_s": { "mean": 52.812307327001356, "median": 52.283, "p95": 68.29979999999999 }, "alpha": { "mean": 0.43244274084124834, "median": 0.4251 }, "tau_mean": 139.79511533242876, "ttft_ms": { "mean": 83.02605156037993, "median": 47.6, "p95": 244.90499999999997 }, "vram_max_mib": 11397, "power_max_w": 151.27, "duration_s": 52.99, "errors": 0, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 53.497669767441856, "median": 53.282, "p95": 60.908899999999996 }, "alpha": { "mean": 0.44095744186046515, "median": 0.439 }, "tau_mean": 143.76511627906976, "speedup": { "mean": 1.6510905504366684, "median": 1.6429804874117824, "agg": 1.6539573729696597, "n_match": 430 } }, "code": { "n": 564, "tok_per_s": { "mean": 53.40724113475177, "median": 52.2595, "p95": 66.3061 }, "alpha": { "mean": 0.44163297872340423, "median": 0.4239 }, "tau_mean": 143.55673758865248, "speedup": { "mean": 1.6527423498652336, "median": 1.6151450800863136, "agg": 1.6511616417833201, "n_match": 564 } }, "chat": { "n": 480, "tok_per_s": { "mean": 51.49928958333333, "median": 50.323, "p95": 74.97179999999997 }, "alpha": { "mean": 0.4140164583333333, "median": 0.3954 }, "tau_mean": 131.81875, "speedup": { "mean": 1.592449508746356, "median": 1.5527616166531488, "agg": 1.5921745765624356, "n_match": 480 } } }, "speedup": { "mean": 1.6326264492605225, "median": 1.6141883280926623, "agg": 1.6327684077969575, "n_match": 1474 } }, "final-qwen-q8-solo": { "run": "final-qwen-q8-solo", "target": "qwen-q8", "family": "qwen", "quant": "q8", "drafter": "solo", "k": 3, "ctx": 8192, "n": 1474, "tok_per_s": { "mean": 32.345253052917236, "median": 32.413, "p95": 32.46 }, "alpha": null, "tau_mean": null, "ttft_ms": { "mean": 70.65502035278155, "median": 42.1, "p95": 203.23999999999995 }, "vram_max_mib": 9881, "power_max_w": 150.75, "duration_s": 78.25, "errors": 0, "per_domain": { "math": { "n": 430, "tok_per_s": { "mean": 32.4010976744186, "median": 32.416, "p95": 32.44155 }, "alpha": null, "tau_mean": null }, "code": { "n": 564, "tok_per_s": { "mean": 32.308576241134745, "median": 32.3695, "p95": 32.46 }, "alpha": null, "tau_mean": null }, "chat": { "n": 480, "tok_per_s": { "mean": 32.338320833333334, "median": 32.4315, "p95": 32.461 }, "alpha": null, "tau_mean": null } } } }, "baselines": [ { "run": "baseline-ctx2048-gemma-q4-solo", "target": "gemma-q4", "ctx": 2048, "prompt_path": "experiments/prompts/old/acc-sample.jsonl", "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "n_tokens": 256, "sampling": { "temperature": 0.0, "top_k": 40, "top_p": 0.95, "seed": 42 }, "n": 175, "excluded": 3, "errors": 2, "tok_per_s": { "mean": 33.47058285714286, "median": 32.657, "p95": 34.8518 }, "tok_per_s_by_domain": { "math": { "mean": 32.767716666666665, "median": 32.206, "p95": 34.69645 }, "code": { "mean": 32.50666666666667, "median": 32.283, "p95": 34.74505 }, "chat": { "mean": 35.28889090909091, "median": 34.711, "p95": 35.876599999999996 } } }, { "run": "baseline-ctx2048-qwen-q4-solo", "target": "qwen-q4", "ctx": 2048, "prompt_path": "experiments/prompts/old/acc-sample.jsonl", "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "n_tokens": 256, "sampling": { "temperature": 0.0, "top_k": 40, "top_p": 0.95, "seed": 42 }, "n": 178, "excluded": 0, "errors": 2, "tok_per_s": { "mean": 51.49112359550562, "median": 50.039, "p95": 53.575900000000004 }, "tok_per_s_by_domain": { "math": { "mean": 53.47833333333333, "median": 53.4995, "p95": 53.5811 }, "code": { "mean": 51.10905000000001, "median": 49.963, "p95": 53.5932 }, "chat": { "mean": 49.83063793103449, "median": 49.902, "p95": 50.011649999999996 } } }, { "run": "baseline-ctx2048-qwen-q8-solo", "target": "qwen-q8", "ctx": 2048, "prompt_path": "experiments/prompts/old/acc-sample.jsonl", "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "n_tokens": 256, "sampling": { "temperature": 0.0, "top_k": 40, "top_p": 0.95, "seed": 42 }, "n": 178, "excluded": 0, "errors": 2, "tok_per_s": { "mean": 31.769084269662915, "median": 32.2735, "p95": 32.34115 }, "tok_per_s_by_domain": { "math": { "mean": 30.842750000000002, "median": 30.124000000000002, "p95": 32.3071 }, "code": { "mean": 32.19831666666666, "median": 32.2655, "p95": 32.34105 }, "chat": { "mean": 32.283327586206894, "median": 32.317499999999995, "p95": 32.345 } } } ], "curves": { "curves-gemma-q4-dflash-f16": { "family": "curves", "target": "gemma-q4", "drafter": "dflash-f16", "k": 3, "n_excluded": 3, "n_matched": 175, "log_lines": 175, "max_alpha_diff": 4.999999999999449e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8875333333333334, "alpha_median": 0.8965000000000001, "ci95_low": 0.8730825000000001, "ci95_high": 0.9003516666666667 }, { "position": 2, "n": 60, "alpha_mean": 0.7715, "alpha_median": 0.7805, "ci95_low": 0.7495995833333332, "ci95_high": 0.7915512499999998 }, { "position": 3, "n": 60, "alpha_mean": 0.6351833333333334, "alpha_median": 0.634, "ci95_low": 0.6084816666666666, "ci95_high": 0.6601525 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.8466833333333333, "alpha_median": 0.86, "ci95_low": 0.8261970833333333, "ci95_high": 0.8677695833333332 }, { "position": 2, "n": 60, "alpha_mean": 0.7134166666666667, "alpha_median": 0.7144999999999999, "ci95_low": 0.6827783333333333, "ci95_high": 0.7451666666666666 }, { "position": 3, "n": 60, "alpha_mean": 0.5915166666666668, "alpha_median": 0.5774999999999999, "ci95_low": 0.5538308333333334, "ci95_high": 0.6293337499999999 } ], "chat": [ { "position": 1, "n": 55, "alpha_mean": 0.6516363636363637, "alpha_median": 0.673, "ci95_low": 0.5917995454545455, "ci95_high": 0.7055977272727272 }, { "position": 2, "n": 55, "alpha_mean": 0.4742, "alpha_median": 0.435, "ci95_low": 0.41159909090909086, "ci95_high": 0.5376786363636363 }, { "position": 3, "n": 55, "alpha_mean": 0.322, "alpha_median": 0.241, "ci95_low": 0.26021772727272724, "ci95_high": 0.38793636363636363 } ] } }, "curves-gemma-q4-dflash-q4": { "family": "curves", "target": "gemma-q4", "drafter": "dflash-q4", "k": 3, "n_excluded": 3, "n_matched": 175, "log_lines": 175, "max_alpha_diff": 4.999999999999449e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8840500000000001, "alpha_median": 0.894, "ci95_low": 0.8702995833333335, "ci95_high": 0.8966183333333333 }, { "position": 2, "n": 60, "alpha_mean": 0.7654833333333334, "alpha_median": 0.7855000000000001, "ci95_low": 0.7415658333333334, "ci95_high": 0.7874345833333332 }, { "position": 3, "n": 60, "alpha_mean": 0.6332499999999999, "alpha_median": 0.634, "ci95_low": 0.6061233333333335, "ci95_high": 0.6591541666666667 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.8459833333333333, "alpha_median": 0.8534999999999999, "ci95_low": 0.8243808333333332, "ci95_high": 0.8675191666666666 }, { "position": 2, "n": 60, "alpha_mean": 0.7023333333333334, "alpha_median": 0.6975, "ci95_low": 0.6700754166666668, "ci95_high": 0.7339170833333334 }, { "position": 3, "n": 60, "alpha_mean": 0.57765, "alpha_median": 0.5865, "ci95_low": 0.5409133333333332, "ci95_high": 0.6155004166666667 } ], "chat": [ { "position": 1, "n": 55, "alpha_mean": 0.6411454545454545, "alpha_median": 0.692, "ci95_low": 0.5844854545454545, "ci95_high": 0.6936559090909092 }, { "position": 2, "n": 55, "alpha_mean": 0.46087272727272727, "alpha_median": 0.453, "ci95_low": 0.4001595454545453, "ci95_high": 0.5217354545454544 }, { "position": 3, "n": 55, "alpha_mean": 0.3072727272727273, "alpha_median": 0.25, "ci95_low": 0.24769772727272724, "ci95_high": 0.37022499999999986 } ] } }, "curves-gemma-q4-dflash-q8": { "family": "curves", "target": "gemma-q4", "drafter": "dflash-q8", "k": 3, "n_excluded": 3, "n_matched": 175, "log_lines": 175, "max_alpha_diff": 4.999999999999449e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8857833333333335, "alpha_median": 0.8965000000000001, "ci95_low": 0.8707291666666668, "ci95_high": 0.8987858333333334 }, { "position": 2, "n": 60, "alpha_mean": 0.7701499999999999, "alpha_median": 0.7805, "ci95_low": 0.7480583333333333, "ci95_high": 0.7899016666666666 }, { "position": 3, "n": 60, "alpha_mean": 0.6345166666666666, "alpha_median": 0.6365000000000001, "ci95_low": 0.6085316666666667, "ci95_high": 0.6598679166666667 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.8447333333333333, "alpha_median": 0.859, "ci95_low": 0.8247666666666666, "ci95_high": 0.8653 }, { "position": 2, "n": 60, "alpha_mean": 0.7095166666666667, "alpha_median": 0.7055, "ci95_low": 0.6789491666666667, "ci95_high": 0.7401 }, { "position": 3, "n": 60, "alpha_mean": 0.59055, "alpha_median": 0.5774999999999999, "ci95_low": 0.553065, "ci95_high": 0.6284370833333334 } ], "chat": [ { "position": 1, "n": 55, "alpha_mean": 0.6510727272727274, "alpha_median": 0.67, "ci95_low": 0.5907809090909091, "ci95_high": 0.7066227272727273 }, { "position": 2, "n": 55, "alpha_mean": 0.4744181818181819, "alpha_median": 0.439, "ci95_low": 0.4124404545454545, "ci95_high": 0.5378390909090909 }, { "position": 3, "n": 55, "alpha_mean": 0.32210909090909096, "alpha_median": 0.239, "ci95_low": 0.2609240909090909, "ci95_high": 0.3875650000000001 } ] } }, "curves-gemma-q4-mtp": { "family": "curves", "target": "gemma-q4", "drafter": "mtp", "k": 3, "n_excluded": 3, "n_matched": 175, "log_lines": 175, "max_alpha_diff": 4.999999999999449e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.9051000000000001, "alpha_median": 0.912, "ci95_low": 0.8935333333333334, "ci95_high": 0.9165504166666667 }, { "position": 2, "n": 60, "alpha_mean": 0.8125499999999999, "alpha_median": 0.8160000000000001, "ci95_low": 0.7919820833333333, "ci95_high": 0.8327345833333333 }, { "position": 3, "n": 60, "alpha_mean": 0.7374333333333333, "alpha_median": 0.752, "ci95_low": 0.71372625, "ci95_high": 0.7597679166666668 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.8862666666666669, "alpha_median": 0.896, "ci95_low": 0.8699804166666667, "ci95_high": 0.9020004166666666 }, { "position": 2, "n": 60, "alpha_mean": 0.7817166666666666, "alpha_median": 0.7775000000000001, "ci95_low": 0.7571645833333334, "ci95_high": 0.8053525 }, { "position": 3, "n": 60, "alpha_mean": 0.6927166666666665, "alpha_median": 0.6885, "ci95_low": 0.6627329166666668, "ci95_high": 0.7230841666666667 } ], "chat": [ { "position": 1, "n": 55, "alpha_mean": 0.7378363636363637, "alpha_median": 0.786, "ci95_low": 0.6822290909090908, "ci95_high": 0.7893086363636364 }, { "position": 2, "n": 55, "alpha_mean": 0.5864363636363636, "alpha_median": 0.574, "ci95_low": 0.5274227272727273, "ci95_high": 0.6486104545454544 }, { "position": 3, "n": 55, "alpha_mean": 0.48025454545454543, "alpha_median": 0.436, "ci95_low": 0.4184490909090909, "ci95_high": 0.5485113636363635 } ] } }, "curves-gemma-q5-dflash-f16": { "family": "curves", "target": "gemma-q5", "drafter": "dflash-f16", "k": 3, "n_excluded": 4, "n_matched": 174, "log_lines": 174, "max_alpha_diff": 5.0000000000050004e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8929333333333334, "alpha_median": 0.905, "ci95_low": 0.8811162500000003, "ci95_high": 0.9041341666666669 }, { "position": 2, "n": 60, "alpha_mean": 0.78665, "alpha_median": 0.797, "ci95_low": 0.7672808333333334, "ci95_high": 0.8055704166666667 }, { "position": 3, "n": 60, "alpha_mean": 0.6724500000000002, "alpha_median": 0.678, "ci95_low": 0.6514829166666668, "ci95_high": 0.6935520833333333 } ], "code": [ { "position": 1, "n": 58, "alpha_mean": 0.8492586206896553, "alpha_median": 0.871, "ci95_low": 0.815323275862069, "ci95_high": 0.8769172413793103 }, { "position": 2, "n": 58, "alpha_mean": 0.7110172413793103, "alpha_median": 0.7264999999999999, "ci95_low": 0.6739077586206896, "ci95_high": 0.7490047413793102 }, { "position": 3, "n": 58, "alpha_mean": 0.5873448275862071, "alpha_median": 0.582, "ci95_low": 0.5447491379310344, "ci95_high": 0.6347806034482758 } ], "chat": [ { "position": 1, "n": 56, "alpha_mean": 0.6457500000000002, "alpha_median": 0.6975, "ci95_low": 0.5908165178571428, "ci95_high": 0.6954464285714286 }, { "position": 2, "n": 56, "alpha_mean": 0.4306964285714286, "alpha_median": 0.421, "ci95_low": 0.3777625, "ci95_high": 0.48348749999999996 }, { "position": 3, "n": 56, "alpha_mean": 0.2739285714285714, "alpha_median": 0.2405, "ci95_low": 0.22422053571428568, "ci95_high": 0.32529330357142855 } ] } }, "curves-gemma-q5-mtp": { "family": "curves", "target": "gemma-q5", "drafter": "mtp", "k": 3, "n_excluded": 4, "n_matched": 174, "log_lines": 174, "max_alpha_diff": 4.999999999999449e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.9157500000000002, "alpha_median": 0.917, "ci95_low": 0.9051816666666667, "ci95_high": 0.9253833333333332 }, { "position": 2, "n": 60, "alpha_mean": 0.8319833333333334, "alpha_median": 0.838, "ci95_low": 0.8126820833333335, "ci95_high": 0.8497679166666667 }, { "position": 3, "n": 60, "alpha_mean": 0.7505833333333333, "alpha_median": 0.757, "ci95_low": 0.7291641666666666, "ci95_high": 0.7722679166666668 } ], "code": [ { "position": 1, "n": 58, "alpha_mean": 0.8730517241379312, "alpha_median": 0.8885000000000001, "ci95_low": 0.8504306034482759, "ci95_high": 0.893867672413793 }, { "position": 2, "n": 58, "alpha_mean": 0.7701206896551723, "alpha_median": 0.7775000000000001, "ci95_low": 0.7438866379310345, "ci95_high": 0.7976043103448276 }, { "position": 3, "n": 58, "alpha_mean": 0.6722241379310345, "alpha_median": 0.667, "ci95_low": 0.6396025862068966, "ci95_high": 0.7066556034482759 } ], "chat": [ { "position": 1, "n": 56, "alpha_mean": 0.7448392857142857, "alpha_median": 0.738, "ci95_low": 0.7026580357142858, "ci95_high": 0.7799857142857142 }, { "position": 2, "n": 56, "alpha_mean": 0.5742857142857144, "alpha_median": 0.572, "ci95_low": 0.5256375000000001, "ci95_high": 0.6192348214285716 }, { "position": 3, "n": 56, "alpha_mean": 0.45112500000000005, "alpha_median": 0.448, "ci95_low": 0.39828526785714286, "ci95_high": 0.5046437500000001 } ] } }, "curves-gemma-q8-dflash-f16": { "family": "curves", "target": "gemma-q8", "drafter": "dflash-f16", "k": 3, "n_excluded": 4, "n_matched": 174, "log_lines": 174, "max_alpha_diff": 5.0000000000050004e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8918666666666668, "alpha_median": 0.9045000000000001, "ci95_low": 0.8784491666666668, "ci95_high": 0.9039854166666665 }, { "position": 2, "n": 60, "alpha_mean": 0.7721166666666668, "alpha_median": 0.793, "ci95_low": 0.7504645833333331, "ci95_high": 0.7939008333333333 }, { "position": 3, "n": 60, "alpha_mean": 0.6459500000000001, "alpha_median": 0.6535, "ci95_low": 0.6191287500000001, "ci95_high": 0.6728866666666666 } ], "code": [ { "position": 1, "n": 58, "alpha_mean": 0.842655172413793, "alpha_median": 0.8594999999999999, "ci95_low": 0.8167754310344827, "ci95_high": 0.8669663793103449 }, { "position": 2, "n": 58, "alpha_mean": 0.7027068965517241, "alpha_median": 0.679, "ci95_low": 0.6676331896551723, "ci95_high": 0.7373814655172414 }, { "position": 3, "n": 58, "alpha_mean": 0.5888620689655173, "alpha_median": 0.5605, "ci95_low": 0.5459797413793104, "ci95_high": 0.6324487068965516 } ], "chat": [ { "position": 1, "n": 56, "alpha_mean": 0.6704821428571428, "alpha_median": 0.672, "ci95_low": 0.6234272321428571, "ci95_high": 0.713859375 }, { "position": 2, "n": 56, "alpha_mean": 0.43603571428571425, "alpha_median": 0.4165, "ci95_low": 0.3837392857142857, "ci95_high": 0.49034910714285707 }, { "position": 3, "n": 56, "alpha_mean": 0.2904642857142857, "alpha_median": 0.2595, "ci95_low": 0.2418611607142857, "ci95_high": 0.3433772321428571 } ] } }, "curves-gemma-q8-mtp": { "family": "curves", "target": "gemma-q8", "drafter": "mtp", "k": 3, "n_excluded": 4, "n_matched": 174, "log_lines": 174, "max_alpha_diff": 4.999999999999449e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.9138833333333335, "alpha_median": 0.9285000000000001, "ci95_low": 0.9010154166666666, "ci95_high": 0.9263 }, { "position": 2, "n": 60, "alpha_mean": 0.8341500000000001, "alpha_median": 0.848, "ci95_low": 0.8140654166666669, "ci95_high": 0.8542520833333334 }, { "position": 3, "n": 60, "alpha_mean": 0.7475666666666666, "alpha_median": 0.7625, "ci95_low": 0.7242616666666667, "ci95_high": 0.7714366666666669 } ], "code": [ { "position": 1, "n": 58, "alpha_mean": 0.8691551724137931, "alpha_median": 0.8654999999999999, "ci95_low": 0.8525469827586206, "ci95_high": 0.8869310344827587 }, { "position": 2, "n": 58, "alpha_mean": 0.7647413793103448, "alpha_median": 0.7455, "ci95_low": 0.7410340517241379, "ci95_high": 0.7910892241379309 }, { "position": 3, "n": 58, "alpha_mean": 0.6648103448275863, "alpha_median": 0.6515, "ci95_low": 0.6358267241379308, "ci95_high": 0.697656896551724 } ], "chat": [ { "position": 1, "n": 56, "alpha_mean": 0.758267857142857, "alpha_median": 0.7555000000000001, "ci95_low": 0.7149366071428571, "ci95_high": 0.7955558035714285 }, { "position": 2, "n": 56, "alpha_mean": 0.5967142857142856, "alpha_median": 0.587, "ci95_low": 0.549794642857143, "ci95_high": 0.641684375 }, { "position": 3, "n": 56, "alpha_mean": 0.4818928571428572, "alpha_median": 0.447, "ci95_low": 0.42783482142857143, "ci95_high": 0.5349348214285714 } ] } }, "curves-qwen-q4-dflash": { "family": "curves", "target": "qwen-q4", "drafter": "dflash-f16", "k": 3, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.0000000000000565e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.019950000000000002, "alpha_median": 0.016, "ci95_low": 0.016049583333333332, "ci95_high": 0.02385 }, { "position": 2, "n": 60, "alpha_mean": 0.0048000000000000004, "alpha_median": 0.004, "ci95_low": 0.0031995833333333334, "ci95_high": 0.006933333333333334 }, { "position": 3, "n": 60, "alpha_mean": 0.0017666666666666668, "alpha_median": 0.0, "ci95_low": 0.0010829166666666667, "ci95_high": 0.0025333333333333336 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.021466666666666672, "alpha_median": 0.02, "ci95_low": 0.01824916666666667, "ci95_high": 0.024750416666666664 }, { "position": 2, "n": 60, "alpha_mean": 0.004333333333333333, "alpha_median": 0.004, "ci95_low": 0.0033, "ci95_high": 0.005383333333333334 }, { "position": 3, "n": 60, "alpha_mean": 0.0017500000000000003, "alpha_median": 0.0, "ci95_low": 0.0011325, "ci95_high": 0.0024500000000000004 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.028413793103448277, "alpha_median": 0.0205, "ci95_low": 0.023344396551724135, "ci95_high": 0.033844827586206896 }, { "position": 2, "n": 58, "alpha_mean": 0.004206896551724138, "alpha_median": 0.004, "ci95_low": 0.003189655172413793, "ci95_high": 0.005276293103448275 }, { "position": 3, "n": 58, "alpha_mean": 0.001603448275862069, "alpha_median": 0.0, "ci95_low": 0.0009995689655172414, "ci95_high": 0.0022413793103448275 } ] } }, "curves-qwen-q4-dspark-p0": { "family": "curves", "target": "qwen-q4", "drafter": "dspark-p0", "k": 3, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 4.999999999999449e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8346833333333333, "alpha_median": 0.851, "ci95_low": 0.8120991666666666, "ci95_high": 0.8569516666666667 }, { "position": 2, "n": 60, "alpha_mean": 0.6921166666666666, "alpha_median": 0.6975, "ci95_low": 0.6581199999999999, "ci95_high": 0.7272737499999999 }, { "position": 3, "n": 60, "alpha_mean": 0.5783499999999999, "alpha_median": 0.5645, "ci95_low": 0.536965, "ci95_high": 0.6212970833333332 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.7782833333333333, "alpha_median": 0.7705, "ci95_low": 0.7597149999999999, "ci95_high": 0.7968850000000001 }, { "position": 2, "n": 60, "alpha_mean": 0.6005333333333333, "alpha_median": 0.593, "ci95_low": 0.5761483333333334, "ci95_high": 0.6253666666666666 }, { "position": 3, "n": 60, "alpha_mean": 0.4507166666666666, "alpha_median": 0.432, "ci95_low": 0.42258125, "ci95_high": 0.48155291666666666 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.6920172413793104, "alpha_median": 0.7335, "ci95_low": 0.6443961206896551, "ci95_high": 0.7338521551724138 }, { "position": 2, "n": 58, "alpha_mean": 0.4768793103448276, "alpha_median": 0.4975, "ci95_low": 0.4254219827586207, "ci95_high": 0.5239168103448276 }, { "position": 3, "n": 58, "alpha_mean": 0.3412068965517242, "alpha_median": 0.3415, "ci95_low": 0.2928254310344828, "ci95_high": 0.3870702586206896 } ] } }, "curves-qwen-q4-dspark-p2": { "family": "curves", "target": "qwen-q4", "drafter": "dspark-p2", "k": 3, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 4.999999999999449e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8345333333333333, "alpha_median": 0.851, "ci95_low": 0.8120466666666666, "ci95_high": 0.8567204166666667 }, { "position": 2, "n": 60, "alpha_mean": 0.6918, "alpha_median": 0.6975, "ci95_low": 0.6576162499999999, "ci95_high": 0.7269570833333333 }, { "position": 3, "n": 60, "alpha_mean": 0.5782833333333331, "alpha_median": 0.5645, "ci95_low": 0.5370287499999999, "ci95_high": 0.6215133333333334 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.7777999999999999, "alpha_median": 0.7705, "ci95_low": 0.7592299999999998, "ci95_high": 0.7964358333333332 }, { "position": 2, "n": 60, "alpha_mean": 0.6002666666666665, "alpha_median": 0.591, "ci95_low": 0.5757958333333333, "ci95_high": 0.6250004166666666 }, { "position": 3, "n": 60, "alpha_mean": 0.45085, "alpha_median": 0.432, "ci95_low": 0.4225320833333335, "ci95_high": 0.48181666666666667 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.6981206896551725, "alpha_median": 0.7404999999999999, "ci95_low": 0.6547439655172413, "ci95_high": 0.737828448275862 }, { "position": 2, "n": 58, "alpha_mean": 0.4825, "alpha_median": 0.5025, "ci95_low": 0.43263318965517245, "ci95_high": 0.5286422413793103 }, { "position": 3, "n": 58, "alpha_mean": 0.34689655172413797, "alpha_median": 0.3605, "ci95_low": 0.2989051724137931, "ci95_high": 0.3935047413793103 } ] } }, "curves-qwen-q4-dspark-p4": { "family": "curves", "target": "qwen-q4", "drafter": "dspark-p4", "k": 3, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.0000000000105516e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8353, "alpha_median": 0.852, "ci95_low": 0.8121125, "ci95_high": 0.8580170833333333 }, { "position": 2, "n": 60, "alpha_mean": 0.6887333333333332, "alpha_median": 0.7, "ci95_low": 0.6534300000000001, "ci95_high": 0.7252525000000002 }, { "position": 3, "n": 60, "alpha_mean": 0.5757499999999999, "alpha_median": 0.556, "ci95_low": 0.53395, "ci95_high": 0.6201391666666666 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.7805666666666666, "alpha_median": 0.789, "ci95_low": 0.7626000000000002, "ci95_high": 0.7979358333333333 }, { "position": 2, "n": 60, "alpha_mean": 0.6028500000000001, "alpha_median": 0.5945, "ci95_low": 0.5796620833333332, "ci95_high": 0.6280012500000001 }, { "position": 3, "n": 60, "alpha_mean": 0.44961666666666666, "alpha_median": 0.4325, "ci95_low": 0.4218633333333333, "ci95_high": 0.47848999999999997 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.7250344827586207, "alpha_median": 0.737, "ci95_low": 0.6934771551724138, "ci95_high": 0.7548646551724139 }, { "position": 2, "n": 58, "alpha_mean": 0.48151724137931035, "alpha_median": 0.48, "ci95_low": 0.43344655172413793, "ci95_high": 0.5258487068965518 }, { "position": 3, "n": 58, "alpha_mean": 0.32974137931034475, "alpha_median": 0.3185, "ci95_low": 0.28190818965517245, "ci95_high": 0.3738103448275862 } ] } }, "curves-qwen-q4-dspark-p6": { "family": "curves", "target": "qwen-q4", "drafter": "dspark-p6", "k": 3, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.0000000000105516e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.84375, "alpha_median": 0.8585, "ci95_low": 0.8235158333333334, "ci95_high": 0.86403375 }, { "position": 2, "n": 60, "alpha_mean": 0.6838833333333333, "alpha_median": 0.696, "ci95_low": 0.6492825000000001, "ci95_high": 0.7188870833333332 }, { "position": 3, "n": 60, "alpha_mean": 0.5639166666666667, "alpha_median": 0.543, "ci95_low": 0.521315, "ci95_high": 0.60840125 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.7965166666666668, "alpha_median": 0.792, "ci95_low": 0.7783312500000001, "ci95_high": 0.8139354166666666 }, { "position": 2, "n": 60, "alpha_mean": 0.5829666666666667, "alpha_median": 0.585, "ci95_low": 0.5583320833333334, "ci95_high": 0.6082204166666667 }, { "position": 3, "n": 60, "alpha_mean": 0.4116666666666667, "alpha_median": 0.4155, "ci95_low": 0.38426375, "ci95_high": 0.4407420833333332 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.7730000000000001, "alpha_median": 0.7805, "ci95_low": 0.7447702586206895, "ci95_high": 0.7983125000000001 }, { "position": 2, "n": 58, "alpha_mean": 0.448551724137931, "alpha_median": 0.446, "ci95_low": 0.403296551724138, "ci95_high": 0.4906043103448276 }, { "position": 3, "n": 58, "alpha_mean": 0.2883448275862069, "alpha_median": 0.27, "ci95_low": 0.24225258620689655, "ci95_high": 0.33291551724137936 } ] } }, "curves-qwen-q4-eagle3": { "family": "curves", "target": "qwen-q4", "drafter": "eagle3", "k": 3, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.0000000000050004e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.65895, "alpha_median": 0.641, "ci95_low": 0.6398808333333335, "ci95_high": 0.6790333333333333 }, { "position": 2, "n": 60, "alpha_mean": 0.4375, "alpha_median": 0.434, "ci95_low": 0.4135162499999999, "ci95_high": 0.46195458333333334 }, { "position": 3, "n": 60, "alpha_mean": 0.23758333333333329, "alpha_median": 0.2375, "ci95_low": 0.21839874999999997, "ci95_high": 0.25863375 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.6724000000000001, "alpha_median": 0.6715, "ci95_low": 0.652665, "ci95_high": 0.6924891666666666 }, { "position": 2, "n": 60, "alpha_mean": 0.43201666666666666, "alpha_median": 0.4195, "ci95_low": 0.40646541666666675, "ci95_high": 0.45819 }, { "position": 3, "n": 60, "alpha_mean": 0.22091666666666668, "alpha_median": 0.1965, "ci95_low": 0.1931775, "ci95_high": 0.2508708333333333 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.5734310344827588, "alpha_median": 0.6125, "ci95_low": 0.5173581896551724, "ci95_high": 0.624796551724138 }, { "position": 2, "n": 58, "alpha_mean": 0.3631379310344828, "alpha_median": 0.3525, "ci95_low": 0.30766336206896555, "ci95_high": 0.41653577586206897 }, { "position": 3, "n": 58, "alpha_mean": 0.21055172413793102, "alpha_median": 0.165, "ci95_low": 0.16535862068965515, "ci95_high": 0.25851896551724135 } ] } }, "curves-qwen-q4-vanilla17b": { "family": "curves", "target": "qwen-q4", "drafter": "vanilla17b", "k": 3, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 4.999999999999449e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8724666666666665, "alpha_median": 0.87, "ci95_low": 0.8594166666666668, "ci95_high": 0.8848845833333332 }, { "position": 2, "n": 60, "alpha_mean": 0.7706000000000002, "alpha_median": 0.776, "ci95_low": 0.7497649999999999, "ci95_high": 0.7904025 }, { "position": 3, "n": 60, "alpha_mean": 0.6854666666666667, "alpha_median": 0.6905, "ci95_low": 0.6592325, "ci95_high": 0.7101183333333334 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.8352166666666668, "alpha_median": 0.829, "ci95_low": 0.8234154166666667, "ci95_high": 0.8467004166666667 }, { "position": 2, "n": 60, "alpha_mean": 0.7095166666666667, "alpha_median": 0.7055, "ci95_low": 0.6907325, "ci95_high": 0.7286520833333333 }, { "position": 3, "n": 60, "alpha_mean": 0.6087000000000001, "alpha_median": 0.6065, "ci95_low": 0.58583125, "ci95_high": 0.6329516666666665 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.7829482758620689, "alpha_median": 0.7745, "ci95_low": 0.7622922413793102, "ci95_high": 0.803225 }, { "position": 2, "n": 58, "alpha_mean": 0.6254310344827587, "alpha_median": 0.608, "ci95_low": 0.5945271551724138, "ci95_high": 0.6549883620689654 }, { "position": 3, "n": 58, "alpha_mean": 0.5063620689655173, "alpha_median": 0.488, "ci95_low": 0.46864137931034483, "ci95_high": 0.5435073275862069 } ] } }, "curves-qwen-q5-dflash": { "family": "curves", "target": "qwen-q5", "drafter": "dflash-f16", "k": 3, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.0000000000000565e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.01966666666666667, "alpha_median": 0.016, "ci95_low": 0.016115, "ci95_high": 0.023299999999999998 }, { "position": 2, "n": 60, "alpha_mean": 0.0044666666666666665, "alpha_median": 0.004, "ci95_low": 0.0032166666666666667, "ci95_high": 0.005866666666666667 }, { "position": 3, "n": 60, "alpha_mean": 0.0022166666666666667, "alpha_median": 0.0, "ci95_low": 0.0014833333333333332, "ci95_high": 0.00305 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.0203, "alpha_median": 0.02, "ci95_low": 0.017382083333333336, "ci95_high": 0.023333749999999997 }, { "position": 2, "n": 60, "alpha_mean": 0.004783333333333334, "alpha_median": 0.004, "ci95_low": 0.00341625, "ci95_high": 0.006250000000000001 }, { "position": 3, "n": 60, "alpha_mean": 0.0018666666666666666, "alpha_median": 0.0, "ci95_low": 0.0012000000000000001, "ci95_high": 0.0026000000000000003 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.02643103448275862, "alpha_median": 0.024, "ci95_low": 0.02167241379310345, "ci95_high": 0.031810344827586205 }, { "position": 2, "n": 58, "alpha_mean": 0.00353448275862069, "alpha_median": 0.004, "ci95_low": 0.00256896551724138, "ci95_high": 0.0044482758620689655 }, { "position": 3, "n": 58, "alpha_mean": 0.0013103448275862072, "alpha_median": 0.0, "ci95_low": 0.0007586206896551724, "ci95_high": 0.001931034482758621 } ] } }, "curves-qwen-q5-dspark-p0": { "family": "curves", "target": "qwen-q5", "drafter": "dspark-p0", "k": 3, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 4.999999999999449e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8288666666666666, "alpha_median": 0.832, "ci95_low": 0.8069320833333334, "ci95_high": 0.8511508333333333 }, { "position": 2, "n": 60, "alpha_mean": 0.6884666666666668, "alpha_median": 0.6895, "ci95_low": 0.65485, "ci95_high": 0.7222670833333332 }, { "position": 3, "n": 60, "alpha_mean": 0.58035, "alpha_median": 0.5834999999999999, "ci95_low": 0.53963125, "ci95_high": 0.62223375 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.7708666666666666, "alpha_median": 0.7685, "ci95_low": 0.7531162500000002, "ci95_high": 0.7895529166666666 }, { "position": 2, "n": 60, "alpha_mean": 0.5833333333333334, "alpha_median": 0.5814999999999999, "ci95_low": 0.5556662500000001, "ci95_high": 0.61250375 }, { "position": 3, "n": 60, "alpha_mean": 0.43556666666666666, "alpha_median": 0.4285, "ci95_low": 0.4027470833333333, "ci95_high": 0.4700704166666666 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.701741379310345, "alpha_median": 0.7355, "ci95_low": 0.6575254310344828, "ci95_high": 0.7423306034482758 }, { "position": 2, "n": 58, "alpha_mean": 0.4863793103448276, "alpha_median": 0.4925, "ci95_low": 0.43637887931034475, "ci95_high": 0.5348375 }, { "position": 3, "n": 58, "alpha_mean": 0.3533965517241379, "alpha_median": 0.3285, "ci95_low": 0.30560000000000004, "ci95_high": 0.40571034482758617 } ] } }, "curves-qwen-q5-eagle3": { "family": "curves", "target": "qwen-q5", "drafter": "eagle3", "k": 3, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.000000000000837e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.6617833333333335, "alpha_median": 0.6515, "ci95_low": 0.6442812500000001, "ci95_high": 0.6797862499999999 }, { "position": 2, "n": 60, "alpha_mean": 0.4344, "alpha_median": 0.429, "ci95_low": 0.41286291666666664, "ci95_high": 0.4559833333333333 }, { "position": 3, "n": 60, "alpha_mean": 0.23328333333333337, "alpha_median": 0.2145, "ci95_low": 0.21618, "ci95_high": 0.2505683333333333 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.6694, "alpha_median": 0.6685000000000001, "ci95_low": 0.6487329166666667, "ci95_high": 0.69080125 }, { "position": 2, "n": 60, "alpha_mean": 0.4252666666666667, "alpha_median": 0.4245, "ci95_low": 0.39917749999999996, "ci95_high": 0.45331958333333333 }, { "position": 3, "n": 60, "alpha_mean": 0.21823333333333333, "alpha_median": 0.196, "ci95_low": 0.19128250000000002, "ci95_high": 0.24852166666666667 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.5931379310344828, "alpha_median": 0.6385000000000001, "ci95_low": 0.5417918103448275, "ci95_high": 0.643225 }, { "position": 2, "n": 58, "alpha_mean": 0.36172413793103453, "alpha_median": 0.362, "ci95_low": 0.3137870689655172, "ci95_high": 0.41031422413793095 }, { "position": 3, "n": 58, "alpha_mean": 0.20020689655172413, "alpha_median": 0.163, "ci95_low": 0.16113534482758618, "ci95_high": 0.24240387931034482 } ] } }, "curves-qwen-q8-dspark-p0": { "family": "curves", "target": "qwen-q8", "drafter": "dspark-p0", "k": 3, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.000000000002225e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8263666666666667, "alpha_median": 0.8365, "ci95_low": 0.8036991666666665, "ci95_high": 0.8494 }, { "position": 2, "n": 60, "alpha_mean": 0.6871833333333334, "alpha_median": 0.692, "ci95_low": 0.6528645833333334, "ci95_high": 0.7224837499999999 }, { "position": 3, "n": 60, "alpha_mean": 0.5735833333333334, "alpha_median": 0.5814999999999999, "ci95_low": 0.5308795833333334, "ci95_high": 0.61690375 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.77365, "alpha_median": 0.772, "ci95_low": 0.7569454166666667, "ci95_high": 0.7908845833333332 }, { "position": 2, "n": 60, "alpha_mean": 0.5854666666666666, "alpha_median": 0.579, "ci95_low": 0.5615266666666666, "ci95_high": 0.6115545833333333 }, { "position": 3, "n": 60, "alpha_mean": 0.44101666666666667, "alpha_median": 0.4405, "ci95_low": 0.41631583333333333, "ci95_high": 0.46805041666666664 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.6959827586206897, "alpha_median": 0.721, "ci95_low": 0.6521508620689656, "ci95_high": 0.7347081896551725 }, { "position": 2, "n": 58, "alpha_mean": 0.4865862068965517, "alpha_median": 0.483, "ci95_low": 0.441, "ci95_high": 0.5306064655172413 }, { "position": 3, "n": 58, "alpha_mean": 0.3454137931034482, "alpha_median": 0.3235, "ci95_low": 0.3001, "ci95_high": 0.39220775862068963 } ] } }, "curves-qwen-q8-eagle3": { "family": "curves", "target": "qwen-q8", "drafter": "eagle3", "k": 3, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.0000000000050004e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.6503833333333334, "alpha_median": 0.6445000000000001, "ci95_low": 0.6334787499999999, "ci95_high": 0.6669016666666667 }, { "position": 2, "n": 60, "alpha_mean": 0.4284833333333333, "alpha_median": 0.418, "ci95_low": 0.41021583333333345, "ci95_high": 0.4492220833333333 }, { "position": 3, "n": 60, "alpha_mean": 0.23021666666666665, "alpha_median": 0.2345, "ci95_low": 0.21416541666666666, "ci95_high": 0.24763791666666662 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.6651166666666666, "alpha_median": 0.6535, "ci95_low": 0.6472791666666666, "ci95_high": 0.6835170833333333 }, { "position": 2, "n": 60, "alpha_mean": 0.42485, "alpha_median": 0.4165, "ci95_low": 0.40288041666666663, "ci95_high": 0.4486516666666667 }, { "position": 3, "n": 60, "alpha_mean": 0.2141, "alpha_median": 0.189, "ci95_low": 0.19004458333333332, "ci95_high": 0.24040083333333337 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.5834310344827586, "alpha_median": 0.637, "ci95_low": 0.5310758620689655, "ci95_high": 0.6309508620689654 }, { "position": 2, "n": 58, "alpha_mean": 0.3570862068965517, "alpha_median": 0.372, "ci95_low": 0.3111849137931035, "ci95_high": 0.4010762931034482 }, { "position": 3, "n": 58, "alpha_mean": 0.1905, "alpha_median": 0.18, "ci95_low": 0.15682155172413792, "ci95_high": 0.22692456896551721 } ] } }, "ksweep-gemma-q4-dflash-f16-k10": { "family": "ksweep", "target": "gemma-q4", "drafter": "dflash-f16", "k": 10, "n_excluded": 3, "n_matched": 175, "log_lines": 175, "max_alpha_diff": 5.0000000000050004e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8581333333333333, "alpha_median": 0.86, "ci95_low": 0.8396154166666667, "ci95_high": 0.8742508333333333 }, { "position": 2, "n": 60, "alpha_mean": 0.7290666666666668, "alpha_median": 0.7255, "ci95_low": 0.70286625, "ci95_high": 0.7540858333333336 }, { "position": 3, "n": 60, "alpha_mean": 0.58595, "alpha_median": 0.5934999999999999, "ci95_low": 0.5582470833333333, "ci95_high": 0.6129883333333332 }, { "position": 4, "n": 60, "alpha_mean": 0.4879833333333333, "alpha_median": 0.471, "ci95_low": 0.4562466666666666, "ci95_high": 0.51908375 }, { "position": 5, "n": 60, "alpha_mean": 0.3957166666666667, "alpha_median": 0.375, "ci95_low": 0.36298125, "ci95_high": 0.42700749999999993 }, { "position": 6, "n": 60, "alpha_mean": 0.3183333333333333, "alpha_median": 0.303, "ci95_low": 0.28596375000000007, "ci95_high": 0.3491191666666666 }, { "position": 7, "n": 60, "alpha_mean": 0.2571666666666667, "alpha_median": 0.2255, "ci95_low": 0.2281825, "ci95_high": 0.284435 }, { "position": 8, "n": 60, "alpha_mean": 0.2046666666666667, "alpha_median": 0.184, "ci95_low": 0.17776083333333334, "ci95_high": 0.23036874999999998 }, { "position": 9, "n": 60, "alpha_mean": 0.1633666666666667, "alpha_median": 0.1615, "ci95_low": 0.14023, "ci95_high": 0.1857675 }, { "position": 10, "n": 60, "alpha_mean": 0.119, "alpha_median": 0.10200000000000001, "ci95_low": 0.09944916666666667, "ci95_high": 0.13833458333333334 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.7975833333333332, "alpha_median": 0.805, "ci95_low": 0.7638662500000001, "ci95_high": 0.8274833333333335 }, { "position": 2, "n": 60, "alpha_mean": 0.6294500000000001, "alpha_median": 0.6485000000000001, "ci95_low": 0.5893558333333334, "ci95_high": 0.66490375 }, { "position": 3, "n": 60, "alpha_mean": 0.4983833333333333, "alpha_median": 0.492, "ci95_low": 0.45793291666666663, "ci95_high": 0.5402670833333332 }, { "position": 4, "n": 60, "alpha_mean": 0.3997666666666666, "alpha_median": 0.3985, "ci95_low": 0.3604479166666667, "ci95_high": 0.44135541666666667 }, { "position": 5, "n": 60, "alpha_mean": 0.3185666666666666, "alpha_median": 0.2845, "ci95_low": 0.28028624999999996, "ci95_high": 0.35915333333333327 }, { "position": 6, "n": 60, "alpha_mean": 0.25938333333333335, "alpha_median": 0.225, "ci95_low": 0.22559875, "ci95_high": 0.29620625 }, { "position": 7, "n": 60, "alpha_mean": 0.20616666666666666, "alpha_median": 0.176, "ci95_low": 0.175915, "ci95_high": 0.24012041666666661 }, { "position": 8, "n": 60, "alpha_mean": 0.16380000000000003, "alpha_median": 0.1355, "ci95_low": 0.13546458333333336, "ci95_high": 0.19675666666666658 }, { "position": 9, "n": 60, "alpha_mean": 0.13241666666666668, "alpha_median": 0.0935, "ci95_low": 0.10526500000000001, "ci95_high": 0.16395499999999996 }, { "position": 10, "n": 60, "alpha_mean": 0.10470000000000002, "alpha_median": 0.0765, "ci95_low": 0.08091625000000001, "ci95_high": 0.13463999999999995 } ], "chat": [ { "position": 1, "n": 55, "alpha_mean": 0.6401454545454547, "alpha_median": 0.667, "ci95_low": 0.5908345454545455, "ci95_high": 0.6839649999999998 }, { "position": 2, "n": 55, "alpha_mean": 0.4230181818181819, "alpha_median": 0.422, "ci95_low": 0.36814090909090913, "ci95_high": 0.47485499999999997 }, { "position": 3, "n": 55, "alpha_mean": 0.2565090909090909, "alpha_median": 0.206, "ci95_low": 0.21030545454545452, "ci95_high": 0.3052913636363636 }, { "position": 4, "n": 55, "alpha_mean": 0.1807272727272727, "alpha_median": 0.137, "ci95_low": 0.13614363636363633, "ci95_high": 0.2264454545454545 }, { "position": 5, "n": 55, "alpha_mean": 0.1324, "alpha_median": 0.082, "ci95_low": 0.09220954545454546, "ci95_high": 0.17708409090909083 }, { "position": 6, "n": 55, "alpha_mean": 0.09461818181818181, "alpha_median": 0.041, "ci95_low": 0.05852681818181819, "ci95_high": 0.13318727272727268 }, { "position": 7, "n": 55, "alpha_mean": 0.07121818181818182, "alpha_median": 0.02, "ci95_low": 0.03836272727272727, "ci95_high": 0.10632727272727273 }, { "position": 8, "n": 55, "alpha_mean": 0.05894545454545455, "alpha_median": 0.011, "ci95_low": 0.029181818181818187, "ci95_high": 0.09329181818181818 }, { "position": 9, "n": 55, "alpha_mean": 0.04472727272727272, "alpha_median": 0.009, "ci95_low": 0.018727272727272728, "ci95_high": 0.07535272727272724 }, { "position": 10, "n": 55, "alpha_mean": 0.03549090909090909, "alpha_median": 0.0, "ci95_low": 0.013397727272727275, "ci95_high": 0.06263681818181818 } ] } }, "ksweep-gemma-q4-dflash-f16-k5": { "family": "ksweep", "target": "gemma-q4", "drafter": "dflash-f16", "k": 5, "n_excluded": 3, "n_matched": 175, "log_lines": 175, "max_alpha_diff": 4.999999999999449e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8648166666666665, "alpha_median": 0.8805000000000001, "ci95_low": 0.84664875, "ci95_high": 0.8817375 }, { "position": 2, "n": 60, "alpha_mean": 0.7476166666666665, "alpha_median": 0.754, "ci95_low": 0.7237820833333334, "ci95_high": 0.7708 }, { "position": 3, "n": 60, "alpha_mean": 0.6148666666666667, "alpha_median": 0.618, "ci95_low": 0.5844333333333334, "ci95_high": 0.6425420833333334 }, { "position": 4, "n": 60, "alpha_mean": 0.5182333333333334, "alpha_median": 0.504, "ci95_low": 0.48459916666666675, "ci95_high": 0.5506170833333333 }, { "position": 5, "n": 60, "alpha_mean": 0.43478333333333324, "alpha_median": 0.437, "ci95_low": 0.40129875000000004, "ci95_high": 0.4672016666666667 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.8290833333333333, "alpha_median": 0.831, "ci95_low": 0.8066158333333333, "ci95_high": 0.8515845833333333 }, { "position": 2, "n": 60, "alpha_mean": 0.6821666666666667, "alpha_median": 0.68, "ci95_low": 0.6523333333333334, "ci95_high": 0.7136345833333332 }, { "position": 3, "n": 60, "alpha_mean": 0.5617, "alpha_median": 0.5665, "ci95_low": 0.5239529166666667, "ci95_high": 0.5996862499999999 }, { "position": 4, "n": 60, "alpha_mean": 0.46573333333333333, "alpha_median": 0.462, "ci95_low": 0.42466125, "ci95_high": 0.5074033333333333 }, { "position": 5, "n": 60, "alpha_mean": 0.39039999999999997, "alpha_median": 0.377, "ci95_low": 0.34859750000000006, "ci95_high": 0.4326854166666667 } ], "chat": [ { "position": 1, "n": 55, "alpha_mean": 0.6564363636363636, "alpha_median": 0.68, "ci95_low": 0.6045904545454546, "ci95_high": 0.7042477272727272 }, { "position": 2, "n": 55, "alpha_mean": 0.4465636363636364, "alpha_median": 0.413, "ci95_low": 0.39149863636363635, "ci95_high": 0.5005781818181817 }, { "position": 3, "n": 55, "alpha_mean": 0.2991818181818182, "alpha_median": 0.25, "ci95_low": 0.24532045454545454, "ci95_high": 0.35314772727272725 }, { "position": 4, "n": 55, "alpha_mean": 0.2236545454545455, "alpha_median": 0.149, "ci95_low": 0.1724536363636364, "ci95_high": 0.2765854545454545 }, { "position": 5, "n": 55, "alpha_mean": 0.1691272727272728, "alpha_median": 0.106, "ci95_low": 0.11848954545454544, "ci95_high": 0.2225631818181817 } ] } }, "ksweep-gemma-q4-dflash-f16-k7": { "family": "ksweep", "target": "gemma-q4", "drafter": "dflash-f16", "k": 7, "n_excluded": 3, "n_matched": 175, "log_lines": 175, "max_alpha_diff": 5.0000000000050004e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8579166666666665, "alpha_median": 0.867, "ci95_low": 0.8395308333333333, "ci95_high": 0.8751016666666668 }, { "position": 2, "n": 60, "alpha_mean": 0.7313999999999999, "alpha_median": 0.7404999999999999, "ci95_low": 0.7069320833333336, "ci95_high": 0.7555179166666667 }, { "position": 3, "n": 60, "alpha_mean": 0.5955666666666666, "alpha_median": 0.605, "ci95_low": 0.5640833333333333, "ci95_high": 0.6246870833333333 }, { "position": 4, "n": 60, "alpha_mean": 0.49323333333333336, "alpha_median": 0.482, "ci95_low": 0.45769624999999997, "ci95_high": 0.5276183333333333 }, { "position": 5, "n": 60, "alpha_mean": 0.4169, "alpha_median": 0.391, "ci95_low": 0.38221625000000004, "ci95_high": 0.4508516666666667 }, { "position": 6, "n": 60, "alpha_mean": 0.35129999999999995, "alpha_median": 0.333, "ci95_low": 0.3176629166666667, "ci95_high": 0.38515333333333324 }, { "position": 7, "n": 60, "alpha_mean": 0.28978333333333334, "alpha_median": 0.2665, "ci95_low": 0.2591608333333334, "ci95_high": 0.32101874999999996 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.8238166666666665, "alpha_median": 0.823, "ci95_low": 0.7995483333333334, "ci95_high": 0.8470179166666668 }, { "position": 2, "n": 60, "alpha_mean": 0.6661166666666667, "alpha_median": 0.667, "ci95_low": 0.633095, "ci95_high": 0.6982841666666669 }, { "position": 3, "n": 60, "alpha_mean": 0.5352166666666667, "alpha_median": 0.534, "ci95_low": 0.49966374999999996, "ci95_high": 0.5731175000000002 }, { "position": 4, "n": 60, "alpha_mean": 0.4425833333333334, "alpha_median": 0.441, "ci95_low": 0.4045970833333334, "ci95_high": 0.48081 }, { "position": 5, "n": 60, "alpha_mean": 0.36938333333333334, "alpha_median": 0.3695, "ci95_low": 0.33084708333333335, "ci95_high": 0.40810291666666665 }, { "position": 6, "n": 60, "alpha_mean": 0.3013, "alpha_median": 0.2845, "ci95_low": 0.26406583333333333, "ci95_high": 0.33970374999999986 }, { "position": 7, "n": 60, "alpha_mean": 0.24645, "alpha_median": 0.2235, "ci95_low": 0.2119533333333333, "ci95_high": 0.2845670833333334 } ], "chat": [ { "position": 1, "n": 55, "alpha_mean": 0.6426181818181819, "alpha_median": 0.658, "ci95_low": 0.5934681818181818, "ci95_high": 0.6863299999999998 }, { "position": 2, "n": 55, "alpha_mean": 0.4292727272727273, "alpha_median": 0.409, "ci95_low": 0.3773263636363636, "ci95_high": 0.4804581818181817 }, { "position": 3, "n": 55, "alpha_mean": 0.2806, "alpha_median": 0.242, "ci95_low": 0.23070636363636363, "ci95_high": 0.33234681818181816 }, { "position": 4, "n": 55, "alpha_mean": 0.20427272727272722, "alpha_median": 0.151, "ci95_low": 0.15617499999999998, "ci95_high": 0.2536563636363636 }, { "position": 5, "n": 55, "alpha_mean": 0.1530363636363636, "alpha_median": 0.087, "ci95_low": 0.10728318181818182, "ci95_high": 0.20192045454545446 }, { "position": 6, "n": 55, "alpha_mean": 0.12018181818181817, "alpha_median": 0.051, "ci95_low": 0.07734409090909093, "ci95_high": 0.1675277272727273 }, { "position": 7, "n": 55, "alpha_mean": 0.09421818181818181, "alpha_median": 0.028, "ci95_low": 0.052829999999999995, "ci95_high": 0.14032772727272727 } ] } }, "ksweep-gemma-q4-mtp-k10": { "family": "ksweep", "target": "gemma-q4", "drafter": "mtp", "k": 10, "n_excluded": 3, "n_matched": 175, "log_lines": 175, "max_alpha_diff": 5.0000000000105516e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8836833333333335, "alpha_median": 0.8905000000000001, "ci95_low": 0.8681633333333334, "ci95_high": 0.8991691666666666 }, { "position": 2, "n": 60, "alpha_mean": 0.7941166666666667, "alpha_median": 0.8, "ci95_low": 0.7705591666666666, "ci95_high": 0.8170041666666669 }, { "position": 3, "n": 60, "alpha_mean": 0.7047999999999999, "alpha_median": 0.7195, "ci95_low": 0.6783975000000001, "ci95_high": 0.7299704166666666 }, { "position": 4, "n": 60, "alpha_mean": 0.6122833333333333, "alpha_median": 0.6385000000000001, "ci95_low": 0.5747329166666666, "ci95_high": 0.6443341666666665 }, { "position": 5, "n": 60, "alpha_mean": 0.5341833333333332, "alpha_median": 0.549, "ci95_low": 0.49784874999999995, "ci95_high": 0.5669670833333333 }, { "position": 6, "n": 60, "alpha_mean": 0.45953333333333324, "alpha_median": 0.463, "ci95_low": 0.42431375, "ci95_high": 0.49294 }, { "position": 7, "n": 60, "alpha_mean": 0.3834, "alpha_median": 0.3735, "ci95_low": 0.3501558333333333, "ci95_high": 0.4155025 }, { "position": 8, "n": 60, "alpha_mean": 0.3148833333333334, "alpha_median": 0.3115, "ci95_low": 0.2848829166666666, "ci95_high": 0.3450333333333333 }, { "position": 9, "n": 60, "alpha_mean": 0.25093333333333334, "alpha_median": 0.2415, "ci95_low": 0.22318125000000003, "ci95_high": 0.28111875000000003 }, { "position": 10, "n": 60, "alpha_mean": 0.20245, "alpha_median": 0.2, "ci95_low": 0.17943208333333335, "ci95_high": 0.22818499999999997 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.8295333333333333, "alpha_median": 0.8434999999999999, "ci95_low": 0.80103125, "ci95_high": 0.8527508333333333 }, { "position": 2, "n": 60, "alpha_mean": 0.7212499999999998, "alpha_median": 0.732, "ci95_low": 0.6896574999999999, "ci95_high": 0.7506525000000001 }, { "position": 3, "n": 60, "alpha_mean": 0.6143499999999998, "alpha_median": 0.604, "ci95_low": 0.5809475, "ci95_high": 0.6462687500000001 }, { "position": 4, "n": 60, "alpha_mean": 0.5350666666666667, "alpha_median": 0.523, "ci95_low": 0.5002779166666668, "ci95_high": 0.5712199999999998 }, { "position": 5, "n": 60, "alpha_mean": 0.46813333333333335, "alpha_median": 0.4495, "ci95_low": 0.43272666666666665, "ci95_high": 0.5032670833333333 }, { "position": 6, "n": 60, "alpha_mean": 0.39171666666666666, "alpha_median": 0.377, "ci95_low": 0.35819833333333334, "ci95_high": 0.4262558333333332 }, { "position": 7, "n": 60, "alpha_mean": 0.3330666666666667, "alpha_median": 0.309, "ci95_low": 0.2998495833333334, "ci95_high": 0.36738375 }, { "position": 8, "n": 60, "alpha_mean": 0.27935, "alpha_median": 0.25, "ci95_low": 0.2476816666666667, "ci95_high": 0.3136854166666667 }, { "position": 9, "n": 60, "alpha_mean": 0.23356666666666667, "alpha_median": 0.214, "ci95_low": 0.2039, "ci95_high": 0.2661545833333333 }, { "position": 10, "n": 60, "alpha_mean": 0.18168333333333336, "alpha_median": 0.159, "ci95_low": 0.1529316666666667, "ci95_high": 0.21411833333333333 } ], "chat": [ { "position": 1, "n": 55, "alpha_mean": 0.7337636363636365, "alpha_median": 0.747, "ci95_low": 0.6890713636363638, "ci95_high": 0.7732472727272728 }, { "position": 2, "n": 55, "alpha_mean": 0.5758181818181819, "alpha_median": 0.569, "ci95_low": 0.5250227272727273, "ci95_high": 0.6269495454545455 }, { "position": 3, "n": 55, "alpha_mean": 0.44114545454545445, "alpha_median": 0.388, "ci95_low": 0.384850909090909, "ci95_high": 0.5014190909090909 }, { "position": 4, "n": 55, "alpha_mean": 0.34834545454545457, "alpha_median": 0.277, "ci95_low": 0.28816, "ci95_high": 0.4101668181818182 }, { "position": 5, "n": 55, "alpha_mean": 0.26996363636363635, "alpha_median": 0.192, "ci95_low": 0.21188772727272728, "ci95_high": 0.3302231818181817 }, { "position": 6, "n": 55, "alpha_mean": 0.22503636363636365, "alpha_median": 0.139, "ci95_low": 0.1671940909090909, "ci95_high": 0.28587045454545446 }, { "position": 7, "n": 55, "alpha_mean": 0.19132727272727276, "alpha_median": 0.092, "ci95_low": 0.13407136363636363, "ci95_high": 0.25535454545454545 }, { "position": 8, "n": 55, "alpha_mean": 0.16399999999999998, "alpha_median": 0.069, "ci95_low": 0.10636363636363637, "ci95_high": 0.22853 }, { "position": 9, "n": 55, "alpha_mean": 0.1394, "alpha_median": 0.043, "ci95_low": 0.08196227272727274, "ci95_high": 0.20359090909090902 }, { "position": 10, "n": 55, "alpha_mean": 0.12441818181818183, "alpha_median": 0.028, "ci95_low": 0.065955, "ci95_high": 0.18841636363636347 } ] } }, "ksweep-gemma-q4-mtp-k5": { "family": "ksweep", "target": "gemma-q4", "drafter": "mtp", "k": 5, "n_excluded": 3, "n_matched": 175, "log_lines": 175, "max_alpha_diff": 5.0000000000050004e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8950166666666667, "alpha_median": 0.8975, "ci95_low": 0.8799124999999999, "ci95_high": 0.9086012499999998 }, { "position": 2, "n": 60, "alpha_mean": 0.8061833333333333, "alpha_median": 0.8085, "ci95_low": 0.7858991666666666, "ci95_high": 0.8246529166666665 }, { "position": 3, "n": 60, "alpha_mean": 0.7245333333333333, "alpha_median": 0.743, "ci95_low": 0.6977758333333333, "ci95_high": 0.7487174999999999 }, { "position": 4, "n": 60, "alpha_mean": 0.6364333333333334, "alpha_median": 0.653, "ci95_low": 0.6066799999999999, "ci95_high": 0.6637341666666666 }, { "position": 5, "n": 60, "alpha_mean": 0.5607833333333334, "alpha_median": 0.56, "ci95_low": 0.5305791666666666, "ci95_high": 0.5900512499999999 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.8677666666666666, "alpha_median": 0.885, "ci95_low": 0.8488662499999999, "ci95_high": 0.88593375 }, { "position": 2, "n": 60, "alpha_mean": 0.7681333333333333, "alpha_median": 0.774, "ci95_low": 0.7418133333333332, "ci95_high": 0.7935254166666666 }, { "position": 3, "n": 60, "alpha_mean": 0.6667833333333332, "alpha_median": 0.661, "ci95_low": 0.6357720833333335, "ci95_high": 0.6976683333333333 }, { "position": 4, "n": 60, "alpha_mean": 0.5806333333333333, "alpha_median": 0.568, "ci95_low": 0.544045, "ci95_high": 0.6166670833333333 }, { "position": 5, "n": 60, "alpha_mean": 0.5109833333333332, "alpha_median": 0.5, "ci95_low": 0.47249791666666663, "ci95_high": 0.5504029166666666 } ], "chat": [ { "position": 1, "n": 55, "alpha_mean": 0.7516363636363635, "alpha_median": 0.757, "ci95_low": 0.7077790909090909, "ci95_high": 0.7898595454545454 }, { "position": 2, "n": 55, "alpha_mean": 0.5932545454545456, "alpha_median": 0.585, "ci95_low": 0.5417622727272728, "ci95_high": 0.6486572727272726 }, { "position": 3, "n": 55, "alpha_mean": 0.48161818181818183, "alpha_median": 0.412, "ci95_low": 0.42208954545454547, "ci95_high": 0.5441836363636364 }, { "position": 4, "n": 55, "alpha_mean": 0.39509090909090916, "alpha_median": 0.325, "ci95_low": 0.3306468181818182, "ci95_high": 0.46265500000000004 }, { "position": 5, "n": 55, "alpha_mean": 0.32367272727272733, "alpha_median": 0.219, "ci95_low": 0.25792818181818183, "ci95_high": 0.3945495454545453 } ] } }, "ksweep-gemma-q4-mtp-k7": { "family": "ksweep", "target": "gemma-q4", "drafter": "mtp", "k": 7, "n_excluded": 3, "n_matched": 175, "log_lines": 175, "max_alpha_diff": 5.0000000000050004e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8807333333333333, "alpha_median": 0.883, "ci95_low": 0.8624995833333334, "ci95_high": 0.8964683333333335 }, { "position": 2, "n": 60, "alpha_mean": 0.79455, "alpha_median": 0.8025, "ci95_low": 0.7706141666666667, "ci95_high": 0.8169733333333333 }, { "position": 3, "n": 60, "alpha_mean": 0.712, "alpha_median": 0.7084999999999999, "ci95_low": 0.6851662500000001, "ci95_high": 0.7384850000000002 }, { "position": 4, "n": 60, "alpha_mean": 0.6325999999999999, "alpha_median": 0.638, "ci95_low": 0.6014795833333332, "ci95_high": 0.6633079166666666 }, { "position": 5, "n": 60, "alpha_mean": 0.5557833333333332, "alpha_median": 0.553, "ci95_low": 0.5222575000000002, "ci95_high": 0.5895004166666666 }, { "position": 6, "n": 60, "alpha_mean": 0.47598333333333337, "alpha_median": 0.4825, "ci95_low": 0.43973083333333335, "ci95_high": 0.5133391666666666 }, { "position": 7, "n": 60, "alpha_mean": 0.4049833333333333, "alpha_median": 0.39, "ci95_low": 0.3712995833333333, "ci95_high": 0.43846416666666665 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.8569333333333334, "alpha_median": 0.874, "ci95_low": 0.8370483333333333, "ci95_high": 0.8763833333333333 }, { "position": 2, "n": 60, "alpha_mean": 0.7463333333333334, "alpha_median": 0.763, "ci95_low": 0.718665, "ci95_high": 0.7722341666666669 }, { "position": 3, "n": 60, "alpha_mean": 0.6522333333333332, "alpha_median": 0.654, "ci95_low": 0.6200466666666665, "ci95_high": 0.6846175 }, { "position": 4, "n": 60, "alpha_mean": 0.5665999999999999, "alpha_median": 0.5680000000000001, "ci95_low": 0.5307308333333333, "ci95_high": 0.6019720833333333 }, { "position": 5, "n": 60, "alpha_mean": 0.49448333333333333, "alpha_median": 0.4755, "ci95_low": 0.4554829166666667, "ci95_high": 0.5327204166666667 }, { "position": 6, "n": 60, "alpha_mean": 0.42051666666666665, "alpha_median": 0.397, "ci95_low": 0.38084958333333335, "ci95_high": 0.4608499999999999 }, { "position": 7, "n": 60, "alpha_mean": 0.3630166666666667, "alpha_median": 0.3365, "ci95_low": 0.3226812499999999, "ci95_high": 0.4040516666666667 } ], "chat": [ { "position": 1, "n": 55, "alpha_mean": 0.7442181818181818, "alpha_median": 0.75, "ci95_low": 0.7010240909090908, "ci95_high": 0.7815731818181817 }, { "position": 2, "n": 55, "alpha_mean": 0.5852181818181819, "alpha_median": 0.561, "ci95_low": 0.53427, "ci95_high": 0.6413472727272729 }, { "position": 3, "n": 55, "alpha_mean": 0.4758545454545455, "alpha_median": 0.411, "ci95_low": 0.41785954545454534, "ci95_high": 0.5385318181818182 }, { "position": 4, "n": 55, "alpha_mean": 0.3937818181818182, "alpha_median": 0.301, "ci95_low": 0.32965363636363637, "ci95_high": 0.4654922727272726 }, { "position": 5, "n": 55, "alpha_mean": 0.32007272727272723, "alpha_median": 0.203, "ci95_low": 0.2553986363636363, "ci95_high": 0.3890754545454545 }, { "position": 6, "n": 55, "alpha_mean": 0.2712, "alpha_median": 0.159, "ci95_low": 0.2051377272727273, "ci95_high": 0.34276909090909086 }, { "position": 7, "n": 55, "alpha_mean": 0.23603636363636366, "alpha_median": 0.113, "ci95_low": 0.16681272727272728, "ci95_high": 0.3091281818181818 } ] } }, "ksweep-qwen-q4-dflash-k10": { "family": "ksweep", "target": "qwen-q4", "drafter": "dflash-f16", "k": 10, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.000000000000143e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.014683333333333335, "alpha_median": 0.01, "ci95_low": 0.011265, "ci95_high": 0.0186 }, { "position": 2, "n": 60, "alpha_mean": 0.0034000000000000002, "alpha_median": 0.0, "ci95_low": 0.001816666666666667, "ci95_high": 0.0056 }, { "position": 3, "n": 60, "alpha_mean": 0.0017, "alpha_median": 0.0, "ci95_low": 0.00095, "ci95_high": 0.002533749999999998 }, { "position": 4, "n": 60, "alpha_mean": 0.0011333333333333334, "alpha_median": 0.0, "ci95_low": 0.0006666666666666666, "ci95_high": 0.0016666666666666668 }, { "position": 5, "n": 60, "alpha_mean": 0.0006000000000000001, "alpha_median": 0.0, "ci95_low": 0.0002666666666666667, "ci95_high": 0.001 }, { "position": 6, "n": 60, "alpha_mean": 0.0003333333333333333, "alpha_median": 0.0, "ci95_low": 6.666666666666667e-05, "ci95_high": 0.0006666666666666666 }, { "position": 7, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 8, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 9, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 10, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.013800000000000003, "alpha_median": 0.012, "ci95_low": 0.011216250000000002, "ci95_high": 0.01651666666666667 }, { "position": 2, "n": 60, "alpha_mean": 0.003916666666666667, "alpha_median": 0.004, "ci95_low": 0.002766666666666667, "ci95_high": 0.005233333333333334 }, { "position": 3, "n": 60, "alpha_mean": 0.00155, "alpha_median": 0.0, "ci95_low": 0.0008666666666666667, "ci95_high": 0.0023833333333333337 }, { "position": 4, "n": 60, "alpha_mean": 0.0006166666666666667, "alpha_median": 0.0, "ci95_low": 0.0002, "ci95_high": 0.0011166666666666666 }, { "position": 5, "n": 60, "alpha_mean": 0.0003333333333333333, "alpha_median": 0.0, "ci95_low": 6.666666666666667e-05, "ci95_high": 0.0006666666666666666 }, { "position": 6, "n": 60, "alpha_mean": 0.0002, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.00046666666666666666 }, { "position": 7, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 8, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 9, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 10, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.021448275862068968, "alpha_median": 0.016, "ci95_low": 0.017085344827586207, "ci95_high": 0.026397413793103446 }, { "position": 2, "n": 58, "alpha_mean": 0.004586206896551724, "alpha_median": 0.004, "ci95_low": 0.003327586206896552, "ci95_high": 0.005983189655172413 }, { "position": 3, "n": 58, "alpha_mean": 0.0018965517241379313, "alpha_median": 0.0, "ci95_low": 0.0011379310344827587, "ci95_high": 0.0027241379310344828 }, { "position": 4, "n": 58, "alpha_mean": 0.0010000000000000002, "alpha_median": 0.0, "ci95_low": 0.00041379310344827585, "ci95_high": 0.0016724137931034483 }, { "position": 5, "n": 58, "alpha_mean": 0.0005172413793103448, "alpha_median": 0.0, "ci95_low": 6.896551724137931e-05, "ci95_high": 0.0011379310344827585 }, { "position": 6, "n": 58, "alpha_mean": 0.0002931034482758621, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.000689655172413793 }, { "position": 7, "n": 58, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 8, "n": 58, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 9, "n": 58, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 10, "n": 58, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 } ] } }, "ksweep-qwen-q4-dflash-k5": { "family": "ksweep", "target": "qwen-q4", "drafter": "dflash-f16", "k": 5, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 4.99999999999997e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.013850000000000001, "alpha_median": 0.012, "ci95_low": 0.0105, "ci95_high": 0.0178 }, { "position": 2, "n": 60, "alpha_mean": 0.003433333333333334, "alpha_median": 0.0, "ci95_low": 0.0018658333333333333, "ci95_high": 0.005633333333333334 }, { "position": 3, "n": 60, "alpha_mean": 0.0012666666666666668, "alpha_median": 0.0, "ci95_low": 0.0007333333333333333, "ci95_high": 0.0018666666666666666 }, { "position": 4, "n": 60, "alpha_mean": 0.0008, "alpha_median": 0.0, "ci95_low": 0.0004, "ci95_high": 0.0012000000000000001 }, { "position": 5, "n": 60, "alpha_mean": 0.0006000000000000001, "alpha_median": 0.0, "ci95_low": 0.0002666666666666667, "ci95_high": 0.001 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.012633333333333333, "alpha_median": 0.012, "ci95_low": 0.010132916666666669, "ci95_high": 0.015167083333333331 }, { "position": 2, "n": 60, "alpha_mean": 0.0030166666666666675, "alpha_median": 0.0, "ci95_low": 0.0020162500000000002, "ci95_high": 0.004 }, { "position": 3, "n": 60, "alpha_mean": 0.0012666666666666668, "alpha_median": 0.0, "ci95_low": 0.0007333333333333333, "ci95_high": 0.0018000000000000002 }, { "position": 4, "n": 60, "alpha_mean": 0.0005333333333333334, "alpha_median": 0.0, "ci95_low": 0.0002, "ci95_high": 0.0009333333333333334 }, { "position": 5, "n": 60, "alpha_mean": 0.0002, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.00046666666666666666 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.020879310344827588, "alpha_median": 0.016, "ci95_low": 0.01656853448275862, "ci95_high": 0.025655603448275863 }, { "position": 2, "n": 58, "alpha_mean": 0.003637931034482759, "alpha_median": 0.0, "ci95_low": 0.0024310344827586207, "ci95_high": 0.005051724137931035 }, { "position": 3, "n": 58, "alpha_mean": 0.0012413793103448277, "alpha_median": 0.0, "ci95_low": 0.0006896551724137933, "ci95_high": 0.0017931034482758623 }, { "position": 4, "n": 58, "alpha_mean": 0.0006206896551724138, "alpha_median": 0.0, "ci95_low": 0.00027586206896551725, "ci95_high": 0.0010344827586206897 }, { "position": 5, "n": 58, "alpha_mean": 0.00020689655172413793, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0004827586206896552 } ] } }, "ksweep-qwen-q4-dflash-k7": { "family": "ksweep", "target": "qwen-q4", "drafter": "dflash-f16", "k": 7, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.000000000000143e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.014683333333333335, "alpha_median": 0.01, "ci95_low": 0.011265, "ci95_high": 0.0186 }, { "position": 2, "n": 60, "alpha_mean": 0.0034000000000000002, "alpha_median": 0.0, "ci95_low": 0.001816666666666667, "ci95_high": 0.0056 }, { "position": 3, "n": 60, "alpha_mean": 0.0017, "alpha_median": 0.0, "ci95_low": 0.00095, "ci95_high": 0.002533749999999998 }, { "position": 4, "n": 60, "alpha_mean": 0.0011333333333333334, "alpha_median": 0.0, "ci95_low": 0.0006666666666666666, "ci95_high": 0.0016666666666666668 }, { "position": 5, "n": 60, "alpha_mean": 0.0006000000000000001, "alpha_median": 0.0, "ci95_low": 0.0002666666666666667, "ci95_high": 0.001 }, { "position": 6, "n": 60, "alpha_mean": 0.0003333333333333333, "alpha_median": 0.0, "ci95_low": 6.666666666666667e-05, "ci95_high": 0.0006666666666666666 }, { "position": 7, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.013800000000000003, "alpha_median": 0.012, "ci95_low": 0.011216250000000002, "ci95_high": 0.01651666666666667 }, { "position": 2, "n": 60, "alpha_mean": 0.003916666666666667, "alpha_median": 0.004, "ci95_low": 0.002766666666666667, "ci95_high": 0.005233333333333334 }, { "position": 3, "n": 60, "alpha_mean": 0.00155, "alpha_median": 0.0, "ci95_low": 0.0008666666666666667, "ci95_high": 0.0023833333333333337 }, { "position": 4, "n": 60, "alpha_mean": 0.0006166666666666667, "alpha_median": 0.0, "ci95_low": 0.0002, "ci95_high": 0.0011166666666666666 }, { "position": 5, "n": 60, "alpha_mean": 0.0003333333333333333, "alpha_median": 0.0, "ci95_low": 6.666666666666667e-05, "ci95_high": 0.0006666666666666666 }, { "position": 6, "n": 60, "alpha_mean": 0.0002, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.00046666666666666666 }, { "position": 7, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.021448275862068968, "alpha_median": 0.016, "ci95_low": 0.017085344827586207, "ci95_high": 0.026397413793103446 }, { "position": 2, "n": 58, "alpha_mean": 0.004586206896551724, "alpha_median": 0.004, "ci95_low": 0.003327586206896552, "ci95_high": 0.005983189655172413 }, { "position": 3, "n": 58, "alpha_mean": 0.0018965517241379313, "alpha_median": 0.0, "ci95_low": 0.0011379310344827587, "ci95_high": 0.0027241379310344828 }, { "position": 4, "n": 58, "alpha_mean": 0.0010000000000000002, "alpha_median": 0.0, "ci95_low": 0.00041379310344827585, "ci95_high": 0.0016724137931034483 }, { "position": 5, "n": 58, "alpha_mean": 0.0005172413793103448, "alpha_median": 0.0, "ci95_low": 6.896551724137931e-05, "ci95_high": 0.0011379310344827585 }, { "position": 6, "n": 58, "alpha_mean": 0.0002931034482758621, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.000689655172413793 }, { "position": 7, "n": 58, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 } ] } }, "ksweep-qwen-q4-dspark-p0-k10": { "family": "ksweep", "target": "qwen-q4", "drafter": "dspark-p0", "k": 10, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.0000000000105516e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8275166666666666, "alpha_median": 0.8234999999999999, "ci95_low": 0.80881625, "ci95_high": 0.8464854166666665 }, { "position": 2, "n": 60, "alpha_mean": 0.6882666666666667, "alpha_median": 0.7064999999999999, "ci95_low": 0.6566654166666668, "ci95_high": 0.7224 }, { "position": 3, "n": 60, "alpha_mean": 0.5771166666666667, "alpha_median": 0.57, "ci95_low": 0.5377991666666667, "ci95_high": 0.6176891666666667 }, { "position": 4, "n": 60, "alpha_mean": 0.48430000000000006, "alpha_median": 0.4585, "ci95_low": 0.4371829166666667, "ci95_high": 0.5336854166666667 }, { "position": 5, "n": 60, "alpha_mean": 0.4117, "alpha_median": 0.4, "ci95_low": 0.36243166666666665, "ci95_high": 0.4655683333333333 }, { "position": 6, "n": 60, "alpha_mean": 0.35425, "alpha_median": 0.339, "ci95_low": 0.30452666666666667, "ci95_high": 0.40827583333333334 }, { "position": 7, "n": 60, "alpha_mean": 0.30565, "alpha_median": 0.295, "ci95_low": 0.2567629166666667, "ci95_high": 0.3597012499999999 }, { "position": 8, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 9, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 10, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.7943666666666667, "alpha_median": 0.808, "ci95_low": 0.77531625, "ci95_high": 0.8127087500000001 }, { "position": 2, "n": 60, "alpha_mean": 0.6135166666666668, "alpha_median": 0.6125, "ci95_low": 0.58530875, "ci95_high": 0.6420545833333332 }, { "position": 3, "n": 60, "alpha_mean": 0.46181666666666665, "alpha_median": 0.4355, "ci95_low": 0.4318120833333333, "ci95_high": 0.4930520833333334 }, { "position": 4, "n": 60, "alpha_mean": 0.3523333333333333, "alpha_median": 0.3255, "ci95_low": 0.3216791666666667, "ci95_high": 0.3854704166666667 }, { "position": 5, "n": 60, "alpha_mean": 0.26839999999999997, "alpha_median": 0.2355, "ci95_low": 0.23628, "ci95_high": 0.3033833333333333 }, { "position": 6, "n": 60, "alpha_mean": 0.19771666666666665, "alpha_median": 0.1725, "ci95_low": 0.16911499999999996, "ci95_high": 0.22833874999999998 }, { "position": 7, "n": 60, "alpha_mean": 0.15255000000000002, "alpha_median": 0.119, "ci95_low": 0.12411666666666668, "ci95_high": 0.18401791666666664 }, { "position": 8, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 9, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 10, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.7089310344827586, "alpha_median": 0.7275, "ci95_low": 0.665080172413793, "ci95_high": 0.747226724137931 }, { "position": 2, "n": 58, "alpha_mean": 0.4922241379310345, "alpha_median": 0.5095000000000001, "ci95_low": 0.44306379310344834, "ci95_high": 0.5365064655172413 }, { "position": 3, "n": 58, "alpha_mean": 0.35253448275862065, "alpha_median": 0.3375, "ci95_low": 0.30437112068965516, "ci95_high": 0.39734525862068965 }, { "position": 4, "n": 58, "alpha_mean": 0.2588793103448276, "alpha_median": 0.2205, "ci95_low": 0.21575560344827588, "ci95_high": 0.30172629310344823 }, { "position": 5, "n": 58, "alpha_mean": 0.19879310344827586, "alpha_median": 0.1545, "ci95_low": 0.15875646551724143, "ci95_high": 0.24043620689655168 }, { "position": 6, "n": 58, "alpha_mean": 0.15515517241379312, "alpha_median": 0.0895, "ci95_low": 0.11734439655172413, "ci95_high": 0.19293146551724136 }, { "position": 7, "n": 58, "alpha_mean": 0.12293103448275863, "alpha_median": 0.065, "ci95_low": 0.0872926724137931, "ci95_high": 0.15955387931034481 }, { "position": 8, "n": 58, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 9, "n": 58, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 10, "n": 58, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 } ] } }, "ksweep-qwen-q4-dspark-p0-k5": { "family": "ksweep", "target": "qwen-q4", "drafter": "dspark-p0", "k": 5, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.0000000000050004e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8351000000000001, "alpha_median": 0.8434999999999999, "ci95_low": 0.8158500000000001, "ci95_high": 0.8545670833333333 }, { "position": 2, "n": 60, "alpha_mean": 0.6908, "alpha_median": 0.699, "ci95_low": 0.6574162500000001, "ci95_high": 0.7251333333333333 }, { "position": 3, "n": 60, "alpha_mean": 0.5831, "alpha_median": 0.5714999999999999, "ci95_low": 0.5408058333333334, "ci95_high": 0.6268512499999997 }, { "position": 4, "n": 60, "alpha_mean": 0.48841666666666667, "alpha_median": 0.488, "ci95_low": 0.44127791666666666, "ci95_high": 0.5375004166666666 }, { "position": 5, "n": 60, "alpha_mean": 0.40971666666666656, "alpha_median": 0.397, "ci95_low": 0.3608645833333333, "ci95_high": 0.4619025 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.7903666666666667, "alpha_median": 0.792, "ci95_low": 0.77183, "ci95_high": 0.80858375 }, { "position": 2, "n": 60, "alpha_mean": 0.6033166666666666, "alpha_median": 0.5945, "ci95_low": 0.5752324999999999, "ci95_high": 0.6310170833333334 }, { "position": 3, "n": 60, "alpha_mean": 0.4645, "alpha_median": 0.454, "ci95_low": 0.4331312499999999, "ci95_high": 0.4965354166666667 }, { "position": 4, "n": 60, "alpha_mean": 0.35390000000000005, "alpha_median": 0.329, "ci95_low": 0.32044125, "ci95_high": 0.38973333333333327 }, { "position": 5, "n": 60, "alpha_mean": 0.26375, "alpha_median": 0.23, "ci95_low": 0.23050958333333335, "ci95_high": 0.29978541666666664 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.7085689655172414, "alpha_median": 0.7555000000000001, "ci95_low": 0.664840948275862, "ci95_high": 0.7459702586206896 }, { "position": 2, "n": 58, "alpha_mean": 0.4858793103448276, "alpha_median": 0.513, "ci95_low": 0.43451724137931036, "ci95_high": 0.5313284482758621 }, { "position": 3, "n": 58, "alpha_mean": 0.352948275862069, "alpha_median": 0.3545, "ci95_low": 0.30419784482758616, "ci95_high": 0.39955258620689643 }, { "position": 4, "n": 58, "alpha_mean": 0.26739655172413795, "alpha_median": 0.2155, "ci95_low": 0.2195293103448276, "ci95_high": 0.3135099137931034 }, { "position": 5, "n": 58, "alpha_mean": 0.2047586206896552, "alpha_median": 0.1445, "ci95_low": 0.16067543103448276, "ci95_high": 0.24851810344827574 } ] } }, "ksweep-qwen-q4-dspark-p0-k7": { "family": "ksweep", "target": "qwen-q4", "drafter": "dspark-p0", "k": 7, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.0000000000105516e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8275166666666666, "alpha_median": 0.8234999999999999, "ci95_low": 0.80881625, "ci95_high": 0.8464854166666665 }, { "position": 2, "n": 60, "alpha_mean": 0.6882666666666667, "alpha_median": 0.7064999999999999, "ci95_low": 0.6566654166666668, "ci95_high": 0.7224 }, { "position": 3, "n": 60, "alpha_mean": 0.5771166666666667, "alpha_median": 0.57, "ci95_low": 0.5377991666666667, "ci95_high": 0.6176891666666667 }, { "position": 4, "n": 60, "alpha_mean": 0.48430000000000006, "alpha_median": 0.4585, "ci95_low": 0.4371829166666667, "ci95_high": 0.5336854166666667 }, { "position": 5, "n": 60, "alpha_mean": 0.4117, "alpha_median": 0.4, "ci95_low": 0.36243166666666665, "ci95_high": 0.4655683333333333 }, { "position": 6, "n": 60, "alpha_mean": 0.35425, "alpha_median": 0.339, "ci95_low": 0.30452666666666667, "ci95_high": 0.40827583333333334 }, { "position": 7, "n": 60, "alpha_mean": 0.30565, "alpha_median": 0.295, "ci95_low": 0.2567629166666667, "ci95_high": 0.3597012499999999 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.7943666666666667, "alpha_median": 0.808, "ci95_low": 0.77531625, "ci95_high": 0.8127087500000001 }, { "position": 2, "n": 60, "alpha_mean": 0.6135166666666668, "alpha_median": 0.6125, "ci95_low": 0.58530875, "ci95_high": 0.6420545833333332 }, { "position": 3, "n": 60, "alpha_mean": 0.46181666666666665, "alpha_median": 0.4355, "ci95_low": 0.4318120833333333, "ci95_high": 0.4930520833333334 }, { "position": 4, "n": 60, "alpha_mean": 0.3523333333333333, "alpha_median": 0.3255, "ci95_low": 0.3216791666666667, "ci95_high": 0.3854704166666667 }, { "position": 5, "n": 60, "alpha_mean": 0.26839999999999997, "alpha_median": 0.2355, "ci95_low": 0.23628, "ci95_high": 0.3033833333333333 }, { "position": 6, "n": 60, "alpha_mean": 0.19771666666666665, "alpha_median": 0.1725, "ci95_low": 0.16911499999999996, "ci95_high": 0.22833874999999998 }, { "position": 7, "n": 60, "alpha_mean": 0.15255000000000002, "alpha_median": 0.119, "ci95_low": 0.12411666666666668, "ci95_high": 0.18401791666666664 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.7089310344827586, "alpha_median": 0.7275, "ci95_low": 0.665080172413793, "ci95_high": 0.747226724137931 }, { "position": 2, "n": 58, "alpha_mean": 0.4922241379310345, "alpha_median": 0.5095000000000001, "ci95_low": 0.44306379310344834, "ci95_high": 0.5365064655172413 }, { "position": 3, "n": 58, "alpha_mean": 0.35253448275862065, "alpha_median": 0.3375, "ci95_low": 0.30437112068965516, "ci95_high": 0.39734525862068965 }, { "position": 4, "n": 58, "alpha_mean": 0.2588793103448276, "alpha_median": 0.2205, "ci95_low": 0.21575560344827588, "ci95_high": 0.30172629310344823 }, { "position": 5, "n": 58, "alpha_mean": 0.19879310344827586, "alpha_median": 0.1545, "ci95_low": 0.15875646551724143, "ci95_high": 0.24043620689655168 }, { "position": 6, "n": 58, "alpha_mean": 0.15515517241379312, "alpha_median": 0.0895, "ci95_low": 0.11734439655172413, "ci95_high": 0.19293146551724136 }, { "position": 7, "n": 58, "alpha_mean": 0.12293103448275863, "alpha_median": 0.065, "ci95_low": 0.0872926724137931, "ci95_high": 0.15955387931034481 } ] } }, "ksweep-qwen-q4-eagle3-k10": { "family": "ksweep", "target": "qwen-q4", "drafter": "eagle3", "k": 10, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.000000000002225e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.6393000000000001, "alpha_median": 0.636, "ci95_low": 0.6221158333333334, "ci95_high": 0.65715 }, { "position": 2, "n": 60, "alpha_mean": 0.41214999999999996, "alpha_median": 0.413, "ci95_low": 0.3911658333333333, "ci95_high": 0.43366874999999994 }, { "position": 3, "n": 60, "alpha_mean": 0.2157, "alpha_median": 0.2145, "ci95_low": 0.20113291666666666, "ci95_high": 0.23085124999999998 }, { "position": 4, "n": 60, "alpha_mean": 0.12154999999999999, "alpha_median": 0.1165, "ci95_low": 0.10949666666666666, "ci95_high": 0.13506666666666664 }, { "position": 5, "n": 60, "alpha_mean": 0.07311666666666668, "alpha_median": 0.07050000000000001, "ci95_low": 0.06314958333333333, "ci95_high": 0.08443333333333333 }, { "position": 6, "n": 60, "alpha_mean": 0.04493333333333333, "alpha_median": 0.038, "ci95_low": 0.037166250000000005, "ci95_high": 0.05385083333333333 }, { "position": 7, "n": 60, "alpha_mean": 0.027366666666666664, "alpha_median": 0.0225, "ci95_low": 0.021782916666666666, "ci95_high": 0.0334675 }, { "position": 8, "n": 60, "alpha_mean": 0.01655, "alpha_median": 0.011, "ci95_low": 0.012550000000000002, "ci95_high": 0.020699999999999996 }, { "position": 9, "n": 60, "alpha_mean": 0.008683333333333333, "alpha_median": 0.0085, "ci95_low": 0.005449166666666666, "ci95_high": 0.012416666666666666 }, { "position": 10, "n": 60, "alpha_mean": 0.005783333333333333, "alpha_median": 0.0, "ci95_low": 0.0031833333333333336, "ci95_high": 0.008583333333333333 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.6601999999999999, "alpha_median": 0.6415, "ci95_low": 0.6419654166666666, "ci95_high": 0.6793508333333333 }, { "position": 2, "n": 60, "alpha_mean": 0.41513333333333335, "alpha_median": 0.399, "ci95_low": 0.3911633333333333, "ci95_high": 0.4415695833333333 }, { "position": 3, "n": 60, "alpha_mean": 0.21558333333333332, "alpha_median": 0.174, "ci95_low": 0.1876145833333333, "ci95_high": 0.24768500000000002 }, { "position": 4, "n": 60, "alpha_mean": 0.12786666666666668, "alpha_median": 0.0925, "ci95_low": 0.10286625, "ci95_high": 0.15703499999999995 }, { "position": 5, "n": 60, "alpha_mean": 0.08738333333333334, "alpha_median": 0.046, "ci95_low": 0.06396333333333334, "ci95_high": 0.11490749999999998 }, { "position": 6, "n": 60, "alpha_mean": 0.057249999999999995, "alpha_median": 0.02, "ci95_low": 0.03816375, "ci95_high": 0.07923541666666666 }, { "position": 7, "n": 60, "alpha_mean": 0.03891666666666666, "alpha_median": 0.01, "ci95_low": 0.022932499999999998, "ci95_high": 0.05705333333333332 }, { "position": 8, "n": 60, "alpha_mean": 0.030533333333333336, "alpha_median": 0.009, "ci95_low": 0.01624916666666667, "ci95_high": 0.047400833333333336 }, { "position": 9, "n": 60, "alpha_mean": 0.021716666666666665, "alpha_median": 0.0, "ci95_low": 0.010549166666666668, "ci95_high": 0.03530249999999999 }, { "position": 10, "n": 60, "alpha_mean": 0.01635, "alpha_median": 0.0, "ci95_low": 0.008066666666666666, "ci95_high": 0.025851666666666658 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.5735517241379311, "alpha_median": 0.6014999999999999, "ci95_low": 0.5235991379310345, "ci95_high": 0.6212129310344826 }, { "position": 2, "n": 58, "alpha_mean": 0.34679310344827585, "alpha_median": 0.3715, "ci95_low": 0.2997573275862069, "ci95_high": 0.39171379310344817 }, { "position": 3, "n": 58, "alpha_mean": 0.18879310344827585, "alpha_median": 0.163, "ci95_low": 0.1524030172413793, "ci95_high": 0.22463879310344825 }, { "position": 4, "n": 58, "alpha_mean": 0.11891379310344827, "alpha_median": 0.079, "ci95_low": 0.08842974137931035, "ci95_high": 0.15122887931034484 }, { "position": 5, "n": 58, "alpha_mean": 0.08306896551724137, "alpha_median": 0.0405, "ci95_low": 0.057982327586206896, "ci95_high": 0.10953577586206896 }, { "position": 6, "n": 58, "alpha_mean": 0.0649655172413793, "alpha_median": 0.0285, "ci95_low": 0.04182456896551724, "ci95_high": 0.08976249999999997 }, { "position": 7, "n": 58, "alpha_mean": 0.048999999999999995, "alpha_median": 0.0095, "ci95_low": 0.029137068965517245, "ci95_high": 0.0712004310344827 }, { "position": 8, "n": 58, "alpha_mean": 0.04074137931034483, "alpha_median": 0.008, "ci95_low": 0.022344827586206897, "ci95_high": 0.06181077586206897 }, { "position": 9, "n": 58, "alpha_mean": 0.034413793103448276, "alpha_median": 0.0, "ci95_low": 0.016790948275862073, "ci95_high": 0.0545176724137931 }, { "position": 10, "n": 58, "alpha_mean": 0.029206896551724138, "alpha_median": 0.0, "ci95_low": 0.013636637931034482, "ci95_high": 0.047277586206896545 } ] } }, "ksweep-qwen-q4-eagle3-k5": { "family": "ksweep", "target": "qwen-q4", "drafter": "eagle3", "k": 5, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.0000000000050004e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.6475333333333333, "alpha_median": 0.6315, "ci95_low": 0.6281633333333333, "ci95_high": 0.6681520833333335 }, { "position": 2, "n": 60, "alpha_mean": 0.4170999999999999, "alpha_median": 0.405, "ci95_low": 0.39436625, "ci95_high": 0.44110166666666667 }, { "position": 3, "n": 60, "alpha_mean": 0.2237666666666667, "alpha_median": 0.218, "ci95_low": 0.20564458333333333, "ci95_high": 0.24383749999999996 }, { "position": 4, "n": 60, "alpha_mean": 0.12868333333333334, "alpha_median": 0.1205, "ci95_low": 0.11228208333333332, "ci95_high": 0.1467341666666667 }, { "position": 5, "n": 60, "alpha_mean": 0.0775, "alpha_median": 0.0695, "ci95_low": 0.06496583333333333, "ci95_high": 0.09210125 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.6593166666666668, "alpha_median": 0.648, "ci95_low": 0.6399995833333333, "ci95_high": 0.6801691666666667 }, { "position": 2, "n": 60, "alpha_mean": 0.4212333333333333, "alpha_median": 0.4105, "ci95_low": 0.39419958333333327, "ci95_high": 0.4513374999999999 }, { "position": 3, "n": 60, "alpha_mean": 0.21413333333333331, "alpha_median": 0.176, "ci95_low": 0.18536458333333336, "ci95_high": 0.24663083333333327 }, { "position": 4, "n": 60, "alpha_mean": 0.1280666666666667, "alpha_median": 0.088, "ci95_low": 0.10104916666666666, "ci95_high": 0.15938333333333332 }, { "position": 5, "n": 60, "alpha_mean": 0.0828, "alpha_median": 0.048, "ci95_low": 0.058109999999999995, "ci95_high": 0.11205333333333332 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.5732241379310344, "alpha_median": 0.597, "ci95_low": 0.5165314655172414, "ci95_high": 0.6251758620689656 }, { "position": 2, "n": 58, "alpha_mean": 0.3543620689655172, "alpha_median": 0.3435, "ci95_low": 0.3023431034482758, "ci95_high": 0.4045181034482759 }, { "position": 3, "n": 58, "alpha_mean": 0.1993103448275862, "alpha_median": 0.1605, "ci95_low": 0.15927284482758625, "ci95_high": 0.24270301724137924 }, { "position": 4, "n": 58, "alpha_mean": 0.1258103448275862, "alpha_median": 0.092, "ci95_low": 0.09251422413793102, "ci95_high": 0.1614176724137931 }, { "position": 5, "n": 58, "alpha_mean": 0.08212068965517243, "alpha_median": 0.0435, "ci95_low": 0.05446120689655173, "ci95_high": 0.11160344827586206 } ] } }, "ksweep-qwen-q4-eagle3-k7": { "family": "ksweep", "target": "qwen-q4", "drafter": "eagle3", "k": 7, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.000000000002225e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.6443333333333333, "alpha_median": 0.626, "ci95_low": 0.6251470833333334, "ci95_high": 0.6649041666666665 }, { "position": 2, "n": 60, "alpha_mean": 0.4131333333333334, "alpha_median": 0.403, "ci95_low": 0.3907324999999999, "ci95_high": 0.4365841666666666 }, { "position": 3, "n": 60, "alpha_mean": 0.21961666666666668, "alpha_median": 0.218, "ci95_low": 0.20243291666666668, "ci95_high": 0.2386175 }, { "position": 4, "n": 60, "alpha_mean": 0.12621666666666667, "alpha_median": 0.118, "ci95_low": 0.1104825, "ci95_high": 0.14348999999999995 }, { "position": 5, "n": 60, "alpha_mean": 0.07546666666666667, "alpha_median": 0.0675, "ci95_low": 0.06334833333333333, "ci95_high": 0.08971749999999999 }, { "position": 6, "n": 60, "alpha_mean": 0.045649999999999996, "alpha_median": 0.036, "ci95_low": 0.03564916666666666, "ci95_high": 0.05713375 }, { "position": 7, "n": 60, "alpha_mean": 0.028, "alpha_median": 0.02, "ci95_low": 0.02039833333333333, "ci95_high": 0.037367083333333335 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.6557499999999999, "alpha_median": 0.6445000000000001, "ci95_low": 0.6375145833333333, "ci95_high": 0.6747520833333334 }, { "position": 2, "n": 60, "alpha_mean": 0.41759999999999997, "alpha_median": 0.4125, "ci95_low": 0.39189124999999997, "ci95_high": 0.4459733333333334 }, { "position": 3, "n": 60, "alpha_mean": 0.21246666666666666, "alpha_median": 0.1815, "ci95_low": 0.18428166666666662, "ci95_high": 0.2451670833333333 }, { "position": 4, "n": 60, "alpha_mean": 0.12625000000000003, "alpha_median": 0.0875, "ci95_low": 0.09834916666666665, "ci95_high": 0.15945666666666664 }, { "position": 5, "n": 60, "alpha_mean": 0.08131666666666669, "alpha_median": 0.049, "ci95_low": 0.05654833333333334, "ci95_high": 0.11175416666666664 }, { "position": 6, "n": 60, "alpha_mean": 0.052649999999999995, "alpha_median": 0.019, "ci95_low": 0.03203333333333333, "ci95_high": 0.07733583333333333 }, { "position": 7, "n": 60, "alpha_mean": 0.0377, "alpha_median": 0.01, "ci95_low": 0.020062916666666666, "ci95_high": 0.05930083333333334 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.5708275862068966, "alpha_median": 0.6045, "ci95_low": 0.5146288793103448, "ci95_high": 0.6221379310344828 }, { "position": 2, "n": 58, "alpha_mean": 0.3496551724137931, "alpha_median": 0.3395, "ci95_low": 0.29932672413793104, "ci95_high": 0.39888405172413793 }, { "position": 3, "n": 58, "alpha_mean": 0.19015517241379312, "alpha_median": 0.158, "ci95_low": 0.15470474137931034, "ci95_high": 0.22732758620689655 }, { "position": 4, "n": 58, "alpha_mean": 0.12032758620689656, "alpha_median": 0.084, "ci95_low": 0.09060258620689655, "ci95_high": 0.15307241379310346 }, { "position": 5, "n": 58, "alpha_mean": 0.07534482758620689, "alpha_median": 0.042499999999999996, "ci95_low": 0.05094008620689655, "ci95_high": 0.10100258620689655 }, { "position": 6, "n": 58, "alpha_mean": 0.057896551724137935, "alpha_median": 0.0205, "ci95_low": 0.035170689655172414, "ci95_high": 0.08256939655172414 }, { "position": 7, "n": 58, "alpha_mean": 0.04268965517241379, "alpha_median": 0.009, "ci95_low": 0.022724137931034484, "ci95_high": 0.06393362068965516 } ] } }, "ksweep-qwen-q4-vanilla17b-k10": { "family": "ksweep", "target": "qwen-q4", "drafter": "vanilla17b", "k": 10, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.000000000002225e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8605833333333334, "alpha_median": 0.864, "ci95_low": 0.8454433333333332, "ci95_high": 0.8753520833333333 }, { "position": 2, "n": 60, "alpha_mean": 0.7415833333333333, "alpha_median": 0.7255, "ci95_low": 0.7151995833333333, "ci95_high": 0.7684404166666667 }, { "position": 3, "n": 60, "alpha_mean": 0.6566500000000001, "alpha_median": 0.6505000000000001, "ci95_low": 0.6250304166666666, "ci95_high": 0.6895837499999998 }, { "position": 4, "n": 60, "alpha_mean": 0.5859333333333334, "alpha_median": 0.5725, "ci95_low": 0.5521279166666666, "ci95_high": 0.6207170833333334 }, { "position": 5, "n": 60, "alpha_mean": 0.5292833333333333, "alpha_median": 0.512, "ci95_low": 0.49318166666666663, "ci95_high": 0.5666537499999998 }, { "position": 6, "n": 60, "alpha_mean": 0.47714999999999996, "alpha_median": 0.469, "ci95_low": 0.44058083333333337, "ci95_high": 0.51552375 }, { "position": 7, "n": 60, "alpha_mean": 0.4276666666666667, "alpha_median": 0.42, "ci95_low": 0.39024958333333337, "ci95_high": 0.4667545833333333 }, { "position": 8, "n": 60, "alpha_mean": 0.3843833333333334, "alpha_median": 0.366, "ci95_low": 0.34676666666666667, "ci95_high": 0.4235545833333333 }, { "position": 9, "n": 60, "alpha_mean": 0.3481, "alpha_median": 0.341, "ci95_low": 0.3101312500000001, "ci95_high": 0.3868700000000001 }, { "position": 10, "n": 60, "alpha_mean": 0.32078333333333325, "alpha_median": 0.314, "ci95_low": 0.28234583333333335, "ci95_high": 0.3583666666666666 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.82835, "alpha_median": 0.835, "ci95_low": 0.8096479166666668, "ci95_high": 0.8476525 }, { "position": 2, "n": 60, "alpha_mean": 0.6865166666666667, "alpha_median": 0.6875, "ci95_low": 0.6631316666666667, "ci95_high": 0.7102224999999999 }, { "position": 3, "n": 60, "alpha_mean": 0.5856499999999999, "alpha_median": 0.578, "ci95_low": 0.5595291666666666, "ci95_high": 0.6141675 }, { "position": 4, "n": 60, "alpha_mean": 0.5046, "alpha_median": 0.484, "ci95_low": 0.47836416666666665, "ci95_high": 0.5368516666666666 }, { "position": 5, "n": 60, "alpha_mean": 0.43585, "alpha_median": 0.415, "ci95_low": 0.4082495833333332, "ci95_high": 0.4698133333333333 }, { "position": 6, "n": 60, "alpha_mean": 0.3830666666666667, "alpha_median": 0.3645, "ci95_low": 0.3531079166666666, "ci95_high": 0.41880291666666664 }, { "position": 7, "n": 60, "alpha_mean": 0.34103333333333335, "alpha_median": 0.3225, "ci95_low": 0.3107825, "ci95_high": 0.3766229166666667 }, { "position": 8, "n": 60, "alpha_mean": 0.30171666666666663, "alpha_median": 0.278, "ci95_low": 0.2721820833333333, "ci95_high": 0.3373029166666667 }, { "position": 9, "n": 60, "alpha_mean": 0.2707166666666667, "alpha_median": 0.242, "ci95_low": 0.24014916666666664, "ci95_high": 0.3073733333333334 }, { "position": 10, "n": 60, "alpha_mean": 0.2393666666666667, "alpha_median": 0.2145, "ci95_low": 0.21163, "ci95_high": 0.27196833333333337 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.7706724137931034, "alpha_median": 0.7444999999999999, "ci95_low": 0.7471025862068965, "ci95_high": 0.7944883620689656 }, { "position": 2, "n": 58, "alpha_mean": 0.5908793103448275, "alpha_median": 0.588, "ci95_low": 0.5608418103448276, "ci95_high": 0.6219202586206896 }, { "position": 3, "n": 58, "alpha_mean": 0.47472413793103446, "alpha_median": 0.478, "ci95_low": 0.43965344827586206, "ci95_high": 0.5126948275862069 }, { "position": 4, "n": 58, "alpha_mean": 0.38355172413793104, "alpha_median": 0.362, "ci95_low": 0.3471965517241379, "ci95_high": 0.4236383620689655 }, { "position": 5, "n": 58, "alpha_mean": 0.3245862068965517, "alpha_median": 0.2845, "ci95_low": 0.28644482758620693, "ci95_high": 0.3674366379310344 }, { "position": 6, "n": 58, "alpha_mean": 0.27910344827586203, "alpha_median": 0.231, "ci95_low": 0.2389810344827586, "ci95_high": 0.32389655172413795 }, { "position": 7, "n": 58, "alpha_mean": 0.24193103448275866, "alpha_median": 0.191, "ci95_low": 0.20145646551724136, "ci95_high": 0.2864323275862069 }, { "position": 8, "n": 58, "alpha_mean": 0.21972413793103449, "alpha_median": 0.16399999999999998, "ci95_low": 0.17875862068965517, "ci95_high": 0.266089224137931 }, { "position": 9, "n": 58, "alpha_mean": 0.19453448275862065, "alpha_median": 0.138, "ci95_low": 0.15435689655172413, "ci95_high": 0.23910603448275858 }, { "position": 10, "n": 58, "alpha_mean": 0.17518965517241378, "alpha_median": 0.11449999999999999, "ci95_low": 0.1350336206896552, "ci95_high": 0.21991681034482755 } ] } }, "ksweep-qwen-q4-vanilla17b-k5": { "family": "ksweep", "target": "qwen-q4", "drafter": "vanilla17b", "k": 5, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.0000000000050004e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8614333333333332, "alpha_median": 0.8694999999999999, "ci95_low": 0.8468833333333333, "ci95_high": 0.8758516666666667 }, { "position": 2, "n": 60, "alpha_mean": 0.7467333333333332, "alpha_median": 0.737, "ci95_low": 0.7221154166666666, "ci95_high": 0.7710595833333332 }, { "position": 3, "n": 60, "alpha_mean": 0.6641666666666667, "alpha_median": 0.6555, "ci95_low": 0.6353624999999999, "ci95_high": 0.6928541666666668 }, { "position": 4, "n": 60, "alpha_mean": 0.5916833333333334, "alpha_median": 0.5945, "ci95_low": 0.5596133333333334, "ci95_high": 0.6232516666666668 }, { "position": 5, "n": 60, "alpha_mean": 0.5365, "alpha_median": 0.5255000000000001, "ci95_low": 0.5021658333333333, "ci95_high": 0.5705020833333334 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.8306333333333333, "alpha_median": 0.8225, "ci95_low": 0.8160666666666666, "ci95_high": 0.8457504166666666 }, { "position": 2, "n": 60, "alpha_mean": 0.6994166666666667, "alpha_median": 0.696, "ci95_low": 0.6790833333333334, "ci95_high": 0.7209337499999998 }, { "position": 3, "n": 60, "alpha_mean": 0.5953000000000002, "alpha_median": 0.585, "ci95_low": 0.5718650000000002, "ci95_high": 0.6201179166666667 }, { "position": 4, "n": 60, "alpha_mean": 0.5117333333333334, "alpha_median": 0.492, "ci95_low": 0.4881979166666667, "ci95_high": 0.5381729166666667 }, { "position": 5, "n": 60, "alpha_mean": 0.44403333333333334, "alpha_median": 0.4175, "ci95_low": 0.4157829166666667, "ci95_high": 0.4744208333333333 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.7753448275862069, "alpha_median": 0.769, "ci95_low": 0.7534068965517241, "ci95_high": 0.7984706896551723 }, { "position": 2, "n": 58, "alpha_mean": 0.6073620689655171, "alpha_median": 0.5805, "ci95_low": 0.575498275862069, "ci95_high": 0.6407241379310344 }, { "position": 3, "n": 58, "alpha_mean": 0.488948275862069, "alpha_median": 0.46950000000000003, "ci95_low": 0.4514625, "ci95_high": 0.5298508620689655 }, { "position": 4, "n": 58, "alpha_mean": 0.40144827586206894, "alpha_median": 0.363, "ci95_low": 0.3599090517241379, "ci95_high": 0.44630129310344824 }, { "position": 5, "n": 58, "alpha_mean": 0.33731034482758626, "alpha_median": 0.283, "ci95_low": 0.2937344827586207, "ci95_high": 0.3847327586206897 } ] } }, "ksweep-qwen-q4-vanilla17b-k7": { "family": "ksweep", "target": "qwen-q4", "drafter": "vanilla17b", "k": 7, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.0000000000105516e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8567666666666667, "alpha_median": 0.857, "ci95_low": 0.8415325, "ci95_high": 0.8714508333333333 }, { "position": 2, "n": 60, "alpha_mean": 0.7372166666666667, "alpha_median": 0.732, "ci95_low": 0.7132141666666667, "ci95_high": 0.7604666666666666 }, { "position": 3, "n": 60, "alpha_mean": 0.6550666666666667, "alpha_median": 0.656, "ci95_low": 0.6266645833333334, "ci95_high": 0.6839679166666665 }, { "position": 4, "n": 60, "alpha_mean": 0.58005, "alpha_median": 0.59, "ci95_low": 0.54862375, "ci95_high": 0.6120524999999999 }, { "position": 5, "n": 60, "alpha_mean": 0.5245666666666667, "alpha_median": 0.521, "ci95_low": 0.4935104166666666, "ci95_high": 0.5572333333333331 }, { "position": 6, "n": 60, "alpha_mean": 0.47533333333333333, "alpha_median": 0.4745, "ci95_low": 0.4426595833333334, "ci95_high": 0.5094366666666666 }, { "position": 7, "n": 60, "alpha_mean": 0.42833333333333334, "alpha_median": 0.417, "ci95_low": 0.39536541666666664, "ci95_high": 0.4637670833333333 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.8221666666666666, "alpha_median": 0.826, "ci95_low": 0.8074325, "ci95_high": 0.8374529166666667 }, { "position": 2, "n": 60, "alpha_mean": 0.6857833333333333, "alpha_median": 0.673, "ci95_low": 0.6650241666666667, "ci95_high": 0.7074675 }, { "position": 3, "n": 60, "alpha_mean": 0.5782500000000002, "alpha_median": 0.564, "ci95_low": 0.5552654166666666, "ci95_high": 0.6034895833333332 }, { "position": 4, "n": 60, "alpha_mean": 0.4988000000000001, "alpha_median": 0.4785, "ci95_low": 0.47428333333333333, "ci95_high": 0.5256604166666666 }, { "position": 5, "n": 60, "alpha_mean": 0.4322833333333333, "alpha_median": 0.415, "ci95_low": 0.4041833333333334, "ci95_high": 0.46378500000000006 }, { "position": 6, "n": 60, "alpha_mean": 0.3756166666666666, "alpha_median": 0.358, "ci95_low": 0.3435120833333334, "ci95_high": 0.40943458333333327 }, { "position": 7, "n": 60, "alpha_mean": 0.3322, "alpha_median": 0.305, "ci95_low": 0.3008808333333333, "ci95_high": 0.3676062500000001 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.7714827586206897, "alpha_median": 0.7615000000000001, "ci95_low": 0.750240948275862, "ci95_high": 0.7938982758620688 }, { "position": 2, "n": 58, "alpha_mean": 0.5965, "alpha_median": 0.5805, "ci95_low": 0.5664987068965518, "ci95_high": 0.6276793103448276 }, { "position": 3, "n": 58, "alpha_mean": 0.4778965517241379, "alpha_median": 0.465, "ci95_low": 0.4421551724137931, "ci95_high": 0.5167913793103448 }, { "position": 4, "n": 58, "alpha_mean": 0.39296551724137935, "alpha_median": 0.3585, "ci95_low": 0.3511181034482759, "ci95_high": 0.43686250000000004 }, { "position": 5, "n": 58, "alpha_mean": 0.3258103448275862, "alpha_median": 0.268, "ci95_low": 0.28316508620689657, "ci95_high": 0.3719310344827586 }, { "position": 6, "n": 58, "alpha_mean": 0.28493103448275864, "alpha_median": 0.214, "ci95_low": 0.24000862068965514, "ci95_high": 0.3335034482758621 }, { "position": 7, "n": 58, "alpha_mean": 0.2499655172413793, "alpha_median": 0.1795, "ci95_low": 0.2041969827586207, "ci95_high": 0.30017543103448274 } ] } }, "ksweep-qwen-q8-eagle3-k10": { "family": "ksweep", "target": "qwen-q8", "drafter": "eagle3", "k": 10, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.000000000002225e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.6383833333333334, "alpha_median": 0.631, "ci95_low": 0.6204491666666665, "ci95_high": 0.6561166666666665 }, { "position": 2, "n": 60, "alpha_mean": 0.4114333333333333, "alpha_median": 0.406, "ci95_low": 0.39191499999999996, "ci95_high": 0.431985 }, { "position": 3, "n": 60, "alpha_mean": 0.2123, "alpha_median": 0.214, "ci95_low": 0.19686250000000002, "ci95_high": 0.22998333333333332 }, { "position": 4, "n": 60, "alpha_mean": 0.11678333333333334, "alpha_median": 0.114, "ci95_low": 0.10509875, "ci95_high": 0.12958583333333332 }, { "position": 5, "n": 60, "alpha_mean": 0.06493333333333333, "alpha_median": 0.062, "ci95_low": 0.05644958333333333, "ci95_high": 0.07463541666666665 }, { "position": 6, "n": 60, "alpha_mean": 0.03548333333333334, "alpha_median": 0.0315, "ci95_low": 0.029499583333333326, "ci95_high": 0.04180083333333333 }, { "position": 7, "n": 60, "alpha_mean": 0.019933333333333338, "alpha_median": 0.019, "ci95_low": 0.0161, "ci95_high": 0.02403416666666666 }, { "position": 8, "n": 60, "alpha_mean": 0.011000000000000001, "alpha_median": 0.01, "ci95_low": 0.008366666666666666, "ci95_high": 0.013767083333333333 }, { "position": 9, "n": 60, "alpha_mean": 0.006816666666666667, "alpha_median": 0.004, "ci95_low": 0.004932916666666666, "ci95_high": 0.008833333333333334 }, { "position": 10, "n": 60, "alpha_mean": 0.0042833333333333334, "alpha_median": 0.0, "ci95_low": 0.002716666666666667, "ci95_high": 0.005933333333333333 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.64665, "alpha_median": 0.65, "ci95_low": 0.6291, "ci95_high": 0.6649175 }, { "position": 2, "n": 60, "alpha_mean": 0.40869999999999995, "alpha_median": 0.401, "ci95_low": 0.38830000000000003, "ci95_high": 0.4312008333333334 }, { "position": 3, "n": 60, "alpha_mean": 0.19943333333333332, "alpha_median": 0.1765, "ci95_low": 0.17841625, "ci95_high": 0.22465374999999999 }, { "position": 4, "n": 60, "alpha_mean": 0.11043333333333333, "alpha_median": 0.092, "ci95_low": 0.09008041666666666, "ci95_high": 0.1367220833333333 }, { "position": 5, "n": 60, "alpha_mean": 0.06545, "alpha_median": 0.045, "ci95_low": 0.04749624999999999, "ci95_high": 0.08988333333333334 }, { "position": 6, "n": 60, "alpha_mean": 0.04083333333333334, "alpha_median": 0.02, "ci95_low": 0.025680416666666664, "ci95_high": 0.06138375000000001 }, { "position": 7, "n": 60, "alpha_mean": 0.028283333333333334, "alpha_median": 0.01, "ci95_low": 0.016983333333333336, "ci95_high": 0.04260249999999999 }, { "position": 8, "n": 60, "alpha_mean": 0.020483333333333336, "alpha_median": 0.008, "ci95_low": 0.011382916666666668, "ci95_high": 0.03171791666666666 }, { "position": 9, "n": 60, "alpha_mean": 0.012733333333333334, "alpha_median": 0.0, "ci95_low": 0.006415833333333332, "ci95_high": 0.021366666666666662 }, { "position": 10, "n": 60, "alpha_mean": 0.009200000000000002, "alpha_median": 0.0, "ci95_low": 0.0034825000000000004, "ci95_high": 0.017300833333333328 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.577948275862069, "alpha_median": 0.609, "ci95_low": 0.5294887931034482, "ci95_high": 0.623278448275862 }, { "position": 2, "n": 58, "alpha_mean": 0.3406379310344828, "alpha_median": 0.345, "ci95_low": 0.2972030172413792, "ci95_high": 0.38113275862068957 }, { "position": 3, "n": 58, "alpha_mean": 0.17277586206896553, "alpha_median": 0.1625, "ci95_low": 0.1397866379310345, "ci95_high": 0.2088625 }, { "position": 4, "n": 58, "alpha_mean": 0.11274137931034484, "alpha_median": 0.0845, "ci95_low": 0.08296508620689654, "ci95_high": 0.14803965517241377 }, { "position": 5, "n": 58, "alpha_mean": 0.06896551724137931, "alpha_median": 0.0455, "ci95_low": 0.046757758620689656, "ci95_high": 0.09586336206896551 }, { "position": 6, "n": 58, "alpha_mean": 0.05379310344827586, "alpha_median": 0.028, "ci95_low": 0.03434439655172414, "ci95_high": 0.07764525862068962 }, { "position": 7, "n": 58, "alpha_mean": 0.0381206896551724, "alpha_median": 0.0095, "ci95_low": 0.02318965517241379, "ci95_high": 0.056193965517241345 }, { "position": 8, "n": 58, "alpha_mean": 0.030896551724137935, "alpha_median": 0.009, "ci95_low": 0.017016810344827587, "ci95_high": 0.04772844827586205 }, { "position": 9, "n": 58, "alpha_mean": 0.023724137931034485, "alpha_median": 0.0, "ci95_low": 0.012065517241379312, "ci95_high": 0.03824999999999995 }, { "position": 10, "n": 58, "alpha_mean": 0.01812068965517241, "alpha_median": 0.0, "ci95_low": 0.00794741379310345, "ci95_high": 0.031227155172413777 } ] } }, "ksweep-qwen-q8-eagle3-k5": { "family": "ksweep", "target": "qwen-q8", "drafter": "eagle3", "k": 5, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 4.999999999999449e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.644, "alpha_median": 0.629, "ci95_low": 0.6263645833333332, "ci95_high": 0.6616704166666665 }, { "position": 2, "n": 60, "alpha_mean": 0.41586666666666666, "alpha_median": 0.422, "ci95_low": 0.39656375, "ci95_high": 0.4358345833333334 }, { "position": 3, "n": 60, "alpha_mean": 0.22018333333333331, "alpha_median": 0.2165, "ci95_low": 0.20504958333333334, "ci95_high": 0.2355683333333333 }, { "position": 4, "n": 60, "alpha_mean": 0.12205, "alpha_median": 0.1155, "ci95_low": 0.11074958333333332, "ci95_high": 0.13483625 }, { "position": 5, "n": 60, "alpha_mean": 0.06903333333333334, "alpha_median": 0.063, "ci95_low": 0.06015000000000001, "ci95_high": 0.07895166666666667 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.6542833333333333, "alpha_median": 0.6465000000000001, "ci95_low": 0.6373291666666667, "ci95_high": 0.6723512500000001 }, { "position": 2, "n": 60, "alpha_mean": 0.41663333333333336, "alpha_median": 0.412, "ci95_low": 0.39533, "ci95_high": 0.4391191666666666 }, { "position": 3, "n": 60, "alpha_mean": 0.20393333333333338, "alpha_median": 0.1845, "ci95_low": 0.18191583333333333, "ci95_high": 0.22860499999999997 }, { "position": 4, "n": 60, "alpha_mean": 0.11448333333333333, "alpha_median": 0.09, "ci95_low": 0.09354500000000002, "ci95_high": 0.13943791666666663 }, { "position": 5, "n": 60, "alpha_mean": 0.06634999999999999, "alpha_median": 0.045, "ci95_low": 0.048914166666666654, "ci95_high": 0.08815166666666666 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.5883793103448276, "alpha_median": 0.6265000000000001, "ci95_low": 0.5389129310344828, "ci95_high": 0.6331443965517242 }, { "position": 2, "n": 58, "alpha_mean": 0.3518275862068965, "alpha_median": 0.37, "ci95_low": 0.3077607758620689, "ci95_high": 0.39405560344827584 }, { "position": 3, "n": 58, "alpha_mean": 0.18241379310344827, "alpha_median": 0.174, "ci95_low": 0.1496987068965517, "ci95_high": 0.21997456896551718 }, { "position": 4, "n": 58, "alpha_mean": 0.11779310344827586, "alpha_median": 0.0885, "ci95_low": 0.09025387931034483, "ci95_high": 0.15170948275862067 }, { "position": 5, "n": 58, "alpha_mean": 0.07225862068965518, "alpha_median": 0.0445, "ci95_low": 0.050567672413793105, "ci95_high": 0.0976306034482758 } ] } }, "ksweep-qwen-q8-eagle3-k7": { "family": "ksweep", "target": "qwen-q8", "drafter": "eagle3", "k": 7, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.000000000002225e-05, "per_domain": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.6422333333333333, "alpha_median": 0.6315, "ci95_low": 0.6244462499999999, "ci95_high": 0.6602345833333334 }, { "position": 2, "n": 60, "alpha_mean": 0.4123833333333334, "alpha_median": 0.4095, "ci95_low": 0.3926487499999999, "ci95_high": 0.43237125 }, { "position": 3, "n": 60, "alpha_mean": 0.21725, "alpha_median": 0.214, "ci95_low": 0.20271458333333336, "ci95_high": 0.2327575 }, { "position": 4, "n": 60, "alpha_mean": 0.11929999999999999, "alpha_median": 0.1155, "ci95_low": 0.10796541666666669, "ci95_high": 0.13148624999999997 }, { "position": 5, "n": 60, "alpha_mean": 0.06716666666666665, "alpha_median": 0.063, "ci95_low": 0.05866583333333334, "ci95_high": 0.0768675 }, { "position": 6, "n": 60, "alpha_mean": 0.03963333333333333, "alpha_median": 0.035500000000000004, "ci95_low": 0.033666666666666664, "ci95_high": 0.04583499999999999 }, { "position": 7, "n": 60, "alpha_mean": 0.022533333333333332, "alpha_median": 0.0195, "ci95_low": 0.01825, "ci95_high": 0.027017083333333334 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.6501833333333332, "alpha_median": 0.641, "ci95_low": 0.6335320833333334, "ci95_high": 0.6674170833333334 }, { "position": 2, "n": 60, "alpha_mean": 0.4082166666666666, "alpha_median": 0.404, "ci95_low": 0.3865483333333332, "ci95_high": 0.4309508333333333 }, { "position": 3, "n": 60, "alpha_mean": 0.19706666666666664, "alpha_median": 0.184, "ci95_low": 0.17731624999999995, "ci95_high": 0.22053708333333333 }, { "position": 4, "n": 60, "alpha_mean": 0.10885000000000002, "alpha_median": 0.088, "ci95_low": 0.09053041666666665, "ci95_high": 0.13218708333333332 }, { "position": 5, "n": 60, "alpha_mean": 0.06378333333333333, "alpha_median": 0.043, "ci95_low": 0.0474325, "ci95_high": 0.08616875 }, { "position": 6, "n": 60, "alpha_mean": 0.03786666666666667, "alpha_median": 0.02, "ci95_low": 0.0248975, "ci95_high": 0.05433458333333333 }, { "position": 7, "n": 60, "alpha_mean": 0.027649999999999997, "alpha_median": 0.009, "ci95_low": 0.016564583333333334, "ci95_high": 0.041801250000000005 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.5866896551724138, "alpha_median": 0.62, "ci95_low": 0.5381525862068965, "ci95_high": 0.6317435344827586 }, { "position": 2, "n": 58, "alpha_mean": 0.34374137931034476, "alpha_median": 0.3615, "ci95_low": 0.3013275862068966, "ci95_high": 0.38341163793103433 }, { "position": 3, "n": 58, "alpha_mean": 0.17375862068965517, "alpha_median": 0.1685, "ci95_low": 0.14348017241379313, "ci95_high": 0.20646594827586204 }, { "position": 4, "n": 58, "alpha_mean": 0.11463793103448276, "alpha_median": 0.08549999999999999, "ci95_low": 0.08758534482758619, "ci95_high": 0.1466258620689655 }, { "position": 5, "n": 58, "alpha_mean": 0.06991379310344828, "alpha_median": 0.041499999999999995, "ci95_low": 0.04782629310344828, "ci95_high": 0.09610560344827586 }, { "position": 6, "n": 58, "alpha_mean": 0.05153448275862069, "alpha_median": 0.019, "ci95_low": 0.031843534482758624, "ci95_high": 0.07505689655172411 }, { "position": 7, "n": 58, "alpha_mean": 0.03632758620689655, "alpha_median": 0.0095, "ci95_low": 0.02136206896551724, "ci95_high": 0.05450474137931032 } ] } } }, "ksweep": { "ksweep-gemma-q4-dflash-f16-k10": { "family": "ksweep", "target": "gemma-q4", "drafter": "dflash-f16", "k": 10, "n_excluded": 3, "n_matched": 175, "log_lines": 175, "max_alpha_diff": 5.0000000000050004e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-gemma-q4-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 175, "baseline_n_matched": 175, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 55 }, "baseline_excluded": 3, "baseline_all": 33.47058285714286, "baseline_by_domain": { "math": 32.767716666666665, "code": 32.50666666666667, "chat": 35.28889090909091 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8581333333333333, "alpha_median": 0.86, "ci95_low": 0.8396154166666667, "ci95_high": 0.8742508333333333 }, { "position": 2, "n": 60, "alpha_mean": 0.7290666666666668, "alpha_median": 0.7255, "ci95_low": 0.70286625, "ci95_high": 0.7540858333333336 }, { "position": 3, "n": 60, "alpha_mean": 0.58595, "alpha_median": 0.5934999999999999, "ci95_low": 0.5582470833333333, "ci95_high": 0.6129883333333332 }, { "position": 4, "n": 60, "alpha_mean": 0.4879833333333333, "alpha_median": 0.471, "ci95_low": 0.4562466666666666, "ci95_high": 0.51908375 }, { "position": 5, "n": 60, "alpha_mean": 0.3957166666666667, "alpha_median": 0.375, "ci95_low": 0.36298125, "ci95_high": 0.42700749999999993 }, { "position": 6, "n": 60, "alpha_mean": 0.3183333333333333, "alpha_median": 0.303, "ci95_low": 0.28596375000000007, "ci95_high": 0.3491191666666666 }, { "position": 7, "n": 60, "alpha_mean": 0.2571666666666667, "alpha_median": 0.2255, "ci95_low": 0.2281825, "ci95_high": 0.284435 }, { "position": 8, "n": 60, "alpha_mean": 0.2046666666666667, "alpha_median": 0.184, "ci95_low": 0.17776083333333334, "ci95_high": 0.23036874999999998 }, { "position": 9, "n": 60, "alpha_mean": 0.1633666666666667, "alpha_median": 0.1615, "ci95_low": 0.14023, "ci95_high": 0.1857675 }, { "position": 10, "n": 60, "alpha_mean": 0.119, "alpha_median": 0.10200000000000001, "ci95_low": 0.09944916666666667, "ci95_high": 0.13833458333333334 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.7975833333333332, "alpha_median": 0.805, "ci95_low": 0.7638662500000001, "ci95_high": 0.8274833333333335 }, { "position": 2, "n": 60, "alpha_mean": 0.6294500000000001, "alpha_median": 0.6485000000000001, "ci95_low": 0.5893558333333334, "ci95_high": 0.66490375 }, { "position": 3, "n": 60, "alpha_mean": 0.4983833333333333, "alpha_median": 0.492, "ci95_low": 0.45793291666666663, "ci95_high": 0.5402670833333332 }, { "position": 4, "n": 60, "alpha_mean": 0.3997666666666666, "alpha_median": 0.3985, "ci95_low": 0.3604479166666667, "ci95_high": 0.44135541666666667 }, { "position": 5, "n": 60, "alpha_mean": 0.3185666666666666, "alpha_median": 0.2845, "ci95_low": 0.28028624999999996, "ci95_high": 0.35915333333333327 }, { "position": 6, "n": 60, "alpha_mean": 0.25938333333333335, "alpha_median": 0.225, "ci95_low": 0.22559875, "ci95_high": 0.29620625 }, { "position": 7, "n": 60, "alpha_mean": 0.20616666666666666, "alpha_median": 0.176, "ci95_low": 0.175915, "ci95_high": 0.24012041666666661 }, { "position": 8, "n": 60, "alpha_mean": 0.16380000000000003, "alpha_median": 0.1355, "ci95_low": 0.13546458333333336, "ci95_high": 0.19675666666666658 }, { "position": 9, "n": 60, "alpha_mean": 0.13241666666666668, "alpha_median": 0.0935, "ci95_low": 0.10526500000000001, "ci95_high": 0.16395499999999996 }, { "position": 10, "n": 60, "alpha_mean": 0.10470000000000002, "alpha_median": 0.0765, "ci95_low": 0.08091625000000001, "ci95_high": 0.13463999999999995 } ], "chat": [ { "position": 1, "n": 55, "alpha_mean": 0.6401454545454547, "alpha_median": 0.667, "ci95_low": 0.5908345454545455, "ci95_high": 0.6839649999999998 }, { "position": 2, "n": 55, "alpha_mean": 0.4230181818181819, "alpha_median": 0.422, "ci95_low": 0.36814090909090913, "ci95_high": 0.47485499999999997 }, { "position": 3, "n": 55, "alpha_mean": 0.2565090909090909, "alpha_median": 0.206, "ci95_low": 0.21030545454545452, "ci95_high": 0.3052913636363636 }, { "position": 4, "n": 55, "alpha_mean": 0.1807272727272727, "alpha_median": 0.137, "ci95_low": 0.13614363636363633, "ci95_high": 0.2264454545454545 }, { "position": 5, "n": 55, "alpha_mean": 0.1324, "alpha_median": 0.082, "ci95_low": 0.09220954545454546, "ci95_high": 0.17708409090909083 }, { "position": 6, "n": 55, "alpha_mean": 0.09461818181818181, "alpha_median": 0.041, "ci95_low": 0.05852681818181819, "ci95_high": 0.13318727272727268 }, { "position": 7, "n": 55, "alpha_mean": 0.07121818181818182, "alpha_median": 0.02, "ci95_low": 0.03836272727272727, "ci95_high": 0.10632727272727273 }, { "position": 8, "n": 55, "alpha_mean": 0.05894545454545455, "alpha_median": 0.011, "ci95_low": 0.029181818181818187, "ci95_high": 0.09329181818181818 }, { "position": 9, "n": 55, "alpha_mean": 0.04472727272727272, "alpha_median": 0.009, "ci95_low": 0.018727272727272728, "ci95_high": 0.07535272727272724 }, { "position": 10, "n": 55, "alpha_mean": 0.03549090909090909, "alpha_median": 0.0, "ci95_low": 0.013397727272727275, "ci95_high": 0.06263681818181818 } ] }, "breakeven_pooled": { "n": 175, "beta": 185.65338930253327, "intercept": 21.412957738796678, "r2": 0.9968518546115529, "alpha_be": 0.06494697006957176, "ci95": 0.00819673142708027 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 182.85802939153714, "intercept": 22.722131583509295, "r2": 0.9975750053585383, "alpha_be": 0.05493652707832522, "ci95": 0.012397537312855884 }, "code": { "n": 60, "beta": 188.60128085464027, "intercept": 20.246533752861733, "r2": 0.9985554499572545, "alpha_be": 0.06500556548846625, "ci95": 0.009562440138197638 }, "chat": { "n": 55, "beta": 183.88182014594216, "intercept": 21.735519912245216, "r2": 0.9904598049419733, "alpha_be": 0.07370696562655699, "ci95": 0.026042021172539336 } } }, "ksweep-gemma-q4-dflash-f16-k5": { "family": "ksweep", "target": "gemma-q4", "drafter": "dflash-f16", "k": 5, "n_excluded": 3, "n_matched": 175, "log_lines": 175, "max_alpha_diff": 4.999999999999449e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-gemma-q4-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 175, "baseline_n_matched": 175, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 55 }, "baseline_excluded": 3, "baseline_all": 33.47058285714286, "baseline_by_domain": { "math": 32.767716666666665, "code": 32.50666666666667, "chat": 35.28889090909091 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8648166666666665, "alpha_median": 0.8805000000000001, "ci95_low": 0.84664875, "ci95_high": 0.8817375 }, { "position": 2, "n": 60, "alpha_mean": 0.7476166666666665, "alpha_median": 0.754, "ci95_low": 0.7237820833333334, "ci95_high": 0.7708 }, { "position": 3, "n": 60, "alpha_mean": 0.6148666666666667, "alpha_median": 0.618, "ci95_low": 0.5844333333333334, "ci95_high": 0.6425420833333334 }, { "position": 4, "n": 60, "alpha_mean": 0.5182333333333334, "alpha_median": 0.504, "ci95_low": 0.48459916666666675, "ci95_high": 0.5506170833333333 }, { "position": 5, "n": 60, "alpha_mean": 0.43478333333333324, "alpha_median": 0.437, "ci95_low": 0.40129875000000004, "ci95_high": 0.4672016666666667 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.8290833333333333, "alpha_median": 0.831, "ci95_low": 0.8066158333333333, "ci95_high": 0.8515845833333333 }, { "position": 2, "n": 60, "alpha_mean": 0.6821666666666667, "alpha_median": 0.68, "ci95_low": 0.6523333333333334, "ci95_high": 0.7136345833333332 }, { "position": 3, "n": 60, "alpha_mean": 0.5617, "alpha_median": 0.5665, "ci95_low": 0.5239529166666667, "ci95_high": 0.5996862499999999 }, { "position": 4, "n": 60, "alpha_mean": 0.46573333333333333, "alpha_median": 0.462, "ci95_low": 0.42466125, "ci95_high": 0.5074033333333333 }, { "position": 5, "n": 60, "alpha_mean": 0.39039999999999997, "alpha_median": 0.377, "ci95_low": 0.34859750000000006, "ci95_high": 0.4326854166666667 } ], "chat": [ { "position": 1, "n": 55, "alpha_mean": 0.6564363636363636, "alpha_median": 0.68, "ci95_low": 0.6045904545454546, "ci95_high": 0.7042477272727272 }, { "position": 2, "n": 55, "alpha_mean": 0.4465636363636364, "alpha_median": 0.413, "ci95_low": 0.39149863636363635, "ci95_high": 0.5005781818181817 }, { "position": 3, "n": 55, "alpha_mean": 0.2991818181818182, "alpha_median": 0.25, "ci95_low": 0.24532045454545454, "ci95_high": 0.35314772727272725 }, { "position": 4, "n": 55, "alpha_mean": 0.2236545454545455, "alpha_median": 0.149, "ci95_low": 0.1724536363636364, "ci95_high": 0.2765854545454545 }, { "position": 5, "n": 55, "alpha_mean": 0.1691272727272728, "alpha_median": 0.106, "ci95_low": 0.11848954545454544, "ci95_high": 0.2225631818181817 } ] }, "breakeven_pooled": { "n": 175, "beta": 96.77710115191583, "intercept": 22.015537871718465, "r2": 0.9931431470600357, "alpha_be": 0.11836524187103765, "ci95": 0.01160099637459681 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 98.1041006766699, "intercept": 21.27439792774494, "r2": 0.998207231404384, "alpha_be": 0.11715431525947363, "ci95": 0.010089068401194437 }, "code": { "n": 60, "beta": 99.30438547933836, "intercept": 20.365855063842325, "r2": 0.9979091343950341, "alpha_be": 0.1222585643546468, "ci95": 0.010933479480822012 }, "chat": { "n": 55, "beta": 95.1663825453732, "intercept": 22.65345979970406, "r2": 0.9799522512950171, "alpha_be": 0.1327720017450759, "ci95": 0.036674618802919556 } } }, "ksweep-gemma-q4-dflash-f16-k7": { "family": "ksweep", "target": "gemma-q4", "drafter": "dflash-f16", "k": 7, "n_excluded": 3, "n_matched": 175, "log_lines": 175, "max_alpha_diff": 5.0000000000050004e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-gemma-q4-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 175, "baseline_n_matched": 175, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 55 }, "baseline_excluded": 3, "baseline_all": 33.47058285714286, "baseline_by_domain": { "math": 32.767716666666665, "code": 32.50666666666667, "chat": 35.28889090909091 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8579166666666665, "alpha_median": 0.867, "ci95_low": 0.8395308333333333, "ci95_high": 0.8751016666666668 }, { "position": 2, "n": 60, "alpha_mean": 0.7313999999999999, "alpha_median": 0.7404999999999999, "ci95_low": 0.7069320833333336, "ci95_high": 0.7555179166666667 }, { "position": 3, "n": 60, "alpha_mean": 0.5955666666666666, "alpha_median": 0.605, "ci95_low": 0.5640833333333333, "ci95_high": 0.6246870833333333 }, { "position": 4, "n": 60, "alpha_mean": 0.49323333333333336, "alpha_median": 0.482, "ci95_low": 0.45769624999999997, "ci95_high": 0.5276183333333333 }, { "position": 5, "n": 60, "alpha_mean": 0.4169, "alpha_median": 0.391, "ci95_low": 0.38221625000000004, "ci95_high": 0.4508516666666667 }, { "position": 6, "n": 60, "alpha_mean": 0.35129999999999995, "alpha_median": 0.333, "ci95_low": 0.3176629166666667, "ci95_high": 0.38515333333333324 }, { "position": 7, "n": 60, "alpha_mean": 0.28978333333333334, "alpha_median": 0.2665, "ci95_low": 0.2591608333333334, "ci95_high": 0.32101874999999996 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.8238166666666665, "alpha_median": 0.823, "ci95_low": 0.7995483333333334, "ci95_high": 0.8470179166666668 }, { "position": 2, "n": 60, "alpha_mean": 0.6661166666666667, "alpha_median": 0.667, "ci95_low": 0.633095, "ci95_high": 0.6982841666666669 }, { "position": 3, "n": 60, "alpha_mean": 0.5352166666666667, "alpha_median": 0.534, "ci95_low": 0.49966374999999996, "ci95_high": 0.5731175000000002 }, { "position": 4, "n": 60, "alpha_mean": 0.4425833333333334, "alpha_median": 0.441, "ci95_low": 0.4045970833333334, "ci95_high": 0.48081 }, { "position": 5, "n": 60, "alpha_mean": 0.36938333333333334, "alpha_median": 0.3695, "ci95_low": 0.33084708333333335, "ci95_high": 0.40810291666666665 }, { "position": 6, "n": 60, "alpha_mean": 0.3013, "alpha_median": 0.2845, "ci95_low": 0.26406583333333333, "ci95_high": 0.33970374999999986 }, { "position": 7, "n": 60, "alpha_mean": 0.24645, "alpha_median": 0.2235, "ci95_low": 0.2119533333333333, "ci95_high": 0.2845670833333334 } ], "chat": [ { "position": 1, "n": 55, "alpha_mean": 0.6426181818181819, "alpha_median": 0.658, "ci95_low": 0.5934681818181818, "ci95_high": 0.6863299999999998 }, { "position": 2, "n": 55, "alpha_mean": 0.4292727272727273, "alpha_median": 0.409, "ci95_low": 0.3773263636363636, "ci95_high": 0.4804581818181817 }, { "position": 3, "n": 55, "alpha_mean": 0.2806, "alpha_median": 0.242, "ci95_low": 0.23070636363636363, "ci95_high": 0.33234681818181816 }, { "position": 4, "n": 55, "alpha_mean": 0.20427272727272722, "alpha_median": 0.151, "ci95_low": 0.15617499999999998, "ci95_high": 0.2536563636363636 }, { "position": 5, "n": 55, "alpha_mean": 0.1530363636363636, "alpha_median": 0.087, "ci95_low": 0.10728318181818182, "ci95_high": 0.20192045454545446 }, { "position": 6, "n": 55, "alpha_mean": 0.12018181818181817, "alpha_median": 0.051, "ci95_low": 0.07734409090909093, "ci95_high": 0.1675277272727273 }, { "position": 7, "n": 55, "alpha_mean": 0.09421818181818181, "alpha_median": 0.028, "ci95_low": 0.052829999999999995, "ci95_high": 0.14032772727272727 } ] }, "breakeven_pooled": { "n": 175, "beta": 110.31798887226992, "intercept": 17.599562990669483, "r2": 0.996198622557111, "alpha_be": 0.14386610949597173, "ci95": 0.008626682072197842 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 110.87923409298594, "intercept": 17.376101245581285, "r2": 0.9994551157203211, "alpha_be": 0.1388142292557469, "ci95": 0.005560251667080671 }, "code": { "n": 60, "beta": 111.79921770706834, "intercept": 16.774345991889742, "r2": 0.9988222078385945, "alpha_be": 0.1407194164452751, "ci95": 0.008232365607627924 }, "chat": { "n": 55, "beta": 109.08969313042472, "intercept": 17.96546411810064, "r2": 0.9873296482279417, "alpha_be": 0.15879984894887222, "ci95": 0.029167721183299196 } } }, "ksweep-gemma-q4-mtp-k10": { "family": "ksweep", "target": "gemma-q4", "drafter": "mtp", "k": 10, "n_excluded": 3, "n_matched": 175, "log_lines": 175, "max_alpha_diff": 5.0000000000105516e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-gemma-q4-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 175, "baseline_n_matched": 175, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 55 }, "baseline_excluded": 3, "baseline_all": 33.47058285714286, "baseline_by_domain": { "math": 32.767716666666665, "code": 32.50666666666667, "chat": 35.28889090909091 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8836833333333335, "alpha_median": 0.8905000000000001, "ci95_low": 0.8681633333333334, "ci95_high": 0.8991691666666666 }, { "position": 2, "n": 60, "alpha_mean": 0.7941166666666667, "alpha_median": 0.8, "ci95_low": 0.7705591666666666, "ci95_high": 0.8170041666666669 }, { "position": 3, "n": 60, "alpha_mean": 0.7047999999999999, "alpha_median": 0.7195, "ci95_low": 0.6783975000000001, "ci95_high": 0.7299704166666666 }, { "position": 4, "n": 60, "alpha_mean": 0.6122833333333333, "alpha_median": 0.6385000000000001, "ci95_low": 0.5747329166666666, "ci95_high": 0.6443341666666665 }, { "position": 5, "n": 60, "alpha_mean": 0.5341833333333332, "alpha_median": 0.549, "ci95_low": 0.49784874999999995, "ci95_high": 0.5669670833333333 }, { "position": 6, "n": 60, "alpha_mean": 0.45953333333333324, "alpha_median": 0.463, "ci95_low": 0.42431375, "ci95_high": 0.49294 }, { "position": 7, "n": 60, "alpha_mean": 0.3834, "alpha_median": 0.3735, "ci95_low": 0.3501558333333333, "ci95_high": 0.4155025 }, { "position": 8, "n": 60, "alpha_mean": 0.3148833333333334, "alpha_median": 0.3115, "ci95_low": 0.2848829166666666, "ci95_high": 0.3450333333333333 }, { "position": 9, "n": 60, "alpha_mean": 0.25093333333333334, "alpha_median": 0.2415, "ci95_low": 0.22318125000000003, "ci95_high": 0.28111875000000003 }, { "position": 10, "n": 60, "alpha_mean": 0.20245, "alpha_median": 0.2, "ci95_low": 0.17943208333333335, "ci95_high": 0.22818499999999997 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.8295333333333333, "alpha_median": 0.8434999999999999, "ci95_low": 0.80103125, "ci95_high": 0.8527508333333333 }, { "position": 2, "n": 60, "alpha_mean": 0.7212499999999998, "alpha_median": 0.732, "ci95_low": 0.6896574999999999, "ci95_high": 0.7506525000000001 }, { "position": 3, "n": 60, "alpha_mean": 0.6143499999999998, "alpha_median": 0.604, "ci95_low": 0.5809475, "ci95_high": 0.6462687500000001 }, { "position": 4, "n": 60, "alpha_mean": 0.5350666666666667, "alpha_median": 0.523, "ci95_low": 0.5002779166666668, "ci95_high": 0.5712199999999998 }, { "position": 5, "n": 60, "alpha_mean": 0.46813333333333335, "alpha_median": 0.4495, "ci95_low": 0.43272666666666665, "ci95_high": 0.5032670833333333 }, { "position": 6, "n": 60, "alpha_mean": 0.39171666666666666, "alpha_median": 0.377, "ci95_low": 0.35819833333333334, "ci95_high": 0.4262558333333332 }, { "position": 7, "n": 60, "alpha_mean": 0.3330666666666667, "alpha_median": 0.309, "ci95_low": 0.2998495833333334, "ci95_high": 0.36738375 }, { "position": 8, "n": 60, "alpha_mean": 0.27935, "alpha_median": 0.25, "ci95_low": 0.2476816666666667, "ci95_high": 0.3136854166666667 }, { "position": 9, "n": 60, "alpha_mean": 0.23356666666666667, "alpha_median": 0.214, "ci95_low": 0.2039, "ci95_high": 0.2661545833333333 }, { "position": 10, "n": 60, "alpha_mean": 0.18168333333333336, "alpha_median": 0.159, "ci95_low": 0.1529316666666667, "ci95_high": 0.21411833333333333 } ], "chat": [ { "position": 1, "n": 55, "alpha_mean": 0.7337636363636365, "alpha_median": 0.747, "ci95_low": 0.6890713636363638, "ci95_high": 0.7732472727272728 }, { "position": 2, "n": 55, "alpha_mean": 0.5758181818181819, "alpha_median": 0.569, "ci95_low": 0.5250227272727273, "ci95_high": 0.6269495454545455 }, { "position": 3, "n": 55, "alpha_mean": 0.44114545454545445, "alpha_median": 0.388, "ci95_low": 0.384850909090909, "ci95_high": 0.5014190909090909 }, { "position": 4, "n": 55, "alpha_mean": 0.34834545454545457, "alpha_median": 0.277, "ci95_low": 0.28816, "ci95_high": 0.4101668181818182 }, { "position": 5, "n": 55, "alpha_mean": 0.26996363636363635, "alpha_median": 0.192, "ci95_low": 0.21188772727272728, "ci95_high": 0.3302231818181817 }, { "position": 6, "n": 55, "alpha_mean": 0.22503636363636365, "alpha_median": 0.139, "ci95_low": 0.1671940909090909, "ci95_high": 0.28587045454545446 }, { "position": 7, "n": 55, "alpha_mean": 0.19132727272727276, "alpha_median": 0.092, "ci95_low": 0.13407136363636363, "ci95_high": 0.25535454545454545 }, { "position": 8, "n": 55, "alpha_mean": 0.16399999999999998, "alpha_median": 0.069, "ci95_low": 0.10636363636363637, "ci95_high": 0.22853 }, { "position": 9, "n": 55, "alpha_mean": 0.1394, "alpha_median": 0.043, "ci95_low": 0.08196227272727274, "ci95_high": 0.20359090909090902 }, { "position": 10, "n": 55, "alpha_mean": 0.12441818181818183, "alpha_median": 0.028, "ci95_low": 0.065955, "ci95_high": 0.18841636363636347 } ] }, "breakeven_pooled": { "n": 175, "beta": 162.16078678337638, "intercept": 19.85372457913967, "r2": 0.995616045136786, "alpha_be": 0.08397133825080266, "ci95": 0.00952341837434763 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 160.9253770995331, "intercept": 20.989059374832642, "r2": 0.9976761270887337, "alpha_be": 0.07319328687699062, "ci95": 0.011947128835343374 }, "code": { "n": 60, "beta": 162.2759948294464, "intercept": 19.606006638373273, "r2": 0.9941057004073529, "alpha_be": 0.07949826492730554, "ci95": 0.019085179496654565 }, "chat": { "n": 55, "beta": 160.61421923484983, "intercept": 20.033221560650826, "r2": 0.9936719153258121, "alpha_be": 0.09498330484758183, "ci95": 0.02082473328687092 } } }, "ksweep-gemma-q4-mtp-k5": { "family": "ksweep", "target": "gemma-q4", "drafter": "mtp", "k": 5, "n_excluded": 3, "n_matched": 175, "log_lines": 175, "max_alpha_diff": 5.0000000000050004e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-gemma-q4-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 175, "baseline_n_matched": 175, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 55 }, "baseline_excluded": 3, "baseline_all": 33.47058285714286, "baseline_by_domain": { "math": 32.767716666666665, "code": 32.50666666666667, "chat": 35.28889090909091 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8950166666666667, "alpha_median": 0.8975, "ci95_low": 0.8799124999999999, "ci95_high": 0.9086012499999998 }, { "position": 2, "n": 60, "alpha_mean": 0.8061833333333333, "alpha_median": 0.8085, "ci95_low": 0.7858991666666666, "ci95_high": 0.8246529166666665 }, { "position": 3, "n": 60, "alpha_mean": 0.7245333333333333, "alpha_median": 0.743, "ci95_low": 0.6977758333333333, "ci95_high": 0.7487174999999999 }, { "position": 4, "n": 60, "alpha_mean": 0.6364333333333334, "alpha_median": 0.653, "ci95_low": 0.6066799999999999, "ci95_high": 0.6637341666666666 }, { "position": 5, "n": 60, "alpha_mean": 0.5607833333333334, "alpha_median": 0.56, "ci95_low": 0.5305791666666666, "ci95_high": 0.5900512499999999 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.8677666666666666, "alpha_median": 0.885, "ci95_low": 0.8488662499999999, "ci95_high": 0.88593375 }, { "position": 2, "n": 60, "alpha_mean": 0.7681333333333333, "alpha_median": 0.774, "ci95_low": 0.7418133333333332, "ci95_high": 0.7935254166666666 }, { "position": 3, "n": 60, "alpha_mean": 0.6667833333333332, "alpha_median": 0.661, "ci95_low": 0.6357720833333335, "ci95_high": 0.6976683333333333 }, { "position": 4, "n": 60, "alpha_mean": 0.5806333333333333, "alpha_median": 0.568, "ci95_low": 0.544045, "ci95_high": 0.6166670833333333 }, { "position": 5, "n": 60, "alpha_mean": 0.5109833333333332, "alpha_median": 0.5, "ci95_low": 0.47249791666666663, "ci95_high": 0.5504029166666666 } ], "chat": [ { "position": 1, "n": 55, "alpha_mean": 0.7516363636363635, "alpha_median": 0.757, "ci95_low": 0.7077790909090909, "ci95_high": 0.7898595454545454 }, { "position": 2, "n": 55, "alpha_mean": 0.5932545454545456, "alpha_median": 0.585, "ci95_low": 0.5417622727272728, "ci95_high": 0.6486572727272726 }, { "position": 3, "n": 55, "alpha_mean": 0.48161818181818183, "alpha_median": 0.412, "ci95_low": 0.42208954545454547, "ci95_high": 0.5441836363636364 }, { "position": 4, "n": 55, "alpha_mean": 0.39509090909090916, "alpha_median": 0.325, "ci95_low": 0.3306468181818182, "ci95_high": 0.46265500000000004 }, { "position": 5, "n": 55, "alpha_mean": 0.32367272727272733, "alpha_median": 0.219, "ci95_low": 0.25792818181818183, "ci95_high": 0.3945495454545453 } ] }, "breakeven_pooled": { "n": 175, "beta": 97.1633436471974, "intercept": 23.028844760741123, "r2": 0.9896940683887518, "alpha_be": 0.1074658168857996, "ci95": 0.014154215802069242 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 98.18364299128696, "intercept": 22.56271377246348, "r2": 0.9969393889040341, "alpha_be": 0.10393791250044371, "ci95": 0.013178769981063175 }, "code": { "n": 60, "beta": 100.48112287228884, "intercept": 20.527863286387202, "r2": 0.99232136023649, "alpha_be": 0.11921446574103758, "ci95": 0.02079684140496075 }, "chat": { "n": 55, "beta": 95.43848076041085, "intercept": 23.863787534554636, "r2": 0.9820395916741389, "alpha_be": 0.11971170625837915, "ci95": 0.03418819646866108 } } }, "ksweep-gemma-q4-mtp-k7": { "family": "ksweep", "target": "gemma-q4", "drafter": "mtp", "k": 7, "n_excluded": 3, "n_matched": 175, "log_lines": 175, "max_alpha_diff": 5.0000000000050004e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-gemma-q4-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 175, "baseline_n_matched": 175, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 55 }, "baseline_excluded": 3, "baseline_all": 33.47058285714286, "baseline_by_domain": { "math": 32.767716666666665, "code": 32.50666666666667, "chat": 35.28889090909091 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8807333333333333, "alpha_median": 0.883, "ci95_low": 0.8624995833333334, "ci95_high": 0.8964683333333335 }, { "position": 2, "n": 60, "alpha_mean": 0.79455, "alpha_median": 0.8025, "ci95_low": 0.7706141666666667, "ci95_high": 0.8169733333333333 }, { "position": 3, "n": 60, "alpha_mean": 0.712, "alpha_median": 0.7084999999999999, "ci95_low": 0.6851662500000001, "ci95_high": 0.7384850000000002 }, { "position": 4, "n": 60, "alpha_mean": 0.6325999999999999, "alpha_median": 0.638, "ci95_low": 0.6014795833333332, "ci95_high": 0.6633079166666666 }, { "position": 5, "n": 60, "alpha_mean": 0.5557833333333332, "alpha_median": 0.553, "ci95_low": 0.5222575000000002, "ci95_high": 0.5895004166666666 }, { "position": 6, "n": 60, "alpha_mean": 0.47598333333333337, "alpha_median": 0.4825, "ci95_low": 0.43973083333333335, "ci95_high": 0.5133391666666666 }, { "position": 7, "n": 60, "alpha_mean": 0.4049833333333333, "alpha_median": 0.39, "ci95_low": 0.3712995833333333, "ci95_high": 0.43846416666666665 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.8569333333333334, "alpha_median": 0.874, "ci95_low": 0.8370483333333333, "ci95_high": 0.8763833333333333 }, { "position": 2, "n": 60, "alpha_mean": 0.7463333333333334, "alpha_median": 0.763, "ci95_low": 0.718665, "ci95_high": 0.7722341666666669 }, { "position": 3, "n": 60, "alpha_mean": 0.6522333333333332, "alpha_median": 0.654, "ci95_low": 0.6200466666666665, "ci95_high": 0.6846175 }, { "position": 4, "n": 60, "alpha_mean": 0.5665999999999999, "alpha_median": 0.5680000000000001, "ci95_low": 0.5307308333333333, "ci95_high": 0.6019720833333333 }, { "position": 5, "n": 60, "alpha_mean": 0.49448333333333333, "alpha_median": 0.4755, "ci95_low": 0.4554829166666667, "ci95_high": 0.5327204166666667 }, { "position": 6, "n": 60, "alpha_mean": 0.42051666666666665, "alpha_median": 0.397, "ci95_low": 0.38084958333333335, "ci95_high": 0.4608499999999999 }, { "position": 7, "n": 60, "alpha_mean": 0.3630166666666667, "alpha_median": 0.3365, "ci95_low": 0.3226812499999999, "ci95_high": 0.4040516666666667 } ], "chat": [ { "position": 1, "n": 55, "alpha_mean": 0.7442181818181818, "alpha_median": 0.75, "ci95_low": 0.7010240909090908, "ci95_high": 0.7815731818181817 }, { "position": 2, "n": 55, "alpha_mean": 0.5852181818181819, "alpha_median": 0.561, "ci95_low": 0.53427, "ci95_high": 0.6413472727272729 }, { "position": 3, "n": 55, "alpha_mean": 0.4758545454545455, "alpha_median": 0.411, "ci95_low": 0.41785954545454534, "ci95_high": 0.5385318181818182 }, { "position": 4, "n": 55, "alpha_mean": 0.3937818181818182, "alpha_median": 0.301, "ci95_low": 0.32965363636363637, "ci95_high": 0.4654922727272726 }, { "position": 5, "n": 55, "alpha_mean": 0.32007272727272723, "alpha_median": 0.203, "ci95_low": 0.2553986363636363, "ci95_high": 0.3890754545454545 }, { "position": 6, "n": 55, "alpha_mean": 0.2712, "alpha_median": 0.159, "ci95_low": 0.2051377272727273, "ci95_high": 0.34276909090909086 }, { "position": 7, "n": 55, "alpha_mean": 0.23603636363636366, "alpha_median": 0.113, "ci95_low": 0.16681272727272728, "ci95_high": 0.3091281818181818 } ] }, "breakeven_pooled": { "n": 175, "beta": 106.382523204685, "intercept": 17.86586129833469, "r2": 0.992550436179729, "alpha_be": 0.14668501073981813, "ci95": 0.011853202195414992 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 107.52488863148434, "intercept": 17.432174324785418, "r2": 0.998683017231229, "alpha_be": 0.14262318740398908, "ci95": 0.00848984843653783 }, "code": { "n": 60, "beta": 107.19500825696319, "intercept": 17.29974734812864, "r2": 0.9960797533568224, "alpha_be": 0.1418621964381463, "ci95": 0.014793095909013997 }, "chat": { "n": 55, "beta": 104.72962189772807, "intercept": 18.35244241291177, "r2": 0.9864485826641572, "alpha_be": 0.16171593279233015, "ci95": 0.029348998495359956 } } }, "ksweep-qwen-q4-dflash-k10": { "family": "ksweep", "target": "qwen-q4", "drafter": "dflash-f16", "k": 10, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.000000000000143e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-qwen-q4-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 178, "baseline_n_matched": 178, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 58 }, "baseline_excluded": 0, "baseline_all": 51.49112359550562, "baseline_by_domain": { "math": 53.47833333333333, "code": 51.10905000000001, "chat": 49.83063793103449 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.014683333333333335, "alpha_median": 0.01, "ci95_low": 0.011265, "ci95_high": 0.0186 }, { "position": 2, "n": 60, "alpha_mean": 0.0034000000000000002, "alpha_median": 0.0, "ci95_low": 0.001816666666666667, "ci95_high": 0.0056 }, { "position": 3, "n": 60, "alpha_mean": 0.0017, "alpha_median": 0.0, "ci95_low": 0.00095, "ci95_high": 0.002533749999999998 }, { "position": 4, "n": 60, "alpha_mean": 0.0011333333333333334, "alpha_median": 0.0, "ci95_low": 0.0006666666666666666, "ci95_high": 0.0016666666666666668 }, { "position": 5, "n": 60, "alpha_mean": 0.0006000000000000001, "alpha_median": 0.0, "ci95_low": 0.0002666666666666667, "ci95_high": 0.001 }, { "position": 6, "n": 60, "alpha_mean": 0.0003333333333333333, "alpha_median": 0.0, "ci95_low": 6.666666666666667e-05, "ci95_high": 0.0006666666666666666 }, { "position": 7, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 8, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 9, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 10, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.013800000000000003, "alpha_median": 0.012, "ci95_low": 0.011216250000000002, "ci95_high": 0.01651666666666667 }, { "position": 2, "n": 60, "alpha_mean": 0.003916666666666667, "alpha_median": 0.004, "ci95_low": 0.002766666666666667, "ci95_high": 0.005233333333333334 }, { "position": 3, "n": 60, "alpha_mean": 0.00155, "alpha_median": 0.0, "ci95_low": 0.0008666666666666667, "ci95_high": 0.0023833333333333337 }, { "position": 4, "n": 60, "alpha_mean": 0.0006166666666666667, "alpha_median": 0.0, "ci95_low": 0.0002, "ci95_high": 0.0011166666666666666 }, { "position": 5, "n": 60, "alpha_mean": 0.0003333333333333333, "alpha_median": 0.0, "ci95_low": 6.666666666666667e-05, "ci95_high": 0.0006666666666666666 }, { "position": 6, "n": 60, "alpha_mean": 0.0002, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.00046666666666666666 }, { "position": 7, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 8, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 9, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 10, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.021448275862068968, "alpha_median": 0.016, "ci95_low": 0.017085344827586207, "ci95_high": 0.026397413793103446 }, { "position": 2, "n": 58, "alpha_mean": 0.004586206896551724, "alpha_median": 0.004, "ci95_low": 0.003327586206896552, "ci95_high": 0.005983189655172413 }, { "position": 3, "n": 58, "alpha_mean": 0.0018965517241379313, "alpha_median": 0.0, "ci95_low": 0.0011379310344827587, "ci95_high": 0.0027241379310344828 }, { "position": 4, "n": 58, "alpha_mean": 0.0010000000000000002, "alpha_median": 0.0, "ci95_low": 0.00041379310344827585, "ci95_high": 0.0016724137931034483 }, { "position": 5, "n": 58, "alpha_mean": 0.0005172413793103448, "alpha_median": 0.0, "ci95_low": 6.896551724137931e-05, "ci95_high": 0.0011379310344827585 }, { "position": 6, "n": 58, "alpha_mean": 0.0002931034482758621, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.000689655172413793 }, { "position": 7, "n": 58, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 8, "n": 58, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 9, "n": 58, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 10, "n": 58, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 } ] }, "breakeven_pooled": { "n": 178, "beta": 155.1157695563617, "intercept": 26.66819869703642, "r2": 0.8579197421864757, "alpha_be": 0.16002837731756042, "ci95": 0.06008432818710706 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 144.1451037644214, "intercept": 26.594035482046554, "r2": 0.7263764843539029, "alpha_be": 0.18650857468751908, "ci95": 0.15784748900271323 }, "code": { "n": 60, "beta": 152.7967926107302, "intercept": 26.696375780270486, "r2": 0.9301975990488072, "alpha_be": 0.15977216407888878, "ci95": 0.07046106558142584 }, "chat": { "n": 58, "beta": 159.838825494361, "intercept": 26.742363432294788, "r2": 0.9804824586963088, "alpha_be": 0.14444722317828992, "ci95": 0.03692654962141036 } } }, "ksweep-qwen-q4-dflash-k5": { "family": "ksweep", "target": "qwen-q4", "drafter": "dflash-f16", "k": 5, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 4.99999999999997e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-qwen-q4-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 178, "baseline_n_matched": 178, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 58 }, "baseline_excluded": 0, "baseline_all": 51.49112359550562, "baseline_by_domain": { "math": 53.47833333333333, "code": 51.10905000000001, "chat": 49.83063793103449 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.013850000000000001, "alpha_median": 0.012, "ci95_low": 0.0105, "ci95_high": 0.0178 }, { "position": 2, "n": 60, "alpha_mean": 0.003433333333333334, "alpha_median": 0.0, "ci95_low": 0.0018658333333333333, "ci95_high": 0.005633333333333334 }, { "position": 3, "n": 60, "alpha_mean": 0.0012666666666666668, "alpha_median": 0.0, "ci95_low": 0.0007333333333333333, "ci95_high": 0.0018666666666666666 }, { "position": 4, "n": 60, "alpha_mean": 0.0008, "alpha_median": 0.0, "ci95_low": 0.0004, "ci95_high": 0.0012000000000000001 }, { "position": 5, "n": 60, "alpha_mean": 0.0006000000000000001, "alpha_median": 0.0, "ci95_low": 0.0002666666666666667, "ci95_high": 0.001 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.012633333333333333, "alpha_median": 0.012, "ci95_low": 0.010132916666666669, "ci95_high": 0.015167083333333331 }, { "position": 2, "n": 60, "alpha_mean": 0.0030166666666666675, "alpha_median": 0.0, "ci95_low": 0.0020162500000000002, "ci95_high": 0.004 }, { "position": 3, "n": 60, "alpha_mean": 0.0012666666666666668, "alpha_median": 0.0, "ci95_low": 0.0007333333333333333, "ci95_high": 0.0018000000000000002 }, { "position": 4, "n": 60, "alpha_mean": 0.0005333333333333334, "alpha_median": 0.0, "ci95_low": 0.0002, "ci95_high": 0.0009333333333333334 }, { "position": 5, "n": 60, "alpha_mean": 0.0002, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.00046666666666666666 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.020879310344827588, "alpha_median": 0.016, "ci95_low": 0.01656853448275862, "ci95_high": 0.025655603448275863 }, { "position": 2, "n": 58, "alpha_mean": 0.003637931034482759, "alpha_median": 0.0, "ci95_low": 0.0024310344827586207, "ci95_high": 0.005051724137931035 }, { "position": 3, "n": 58, "alpha_mean": 0.0012413793103448277, "alpha_median": 0.0, "ci95_low": 0.0006896551724137933, "ci95_high": 0.0017931034482758623 }, { "position": 4, "n": 58, "alpha_mean": 0.0006206896551724138, "alpha_median": 0.0, "ci95_low": 0.00027586206896551725, "ci95_high": 0.0010344827586206897 }, { "position": 5, "n": 58, "alpha_mean": 0.00020689655172413793, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0004827586206896552 } ] }, "breakeven_pooled": { "n": 178, "beta": 142.46581957111744, "intercept": 28.828018645675503, "r2": 0.9628669806519348, "alpha_be": 0.15907748973091004, "ci95": 0.028993313293994443 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 140.20844229250247, "intercept": 28.85900754573448, "r2": 0.9953218488460952, "alpha_be": 0.17559089442159326, "ci95": 0.01763149923102116 }, "code": { "n": 60, "beta": 139.53866272524186, "intercept": 28.78510312523454, "r2": 0.9052874615079485, "alpha_be": 0.15998395311213756, "ci95": 0.08319621564077606 }, "chat": { "n": 58, "beta": 142.81525573315085, "intercept": 28.858762630748114, "r2": 0.9682201355429803, "alpha_be": 0.14684618385218, "ci95": 0.047414082345922426 } } }, "ksweep-qwen-q4-dflash-k7": { "family": "ksweep", "target": "qwen-q4", "drafter": "dflash-f16", "k": 7, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.000000000000143e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-qwen-q4-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 178, "baseline_n_matched": 178, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 58 }, "baseline_excluded": 0, "baseline_all": 51.49112359550562, "baseline_by_domain": { "math": 53.47833333333333, "code": 51.10905000000001, "chat": 49.83063793103449 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.014683333333333335, "alpha_median": 0.01, "ci95_low": 0.011265, "ci95_high": 0.0186 }, { "position": 2, "n": 60, "alpha_mean": 0.0034000000000000002, "alpha_median": 0.0, "ci95_low": 0.001816666666666667, "ci95_high": 0.0056 }, { "position": 3, "n": 60, "alpha_mean": 0.0017, "alpha_median": 0.0, "ci95_low": 0.00095, "ci95_high": 0.002533749999999998 }, { "position": 4, "n": 60, "alpha_mean": 0.0011333333333333334, "alpha_median": 0.0, "ci95_low": 0.0006666666666666666, "ci95_high": 0.0016666666666666668 }, { "position": 5, "n": 60, "alpha_mean": 0.0006000000000000001, "alpha_median": 0.0, "ci95_low": 0.0002666666666666667, "ci95_high": 0.001 }, { "position": 6, "n": 60, "alpha_mean": 0.0003333333333333333, "alpha_median": 0.0, "ci95_low": 6.666666666666667e-05, "ci95_high": 0.0006666666666666666 }, { "position": 7, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.013800000000000003, "alpha_median": 0.012, "ci95_low": 0.011216250000000002, "ci95_high": 0.01651666666666667 }, { "position": 2, "n": 60, "alpha_mean": 0.003916666666666667, "alpha_median": 0.004, "ci95_low": 0.002766666666666667, "ci95_high": 0.005233333333333334 }, { "position": 3, "n": 60, "alpha_mean": 0.00155, "alpha_median": 0.0, "ci95_low": 0.0008666666666666667, "ci95_high": 0.0023833333333333337 }, { "position": 4, "n": 60, "alpha_mean": 0.0006166666666666667, "alpha_median": 0.0, "ci95_low": 0.0002, "ci95_high": 0.0011166666666666666 }, { "position": 5, "n": 60, "alpha_mean": 0.0003333333333333333, "alpha_median": 0.0, "ci95_low": 6.666666666666667e-05, "ci95_high": 0.0006666666666666666 }, { "position": 6, "n": 60, "alpha_mean": 0.0002, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.00046666666666666666 }, { "position": 7, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.021448275862068968, "alpha_median": 0.016, "ci95_low": 0.017085344827586207, "ci95_high": 0.026397413793103446 }, { "position": 2, "n": 58, "alpha_mean": 0.004586206896551724, "alpha_median": 0.004, "ci95_low": 0.003327586206896552, "ci95_high": 0.005983189655172413 }, { "position": 3, "n": 58, "alpha_mean": 0.0018965517241379313, "alpha_median": 0.0, "ci95_low": 0.0011379310344827587, "ci95_high": 0.0027241379310344828 }, { "position": 4, "n": 58, "alpha_mean": 0.0010000000000000002, "alpha_median": 0.0, "ci95_low": 0.00041379310344827585, "ci95_high": 0.0016724137931034483 }, { "position": 5, "n": 58, "alpha_mean": 0.0005172413793103448, "alpha_median": 0.0, "ci95_low": 6.896551724137931e-05, "ci95_high": 0.0011379310344827585 }, { "position": 6, "n": 58, "alpha_mean": 0.0002931034482758621, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.000689655172413793 }, { "position": 7, "n": 58, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 } ] }, "breakeven_pooled": { "n": 178, "beta": 157.20818100115525, "intercept": 26.731094438453738, "r2": 0.9742909534426881, "alpha_be": 0.15749835027268672, "ci95": 0.023983953977647077 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 155.24845144897012, "intercept": 26.766995739962496, "r2": 0.9967171636491443, "alpha_be": 0.17205542048289482, "ci95": 0.014760591731619897 }, "code": { "n": 60, "beta": 152.59434518310505, "intercept": 26.704993927273883, "r2": 0.9321570455788156, "alpha_be": 0.15992765684365673, "ci95": 0.06939197242113475 }, "chat": { "n": 58, "beta": 160.08544693486283, "intercept": 26.73051794741944, "r2": 0.9821992129025822, "alpha_be": 0.1442986881438027, "ci95": 0.035234347004426285 } } }, "ksweep-qwen-q4-dspark-p0-k10": { "family": "ksweep", "target": "qwen-q4", "drafter": "dspark-p0", "k": 10, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.0000000000105516e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-qwen-q4-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 178, "baseline_n_matched": 178, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 58 }, "baseline_excluded": 0, "baseline_all": 51.49112359550562, "baseline_by_domain": { "math": 53.47833333333333, "code": 51.10905000000001, "chat": 49.83063793103449 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8275166666666666, "alpha_median": 0.8234999999999999, "ci95_low": 0.80881625, "ci95_high": 0.8464854166666665 }, { "position": 2, "n": 60, "alpha_mean": 0.6882666666666667, "alpha_median": 0.7064999999999999, "ci95_low": 0.6566654166666668, "ci95_high": 0.7224 }, { "position": 3, "n": 60, "alpha_mean": 0.5771166666666667, "alpha_median": 0.57, "ci95_low": 0.5377991666666667, "ci95_high": 0.6176891666666667 }, { "position": 4, "n": 60, "alpha_mean": 0.48430000000000006, "alpha_median": 0.4585, "ci95_low": 0.4371829166666667, "ci95_high": 0.5336854166666667 }, { "position": 5, "n": 60, "alpha_mean": 0.4117, "alpha_median": 0.4, "ci95_low": 0.36243166666666665, "ci95_high": 0.4655683333333333 }, { "position": 6, "n": 60, "alpha_mean": 0.35425, "alpha_median": 0.339, "ci95_low": 0.30452666666666667, "ci95_high": 0.40827583333333334 }, { "position": 7, "n": 60, "alpha_mean": 0.30565, "alpha_median": 0.295, "ci95_low": 0.2567629166666667, "ci95_high": 0.3597012499999999 }, { "position": 8, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 9, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 10, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.7943666666666667, "alpha_median": 0.808, "ci95_low": 0.77531625, "ci95_high": 0.8127087500000001 }, { "position": 2, "n": 60, "alpha_mean": 0.6135166666666668, "alpha_median": 0.6125, "ci95_low": 0.58530875, "ci95_high": 0.6420545833333332 }, { "position": 3, "n": 60, "alpha_mean": 0.46181666666666665, "alpha_median": 0.4355, "ci95_low": 0.4318120833333333, "ci95_high": 0.4930520833333334 }, { "position": 4, "n": 60, "alpha_mean": 0.3523333333333333, "alpha_median": 0.3255, "ci95_low": 0.3216791666666667, "ci95_high": 0.3854704166666667 }, { "position": 5, "n": 60, "alpha_mean": 0.26839999999999997, "alpha_median": 0.2355, "ci95_low": 0.23628, "ci95_high": 0.3033833333333333 }, { "position": 6, "n": 60, "alpha_mean": 0.19771666666666665, "alpha_median": 0.1725, "ci95_low": 0.16911499999999996, "ci95_high": 0.22833874999999998 }, { "position": 7, "n": 60, "alpha_mean": 0.15255000000000002, "alpha_median": 0.119, "ci95_low": 0.12411666666666668, "ci95_high": 0.18401791666666664 }, { "position": 8, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 9, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 10, "n": 60, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.7089310344827586, "alpha_median": 0.7275, "ci95_low": 0.665080172413793, "ci95_high": 0.747226724137931 }, { "position": 2, "n": 58, "alpha_mean": 0.4922241379310345, "alpha_median": 0.5095000000000001, "ci95_low": 0.44306379310344834, "ci95_high": 0.5365064655172413 }, { "position": 3, "n": 58, "alpha_mean": 0.35253448275862065, "alpha_median": 0.3375, "ci95_low": 0.30437112068965516, "ci95_high": 0.39734525862068965 }, { "position": 4, "n": 58, "alpha_mean": 0.2588793103448276, "alpha_median": 0.2205, "ci95_low": 0.21575560344827588, "ci95_high": 0.30172629310344823 }, { "position": 5, "n": 58, "alpha_mean": 0.19879310344827586, "alpha_median": 0.1545, "ci95_low": 0.15875646551724143, "ci95_high": 0.24043620689655168 }, { "position": 6, "n": 58, "alpha_mean": 0.15515517241379312, "alpha_median": 0.0895, "ci95_low": 0.11734439655172413, "ci95_high": 0.19293146551724136 }, { "position": 7, "n": 58, "alpha_mean": 0.12293103448275863, "alpha_median": 0.065, "ci95_low": 0.0872926724137931, "ci95_high": 0.15955387931034481 }, { "position": 8, "n": 58, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 9, "n": 58, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 }, { "position": 10, "n": 58, "alpha_mean": 0.0, "alpha_median": 0.0, "ci95_low": 0.0, "ci95_high": 0.0 } ] }, "breakeven_pooled": { "n": 178, "beta": 152.84007000018624, "intercept": 23.524191882241304, "r2": 0.9993371112986554, "alpha_be": 0.18298167302089194, "ci95": 0.0035118194171447725 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 151.1887495856207, "intercept": 24.38020425585159, "r2": 0.9989825515615268, "alpha_be": 0.19246226427055, "ci95": 0.007384782940490939 }, "code": { "n": 60, "beta": 153.79218465642955, "intercept": 23.207569625410486, "r2": 0.9989280150504728, "alpha_be": 0.18142326566802597, "ci95": 0.007804747017042869 }, "chat": { "n": 58, "beta": 154.14593034109316, "intercept": 23.029808511584648, "r2": 0.999626560816225, "alpha_be": 0.1738666039391707, "ci95": 0.0047728591746651755 } } }, "ksweep-qwen-q4-dspark-p0-k5": { "family": "ksweep", "target": "qwen-q4", "drafter": "dspark-p0", "k": 5, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.0000000000050004e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-qwen-q4-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 178, "baseline_n_matched": 178, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 58 }, "baseline_excluded": 0, "baseline_all": 51.49112359550562, "baseline_by_domain": { "math": 53.47833333333333, "code": 51.10905000000001, "chat": 49.83063793103449 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8351000000000001, "alpha_median": 0.8434999999999999, "ci95_low": 0.8158500000000001, "ci95_high": 0.8545670833333333 }, { "position": 2, "n": 60, "alpha_mean": 0.6908, "alpha_median": 0.699, "ci95_low": 0.6574162500000001, "ci95_high": 0.7251333333333333 }, { "position": 3, "n": 60, "alpha_mean": 0.5831, "alpha_median": 0.5714999999999999, "ci95_low": 0.5408058333333334, "ci95_high": 0.6268512499999997 }, { "position": 4, "n": 60, "alpha_mean": 0.48841666666666667, "alpha_median": 0.488, "ci95_low": 0.44127791666666666, "ci95_high": 0.5375004166666666 }, { "position": 5, "n": 60, "alpha_mean": 0.40971666666666656, "alpha_median": 0.397, "ci95_low": 0.3608645833333333, "ci95_high": 0.4619025 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.7903666666666667, "alpha_median": 0.792, "ci95_low": 0.77183, "ci95_high": 0.80858375 }, { "position": 2, "n": 60, "alpha_mean": 0.6033166666666666, "alpha_median": 0.5945, "ci95_low": 0.5752324999999999, "ci95_high": 0.6310170833333334 }, { "position": 3, "n": 60, "alpha_mean": 0.4645, "alpha_median": 0.454, "ci95_low": 0.4331312499999999, "ci95_high": 0.4965354166666667 }, { "position": 4, "n": 60, "alpha_mean": 0.35390000000000005, "alpha_median": 0.329, "ci95_low": 0.32044125, "ci95_high": 0.38973333333333327 }, { "position": 5, "n": 60, "alpha_mean": 0.26375, "alpha_median": 0.23, "ci95_low": 0.23050958333333335, "ci95_high": 0.29978541666666664 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.7085689655172414, "alpha_median": 0.7555000000000001, "ci95_low": 0.664840948275862, "ci95_high": 0.7459702586206896 }, { "position": 2, "n": 58, "alpha_mean": 0.4858793103448276, "alpha_median": 0.513, "ci95_low": 0.43451724137931036, "ci95_high": 0.5313284482758621 }, { "position": 3, "n": 58, "alpha_mean": 0.352948275862069, "alpha_median": 0.3545, "ci95_low": 0.30419784482758616, "ci95_high": 0.39955258620689643 }, { "position": 4, "n": 58, "alpha_mean": 0.26739655172413795, "alpha_median": 0.2155, "ci95_low": 0.2195293103448276, "ci95_high": 0.3135099137931034 }, { "position": 5, "n": 58, "alpha_mean": 0.2047586206896552, "alpha_median": 0.1445, "ci95_low": 0.16067543103448276, "ci95_high": 0.24851810344827574 } ] }, "breakeven_pooled": { "n": 178, "beta": 126.64769809879108, "intercept": 27.7752026738876, "r2": 0.9921790944292123, "alpha_be": 0.18725899702589544, "ci95": 0.011883994782301592 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 125.79183572031826, "intercept": 28.025430131666013, "r2": 0.9884348158282333, "alpha_be": 0.20234145607238396, "ci95": 0.02443949316259995 }, "code": { "n": 60, "beta": 128.71278207683835, "intercept": 26.728004856515412, "r2": 0.9817423471835494, "alpha_be": 0.1894220974023366, "ci95": 0.03179177306281013 }, "chat": { "n": 58, "beta": 128.4224399451065, "intercept": 27.34948644798619, "r2": 0.9952543165271275, "alpha_be": 0.17505625568753985, "ci95": 0.01680662933104417 } } }, "ksweep-qwen-q4-dspark-p0-k7": { "family": "ksweep", "target": "qwen-q4", "drafter": "dspark-p0", "k": 7, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.0000000000105516e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-qwen-q4-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 178, "baseline_n_matched": 178, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 58 }, "baseline_excluded": 0, "baseline_all": 51.49112359550562, "baseline_by_domain": { "math": 53.47833333333333, "code": 51.10905000000001, "chat": 49.83063793103449 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8275166666666666, "alpha_median": 0.8234999999999999, "ci95_low": 0.80881625, "ci95_high": 0.8464854166666665 }, { "position": 2, "n": 60, "alpha_mean": 0.6882666666666667, "alpha_median": 0.7064999999999999, "ci95_low": 0.6566654166666668, "ci95_high": 0.7224 }, { "position": 3, "n": 60, "alpha_mean": 0.5771166666666667, "alpha_median": 0.57, "ci95_low": 0.5377991666666667, "ci95_high": 0.6176891666666667 }, { "position": 4, "n": 60, "alpha_mean": 0.48430000000000006, "alpha_median": 0.4585, "ci95_low": 0.4371829166666667, "ci95_high": 0.5336854166666667 }, { "position": 5, "n": 60, "alpha_mean": 0.4117, "alpha_median": 0.4, "ci95_low": 0.36243166666666665, "ci95_high": 0.4655683333333333 }, { "position": 6, "n": 60, "alpha_mean": 0.35425, "alpha_median": 0.339, "ci95_low": 0.30452666666666667, "ci95_high": 0.40827583333333334 }, { "position": 7, "n": 60, "alpha_mean": 0.30565, "alpha_median": 0.295, "ci95_low": 0.2567629166666667, "ci95_high": 0.3597012499999999 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.7943666666666667, "alpha_median": 0.808, "ci95_low": 0.77531625, "ci95_high": 0.8127087500000001 }, { "position": 2, "n": 60, "alpha_mean": 0.6135166666666668, "alpha_median": 0.6125, "ci95_low": 0.58530875, "ci95_high": 0.6420545833333332 }, { "position": 3, "n": 60, "alpha_mean": 0.46181666666666665, "alpha_median": 0.4355, "ci95_low": 0.4318120833333333, "ci95_high": 0.4930520833333334 }, { "position": 4, "n": 60, "alpha_mean": 0.3523333333333333, "alpha_median": 0.3255, "ci95_low": 0.3216791666666667, "ci95_high": 0.3854704166666667 }, { "position": 5, "n": 60, "alpha_mean": 0.26839999999999997, "alpha_median": 0.2355, "ci95_low": 0.23628, "ci95_high": 0.3033833333333333 }, { "position": 6, "n": 60, "alpha_mean": 0.19771666666666665, "alpha_median": 0.1725, "ci95_low": 0.16911499999999996, "ci95_high": 0.22833874999999998 }, { "position": 7, "n": 60, "alpha_mean": 0.15255000000000002, "alpha_median": 0.119, "ci95_low": 0.12411666666666668, "ci95_high": 0.18401791666666664 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.7089310344827586, "alpha_median": 0.7275, "ci95_low": 0.665080172413793, "ci95_high": 0.747226724137931 }, { "position": 2, "n": 58, "alpha_mean": 0.4922241379310345, "alpha_median": 0.5095000000000001, "ci95_low": 0.44306379310344834, "ci95_high": 0.5365064655172413 }, { "position": 3, "n": 58, "alpha_mean": 0.35253448275862065, "alpha_median": 0.3375, "ci95_low": 0.30437112068965516, "ci95_high": 0.39734525862068965 }, { "position": 4, "n": 58, "alpha_mean": 0.2588793103448276, "alpha_median": 0.2205, "ci95_low": 0.21575560344827588, "ci95_high": 0.30172629310344823 }, { "position": 5, "n": 58, "alpha_mean": 0.19879310344827586, "alpha_median": 0.1545, "ci95_low": 0.15875646551724143, "ci95_high": 0.24043620689655168 }, { "position": 6, "n": 58, "alpha_mean": 0.15515517241379312, "alpha_median": 0.0895, "ci95_low": 0.11734439655172413, "ci95_high": 0.19293146551724136 }, { "position": 7, "n": 58, "alpha_mean": 0.12293103448275863, "alpha_median": 0.065, "ci95_low": 0.0872926724137931, "ci95_high": 0.15955387931034481 } ] }, "breakeven_pooled": { "n": 178, "beta": 150.91904491850252, "intercept": 23.989659108729462, "r2": 0.9956257051108182, "alpha_be": 0.18222660037125973, "ci95": 0.009041124980654966 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 149.53095147695947, "intercept": 24.162558865235876, "r2": 0.9900120544340701, "alpha_be": 0.1960515477132812, "ci95": 0.02319392037908231 }, "code": { "n": 60, "beta": 154.4545454230679, "intercept": 22.800519103828318, "r2": 0.997135142522458, "alpha_be": 0.18328065916500888, "ci95": 0.012759990054545468 }, "chat": { "n": 58, "beta": 154.71708196604047, "intercept": 23.039200191852306, "r2": 0.9995377240487928, "alpha_be": 0.17316405789674053, "ci95": 0.005311831991766195 } } }, "ksweep-qwen-q4-eagle3-k10": { "family": "ksweep", "target": "qwen-q4", "drafter": "eagle3", "k": 10, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.000000000002225e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-qwen-q4-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 178, "baseline_n_matched": 178, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 58 }, "baseline_excluded": 0, "baseline_all": 51.49112359550562, "baseline_by_domain": { "math": 53.47833333333333, "code": 51.10905000000001, "chat": 49.83063793103449 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.6393000000000001, "alpha_median": 0.636, "ci95_low": 0.6221158333333334, "ci95_high": 0.65715 }, { "position": 2, "n": 60, "alpha_mean": 0.41214999999999996, "alpha_median": 0.413, "ci95_low": 0.3911658333333333, "ci95_high": 0.43366874999999994 }, { "position": 3, "n": 60, "alpha_mean": 0.2157, "alpha_median": 0.2145, "ci95_low": 0.20113291666666666, "ci95_high": 0.23085124999999998 }, { "position": 4, "n": 60, "alpha_mean": 0.12154999999999999, "alpha_median": 0.1165, "ci95_low": 0.10949666666666666, "ci95_high": 0.13506666666666664 }, { "position": 5, "n": 60, "alpha_mean": 0.07311666666666668, "alpha_median": 0.07050000000000001, "ci95_low": 0.06314958333333333, "ci95_high": 0.08443333333333333 }, { "position": 6, "n": 60, "alpha_mean": 0.04493333333333333, "alpha_median": 0.038, "ci95_low": 0.037166250000000005, "ci95_high": 0.05385083333333333 }, { "position": 7, "n": 60, "alpha_mean": 0.027366666666666664, "alpha_median": 0.0225, "ci95_low": 0.021782916666666666, "ci95_high": 0.0334675 }, { "position": 8, "n": 60, "alpha_mean": 0.01655, "alpha_median": 0.011, "ci95_low": 0.012550000000000002, "ci95_high": 0.020699999999999996 }, { "position": 9, "n": 60, "alpha_mean": 0.008683333333333333, "alpha_median": 0.0085, "ci95_low": 0.005449166666666666, "ci95_high": 0.012416666666666666 }, { "position": 10, "n": 60, "alpha_mean": 0.005783333333333333, "alpha_median": 0.0, "ci95_low": 0.0031833333333333336, "ci95_high": 0.008583333333333333 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.6601999999999999, "alpha_median": 0.6415, "ci95_low": 0.6419654166666666, "ci95_high": 0.6793508333333333 }, { "position": 2, "n": 60, "alpha_mean": 0.41513333333333335, "alpha_median": 0.399, "ci95_low": 0.3911633333333333, "ci95_high": 0.4415695833333333 }, { "position": 3, "n": 60, "alpha_mean": 0.21558333333333332, "alpha_median": 0.174, "ci95_low": 0.1876145833333333, "ci95_high": 0.24768500000000002 }, { "position": 4, "n": 60, "alpha_mean": 0.12786666666666668, "alpha_median": 0.0925, "ci95_low": 0.10286625, "ci95_high": 0.15703499999999995 }, { "position": 5, "n": 60, "alpha_mean": 0.08738333333333334, "alpha_median": 0.046, "ci95_low": 0.06396333333333334, "ci95_high": 0.11490749999999998 }, { "position": 6, "n": 60, "alpha_mean": 0.057249999999999995, "alpha_median": 0.02, "ci95_low": 0.03816375, "ci95_high": 0.07923541666666666 }, { "position": 7, "n": 60, "alpha_mean": 0.03891666666666666, "alpha_median": 0.01, "ci95_low": 0.022932499999999998, "ci95_high": 0.05705333333333332 }, { "position": 8, "n": 60, "alpha_mean": 0.030533333333333336, "alpha_median": 0.009, "ci95_low": 0.01624916666666667, "ci95_high": 0.047400833333333336 }, { "position": 9, "n": 60, "alpha_mean": 0.021716666666666665, "alpha_median": 0.0, "ci95_low": 0.010549166666666668, "ci95_high": 0.03530249999999999 }, { "position": 10, "n": 60, "alpha_mean": 0.01635, "alpha_median": 0.0, "ci95_low": 0.008066666666666666, "ci95_high": 0.025851666666666658 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.5735517241379311, "alpha_median": 0.6014999999999999, "ci95_low": 0.5235991379310345, "ci95_high": 0.6212129310344826 }, { "position": 2, "n": 58, "alpha_mean": 0.34679310344827585, "alpha_median": 0.3715, "ci95_low": 0.2997573275862069, "ci95_high": 0.39171379310344817 }, { "position": 3, "n": 58, "alpha_mean": 0.18879310344827585, "alpha_median": 0.163, "ci95_low": 0.1524030172413793, "ci95_high": 0.22463879310344825 }, { "position": 4, "n": 58, "alpha_mean": 0.11891379310344827, "alpha_median": 0.079, "ci95_low": 0.08842974137931035, "ci95_high": 0.15122887931034484 }, { "position": 5, "n": 58, "alpha_mean": 0.08306896551724137, "alpha_median": 0.0405, "ci95_low": 0.057982327586206896, "ci95_high": 0.10953577586206896 }, { "position": 6, "n": 58, "alpha_mean": 0.0649655172413793, "alpha_median": 0.0285, "ci95_low": 0.04182456896551724, "ci95_high": 0.08976249999999997 }, { "position": 7, "n": 58, "alpha_mean": 0.048999999999999995, "alpha_median": 0.0095, "ci95_low": 0.029137068965517245, "ci95_high": 0.0712004310344827 }, { "position": 8, "n": 58, "alpha_mean": 0.04074137931034483, "alpha_median": 0.008, "ci95_low": 0.022344827586206897, "ci95_high": 0.06181077586206897 }, { "position": 9, "n": 58, "alpha_mean": 0.034413793103448276, "alpha_median": 0.0, "ci95_low": 0.016790948275862073, "ci95_high": 0.0545176724137931 }, { "position": 10, "n": 58, "alpha_mean": 0.029206896551724138, "alpha_median": 0.0, "ci95_low": 0.013636637931034482, "ci95_high": 0.047277586206896545 } ] }, "breakeven_pooled": { "n": 178, "beta": 179.9698152641095, "intercept": 18.174912990606554, "r2": 0.9990778895392142, "alpha_be": 0.18512110242490845, "ci95": 0.004354675513391534 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 180.14654989804825, "intercept": 18.16128445739837, "r2": 0.9981746959379467, "alpha_be": 0.196046212907892, "ci95": 0.01066214417874208 }, "code": { "n": 60, "beta": 183.48844434805207, "intercept": 17.5300420163385, "r2": 0.9993391781821137, "alpha_be": 0.18300339349962988, "ci95": 0.006413288845512137 }, "chat": { "n": 58, "beta": 178.32304557547923, "intercept": 18.466067871088224, "r2": 0.9993181610317233, "alpha_be": 0.17588624038316183, "ci95": 0.006655553072379677 } } }, "ksweep-qwen-q4-eagle3-k5": { "family": "ksweep", "target": "qwen-q4", "drafter": "eagle3", "k": 5, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.0000000000050004e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-qwen-q4-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 178, "baseline_n_matched": 178, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 58 }, "baseline_excluded": 0, "baseline_all": 51.49112359550562, "baseline_by_domain": { "math": 53.47833333333333, "code": 51.10905000000001, "chat": 49.83063793103449 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.6475333333333333, "alpha_median": 0.6315, "ci95_low": 0.6281633333333333, "ci95_high": 0.6681520833333335 }, { "position": 2, "n": 60, "alpha_mean": 0.4170999999999999, "alpha_median": 0.405, "ci95_low": 0.39436625, "ci95_high": 0.44110166666666667 }, { "position": 3, "n": 60, "alpha_mean": 0.2237666666666667, "alpha_median": 0.218, "ci95_low": 0.20564458333333333, "ci95_high": 0.24383749999999996 }, { "position": 4, "n": 60, "alpha_mean": 0.12868333333333334, "alpha_median": 0.1205, "ci95_low": 0.11228208333333332, "ci95_high": 0.1467341666666667 }, { "position": 5, "n": 60, "alpha_mean": 0.0775, "alpha_median": 0.0695, "ci95_low": 0.06496583333333333, "ci95_high": 0.09210125 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.6593166666666668, "alpha_median": 0.648, "ci95_low": 0.6399995833333333, "ci95_high": 0.6801691666666667 }, { "position": 2, "n": 60, "alpha_mean": 0.4212333333333333, "alpha_median": 0.4105, "ci95_low": 0.39419958333333327, "ci95_high": 0.4513374999999999 }, { "position": 3, "n": 60, "alpha_mean": 0.21413333333333331, "alpha_median": 0.176, "ci95_low": 0.18536458333333336, "ci95_high": 0.24663083333333327 }, { "position": 4, "n": 60, "alpha_mean": 0.1280666666666667, "alpha_median": 0.088, "ci95_low": 0.10104916666666666, "ci95_high": 0.15938333333333332 }, { "position": 5, "n": 60, "alpha_mean": 0.0828, "alpha_median": 0.048, "ci95_low": 0.058109999999999995, "ci95_high": 0.11205333333333332 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.5732241379310344, "alpha_median": 0.597, "ci95_low": 0.5165314655172414, "ci95_high": 0.6251758620689656 }, { "position": 2, "n": 58, "alpha_mean": 0.3543620689655172, "alpha_median": 0.3435, "ci95_low": 0.3023431034482758, "ci95_high": 0.4045181034482759 }, { "position": 3, "n": 58, "alpha_mean": 0.1993103448275862, "alpha_median": 0.1605, "ci95_low": 0.15927284482758625, "ci95_high": 0.24270301724137924 }, { "position": 4, "n": 58, "alpha_mean": 0.1258103448275862, "alpha_median": 0.092, "ci95_low": 0.09251422413793102, "ci95_high": 0.1614176724137931 }, { "position": 5, "n": 58, "alpha_mean": 0.08212068965517243, "alpha_median": 0.0435, "ci95_low": 0.05446120689655173, "ci95_high": 0.11160344827586206 } ] }, "breakeven_pooled": { "n": 178, "beta": 124.00546088925626, "intercept": 25.168637372730156, "r2": 0.9995948082589274, "alpha_be": 0.21226876650442758, "ci95": 0.0027914808962265377 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 123.4033847196296, "intercept": 25.34221243118326, "r2": 0.9993063214061196, "alpha_be": 0.2280012089301672, "ci95": 0.006316104226265306 }, "code": { "n": 60, "beta": 124.51876657128227, "intercept": 24.95940146575053, "r2": 0.9993914722547149, "alpha_be": 0.2100056823103833, "ci95": 0.005947692775158475 }, "chat": { "n": 58, "beta": 124.00003816146088, "intercept": 25.233496633487402, "r2": 0.999762288937174, "alpha_be": 0.1983639816749013, "ci95": 0.003825103461249379 } } }, "ksweep-qwen-q4-eagle3-k7": { "family": "ksweep", "target": "qwen-q4", "drafter": "eagle3", "k": 7, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.000000000002225e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-qwen-q4-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 178, "baseline_n_matched": 178, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 58 }, "baseline_excluded": 0, "baseline_all": 51.49112359550562, "baseline_by_domain": { "math": 53.47833333333333, "code": 51.10905000000001, "chat": 49.83063793103449 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.6443333333333333, "alpha_median": 0.626, "ci95_low": 0.6251470833333334, "ci95_high": 0.6649041666666665 }, { "position": 2, "n": 60, "alpha_mean": 0.4131333333333334, "alpha_median": 0.403, "ci95_low": 0.3907324999999999, "ci95_high": 0.4365841666666666 }, { "position": 3, "n": 60, "alpha_mean": 0.21961666666666668, "alpha_median": 0.218, "ci95_low": 0.20243291666666668, "ci95_high": 0.2386175 }, { "position": 4, "n": 60, "alpha_mean": 0.12621666666666667, "alpha_median": 0.118, "ci95_low": 0.1104825, "ci95_high": 0.14348999999999995 }, { "position": 5, "n": 60, "alpha_mean": 0.07546666666666667, "alpha_median": 0.0675, "ci95_low": 0.06334833333333333, "ci95_high": 0.08971749999999999 }, { "position": 6, "n": 60, "alpha_mean": 0.045649999999999996, "alpha_median": 0.036, "ci95_low": 0.03564916666666666, "ci95_high": 0.05713375 }, { "position": 7, "n": 60, "alpha_mean": 0.028, "alpha_median": 0.02, "ci95_low": 0.02039833333333333, "ci95_high": 0.037367083333333335 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.6557499999999999, "alpha_median": 0.6445000000000001, "ci95_low": 0.6375145833333333, "ci95_high": 0.6747520833333334 }, { "position": 2, "n": 60, "alpha_mean": 0.41759999999999997, "alpha_median": 0.4125, "ci95_low": 0.39189124999999997, "ci95_high": 0.4459733333333334 }, { "position": 3, "n": 60, "alpha_mean": 0.21246666666666666, "alpha_median": 0.1815, "ci95_low": 0.18428166666666662, "ci95_high": 0.2451670833333333 }, { "position": 4, "n": 60, "alpha_mean": 0.12625000000000003, "alpha_median": 0.0875, "ci95_low": 0.09834916666666665, "ci95_high": 0.15945666666666664 }, { "position": 5, "n": 60, "alpha_mean": 0.08131666666666669, "alpha_median": 0.049, "ci95_low": 0.05654833333333334, "ci95_high": 0.11175416666666664 }, { "position": 6, "n": 60, "alpha_mean": 0.052649999999999995, "alpha_median": 0.019, "ci95_low": 0.03203333333333333, "ci95_high": 0.07733583333333333 }, { "position": 7, "n": 60, "alpha_mean": 0.0377, "alpha_median": 0.01, "ci95_low": 0.020062916666666666, "ci95_high": 0.05930083333333334 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.5708275862068966, "alpha_median": 0.6045, "ci95_low": 0.5146288793103448, "ci95_high": 0.6221379310344828 }, { "position": 2, "n": 58, "alpha_mean": 0.3496551724137931, "alpha_median": 0.3395, "ci95_low": 0.29932672413793104, "ci95_high": 0.39888405172413793 }, { "position": 3, "n": 58, "alpha_mean": 0.19015517241379312, "alpha_median": 0.158, "ci95_low": 0.15470474137931034, "ci95_high": 0.22732758620689655 }, { "position": 4, "n": 58, "alpha_mean": 0.12032758620689656, "alpha_median": 0.084, "ci95_low": 0.09060258620689655, "ci95_high": 0.15307241379310346 }, { "position": 5, "n": 58, "alpha_mean": 0.07534482758620689, "alpha_median": 0.042499999999999996, "ci95_low": 0.05094008620689655, "ci95_high": 0.10100258620689655 }, { "position": 6, "n": 58, "alpha_mean": 0.057896551724137935, "alpha_median": 0.0205, "ci95_low": 0.035170689655172414, "ci95_high": 0.08256939655172414 }, { "position": 7, "n": 58, "alpha_mean": 0.04268965517241379, "alpha_median": 0.009, "ci95_low": 0.022724137931034484, "ci95_high": 0.06393362068965516 } ] }, "breakeven_pooled": { "n": 178, "beta": 136.38572009224174, "intercept": 19.785953970540383, "r2": 0.9997089204363768, "alpha_be": 0.23246692984809614, "ci95": 0.002393383776401263 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 136.00101179865075, "intercept": 19.85562650831181, "r2": 0.9992706631099841, "alpha_be": 0.24722394620710528, "ci95": 0.00656820292046481 }, "code": { "n": 60, "beta": 137.56683633503212, "intercept": 19.51519942035037, "r2": 0.9997118358385192, "alpha_be": 0.22966182418199654, "ci95": 0.004141243316511085 }, "chat": { "n": 58, "beta": 135.8092740235272, "intercept": 19.921302976708613, "r2": 0.9998608796338377, "alpha_be": 0.2202304310171371, "ci95": 0.0029523502222129163 } } }, "ksweep-qwen-q4-vanilla17b-k10": { "family": "ksweep", "target": "qwen-q4", "drafter": "vanilla17b", "k": 10, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.000000000002225e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-qwen-q4-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 178, "baseline_n_matched": 178, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 58 }, "baseline_excluded": 0, "baseline_all": 51.49112359550562, "baseline_by_domain": { "math": 53.47833333333333, "code": 51.10905000000001, "chat": 49.83063793103449 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8605833333333334, "alpha_median": 0.864, "ci95_low": 0.8454433333333332, "ci95_high": 0.8753520833333333 }, { "position": 2, "n": 60, "alpha_mean": 0.7415833333333333, "alpha_median": 0.7255, "ci95_low": 0.7151995833333333, "ci95_high": 0.7684404166666667 }, { "position": 3, "n": 60, "alpha_mean": 0.6566500000000001, "alpha_median": 0.6505000000000001, "ci95_low": 0.6250304166666666, "ci95_high": 0.6895837499999998 }, { "position": 4, "n": 60, "alpha_mean": 0.5859333333333334, "alpha_median": 0.5725, "ci95_low": 0.5521279166666666, "ci95_high": 0.6207170833333334 }, { "position": 5, "n": 60, "alpha_mean": 0.5292833333333333, "alpha_median": 0.512, "ci95_low": 0.49318166666666663, "ci95_high": 0.5666537499999998 }, { "position": 6, "n": 60, "alpha_mean": 0.47714999999999996, "alpha_median": 0.469, "ci95_low": 0.44058083333333337, "ci95_high": 0.51552375 }, { "position": 7, "n": 60, "alpha_mean": 0.4276666666666667, "alpha_median": 0.42, "ci95_low": 0.39024958333333337, "ci95_high": 0.4667545833333333 }, { "position": 8, "n": 60, "alpha_mean": 0.3843833333333334, "alpha_median": 0.366, "ci95_low": 0.34676666666666667, "ci95_high": 0.4235545833333333 }, { "position": 9, "n": 60, "alpha_mean": 0.3481, "alpha_median": 0.341, "ci95_low": 0.3101312500000001, "ci95_high": 0.3868700000000001 }, { "position": 10, "n": 60, "alpha_mean": 0.32078333333333325, "alpha_median": 0.314, "ci95_low": 0.28234583333333335, "ci95_high": 0.3583666666666666 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.82835, "alpha_median": 0.835, "ci95_low": 0.8096479166666668, "ci95_high": 0.8476525 }, { "position": 2, "n": 60, "alpha_mean": 0.6865166666666667, "alpha_median": 0.6875, "ci95_low": 0.6631316666666667, "ci95_high": 0.7102224999999999 }, { "position": 3, "n": 60, "alpha_mean": 0.5856499999999999, "alpha_median": 0.578, "ci95_low": 0.5595291666666666, "ci95_high": 0.6141675 }, { "position": 4, "n": 60, "alpha_mean": 0.5046, "alpha_median": 0.484, "ci95_low": 0.47836416666666665, "ci95_high": 0.5368516666666666 }, { "position": 5, "n": 60, "alpha_mean": 0.43585, "alpha_median": 0.415, "ci95_low": 0.4082495833333332, "ci95_high": 0.4698133333333333 }, { "position": 6, "n": 60, "alpha_mean": 0.3830666666666667, "alpha_median": 0.3645, "ci95_low": 0.3531079166666666, "ci95_high": 0.41880291666666664 }, { "position": 7, "n": 60, "alpha_mean": 0.34103333333333335, "alpha_median": 0.3225, "ci95_low": 0.3107825, "ci95_high": 0.3766229166666667 }, { "position": 8, "n": 60, "alpha_mean": 0.30171666666666663, "alpha_median": 0.278, "ci95_low": 0.2721820833333333, "ci95_high": 0.3373029166666667 }, { "position": 9, "n": 60, "alpha_mean": 0.2707166666666667, "alpha_median": 0.242, "ci95_low": 0.24014916666666664, "ci95_high": 0.3073733333333334 }, { "position": 10, "n": 60, "alpha_mean": 0.2393666666666667, "alpha_median": 0.2145, "ci95_low": 0.21163, "ci95_high": 0.27196833333333337 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.7706724137931034, "alpha_median": 0.7444999999999999, "ci95_low": 0.7471025862068965, "ci95_high": 0.7944883620689656 }, { "position": 2, "n": 58, "alpha_mean": 0.5908793103448275, "alpha_median": 0.588, "ci95_low": 0.5608418103448276, "ci95_high": 0.6219202586206896 }, { "position": 3, "n": 58, "alpha_mean": 0.47472413793103446, "alpha_median": 0.478, "ci95_low": 0.43965344827586206, "ci95_high": 0.5126948275862069 }, { "position": 4, "n": 58, "alpha_mean": 0.38355172413793104, "alpha_median": 0.362, "ci95_low": 0.3471965517241379, "ci95_high": 0.4236383620689655 }, { "position": 5, "n": 58, "alpha_mean": 0.3245862068965517, "alpha_median": 0.2845, "ci95_low": 0.28644482758620693, "ci95_high": 0.3674366379310344 }, { "position": 6, "n": 58, "alpha_mean": 0.27910344827586203, "alpha_median": 0.231, "ci95_low": 0.2389810344827586, "ci95_high": 0.32389655172413795 }, { "position": 7, "n": 58, "alpha_mean": 0.24193103448275866, "alpha_median": 0.191, "ci95_low": 0.20145646551724136, "ci95_high": 0.2864323275862069 }, { "position": 8, "n": 58, "alpha_mean": 0.21972413793103449, "alpha_median": 0.16399999999999998, "ci95_low": 0.17875862068965517, "ci95_high": 0.266089224137931 }, { "position": 9, "n": 58, "alpha_mean": 0.19453448275862065, "alpha_median": 0.138, "ci95_low": 0.15435689655172413, "ci95_high": 0.23910603448275858 }, { "position": 10, "n": 58, "alpha_mean": 0.17518965517241378, "alpha_median": 0.11449999999999999, "ci95_low": 0.1350336206896552, "ci95_high": 0.21991681034482755 } ] }, "breakeven_pooled": { "n": 178, "beta": 116.9362891623002, "intercept": 13.17375042186088, "r2": 0.9963018294654568, "alpha_be": 0.32767734847873137, "ci95": 0.007653783318616962 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 117.82313685422277, "intercept": 13.034395764939275, "r2": 0.9992943504470627, "alpha_be": 0.34325972511183017, "ci95": 0.005573941092610634 }, "code": { "n": 60, "beta": 117.08214935162073, "intercept": 12.639432519936767, "r2": 0.9896060474900664, "alpha_be": 0.32856945053623343, "ci95": 0.02235528858074378 }, "chat": { "n": 58, "beta": 115.43701539092281, "intercept": 13.859326934794522, "r2": 0.9967365947791387, "alpha_be": 0.3116098495307122, "ci95": 0.013270427908675058 } } }, "ksweep-qwen-q4-vanilla17b-k5": { "family": "ksweep", "target": "qwen-q4", "drafter": "vanilla17b", "k": 5, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.0000000000050004e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-qwen-q4-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 178, "baseline_n_matched": 178, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 58 }, "baseline_excluded": 0, "baseline_all": 51.49112359550562, "baseline_by_domain": { "math": 53.47833333333333, "code": 51.10905000000001, "chat": 49.83063793103449 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8614333333333332, "alpha_median": 0.8694999999999999, "ci95_low": 0.8468833333333333, "ci95_high": 0.8758516666666667 }, { "position": 2, "n": 60, "alpha_mean": 0.7467333333333332, "alpha_median": 0.737, "ci95_low": 0.7221154166666666, "ci95_high": 0.7710595833333332 }, { "position": 3, "n": 60, "alpha_mean": 0.6641666666666667, "alpha_median": 0.6555, "ci95_low": 0.6353624999999999, "ci95_high": 0.6928541666666668 }, { "position": 4, "n": 60, "alpha_mean": 0.5916833333333334, "alpha_median": 0.5945, "ci95_low": 0.5596133333333334, "ci95_high": 0.6232516666666668 }, { "position": 5, "n": 60, "alpha_mean": 0.5365, "alpha_median": 0.5255000000000001, "ci95_low": 0.5021658333333333, "ci95_high": 0.5705020833333334 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.8306333333333333, "alpha_median": 0.8225, "ci95_low": 0.8160666666666666, "ci95_high": 0.8457504166666666 }, { "position": 2, "n": 60, "alpha_mean": 0.6994166666666667, "alpha_median": 0.696, "ci95_low": 0.6790833333333334, "ci95_high": 0.7209337499999998 }, { "position": 3, "n": 60, "alpha_mean": 0.5953000000000002, "alpha_median": 0.585, "ci95_low": 0.5718650000000002, "ci95_high": 0.6201179166666667 }, { "position": 4, "n": 60, "alpha_mean": 0.5117333333333334, "alpha_median": 0.492, "ci95_low": 0.4881979166666667, "ci95_high": 0.5381729166666667 }, { "position": 5, "n": 60, "alpha_mean": 0.44403333333333334, "alpha_median": 0.4175, "ci95_low": 0.4157829166666667, "ci95_high": 0.4744208333333333 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.7753448275862069, "alpha_median": 0.769, "ci95_low": 0.7534068965517241, "ci95_high": 0.7984706896551723 }, { "position": 2, "n": 58, "alpha_mean": 0.6073620689655171, "alpha_median": 0.5805, "ci95_low": 0.575498275862069, "ci95_high": 0.6407241379310344 }, { "position": 3, "n": 58, "alpha_mean": 0.488948275862069, "alpha_median": 0.46950000000000003, "ci95_low": 0.4514625, "ci95_high": 0.5298508620689655 }, { "position": 4, "n": 58, "alpha_mean": 0.40144827586206894, "alpha_median": 0.363, "ci95_low": 0.3599090517241379, "ci95_high": 0.44630129310344824 }, { "position": 5, "n": 58, "alpha_mean": 0.33731034482758626, "alpha_median": 0.283, "ci95_low": 0.2937344827586207, "ci95_high": 0.3847327586206897 } ] }, "breakeven_pooled": { "n": 178, "beta": 89.12762701123346, "intercept": 16.186078279775984, "r2": 0.977793087527451, "alpha_be": 0.3961178649048949, "ci95": 0.016906936522748488 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 85.96841262566369, "intercept": 19.23588158773496, "r2": 0.9971865980783139, "alpha_be": 0.39831434243995983, "ci95": 0.009950751827095662 }, "code": { "n": 60, "beta": 85.82487459287938, "intercept": 18.39320437998536, "r2": 0.961332243154551, "alpha_be": 0.3811930489290686, "ci95": 0.03943974692519082 }, "chat": { "n": 58, "beta": 83.92535023584126, "intercept": 17.847296864028984, "r2": 0.971428188831581, "alpha_be": 0.38109273273365096, "ci95": 0.035992084267018956 } } }, "ksweep-qwen-q4-vanilla17b-k7": { "family": "ksweep", "target": "qwen-q4", "drafter": "vanilla17b", "k": 7, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.0000000000105516e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-qwen-q4-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 178, "baseline_n_matched": 178, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 58 }, "baseline_excluded": 0, "baseline_all": 51.49112359550562, "baseline_by_domain": { "math": 53.47833333333333, "code": 51.10905000000001, "chat": 49.83063793103449 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.8567666666666667, "alpha_median": 0.857, "ci95_low": 0.8415325, "ci95_high": 0.8714508333333333 }, { "position": 2, "n": 60, "alpha_mean": 0.7372166666666667, "alpha_median": 0.732, "ci95_low": 0.7132141666666667, "ci95_high": 0.7604666666666666 }, { "position": 3, "n": 60, "alpha_mean": 0.6550666666666667, "alpha_median": 0.656, "ci95_low": 0.6266645833333334, "ci95_high": 0.6839679166666665 }, { "position": 4, "n": 60, "alpha_mean": 0.58005, "alpha_median": 0.59, "ci95_low": 0.54862375, "ci95_high": 0.6120524999999999 }, { "position": 5, "n": 60, "alpha_mean": 0.5245666666666667, "alpha_median": 0.521, "ci95_low": 0.4935104166666666, "ci95_high": 0.5572333333333331 }, { "position": 6, "n": 60, "alpha_mean": 0.47533333333333333, "alpha_median": 0.4745, "ci95_low": 0.4426595833333334, "ci95_high": 0.5094366666666666 }, { "position": 7, "n": 60, "alpha_mean": 0.42833333333333334, "alpha_median": 0.417, "ci95_low": 0.39536541666666664, "ci95_high": 0.4637670833333333 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.8221666666666666, "alpha_median": 0.826, "ci95_low": 0.8074325, "ci95_high": 0.8374529166666667 }, { "position": 2, "n": 60, "alpha_mean": 0.6857833333333333, "alpha_median": 0.673, "ci95_low": 0.6650241666666667, "ci95_high": 0.7074675 }, { "position": 3, "n": 60, "alpha_mean": 0.5782500000000002, "alpha_median": 0.564, "ci95_low": 0.5552654166666666, "ci95_high": 0.6034895833333332 }, { "position": 4, "n": 60, "alpha_mean": 0.4988000000000001, "alpha_median": 0.4785, "ci95_low": 0.47428333333333333, "ci95_high": 0.5256604166666666 }, { "position": 5, "n": 60, "alpha_mean": 0.4322833333333333, "alpha_median": 0.415, "ci95_low": 0.4041833333333334, "ci95_high": 0.46378500000000006 }, { "position": 6, "n": 60, "alpha_mean": 0.3756166666666666, "alpha_median": 0.358, "ci95_low": 0.3435120833333334, "ci95_high": 0.40943458333333327 }, { "position": 7, "n": 60, "alpha_mean": 0.3322, "alpha_median": 0.305, "ci95_low": 0.3008808333333333, "ci95_high": 0.3676062500000001 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.7714827586206897, "alpha_median": 0.7615000000000001, "ci95_low": 0.750240948275862, "ci95_high": 0.7938982758620688 }, { "position": 2, "n": 58, "alpha_mean": 0.5965, "alpha_median": 0.5805, "ci95_low": 0.5664987068965518, "ci95_high": 0.6276793103448276 }, { "position": 3, "n": 58, "alpha_mean": 0.4778965517241379, "alpha_median": 0.465, "ci95_low": 0.4421551724137931, "ci95_high": 0.5167913793103448 }, { "position": 4, "n": 58, "alpha_mean": 0.39296551724137935, "alpha_median": 0.3585, "ci95_low": 0.3511181034482759, "ci95_high": 0.43686250000000004 }, { "position": 5, "n": 58, "alpha_mean": 0.3258103448275862, "alpha_median": 0.268, "ci95_low": 0.28316508620689657, "ci95_high": 0.3719310344827586 }, { "position": 6, "n": 58, "alpha_mean": 0.28493103448275864, "alpha_median": 0.214, "ci95_low": 0.24000862068965514, "ci95_high": 0.3335034482758621 }, { "position": 7, "n": 58, "alpha_mean": 0.2499655172413793, "alpha_median": 0.1795, "ci95_low": 0.2041969827586207, "ci95_high": 0.30017543103448274 } ] }, "breakeven_pooled": { "n": 178, "beta": 93.02021396622774, "intercept": 14.34549676133501, "r2": 0.9965214788429118, "alpha_be": 0.3993285464561159, "ci95": 0.006881675222664475 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 93.31148375702463, "intercept": 14.462993336313076, "r2": 0.9993480699244549, "alpha_be": 0.41811938280408195, "ci95": 0.004892051250056488 }, "code": { "n": 60, "beta": 93.35757091710133, "intercept": 13.791354462793384, "r2": 0.989043007464966, "alpha_be": 0.3997286472925008, "ci95": 0.021289320742869745 }, "chat": { "n": 58, "beta": 91.96354516426943, "intercept": 14.897885904910126, "r2": 0.9975328281585035, "alpha_be": 0.3798543429760771, "ci95": 0.010832394076832803 } } }, "ksweep-qwen-q8-eagle3-k10": { "family": "ksweep", "target": "qwen-q8", "drafter": "eagle3", "k": 10, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.000000000002225e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-qwen-q8-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 178, "baseline_n_matched": 178, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 58 }, "baseline_excluded": 0, "baseline_all": 31.769084269662915, "baseline_by_domain": { "math": 30.842750000000002, "code": 32.19831666666666, "chat": 32.283327586206894 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.6383833333333334, "alpha_median": 0.631, "ci95_low": 0.6204491666666665, "ci95_high": 0.6561166666666665 }, { "position": 2, "n": 60, "alpha_mean": 0.4114333333333333, "alpha_median": 0.406, "ci95_low": 0.39191499999999996, "ci95_high": 0.431985 }, { "position": 3, "n": 60, "alpha_mean": 0.2123, "alpha_median": 0.214, "ci95_low": 0.19686250000000002, "ci95_high": 0.22998333333333332 }, { "position": 4, "n": 60, "alpha_mean": 0.11678333333333334, "alpha_median": 0.114, "ci95_low": 0.10509875, "ci95_high": 0.12958583333333332 }, { "position": 5, "n": 60, "alpha_mean": 0.06493333333333333, "alpha_median": 0.062, "ci95_low": 0.05644958333333333, "ci95_high": 0.07463541666666665 }, { "position": 6, "n": 60, "alpha_mean": 0.03548333333333334, "alpha_median": 0.0315, "ci95_low": 0.029499583333333326, "ci95_high": 0.04180083333333333 }, { "position": 7, "n": 60, "alpha_mean": 0.019933333333333338, "alpha_median": 0.019, "ci95_low": 0.0161, "ci95_high": 0.02403416666666666 }, { "position": 8, "n": 60, "alpha_mean": 0.011000000000000001, "alpha_median": 0.01, "ci95_low": 0.008366666666666666, "ci95_high": 0.013767083333333333 }, { "position": 9, "n": 60, "alpha_mean": 0.006816666666666667, "alpha_median": 0.004, "ci95_low": 0.004932916666666666, "ci95_high": 0.008833333333333334 }, { "position": 10, "n": 60, "alpha_mean": 0.0042833333333333334, "alpha_median": 0.0, "ci95_low": 0.002716666666666667, "ci95_high": 0.005933333333333333 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.64665, "alpha_median": 0.65, "ci95_low": 0.6291, "ci95_high": 0.6649175 }, { "position": 2, "n": 60, "alpha_mean": 0.40869999999999995, "alpha_median": 0.401, "ci95_low": 0.38830000000000003, "ci95_high": 0.4312008333333334 }, { "position": 3, "n": 60, "alpha_mean": 0.19943333333333332, "alpha_median": 0.1765, "ci95_low": 0.17841625, "ci95_high": 0.22465374999999999 }, { "position": 4, "n": 60, "alpha_mean": 0.11043333333333333, "alpha_median": 0.092, "ci95_low": 0.09008041666666666, "ci95_high": 0.1367220833333333 }, { "position": 5, "n": 60, "alpha_mean": 0.06545, "alpha_median": 0.045, "ci95_low": 0.04749624999999999, "ci95_high": 0.08988333333333334 }, { "position": 6, "n": 60, "alpha_mean": 0.04083333333333334, "alpha_median": 0.02, "ci95_low": 0.025680416666666664, "ci95_high": 0.06138375000000001 }, { "position": 7, "n": 60, "alpha_mean": 0.028283333333333334, "alpha_median": 0.01, "ci95_low": 0.016983333333333336, "ci95_high": 0.04260249999999999 }, { "position": 8, "n": 60, "alpha_mean": 0.020483333333333336, "alpha_median": 0.008, "ci95_low": 0.011382916666666668, "ci95_high": 0.03171791666666666 }, { "position": 9, "n": 60, "alpha_mean": 0.012733333333333334, "alpha_median": 0.0, "ci95_low": 0.006415833333333332, "ci95_high": 0.021366666666666662 }, { "position": 10, "n": 60, "alpha_mean": 0.009200000000000002, "alpha_median": 0.0, "ci95_low": 0.0034825000000000004, "ci95_high": 0.017300833333333328 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.577948275862069, "alpha_median": 0.609, "ci95_low": 0.5294887931034482, "ci95_high": 0.623278448275862 }, { "position": 2, "n": 58, "alpha_mean": 0.3406379310344828, "alpha_median": 0.345, "ci95_low": 0.2972030172413792, "ci95_high": 0.38113275862068957 }, { "position": 3, "n": 58, "alpha_mean": 0.17277586206896553, "alpha_median": 0.1625, "ci95_low": 0.1397866379310345, "ci95_high": 0.2088625 }, { "position": 4, "n": 58, "alpha_mean": 0.11274137931034484, "alpha_median": 0.0845, "ci95_low": 0.08296508620689654, "ci95_high": 0.14803965517241377 }, { "position": 5, "n": 58, "alpha_mean": 0.06896551724137931, "alpha_median": 0.0455, "ci95_low": 0.046757758620689656, "ci95_high": 0.09586336206896551 }, { "position": 6, "n": 58, "alpha_mean": 0.05379310344827586, "alpha_median": 0.028, "ci95_low": 0.03434439655172414, "ci95_high": 0.07764525862068962 }, { "position": 7, "n": 58, "alpha_mean": 0.0381206896551724, "alpha_median": 0.0095, "ci95_low": 0.02318965517241379, "ci95_high": 0.056193965517241345 }, { "position": 8, "n": 58, "alpha_mean": 0.030896551724137935, "alpha_median": 0.009, "ci95_low": 0.017016810344827587, "ci95_high": 0.04772844827586205 }, { "position": 9, "n": 58, "alpha_mean": 0.023724137931034485, "alpha_median": 0.0, "ci95_low": 0.012065517241379312, "ci95_high": 0.03824999999999995 }, { "position": 10, "n": 58, "alpha_mean": 0.01812068965517241, "alpha_median": 0.0, "ci95_low": 0.00794741379310345, "ci95_high": 0.031227155172413777 } ] }, "breakeven_pooled": { "n": 178, "beta": 149.50139451333408, "intercept": 15.163527656104614, "r2": 0.999363221682926, "alpha_be": 0.1110729212099573, "ci95": 0.0036662264954498746 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 148.7914088142256, "intercept": 15.265950150965722, "r2": 0.9980829640106651, "alpha_be": 0.10468883904770863, "ci95": 0.011096230647999271 }, "code": { "n": 60, "beta": 151.30654692020363, "intercept": 14.813219539185342, "r2": 0.9992744437981491, "alpha_be": 0.11489983402139177, "ci95": 0.006809977694709568 }, "chat": { "n": 58, "beta": 148.95512228331646, "intercept": 15.320449448641202, "r2": 0.9996799543556998, "alpha_be": 0.11387911927796524, "ci95": 0.004608823700100185 } } }, "ksweep-qwen-q8-eagle3-k5": { "family": "ksweep", "target": "qwen-q8", "drafter": "eagle3", "k": 5, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 4.999999999999449e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-qwen-q8-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 178, "baseline_n_matched": 178, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 58 }, "baseline_excluded": 0, "baseline_all": 31.769084269662915, "baseline_by_domain": { "math": 30.842750000000002, "code": 32.19831666666666, "chat": 32.283327586206894 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.644, "alpha_median": 0.629, "ci95_low": 0.6263645833333332, "ci95_high": 0.6616704166666665 }, { "position": 2, "n": 60, "alpha_mean": 0.41586666666666666, "alpha_median": 0.422, "ci95_low": 0.39656375, "ci95_high": 0.4358345833333334 }, { "position": 3, "n": 60, "alpha_mean": 0.22018333333333331, "alpha_median": 0.2165, "ci95_low": 0.20504958333333334, "ci95_high": 0.2355683333333333 }, { "position": 4, "n": 60, "alpha_mean": 0.12205, "alpha_median": 0.1155, "ci95_low": 0.11074958333333332, "ci95_high": 0.13483625 }, { "position": 5, "n": 60, "alpha_mean": 0.06903333333333334, "alpha_median": 0.063, "ci95_low": 0.06015000000000001, "ci95_high": 0.07895166666666667 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.6542833333333333, "alpha_median": 0.6465000000000001, "ci95_low": 0.6373291666666667, "ci95_high": 0.6723512500000001 }, { "position": 2, "n": 60, "alpha_mean": 0.41663333333333336, "alpha_median": 0.412, "ci95_low": 0.39533, "ci95_high": 0.4391191666666666 }, { "position": 3, "n": 60, "alpha_mean": 0.20393333333333338, "alpha_median": 0.1845, "ci95_low": 0.18191583333333333, "ci95_high": 0.22860499999999997 }, { "position": 4, "n": 60, "alpha_mean": 0.11448333333333333, "alpha_median": 0.09, "ci95_low": 0.09354500000000002, "ci95_high": 0.13943791666666663 }, { "position": 5, "n": 60, "alpha_mean": 0.06634999999999999, "alpha_median": 0.045, "ci95_low": 0.048914166666666654, "ci95_high": 0.08815166666666666 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.5883793103448276, "alpha_median": 0.6265000000000001, "ci95_low": 0.5389129310344828, "ci95_high": 0.6331443965517242 }, { "position": 2, "n": 58, "alpha_mean": 0.3518275862068965, "alpha_median": 0.37, "ci95_low": 0.3077607758620689, "ci95_high": 0.39405560344827584 }, { "position": 3, "n": 58, "alpha_mean": 0.18241379310344827, "alpha_median": 0.174, "ci95_low": 0.1496987068965517, "ci95_high": 0.21997456896551718 }, { "position": 4, "n": 58, "alpha_mean": 0.11779310344827586, "alpha_median": 0.0885, "ci95_low": 0.09025387931034483, "ci95_high": 0.15170948275862067 }, { "position": 5, "n": 58, "alpha_mean": 0.07225862068965518, "alpha_median": 0.0445, "ci95_low": 0.050567672413793105, "ci95_high": 0.0976306034482758 } ] }, "breakeven_pooled": { "n": 178, "beta": 99.7066125419974, "intercept": 20.17060001572036, "r2": 0.9992274909759372, "alpha_be": 0.11632612881173914, "ci95": 0.003972027492441261 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 99.07719826452487, "intercept": 20.42238174387858, "r2": 0.9986829293580858, "alpha_be": 0.10517423220123989, "ci95": 0.009054847988836469 }, "code": { "n": 60, "beta": 101.20372921309948, "intercept": 19.699544375558144, "r2": 0.9987999302009661, "alpha_be": 0.12350110404321661, "ci95": 0.008598338069620702 }, "chat": { "n": 58, "beta": 99.22733195847222, "intercept": 20.263225257552794, "r2": 0.9995787696442888, "alpha_be": 0.12113701025121437, "ci95": 0.0052052490612202905 } } }, "ksweep-qwen-q8-eagle3-k7": { "family": "ksweep", "target": "qwen-q8", "drafter": "eagle3", "k": 7, "n_excluded": 0, "n_matched": 178, "log_lines": 178, "max_alpha_diff": 5.000000000002225e-05, "ctx": 2048, "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_run": "baseline-ctx2048-qwen-q8-solo", "baseline_ctx": 2048, "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", "baseline_n": 178, "baseline_n_matched": 178, "baseline_n_by_domain": { "math": 60, "code": 60, "chat": 58 }, "baseline_excluded": 0, "baseline_all": 31.769084269662915, "baseline_by_domain": { "math": 30.842750000000002, "code": 32.19831666666666, "chat": 32.283327586206894 }, "per_domain_positions": { "math": [ { "position": 1, "n": 60, "alpha_mean": 0.6422333333333333, "alpha_median": 0.6315, "ci95_low": 0.6244462499999999, "ci95_high": 0.6602345833333334 }, { "position": 2, "n": 60, "alpha_mean": 0.4123833333333334, "alpha_median": 0.4095, "ci95_low": 0.3926487499999999, "ci95_high": 0.43237125 }, { "position": 3, "n": 60, "alpha_mean": 0.21725, "alpha_median": 0.214, "ci95_low": 0.20271458333333336, "ci95_high": 0.2327575 }, { "position": 4, "n": 60, "alpha_mean": 0.11929999999999999, "alpha_median": 0.1155, "ci95_low": 0.10796541666666669, "ci95_high": 0.13148624999999997 }, { "position": 5, "n": 60, "alpha_mean": 0.06716666666666665, "alpha_median": 0.063, "ci95_low": 0.05866583333333334, "ci95_high": 0.0768675 }, { "position": 6, "n": 60, "alpha_mean": 0.03963333333333333, "alpha_median": 0.035500000000000004, "ci95_low": 0.033666666666666664, "ci95_high": 0.04583499999999999 }, { "position": 7, "n": 60, "alpha_mean": 0.022533333333333332, "alpha_median": 0.0195, "ci95_low": 0.01825, "ci95_high": 0.027017083333333334 } ], "code": [ { "position": 1, "n": 60, "alpha_mean": 0.6501833333333332, "alpha_median": 0.641, "ci95_low": 0.6335320833333334, "ci95_high": 0.6674170833333334 }, { "position": 2, "n": 60, "alpha_mean": 0.4082166666666666, "alpha_median": 0.404, "ci95_low": 0.3865483333333332, "ci95_high": 0.4309508333333333 }, { "position": 3, "n": 60, "alpha_mean": 0.19706666666666664, "alpha_median": 0.184, "ci95_low": 0.17731624999999995, "ci95_high": 0.22053708333333333 }, { "position": 4, "n": 60, "alpha_mean": 0.10885000000000002, "alpha_median": 0.088, "ci95_low": 0.09053041666666665, "ci95_high": 0.13218708333333332 }, { "position": 5, "n": 60, "alpha_mean": 0.06378333333333333, "alpha_median": 0.043, "ci95_low": 0.0474325, "ci95_high": 0.08616875 }, { "position": 6, "n": 60, "alpha_mean": 0.03786666666666667, "alpha_median": 0.02, "ci95_low": 0.0248975, "ci95_high": 0.05433458333333333 }, { "position": 7, "n": 60, "alpha_mean": 0.027649999999999997, "alpha_median": 0.009, "ci95_low": 0.016564583333333334, "ci95_high": 0.041801250000000005 } ], "chat": [ { "position": 1, "n": 58, "alpha_mean": 0.5866896551724138, "alpha_median": 0.62, "ci95_low": 0.5381525862068965, "ci95_high": 0.6317435344827586 }, { "position": 2, "n": 58, "alpha_mean": 0.34374137931034476, "alpha_median": 0.3615, "ci95_low": 0.3013275862068966, "ci95_high": 0.38341163793103433 }, { "position": 3, "n": 58, "alpha_mean": 0.17375862068965517, "alpha_median": 0.1685, "ci95_low": 0.14348017241379313, "ci95_high": 0.20646594827586204 }, { "position": 4, "n": 58, "alpha_mean": 0.11463793103448276, "alpha_median": 0.08549999999999999, "ci95_low": 0.08758534482758619, "ci95_high": 0.1466258620689655 }, { "position": 5, "n": 58, "alpha_mean": 0.06991379310344828, "alpha_median": 0.041499999999999995, "ci95_low": 0.04782629310344828, "ci95_high": 0.09610560344827586 }, { "position": 6, "n": 58, "alpha_mean": 0.05153448275862069, "alpha_median": 0.019, "ci95_low": 0.031843534482758624, "ci95_high": 0.07505689655172411 }, { "position": 7, "n": 58, "alpha_mean": 0.03632758620689655, "alpha_median": 0.0095, "ci95_low": 0.02136206896551724, "ci95_high": 0.05450474137931032 } ] }, "breakeven_pooled": { "n": 178, "beta": 124.64048221408584, "intercept": 18.020365612671316, "r2": 0.9992895367836464, "alpha_be": 0.11030700790595813, "ci95": 0.0038476324467990285 }, "breakeven_per_domain": { "math": { "n": 60, "beta": 123.38545805979774, "intercept": 18.2734774522408, "r2": 0.9976469763893474, "alpha_be": 0.10186996705614698, "ci95": 0.012219163514663685 }, "code": { "n": 60, "beta": 125.76313992372975, "intercept": 17.724757997176432, "r2": 0.9990909026094454, "alpha_be": 0.11508585646213873, "ci95": 0.007570864014882201 }, "chat": { "n": 58, "beta": 124.54104706651054, "intercept": 18.1188842661618, "r2": 0.9996783226593118, "alpha_be": 0.11373313179614301, "ci95": 0.004592603267583699 } } } } }