| { |
| "meta": { |
| "runs_total": 67, |
| "final_runs": 26, |
| "curves_runs": 20, |
| "ksweep_runs": 21, |
| "baseline_runs": 3, |
| "baseline_records_excluded_total": 3, |
| "records_excluded_total": 271, |
| "exclusions_per_run": { |
| "final-gemma-q4-dflash-f16": { |
| "total": 1473, |
| "kept": 1455, |
| "excluded": 18 |
| }, |
| "final-gemma-q4-dflash-q4": { |
| "total": 1473, |
| "kept": 1455, |
| "excluded": 18 |
| }, |
| "final-gemma-q4-dflash-q8": { |
| "total": 1473, |
| "kept": 1455, |
| "excluded": 18 |
| }, |
| "final-gemma-q4-mtp": { |
| "total": 1473, |
| "kept": 1455, |
| "excluded": 18 |
| }, |
| "final-gemma-q4-solo": { |
| "total": 1473, |
| "kept": 1455, |
| "excluded": 18 |
| }, |
| "final-gemma-q5-dflash-f16": { |
| "total": 1474, |
| "kept": 1449, |
| "excluded": 25 |
| }, |
| "final-gemma-q5-mtp": { |
| "total": 1474, |
| "kept": 1449, |
| "excluded": 25 |
| }, |
| "final-gemma-q5-solo": { |
| "total": 1474, |
| "kept": 1449, |
| "excluded": 25 |
| }, |
| "final-gemma-q8-dflash-f16": { |
| "total": 1472, |
| "kept": 1452, |
| "excluded": 20 |
| }, |
| "final-gemma-q8-mtp": { |
| "total": 1472, |
| "kept": 1452, |
| "excluded": 20 |
| }, |
| "final-gemma-q8-solo": { |
| "total": 1472, |
| "kept": 1452, |
| "excluded": 20 |
| }, |
| "final-qwen-q4-dflash": { |
| "total": 1474, |
| "kept": 1474, |
| "excluded": 0 |
| }, |
| "final-qwen-q4-dspark-p0": { |
| "total": 1474, |
| "kept": 1474, |
| "excluded": 0 |
| }, |
| "final-qwen-q4-dspark-p2": { |
| "total": 1474, |
| "kept": 1474, |
| "excluded": 0 |
| }, |
| "final-qwen-q4-dspark-p4": { |
| "total": 1474, |
| "kept": 1474, |
| "excluded": 0 |
| }, |
| "final-qwen-q4-dspark-p6": { |
| "total": 1474, |
| "kept": 1474, |
| "excluded": 0 |
| }, |
| "final-qwen-q4-eagle3": { |
| "total": 1474, |
| "kept": 1474, |
| "excluded": 0 |
| }, |
| "final-qwen-q4-solo": { |
| "total": 1474, |
| "kept": 1474, |
| "excluded": 0 |
| }, |
| "final-qwen-q4-vanilla17b": { |
| "total": 1474, |
| "kept": 1474, |
| "excluded": 0 |
| }, |
| "final-qwen-q5-dflash": { |
| "total": 1474, |
| "kept": 1474, |
| "excluded": 0 |
| }, |
| "final-qwen-q5-dspark-p0": { |
| "total": 1474, |
| "kept": 1474, |
| "excluded": 0 |
| }, |
| "final-qwen-q5-eagle3": { |
| "total": 1474, |
| "kept": 1474, |
| "excluded": 0 |
| }, |
| "final-qwen-q5-solo": { |
| "total": 1474, |
| "kept": 1474, |
| "excluded": 0 |
| }, |
| "final-qwen-q8-dspark-p0": { |
| "total": 1474, |
| "kept": 1474, |
| "excluded": 0 |
| }, |
| "final-qwen-q8-eagle3": { |
| "total": 1474, |
| "kept": 1474, |
| "excluded": 0 |
| }, |
| "final-qwen-q8-solo": { |
| "total": 1474, |
| "kept": 1474, |
| "excluded": 0 |
| } |
| }, |
| "log_mapping": { |
| "curves-gemma-q4-dflash-f16": { |
| "log_lines": 175, |
| "n_matched": 175, |
| "max_alpha_diff": 4.999999999999449e-05 |
| }, |
| "curves-gemma-q4-dflash-q4": { |
| "log_lines": 175, |
| "n_matched": 175, |
| "max_alpha_diff": 4.999999999999449e-05 |
| }, |
| "curves-gemma-q4-dflash-q8": { |
| "log_lines": 175, |
| "n_matched": 175, |
| "max_alpha_diff": 4.999999999999449e-05 |
| }, |
| "curves-gemma-q4-mtp": { |
| "log_lines": 175, |
| "n_matched": 175, |
| "max_alpha_diff": 4.999999999999449e-05 |
| }, |
| "curves-gemma-q5-dflash-f16": { |
| "log_lines": 174, |
| "n_matched": 174, |
| "max_alpha_diff": 5.0000000000050004e-05 |
| }, |
| "curves-gemma-q5-mtp": { |
| "log_lines": 174, |
| "n_matched": 174, |
| "max_alpha_diff": 4.999999999999449e-05 |
| }, |
| "curves-gemma-q8-dflash-f16": { |
| "log_lines": 174, |
| "n_matched": 174, |
| "max_alpha_diff": 5.0000000000050004e-05 |
| }, |
| "curves-gemma-q8-mtp": { |
| "log_lines": 174, |
| "n_matched": 174, |
| "max_alpha_diff": 4.999999999999449e-05 |
| }, |
| "curves-qwen-q4-dflash": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.0000000000000565e-05 |
| }, |
| "curves-qwen-q4-dspark-p0": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 4.999999999999449e-05 |
| }, |
| "curves-qwen-q4-dspark-p2": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 4.999999999999449e-05 |
| }, |
| "curves-qwen-q4-dspark-p4": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.0000000000105516e-05 |
| }, |
| "curves-qwen-q4-dspark-p6": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.0000000000105516e-05 |
| }, |
| "curves-qwen-q4-eagle3": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.0000000000050004e-05 |
| }, |
| "curves-qwen-q4-vanilla17b": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 4.999999999999449e-05 |
| }, |
| "curves-qwen-q5-dflash": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.0000000000000565e-05 |
| }, |
| "curves-qwen-q5-dspark-p0": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 4.999999999999449e-05 |
| }, |
| "curves-qwen-q5-eagle3": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.000000000000837e-05 |
| }, |
| "curves-qwen-q8-dspark-p0": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.000000000002225e-05 |
| }, |
| "curves-qwen-q8-eagle3": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.0000000000050004e-05 |
| }, |
| "ksweep-gemma-q4-dflash-f16-k10": { |
| "log_lines": 175, |
| "n_matched": 175, |
| "max_alpha_diff": 5.0000000000050004e-05 |
| }, |
| "ksweep-gemma-q4-dflash-f16-k5": { |
| "log_lines": 175, |
| "n_matched": 175, |
| "max_alpha_diff": 4.999999999999449e-05 |
| }, |
| "ksweep-gemma-q4-dflash-f16-k7": { |
| "log_lines": 175, |
| "n_matched": 175, |
| "max_alpha_diff": 5.0000000000050004e-05 |
| }, |
| "ksweep-gemma-q4-mtp-k10": { |
| "log_lines": 175, |
| "n_matched": 175, |
| "max_alpha_diff": 5.0000000000105516e-05 |
| }, |
| "ksweep-gemma-q4-mtp-k5": { |
| "log_lines": 175, |
| "n_matched": 175, |
| "max_alpha_diff": 5.0000000000050004e-05 |
| }, |
| "ksweep-gemma-q4-mtp-k7": { |
| "log_lines": 175, |
| "n_matched": 175, |
| "max_alpha_diff": 5.0000000000050004e-05 |
| }, |
| "ksweep-qwen-q4-dflash-k10": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.000000000000143e-05 |
| }, |
| "ksweep-qwen-q4-dflash-k5": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 4.99999999999997e-05 |
| }, |
| "ksweep-qwen-q4-dflash-k7": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.000000000000143e-05 |
| }, |
| "ksweep-qwen-q4-dspark-p0-k10": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.0000000000105516e-05 |
| }, |
| "ksweep-qwen-q4-dspark-p0-k5": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.0000000000050004e-05 |
| }, |
| "ksweep-qwen-q4-dspark-p0-k7": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.0000000000105516e-05 |
| }, |
| "ksweep-qwen-q4-eagle3-k10": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.000000000002225e-05 |
| }, |
| "ksweep-qwen-q4-eagle3-k5": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.0000000000050004e-05 |
| }, |
| "ksweep-qwen-q4-eagle3-k7": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.000000000002225e-05 |
| }, |
| "ksweep-qwen-q4-vanilla17b-k10": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.000000000002225e-05 |
| }, |
| "ksweep-qwen-q4-vanilla17b-k5": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.0000000000050004e-05 |
| }, |
| "ksweep-qwen-q4-vanilla17b-k7": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.0000000000105516e-05 |
| }, |
| "ksweep-qwen-q8-eagle3-k10": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.000000000002225e-05 |
| }, |
| "ksweep-qwen-q8-eagle3-k5": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 4.999999999999449e-05 |
| }, |
| "ksweep-qwen-q8-eagle3-k7": { |
| "log_lines": 178, |
| "n_matched": 178, |
| "max_alpha_diff": 5.000000000002225e-05 |
| } |
| }, |
| "csv_files": { |
| "curves-gemma-q4-dflash-f16": { |
| "csv": "curves/acc_by_pos_curves-gemma-q4-dflash-f16.csv", |
| "k": 3, |
| "n_matched": 175 |
| }, |
| "curves-gemma-q4-dflash-q4": { |
| "csv": "curves/acc_by_pos_curves-gemma-q4-dflash-q4.csv", |
| "k": 3, |
| "n_matched": 175 |
| }, |
| "curves-gemma-q4-dflash-q8": { |
| "csv": "curves/acc_by_pos_curves-gemma-q4-dflash-q8.csv", |
| "k": 3, |
| "n_matched": 175 |
| }, |
| "curves-gemma-q4-mtp": { |
| "csv": "curves/acc_by_pos_curves-gemma-q4-mtp.csv", |
| "k": 3, |
| "n_matched": 175 |
| }, |
| "curves-gemma-q5-dflash-f16": { |
| "csv": "curves/acc_by_pos_curves-gemma-q5-dflash-f16.csv", |
| "k": 3, |
| "n_matched": 174 |
| }, |
| "curves-gemma-q5-mtp": { |
| "csv": "curves/acc_by_pos_curves-gemma-q5-mtp.csv", |
| "k": 3, |
| "n_matched": 174 |
| }, |
| "curves-gemma-q8-dflash-f16": { |
| "csv": "curves/acc_by_pos_curves-gemma-q8-dflash-f16.csv", |
| "k": 3, |
| "n_matched": 174 |
| }, |
| "curves-gemma-q8-mtp": { |
| "csv": "curves/acc_by_pos_curves-gemma-q8-mtp.csv", |
| "k": 3, |
| "n_matched": 174 |
| }, |
| "curves-qwen-q4-dflash": { |
| "csv": "curves/acc_by_pos_curves-qwen-q4-dflash.csv", |
| "k": 3, |
| "n_matched": 178 |
| }, |
| "curves-qwen-q4-dspark-p0": { |
| "csv": "curves/acc_by_pos_curves-qwen-q4-dspark-p0.csv", |
| "k": 3, |
| "n_matched": 178 |
| }, |
| "curves-qwen-q4-dspark-p2": { |
| "csv": "curves/acc_by_pos_curves-qwen-q4-dspark-p2.csv", |
| "k": 3, |
| "n_matched": 178 |
| }, |
| "curves-qwen-q4-dspark-p4": { |
| "csv": "curves/acc_by_pos_curves-qwen-q4-dspark-p4.csv", |
| "k": 3, |
| "n_matched": 178 |
| }, |
| "curves-qwen-q4-dspark-p6": { |
| "csv": "curves/acc_by_pos_curves-qwen-q4-dspark-p6.csv", |
| "k": 3, |
| "n_matched": 178 |
| }, |
| "curves-qwen-q4-eagle3": { |
| "csv": "curves/acc_by_pos_curves-qwen-q4-eagle3.csv", |
| "k": 3, |
| "n_matched": 178 |
| }, |
| "curves-qwen-q4-vanilla17b": { |
| "csv": "curves/acc_by_pos_curves-qwen-q4-vanilla17b.csv", |
| "k": 3, |
| "n_matched": 178 |
| }, |
| "curves-qwen-q5-dflash": { |
| "csv": "curves/acc_by_pos_curves-qwen-q5-dflash.csv", |
| "k": 3, |
| "n_matched": 178 |
| }, |
| "curves-qwen-q5-dspark-p0": { |
| "csv": "curves/acc_by_pos_curves-qwen-q5-dspark-p0.csv", |
| "k": 3, |
| "n_matched": 178 |
| }, |
| "curves-qwen-q5-eagle3": { |
| "csv": "curves/acc_by_pos_curves-qwen-q5-eagle3.csv", |
| "k": 3, |
| "n_matched": 178 |
| }, |
| "curves-qwen-q8-dspark-p0": { |
| "csv": "curves/acc_by_pos_curves-qwen-q8-dspark-p0.csv", |
| "k": 3, |
| "n_matched": 178 |
| }, |
| "curves-qwen-q8-eagle3": { |
| "csv": "curves/acc_by_pos_curves-qwen-q8-eagle3.csv", |
| "k": 3, |
| "n_matched": 178 |
| }, |
| "ksweep-gemma-q4-dflash-f16-k10": { |
| "csv": "curves/acc_by_pos_ksweep-gemma-q4-dflash-f16-k10.csv", |
| "k": 10, |
| "n_matched": 175 |
| }, |
| "ksweep-gemma-q4-dflash-f16-k5": { |
| "csv": "curves/acc_by_pos_ksweep-gemma-q4-dflash-f16-k5.csv", |
| "k": 5, |
| "n_matched": 175 |
| }, |
| "ksweep-gemma-q4-dflash-f16-k7": { |
| "csv": "curves/acc_by_pos_ksweep-gemma-q4-dflash-f16-k7.csv", |
| "k": 7, |
| "n_matched": 175 |
| }, |
| "ksweep-gemma-q4-mtp-k10": { |
| "csv": "curves/acc_by_pos_ksweep-gemma-q4-mtp-k10.csv", |
| "k": 10, |
| "n_matched": 175 |
| }, |
| "ksweep-gemma-q4-mtp-k5": { |
| "csv": "curves/acc_by_pos_ksweep-gemma-q4-mtp-k5.csv", |
| "k": 5, |
| "n_matched": 175 |
| }, |
| "ksweep-gemma-q4-mtp-k7": { |
| "csv": "curves/acc_by_pos_ksweep-gemma-q4-mtp-k7.csv", |
| "k": 7, |
| "n_matched": 175 |
| }, |
| "ksweep-qwen-q4-dflash-k10": { |
| "csv": "curves/acc_by_pos_ksweep-qwen-q4-dflash-k10.csv", |
| "k": 10, |
| "n_matched": 178 |
| }, |
| "ksweep-qwen-q4-dflash-k5": { |
| "csv": "curves/acc_by_pos_ksweep-qwen-q4-dflash-k5.csv", |
| "k": 5, |
| "n_matched": 178 |
| }, |
| "ksweep-qwen-q4-dflash-k7": { |
| "csv": "curves/acc_by_pos_ksweep-qwen-q4-dflash-k7.csv", |
| "k": 7, |
| "n_matched": 178 |
| }, |
| "ksweep-qwen-q4-dspark-p0-k10": { |
| "csv": "curves/acc_by_pos_ksweep-qwen-q4-dspark-p0-k10.csv", |
| "k": 10, |
| "n_matched": 178 |
| }, |
| "ksweep-qwen-q4-dspark-p0-k5": { |
| "csv": "curves/acc_by_pos_ksweep-qwen-q4-dspark-p0-k5.csv", |
| "k": 5, |
| "n_matched": 178 |
| }, |
| "ksweep-qwen-q4-dspark-p0-k7": { |
| "csv": "curves/acc_by_pos_ksweep-qwen-q4-dspark-p0-k7.csv", |
| "k": 7, |
| "n_matched": 178 |
| }, |
| "ksweep-qwen-q4-eagle3-k10": { |
| "csv": "curves/acc_by_pos_ksweep-qwen-q4-eagle3-k10.csv", |
| "k": 10, |
| "n_matched": 178 |
| }, |
| "ksweep-qwen-q4-eagle3-k5": { |
| "csv": "curves/acc_by_pos_ksweep-qwen-q4-eagle3-k5.csv", |
| "k": 5, |
| "n_matched": 178 |
| }, |
| "ksweep-qwen-q4-eagle3-k7": { |
| "csv": "curves/acc_by_pos_ksweep-qwen-q4-eagle3-k7.csv", |
| "k": 7, |
| "n_matched": 178 |
| }, |
| "ksweep-qwen-q4-vanilla17b-k10": { |
| "csv": "curves/acc_by_pos_ksweep-qwen-q4-vanilla17b-k10.csv", |
| "k": 10, |
| "n_matched": 178 |
| }, |
| "ksweep-qwen-q4-vanilla17b-k5": { |
| "csv": "curves/acc_by_pos_ksweep-qwen-q4-vanilla17b-k5.csv", |
| "k": 5, |
| "n_matched": 178 |
| }, |
| "ksweep-qwen-q4-vanilla17b-k7": { |
| "csv": "curves/acc_by_pos_ksweep-qwen-q4-vanilla17b-k7.csv", |
| "k": 7, |
| "n_matched": 178 |
| }, |
| "ksweep-qwen-q8-eagle3-k10": { |
| "csv": "curves/acc_by_pos_ksweep-qwen-q8-eagle3-k10.csv", |
| "k": 10, |
| "n_matched": 178 |
| }, |
| "ksweep-qwen-q8-eagle3-k5": { |
| "csv": "curves/acc_by_pos_ksweep-qwen-q8-eagle3-k5.csv", |
| "k": 5, |
| "n_matched": 178 |
| }, |
| "ksweep-qwen-q8-eagle3-k7": { |
| "csv": "curves/acc_by_pos_ksweep-qwen-q8-eagle3-k7.csv", |
| "k": 7, |
| "n_matched": 178 |
| } |
| }, |
| "method": "sentinel exclusion: tok_per_s >= 1e5 or predicted_ms <= 0; log<->record by order of appearance; OLS alpha_be per paper #32" |
| }, |
| "final": { |
| "final-gemma-q4-dflash-f16": { |
| "run": "final-gemma-q4-dflash-f16", |
| "target": "gemma-q4", |
| "family": "gemma", |
| "quant": "q4", |
| "drafter": "dflash-f16", |
| "k": 3, |
| "ctx": 16384, |
| "n": 1455, |
| "tok_per_s": { |
| "mean": 72.2039910652921, |
| "median": 75.462, |
| "p95": 88.8179 |
| }, |
| "alpha": { |
| "mean": 0.6566116838487973, |
| "median": 0.7033 |
| }, |
| "tau_mean": 156.13264604810996, |
| "ttft_ms": { |
| "mean": 178.2487972508591, |
| "median": 91.0, |
| "p95": 558.1899999999999 |
| }, |
| "vram_max_mib": 10766, |
| "power_max_w": 157.97, |
| "duration_s": 29.1, |
| "errors": 1, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 80.84256511627908, |
| "median": 81.82149999999999, |
| "p95": 88.83569999999999 |
| }, |
| "alpha": { |
| "mean": 0.7758225581395349, |
| "median": 0.7851 |
| }, |
| "tau_mean": 174.85581395348837, |
| "speedup": { |
| "mean": 2.33136130709954, |
| "median": 2.3507745326695653, |
| "agg": 2.332358114519116, |
| "n_match": 430 |
| } |
| }, |
| "code": { |
| "n": 555, |
| "tok_per_s": { |
| "mean": 76.32085045045045, |
| "median": 76.781, |
| "p95": 90.40169999999999 |
| }, |
| "alpha": { |
| "mean": 0.718401981981982, |
| "median": 0.722 |
| }, |
| "tau_mean": 167.1873873873874, |
| "speedup": { |
| "mean": 2.2174172083676087, |
| "median": 2.2194261119081777, |
| "agg": 2.2019038435887417, |
| "n_match": 555 |
| } |
| }, |
| "chat": { |
| "n": 470, |
| "tok_per_s": { |
| "mean": 59.4392170212766, |
| "median": 58.281499999999994, |
| "p95": 83.13255 |
| }, |
| "alpha": { |
| "mean": 0.4745812765957446, |
| "median": 0.46094999999999997 |
| }, |
| "tau_mean": 125.94893617021276, |
| "speedup": { |
| "mean": 1.7100611460880542, |
| "median": 1.6710309129443266, |
| "agg": 1.7148582549407616, |
| "n_match": 470 |
| } |
| } |
| }, |
| "speedup": { |
| "mean": 2.0872031968097664, |
| "median": 2.185492473628067, |
| "agg": 2.0831298984585085, |
| "n_match": 1455 |
| } |
| }, |
| "final-gemma-q4-dflash-q4": { |
| "run": "final-gemma-q4-dflash-q4", |
| "target": "gemma-q4", |
| "family": "gemma", |
| "quant": "q4", |
| "drafter": "dflash-q4", |
| "k": 3, |
| "ctx": 16384, |
| "n": 1455, |
| "tok_per_s": { |
| "mean": 78.35855463917525, |
| "median": 81.919, |
| "p95": 97.1592 |
| }, |
| "alpha": { |
| "mean": 0.64725470790378, |
| "median": 0.6964 |
| }, |
| "tau_mean": 155.11340206185568, |
| "ttft_ms": { |
| "mean": 177.08604810996565, |
| "median": 90.0, |
| "p95": 555.7299999999999 |
| }, |
| "vram_max_mib": 9730, |
| "power_max_w": 164.05, |
| "duration_s": 22.82, |
| "errors": 1, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 88.11804186046513, |
| "median": 88.844, |
| "p95": 97.2851 |
| }, |
| "alpha": { |
| "mean": 0.7701983720930232, |
| "median": 0.7807 |
| }, |
| "tau_mean": 174.37441860465117, |
| "speedup": { |
| "mean": 2.541166057205258, |
| "median": 2.5700507801313335, |
| "agg": 2.5422601283516837, |
| "n_match": 430 |
| } |
| }, |
| "code": { |
| "n": 555, |
| "tok_per_s": { |
| "mean": 82.80627207207208, |
| "median": 83.517, |
| "p95": 99.07239999999999 |
| }, |
| "alpha": { |
| "mean": 0.708456036036036, |
| "median": 0.716 |
| }, |
| "tau_mean": 166.16936936936938, |
| "speedup": { |
| "mean": 2.405611106727141, |
| "median": 2.420483478758404, |
| "agg": 2.389012277413301, |
| "n_match": 555 |
| } |
| }, |
| "chat": { |
| "n": 470, |
| "tok_per_s": { |
| "mean": 64.17757021276596, |
| "median": 62.519999999999996, |
| "p95": 90.13965 |
| }, |
| "alpha": { |
| "mean": 0.4625046808510639, |
| "median": 0.4477 |
| }, |
| "tau_mean": 124.43617021276596, |
| "speedup": { |
| "mean": 1.8461266568689085, |
| "median": 1.808100534428661, |
| "agg": 1.8515626816215813, |
| "n_match": 470 |
| } |
| } |
| }, |
| "speedup": { |
| "mean": 2.264945084233822, |
| "median": 2.378694381292627, |
| "agg": 2.2606928725207895, |
| "n_match": 1455 |
| } |
| }, |
| "final-gemma-q4-dflash-q8": { |
| "run": "final-gemma-q4-dflash-q8", |
| "target": "gemma-q4", |
| "family": "gemma", |
| "quant": "q4", |
| "drafter": "dflash-q8", |
| "k": 3, |
| "ctx": 16384, |
| "n": 1455, |
| "tok_per_s": { |
| "mean": 76.57275051546392, |
| "median": 80.034, |
| "p95": 94.3417 |
| }, |
| "alpha": { |
| "mean": 0.6557245360824743, |
| "median": 0.7033 |
| }, |
| "tau_mean": 156.05223367697596, |
| "ttft_ms": { |
| "mean": 177.27353951890035, |
| "median": 90.2, |
| "p95": 552.8999999999999 |
| }, |
| "vram_max_mib": 10043, |
| "power_max_w": 161.39, |
| "duration_s": 24.26, |
| "errors": 1, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 85.73577674418605, |
| "median": 86.5975, |
| "p95": 94.36255 |
| }, |
| "alpha": { |
| "mean": 0.7747402325581394, |
| "median": 0.7851 |
| }, |
| "tau_mean": 174.77209302325582, |
| "speedup": { |
| "mean": 2.472454081682774, |
| "median": 2.490608393466564, |
| "agg": 2.4735303030808318, |
| "n_match": 430 |
| } |
| }, |
| "code": { |
| "n": 555, |
| "tok_per_s": { |
| "mean": 80.98567207207208, |
| "median": 81.217, |
| "p95": 96.22209999999998 |
| }, |
| "alpha": { |
| "mean": 0.7176104504504505, |
| "median": 0.719 |
| }, |
| "tau_mean": 167.13693693693693, |
| "speedup": { |
| "mean": 2.3529290187964063, |
| "median": 2.351740524447498, |
| "agg": 2.336486838899742, |
| "n_match": 555 |
| } |
| }, |
| "chat": { |
| "n": 470, |
| "tok_per_s": { |
| "mean": 62.97855319148936, |
| "median": 61.8125, |
| "p95": 88.3801 |
| }, |
| "alpha": { |
| "mean": 0.47375978723404255, |
| "median": 0.45865 |
| }, |
| "tau_mean": 125.83617021276596, |
| "speedup": { |
| "mean": 1.811936448286366, |
| "median": 1.7707875599146599, |
| "agg": 1.8169702973374033, |
| "n_match": 470 |
| } |
| } |
| }, |
| "speedup": { |
| "mean": 2.2134989630585498, |
| "median": 2.3199483722931307, |
| "agg": 2.2091713166066627, |
| "n_match": 1455 |
| } |
| }, |
| "final-gemma-q4-mtp": { |
| "run": "final-gemma-q4-mtp", |
| "target": "gemma-q4", |
| "family": "gemma", |
| "quant": "q4", |
| "drafter": "mtp", |
| "k": 3, |
| "ctx": 16384, |
| "n": 1455, |
| "tok_per_s": { |
| "mean": 82.66167628865979, |
| "median": 85.34, |
| "p95": 97.2279 |
| }, |
| "alpha": { |
| "mean": 0.7388333333333333, |
| "median": 0.7773 |
| }, |
| "tau_mean": 163.71546391752577, |
| "ttft_ms": { |
| "mean": 164.81738831615118, |
| "median": 85.4, |
| "p95": 504.21999999999997 |
| }, |
| "vram_max_mib": 9135, |
| "power_max_w": 164.41, |
| "duration_s": 19.91, |
| "errors": 1, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 89.76367209302326, |
| "median": 90.7055, |
| "p95": 97.2355 |
| }, |
| "alpha": { |
| "mean": 0.8303902325581396, |
| "median": 0.8472 |
| }, |
| "tau_mean": 178.76511627906976, |
| "speedup": { |
| "mean": 2.5891424623179917, |
| "median": 2.6083989781984407, |
| "agg": 2.5897375806181264, |
| "n_match": 430 |
| } |
| }, |
| "code": { |
| "n": 555, |
| "tok_per_s": { |
| "mean": 85.64304684684686, |
| "median": 85.717, |
| "p95": 97.7571 |
| }, |
| "alpha": { |
| "mean": 0.7827306306306306, |
| "median": 0.7841 |
| }, |
| "tau_mean": 172.36756756756756, |
| "speedup": { |
| "mean": 2.4891408054990882, |
| "median": 2.4961926737813673, |
| "agg": 2.470854988063223, |
| "n_match": 555 |
| } |
| }, |
| "chat": { |
| "n": 470, |
| "tok_per_s": { |
| "mean": 72.6435510638298, |
| "median": 71.071, |
| "p95": 95.61234999999998 |
| }, |
| "alpha": { |
| "mean": 0.603232340425532, |
| "median": 0.58805 |
| }, |
| "tau_mean": 139.72978723404256, |
| "speedup": { |
| "mean": 2.0902575697224277, |
| "median": 2.049779416756877, |
| "agg": 2.095811476881123, |
| "n_match": 470 |
| } |
| } |
| }, |
| "speedup": { |
| "mean": 2.3898456794627294, |
| "median": 2.4744594439995424, |
| "agg": 2.3848405994329966, |
| "n_match": 1455 |
| } |
| }, |
| "final-gemma-q4-solo": { |
| "run": "final-gemma-q4-solo", |
| "target": "gemma-q4", |
| "family": "gemma", |
| "quant": "q4", |
| "drafter": "solo", |
| "k": 3, |
| "ctx": 16384, |
| "n": 1455, |
| "tok_per_s": { |
| "mean": 34.661300343642615, |
| "median": 34.829, |
| "p95": 35.0786 |
| }, |
| "alpha": null, |
| "tau_mean": null, |
| "ttft_ms": { |
| "mean": 148.97491408934707, |
| "median": 83.1, |
| "p95": 426.55999999999995 |
| }, |
| "vram_max_mib": 8616, |
| "power_max_w": 164.21, |
| "duration_s": 31.91, |
| "errors": 1, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 34.68004651162791, |
| "median": 34.831500000000005, |
| "p95": 34.930099999999996 |
| }, |
| "alpha": null, |
| "tau_mean": null |
| }, |
| "code": { |
| "n": 555, |
| "tok_per_s": { |
| "mean": 34.40588828828829, |
| "median": 34.654, |
| "p95": 34.964 |
| }, |
| "alpha": null, |
| "tau_mean": null |
| }, |
| "chat": { |
| "n": 470, |
| "tok_per_s": { |
| "mean": 34.945753191489366, |
| "median": 34.9105, |
| "p95": 36.13855 |
| }, |
| "alpha": null, |
| "tau_mean": null |
| } |
| } |
| }, |
| "final-gemma-q5-dflash-f16": { |
| "run": "final-gemma-q5-dflash-f16", |
| "target": "gemma-q5", |
| "family": "gemma", |
| "quant": "q5", |
| "drafter": "dflash-f16", |
| "k": 3, |
| "ctx": 16384, |
| "n": 1449, |
| "tok_per_s": { |
| "mean": 64.50848516218082, |
| "median": 67.763, |
| "p95": 79.3998 |
| }, |
| "alpha": { |
| "mean": 0.659056314699793, |
| "median": 0.709 |
| }, |
| "tau_mean": 156.50103519668738, |
| "ttft_ms": { |
| "mean": 187.51076604554865, |
| "median": 98.2, |
| "p95": 573.2799999999997 |
| }, |
| "vram_max_mib": 11980, |
| "power_max_w": 160.82, |
| "duration_s": 27.93, |
| "errors": 0, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 72.45127209302326, |
| "median": 73.244, |
| "p95": 79.164 |
| }, |
| "alpha": { |
| "mean": 0.7816416279069768, |
| "median": 0.7956 |
| }, |
| "tau_mean": 175.6860465116279, |
| "speedup": { |
| "mean": 2.419694650889923, |
| "median": 2.4436636828827507, |
| "agg": 2.425081750618513, |
| "n_match": 430 |
| } |
| }, |
| "code": { |
| "n": 553, |
| "tok_per_s": { |
| "mean": 68.42736166365282, |
| "median": 68.642, |
| "p95": 81.0476 |
| }, |
| "alpha": { |
| "mean": 0.7238960216998191, |
| "median": 0.725 |
| }, |
| "tau_mean": 167.06690777576853, |
| "speedup": { |
| "mean": 2.301476263202495, |
| "median": 2.306608809450012, |
| "agg": 2.29039382221526, |
| "n_match": 553 |
| } |
| }, |
| "chat": { |
| "n": 466, |
| "tok_per_s": { |
| "mean": 52.528791845493565, |
| "median": 51.403, |
| "p95": 72.68025 |
| }, |
| "alpha": { |
| "mean": 0.4689961373390558, |
| "median": 0.4506 |
| }, |
| "tau_mean": 126.25965665236052, |
| "speedup": { |
| "mean": 1.7535148322519638, |
| "median": 1.7142457329029552, |
| "agg": 1.758238479553374, |
| "n_match": 466 |
| } |
| } |
| }, |
| "speedup": { |
| "mean": 2.1603333231629134, |
| "median": 2.2665265124970806, |
| "agg": 2.159221578966785, |
| "n_match": 1449 |
| } |
| }, |
| "final-gemma-q5-mtp": { |
| "run": "final-gemma-q5-mtp", |
| "target": "gemma-q5", |
| "family": "gemma", |
| "quant": "q5", |
| "drafter": "mtp", |
| "k": 3, |
| "ctx": 16384, |
| "n": 1449, |
| "tok_per_s": { |
| "mean": 75.64226777087647, |
| "median": 78.085, |
| "p95": 88.46039999999999 |
| }, |
| "alpha": { |
| "mean": 0.7446065562456866, |
| "median": 0.7807 |
| }, |
| "tau_mean": 164.80193236714976, |
| "ttft_ms": { |
| "mean": 172.72298136645963, |
| "median": 92.4, |
| "p95": 521.18 |
| }, |
| "vram_max_mib": 10366, |
| "power_max_w": 164.08, |
| "duration_s": 13.39, |
| "errors": 0, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 82.26693488372095, |
| "median": 82.7995, |
| "p95": 88.4153 |
| }, |
| "alpha": { |
| "mean": 0.8372979069767441, |
| "median": 0.84685 |
| }, |
| "tau_mean": 179.5186046511628, |
| "speedup": { |
| "mean": 2.7472747862800895, |
| "median": 2.7623163605961403, |
| "agg": 2.7536306361837446, |
| "n_match": 430 |
| } |
| }, |
| "code": { |
| "n": 553, |
| "tok_per_s": { |
| "mean": 78.36189150090416, |
| "median": 78.361, |
| "p95": 89.0876 |
| }, |
| "alpha": { |
| "mean": 0.7879855334538879, |
| "median": 0.7885 |
| }, |
| "tau_mean": 172.3254972875226, |
| "speedup": { |
| "mean": 2.6363271005548405, |
| "median": 2.642533786032887, |
| "agg": 2.622921413702688, |
| "n_match": 553 |
| } |
| }, |
| "chat": { |
| "n": 466, |
| "tok_per_s": { |
| "mean": 66.30201287553648, |
| "median": 64.481, |
| "p95": 86.9225 |
| }, |
| "alpha": { |
| "mean": 0.6075982832618027, |
| "median": 0.5848 |
| }, |
| "tau_mean": 142.293991416309, |
| "speedup": { |
| "mean": 2.2134025387198037, |
| "median": 2.1624934401359086, |
| "agg": 2.2192543596376737, |
| "n_match": 466 |
| } |
| } |
| }, |
| "speedup": { |
| "mean": 2.5332385284683876, |
| "median": 2.6139016743379364, |
| "agg": 2.5318904395636666, |
| "n_match": 1449 |
| } |
| }, |
| "final-gemma-q5-solo": { |
| "run": "final-gemma-q5-solo", |
| "target": "gemma-q5", |
| "family": "gemma", |
| "quant": "q5", |
| "drafter": "solo", |
| "k": 3, |
| "ctx": 16384, |
| "n": 1449, |
| "tok_per_s": { |
| "mean": 29.875806073153896, |
| "median": 29.952, |
| "p95": 30.119600000000002 |
| }, |
| "alpha": null, |
| "tau_mean": null, |
| "ttft_ms": { |
| "mean": 157.97805383022776, |
| "median": 90.2, |
| "p95": 442.0199999999997 |
| }, |
| "vram_max_mib": 9848, |
| "power_max_w": 164.34, |
| "duration_s": 18.6, |
| "errors": 0, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 29.946332558139538, |
| "median": 29.955, |
| "p95": 30.026 |
| }, |
| "alpha": null, |
| "tau_mean": null |
| }, |
| "code": { |
| "n": 553, |
| "tok_per_s": { |
| "mean": 29.718493670886073, |
| "median": 29.837, |
| "p95": 30.0494 |
| }, |
| "alpha": null, |
| "tau_mean": null |
| }, |
| "chat": { |
| "n": 466, |
| "tok_per_s": { |
| "mean": 29.99740987124464, |
| "median": 29.9895, |
| "p95": 30.8305 |
| }, |
| "alpha": null, |
| "tau_mean": null |
| } |
| } |
| }, |
| "final-gemma-q8-dflash-f16": { |
| "run": "final-gemma-q8-dflash-f16", |
| "target": "gemma-q8", |
| "family": "gemma", |
| "quant": "q8", |
| "drafter": "dflash-f16", |
| "k": 3, |
| "ctx": 8192, |
| "n": 1452, |
| "tok_per_s": { |
| "mean": 45.707994490358125, |
| "median": 47.832499999999996, |
| "p95": 56.6311 |
| }, |
| "alpha": { |
| "mean": 0.6583433195592286, |
| "median": 0.709 |
| }, |
| "tau_mean": 155.7555096418733, |
| "ttft_ms": { |
| "mean": 211.7603305785124, |
| "median": 130.14999999999998, |
| "p95": 580.89 |
| }, |
| "vram_max_mib": 15888, |
| "power_max_w": 149.25, |
| "duration_s": 104.6, |
| "errors": 2, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 51.26042325581395, |
| "median": 51.679500000000004, |
| "p95": 56.893049999999995 |
| }, |
| "alpha": { |
| "mean": 0.7795267441860465, |
| "median": 0.7898000000000001 |
| }, |
| "tau_mean": 175.41162790697675, |
| "speedup": { |
| "mean": 2.5753097406055074, |
| "median": 2.593730133667818, |
| "agg": 2.5714690229558856, |
| "n_match": 430 |
| } |
| }, |
| "code": { |
| "n": 558, |
| "tok_per_s": { |
| "mean": 48.73938888888889, |
| "median": 49.062, |
| "p95": 57.54775 |
| }, |
| "alpha": { |
| "mean": 0.7266213261648745, |
| "median": 0.7322 |
| }, |
| "tau_mean": 166.96236559139786, |
| "speedup": { |
| "mean": 2.4570797043746073, |
| "median": 2.4570426569392354, |
| "agg": 2.4450018311419806, |
| "n_match": 558 |
| } |
| }, |
| "chat": { |
| "n": 464, |
| "tok_per_s": { |
| "mean": 36.91691163793104, |
| "median": 36.471000000000004, |
| "p95": 51.6553 |
| }, |
| "alpha": { |
| "mean": 0.4639295258620689, |
| "median": 0.4534 |
| }, |
| "tau_mean": 124.0625, |
| "speedup": { |
| "mean": 1.8473760999433333, |
| "median": 1.8348887305160884, |
| "agg": 1.851929591497711, |
| "n_match": 464 |
| } |
| } |
| }, |
| "speedup": { |
| "mean": 2.297256318095803, |
| "median": 2.413130461150101, |
| "agg": 2.29293252899669, |
| "n_match": 1452 |
| } |
| }, |
| "final-gemma-q8-mtp": { |
| "run": "final-gemma-q8-mtp", |
| "target": "gemma-q8", |
| "family": "gemma", |
| "quant": "q8", |
| "drafter": "mtp", |
| "k": 3, |
| "ctx": 8192, |
| "n": 1452, |
| "tok_per_s": { |
| "mean": 53.95811019283746, |
| "median": 55.691500000000005, |
| "p95": 63.40005000000001 |
| }, |
| "alpha": { |
| "mean": 0.7454163911845729, |
| "median": 0.7773 |
| }, |
| "tau_mean": 164.02685950413223, |
| "ttft_ms": { |
| "mean": 195.49958677685947, |
| "median": 122.75, |
| "p95": 535.12 |
| }, |
| "vram_max_mib": 14251, |
| "power_max_w": 166.57, |
| "duration_s": 77.81, |
| "errors": 2, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 58.54481627906977, |
| "median": 59.262, |
| "p95": 63.0375 |
| }, |
| "alpha": { |
| "mean": 0.8365709302325581, |
| "median": 0.8472 |
| }, |
| "tau_mean": 179.35813953488372, |
| "speedup": { |
| "mean": 2.9405015463944766, |
| "median": 2.9678438251708847, |
| "agg": 2.936889162326541, |
| "n_match": 430 |
| } |
| }, |
| "code": { |
| "n": 558, |
| "tok_per_s": { |
| "mean": 56.03362544802867, |
| "median": 55.9885, |
| "p95": 63.89855 |
| }, |
| "alpha": { |
| "mean": 0.7901008960573477, |
| "median": 0.7885 |
| }, |
| "tau_mean": 172.20967741935485, |
| "speedup": { |
| "mean": 2.8249630349993744, |
| "median": 2.816732296284015, |
| "agg": 2.8109157695488953, |
| "n_match": 558 |
| } |
| }, |
| "chat": { |
| "n": 464, |
| "tok_per_s": { |
| "mean": 47.211512931034484, |
| "median": 46.5275, |
| "p95": 61.2943 |
| }, |
| "alpha": { |
| "mean": 0.6072043103448276, |
| "median": 0.5926 |
| }, |
| "tau_mean": 139.97844827586206, |
| "speedup": { |
| "mean": 2.3622201722888154, |
| "median": 2.3423838847443745, |
| "agg": 2.368356235046633, |
| "n_match": 464 |
| } |
| } |
| }, |
| "speedup": { |
| "mean": 2.711305233072511, |
| "median": 2.7997846830116755, |
| "agg": 2.706797956109045, |
| "n_match": 1452 |
| } |
| }, |
| "final-gemma-q8-solo": { |
| "run": "final-gemma-q8-solo", |
| "target": "gemma-q8", |
| "family": "gemma", |
| "quant": "q8", |
| "drafter": "solo", |
| "k": 3, |
| "ctx": 8192, |
| "n": 1452, |
| "tok_per_s": { |
| "mean": 19.934295454545452, |
| "median": 19.811500000000002, |
| "p95": 20.451 |
| }, |
| "alpha": null, |
| "tau_mean": null, |
| "ttft_ms": { |
| "mean": 178.38264462809917, |
| "median": 119.8, |
| "p95": 445.27000000000015 |
| }, |
| "vram_max_mib": 13741, |
| "power_max_w": 149.04, |
| "duration_s": 138.21, |
| "errors": 2, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 19.917351162790695, |
| "median": 19.7945, |
| "p95": 20.428 |
| }, |
| "alpha": null, |
| "tau_mean": null |
| }, |
| "code": { |
| "n": 558, |
| "tok_per_s": { |
| "mean": 19.839224014336917, |
| "median": 19.809, |
| "p95": 20.447 |
| }, |
| "alpha": null, |
| "tau_mean": null |
| }, |
| "chat": { |
| "n": 464, |
| "tok_per_s": { |
| "mean": 20.06432974137931, |
| "median": 19.823500000000003, |
| "p95": 20.6116 |
| }, |
| "alpha": null, |
| "tau_mean": null |
| } |
| } |
| }, |
| "final-qwen-q4-dflash": { |
| "run": "final-qwen-q4-dflash", |
| "target": "qwen-q4", |
| "family": "qwen", |
| "quant": "q4", |
| "drafter": "dflash-f16", |
| "k": 3, |
| "ctx": 8192, |
| "n": 1474, |
| "tok_per_s": { |
| "mean": 33.42025169606513, |
| "median": 33.329, |
| "p95": 34.609049999999996 |
| }, |
| "alpha": { |
| "mean": 0.009222184531886024, |
| "median": 0.0081 |
| }, |
| "tau_mean": 6.700135685210312, |
| "ttft_ms": { |
| "mean": 76.68419267299865, |
| "median": 36.3, |
| "p95": 260.24499999999995 |
| }, |
| "vram_max_mib": 9288, |
| "power_max_w": 161.75, |
| "duration_s": 88.06, |
| "errors": 0, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 33.448123255813954, |
| "median": 33.2755, |
| "p95": 34.83115 |
| }, |
| "alpha": { |
| "mean": 0.00873953488372093, |
| "median": 0.0067 |
| }, |
| "tau_mean": 6.341860465116279, |
| "speedup": { |
| "mean": 0.625348238489337, |
| "median": 0.6215996714030199, |
| "agg": 0.6267435724021049, |
| "n_match": 430 |
| } |
| }, |
| "code": { |
| "n": 564, |
| "tok_per_s": { |
| "mean": 33.3616134751773, |
| "median": 33.329, |
| "p95": 34.39455 |
| }, |
| "alpha": { |
| "mean": 0.009304432624113476, |
| "median": 0.0081 |
| }, |
| "tau_mean": 6.792553191489362, |
| "speedup": { |
| "mean": 0.6268145533192508, |
| "median": 0.6247902486921681, |
| "agg": 0.6251225711713553, |
| "n_match": 564 |
| } |
| }, |
| "chat": { |
| "n": 480, |
| "tok_per_s": { |
| "mean": 33.46418333333334, |
| "median": 33.411500000000004, |
| "p95": 34.70445 |
| }, |
| "alpha": { |
| "mean": 0.009557916666666666, |
| "median": 0.0081 |
| }, |
| "tau_mean": 6.9125, |
| "speedup": { |
| "mean": 0.6266434326195484, |
| "median": 0.6238302998776759, |
| "agg": 0.6270445026002082, |
| "n_match": 480 |
| } |
| } |
| }, |
| "speedup": { |
| "mean": 0.6263310707461707, |
| "median": 0.6235980937434453, |
| "agg": 0.626221321249423, |
| "n_match": 1474 |
| } |
| }, |
| "final-qwen-q4-dspark-p0": { |
| "run": "final-qwen-q4-dspark-p0", |
| "target": "qwen-q4", |
| "family": "qwen", |
| "quant": "q4", |
| "drafter": "dspark-p0", |
| "k": 3, |
| "ctx": 8192, |
| "n": 1474, |
| "tok_per_s": { |
| "mean": 87.61712075983718, |
| "median": 86.888, |
| "p95": 109.27405 |
| }, |
| "alpha": { |
| "mean": 0.6163681818181819, |
| "median": 0.6074 |
| }, |
| "tau_mean": 162.40773405698778, |
| "ttft_ms": { |
| "mean": 76.93622795115333, |
| "median": 36.6, |
| "p95": 259.07 |
| }, |
| "vram_max_mib": 9429, |
| "power_max_w": 158.6, |
| "duration_s": 43.65, |
| "errors": 0, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 95.87322790697674, |
| "median": 96.12450000000001, |
| "p95": 111.8299 |
| }, |
| "alpha": { |
| "mean": 0.7061618604651162, |
| "median": 0.709 |
| }, |
| "tau_mean": 171.12558139534883, |
| "speedup": { |
| "mean": 1.792389094534062, |
| "median": 1.8095527871081896, |
| "agg": 1.7964514450207694, |
| "n_match": 430 |
| } |
| }, |
| "code": { |
| "n": 564, |
| "tok_per_s": { |
| "mean": 86.51806382978724, |
| "median": 85.81049999999999, |
| "p95": 101.14075 |
| }, |
| "alpha": { |
| "mean": 0.6055257092198582, |
| "median": 0.5971 |
| }, |
| "tau_mean": 163.10106382978722, |
| "speedup": { |
| "mean": 1.6257053204274554, |
| "median": 1.6120584527896276, |
| "agg": 1.6211564393995967, |
| "n_match": 564 |
| } |
| }, |
| "chat": { |
| "n": 480, |
| "tok_per_s": { |
| "mean": 81.51241666666668, |
| "median": 80.22, |
| "p95": 109.78285 |
| }, |
| "alpha": { |
| "mean": 0.5486679166666667, |
| "median": 0.534 |
| }, |
| "tau_mean": 153.78333333333333, |
| "speedup": { |
| "mean": 1.5262824570915052, |
| "median": 1.4997059758476776, |
| "agg": 1.5273617244852606, |
| "n_match": 480 |
| } |
| } |
| }, |
| "speedup": { |
| "mean": 1.6419543356680153, |
| "median": 1.6271307344012569, |
| "agg": 1.6417503262776312, |
| "n_match": 1474 |
| } |
| }, |
| "final-qwen-q4-dspark-p2": { |
| "run": "final-qwen-q4-dspark-p2", |
| "target": "qwen-q4", |
| "family": "qwen", |
| "quant": "q4", |
| "drafter": "dspark-p2", |
| "k": 3, |
| "ctx": 8192, |
| "n": 1474, |
| "tok_per_s": { |
| "mean": 86.67783649932157, |
| "median": 85.97399999999999, |
| "p95": 108.11829999999999 |
| }, |
| "alpha": { |
| "mean": 0.6207055630936228, |
| "median": 0.6104 |
| }, |
| "tau_mean": 162.35888738127545, |
| "ttft_ms": { |
| "mean": 76.96702849389418, |
| "median": 36.5, |
| "p95": 260.44999999999993 |
| }, |
| "vram_max_mib": 9426, |
| "power_max_w": 158.01, |
| "duration_s": 40.92, |
| "errors": 0, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 94.99026744186047, |
| "median": 95.7945, |
| "p95": 110.67115 |
| }, |
| "alpha": { |
| "mean": 0.7067206976744186, |
| "median": 0.71045 |
| }, |
| "tau_mean": 171.146511627907, |
| "speedup": { |
| "mean": 1.7758914666828969, |
| "median": 1.7920045412645096, |
| "agg": 1.7799067261447827, |
| "n_match": 430 |
| } |
| }, |
| "code": { |
| "n": 564, |
| "tok_per_s": { |
| "mean": 85.59933865248227, |
| "median": 84.9105, |
| "p95": 100.38805 |
| }, |
| "alpha": { |
| "mean": 0.6058659574468085, |
| "median": 0.5971 |
| }, |
| "tau_mean": 163.08510638297872, |
| "speedup": { |
| "mean": 1.6084466080758277, |
| "median": 1.591466231502861, |
| "agg": 1.6039415692176116, |
| "n_match": 564 |
| } |
| }, |
| "chat": { |
| "n": 480, |
| "tok_per_s": { |
| "mean": 80.49851875, |
| "median": 79.1845, |
| "p95": 108.44174999999997 |
| }, |
| "alpha": { |
| "mean": 0.5610868750000001, |
| "median": 0.54575 |
| }, |
| "tau_mean": 153.63333333333333, |
| "speedup": { |
| "mean": 1.507203142793631, |
| "median": 1.4820706392926062, |
| "agg": 1.5083635284584547, |
| "n_match": 480 |
| } |
| } |
| }, |
| "speedup": { |
| "mean": 1.6243247803048544, |
| "median": 1.6104524664676467, |
| "agg": 1.6241502245190282, |
| "n_match": 1474 |
| } |
| }, |
| "final-qwen-q4-dspark-p4": { |
| "run": "final-qwen-q4-dspark-p4", |
| "target": "qwen-q4", |
| "family": "qwen", |
| "quant": "q4", |
| "drafter": "dspark-p4", |
| "k": 3, |
| "ctx": 8192, |
| "n": 1474, |
| "tok_per_s": { |
| "mean": 85.41618995929444, |
| "median": 85.232, |
| "p95": 107.8453 |
| }, |
| "alpha": { |
| "mean": 0.6464877204884668, |
| "median": 0.6293 |
| }, |
| "tau_mean": 160.86227951153325, |
| "ttft_ms": { |
| "mean": 76.99735413839892, |
| "median": 36.6, |
| "p95": 260.0199999999999 |
| }, |
| "vram_max_mib": 9418, |
| "power_max_w": 155.91, |
| "duration_s": 41.37, |
| "errors": 0, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 94.57915581395348, |
| "median": 95.011, |
| "p95": 110.8468 |
| }, |
| "alpha": { |
| "mean": 0.7119279069767442, |
| "median": 0.7169 |
| }, |
| "tau_mean": 170.88372093023256, |
| "speedup": { |
| "mean": 1.7682082029834372, |
| "median": 1.7870420187369391, |
| "agg": 1.7722034069373087, |
| "n_match": 430 |
| } |
| }, |
| "code": { |
| "n": 564, |
| "tok_per_s": { |
| "mean": 84.81273226950354, |
| "median": 84.136, |
| "p95": 99.644 |
| }, |
| "alpha": { |
| "mean": 0.6222294326241136, |
| "median": 0.6157 |
| }, |
| "tau_mean": 162.5904255319149, |
| "speedup": { |
| "mean": 1.5937010350638858, |
| "median": 1.5782483066579922, |
| "agg": 1.589202311927393, |
| "n_match": 564 |
| } |
| }, |
| "chat": { |
| "n": 480, |
| "tok_per_s": { |
| "mean": 77.9167625, |
| "median": 76.28450000000001, |
| "p95": 107.14879999999998 |
| }, |
| "alpha": { |
| "mean": 0.6163677083333333, |
| "median": 0.5869 |
| }, |
| "tau_mean": 149.85416666666666, |
| "speedup": { |
| "mean": 1.458965926898531, |
| "median": 1.4277669171898328, |
| "agg": 1.4599871480313344, |
| "n_match": 480 |
| } |
| } |
| }, |
| "speedup": { |
| "mean": 1.6007330773203559, |
| "median": 1.5970623063583436, |
| "agg": 1.6005097693114878, |
| "n_match": 1474 |
| } |
| }, |
| "final-qwen-q4-dspark-p6": { |
| "run": "final-qwen-q4-dspark-p6", |
| "target": "qwen-q4", |
| "family": "qwen", |
| "quant": "q4", |
| "drafter": "dspark-p6", |
| "k": 3, |
| "ctx": 8192, |
| "n": 1474, |
| "tok_per_s": { |
| "mean": 80.73099660786974, |
| "median": 80.082, |
| "p95": 106.61354999999999 |
| }, |
| "alpha": { |
| "mean": 0.7138444369063772, |
| "median": 0.7036 |
| }, |
| "tau_mean": 154.219131614654, |
| "ttft_ms": { |
| "mean": 77.25006784260516, |
| "median": 36.9, |
| "p95": 260.78999999999985 |
| }, |
| "vram_max_mib": 9417, |
| "power_max_w": 158.04, |
| "duration_s": 45.09, |
| "errors": 0, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 92.6727976744186, |
| "median": 92.857, |
| "p95": 110.70694999999999 |
| }, |
| "alpha": { |
| "mean": 0.7426737209302325, |
| "median": 0.75215 |
| }, |
| "tau_mean": 169.05348837209303, |
| "speedup": { |
| "mean": 1.7325356529294538, |
| "median": 1.7405452449155936, |
| "agg": 1.7364824876644391, |
| "n_match": 430 |
| } |
| }, |
| "code": { |
| "n": 564, |
| "tok_per_s": { |
| "mean": 79.6881755319149, |
| "median": 79.1105, |
| "p95": 95.50435000000002 |
| }, |
| "alpha": { |
| "mean": 0.6840361702127659, |
| "median": 0.6807 |
| }, |
| "tau_mean": 156.5549645390071, |
| "speedup": { |
| "mean": 1.4974874919466474, |
| "median": 1.484013217389872, |
| "agg": 1.493179495575946, |
| "n_match": 564 |
| } |
| }, |
| "chat": { |
| "n": 480, |
| "tok_per_s": { |
| "mean": 71.25844791666665, |
| "median": 68.9305, |
| "p95": 103.77059999999997 |
| }, |
| "alpha": { |
| "mean": 0.7230429166666668, |
| "median": 0.7072499999999999 |
| }, |
| "tau_mean": 138.18541666666667, |
| "speedup": { |
| "mean": 1.3341349356936296, |
| "median": 1.284632294241261, |
| "agg": 1.3352251147112733, |
| "n_match": 480 |
| } |
| } |
| }, |
| "speedup": { |
| "mean": 1.5128616318524535, |
| "median": 1.49748294695411, |
| "agg": 1.5127196473961693, |
| "n_match": 1474 |
| } |
| }, |
| "final-qwen-q4-eagle3": { |
| "run": "final-qwen-q4-eagle3", |
| "target": "qwen-q4", |
| "family": "qwen", |
| "quant": "q4", |
| "drafter": "eagle3", |
| "k": 3, |
| "ctx": 8192, |
| "n": 1474, |
| "tok_per_s": { |
| "mean": 74.37096675712347, |
| "median": 73.1925, |
| "p95": 97.836 |
| }, |
| "alpha": { |
| "mean": 0.44020434192673, |
| "median": 0.4294 |
| }, |
| "tau_mean": 140.5407055630936, |
| "ttft_ms": { |
| "mean": 74.8416553595658, |
| "median": 35.4, |
| "p95": 253.77499999999995 |
| }, |
| "vram_max_mib": 8191, |
| "power_max_w": 159.02, |
| "duration_s": 43.6, |
| "errors": 0, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 74.98914186046511, |
| "median": 74.6525, |
| "p95": 86.56679999999999 |
| }, |
| "alpha": { |
| "mean": 0.4451893023255814, |
| "median": 0.4421 |
| }, |
| "tau_mean": 143.8860465116279, |
| "speedup": { |
| "mean": 1.401978119769266, |
| "median": 1.3938344426680405, |
| "agg": 1.4051300367899346, |
| "n_match": 430 |
| } |
| }, |
| "code": { |
| "n": 564, |
| "tok_per_s": { |
| "mean": 74.86378723404256, |
| "median": 73.24950000000001, |
| "p95": 93.90325 |
| }, |
| "alpha": { |
| "mean": 0.4467317375886525, |
| "median": 0.4294 |
| }, |
| "tau_mean": 144.17021276595744, |
| "speedup": { |
| "mean": 1.4063464266699996, |
| "median": 1.3791478278236826, |
| "agg": 1.4027811693877092, |
| "n_match": 564 |
| } |
| }, |
| "chat": { |
| "n": 480, |
| "tok_per_s": { |
| "mean": 73.23812083333333, |
| "median": 70.8945, |
| "p95": 107.34859999999993 |
| }, |
| "alpha": { |
| "mean": 0.4280689583333333, |
| "median": 0.4029 |
| }, |
| "tau_mean": 133.27916666666667, |
| "speedup": { |
| "mean": 1.3715519431293326, |
| "median": 1.3239886829768142, |
| "agg": 1.372319790143136, |
| "n_match": 480 |
| } |
| } |
| }, |
| "speedup": { |
| "mean": 1.3937414578322551, |
| "median": 1.3727328100680438, |
| "agg": 1.3935468077496915, |
| "n_match": 1474 |
| } |
| }, |
| "final-qwen-q4-solo": { |
| "run": "final-qwen-q4-solo", |
| "target": "qwen-q4", |
| "family": "qwen", |
| "quant": "q4", |
| "drafter": "solo", |
| "k": 3, |
| "ctx": 8192, |
| "n": 1474, |
| "tok_per_s": { |
| "mean": 53.368115332428765, |
| "median": 53.703, |
| "p95": 53.862 |
| }, |
| "alpha": null, |
| "tau_mean": null, |
| "ttft_ms": { |
| "mean": 65.15162822252374, |
| "median": 32.05, |
| "p95": 213.54499999999993 |
| }, |
| "vram_max_mib": 6683, |
| "power_max_w": 158.15, |
| "duration_s": 53.79, |
| "errors": 0, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 53.496337209302325, |
| "median": 53.731, |
| "p95": 53.815200000000004 |
| }, |
| "alpha": null, |
| "tau_mean": null |
| }, |
| "code": { |
| "n": 564, |
| "tok_per_s": { |
| "mean": 53.23322872340426, |
| "median": 53.432500000000005, |
| "p95": 53.8617 |
| }, |
| "alpha": null, |
| "tau_mean": null |
| }, |
| "chat": { |
| "n": 480, |
| "tok_per_s": { |
| "mean": 53.411741666666664, |
| "median": 53.781499999999994, |
| "p95": 53.86905 |
| }, |
| "alpha": null, |
| "tau_mean": null |
| } |
| } |
| }, |
| "final-qwen-q4-vanilla17b": { |
| "run": "final-qwen-q4-vanilla17b", |
| "target": "qwen-q4", |
| "family": "qwen", |
| "quant": "q4", |
| "drafter": "vanilla17b", |
| "k": 3, |
| "ctx": 8192, |
| "n": 1474, |
| "tok_per_s": { |
| "mean": 75.45253731343284, |
| "median": 75.6645, |
| "p95": 87.57839999999999 |
| }, |
| "alpha": { |
| "mean": 0.7251561058344641, |
| "median": 0.7292 |
| }, |
| "tau_mean": 173.27272727272728, |
| "ttft_ms": { |
| "mean": 79.04545454545455, |
| "median": 39.3, |
| "p95": 253.23499999999999 |
| }, |
| "vram_max_mib": 8691, |
| "power_max_w": 164.61, |
| "duration_s": 49.07, |
| "errors": 0, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 79.97231162790698, |
| "median": 80.05449999999999, |
| "p95": 87.7595 |
| }, |
| "alpha": { |
| "mean": 0.7835525581395348, |
| "median": 0.7851 |
| }, |
| "tau_mean": 177.61162790697674, |
| "speedup": { |
| "mean": 1.4951547492953159, |
| "median": 1.4965815573078318, |
| "agg": 1.498503575210803, |
| "n_match": 430 |
| } |
| }, |
| "code": { |
| "n": 564, |
| "tok_per_s": { |
| "mean": 74.41387234042553, |
| "median": 74.43299999999999, |
| "p95": 83.16835 |
| }, |
| "alpha": { |
| "mean": 0.7156375886524823, |
| "median": 0.7149 |
| }, |
| "tau_mean": 173.18971631205673, |
| "speedup": { |
| "mean": 1.398139174151506, |
| "median": 1.394663418381457, |
| "agg": 1.39435076312707, |
| "n_match": 564 |
| } |
| }, |
| "chat": { |
| "n": 480, |
| "tok_per_s": { |
| "mean": 72.62400416666668, |
| "median": 71.2925, |
| "p95": 90.07265 |
| }, |
| "alpha": { |
| "mean": 0.684026875, |
| "median": 0.6576 |
| }, |
| "tau_mean": 169.48333333333332, |
| "speedup": { |
| "mean": 1.359868566571217, |
| "median": 1.327286770288385, |
| "agg": 1.36081260719614, |
| "n_match": 480 |
| } |
| } |
| }, |
| "speedup": { |
| "mean": 1.41397825534099, |
| "median": 1.4146129905513134, |
| "agg": 1.4138130388049253, |
| "n_match": 1474 |
| } |
| }, |
| "final-qwen-q5-dflash": { |
| "run": "final-qwen-q5-dflash", |
| "target": "qwen-q5", |
| "family": "qwen", |
| "quant": "q5", |
| "drafter": "dflash-f16", |
| "k": 3, |
| "ctx": 8192, |
| "n": 1474, |
| "tok_per_s": { |
| "mean": 30.77782089552239, |
| "median": 30.700000000000003, |
| "p95": 31.74435 |
| }, |
| "alpha": { |
| "mean": 0.009008683853459973, |
| "median": 0.0081 |
| }, |
| "tau_mean": 6.550203527815468, |
| "ttft_ms": { |
| "mean": 79.47089552238806, |
| "median": 38.0, |
| "p95": 264.34499999999997 |
| }, |
| "vram_max_mib": 9982, |
| "power_max_w": 162.39, |
| "duration_s": 92.1, |
| "errors": 0, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 30.84132093023256, |
| "median": 30.722, |
| "p95": 32.03855 |
| }, |
| "alpha": { |
| "mean": 0.00911813953488372, |
| "median": 0.0081 |
| }, |
| "tau_mean": 6.611627906976744, |
| "speedup": { |
| "mean": 0.6574222739183073, |
| "median": 0.654970066684825, |
| "agg": 0.6590076471548982, |
| "n_match": 430 |
| } |
| }, |
| "code": { |
| "n": 564, |
| "tok_per_s": { |
| "mean": 30.707031914893616, |
| "median": 30.6725, |
| "p95": 31.5744 |
| }, |
| "alpha": { |
| "mean": 0.00880372340425532, |
| "median": 0.0081 |
| }, |
| "tau_mean": 6.425531914893617, |
| "speedup": { |
| "mean": 0.6573459269265252, |
| "median": 0.6557818202526157, |
| "agg": 0.6561382017041842, |
| "n_match": 564 |
| } |
| }, |
| "chat": { |
| "n": 480, |
| "tok_per_s": { |
| "mean": 30.8041125, |
| "median": 30.741500000000002, |
| "p95": 31.91105 |
| }, |
| "alpha": { |
| "mean": 0.009151458333333333, |
| "median": 0.0081 |
| }, |
| "tau_mean": 6.641666666666667, |
| "speedup": { |
| "mean": 0.6582839316051012, |
| "median": 0.6558193530744056, |
| "agg": 0.6582125891183973, |
| "n_match": 480 |
| } |
| } |
| }, |
| "speedup": { |
| "mean": 0.6576736551844512, |
| "median": 0.6554352371876343, |
| "agg": 0.6576507983816804, |
| "n_match": 1474 |
| } |
| }, |
| "final-qwen-q5-dspark-p0": { |
| "run": "final-qwen-q5-dspark-p0", |
| "target": "qwen-q5", |
| "family": "qwen", |
| "quant": "q5", |
| "drafter": "dspark-p0", |
| "k": 3, |
| "ctx": 8192, |
| "n": 1474, |
| "tok_per_s": { |
| "mean": 80.64056105834464, |
| "median": 80.6435, |
| "p95": 100.58834999999999 |
| }, |
| "alpha": { |
| "mean": 0.6133860922659431, |
| "median": 0.6134 |
| }, |
| "tau_mean": 162.07123473541384, |
| "ttft_ms": { |
| "mean": 79.55481682496607, |
| "median": 37.95, |
| "p95": 266.24499999999995 |
| }, |
| "vram_max_mib": 10130, |
| "power_max_w": 161.21, |
| "duration_s": 43.83, |
| "errors": 0, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 87.93867209302326, |
| "median": 87.176, |
| "p95": 103.6909 |
| }, |
| "alpha": { |
| "mean": 0.6988960465116278, |
| "median": 0.69215 |
| }, |
| "tau_mean": 170.9232558139535, |
| "speedup": { |
| "mean": 1.8745434770684581, |
| "median": 1.8641986168364166, |
| "agg": 1.8790458917452204, |
| "n_match": 430 |
| } |
| }, |
| "code": { |
| "n": 564, |
| "tok_per_s": { |
| "mean": 80.11799290780142, |
| "median": 79.6585, |
| "p95": 94.96770000000001 |
| }, |
| "alpha": { |
| "mean": 0.6089276595744681, |
| "median": 0.6052 |
| }, |
| "tau_mean": 163.13120567375887, |
| "speedup": { |
| "mean": 1.7150924740926523, |
| "median": 1.7078710466918818, |
| "agg": 1.7119360782367408, |
| "n_match": 564 |
| } |
| }, |
| "chat": { |
| "n": 480, |
| "tok_per_s": { |
| "mean": 74.7166875, |
| "median": 73.3435, |
| "p95": 98.82169999999999 |
| }, |
| "alpha": { |
| "mean": 0.5420220833333335, |
| "median": 0.5271 |
| }, |
| "tau_mean": 152.89583333333334, |
| "speedup": { |
| "mean": 1.5971444539851656, |
| "median": 1.5693763901165734, |
| "agg": 1.5965226827984476, |
| "n_match": 480 |
| } |
| } |
| }, |
| "speedup": { |
| "mean": 1.7231989066761009, |
| "median": 1.724499600708512, |
| "agg": 1.7231021501487256, |
| "n_match": 1474 |
| } |
| }, |
| "final-qwen-q5-eagle3": { |
| "run": "final-qwen-q5-eagle3", |
| "target": "qwen-q5", |
| "family": "qwen", |
| "quant": "q5", |
| "drafter": "eagle3", |
| "k": 3, |
| "ctx": 8192, |
| "n": 1474, |
| "tok_per_s": { |
| "mean": 68.33110108548169, |
| "median": 67.4005, |
| "p95": 89.91565 |
| }, |
| "alpha": { |
| "mean": 0.4393555630936229, |
| "median": 0.4281 |
| }, |
| "tau_mean": 140.3697421981004, |
| "ttft_ms": { |
| "mean": 78.1544776119403, |
| "median": 37.1, |
| "p95": 258.635 |
| }, |
| "vram_max_mib": 8902, |
| "power_max_w": 159.27, |
| "duration_s": 46.69, |
| "errors": 0, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 69.03269069767443, |
| "median": 68.809, |
| "p95": 80.3728 |
| }, |
| "alpha": { |
| "mean": 0.44576348837209306, |
| "median": 0.44275 |
| }, |
| "tau_mean": 144.36279069767443, |
| "speedup": { |
| "mean": 1.4714964823037155, |
| "median": 1.4654598663043426, |
| "agg": 1.4750688265382028, |
| "n_match": 430 |
| } |
| }, |
| "code": { |
| "n": 564, |
| "tok_per_s": { |
| "mean": 69.14667907801419, |
| "median": 67.2295, |
| "p95": 86.0588 |
| }, |
| "alpha": { |
| "mean": 0.44972446808510635, |
| "median": 0.4294 |
| }, |
| "tau_mean": 144.2464539007092, |
| "speedup": { |
| "mean": 1.4797767426011872, |
| "median": 1.44102914860788, |
| "agg": 1.4775044944040712, |
| "n_match": 564 |
| } |
| }, |
| "chat": { |
| "n": 480, |
| "tok_per_s": { |
| "mean": 66.74428958333333, |
| "median": 65.0515, |
| "p95": 97.40849999999996 |
| }, |
| "alpha": { |
| "mean": 0.4214316666666667, |
| "median": 0.4017 |
| }, |
| "tau_mean": 132.2375, |
| "speedup": { |
| "mean": 1.4267358444390625, |
| "median": 1.3892576501221157, |
| "agg": 1.42617099114652, |
| "n_match": 480 |
| } |
| } |
| }, |
| "speedup": { |
| "mean": 1.4600887215389535, |
| "median": 1.4391020701542812, |
| "agg": 1.4600774803294798, |
| "n_match": 1474 |
| } |
| }, |
| "final-qwen-q5-solo": { |
| "run": "final-qwen-q5-solo", |
| "target": "qwen-q5", |
| "family": "qwen", |
| "quant": "q5", |
| "drafter": "solo", |
| "k": 3, |
| "ctx": 8192, |
| "n": 1474, |
| "tok_per_s": { |
| "mean": 46.79964043419268, |
| "median": 46.951, |
| "p95": 47.038 |
| }, |
| "alpha": null, |
| "tau_mean": null, |
| "ttft_ms": { |
| "mean": 68.04660786974219, |
| "median": 33.8, |
| "p95": 220.17999999999992 |
| }, |
| "vram_max_mib": 7396, |
| "power_max_w": 164.2, |
| "duration_s": 59.33, |
| "errors": 0, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 46.91275813953488, |
| "median": 46.953, |
| "p95": 47.00555 |
| }, |
| "alpha": null, |
| "tau_mean": null |
| }, |
| "code": { |
| "n": 564, |
| "tok_per_s": { |
| "mean": 46.71516666666667, |
| "median": 46.852, |
| "p95": 47.04 |
| }, |
| "alpha": null, |
| "tau_mean": null |
| }, |
| "chat": { |
| "n": 480, |
| "tok_per_s": { |
| "mean": 46.797562500000005, |
| "median": 46.995, |
| "p95": 47.04205 |
| }, |
| "alpha": null, |
| "tau_mean": null |
| } |
| } |
| }, |
| "final-qwen-q8-dspark-p0": { |
| "run": "final-qwen-q8-dspark-p0", |
| "target": "qwen-q8", |
| "family": "qwen", |
| "quant": "q8", |
| "drafter": "dspark-p0", |
| "k": 3, |
| "ctx": 8192, |
| "n": 1474, |
| "tok_per_s": { |
| "mean": 62.331263907734055, |
| "median": 62.384, |
| "p95": 78.3239 |
| }, |
| "alpha": { |
| "mean": 0.6108656037991859, |
| "median": 0.6111 |
| }, |
| "tau_mean": 162.01085481682497, |
| "ttft_ms": { |
| "mean": 84.01906377204885, |
| "median": 47.9, |
| "p95": 250.26999999999978 |
| }, |
| "vram_max_mib": 12628, |
| "power_max_w": 138.36, |
| "duration_s": 57.82, |
| "errors": 0, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 68.43227209302326, |
| "median": 67.428, |
| "p95": 80.74814999999998 |
| }, |
| "alpha": { |
| "mean": 0.702976511627907, |
| "median": 0.6909000000000001 |
| }, |
| "tau_mean": 171.30232558139534, |
| "speedup": { |
| "mean": 2.112076473512988, |
| "median": 2.0797631263157514, |
| "agg": 2.1156820749266427, |
| "n_match": 430 |
| } |
| }, |
| "code": { |
| "n": 564, |
| "tok_per_s": { |
| "mean": 61.80902836879433, |
| "median": 61.689499999999995, |
| "p95": 71.42160000000001 |
| }, |
| "alpha": { |
| "mean": 0.6026237588652482, |
| "median": 0.6029 |
| }, |
| "tau_mean": 162.93262411347519, |
| "speedup": { |
| "mean": 1.913091765874063, |
| "median": 1.9143401886550064, |
| "agg": 1.9109149731391493, |
| "n_match": 564 |
| } |
| }, |
| "chat": { |
| "n": 480, |
| "tok_per_s": { |
| "mean": 57.47940416666667, |
| "median": 56.7975, |
| "p95": 77.8765 |
| }, |
| "alpha": { |
| "mean": 0.5380337500000001, |
| "median": 0.5271 |
| }, |
| "tau_mean": 152.60416666666666, |
| "speedup": { |
| "mean": 1.777518476338368, |
| "median": 1.7520308139870606, |
| "agg": 1.7770584163502956, |
| "n_match": 480 |
| } |
| } |
| }, |
| "speedup": { |
| "mean": 1.926991525241501, |
| "median": 1.9257494441741478, |
| "agg": 1.927060635628954, |
| "n_match": 1474 |
| } |
| }, |
| "final-qwen-q8-eagle3": { |
| "run": "final-qwen-q8-eagle3", |
| "target": "qwen-q8", |
| "family": "qwen", |
| "quant": "q8", |
| "drafter": "eagle3", |
| "k": 3, |
| "ctx": 8192, |
| "n": 1474, |
| "tok_per_s": { |
| "mean": 52.812307327001356, |
| "median": 52.283, |
| "p95": 68.29979999999999 |
| }, |
| "alpha": { |
| "mean": 0.43244274084124834, |
| "median": 0.4251 |
| }, |
| "tau_mean": 139.79511533242876, |
| "ttft_ms": { |
| "mean": 83.02605156037993, |
| "median": 47.6, |
| "p95": 244.90499999999997 |
| }, |
| "vram_max_mib": 11397, |
| "power_max_w": 151.27, |
| "duration_s": 52.99, |
| "errors": 0, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 53.497669767441856, |
| "median": 53.282, |
| "p95": 60.908899999999996 |
| }, |
| "alpha": { |
| "mean": 0.44095744186046515, |
| "median": 0.439 |
| }, |
| "tau_mean": 143.76511627906976, |
| "speedup": { |
| "mean": 1.6510905504366684, |
| "median": 1.6429804874117824, |
| "agg": 1.6539573729696597, |
| "n_match": 430 |
| } |
| }, |
| "code": { |
| "n": 564, |
| "tok_per_s": { |
| "mean": 53.40724113475177, |
| "median": 52.2595, |
| "p95": 66.3061 |
| }, |
| "alpha": { |
| "mean": 0.44163297872340423, |
| "median": 0.4239 |
| }, |
| "tau_mean": 143.55673758865248, |
| "speedup": { |
| "mean": 1.6527423498652336, |
| "median": 1.6151450800863136, |
| "agg": 1.6511616417833201, |
| "n_match": 564 |
| } |
| }, |
| "chat": { |
| "n": 480, |
| "tok_per_s": { |
| "mean": 51.49928958333333, |
| "median": 50.323, |
| "p95": 74.97179999999997 |
| }, |
| "alpha": { |
| "mean": 0.4140164583333333, |
| "median": 0.3954 |
| }, |
| "tau_mean": 131.81875, |
| "speedup": { |
| "mean": 1.592449508746356, |
| "median": 1.5527616166531488, |
| "agg": 1.5921745765624356, |
| "n_match": 480 |
| } |
| } |
| }, |
| "speedup": { |
| "mean": 1.6326264492605225, |
| "median": 1.6141883280926623, |
| "agg": 1.6327684077969575, |
| "n_match": 1474 |
| } |
| }, |
| "final-qwen-q8-solo": { |
| "run": "final-qwen-q8-solo", |
| "target": "qwen-q8", |
| "family": "qwen", |
| "quant": "q8", |
| "drafter": "solo", |
| "k": 3, |
| "ctx": 8192, |
| "n": 1474, |
| "tok_per_s": { |
| "mean": 32.345253052917236, |
| "median": 32.413, |
| "p95": 32.46 |
| }, |
| "alpha": null, |
| "tau_mean": null, |
| "ttft_ms": { |
| "mean": 70.65502035278155, |
| "median": 42.1, |
| "p95": 203.23999999999995 |
| }, |
| "vram_max_mib": 9881, |
| "power_max_w": 150.75, |
| "duration_s": 78.25, |
| "errors": 0, |
| "per_domain": { |
| "math": { |
| "n": 430, |
| "tok_per_s": { |
| "mean": 32.4010976744186, |
| "median": 32.416, |
| "p95": 32.44155 |
| }, |
| "alpha": null, |
| "tau_mean": null |
| }, |
| "code": { |
| "n": 564, |
| "tok_per_s": { |
| "mean": 32.308576241134745, |
| "median": 32.3695, |
| "p95": 32.46 |
| }, |
| "alpha": null, |
| "tau_mean": null |
| }, |
| "chat": { |
| "n": 480, |
| "tok_per_s": { |
| "mean": 32.338320833333334, |
| "median": 32.4315, |
| "p95": 32.461 |
| }, |
| "alpha": null, |
| "tau_mean": null |
| } |
| } |
| } |
| }, |
| "baselines": [ |
| { |
| "run": "baseline-ctx2048-gemma-q4-solo", |
| "target": "gemma-q4", |
| "ctx": 2048, |
| "prompt_path": "experiments/prompts/old/acc-sample.jsonl", |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "n_tokens": 256, |
| "sampling": { |
| "temperature": 0.0, |
| "top_k": 40, |
| "top_p": 0.95, |
| "seed": 42 |
| }, |
| "n": 175, |
| "excluded": 3, |
| "errors": 2, |
| "tok_per_s": { |
| "mean": 33.47058285714286, |
| "median": 32.657, |
| "p95": 34.8518 |
| }, |
| "tok_per_s_by_domain": { |
| "math": { |
| "mean": 32.767716666666665, |
| "median": 32.206, |
| "p95": 34.69645 |
| }, |
| "code": { |
| "mean": 32.50666666666667, |
| "median": 32.283, |
| "p95": 34.74505 |
| }, |
| "chat": { |
| "mean": 35.28889090909091, |
| "median": 34.711, |
| "p95": 35.876599999999996 |
| } |
| } |
| }, |
| { |
| "run": "baseline-ctx2048-qwen-q4-solo", |
| "target": "qwen-q4", |
| "ctx": 2048, |
| "prompt_path": "experiments/prompts/old/acc-sample.jsonl", |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "n_tokens": 256, |
| "sampling": { |
| "temperature": 0.0, |
| "top_k": 40, |
| "top_p": 0.95, |
| "seed": 42 |
| }, |
| "n": 178, |
| "excluded": 0, |
| "errors": 2, |
| "tok_per_s": { |
| "mean": 51.49112359550562, |
| "median": 50.039, |
| "p95": 53.575900000000004 |
| }, |
| "tok_per_s_by_domain": { |
| "math": { |
| "mean": 53.47833333333333, |
| "median": 53.4995, |
| "p95": 53.5811 |
| }, |
| "code": { |
| "mean": 51.10905000000001, |
| "median": 49.963, |
| "p95": 53.5932 |
| }, |
| "chat": { |
| "mean": 49.83063793103449, |
| "median": 49.902, |
| "p95": 50.011649999999996 |
| } |
| } |
| }, |
| { |
| "run": "baseline-ctx2048-qwen-q8-solo", |
| "target": "qwen-q8", |
| "ctx": 2048, |
| "prompt_path": "experiments/prompts/old/acc-sample.jsonl", |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "n_tokens": 256, |
| "sampling": { |
| "temperature": 0.0, |
| "top_k": 40, |
| "top_p": 0.95, |
| "seed": 42 |
| }, |
| "n": 178, |
| "excluded": 0, |
| "errors": 2, |
| "tok_per_s": { |
| "mean": 31.769084269662915, |
| "median": 32.2735, |
| "p95": 32.34115 |
| }, |
| "tok_per_s_by_domain": { |
| "math": { |
| "mean": 30.842750000000002, |
| "median": 30.124000000000002, |
| "p95": 32.3071 |
| }, |
| "code": { |
| "mean": 32.19831666666666, |
| "median": 32.2655, |
| "p95": 32.34105 |
| }, |
| "chat": { |
| "mean": 32.283327586206894, |
| "median": 32.317499999999995, |
| "p95": 32.345 |
| } |
| } |
| } |
| ], |
| "curves": { |
| "curves-gemma-q4-dflash-f16": { |
| "family": "curves", |
| "target": "gemma-q4", |
| "drafter": "dflash-f16", |
| "k": 3, |
| "n_excluded": 3, |
| "n_matched": 175, |
| "log_lines": 175, |
| "max_alpha_diff": 4.999999999999449e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8875333333333334, |
| "alpha_median": 0.8965000000000001, |
| "ci95_low": 0.8730825000000001, |
| "ci95_high": 0.9003516666666667 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7715, |
| "alpha_median": 0.7805, |
| "ci95_low": 0.7495995833333332, |
| "ci95_high": 0.7915512499999998 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6351833333333334, |
| "alpha_median": 0.634, |
| "ci95_low": 0.6084816666666666, |
| "ci95_high": 0.6601525 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8466833333333333, |
| "alpha_median": 0.86, |
| "ci95_low": 0.8261970833333333, |
| "ci95_high": 0.8677695833333332 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7134166666666667, |
| "alpha_median": 0.7144999999999999, |
| "ci95_low": 0.6827783333333333, |
| "ci95_high": 0.7451666666666666 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5915166666666668, |
| "alpha_median": 0.5774999999999999, |
| "ci95_low": 0.5538308333333334, |
| "ci95_high": 0.6293337499999999 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 55, |
| "alpha_mean": 0.6516363636363637, |
| "alpha_median": 0.673, |
| "ci95_low": 0.5917995454545455, |
| "ci95_high": 0.7055977272727272 |
| }, |
| { |
| "position": 2, |
| "n": 55, |
| "alpha_mean": 0.4742, |
| "alpha_median": 0.435, |
| "ci95_low": 0.41159909090909086, |
| "ci95_high": 0.5376786363636363 |
| }, |
| { |
| "position": 3, |
| "n": 55, |
| "alpha_mean": 0.322, |
| "alpha_median": 0.241, |
| "ci95_low": 0.26021772727272724, |
| "ci95_high": 0.38793636363636363 |
| } |
| ] |
| } |
| }, |
| "curves-gemma-q4-dflash-q4": { |
| "family": "curves", |
| "target": "gemma-q4", |
| "drafter": "dflash-q4", |
| "k": 3, |
| "n_excluded": 3, |
| "n_matched": 175, |
| "log_lines": 175, |
| "max_alpha_diff": 4.999999999999449e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8840500000000001, |
| "alpha_median": 0.894, |
| "ci95_low": 0.8702995833333335, |
| "ci95_high": 0.8966183333333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7654833333333334, |
| "alpha_median": 0.7855000000000001, |
| "ci95_low": 0.7415658333333334, |
| "ci95_high": 0.7874345833333332 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6332499999999999, |
| "alpha_median": 0.634, |
| "ci95_low": 0.6061233333333335, |
| "ci95_high": 0.6591541666666667 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8459833333333333, |
| "alpha_median": 0.8534999999999999, |
| "ci95_low": 0.8243808333333332, |
| "ci95_high": 0.8675191666666666 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7023333333333334, |
| "alpha_median": 0.6975, |
| "ci95_low": 0.6700754166666668, |
| "ci95_high": 0.7339170833333334 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.57765, |
| "alpha_median": 0.5865, |
| "ci95_low": 0.5409133333333332, |
| "ci95_high": 0.6155004166666667 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 55, |
| "alpha_mean": 0.6411454545454545, |
| "alpha_median": 0.692, |
| "ci95_low": 0.5844854545454545, |
| "ci95_high": 0.6936559090909092 |
| }, |
| { |
| "position": 2, |
| "n": 55, |
| "alpha_mean": 0.46087272727272727, |
| "alpha_median": 0.453, |
| "ci95_low": 0.4001595454545453, |
| "ci95_high": 0.5217354545454544 |
| }, |
| { |
| "position": 3, |
| "n": 55, |
| "alpha_mean": 0.3072727272727273, |
| "alpha_median": 0.25, |
| "ci95_low": 0.24769772727272724, |
| "ci95_high": 0.37022499999999986 |
| } |
| ] |
| } |
| }, |
| "curves-gemma-q4-dflash-q8": { |
| "family": "curves", |
| "target": "gemma-q4", |
| "drafter": "dflash-q8", |
| "k": 3, |
| "n_excluded": 3, |
| "n_matched": 175, |
| "log_lines": 175, |
| "max_alpha_diff": 4.999999999999449e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8857833333333335, |
| "alpha_median": 0.8965000000000001, |
| "ci95_low": 0.8707291666666668, |
| "ci95_high": 0.8987858333333334 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7701499999999999, |
| "alpha_median": 0.7805, |
| "ci95_low": 0.7480583333333333, |
| "ci95_high": 0.7899016666666666 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6345166666666666, |
| "alpha_median": 0.6365000000000001, |
| "ci95_low": 0.6085316666666667, |
| "ci95_high": 0.6598679166666667 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8447333333333333, |
| "alpha_median": 0.859, |
| "ci95_low": 0.8247666666666666, |
| "ci95_high": 0.8653 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7095166666666667, |
| "alpha_median": 0.7055, |
| "ci95_low": 0.6789491666666667, |
| "ci95_high": 0.7401 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.59055, |
| "alpha_median": 0.5774999999999999, |
| "ci95_low": 0.553065, |
| "ci95_high": 0.6284370833333334 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 55, |
| "alpha_mean": 0.6510727272727274, |
| "alpha_median": 0.67, |
| "ci95_low": 0.5907809090909091, |
| "ci95_high": 0.7066227272727273 |
| }, |
| { |
| "position": 2, |
| "n": 55, |
| "alpha_mean": 0.4744181818181819, |
| "alpha_median": 0.439, |
| "ci95_low": 0.4124404545454545, |
| "ci95_high": 0.5378390909090909 |
| }, |
| { |
| "position": 3, |
| "n": 55, |
| "alpha_mean": 0.32210909090909096, |
| "alpha_median": 0.239, |
| "ci95_low": 0.2609240909090909, |
| "ci95_high": 0.3875650000000001 |
| } |
| ] |
| } |
| }, |
| "curves-gemma-q4-mtp": { |
| "family": "curves", |
| "target": "gemma-q4", |
| "drafter": "mtp", |
| "k": 3, |
| "n_excluded": 3, |
| "n_matched": 175, |
| "log_lines": 175, |
| "max_alpha_diff": 4.999999999999449e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.9051000000000001, |
| "alpha_median": 0.912, |
| "ci95_low": 0.8935333333333334, |
| "ci95_high": 0.9165504166666667 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.8125499999999999, |
| "alpha_median": 0.8160000000000001, |
| "ci95_low": 0.7919820833333333, |
| "ci95_high": 0.8327345833333333 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.7374333333333333, |
| "alpha_median": 0.752, |
| "ci95_low": 0.71372625, |
| "ci95_high": 0.7597679166666668 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8862666666666669, |
| "alpha_median": 0.896, |
| "ci95_low": 0.8699804166666667, |
| "ci95_high": 0.9020004166666666 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7817166666666666, |
| "alpha_median": 0.7775000000000001, |
| "ci95_low": 0.7571645833333334, |
| "ci95_high": 0.8053525 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6927166666666665, |
| "alpha_median": 0.6885, |
| "ci95_low": 0.6627329166666668, |
| "ci95_high": 0.7230841666666667 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 55, |
| "alpha_mean": 0.7378363636363637, |
| "alpha_median": 0.786, |
| "ci95_low": 0.6822290909090908, |
| "ci95_high": 0.7893086363636364 |
| }, |
| { |
| "position": 2, |
| "n": 55, |
| "alpha_mean": 0.5864363636363636, |
| "alpha_median": 0.574, |
| "ci95_low": 0.5274227272727273, |
| "ci95_high": 0.6486104545454544 |
| }, |
| { |
| "position": 3, |
| "n": 55, |
| "alpha_mean": 0.48025454545454543, |
| "alpha_median": 0.436, |
| "ci95_low": 0.4184490909090909, |
| "ci95_high": 0.5485113636363635 |
| } |
| ] |
| } |
| }, |
| "curves-gemma-q5-dflash-f16": { |
| "family": "curves", |
| "target": "gemma-q5", |
| "drafter": "dflash-f16", |
| "k": 3, |
| "n_excluded": 4, |
| "n_matched": 174, |
| "log_lines": 174, |
| "max_alpha_diff": 5.0000000000050004e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8929333333333334, |
| "alpha_median": 0.905, |
| "ci95_low": 0.8811162500000003, |
| "ci95_high": 0.9041341666666669 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.78665, |
| "alpha_median": 0.797, |
| "ci95_low": 0.7672808333333334, |
| "ci95_high": 0.8055704166666667 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6724500000000002, |
| "alpha_median": 0.678, |
| "ci95_low": 0.6514829166666668, |
| "ci95_high": 0.6935520833333333 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.8492586206896553, |
| "alpha_median": 0.871, |
| "ci95_low": 0.815323275862069, |
| "ci95_high": 0.8769172413793103 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.7110172413793103, |
| "alpha_median": 0.7264999999999999, |
| "ci95_low": 0.6739077586206896, |
| "ci95_high": 0.7490047413793102 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.5873448275862071, |
| "alpha_median": 0.582, |
| "ci95_low": 0.5447491379310344, |
| "ci95_high": 0.6347806034482758 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 56, |
| "alpha_mean": 0.6457500000000002, |
| "alpha_median": 0.6975, |
| "ci95_low": 0.5908165178571428, |
| "ci95_high": 0.6954464285714286 |
| }, |
| { |
| "position": 2, |
| "n": 56, |
| "alpha_mean": 0.4306964285714286, |
| "alpha_median": 0.421, |
| "ci95_low": 0.3777625, |
| "ci95_high": 0.48348749999999996 |
| }, |
| { |
| "position": 3, |
| "n": 56, |
| "alpha_mean": 0.2739285714285714, |
| "alpha_median": 0.2405, |
| "ci95_low": 0.22422053571428568, |
| "ci95_high": 0.32529330357142855 |
| } |
| ] |
| } |
| }, |
| "curves-gemma-q5-mtp": { |
| "family": "curves", |
| "target": "gemma-q5", |
| "drafter": "mtp", |
| "k": 3, |
| "n_excluded": 4, |
| "n_matched": 174, |
| "log_lines": 174, |
| "max_alpha_diff": 4.999999999999449e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.9157500000000002, |
| "alpha_median": 0.917, |
| "ci95_low": 0.9051816666666667, |
| "ci95_high": 0.9253833333333332 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.8319833333333334, |
| "alpha_median": 0.838, |
| "ci95_low": 0.8126820833333335, |
| "ci95_high": 0.8497679166666667 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.7505833333333333, |
| "alpha_median": 0.757, |
| "ci95_low": 0.7291641666666666, |
| "ci95_high": 0.7722679166666668 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.8730517241379312, |
| "alpha_median": 0.8885000000000001, |
| "ci95_low": 0.8504306034482759, |
| "ci95_high": 0.893867672413793 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.7701206896551723, |
| "alpha_median": 0.7775000000000001, |
| "ci95_low": 0.7438866379310345, |
| "ci95_high": 0.7976043103448276 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.6722241379310345, |
| "alpha_median": 0.667, |
| "ci95_low": 0.6396025862068966, |
| "ci95_high": 0.7066556034482759 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 56, |
| "alpha_mean": 0.7448392857142857, |
| "alpha_median": 0.738, |
| "ci95_low": 0.7026580357142858, |
| "ci95_high": 0.7799857142857142 |
| }, |
| { |
| "position": 2, |
| "n": 56, |
| "alpha_mean": 0.5742857142857144, |
| "alpha_median": 0.572, |
| "ci95_low": 0.5256375000000001, |
| "ci95_high": 0.6192348214285716 |
| }, |
| { |
| "position": 3, |
| "n": 56, |
| "alpha_mean": 0.45112500000000005, |
| "alpha_median": 0.448, |
| "ci95_low": 0.39828526785714286, |
| "ci95_high": 0.5046437500000001 |
| } |
| ] |
| } |
| }, |
| "curves-gemma-q8-dflash-f16": { |
| "family": "curves", |
| "target": "gemma-q8", |
| "drafter": "dflash-f16", |
| "k": 3, |
| "n_excluded": 4, |
| "n_matched": 174, |
| "log_lines": 174, |
| "max_alpha_diff": 5.0000000000050004e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8918666666666668, |
| "alpha_median": 0.9045000000000001, |
| "ci95_low": 0.8784491666666668, |
| "ci95_high": 0.9039854166666665 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7721166666666668, |
| "alpha_median": 0.793, |
| "ci95_low": 0.7504645833333331, |
| "ci95_high": 0.7939008333333333 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6459500000000001, |
| "alpha_median": 0.6535, |
| "ci95_low": 0.6191287500000001, |
| "ci95_high": 0.6728866666666666 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.842655172413793, |
| "alpha_median": 0.8594999999999999, |
| "ci95_low": 0.8167754310344827, |
| "ci95_high": 0.8669663793103449 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.7027068965517241, |
| "alpha_median": 0.679, |
| "ci95_low": 0.6676331896551723, |
| "ci95_high": 0.7373814655172414 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.5888620689655173, |
| "alpha_median": 0.5605, |
| "ci95_low": 0.5459797413793104, |
| "ci95_high": 0.6324487068965516 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 56, |
| "alpha_mean": 0.6704821428571428, |
| "alpha_median": 0.672, |
| "ci95_low": 0.6234272321428571, |
| "ci95_high": 0.713859375 |
| }, |
| { |
| "position": 2, |
| "n": 56, |
| "alpha_mean": 0.43603571428571425, |
| "alpha_median": 0.4165, |
| "ci95_low": 0.3837392857142857, |
| "ci95_high": 0.49034910714285707 |
| }, |
| { |
| "position": 3, |
| "n": 56, |
| "alpha_mean": 0.2904642857142857, |
| "alpha_median": 0.2595, |
| "ci95_low": 0.2418611607142857, |
| "ci95_high": 0.3433772321428571 |
| } |
| ] |
| } |
| }, |
| "curves-gemma-q8-mtp": { |
| "family": "curves", |
| "target": "gemma-q8", |
| "drafter": "mtp", |
| "k": 3, |
| "n_excluded": 4, |
| "n_matched": 174, |
| "log_lines": 174, |
| "max_alpha_diff": 4.999999999999449e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.9138833333333335, |
| "alpha_median": 0.9285000000000001, |
| "ci95_low": 0.9010154166666666, |
| "ci95_high": 0.9263 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.8341500000000001, |
| "alpha_median": 0.848, |
| "ci95_low": 0.8140654166666669, |
| "ci95_high": 0.8542520833333334 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.7475666666666666, |
| "alpha_median": 0.7625, |
| "ci95_low": 0.7242616666666667, |
| "ci95_high": 0.7714366666666669 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.8691551724137931, |
| "alpha_median": 0.8654999999999999, |
| "ci95_low": 0.8525469827586206, |
| "ci95_high": 0.8869310344827587 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.7647413793103448, |
| "alpha_median": 0.7455, |
| "ci95_low": 0.7410340517241379, |
| "ci95_high": 0.7910892241379309 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.6648103448275863, |
| "alpha_median": 0.6515, |
| "ci95_low": 0.6358267241379308, |
| "ci95_high": 0.697656896551724 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 56, |
| "alpha_mean": 0.758267857142857, |
| "alpha_median": 0.7555000000000001, |
| "ci95_low": 0.7149366071428571, |
| "ci95_high": 0.7955558035714285 |
| }, |
| { |
| "position": 2, |
| "n": 56, |
| "alpha_mean": 0.5967142857142856, |
| "alpha_median": 0.587, |
| "ci95_low": 0.549794642857143, |
| "ci95_high": 0.641684375 |
| }, |
| { |
| "position": 3, |
| "n": 56, |
| "alpha_mean": 0.4818928571428572, |
| "alpha_median": 0.447, |
| "ci95_low": 0.42783482142857143, |
| "ci95_high": 0.5349348214285714 |
| } |
| ] |
| } |
| }, |
| "curves-qwen-q4-dflash": { |
| "family": "curves", |
| "target": "qwen-q4", |
| "drafter": "dflash-f16", |
| "k": 3, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.0000000000000565e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.019950000000000002, |
| "alpha_median": 0.016, |
| "ci95_low": 0.016049583333333332, |
| "ci95_high": 0.02385 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.0048000000000000004, |
| "alpha_median": 0.004, |
| "ci95_low": 0.0031995833333333334, |
| "ci95_high": 0.006933333333333334 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.0017666666666666668, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0010829166666666667, |
| "ci95_high": 0.0025333333333333336 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.021466666666666672, |
| "alpha_median": 0.02, |
| "ci95_low": 0.01824916666666667, |
| "ci95_high": 0.024750416666666664 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.004333333333333333, |
| "alpha_median": 0.004, |
| "ci95_low": 0.0033, |
| "ci95_high": 0.005383333333333334 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.0017500000000000003, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0011325, |
| "ci95_high": 0.0024500000000000004 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.028413793103448277, |
| "alpha_median": 0.0205, |
| "ci95_low": 0.023344396551724135, |
| "ci95_high": 0.033844827586206896 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.004206896551724138, |
| "alpha_median": 0.004, |
| "ci95_low": 0.003189655172413793, |
| "ci95_high": 0.005276293103448275 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.001603448275862069, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0009995689655172414, |
| "ci95_high": 0.0022413793103448275 |
| } |
| ] |
| } |
| }, |
| "curves-qwen-q4-dspark-p0": { |
| "family": "curves", |
| "target": "qwen-q4", |
| "drafter": "dspark-p0", |
| "k": 3, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 4.999999999999449e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8346833333333333, |
| "alpha_median": 0.851, |
| "ci95_low": 0.8120991666666666, |
| "ci95_high": 0.8569516666666667 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6921166666666666, |
| "alpha_median": 0.6975, |
| "ci95_low": 0.6581199999999999, |
| "ci95_high": 0.7272737499999999 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5783499999999999, |
| "alpha_median": 0.5645, |
| "ci95_low": 0.536965, |
| "ci95_high": 0.6212970833333332 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.7782833333333333, |
| "alpha_median": 0.7705, |
| "ci95_low": 0.7597149999999999, |
| "ci95_high": 0.7968850000000001 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6005333333333333, |
| "alpha_median": 0.593, |
| "ci95_low": 0.5761483333333334, |
| "ci95_high": 0.6253666666666666 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.4507166666666666, |
| "alpha_median": 0.432, |
| "ci95_low": 0.42258125, |
| "ci95_high": 0.48155291666666666 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.6920172413793104, |
| "alpha_median": 0.7335, |
| "ci95_low": 0.6443961206896551, |
| "ci95_high": 0.7338521551724138 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.4768793103448276, |
| "alpha_median": 0.4975, |
| "ci95_low": 0.4254219827586207, |
| "ci95_high": 0.5239168103448276 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.3412068965517242, |
| "alpha_median": 0.3415, |
| "ci95_low": 0.2928254310344828, |
| "ci95_high": 0.3870702586206896 |
| } |
| ] |
| } |
| }, |
| "curves-qwen-q4-dspark-p2": { |
| "family": "curves", |
| "target": "qwen-q4", |
| "drafter": "dspark-p2", |
| "k": 3, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 4.999999999999449e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8345333333333333, |
| "alpha_median": 0.851, |
| "ci95_low": 0.8120466666666666, |
| "ci95_high": 0.8567204166666667 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6918, |
| "alpha_median": 0.6975, |
| "ci95_low": 0.6576162499999999, |
| "ci95_high": 0.7269570833333333 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5782833333333331, |
| "alpha_median": 0.5645, |
| "ci95_low": 0.5370287499999999, |
| "ci95_high": 0.6215133333333334 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.7777999999999999, |
| "alpha_median": 0.7705, |
| "ci95_low": 0.7592299999999998, |
| "ci95_high": 0.7964358333333332 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6002666666666665, |
| "alpha_median": 0.591, |
| "ci95_low": 0.5757958333333333, |
| "ci95_high": 0.6250004166666666 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.45085, |
| "alpha_median": 0.432, |
| "ci95_low": 0.4225320833333335, |
| "ci95_high": 0.48181666666666667 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.6981206896551725, |
| "alpha_median": 0.7404999999999999, |
| "ci95_low": 0.6547439655172413, |
| "ci95_high": 0.737828448275862 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.4825, |
| "alpha_median": 0.5025, |
| "ci95_low": 0.43263318965517245, |
| "ci95_high": 0.5286422413793103 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.34689655172413797, |
| "alpha_median": 0.3605, |
| "ci95_low": 0.2989051724137931, |
| "ci95_high": 0.3935047413793103 |
| } |
| ] |
| } |
| }, |
| "curves-qwen-q4-dspark-p4": { |
| "family": "curves", |
| "target": "qwen-q4", |
| "drafter": "dspark-p4", |
| "k": 3, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.0000000000105516e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8353, |
| "alpha_median": 0.852, |
| "ci95_low": 0.8121125, |
| "ci95_high": 0.8580170833333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6887333333333332, |
| "alpha_median": 0.7, |
| "ci95_low": 0.6534300000000001, |
| "ci95_high": 0.7252525000000002 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5757499999999999, |
| "alpha_median": 0.556, |
| "ci95_low": 0.53395, |
| "ci95_high": 0.6201391666666666 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.7805666666666666, |
| "alpha_median": 0.789, |
| "ci95_low": 0.7626000000000002, |
| "ci95_high": 0.7979358333333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6028500000000001, |
| "alpha_median": 0.5945, |
| "ci95_low": 0.5796620833333332, |
| "ci95_high": 0.6280012500000001 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.44961666666666666, |
| "alpha_median": 0.4325, |
| "ci95_low": 0.4218633333333333, |
| "ci95_high": 0.47848999999999997 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.7250344827586207, |
| "alpha_median": 0.737, |
| "ci95_low": 0.6934771551724138, |
| "ci95_high": 0.7548646551724139 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.48151724137931035, |
| "alpha_median": 0.48, |
| "ci95_low": 0.43344655172413793, |
| "ci95_high": 0.5258487068965518 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.32974137931034475, |
| "alpha_median": 0.3185, |
| "ci95_low": 0.28190818965517245, |
| "ci95_high": 0.3738103448275862 |
| } |
| ] |
| } |
| }, |
| "curves-qwen-q4-dspark-p6": { |
| "family": "curves", |
| "target": "qwen-q4", |
| "drafter": "dspark-p6", |
| "k": 3, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.0000000000105516e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.84375, |
| "alpha_median": 0.8585, |
| "ci95_low": 0.8235158333333334, |
| "ci95_high": 0.86403375 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6838833333333333, |
| "alpha_median": 0.696, |
| "ci95_low": 0.6492825000000001, |
| "ci95_high": 0.7188870833333332 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5639166666666667, |
| "alpha_median": 0.543, |
| "ci95_low": 0.521315, |
| "ci95_high": 0.60840125 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.7965166666666668, |
| "alpha_median": 0.792, |
| "ci95_low": 0.7783312500000001, |
| "ci95_high": 0.8139354166666666 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.5829666666666667, |
| "alpha_median": 0.585, |
| "ci95_low": 0.5583320833333334, |
| "ci95_high": 0.6082204166666667 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.4116666666666667, |
| "alpha_median": 0.4155, |
| "ci95_low": 0.38426375, |
| "ci95_high": 0.4407420833333332 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.7730000000000001, |
| "alpha_median": 0.7805, |
| "ci95_low": 0.7447702586206895, |
| "ci95_high": 0.7983125000000001 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.448551724137931, |
| "alpha_median": 0.446, |
| "ci95_low": 0.403296551724138, |
| "ci95_high": 0.4906043103448276 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.2883448275862069, |
| "alpha_median": 0.27, |
| "ci95_low": 0.24225258620689655, |
| "ci95_high": 0.33291551724137936 |
| } |
| ] |
| } |
| }, |
| "curves-qwen-q4-eagle3": { |
| "family": "curves", |
| "target": "qwen-q4", |
| "drafter": "eagle3", |
| "k": 3, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.0000000000050004e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.65895, |
| "alpha_median": 0.641, |
| "ci95_low": 0.6398808333333335, |
| "ci95_high": 0.6790333333333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.4375, |
| "alpha_median": 0.434, |
| "ci95_low": 0.4135162499999999, |
| "ci95_high": 0.46195458333333334 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.23758333333333329, |
| "alpha_median": 0.2375, |
| "ci95_low": 0.21839874999999997, |
| "ci95_high": 0.25863375 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6724000000000001, |
| "alpha_median": 0.6715, |
| "ci95_low": 0.652665, |
| "ci95_high": 0.6924891666666666 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.43201666666666666, |
| "alpha_median": 0.4195, |
| "ci95_low": 0.40646541666666675, |
| "ci95_high": 0.45819 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.22091666666666668, |
| "alpha_median": 0.1965, |
| "ci95_low": 0.1931775, |
| "ci95_high": 0.2508708333333333 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.5734310344827588, |
| "alpha_median": 0.6125, |
| "ci95_low": 0.5173581896551724, |
| "ci95_high": 0.624796551724138 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.3631379310344828, |
| "alpha_median": 0.3525, |
| "ci95_low": 0.30766336206896555, |
| "ci95_high": 0.41653577586206897 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.21055172413793102, |
| "alpha_median": 0.165, |
| "ci95_low": 0.16535862068965515, |
| "ci95_high": 0.25851896551724135 |
| } |
| ] |
| } |
| }, |
| "curves-qwen-q4-vanilla17b": { |
| "family": "curves", |
| "target": "qwen-q4", |
| "drafter": "vanilla17b", |
| "k": 3, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 4.999999999999449e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8724666666666665, |
| "alpha_median": 0.87, |
| "ci95_low": 0.8594166666666668, |
| "ci95_high": 0.8848845833333332 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7706000000000002, |
| "alpha_median": 0.776, |
| "ci95_low": 0.7497649999999999, |
| "ci95_high": 0.7904025 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6854666666666667, |
| "alpha_median": 0.6905, |
| "ci95_low": 0.6592325, |
| "ci95_high": 0.7101183333333334 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8352166666666668, |
| "alpha_median": 0.829, |
| "ci95_low": 0.8234154166666667, |
| "ci95_high": 0.8467004166666667 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7095166666666667, |
| "alpha_median": 0.7055, |
| "ci95_low": 0.6907325, |
| "ci95_high": 0.7286520833333333 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6087000000000001, |
| "alpha_median": 0.6065, |
| "ci95_low": 0.58583125, |
| "ci95_high": 0.6329516666666665 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.7829482758620689, |
| "alpha_median": 0.7745, |
| "ci95_low": 0.7622922413793102, |
| "ci95_high": 0.803225 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.6254310344827587, |
| "alpha_median": 0.608, |
| "ci95_low": 0.5945271551724138, |
| "ci95_high": 0.6549883620689654 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.5063620689655173, |
| "alpha_median": 0.488, |
| "ci95_low": 0.46864137931034483, |
| "ci95_high": 0.5435073275862069 |
| } |
| ] |
| } |
| }, |
| "curves-qwen-q5-dflash": { |
| "family": "curves", |
| "target": "qwen-q5", |
| "drafter": "dflash-f16", |
| "k": 3, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.0000000000000565e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.01966666666666667, |
| "alpha_median": 0.016, |
| "ci95_low": 0.016115, |
| "ci95_high": 0.023299999999999998 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.0044666666666666665, |
| "alpha_median": 0.004, |
| "ci95_low": 0.0032166666666666667, |
| "ci95_high": 0.005866666666666667 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.0022166666666666667, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0014833333333333332, |
| "ci95_high": 0.00305 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.0203, |
| "alpha_median": 0.02, |
| "ci95_low": 0.017382083333333336, |
| "ci95_high": 0.023333749999999997 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.004783333333333334, |
| "alpha_median": 0.004, |
| "ci95_low": 0.00341625, |
| "ci95_high": 0.006250000000000001 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.0018666666666666666, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0012000000000000001, |
| "ci95_high": 0.0026000000000000003 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.02643103448275862, |
| "alpha_median": 0.024, |
| "ci95_low": 0.02167241379310345, |
| "ci95_high": 0.031810344827586205 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.00353448275862069, |
| "alpha_median": 0.004, |
| "ci95_low": 0.00256896551724138, |
| "ci95_high": 0.0044482758620689655 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.0013103448275862072, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0007586206896551724, |
| "ci95_high": 0.001931034482758621 |
| } |
| ] |
| } |
| }, |
| "curves-qwen-q5-dspark-p0": { |
| "family": "curves", |
| "target": "qwen-q5", |
| "drafter": "dspark-p0", |
| "k": 3, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 4.999999999999449e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8288666666666666, |
| "alpha_median": 0.832, |
| "ci95_low": 0.8069320833333334, |
| "ci95_high": 0.8511508333333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6884666666666668, |
| "alpha_median": 0.6895, |
| "ci95_low": 0.65485, |
| "ci95_high": 0.7222670833333332 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.58035, |
| "alpha_median": 0.5834999999999999, |
| "ci95_low": 0.53963125, |
| "ci95_high": 0.62223375 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.7708666666666666, |
| "alpha_median": 0.7685, |
| "ci95_low": 0.7531162500000002, |
| "ci95_high": 0.7895529166666666 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.5833333333333334, |
| "alpha_median": 0.5814999999999999, |
| "ci95_low": 0.5556662500000001, |
| "ci95_high": 0.61250375 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.43556666666666666, |
| "alpha_median": 0.4285, |
| "ci95_low": 0.4027470833333333, |
| "ci95_high": 0.4700704166666666 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.701741379310345, |
| "alpha_median": 0.7355, |
| "ci95_low": 0.6575254310344828, |
| "ci95_high": 0.7423306034482758 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.4863793103448276, |
| "alpha_median": 0.4925, |
| "ci95_low": 0.43637887931034475, |
| "ci95_high": 0.5348375 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.3533965517241379, |
| "alpha_median": 0.3285, |
| "ci95_low": 0.30560000000000004, |
| "ci95_high": 0.40571034482758617 |
| } |
| ] |
| } |
| }, |
| "curves-qwen-q5-eagle3": { |
| "family": "curves", |
| "target": "qwen-q5", |
| "drafter": "eagle3", |
| "k": 3, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.000000000000837e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6617833333333335, |
| "alpha_median": 0.6515, |
| "ci95_low": 0.6442812500000001, |
| "ci95_high": 0.6797862499999999 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.4344, |
| "alpha_median": 0.429, |
| "ci95_low": 0.41286291666666664, |
| "ci95_high": 0.4559833333333333 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.23328333333333337, |
| "alpha_median": 0.2145, |
| "ci95_low": 0.21618, |
| "ci95_high": 0.2505683333333333 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6694, |
| "alpha_median": 0.6685000000000001, |
| "ci95_low": 0.6487329166666667, |
| "ci95_high": 0.69080125 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.4252666666666667, |
| "alpha_median": 0.4245, |
| "ci95_low": 0.39917749999999996, |
| "ci95_high": 0.45331958333333333 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.21823333333333333, |
| "alpha_median": 0.196, |
| "ci95_low": 0.19128250000000002, |
| "ci95_high": 0.24852166666666667 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.5931379310344828, |
| "alpha_median": 0.6385000000000001, |
| "ci95_low": 0.5417918103448275, |
| "ci95_high": 0.643225 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.36172413793103453, |
| "alpha_median": 0.362, |
| "ci95_low": 0.3137870689655172, |
| "ci95_high": 0.41031422413793095 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.20020689655172413, |
| "alpha_median": 0.163, |
| "ci95_low": 0.16113534482758618, |
| "ci95_high": 0.24240387931034482 |
| } |
| ] |
| } |
| }, |
| "curves-qwen-q8-dspark-p0": { |
| "family": "curves", |
| "target": "qwen-q8", |
| "drafter": "dspark-p0", |
| "k": 3, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.000000000002225e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8263666666666667, |
| "alpha_median": 0.8365, |
| "ci95_low": 0.8036991666666665, |
| "ci95_high": 0.8494 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6871833333333334, |
| "alpha_median": 0.692, |
| "ci95_low": 0.6528645833333334, |
| "ci95_high": 0.7224837499999999 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5735833333333334, |
| "alpha_median": 0.5814999999999999, |
| "ci95_low": 0.5308795833333334, |
| "ci95_high": 0.61690375 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.77365, |
| "alpha_median": 0.772, |
| "ci95_low": 0.7569454166666667, |
| "ci95_high": 0.7908845833333332 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.5854666666666666, |
| "alpha_median": 0.579, |
| "ci95_low": 0.5615266666666666, |
| "ci95_high": 0.6115545833333333 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.44101666666666667, |
| "alpha_median": 0.4405, |
| "ci95_low": 0.41631583333333333, |
| "ci95_high": 0.46805041666666664 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.6959827586206897, |
| "alpha_median": 0.721, |
| "ci95_low": 0.6521508620689656, |
| "ci95_high": 0.7347081896551725 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.4865862068965517, |
| "alpha_median": 0.483, |
| "ci95_low": 0.441, |
| "ci95_high": 0.5306064655172413 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.3454137931034482, |
| "alpha_median": 0.3235, |
| "ci95_low": 0.3001, |
| "ci95_high": 0.39220775862068963 |
| } |
| ] |
| } |
| }, |
| "curves-qwen-q8-eagle3": { |
| "family": "curves", |
| "target": "qwen-q8", |
| "drafter": "eagle3", |
| "k": 3, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.0000000000050004e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6503833333333334, |
| "alpha_median": 0.6445000000000001, |
| "ci95_low": 0.6334787499999999, |
| "ci95_high": 0.6669016666666667 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.4284833333333333, |
| "alpha_median": 0.418, |
| "ci95_low": 0.41021583333333345, |
| "ci95_high": 0.4492220833333333 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.23021666666666665, |
| "alpha_median": 0.2345, |
| "ci95_low": 0.21416541666666666, |
| "ci95_high": 0.24763791666666662 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6651166666666666, |
| "alpha_median": 0.6535, |
| "ci95_low": 0.6472791666666666, |
| "ci95_high": 0.6835170833333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.42485, |
| "alpha_median": 0.4165, |
| "ci95_low": 0.40288041666666663, |
| "ci95_high": 0.4486516666666667 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.2141, |
| "alpha_median": 0.189, |
| "ci95_low": 0.19004458333333332, |
| "ci95_high": 0.24040083333333337 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.5834310344827586, |
| "alpha_median": 0.637, |
| "ci95_low": 0.5310758620689655, |
| "ci95_high": 0.6309508620689654 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.3570862068965517, |
| "alpha_median": 0.372, |
| "ci95_low": 0.3111849137931035, |
| "ci95_high": 0.4010762931034482 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.1905, |
| "alpha_median": 0.18, |
| "ci95_low": 0.15682155172413792, |
| "ci95_high": 0.22692456896551721 |
| } |
| ] |
| } |
| }, |
| "ksweep-gemma-q4-dflash-f16-k10": { |
| "family": "ksweep", |
| "target": "gemma-q4", |
| "drafter": "dflash-f16", |
| "k": 10, |
| "n_excluded": 3, |
| "n_matched": 175, |
| "log_lines": 175, |
| "max_alpha_diff": 5.0000000000050004e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8581333333333333, |
| "alpha_median": 0.86, |
| "ci95_low": 0.8396154166666667, |
| "ci95_high": 0.8742508333333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7290666666666668, |
| "alpha_median": 0.7255, |
| "ci95_low": 0.70286625, |
| "ci95_high": 0.7540858333333336 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.58595, |
| "alpha_median": 0.5934999999999999, |
| "ci95_low": 0.5582470833333333, |
| "ci95_high": 0.6129883333333332 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.4879833333333333, |
| "alpha_median": 0.471, |
| "ci95_low": 0.4562466666666666, |
| "ci95_high": 0.51908375 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.3957166666666667, |
| "alpha_median": 0.375, |
| "ci95_low": 0.36298125, |
| "ci95_high": 0.42700749999999993 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.3183333333333333, |
| "alpha_median": 0.303, |
| "ci95_low": 0.28596375000000007, |
| "ci95_high": 0.3491191666666666 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.2571666666666667, |
| "alpha_median": 0.2255, |
| "ci95_low": 0.2281825, |
| "ci95_high": 0.284435 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.2046666666666667, |
| "alpha_median": 0.184, |
| "ci95_low": 0.17776083333333334, |
| "ci95_high": 0.23036874999999998 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.1633666666666667, |
| "alpha_median": 0.1615, |
| "ci95_low": 0.14023, |
| "ci95_high": 0.1857675 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.119, |
| "alpha_median": 0.10200000000000001, |
| "ci95_low": 0.09944916666666667, |
| "ci95_high": 0.13833458333333334 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.7975833333333332, |
| "alpha_median": 0.805, |
| "ci95_low": 0.7638662500000001, |
| "ci95_high": 0.8274833333333335 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6294500000000001, |
| "alpha_median": 0.6485000000000001, |
| "ci95_low": 0.5893558333333334, |
| "ci95_high": 0.66490375 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.4983833333333333, |
| "alpha_median": 0.492, |
| "ci95_low": 0.45793291666666663, |
| "ci95_high": 0.5402670833333332 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.3997666666666666, |
| "alpha_median": 0.3985, |
| "ci95_low": 0.3604479166666667, |
| "ci95_high": 0.44135541666666667 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.3185666666666666, |
| "alpha_median": 0.2845, |
| "ci95_low": 0.28028624999999996, |
| "ci95_high": 0.35915333333333327 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.25938333333333335, |
| "alpha_median": 0.225, |
| "ci95_low": 0.22559875, |
| "ci95_high": 0.29620625 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.20616666666666666, |
| "alpha_median": 0.176, |
| "ci95_low": 0.175915, |
| "ci95_high": 0.24012041666666661 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.16380000000000003, |
| "alpha_median": 0.1355, |
| "ci95_low": 0.13546458333333336, |
| "ci95_high": 0.19675666666666658 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.13241666666666668, |
| "alpha_median": 0.0935, |
| "ci95_low": 0.10526500000000001, |
| "ci95_high": 0.16395499999999996 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.10470000000000002, |
| "alpha_median": 0.0765, |
| "ci95_low": 0.08091625000000001, |
| "ci95_high": 0.13463999999999995 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 55, |
| "alpha_mean": 0.6401454545454547, |
| "alpha_median": 0.667, |
| "ci95_low": 0.5908345454545455, |
| "ci95_high": 0.6839649999999998 |
| }, |
| { |
| "position": 2, |
| "n": 55, |
| "alpha_mean": 0.4230181818181819, |
| "alpha_median": 0.422, |
| "ci95_low": 0.36814090909090913, |
| "ci95_high": 0.47485499999999997 |
| }, |
| { |
| "position": 3, |
| "n": 55, |
| "alpha_mean": 0.2565090909090909, |
| "alpha_median": 0.206, |
| "ci95_low": 0.21030545454545452, |
| "ci95_high": 0.3052913636363636 |
| }, |
| { |
| "position": 4, |
| "n": 55, |
| "alpha_mean": 0.1807272727272727, |
| "alpha_median": 0.137, |
| "ci95_low": 0.13614363636363633, |
| "ci95_high": 0.2264454545454545 |
| }, |
| { |
| "position": 5, |
| "n": 55, |
| "alpha_mean": 0.1324, |
| "alpha_median": 0.082, |
| "ci95_low": 0.09220954545454546, |
| "ci95_high": 0.17708409090909083 |
| }, |
| { |
| "position": 6, |
| "n": 55, |
| "alpha_mean": 0.09461818181818181, |
| "alpha_median": 0.041, |
| "ci95_low": 0.05852681818181819, |
| "ci95_high": 0.13318727272727268 |
| }, |
| { |
| "position": 7, |
| "n": 55, |
| "alpha_mean": 0.07121818181818182, |
| "alpha_median": 0.02, |
| "ci95_low": 0.03836272727272727, |
| "ci95_high": 0.10632727272727273 |
| }, |
| { |
| "position": 8, |
| "n": 55, |
| "alpha_mean": 0.05894545454545455, |
| "alpha_median": 0.011, |
| "ci95_low": 0.029181818181818187, |
| "ci95_high": 0.09329181818181818 |
| }, |
| { |
| "position": 9, |
| "n": 55, |
| "alpha_mean": 0.04472727272727272, |
| "alpha_median": 0.009, |
| "ci95_low": 0.018727272727272728, |
| "ci95_high": 0.07535272727272724 |
| }, |
| { |
| "position": 10, |
| "n": 55, |
| "alpha_mean": 0.03549090909090909, |
| "alpha_median": 0.0, |
| "ci95_low": 0.013397727272727275, |
| "ci95_high": 0.06263681818181818 |
| } |
| ] |
| } |
| }, |
| "ksweep-gemma-q4-dflash-f16-k5": { |
| "family": "ksweep", |
| "target": "gemma-q4", |
| "drafter": "dflash-f16", |
| "k": 5, |
| "n_excluded": 3, |
| "n_matched": 175, |
| "log_lines": 175, |
| "max_alpha_diff": 4.999999999999449e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8648166666666665, |
| "alpha_median": 0.8805000000000001, |
| "ci95_low": 0.84664875, |
| "ci95_high": 0.8817375 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7476166666666665, |
| "alpha_median": 0.754, |
| "ci95_low": 0.7237820833333334, |
| "ci95_high": 0.7708 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6148666666666667, |
| "alpha_median": 0.618, |
| "ci95_low": 0.5844333333333334, |
| "ci95_high": 0.6425420833333334 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.5182333333333334, |
| "alpha_median": 0.504, |
| "ci95_low": 0.48459916666666675, |
| "ci95_high": 0.5506170833333333 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.43478333333333324, |
| "alpha_median": 0.437, |
| "ci95_low": 0.40129875000000004, |
| "ci95_high": 0.4672016666666667 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8290833333333333, |
| "alpha_median": 0.831, |
| "ci95_low": 0.8066158333333333, |
| "ci95_high": 0.8515845833333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6821666666666667, |
| "alpha_median": 0.68, |
| "ci95_low": 0.6523333333333334, |
| "ci95_high": 0.7136345833333332 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5617, |
| "alpha_median": 0.5665, |
| "ci95_low": 0.5239529166666667, |
| "ci95_high": 0.5996862499999999 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.46573333333333333, |
| "alpha_median": 0.462, |
| "ci95_low": 0.42466125, |
| "ci95_high": 0.5074033333333333 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.39039999999999997, |
| "alpha_median": 0.377, |
| "ci95_low": 0.34859750000000006, |
| "ci95_high": 0.4326854166666667 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 55, |
| "alpha_mean": 0.6564363636363636, |
| "alpha_median": 0.68, |
| "ci95_low": 0.6045904545454546, |
| "ci95_high": 0.7042477272727272 |
| }, |
| { |
| "position": 2, |
| "n": 55, |
| "alpha_mean": 0.4465636363636364, |
| "alpha_median": 0.413, |
| "ci95_low": 0.39149863636363635, |
| "ci95_high": 0.5005781818181817 |
| }, |
| { |
| "position": 3, |
| "n": 55, |
| "alpha_mean": 0.2991818181818182, |
| "alpha_median": 0.25, |
| "ci95_low": 0.24532045454545454, |
| "ci95_high": 0.35314772727272725 |
| }, |
| { |
| "position": 4, |
| "n": 55, |
| "alpha_mean": 0.2236545454545455, |
| "alpha_median": 0.149, |
| "ci95_low": 0.1724536363636364, |
| "ci95_high": 0.2765854545454545 |
| }, |
| { |
| "position": 5, |
| "n": 55, |
| "alpha_mean": 0.1691272727272728, |
| "alpha_median": 0.106, |
| "ci95_low": 0.11848954545454544, |
| "ci95_high": 0.2225631818181817 |
| } |
| ] |
| } |
| }, |
| "ksweep-gemma-q4-dflash-f16-k7": { |
| "family": "ksweep", |
| "target": "gemma-q4", |
| "drafter": "dflash-f16", |
| "k": 7, |
| "n_excluded": 3, |
| "n_matched": 175, |
| "log_lines": 175, |
| "max_alpha_diff": 5.0000000000050004e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8579166666666665, |
| "alpha_median": 0.867, |
| "ci95_low": 0.8395308333333333, |
| "ci95_high": 0.8751016666666668 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7313999999999999, |
| "alpha_median": 0.7404999999999999, |
| "ci95_low": 0.7069320833333336, |
| "ci95_high": 0.7555179166666667 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5955666666666666, |
| "alpha_median": 0.605, |
| "ci95_low": 0.5640833333333333, |
| "ci95_high": 0.6246870833333333 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.49323333333333336, |
| "alpha_median": 0.482, |
| "ci95_low": 0.45769624999999997, |
| "ci95_high": 0.5276183333333333 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.4169, |
| "alpha_median": 0.391, |
| "ci95_low": 0.38221625000000004, |
| "ci95_high": 0.4508516666666667 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.35129999999999995, |
| "alpha_median": 0.333, |
| "ci95_low": 0.3176629166666667, |
| "ci95_high": 0.38515333333333324 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.28978333333333334, |
| "alpha_median": 0.2665, |
| "ci95_low": 0.2591608333333334, |
| "ci95_high": 0.32101874999999996 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8238166666666665, |
| "alpha_median": 0.823, |
| "ci95_low": 0.7995483333333334, |
| "ci95_high": 0.8470179166666668 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6661166666666667, |
| "alpha_median": 0.667, |
| "ci95_low": 0.633095, |
| "ci95_high": 0.6982841666666669 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5352166666666667, |
| "alpha_median": 0.534, |
| "ci95_low": 0.49966374999999996, |
| "ci95_high": 0.5731175000000002 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.4425833333333334, |
| "alpha_median": 0.441, |
| "ci95_low": 0.4045970833333334, |
| "ci95_high": 0.48081 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.36938333333333334, |
| "alpha_median": 0.3695, |
| "ci95_low": 0.33084708333333335, |
| "ci95_high": 0.40810291666666665 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.3013, |
| "alpha_median": 0.2845, |
| "ci95_low": 0.26406583333333333, |
| "ci95_high": 0.33970374999999986 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.24645, |
| "alpha_median": 0.2235, |
| "ci95_low": 0.2119533333333333, |
| "ci95_high": 0.2845670833333334 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 55, |
| "alpha_mean": 0.6426181818181819, |
| "alpha_median": 0.658, |
| "ci95_low": 0.5934681818181818, |
| "ci95_high": 0.6863299999999998 |
| }, |
| { |
| "position": 2, |
| "n": 55, |
| "alpha_mean": 0.4292727272727273, |
| "alpha_median": 0.409, |
| "ci95_low": 0.3773263636363636, |
| "ci95_high": 0.4804581818181817 |
| }, |
| { |
| "position": 3, |
| "n": 55, |
| "alpha_mean": 0.2806, |
| "alpha_median": 0.242, |
| "ci95_low": 0.23070636363636363, |
| "ci95_high": 0.33234681818181816 |
| }, |
| { |
| "position": 4, |
| "n": 55, |
| "alpha_mean": 0.20427272727272722, |
| "alpha_median": 0.151, |
| "ci95_low": 0.15617499999999998, |
| "ci95_high": 0.2536563636363636 |
| }, |
| { |
| "position": 5, |
| "n": 55, |
| "alpha_mean": 0.1530363636363636, |
| "alpha_median": 0.087, |
| "ci95_low": 0.10728318181818182, |
| "ci95_high": 0.20192045454545446 |
| }, |
| { |
| "position": 6, |
| "n": 55, |
| "alpha_mean": 0.12018181818181817, |
| "alpha_median": 0.051, |
| "ci95_low": 0.07734409090909093, |
| "ci95_high": 0.1675277272727273 |
| }, |
| { |
| "position": 7, |
| "n": 55, |
| "alpha_mean": 0.09421818181818181, |
| "alpha_median": 0.028, |
| "ci95_low": 0.052829999999999995, |
| "ci95_high": 0.14032772727272727 |
| } |
| ] |
| } |
| }, |
| "ksweep-gemma-q4-mtp-k10": { |
| "family": "ksweep", |
| "target": "gemma-q4", |
| "drafter": "mtp", |
| "k": 10, |
| "n_excluded": 3, |
| "n_matched": 175, |
| "log_lines": 175, |
| "max_alpha_diff": 5.0000000000105516e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8836833333333335, |
| "alpha_median": 0.8905000000000001, |
| "ci95_low": 0.8681633333333334, |
| "ci95_high": 0.8991691666666666 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7941166666666667, |
| "alpha_median": 0.8, |
| "ci95_low": 0.7705591666666666, |
| "ci95_high": 0.8170041666666669 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.7047999999999999, |
| "alpha_median": 0.7195, |
| "ci95_low": 0.6783975000000001, |
| "ci95_high": 0.7299704166666666 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.6122833333333333, |
| "alpha_median": 0.6385000000000001, |
| "ci95_low": 0.5747329166666666, |
| "ci95_high": 0.6443341666666665 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.5341833333333332, |
| "alpha_median": 0.549, |
| "ci95_low": 0.49784874999999995, |
| "ci95_high": 0.5669670833333333 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.45953333333333324, |
| "alpha_median": 0.463, |
| "ci95_low": 0.42431375, |
| "ci95_high": 0.49294 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.3834, |
| "alpha_median": 0.3735, |
| "ci95_low": 0.3501558333333333, |
| "ci95_high": 0.4155025 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.3148833333333334, |
| "alpha_median": 0.3115, |
| "ci95_low": 0.2848829166666666, |
| "ci95_high": 0.3450333333333333 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.25093333333333334, |
| "alpha_median": 0.2415, |
| "ci95_low": 0.22318125000000003, |
| "ci95_high": 0.28111875000000003 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.20245, |
| "alpha_median": 0.2, |
| "ci95_low": 0.17943208333333335, |
| "ci95_high": 0.22818499999999997 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8295333333333333, |
| "alpha_median": 0.8434999999999999, |
| "ci95_low": 0.80103125, |
| "ci95_high": 0.8527508333333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7212499999999998, |
| "alpha_median": 0.732, |
| "ci95_low": 0.6896574999999999, |
| "ci95_high": 0.7506525000000001 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6143499999999998, |
| "alpha_median": 0.604, |
| "ci95_low": 0.5809475, |
| "ci95_high": 0.6462687500000001 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.5350666666666667, |
| "alpha_median": 0.523, |
| "ci95_low": 0.5002779166666668, |
| "ci95_high": 0.5712199999999998 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.46813333333333335, |
| "alpha_median": 0.4495, |
| "ci95_low": 0.43272666666666665, |
| "ci95_high": 0.5032670833333333 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.39171666666666666, |
| "alpha_median": 0.377, |
| "ci95_low": 0.35819833333333334, |
| "ci95_high": 0.4262558333333332 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.3330666666666667, |
| "alpha_median": 0.309, |
| "ci95_low": 0.2998495833333334, |
| "ci95_high": 0.36738375 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.27935, |
| "alpha_median": 0.25, |
| "ci95_low": 0.2476816666666667, |
| "ci95_high": 0.3136854166666667 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.23356666666666667, |
| "alpha_median": 0.214, |
| "ci95_low": 0.2039, |
| "ci95_high": 0.2661545833333333 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.18168333333333336, |
| "alpha_median": 0.159, |
| "ci95_low": 0.1529316666666667, |
| "ci95_high": 0.21411833333333333 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 55, |
| "alpha_mean": 0.7337636363636365, |
| "alpha_median": 0.747, |
| "ci95_low": 0.6890713636363638, |
| "ci95_high": 0.7732472727272728 |
| }, |
| { |
| "position": 2, |
| "n": 55, |
| "alpha_mean": 0.5758181818181819, |
| "alpha_median": 0.569, |
| "ci95_low": 0.5250227272727273, |
| "ci95_high": 0.6269495454545455 |
| }, |
| { |
| "position": 3, |
| "n": 55, |
| "alpha_mean": 0.44114545454545445, |
| "alpha_median": 0.388, |
| "ci95_low": 0.384850909090909, |
| "ci95_high": 0.5014190909090909 |
| }, |
| { |
| "position": 4, |
| "n": 55, |
| "alpha_mean": 0.34834545454545457, |
| "alpha_median": 0.277, |
| "ci95_low": 0.28816, |
| "ci95_high": 0.4101668181818182 |
| }, |
| { |
| "position": 5, |
| "n": 55, |
| "alpha_mean": 0.26996363636363635, |
| "alpha_median": 0.192, |
| "ci95_low": 0.21188772727272728, |
| "ci95_high": 0.3302231818181817 |
| }, |
| { |
| "position": 6, |
| "n": 55, |
| "alpha_mean": 0.22503636363636365, |
| "alpha_median": 0.139, |
| "ci95_low": 0.1671940909090909, |
| "ci95_high": 0.28587045454545446 |
| }, |
| { |
| "position": 7, |
| "n": 55, |
| "alpha_mean": 0.19132727272727276, |
| "alpha_median": 0.092, |
| "ci95_low": 0.13407136363636363, |
| "ci95_high": 0.25535454545454545 |
| }, |
| { |
| "position": 8, |
| "n": 55, |
| "alpha_mean": 0.16399999999999998, |
| "alpha_median": 0.069, |
| "ci95_low": 0.10636363636363637, |
| "ci95_high": 0.22853 |
| }, |
| { |
| "position": 9, |
| "n": 55, |
| "alpha_mean": 0.1394, |
| "alpha_median": 0.043, |
| "ci95_low": 0.08196227272727274, |
| "ci95_high": 0.20359090909090902 |
| }, |
| { |
| "position": 10, |
| "n": 55, |
| "alpha_mean": 0.12441818181818183, |
| "alpha_median": 0.028, |
| "ci95_low": 0.065955, |
| "ci95_high": 0.18841636363636347 |
| } |
| ] |
| } |
| }, |
| "ksweep-gemma-q4-mtp-k5": { |
| "family": "ksweep", |
| "target": "gemma-q4", |
| "drafter": "mtp", |
| "k": 5, |
| "n_excluded": 3, |
| "n_matched": 175, |
| "log_lines": 175, |
| "max_alpha_diff": 5.0000000000050004e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8950166666666667, |
| "alpha_median": 0.8975, |
| "ci95_low": 0.8799124999999999, |
| "ci95_high": 0.9086012499999998 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.8061833333333333, |
| "alpha_median": 0.8085, |
| "ci95_low": 0.7858991666666666, |
| "ci95_high": 0.8246529166666665 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.7245333333333333, |
| "alpha_median": 0.743, |
| "ci95_low": 0.6977758333333333, |
| "ci95_high": 0.7487174999999999 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.6364333333333334, |
| "alpha_median": 0.653, |
| "ci95_low": 0.6066799999999999, |
| "ci95_high": 0.6637341666666666 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.5607833333333334, |
| "alpha_median": 0.56, |
| "ci95_low": 0.5305791666666666, |
| "ci95_high": 0.5900512499999999 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8677666666666666, |
| "alpha_median": 0.885, |
| "ci95_low": 0.8488662499999999, |
| "ci95_high": 0.88593375 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7681333333333333, |
| "alpha_median": 0.774, |
| "ci95_low": 0.7418133333333332, |
| "ci95_high": 0.7935254166666666 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6667833333333332, |
| "alpha_median": 0.661, |
| "ci95_low": 0.6357720833333335, |
| "ci95_high": 0.6976683333333333 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.5806333333333333, |
| "alpha_median": 0.568, |
| "ci95_low": 0.544045, |
| "ci95_high": 0.6166670833333333 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.5109833333333332, |
| "alpha_median": 0.5, |
| "ci95_low": 0.47249791666666663, |
| "ci95_high": 0.5504029166666666 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 55, |
| "alpha_mean": 0.7516363636363635, |
| "alpha_median": 0.757, |
| "ci95_low": 0.7077790909090909, |
| "ci95_high": 0.7898595454545454 |
| }, |
| { |
| "position": 2, |
| "n": 55, |
| "alpha_mean": 0.5932545454545456, |
| "alpha_median": 0.585, |
| "ci95_low": 0.5417622727272728, |
| "ci95_high": 0.6486572727272726 |
| }, |
| { |
| "position": 3, |
| "n": 55, |
| "alpha_mean": 0.48161818181818183, |
| "alpha_median": 0.412, |
| "ci95_low": 0.42208954545454547, |
| "ci95_high": 0.5441836363636364 |
| }, |
| { |
| "position": 4, |
| "n": 55, |
| "alpha_mean": 0.39509090909090916, |
| "alpha_median": 0.325, |
| "ci95_low": 0.3306468181818182, |
| "ci95_high": 0.46265500000000004 |
| }, |
| { |
| "position": 5, |
| "n": 55, |
| "alpha_mean": 0.32367272727272733, |
| "alpha_median": 0.219, |
| "ci95_low": 0.25792818181818183, |
| "ci95_high": 0.3945495454545453 |
| } |
| ] |
| } |
| }, |
| "ksweep-gemma-q4-mtp-k7": { |
| "family": "ksweep", |
| "target": "gemma-q4", |
| "drafter": "mtp", |
| "k": 7, |
| "n_excluded": 3, |
| "n_matched": 175, |
| "log_lines": 175, |
| "max_alpha_diff": 5.0000000000050004e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8807333333333333, |
| "alpha_median": 0.883, |
| "ci95_low": 0.8624995833333334, |
| "ci95_high": 0.8964683333333335 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.79455, |
| "alpha_median": 0.8025, |
| "ci95_low": 0.7706141666666667, |
| "ci95_high": 0.8169733333333333 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.712, |
| "alpha_median": 0.7084999999999999, |
| "ci95_low": 0.6851662500000001, |
| "ci95_high": 0.7384850000000002 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.6325999999999999, |
| "alpha_median": 0.638, |
| "ci95_low": 0.6014795833333332, |
| "ci95_high": 0.6633079166666666 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.5557833333333332, |
| "alpha_median": 0.553, |
| "ci95_low": 0.5222575000000002, |
| "ci95_high": 0.5895004166666666 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.47598333333333337, |
| "alpha_median": 0.4825, |
| "ci95_low": 0.43973083333333335, |
| "ci95_high": 0.5133391666666666 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.4049833333333333, |
| "alpha_median": 0.39, |
| "ci95_low": 0.3712995833333333, |
| "ci95_high": 0.43846416666666665 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8569333333333334, |
| "alpha_median": 0.874, |
| "ci95_low": 0.8370483333333333, |
| "ci95_high": 0.8763833333333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7463333333333334, |
| "alpha_median": 0.763, |
| "ci95_low": 0.718665, |
| "ci95_high": 0.7722341666666669 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6522333333333332, |
| "alpha_median": 0.654, |
| "ci95_low": 0.6200466666666665, |
| "ci95_high": 0.6846175 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.5665999999999999, |
| "alpha_median": 0.5680000000000001, |
| "ci95_low": 0.5307308333333333, |
| "ci95_high": 0.6019720833333333 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.49448333333333333, |
| "alpha_median": 0.4755, |
| "ci95_low": 0.4554829166666667, |
| "ci95_high": 0.5327204166666667 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.42051666666666665, |
| "alpha_median": 0.397, |
| "ci95_low": 0.38084958333333335, |
| "ci95_high": 0.4608499999999999 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.3630166666666667, |
| "alpha_median": 0.3365, |
| "ci95_low": 0.3226812499999999, |
| "ci95_high": 0.4040516666666667 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 55, |
| "alpha_mean": 0.7442181818181818, |
| "alpha_median": 0.75, |
| "ci95_low": 0.7010240909090908, |
| "ci95_high": 0.7815731818181817 |
| }, |
| { |
| "position": 2, |
| "n": 55, |
| "alpha_mean": 0.5852181818181819, |
| "alpha_median": 0.561, |
| "ci95_low": 0.53427, |
| "ci95_high": 0.6413472727272729 |
| }, |
| { |
| "position": 3, |
| "n": 55, |
| "alpha_mean": 0.4758545454545455, |
| "alpha_median": 0.411, |
| "ci95_low": 0.41785954545454534, |
| "ci95_high": 0.5385318181818182 |
| }, |
| { |
| "position": 4, |
| "n": 55, |
| "alpha_mean": 0.3937818181818182, |
| "alpha_median": 0.301, |
| "ci95_low": 0.32965363636363637, |
| "ci95_high": 0.4654922727272726 |
| }, |
| { |
| "position": 5, |
| "n": 55, |
| "alpha_mean": 0.32007272727272723, |
| "alpha_median": 0.203, |
| "ci95_low": 0.2553986363636363, |
| "ci95_high": 0.3890754545454545 |
| }, |
| { |
| "position": 6, |
| "n": 55, |
| "alpha_mean": 0.2712, |
| "alpha_median": 0.159, |
| "ci95_low": 0.2051377272727273, |
| "ci95_high": 0.34276909090909086 |
| }, |
| { |
| "position": 7, |
| "n": 55, |
| "alpha_mean": 0.23603636363636366, |
| "alpha_median": 0.113, |
| "ci95_low": 0.16681272727272728, |
| "ci95_high": 0.3091281818181818 |
| } |
| ] |
| } |
| }, |
| "ksweep-qwen-q4-dflash-k10": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "dflash-f16", |
| "k": 10, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.000000000000143e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.014683333333333335, |
| "alpha_median": 0.01, |
| "ci95_low": 0.011265, |
| "ci95_high": 0.0186 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.0034000000000000002, |
| "alpha_median": 0.0, |
| "ci95_low": 0.001816666666666667, |
| "ci95_high": 0.0056 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.0017, |
| "alpha_median": 0.0, |
| "ci95_low": 0.00095, |
| "ci95_high": 0.002533749999999998 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.0011333333333333334, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0006666666666666666, |
| "ci95_high": 0.0016666666666666668 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.0006000000000000001, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0002666666666666667, |
| "ci95_high": 0.001 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.0003333333333333333, |
| "alpha_median": 0.0, |
| "ci95_low": 6.666666666666667e-05, |
| "ci95_high": 0.0006666666666666666 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.013800000000000003, |
| "alpha_median": 0.012, |
| "ci95_low": 0.011216250000000002, |
| "ci95_high": 0.01651666666666667 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.003916666666666667, |
| "alpha_median": 0.004, |
| "ci95_low": 0.002766666666666667, |
| "ci95_high": 0.005233333333333334 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.00155, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0008666666666666667, |
| "ci95_high": 0.0023833333333333337 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.0006166666666666667, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0002, |
| "ci95_high": 0.0011166666666666666 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.0003333333333333333, |
| "alpha_median": 0.0, |
| "ci95_low": 6.666666666666667e-05, |
| "ci95_high": 0.0006666666666666666 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.0002, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.00046666666666666666 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.021448275862068968, |
| "alpha_median": 0.016, |
| "ci95_low": 0.017085344827586207, |
| "ci95_high": 0.026397413793103446 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.004586206896551724, |
| "alpha_median": 0.004, |
| "ci95_low": 0.003327586206896552, |
| "ci95_high": 0.005983189655172413 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.0018965517241379313, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0011379310344827587, |
| "ci95_high": 0.0027241379310344828 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.0010000000000000002, |
| "alpha_median": 0.0, |
| "ci95_low": 0.00041379310344827585, |
| "ci95_high": 0.0016724137931034483 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.0005172413793103448, |
| "alpha_median": 0.0, |
| "ci95_low": 6.896551724137931e-05, |
| "ci95_high": 0.0011379310344827585 |
| }, |
| { |
| "position": 6, |
| "n": 58, |
| "alpha_mean": 0.0002931034482758621, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.000689655172413793 |
| }, |
| { |
| "position": 7, |
| "n": 58, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 8, |
| "n": 58, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 9, |
| "n": 58, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 10, |
| "n": 58, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| } |
| ] |
| } |
| }, |
| "ksweep-qwen-q4-dflash-k5": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "dflash-f16", |
| "k": 5, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 4.99999999999997e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.013850000000000001, |
| "alpha_median": 0.012, |
| "ci95_low": 0.0105, |
| "ci95_high": 0.0178 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.003433333333333334, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0018658333333333333, |
| "ci95_high": 0.005633333333333334 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.0012666666666666668, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0007333333333333333, |
| "ci95_high": 0.0018666666666666666 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.0008, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0004, |
| "ci95_high": 0.0012000000000000001 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.0006000000000000001, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0002666666666666667, |
| "ci95_high": 0.001 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.012633333333333333, |
| "alpha_median": 0.012, |
| "ci95_low": 0.010132916666666669, |
| "ci95_high": 0.015167083333333331 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.0030166666666666675, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0020162500000000002, |
| "ci95_high": 0.004 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.0012666666666666668, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0007333333333333333, |
| "ci95_high": 0.0018000000000000002 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.0005333333333333334, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0002, |
| "ci95_high": 0.0009333333333333334 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.0002, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.00046666666666666666 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.020879310344827588, |
| "alpha_median": 0.016, |
| "ci95_low": 0.01656853448275862, |
| "ci95_high": 0.025655603448275863 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.003637931034482759, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0024310344827586207, |
| "ci95_high": 0.005051724137931035 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.0012413793103448277, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0006896551724137933, |
| "ci95_high": 0.0017931034482758623 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.0006206896551724138, |
| "alpha_median": 0.0, |
| "ci95_low": 0.00027586206896551725, |
| "ci95_high": 0.0010344827586206897 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.00020689655172413793, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0004827586206896552 |
| } |
| ] |
| } |
| }, |
| "ksweep-qwen-q4-dflash-k7": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "dflash-f16", |
| "k": 7, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.000000000000143e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.014683333333333335, |
| "alpha_median": 0.01, |
| "ci95_low": 0.011265, |
| "ci95_high": 0.0186 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.0034000000000000002, |
| "alpha_median": 0.0, |
| "ci95_low": 0.001816666666666667, |
| "ci95_high": 0.0056 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.0017, |
| "alpha_median": 0.0, |
| "ci95_low": 0.00095, |
| "ci95_high": 0.002533749999999998 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.0011333333333333334, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0006666666666666666, |
| "ci95_high": 0.0016666666666666668 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.0006000000000000001, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0002666666666666667, |
| "ci95_high": 0.001 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.0003333333333333333, |
| "alpha_median": 0.0, |
| "ci95_low": 6.666666666666667e-05, |
| "ci95_high": 0.0006666666666666666 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.013800000000000003, |
| "alpha_median": 0.012, |
| "ci95_low": 0.011216250000000002, |
| "ci95_high": 0.01651666666666667 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.003916666666666667, |
| "alpha_median": 0.004, |
| "ci95_low": 0.002766666666666667, |
| "ci95_high": 0.005233333333333334 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.00155, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0008666666666666667, |
| "ci95_high": 0.0023833333333333337 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.0006166666666666667, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0002, |
| "ci95_high": 0.0011166666666666666 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.0003333333333333333, |
| "alpha_median": 0.0, |
| "ci95_low": 6.666666666666667e-05, |
| "ci95_high": 0.0006666666666666666 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.0002, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.00046666666666666666 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.021448275862068968, |
| "alpha_median": 0.016, |
| "ci95_low": 0.017085344827586207, |
| "ci95_high": 0.026397413793103446 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.004586206896551724, |
| "alpha_median": 0.004, |
| "ci95_low": 0.003327586206896552, |
| "ci95_high": 0.005983189655172413 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.0018965517241379313, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0011379310344827587, |
| "ci95_high": 0.0027241379310344828 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.0010000000000000002, |
| "alpha_median": 0.0, |
| "ci95_low": 0.00041379310344827585, |
| "ci95_high": 0.0016724137931034483 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.0005172413793103448, |
| "alpha_median": 0.0, |
| "ci95_low": 6.896551724137931e-05, |
| "ci95_high": 0.0011379310344827585 |
| }, |
| { |
| "position": 6, |
| "n": 58, |
| "alpha_mean": 0.0002931034482758621, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.000689655172413793 |
| }, |
| { |
| "position": 7, |
| "n": 58, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| } |
| ] |
| } |
| }, |
| "ksweep-qwen-q4-dspark-p0-k10": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "dspark-p0", |
| "k": 10, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.0000000000105516e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8275166666666666, |
| "alpha_median": 0.8234999999999999, |
| "ci95_low": 0.80881625, |
| "ci95_high": 0.8464854166666665 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6882666666666667, |
| "alpha_median": 0.7064999999999999, |
| "ci95_low": 0.6566654166666668, |
| "ci95_high": 0.7224 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5771166666666667, |
| "alpha_median": 0.57, |
| "ci95_low": 0.5377991666666667, |
| "ci95_high": 0.6176891666666667 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.48430000000000006, |
| "alpha_median": 0.4585, |
| "ci95_low": 0.4371829166666667, |
| "ci95_high": 0.5336854166666667 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.4117, |
| "alpha_median": 0.4, |
| "ci95_low": 0.36243166666666665, |
| "ci95_high": 0.4655683333333333 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.35425, |
| "alpha_median": 0.339, |
| "ci95_low": 0.30452666666666667, |
| "ci95_high": 0.40827583333333334 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.30565, |
| "alpha_median": 0.295, |
| "ci95_low": 0.2567629166666667, |
| "ci95_high": 0.3597012499999999 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.7943666666666667, |
| "alpha_median": 0.808, |
| "ci95_low": 0.77531625, |
| "ci95_high": 0.8127087500000001 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6135166666666668, |
| "alpha_median": 0.6125, |
| "ci95_low": 0.58530875, |
| "ci95_high": 0.6420545833333332 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.46181666666666665, |
| "alpha_median": 0.4355, |
| "ci95_low": 0.4318120833333333, |
| "ci95_high": 0.4930520833333334 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.3523333333333333, |
| "alpha_median": 0.3255, |
| "ci95_low": 0.3216791666666667, |
| "ci95_high": 0.3854704166666667 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.26839999999999997, |
| "alpha_median": 0.2355, |
| "ci95_low": 0.23628, |
| "ci95_high": 0.3033833333333333 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.19771666666666665, |
| "alpha_median": 0.1725, |
| "ci95_low": 0.16911499999999996, |
| "ci95_high": 0.22833874999999998 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.15255000000000002, |
| "alpha_median": 0.119, |
| "ci95_low": 0.12411666666666668, |
| "ci95_high": 0.18401791666666664 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.7089310344827586, |
| "alpha_median": 0.7275, |
| "ci95_low": 0.665080172413793, |
| "ci95_high": 0.747226724137931 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.4922241379310345, |
| "alpha_median": 0.5095000000000001, |
| "ci95_low": 0.44306379310344834, |
| "ci95_high": 0.5365064655172413 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.35253448275862065, |
| "alpha_median": 0.3375, |
| "ci95_low": 0.30437112068965516, |
| "ci95_high": 0.39734525862068965 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.2588793103448276, |
| "alpha_median": 0.2205, |
| "ci95_low": 0.21575560344827588, |
| "ci95_high": 0.30172629310344823 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.19879310344827586, |
| "alpha_median": 0.1545, |
| "ci95_low": 0.15875646551724143, |
| "ci95_high": 0.24043620689655168 |
| }, |
| { |
| "position": 6, |
| "n": 58, |
| "alpha_mean": 0.15515517241379312, |
| "alpha_median": 0.0895, |
| "ci95_low": 0.11734439655172413, |
| "ci95_high": 0.19293146551724136 |
| }, |
| { |
| "position": 7, |
| "n": 58, |
| "alpha_mean": 0.12293103448275863, |
| "alpha_median": 0.065, |
| "ci95_low": 0.0872926724137931, |
| "ci95_high": 0.15955387931034481 |
| }, |
| { |
| "position": 8, |
| "n": 58, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 9, |
| "n": 58, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 10, |
| "n": 58, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| } |
| ] |
| } |
| }, |
| "ksweep-qwen-q4-dspark-p0-k5": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "dspark-p0", |
| "k": 5, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.0000000000050004e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8351000000000001, |
| "alpha_median": 0.8434999999999999, |
| "ci95_low": 0.8158500000000001, |
| "ci95_high": 0.8545670833333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6908, |
| "alpha_median": 0.699, |
| "ci95_low": 0.6574162500000001, |
| "ci95_high": 0.7251333333333333 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5831, |
| "alpha_median": 0.5714999999999999, |
| "ci95_low": 0.5408058333333334, |
| "ci95_high": 0.6268512499999997 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.48841666666666667, |
| "alpha_median": 0.488, |
| "ci95_low": 0.44127791666666666, |
| "ci95_high": 0.5375004166666666 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.40971666666666656, |
| "alpha_median": 0.397, |
| "ci95_low": 0.3608645833333333, |
| "ci95_high": 0.4619025 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.7903666666666667, |
| "alpha_median": 0.792, |
| "ci95_low": 0.77183, |
| "ci95_high": 0.80858375 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6033166666666666, |
| "alpha_median": 0.5945, |
| "ci95_low": 0.5752324999999999, |
| "ci95_high": 0.6310170833333334 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.4645, |
| "alpha_median": 0.454, |
| "ci95_low": 0.4331312499999999, |
| "ci95_high": 0.4965354166666667 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.35390000000000005, |
| "alpha_median": 0.329, |
| "ci95_low": 0.32044125, |
| "ci95_high": 0.38973333333333327 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.26375, |
| "alpha_median": 0.23, |
| "ci95_low": 0.23050958333333335, |
| "ci95_high": 0.29978541666666664 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.7085689655172414, |
| "alpha_median": 0.7555000000000001, |
| "ci95_low": 0.664840948275862, |
| "ci95_high": 0.7459702586206896 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.4858793103448276, |
| "alpha_median": 0.513, |
| "ci95_low": 0.43451724137931036, |
| "ci95_high": 0.5313284482758621 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.352948275862069, |
| "alpha_median": 0.3545, |
| "ci95_low": 0.30419784482758616, |
| "ci95_high": 0.39955258620689643 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.26739655172413795, |
| "alpha_median": 0.2155, |
| "ci95_low": 0.2195293103448276, |
| "ci95_high": 0.3135099137931034 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.2047586206896552, |
| "alpha_median": 0.1445, |
| "ci95_low": 0.16067543103448276, |
| "ci95_high": 0.24851810344827574 |
| } |
| ] |
| } |
| }, |
| "ksweep-qwen-q4-dspark-p0-k7": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "dspark-p0", |
| "k": 7, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.0000000000105516e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8275166666666666, |
| "alpha_median": 0.8234999999999999, |
| "ci95_low": 0.80881625, |
| "ci95_high": 0.8464854166666665 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6882666666666667, |
| "alpha_median": 0.7064999999999999, |
| "ci95_low": 0.6566654166666668, |
| "ci95_high": 0.7224 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5771166666666667, |
| "alpha_median": 0.57, |
| "ci95_low": 0.5377991666666667, |
| "ci95_high": 0.6176891666666667 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.48430000000000006, |
| "alpha_median": 0.4585, |
| "ci95_low": 0.4371829166666667, |
| "ci95_high": 0.5336854166666667 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.4117, |
| "alpha_median": 0.4, |
| "ci95_low": 0.36243166666666665, |
| "ci95_high": 0.4655683333333333 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.35425, |
| "alpha_median": 0.339, |
| "ci95_low": 0.30452666666666667, |
| "ci95_high": 0.40827583333333334 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.30565, |
| "alpha_median": 0.295, |
| "ci95_low": 0.2567629166666667, |
| "ci95_high": 0.3597012499999999 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.7943666666666667, |
| "alpha_median": 0.808, |
| "ci95_low": 0.77531625, |
| "ci95_high": 0.8127087500000001 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6135166666666668, |
| "alpha_median": 0.6125, |
| "ci95_low": 0.58530875, |
| "ci95_high": 0.6420545833333332 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.46181666666666665, |
| "alpha_median": 0.4355, |
| "ci95_low": 0.4318120833333333, |
| "ci95_high": 0.4930520833333334 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.3523333333333333, |
| "alpha_median": 0.3255, |
| "ci95_low": 0.3216791666666667, |
| "ci95_high": 0.3854704166666667 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.26839999999999997, |
| "alpha_median": 0.2355, |
| "ci95_low": 0.23628, |
| "ci95_high": 0.3033833333333333 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.19771666666666665, |
| "alpha_median": 0.1725, |
| "ci95_low": 0.16911499999999996, |
| "ci95_high": 0.22833874999999998 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.15255000000000002, |
| "alpha_median": 0.119, |
| "ci95_low": 0.12411666666666668, |
| "ci95_high": 0.18401791666666664 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.7089310344827586, |
| "alpha_median": 0.7275, |
| "ci95_low": 0.665080172413793, |
| "ci95_high": 0.747226724137931 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.4922241379310345, |
| "alpha_median": 0.5095000000000001, |
| "ci95_low": 0.44306379310344834, |
| "ci95_high": 0.5365064655172413 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.35253448275862065, |
| "alpha_median": 0.3375, |
| "ci95_low": 0.30437112068965516, |
| "ci95_high": 0.39734525862068965 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.2588793103448276, |
| "alpha_median": 0.2205, |
| "ci95_low": 0.21575560344827588, |
| "ci95_high": 0.30172629310344823 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.19879310344827586, |
| "alpha_median": 0.1545, |
| "ci95_low": 0.15875646551724143, |
| "ci95_high": 0.24043620689655168 |
| }, |
| { |
| "position": 6, |
| "n": 58, |
| "alpha_mean": 0.15515517241379312, |
| "alpha_median": 0.0895, |
| "ci95_low": 0.11734439655172413, |
| "ci95_high": 0.19293146551724136 |
| }, |
| { |
| "position": 7, |
| "n": 58, |
| "alpha_mean": 0.12293103448275863, |
| "alpha_median": 0.065, |
| "ci95_low": 0.0872926724137931, |
| "ci95_high": 0.15955387931034481 |
| } |
| ] |
| } |
| }, |
| "ksweep-qwen-q4-eagle3-k10": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "eagle3", |
| "k": 10, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.000000000002225e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6393000000000001, |
| "alpha_median": 0.636, |
| "ci95_low": 0.6221158333333334, |
| "ci95_high": 0.65715 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.41214999999999996, |
| "alpha_median": 0.413, |
| "ci95_low": 0.3911658333333333, |
| "ci95_high": 0.43366874999999994 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.2157, |
| "alpha_median": 0.2145, |
| "ci95_low": 0.20113291666666666, |
| "ci95_high": 0.23085124999999998 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.12154999999999999, |
| "alpha_median": 0.1165, |
| "ci95_low": 0.10949666666666666, |
| "ci95_high": 0.13506666666666664 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.07311666666666668, |
| "alpha_median": 0.07050000000000001, |
| "ci95_low": 0.06314958333333333, |
| "ci95_high": 0.08443333333333333 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.04493333333333333, |
| "alpha_median": 0.038, |
| "ci95_low": 0.037166250000000005, |
| "ci95_high": 0.05385083333333333 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.027366666666666664, |
| "alpha_median": 0.0225, |
| "ci95_low": 0.021782916666666666, |
| "ci95_high": 0.0334675 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.01655, |
| "alpha_median": 0.011, |
| "ci95_low": 0.012550000000000002, |
| "ci95_high": 0.020699999999999996 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.008683333333333333, |
| "alpha_median": 0.0085, |
| "ci95_low": 0.005449166666666666, |
| "ci95_high": 0.012416666666666666 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.005783333333333333, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0031833333333333336, |
| "ci95_high": 0.008583333333333333 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6601999999999999, |
| "alpha_median": 0.6415, |
| "ci95_low": 0.6419654166666666, |
| "ci95_high": 0.6793508333333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.41513333333333335, |
| "alpha_median": 0.399, |
| "ci95_low": 0.3911633333333333, |
| "ci95_high": 0.4415695833333333 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.21558333333333332, |
| "alpha_median": 0.174, |
| "ci95_low": 0.1876145833333333, |
| "ci95_high": 0.24768500000000002 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.12786666666666668, |
| "alpha_median": 0.0925, |
| "ci95_low": 0.10286625, |
| "ci95_high": 0.15703499999999995 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.08738333333333334, |
| "alpha_median": 0.046, |
| "ci95_low": 0.06396333333333334, |
| "ci95_high": 0.11490749999999998 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.057249999999999995, |
| "alpha_median": 0.02, |
| "ci95_low": 0.03816375, |
| "ci95_high": 0.07923541666666666 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.03891666666666666, |
| "alpha_median": 0.01, |
| "ci95_low": 0.022932499999999998, |
| "ci95_high": 0.05705333333333332 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.030533333333333336, |
| "alpha_median": 0.009, |
| "ci95_low": 0.01624916666666667, |
| "ci95_high": 0.047400833333333336 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.021716666666666665, |
| "alpha_median": 0.0, |
| "ci95_low": 0.010549166666666668, |
| "ci95_high": 0.03530249999999999 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.01635, |
| "alpha_median": 0.0, |
| "ci95_low": 0.008066666666666666, |
| "ci95_high": 0.025851666666666658 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.5735517241379311, |
| "alpha_median": 0.6014999999999999, |
| "ci95_low": 0.5235991379310345, |
| "ci95_high": 0.6212129310344826 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.34679310344827585, |
| "alpha_median": 0.3715, |
| "ci95_low": 0.2997573275862069, |
| "ci95_high": 0.39171379310344817 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.18879310344827585, |
| "alpha_median": 0.163, |
| "ci95_low": 0.1524030172413793, |
| "ci95_high": 0.22463879310344825 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.11891379310344827, |
| "alpha_median": 0.079, |
| "ci95_low": 0.08842974137931035, |
| "ci95_high": 0.15122887931034484 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.08306896551724137, |
| "alpha_median": 0.0405, |
| "ci95_low": 0.057982327586206896, |
| "ci95_high": 0.10953577586206896 |
| }, |
| { |
| "position": 6, |
| "n": 58, |
| "alpha_mean": 0.0649655172413793, |
| "alpha_median": 0.0285, |
| "ci95_low": 0.04182456896551724, |
| "ci95_high": 0.08976249999999997 |
| }, |
| { |
| "position": 7, |
| "n": 58, |
| "alpha_mean": 0.048999999999999995, |
| "alpha_median": 0.0095, |
| "ci95_low": 0.029137068965517245, |
| "ci95_high": 0.0712004310344827 |
| }, |
| { |
| "position": 8, |
| "n": 58, |
| "alpha_mean": 0.04074137931034483, |
| "alpha_median": 0.008, |
| "ci95_low": 0.022344827586206897, |
| "ci95_high": 0.06181077586206897 |
| }, |
| { |
| "position": 9, |
| "n": 58, |
| "alpha_mean": 0.034413793103448276, |
| "alpha_median": 0.0, |
| "ci95_low": 0.016790948275862073, |
| "ci95_high": 0.0545176724137931 |
| }, |
| { |
| "position": 10, |
| "n": 58, |
| "alpha_mean": 0.029206896551724138, |
| "alpha_median": 0.0, |
| "ci95_low": 0.013636637931034482, |
| "ci95_high": 0.047277586206896545 |
| } |
| ] |
| } |
| }, |
| "ksweep-qwen-q4-eagle3-k5": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "eagle3", |
| "k": 5, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.0000000000050004e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6475333333333333, |
| "alpha_median": 0.6315, |
| "ci95_low": 0.6281633333333333, |
| "ci95_high": 0.6681520833333335 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.4170999999999999, |
| "alpha_median": 0.405, |
| "ci95_low": 0.39436625, |
| "ci95_high": 0.44110166666666667 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.2237666666666667, |
| "alpha_median": 0.218, |
| "ci95_low": 0.20564458333333333, |
| "ci95_high": 0.24383749999999996 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.12868333333333334, |
| "alpha_median": 0.1205, |
| "ci95_low": 0.11228208333333332, |
| "ci95_high": 0.1467341666666667 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.0775, |
| "alpha_median": 0.0695, |
| "ci95_low": 0.06496583333333333, |
| "ci95_high": 0.09210125 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6593166666666668, |
| "alpha_median": 0.648, |
| "ci95_low": 0.6399995833333333, |
| "ci95_high": 0.6801691666666667 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.4212333333333333, |
| "alpha_median": 0.4105, |
| "ci95_low": 0.39419958333333327, |
| "ci95_high": 0.4513374999999999 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.21413333333333331, |
| "alpha_median": 0.176, |
| "ci95_low": 0.18536458333333336, |
| "ci95_high": 0.24663083333333327 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.1280666666666667, |
| "alpha_median": 0.088, |
| "ci95_low": 0.10104916666666666, |
| "ci95_high": 0.15938333333333332 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.0828, |
| "alpha_median": 0.048, |
| "ci95_low": 0.058109999999999995, |
| "ci95_high": 0.11205333333333332 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.5732241379310344, |
| "alpha_median": 0.597, |
| "ci95_low": 0.5165314655172414, |
| "ci95_high": 0.6251758620689656 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.3543620689655172, |
| "alpha_median": 0.3435, |
| "ci95_low": 0.3023431034482758, |
| "ci95_high": 0.4045181034482759 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.1993103448275862, |
| "alpha_median": 0.1605, |
| "ci95_low": 0.15927284482758625, |
| "ci95_high": 0.24270301724137924 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.1258103448275862, |
| "alpha_median": 0.092, |
| "ci95_low": 0.09251422413793102, |
| "ci95_high": 0.1614176724137931 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.08212068965517243, |
| "alpha_median": 0.0435, |
| "ci95_low": 0.05446120689655173, |
| "ci95_high": 0.11160344827586206 |
| } |
| ] |
| } |
| }, |
| "ksweep-qwen-q4-eagle3-k7": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "eagle3", |
| "k": 7, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.000000000002225e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6443333333333333, |
| "alpha_median": 0.626, |
| "ci95_low": 0.6251470833333334, |
| "ci95_high": 0.6649041666666665 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.4131333333333334, |
| "alpha_median": 0.403, |
| "ci95_low": 0.3907324999999999, |
| "ci95_high": 0.4365841666666666 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.21961666666666668, |
| "alpha_median": 0.218, |
| "ci95_low": 0.20243291666666668, |
| "ci95_high": 0.2386175 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.12621666666666667, |
| "alpha_median": 0.118, |
| "ci95_low": 0.1104825, |
| "ci95_high": 0.14348999999999995 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.07546666666666667, |
| "alpha_median": 0.0675, |
| "ci95_low": 0.06334833333333333, |
| "ci95_high": 0.08971749999999999 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.045649999999999996, |
| "alpha_median": 0.036, |
| "ci95_low": 0.03564916666666666, |
| "ci95_high": 0.05713375 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.028, |
| "alpha_median": 0.02, |
| "ci95_low": 0.02039833333333333, |
| "ci95_high": 0.037367083333333335 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6557499999999999, |
| "alpha_median": 0.6445000000000001, |
| "ci95_low": 0.6375145833333333, |
| "ci95_high": 0.6747520833333334 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.41759999999999997, |
| "alpha_median": 0.4125, |
| "ci95_low": 0.39189124999999997, |
| "ci95_high": 0.4459733333333334 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.21246666666666666, |
| "alpha_median": 0.1815, |
| "ci95_low": 0.18428166666666662, |
| "ci95_high": 0.2451670833333333 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.12625000000000003, |
| "alpha_median": 0.0875, |
| "ci95_low": 0.09834916666666665, |
| "ci95_high": 0.15945666666666664 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.08131666666666669, |
| "alpha_median": 0.049, |
| "ci95_low": 0.05654833333333334, |
| "ci95_high": 0.11175416666666664 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.052649999999999995, |
| "alpha_median": 0.019, |
| "ci95_low": 0.03203333333333333, |
| "ci95_high": 0.07733583333333333 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.0377, |
| "alpha_median": 0.01, |
| "ci95_low": 0.020062916666666666, |
| "ci95_high": 0.05930083333333334 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.5708275862068966, |
| "alpha_median": 0.6045, |
| "ci95_low": 0.5146288793103448, |
| "ci95_high": 0.6221379310344828 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.3496551724137931, |
| "alpha_median": 0.3395, |
| "ci95_low": 0.29932672413793104, |
| "ci95_high": 0.39888405172413793 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.19015517241379312, |
| "alpha_median": 0.158, |
| "ci95_low": 0.15470474137931034, |
| "ci95_high": 0.22732758620689655 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.12032758620689656, |
| "alpha_median": 0.084, |
| "ci95_low": 0.09060258620689655, |
| "ci95_high": 0.15307241379310346 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.07534482758620689, |
| "alpha_median": 0.042499999999999996, |
| "ci95_low": 0.05094008620689655, |
| "ci95_high": 0.10100258620689655 |
| }, |
| { |
| "position": 6, |
| "n": 58, |
| "alpha_mean": 0.057896551724137935, |
| "alpha_median": 0.0205, |
| "ci95_low": 0.035170689655172414, |
| "ci95_high": 0.08256939655172414 |
| }, |
| { |
| "position": 7, |
| "n": 58, |
| "alpha_mean": 0.04268965517241379, |
| "alpha_median": 0.009, |
| "ci95_low": 0.022724137931034484, |
| "ci95_high": 0.06393362068965516 |
| } |
| ] |
| } |
| }, |
| "ksweep-qwen-q4-vanilla17b-k10": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "vanilla17b", |
| "k": 10, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.000000000002225e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8605833333333334, |
| "alpha_median": 0.864, |
| "ci95_low": 0.8454433333333332, |
| "ci95_high": 0.8753520833333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7415833333333333, |
| "alpha_median": 0.7255, |
| "ci95_low": 0.7151995833333333, |
| "ci95_high": 0.7684404166666667 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6566500000000001, |
| "alpha_median": 0.6505000000000001, |
| "ci95_low": 0.6250304166666666, |
| "ci95_high": 0.6895837499999998 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.5859333333333334, |
| "alpha_median": 0.5725, |
| "ci95_low": 0.5521279166666666, |
| "ci95_high": 0.6207170833333334 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.5292833333333333, |
| "alpha_median": 0.512, |
| "ci95_low": 0.49318166666666663, |
| "ci95_high": 0.5666537499999998 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.47714999999999996, |
| "alpha_median": 0.469, |
| "ci95_low": 0.44058083333333337, |
| "ci95_high": 0.51552375 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.4276666666666667, |
| "alpha_median": 0.42, |
| "ci95_low": 0.39024958333333337, |
| "ci95_high": 0.4667545833333333 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.3843833333333334, |
| "alpha_median": 0.366, |
| "ci95_low": 0.34676666666666667, |
| "ci95_high": 0.4235545833333333 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.3481, |
| "alpha_median": 0.341, |
| "ci95_low": 0.3101312500000001, |
| "ci95_high": 0.3868700000000001 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.32078333333333325, |
| "alpha_median": 0.314, |
| "ci95_low": 0.28234583333333335, |
| "ci95_high": 0.3583666666666666 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.82835, |
| "alpha_median": 0.835, |
| "ci95_low": 0.8096479166666668, |
| "ci95_high": 0.8476525 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6865166666666667, |
| "alpha_median": 0.6875, |
| "ci95_low": 0.6631316666666667, |
| "ci95_high": 0.7102224999999999 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5856499999999999, |
| "alpha_median": 0.578, |
| "ci95_low": 0.5595291666666666, |
| "ci95_high": 0.6141675 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.5046, |
| "alpha_median": 0.484, |
| "ci95_low": 0.47836416666666665, |
| "ci95_high": 0.5368516666666666 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.43585, |
| "alpha_median": 0.415, |
| "ci95_low": 0.4082495833333332, |
| "ci95_high": 0.4698133333333333 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.3830666666666667, |
| "alpha_median": 0.3645, |
| "ci95_low": 0.3531079166666666, |
| "ci95_high": 0.41880291666666664 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.34103333333333335, |
| "alpha_median": 0.3225, |
| "ci95_low": 0.3107825, |
| "ci95_high": 0.3766229166666667 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.30171666666666663, |
| "alpha_median": 0.278, |
| "ci95_low": 0.2721820833333333, |
| "ci95_high": 0.3373029166666667 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.2707166666666667, |
| "alpha_median": 0.242, |
| "ci95_low": 0.24014916666666664, |
| "ci95_high": 0.3073733333333334 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.2393666666666667, |
| "alpha_median": 0.2145, |
| "ci95_low": 0.21163, |
| "ci95_high": 0.27196833333333337 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.7706724137931034, |
| "alpha_median": 0.7444999999999999, |
| "ci95_low": 0.7471025862068965, |
| "ci95_high": 0.7944883620689656 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.5908793103448275, |
| "alpha_median": 0.588, |
| "ci95_low": 0.5608418103448276, |
| "ci95_high": 0.6219202586206896 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.47472413793103446, |
| "alpha_median": 0.478, |
| "ci95_low": 0.43965344827586206, |
| "ci95_high": 0.5126948275862069 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.38355172413793104, |
| "alpha_median": 0.362, |
| "ci95_low": 0.3471965517241379, |
| "ci95_high": 0.4236383620689655 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.3245862068965517, |
| "alpha_median": 0.2845, |
| "ci95_low": 0.28644482758620693, |
| "ci95_high": 0.3674366379310344 |
| }, |
| { |
| "position": 6, |
| "n": 58, |
| "alpha_mean": 0.27910344827586203, |
| "alpha_median": 0.231, |
| "ci95_low": 0.2389810344827586, |
| "ci95_high": 0.32389655172413795 |
| }, |
| { |
| "position": 7, |
| "n": 58, |
| "alpha_mean": 0.24193103448275866, |
| "alpha_median": 0.191, |
| "ci95_low": 0.20145646551724136, |
| "ci95_high": 0.2864323275862069 |
| }, |
| { |
| "position": 8, |
| "n": 58, |
| "alpha_mean": 0.21972413793103449, |
| "alpha_median": 0.16399999999999998, |
| "ci95_low": 0.17875862068965517, |
| "ci95_high": 0.266089224137931 |
| }, |
| { |
| "position": 9, |
| "n": 58, |
| "alpha_mean": 0.19453448275862065, |
| "alpha_median": 0.138, |
| "ci95_low": 0.15435689655172413, |
| "ci95_high": 0.23910603448275858 |
| }, |
| { |
| "position": 10, |
| "n": 58, |
| "alpha_mean": 0.17518965517241378, |
| "alpha_median": 0.11449999999999999, |
| "ci95_low": 0.1350336206896552, |
| "ci95_high": 0.21991681034482755 |
| } |
| ] |
| } |
| }, |
| "ksweep-qwen-q4-vanilla17b-k5": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "vanilla17b", |
| "k": 5, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.0000000000050004e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8614333333333332, |
| "alpha_median": 0.8694999999999999, |
| "ci95_low": 0.8468833333333333, |
| "ci95_high": 0.8758516666666667 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7467333333333332, |
| "alpha_median": 0.737, |
| "ci95_low": 0.7221154166666666, |
| "ci95_high": 0.7710595833333332 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6641666666666667, |
| "alpha_median": 0.6555, |
| "ci95_low": 0.6353624999999999, |
| "ci95_high": 0.6928541666666668 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.5916833333333334, |
| "alpha_median": 0.5945, |
| "ci95_low": 0.5596133333333334, |
| "ci95_high": 0.6232516666666668 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.5365, |
| "alpha_median": 0.5255000000000001, |
| "ci95_low": 0.5021658333333333, |
| "ci95_high": 0.5705020833333334 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8306333333333333, |
| "alpha_median": 0.8225, |
| "ci95_low": 0.8160666666666666, |
| "ci95_high": 0.8457504166666666 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6994166666666667, |
| "alpha_median": 0.696, |
| "ci95_low": 0.6790833333333334, |
| "ci95_high": 0.7209337499999998 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5953000000000002, |
| "alpha_median": 0.585, |
| "ci95_low": 0.5718650000000002, |
| "ci95_high": 0.6201179166666667 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.5117333333333334, |
| "alpha_median": 0.492, |
| "ci95_low": 0.4881979166666667, |
| "ci95_high": 0.5381729166666667 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.44403333333333334, |
| "alpha_median": 0.4175, |
| "ci95_low": 0.4157829166666667, |
| "ci95_high": 0.4744208333333333 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.7753448275862069, |
| "alpha_median": 0.769, |
| "ci95_low": 0.7534068965517241, |
| "ci95_high": 0.7984706896551723 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.6073620689655171, |
| "alpha_median": 0.5805, |
| "ci95_low": 0.575498275862069, |
| "ci95_high": 0.6407241379310344 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.488948275862069, |
| "alpha_median": 0.46950000000000003, |
| "ci95_low": 0.4514625, |
| "ci95_high": 0.5298508620689655 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.40144827586206894, |
| "alpha_median": 0.363, |
| "ci95_low": 0.3599090517241379, |
| "ci95_high": 0.44630129310344824 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.33731034482758626, |
| "alpha_median": 0.283, |
| "ci95_low": 0.2937344827586207, |
| "ci95_high": 0.3847327586206897 |
| } |
| ] |
| } |
| }, |
| "ksweep-qwen-q4-vanilla17b-k7": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "vanilla17b", |
| "k": 7, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.0000000000105516e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8567666666666667, |
| "alpha_median": 0.857, |
| "ci95_low": 0.8415325, |
| "ci95_high": 0.8714508333333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7372166666666667, |
| "alpha_median": 0.732, |
| "ci95_low": 0.7132141666666667, |
| "ci95_high": 0.7604666666666666 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6550666666666667, |
| "alpha_median": 0.656, |
| "ci95_low": 0.6266645833333334, |
| "ci95_high": 0.6839679166666665 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.58005, |
| "alpha_median": 0.59, |
| "ci95_low": 0.54862375, |
| "ci95_high": 0.6120524999999999 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.5245666666666667, |
| "alpha_median": 0.521, |
| "ci95_low": 0.4935104166666666, |
| "ci95_high": 0.5572333333333331 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.47533333333333333, |
| "alpha_median": 0.4745, |
| "ci95_low": 0.4426595833333334, |
| "ci95_high": 0.5094366666666666 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.42833333333333334, |
| "alpha_median": 0.417, |
| "ci95_low": 0.39536541666666664, |
| "ci95_high": 0.4637670833333333 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8221666666666666, |
| "alpha_median": 0.826, |
| "ci95_low": 0.8074325, |
| "ci95_high": 0.8374529166666667 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6857833333333333, |
| "alpha_median": 0.673, |
| "ci95_low": 0.6650241666666667, |
| "ci95_high": 0.7074675 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5782500000000002, |
| "alpha_median": 0.564, |
| "ci95_low": 0.5552654166666666, |
| "ci95_high": 0.6034895833333332 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.4988000000000001, |
| "alpha_median": 0.4785, |
| "ci95_low": 0.47428333333333333, |
| "ci95_high": 0.5256604166666666 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.4322833333333333, |
| "alpha_median": 0.415, |
| "ci95_low": 0.4041833333333334, |
| "ci95_high": 0.46378500000000006 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.3756166666666666, |
| "alpha_median": 0.358, |
| "ci95_low": 0.3435120833333334, |
| "ci95_high": 0.40943458333333327 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.3322, |
| "alpha_median": 0.305, |
| "ci95_low": 0.3008808333333333, |
| "ci95_high": 0.3676062500000001 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.7714827586206897, |
| "alpha_median": 0.7615000000000001, |
| "ci95_low": 0.750240948275862, |
| "ci95_high": 0.7938982758620688 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.5965, |
| "alpha_median": 0.5805, |
| "ci95_low": 0.5664987068965518, |
| "ci95_high": 0.6276793103448276 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.4778965517241379, |
| "alpha_median": 0.465, |
| "ci95_low": 0.4421551724137931, |
| "ci95_high": 0.5167913793103448 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.39296551724137935, |
| "alpha_median": 0.3585, |
| "ci95_low": 0.3511181034482759, |
| "ci95_high": 0.43686250000000004 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.3258103448275862, |
| "alpha_median": 0.268, |
| "ci95_low": 0.28316508620689657, |
| "ci95_high": 0.3719310344827586 |
| }, |
| { |
| "position": 6, |
| "n": 58, |
| "alpha_mean": 0.28493103448275864, |
| "alpha_median": 0.214, |
| "ci95_low": 0.24000862068965514, |
| "ci95_high": 0.3335034482758621 |
| }, |
| { |
| "position": 7, |
| "n": 58, |
| "alpha_mean": 0.2499655172413793, |
| "alpha_median": 0.1795, |
| "ci95_low": 0.2041969827586207, |
| "ci95_high": 0.30017543103448274 |
| } |
| ] |
| } |
| }, |
| "ksweep-qwen-q8-eagle3-k10": { |
| "family": "ksweep", |
| "target": "qwen-q8", |
| "drafter": "eagle3", |
| "k": 10, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.000000000002225e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6383833333333334, |
| "alpha_median": 0.631, |
| "ci95_low": 0.6204491666666665, |
| "ci95_high": 0.6561166666666665 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.4114333333333333, |
| "alpha_median": 0.406, |
| "ci95_low": 0.39191499999999996, |
| "ci95_high": 0.431985 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.2123, |
| "alpha_median": 0.214, |
| "ci95_low": 0.19686250000000002, |
| "ci95_high": 0.22998333333333332 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.11678333333333334, |
| "alpha_median": 0.114, |
| "ci95_low": 0.10509875, |
| "ci95_high": 0.12958583333333332 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.06493333333333333, |
| "alpha_median": 0.062, |
| "ci95_low": 0.05644958333333333, |
| "ci95_high": 0.07463541666666665 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.03548333333333334, |
| "alpha_median": 0.0315, |
| "ci95_low": 0.029499583333333326, |
| "ci95_high": 0.04180083333333333 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.019933333333333338, |
| "alpha_median": 0.019, |
| "ci95_low": 0.0161, |
| "ci95_high": 0.02403416666666666 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.011000000000000001, |
| "alpha_median": 0.01, |
| "ci95_low": 0.008366666666666666, |
| "ci95_high": 0.013767083333333333 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.006816666666666667, |
| "alpha_median": 0.004, |
| "ci95_low": 0.004932916666666666, |
| "ci95_high": 0.008833333333333334 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.0042833333333333334, |
| "alpha_median": 0.0, |
| "ci95_low": 0.002716666666666667, |
| "ci95_high": 0.005933333333333333 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.64665, |
| "alpha_median": 0.65, |
| "ci95_low": 0.6291, |
| "ci95_high": 0.6649175 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.40869999999999995, |
| "alpha_median": 0.401, |
| "ci95_low": 0.38830000000000003, |
| "ci95_high": 0.4312008333333334 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.19943333333333332, |
| "alpha_median": 0.1765, |
| "ci95_low": 0.17841625, |
| "ci95_high": 0.22465374999999999 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.11043333333333333, |
| "alpha_median": 0.092, |
| "ci95_low": 0.09008041666666666, |
| "ci95_high": 0.1367220833333333 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.06545, |
| "alpha_median": 0.045, |
| "ci95_low": 0.04749624999999999, |
| "ci95_high": 0.08988333333333334 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.04083333333333334, |
| "alpha_median": 0.02, |
| "ci95_low": 0.025680416666666664, |
| "ci95_high": 0.06138375000000001 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.028283333333333334, |
| "alpha_median": 0.01, |
| "ci95_low": 0.016983333333333336, |
| "ci95_high": 0.04260249999999999 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.020483333333333336, |
| "alpha_median": 0.008, |
| "ci95_low": 0.011382916666666668, |
| "ci95_high": 0.03171791666666666 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.012733333333333334, |
| "alpha_median": 0.0, |
| "ci95_low": 0.006415833333333332, |
| "ci95_high": 0.021366666666666662 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.009200000000000002, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0034825000000000004, |
| "ci95_high": 0.017300833333333328 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.577948275862069, |
| "alpha_median": 0.609, |
| "ci95_low": 0.5294887931034482, |
| "ci95_high": 0.623278448275862 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.3406379310344828, |
| "alpha_median": 0.345, |
| "ci95_low": 0.2972030172413792, |
| "ci95_high": 0.38113275862068957 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.17277586206896553, |
| "alpha_median": 0.1625, |
| "ci95_low": 0.1397866379310345, |
| "ci95_high": 0.2088625 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.11274137931034484, |
| "alpha_median": 0.0845, |
| "ci95_low": 0.08296508620689654, |
| "ci95_high": 0.14803965517241377 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.06896551724137931, |
| "alpha_median": 0.0455, |
| "ci95_low": 0.046757758620689656, |
| "ci95_high": 0.09586336206896551 |
| }, |
| { |
| "position": 6, |
| "n": 58, |
| "alpha_mean": 0.05379310344827586, |
| "alpha_median": 0.028, |
| "ci95_low": 0.03434439655172414, |
| "ci95_high": 0.07764525862068962 |
| }, |
| { |
| "position": 7, |
| "n": 58, |
| "alpha_mean": 0.0381206896551724, |
| "alpha_median": 0.0095, |
| "ci95_low": 0.02318965517241379, |
| "ci95_high": 0.056193965517241345 |
| }, |
| { |
| "position": 8, |
| "n": 58, |
| "alpha_mean": 0.030896551724137935, |
| "alpha_median": 0.009, |
| "ci95_low": 0.017016810344827587, |
| "ci95_high": 0.04772844827586205 |
| }, |
| { |
| "position": 9, |
| "n": 58, |
| "alpha_mean": 0.023724137931034485, |
| "alpha_median": 0.0, |
| "ci95_low": 0.012065517241379312, |
| "ci95_high": 0.03824999999999995 |
| }, |
| { |
| "position": 10, |
| "n": 58, |
| "alpha_mean": 0.01812068965517241, |
| "alpha_median": 0.0, |
| "ci95_low": 0.00794741379310345, |
| "ci95_high": 0.031227155172413777 |
| } |
| ] |
| } |
| }, |
| "ksweep-qwen-q8-eagle3-k5": { |
| "family": "ksweep", |
| "target": "qwen-q8", |
| "drafter": "eagle3", |
| "k": 5, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 4.999999999999449e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.644, |
| "alpha_median": 0.629, |
| "ci95_low": 0.6263645833333332, |
| "ci95_high": 0.6616704166666665 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.41586666666666666, |
| "alpha_median": 0.422, |
| "ci95_low": 0.39656375, |
| "ci95_high": 0.4358345833333334 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.22018333333333331, |
| "alpha_median": 0.2165, |
| "ci95_low": 0.20504958333333334, |
| "ci95_high": 0.2355683333333333 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.12205, |
| "alpha_median": 0.1155, |
| "ci95_low": 0.11074958333333332, |
| "ci95_high": 0.13483625 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.06903333333333334, |
| "alpha_median": 0.063, |
| "ci95_low": 0.06015000000000001, |
| "ci95_high": 0.07895166666666667 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6542833333333333, |
| "alpha_median": 0.6465000000000001, |
| "ci95_low": 0.6373291666666667, |
| "ci95_high": 0.6723512500000001 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.41663333333333336, |
| "alpha_median": 0.412, |
| "ci95_low": 0.39533, |
| "ci95_high": 0.4391191666666666 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.20393333333333338, |
| "alpha_median": 0.1845, |
| "ci95_low": 0.18191583333333333, |
| "ci95_high": 0.22860499999999997 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.11448333333333333, |
| "alpha_median": 0.09, |
| "ci95_low": 0.09354500000000002, |
| "ci95_high": 0.13943791666666663 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.06634999999999999, |
| "alpha_median": 0.045, |
| "ci95_low": 0.048914166666666654, |
| "ci95_high": 0.08815166666666666 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.5883793103448276, |
| "alpha_median": 0.6265000000000001, |
| "ci95_low": 0.5389129310344828, |
| "ci95_high": 0.6331443965517242 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.3518275862068965, |
| "alpha_median": 0.37, |
| "ci95_low": 0.3077607758620689, |
| "ci95_high": 0.39405560344827584 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.18241379310344827, |
| "alpha_median": 0.174, |
| "ci95_low": 0.1496987068965517, |
| "ci95_high": 0.21997456896551718 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.11779310344827586, |
| "alpha_median": 0.0885, |
| "ci95_low": 0.09025387931034483, |
| "ci95_high": 0.15170948275862067 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.07225862068965518, |
| "alpha_median": 0.0445, |
| "ci95_low": 0.050567672413793105, |
| "ci95_high": 0.0976306034482758 |
| } |
| ] |
| } |
| }, |
| "ksweep-qwen-q8-eagle3-k7": { |
| "family": "ksweep", |
| "target": "qwen-q8", |
| "drafter": "eagle3", |
| "k": 7, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.000000000002225e-05, |
| "per_domain": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6422333333333333, |
| "alpha_median": 0.6315, |
| "ci95_low": 0.6244462499999999, |
| "ci95_high": 0.6602345833333334 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.4123833333333334, |
| "alpha_median": 0.4095, |
| "ci95_low": 0.3926487499999999, |
| "ci95_high": 0.43237125 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.21725, |
| "alpha_median": 0.214, |
| "ci95_low": 0.20271458333333336, |
| "ci95_high": 0.2327575 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.11929999999999999, |
| "alpha_median": 0.1155, |
| "ci95_low": 0.10796541666666669, |
| "ci95_high": 0.13148624999999997 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.06716666666666665, |
| "alpha_median": 0.063, |
| "ci95_low": 0.05866583333333334, |
| "ci95_high": 0.0768675 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.03963333333333333, |
| "alpha_median": 0.035500000000000004, |
| "ci95_low": 0.033666666666666664, |
| "ci95_high": 0.04583499999999999 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.022533333333333332, |
| "alpha_median": 0.0195, |
| "ci95_low": 0.01825, |
| "ci95_high": 0.027017083333333334 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6501833333333332, |
| "alpha_median": 0.641, |
| "ci95_low": 0.6335320833333334, |
| "ci95_high": 0.6674170833333334 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.4082166666666666, |
| "alpha_median": 0.404, |
| "ci95_low": 0.3865483333333332, |
| "ci95_high": 0.4309508333333333 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.19706666666666664, |
| "alpha_median": 0.184, |
| "ci95_low": 0.17731624999999995, |
| "ci95_high": 0.22053708333333333 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.10885000000000002, |
| "alpha_median": 0.088, |
| "ci95_low": 0.09053041666666665, |
| "ci95_high": 0.13218708333333332 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.06378333333333333, |
| "alpha_median": 0.043, |
| "ci95_low": 0.0474325, |
| "ci95_high": 0.08616875 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.03786666666666667, |
| "alpha_median": 0.02, |
| "ci95_low": 0.0248975, |
| "ci95_high": 0.05433458333333333 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.027649999999999997, |
| "alpha_median": 0.009, |
| "ci95_low": 0.016564583333333334, |
| "ci95_high": 0.041801250000000005 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.5866896551724138, |
| "alpha_median": 0.62, |
| "ci95_low": 0.5381525862068965, |
| "ci95_high": 0.6317435344827586 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.34374137931034476, |
| "alpha_median": 0.3615, |
| "ci95_low": 0.3013275862068966, |
| "ci95_high": 0.38341163793103433 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.17375862068965517, |
| "alpha_median": 0.1685, |
| "ci95_low": 0.14348017241379313, |
| "ci95_high": 0.20646594827586204 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.11463793103448276, |
| "alpha_median": 0.08549999999999999, |
| "ci95_low": 0.08758534482758619, |
| "ci95_high": 0.1466258620689655 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.06991379310344828, |
| "alpha_median": 0.041499999999999995, |
| "ci95_low": 0.04782629310344828, |
| "ci95_high": 0.09610560344827586 |
| }, |
| { |
| "position": 6, |
| "n": 58, |
| "alpha_mean": 0.05153448275862069, |
| "alpha_median": 0.019, |
| "ci95_low": 0.031843534482758624, |
| "ci95_high": 0.07505689655172411 |
| }, |
| { |
| "position": 7, |
| "n": 58, |
| "alpha_mean": 0.03632758620689655, |
| "alpha_median": 0.0095, |
| "ci95_low": 0.02136206896551724, |
| "ci95_high": 0.05450474137931032 |
| } |
| ] |
| } |
| } |
| }, |
| "ksweep": { |
| "ksweep-gemma-q4-dflash-f16-k10": { |
| "family": "ksweep", |
| "target": "gemma-q4", |
| "drafter": "dflash-f16", |
| "k": 10, |
| "n_excluded": 3, |
| "n_matched": 175, |
| "log_lines": 175, |
| "max_alpha_diff": 5.0000000000050004e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-gemma-q4-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 175, |
| "baseline_n_matched": 175, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 55 |
| }, |
| "baseline_excluded": 3, |
| "baseline_all": 33.47058285714286, |
| "baseline_by_domain": { |
| "math": 32.767716666666665, |
| "code": 32.50666666666667, |
| "chat": 35.28889090909091 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8581333333333333, |
| "alpha_median": 0.86, |
| "ci95_low": 0.8396154166666667, |
| "ci95_high": 0.8742508333333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7290666666666668, |
| "alpha_median": 0.7255, |
| "ci95_low": 0.70286625, |
| "ci95_high": 0.7540858333333336 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.58595, |
| "alpha_median": 0.5934999999999999, |
| "ci95_low": 0.5582470833333333, |
| "ci95_high": 0.6129883333333332 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.4879833333333333, |
| "alpha_median": 0.471, |
| "ci95_low": 0.4562466666666666, |
| "ci95_high": 0.51908375 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.3957166666666667, |
| "alpha_median": 0.375, |
| "ci95_low": 0.36298125, |
| "ci95_high": 0.42700749999999993 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.3183333333333333, |
| "alpha_median": 0.303, |
| "ci95_low": 0.28596375000000007, |
| "ci95_high": 0.3491191666666666 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.2571666666666667, |
| "alpha_median": 0.2255, |
| "ci95_low": 0.2281825, |
| "ci95_high": 0.284435 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.2046666666666667, |
| "alpha_median": 0.184, |
| "ci95_low": 0.17776083333333334, |
| "ci95_high": 0.23036874999999998 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.1633666666666667, |
| "alpha_median": 0.1615, |
| "ci95_low": 0.14023, |
| "ci95_high": 0.1857675 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.119, |
| "alpha_median": 0.10200000000000001, |
| "ci95_low": 0.09944916666666667, |
| "ci95_high": 0.13833458333333334 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.7975833333333332, |
| "alpha_median": 0.805, |
| "ci95_low": 0.7638662500000001, |
| "ci95_high": 0.8274833333333335 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6294500000000001, |
| "alpha_median": 0.6485000000000001, |
| "ci95_low": 0.5893558333333334, |
| "ci95_high": 0.66490375 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.4983833333333333, |
| "alpha_median": 0.492, |
| "ci95_low": 0.45793291666666663, |
| "ci95_high": 0.5402670833333332 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.3997666666666666, |
| "alpha_median": 0.3985, |
| "ci95_low": 0.3604479166666667, |
| "ci95_high": 0.44135541666666667 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.3185666666666666, |
| "alpha_median": 0.2845, |
| "ci95_low": 0.28028624999999996, |
| "ci95_high": 0.35915333333333327 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.25938333333333335, |
| "alpha_median": 0.225, |
| "ci95_low": 0.22559875, |
| "ci95_high": 0.29620625 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.20616666666666666, |
| "alpha_median": 0.176, |
| "ci95_low": 0.175915, |
| "ci95_high": 0.24012041666666661 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.16380000000000003, |
| "alpha_median": 0.1355, |
| "ci95_low": 0.13546458333333336, |
| "ci95_high": 0.19675666666666658 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.13241666666666668, |
| "alpha_median": 0.0935, |
| "ci95_low": 0.10526500000000001, |
| "ci95_high": 0.16395499999999996 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.10470000000000002, |
| "alpha_median": 0.0765, |
| "ci95_low": 0.08091625000000001, |
| "ci95_high": 0.13463999999999995 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 55, |
| "alpha_mean": 0.6401454545454547, |
| "alpha_median": 0.667, |
| "ci95_low": 0.5908345454545455, |
| "ci95_high": 0.6839649999999998 |
| }, |
| { |
| "position": 2, |
| "n": 55, |
| "alpha_mean": 0.4230181818181819, |
| "alpha_median": 0.422, |
| "ci95_low": 0.36814090909090913, |
| "ci95_high": 0.47485499999999997 |
| }, |
| { |
| "position": 3, |
| "n": 55, |
| "alpha_mean": 0.2565090909090909, |
| "alpha_median": 0.206, |
| "ci95_low": 0.21030545454545452, |
| "ci95_high": 0.3052913636363636 |
| }, |
| { |
| "position": 4, |
| "n": 55, |
| "alpha_mean": 0.1807272727272727, |
| "alpha_median": 0.137, |
| "ci95_low": 0.13614363636363633, |
| "ci95_high": 0.2264454545454545 |
| }, |
| { |
| "position": 5, |
| "n": 55, |
| "alpha_mean": 0.1324, |
| "alpha_median": 0.082, |
| "ci95_low": 0.09220954545454546, |
| "ci95_high": 0.17708409090909083 |
| }, |
| { |
| "position": 6, |
| "n": 55, |
| "alpha_mean": 0.09461818181818181, |
| "alpha_median": 0.041, |
| "ci95_low": 0.05852681818181819, |
| "ci95_high": 0.13318727272727268 |
| }, |
| { |
| "position": 7, |
| "n": 55, |
| "alpha_mean": 0.07121818181818182, |
| "alpha_median": 0.02, |
| "ci95_low": 0.03836272727272727, |
| "ci95_high": 0.10632727272727273 |
| }, |
| { |
| "position": 8, |
| "n": 55, |
| "alpha_mean": 0.05894545454545455, |
| "alpha_median": 0.011, |
| "ci95_low": 0.029181818181818187, |
| "ci95_high": 0.09329181818181818 |
| }, |
| { |
| "position": 9, |
| "n": 55, |
| "alpha_mean": 0.04472727272727272, |
| "alpha_median": 0.009, |
| "ci95_low": 0.018727272727272728, |
| "ci95_high": 0.07535272727272724 |
| }, |
| { |
| "position": 10, |
| "n": 55, |
| "alpha_mean": 0.03549090909090909, |
| "alpha_median": 0.0, |
| "ci95_low": 0.013397727272727275, |
| "ci95_high": 0.06263681818181818 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 175, |
| "beta": 185.65338930253327, |
| "intercept": 21.412957738796678, |
| "r2": 0.9968518546115529, |
| "alpha_be": 0.06494697006957176, |
| "ci95": 0.00819673142708027 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 182.85802939153714, |
| "intercept": 22.722131583509295, |
| "r2": 0.9975750053585383, |
| "alpha_be": 0.05493652707832522, |
| "ci95": 0.012397537312855884 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 188.60128085464027, |
| "intercept": 20.246533752861733, |
| "r2": 0.9985554499572545, |
| "alpha_be": 0.06500556548846625, |
| "ci95": 0.009562440138197638 |
| }, |
| "chat": { |
| "n": 55, |
| "beta": 183.88182014594216, |
| "intercept": 21.735519912245216, |
| "r2": 0.9904598049419733, |
| "alpha_be": 0.07370696562655699, |
| "ci95": 0.026042021172539336 |
| } |
| } |
| }, |
| "ksweep-gemma-q4-dflash-f16-k5": { |
| "family": "ksweep", |
| "target": "gemma-q4", |
| "drafter": "dflash-f16", |
| "k": 5, |
| "n_excluded": 3, |
| "n_matched": 175, |
| "log_lines": 175, |
| "max_alpha_diff": 4.999999999999449e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-gemma-q4-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 175, |
| "baseline_n_matched": 175, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 55 |
| }, |
| "baseline_excluded": 3, |
| "baseline_all": 33.47058285714286, |
| "baseline_by_domain": { |
| "math": 32.767716666666665, |
| "code": 32.50666666666667, |
| "chat": 35.28889090909091 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8648166666666665, |
| "alpha_median": 0.8805000000000001, |
| "ci95_low": 0.84664875, |
| "ci95_high": 0.8817375 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7476166666666665, |
| "alpha_median": 0.754, |
| "ci95_low": 0.7237820833333334, |
| "ci95_high": 0.7708 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6148666666666667, |
| "alpha_median": 0.618, |
| "ci95_low": 0.5844333333333334, |
| "ci95_high": 0.6425420833333334 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.5182333333333334, |
| "alpha_median": 0.504, |
| "ci95_low": 0.48459916666666675, |
| "ci95_high": 0.5506170833333333 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.43478333333333324, |
| "alpha_median": 0.437, |
| "ci95_low": 0.40129875000000004, |
| "ci95_high": 0.4672016666666667 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8290833333333333, |
| "alpha_median": 0.831, |
| "ci95_low": 0.8066158333333333, |
| "ci95_high": 0.8515845833333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6821666666666667, |
| "alpha_median": 0.68, |
| "ci95_low": 0.6523333333333334, |
| "ci95_high": 0.7136345833333332 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5617, |
| "alpha_median": 0.5665, |
| "ci95_low": 0.5239529166666667, |
| "ci95_high": 0.5996862499999999 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.46573333333333333, |
| "alpha_median": 0.462, |
| "ci95_low": 0.42466125, |
| "ci95_high": 0.5074033333333333 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.39039999999999997, |
| "alpha_median": 0.377, |
| "ci95_low": 0.34859750000000006, |
| "ci95_high": 0.4326854166666667 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 55, |
| "alpha_mean": 0.6564363636363636, |
| "alpha_median": 0.68, |
| "ci95_low": 0.6045904545454546, |
| "ci95_high": 0.7042477272727272 |
| }, |
| { |
| "position": 2, |
| "n": 55, |
| "alpha_mean": 0.4465636363636364, |
| "alpha_median": 0.413, |
| "ci95_low": 0.39149863636363635, |
| "ci95_high": 0.5005781818181817 |
| }, |
| { |
| "position": 3, |
| "n": 55, |
| "alpha_mean": 0.2991818181818182, |
| "alpha_median": 0.25, |
| "ci95_low": 0.24532045454545454, |
| "ci95_high": 0.35314772727272725 |
| }, |
| { |
| "position": 4, |
| "n": 55, |
| "alpha_mean": 0.2236545454545455, |
| "alpha_median": 0.149, |
| "ci95_low": 0.1724536363636364, |
| "ci95_high": 0.2765854545454545 |
| }, |
| { |
| "position": 5, |
| "n": 55, |
| "alpha_mean": 0.1691272727272728, |
| "alpha_median": 0.106, |
| "ci95_low": 0.11848954545454544, |
| "ci95_high": 0.2225631818181817 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 175, |
| "beta": 96.77710115191583, |
| "intercept": 22.015537871718465, |
| "r2": 0.9931431470600357, |
| "alpha_be": 0.11836524187103765, |
| "ci95": 0.01160099637459681 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 98.1041006766699, |
| "intercept": 21.27439792774494, |
| "r2": 0.998207231404384, |
| "alpha_be": 0.11715431525947363, |
| "ci95": 0.010089068401194437 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 99.30438547933836, |
| "intercept": 20.365855063842325, |
| "r2": 0.9979091343950341, |
| "alpha_be": 0.1222585643546468, |
| "ci95": 0.010933479480822012 |
| }, |
| "chat": { |
| "n": 55, |
| "beta": 95.1663825453732, |
| "intercept": 22.65345979970406, |
| "r2": 0.9799522512950171, |
| "alpha_be": 0.1327720017450759, |
| "ci95": 0.036674618802919556 |
| } |
| } |
| }, |
| "ksweep-gemma-q4-dflash-f16-k7": { |
| "family": "ksweep", |
| "target": "gemma-q4", |
| "drafter": "dflash-f16", |
| "k": 7, |
| "n_excluded": 3, |
| "n_matched": 175, |
| "log_lines": 175, |
| "max_alpha_diff": 5.0000000000050004e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-gemma-q4-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 175, |
| "baseline_n_matched": 175, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 55 |
| }, |
| "baseline_excluded": 3, |
| "baseline_all": 33.47058285714286, |
| "baseline_by_domain": { |
| "math": 32.767716666666665, |
| "code": 32.50666666666667, |
| "chat": 35.28889090909091 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8579166666666665, |
| "alpha_median": 0.867, |
| "ci95_low": 0.8395308333333333, |
| "ci95_high": 0.8751016666666668 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7313999999999999, |
| "alpha_median": 0.7404999999999999, |
| "ci95_low": 0.7069320833333336, |
| "ci95_high": 0.7555179166666667 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5955666666666666, |
| "alpha_median": 0.605, |
| "ci95_low": 0.5640833333333333, |
| "ci95_high": 0.6246870833333333 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.49323333333333336, |
| "alpha_median": 0.482, |
| "ci95_low": 0.45769624999999997, |
| "ci95_high": 0.5276183333333333 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.4169, |
| "alpha_median": 0.391, |
| "ci95_low": 0.38221625000000004, |
| "ci95_high": 0.4508516666666667 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.35129999999999995, |
| "alpha_median": 0.333, |
| "ci95_low": 0.3176629166666667, |
| "ci95_high": 0.38515333333333324 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.28978333333333334, |
| "alpha_median": 0.2665, |
| "ci95_low": 0.2591608333333334, |
| "ci95_high": 0.32101874999999996 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8238166666666665, |
| "alpha_median": 0.823, |
| "ci95_low": 0.7995483333333334, |
| "ci95_high": 0.8470179166666668 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6661166666666667, |
| "alpha_median": 0.667, |
| "ci95_low": 0.633095, |
| "ci95_high": 0.6982841666666669 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5352166666666667, |
| "alpha_median": 0.534, |
| "ci95_low": 0.49966374999999996, |
| "ci95_high": 0.5731175000000002 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.4425833333333334, |
| "alpha_median": 0.441, |
| "ci95_low": 0.4045970833333334, |
| "ci95_high": 0.48081 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.36938333333333334, |
| "alpha_median": 0.3695, |
| "ci95_low": 0.33084708333333335, |
| "ci95_high": 0.40810291666666665 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.3013, |
| "alpha_median": 0.2845, |
| "ci95_low": 0.26406583333333333, |
| "ci95_high": 0.33970374999999986 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.24645, |
| "alpha_median": 0.2235, |
| "ci95_low": 0.2119533333333333, |
| "ci95_high": 0.2845670833333334 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 55, |
| "alpha_mean": 0.6426181818181819, |
| "alpha_median": 0.658, |
| "ci95_low": 0.5934681818181818, |
| "ci95_high": 0.6863299999999998 |
| }, |
| { |
| "position": 2, |
| "n": 55, |
| "alpha_mean": 0.4292727272727273, |
| "alpha_median": 0.409, |
| "ci95_low": 0.3773263636363636, |
| "ci95_high": 0.4804581818181817 |
| }, |
| { |
| "position": 3, |
| "n": 55, |
| "alpha_mean": 0.2806, |
| "alpha_median": 0.242, |
| "ci95_low": 0.23070636363636363, |
| "ci95_high": 0.33234681818181816 |
| }, |
| { |
| "position": 4, |
| "n": 55, |
| "alpha_mean": 0.20427272727272722, |
| "alpha_median": 0.151, |
| "ci95_low": 0.15617499999999998, |
| "ci95_high": 0.2536563636363636 |
| }, |
| { |
| "position": 5, |
| "n": 55, |
| "alpha_mean": 0.1530363636363636, |
| "alpha_median": 0.087, |
| "ci95_low": 0.10728318181818182, |
| "ci95_high": 0.20192045454545446 |
| }, |
| { |
| "position": 6, |
| "n": 55, |
| "alpha_mean": 0.12018181818181817, |
| "alpha_median": 0.051, |
| "ci95_low": 0.07734409090909093, |
| "ci95_high": 0.1675277272727273 |
| }, |
| { |
| "position": 7, |
| "n": 55, |
| "alpha_mean": 0.09421818181818181, |
| "alpha_median": 0.028, |
| "ci95_low": 0.052829999999999995, |
| "ci95_high": 0.14032772727272727 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 175, |
| "beta": 110.31798887226992, |
| "intercept": 17.599562990669483, |
| "r2": 0.996198622557111, |
| "alpha_be": 0.14386610949597173, |
| "ci95": 0.008626682072197842 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 110.87923409298594, |
| "intercept": 17.376101245581285, |
| "r2": 0.9994551157203211, |
| "alpha_be": 0.1388142292557469, |
| "ci95": 0.005560251667080671 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 111.79921770706834, |
| "intercept": 16.774345991889742, |
| "r2": 0.9988222078385945, |
| "alpha_be": 0.1407194164452751, |
| "ci95": 0.008232365607627924 |
| }, |
| "chat": { |
| "n": 55, |
| "beta": 109.08969313042472, |
| "intercept": 17.96546411810064, |
| "r2": 0.9873296482279417, |
| "alpha_be": 0.15879984894887222, |
| "ci95": 0.029167721183299196 |
| } |
| } |
| }, |
| "ksweep-gemma-q4-mtp-k10": { |
| "family": "ksweep", |
| "target": "gemma-q4", |
| "drafter": "mtp", |
| "k": 10, |
| "n_excluded": 3, |
| "n_matched": 175, |
| "log_lines": 175, |
| "max_alpha_diff": 5.0000000000105516e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-gemma-q4-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 175, |
| "baseline_n_matched": 175, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 55 |
| }, |
| "baseline_excluded": 3, |
| "baseline_all": 33.47058285714286, |
| "baseline_by_domain": { |
| "math": 32.767716666666665, |
| "code": 32.50666666666667, |
| "chat": 35.28889090909091 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8836833333333335, |
| "alpha_median": 0.8905000000000001, |
| "ci95_low": 0.8681633333333334, |
| "ci95_high": 0.8991691666666666 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7941166666666667, |
| "alpha_median": 0.8, |
| "ci95_low": 0.7705591666666666, |
| "ci95_high": 0.8170041666666669 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.7047999999999999, |
| "alpha_median": 0.7195, |
| "ci95_low": 0.6783975000000001, |
| "ci95_high": 0.7299704166666666 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.6122833333333333, |
| "alpha_median": 0.6385000000000001, |
| "ci95_low": 0.5747329166666666, |
| "ci95_high": 0.6443341666666665 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.5341833333333332, |
| "alpha_median": 0.549, |
| "ci95_low": 0.49784874999999995, |
| "ci95_high": 0.5669670833333333 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.45953333333333324, |
| "alpha_median": 0.463, |
| "ci95_low": 0.42431375, |
| "ci95_high": 0.49294 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.3834, |
| "alpha_median": 0.3735, |
| "ci95_low": 0.3501558333333333, |
| "ci95_high": 0.4155025 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.3148833333333334, |
| "alpha_median": 0.3115, |
| "ci95_low": 0.2848829166666666, |
| "ci95_high": 0.3450333333333333 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.25093333333333334, |
| "alpha_median": 0.2415, |
| "ci95_low": 0.22318125000000003, |
| "ci95_high": 0.28111875000000003 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.20245, |
| "alpha_median": 0.2, |
| "ci95_low": 0.17943208333333335, |
| "ci95_high": 0.22818499999999997 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8295333333333333, |
| "alpha_median": 0.8434999999999999, |
| "ci95_low": 0.80103125, |
| "ci95_high": 0.8527508333333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7212499999999998, |
| "alpha_median": 0.732, |
| "ci95_low": 0.6896574999999999, |
| "ci95_high": 0.7506525000000001 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6143499999999998, |
| "alpha_median": 0.604, |
| "ci95_low": 0.5809475, |
| "ci95_high": 0.6462687500000001 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.5350666666666667, |
| "alpha_median": 0.523, |
| "ci95_low": 0.5002779166666668, |
| "ci95_high": 0.5712199999999998 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.46813333333333335, |
| "alpha_median": 0.4495, |
| "ci95_low": 0.43272666666666665, |
| "ci95_high": 0.5032670833333333 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.39171666666666666, |
| "alpha_median": 0.377, |
| "ci95_low": 0.35819833333333334, |
| "ci95_high": 0.4262558333333332 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.3330666666666667, |
| "alpha_median": 0.309, |
| "ci95_low": 0.2998495833333334, |
| "ci95_high": 0.36738375 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.27935, |
| "alpha_median": 0.25, |
| "ci95_low": 0.2476816666666667, |
| "ci95_high": 0.3136854166666667 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.23356666666666667, |
| "alpha_median": 0.214, |
| "ci95_low": 0.2039, |
| "ci95_high": 0.2661545833333333 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.18168333333333336, |
| "alpha_median": 0.159, |
| "ci95_low": 0.1529316666666667, |
| "ci95_high": 0.21411833333333333 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 55, |
| "alpha_mean": 0.7337636363636365, |
| "alpha_median": 0.747, |
| "ci95_low": 0.6890713636363638, |
| "ci95_high": 0.7732472727272728 |
| }, |
| { |
| "position": 2, |
| "n": 55, |
| "alpha_mean": 0.5758181818181819, |
| "alpha_median": 0.569, |
| "ci95_low": 0.5250227272727273, |
| "ci95_high": 0.6269495454545455 |
| }, |
| { |
| "position": 3, |
| "n": 55, |
| "alpha_mean": 0.44114545454545445, |
| "alpha_median": 0.388, |
| "ci95_low": 0.384850909090909, |
| "ci95_high": 0.5014190909090909 |
| }, |
| { |
| "position": 4, |
| "n": 55, |
| "alpha_mean": 0.34834545454545457, |
| "alpha_median": 0.277, |
| "ci95_low": 0.28816, |
| "ci95_high": 0.4101668181818182 |
| }, |
| { |
| "position": 5, |
| "n": 55, |
| "alpha_mean": 0.26996363636363635, |
| "alpha_median": 0.192, |
| "ci95_low": 0.21188772727272728, |
| "ci95_high": 0.3302231818181817 |
| }, |
| { |
| "position": 6, |
| "n": 55, |
| "alpha_mean": 0.22503636363636365, |
| "alpha_median": 0.139, |
| "ci95_low": 0.1671940909090909, |
| "ci95_high": 0.28587045454545446 |
| }, |
| { |
| "position": 7, |
| "n": 55, |
| "alpha_mean": 0.19132727272727276, |
| "alpha_median": 0.092, |
| "ci95_low": 0.13407136363636363, |
| "ci95_high": 0.25535454545454545 |
| }, |
| { |
| "position": 8, |
| "n": 55, |
| "alpha_mean": 0.16399999999999998, |
| "alpha_median": 0.069, |
| "ci95_low": 0.10636363636363637, |
| "ci95_high": 0.22853 |
| }, |
| { |
| "position": 9, |
| "n": 55, |
| "alpha_mean": 0.1394, |
| "alpha_median": 0.043, |
| "ci95_low": 0.08196227272727274, |
| "ci95_high": 0.20359090909090902 |
| }, |
| { |
| "position": 10, |
| "n": 55, |
| "alpha_mean": 0.12441818181818183, |
| "alpha_median": 0.028, |
| "ci95_low": 0.065955, |
| "ci95_high": 0.18841636363636347 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 175, |
| "beta": 162.16078678337638, |
| "intercept": 19.85372457913967, |
| "r2": 0.995616045136786, |
| "alpha_be": 0.08397133825080266, |
| "ci95": 0.00952341837434763 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 160.9253770995331, |
| "intercept": 20.989059374832642, |
| "r2": 0.9976761270887337, |
| "alpha_be": 0.07319328687699062, |
| "ci95": 0.011947128835343374 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 162.2759948294464, |
| "intercept": 19.606006638373273, |
| "r2": 0.9941057004073529, |
| "alpha_be": 0.07949826492730554, |
| "ci95": 0.019085179496654565 |
| }, |
| "chat": { |
| "n": 55, |
| "beta": 160.61421923484983, |
| "intercept": 20.033221560650826, |
| "r2": 0.9936719153258121, |
| "alpha_be": 0.09498330484758183, |
| "ci95": 0.02082473328687092 |
| } |
| } |
| }, |
| "ksweep-gemma-q4-mtp-k5": { |
| "family": "ksweep", |
| "target": "gemma-q4", |
| "drafter": "mtp", |
| "k": 5, |
| "n_excluded": 3, |
| "n_matched": 175, |
| "log_lines": 175, |
| "max_alpha_diff": 5.0000000000050004e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-gemma-q4-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 175, |
| "baseline_n_matched": 175, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 55 |
| }, |
| "baseline_excluded": 3, |
| "baseline_all": 33.47058285714286, |
| "baseline_by_domain": { |
| "math": 32.767716666666665, |
| "code": 32.50666666666667, |
| "chat": 35.28889090909091 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8950166666666667, |
| "alpha_median": 0.8975, |
| "ci95_low": 0.8799124999999999, |
| "ci95_high": 0.9086012499999998 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.8061833333333333, |
| "alpha_median": 0.8085, |
| "ci95_low": 0.7858991666666666, |
| "ci95_high": 0.8246529166666665 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.7245333333333333, |
| "alpha_median": 0.743, |
| "ci95_low": 0.6977758333333333, |
| "ci95_high": 0.7487174999999999 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.6364333333333334, |
| "alpha_median": 0.653, |
| "ci95_low": 0.6066799999999999, |
| "ci95_high": 0.6637341666666666 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.5607833333333334, |
| "alpha_median": 0.56, |
| "ci95_low": 0.5305791666666666, |
| "ci95_high": 0.5900512499999999 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8677666666666666, |
| "alpha_median": 0.885, |
| "ci95_low": 0.8488662499999999, |
| "ci95_high": 0.88593375 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7681333333333333, |
| "alpha_median": 0.774, |
| "ci95_low": 0.7418133333333332, |
| "ci95_high": 0.7935254166666666 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6667833333333332, |
| "alpha_median": 0.661, |
| "ci95_low": 0.6357720833333335, |
| "ci95_high": 0.6976683333333333 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.5806333333333333, |
| "alpha_median": 0.568, |
| "ci95_low": 0.544045, |
| "ci95_high": 0.6166670833333333 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.5109833333333332, |
| "alpha_median": 0.5, |
| "ci95_low": 0.47249791666666663, |
| "ci95_high": 0.5504029166666666 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 55, |
| "alpha_mean": 0.7516363636363635, |
| "alpha_median": 0.757, |
| "ci95_low": 0.7077790909090909, |
| "ci95_high": 0.7898595454545454 |
| }, |
| { |
| "position": 2, |
| "n": 55, |
| "alpha_mean": 0.5932545454545456, |
| "alpha_median": 0.585, |
| "ci95_low": 0.5417622727272728, |
| "ci95_high": 0.6486572727272726 |
| }, |
| { |
| "position": 3, |
| "n": 55, |
| "alpha_mean": 0.48161818181818183, |
| "alpha_median": 0.412, |
| "ci95_low": 0.42208954545454547, |
| "ci95_high": 0.5441836363636364 |
| }, |
| { |
| "position": 4, |
| "n": 55, |
| "alpha_mean": 0.39509090909090916, |
| "alpha_median": 0.325, |
| "ci95_low": 0.3306468181818182, |
| "ci95_high": 0.46265500000000004 |
| }, |
| { |
| "position": 5, |
| "n": 55, |
| "alpha_mean": 0.32367272727272733, |
| "alpha_median": 0.219, |
| "ci95_low": 0.25792818181818183, |
| "ci95_high": 0.3945495454545453 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 175, |
| "beta": 97.1633436471974, |
| "intercept": 23.028844760741123, |
| "r2": 0.9896940683887518, |
| "alpha_be": 0.1074658168857996, |
| "ci95": 0.014154215802069242 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 98.18364299128696, |
| "intercept": 22.56271377246348, |
| "r2": 0.9969393889040341, |
| "alpha_be": 0.10393791250044371, |
| "ci95": 0.013178769981063175 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 100.48112287228884, |
| "intercept": 20.527863286387202, |
| "r2": 0.99232136023649, |
| "alpha_be": 0.11921446574103758, |
| "ci95": 0.02079684140496075 |
| }, |
| "chat": { |
| "n": 55, |
| "beta": 95.43848076041085, |
| "intercept": 23.863787534554636, |
| "r2": 0.9820395916741389, |
| "alpha_be": 0.11971170625837915, |
| "ci95": 0.03418819646866108 |
| } |
| } |
| }, |
| "ksweep-gemma-q4-mtp-k7": { |
| "family": "ksweep", |
| "target": "gemma-q4", |
| "drafter": "mtp", |
| "k": 7, |
| "n_excluded": 3, |
| "n_matched": 175, |
| "log_lines": 175, |
| "max_alpha_diff": 5.0000000000050004e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-gemma-q4-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 175, |
| "baseline_n_matched": 175, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 55 |
| }, |
| "baseline_excluded": 3, |
| "baseline_all": 33.47058285714286, |
| "baseline_by_domain": { |
| "math": 32.767716666666665, |
| "code": 32.50666666666667, |
| "chat": 35.28889090909091 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8807333333333333, |
| "alpha_median": 0.883, |
| "ci95_low": 0.8624995833333334, |
| "ci95_high": 0.8964683333333335 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.79455, |
| "alpha_median": 0.8025, |
| "ci95_low": 0.7706141666666667, |
| "ci95_high": 0.8169733333333333 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.712, |
| "alpha_median": 0.7084999999999999, |
| "ci95_low": 0.6851662500000001, |
| "ci95_high": 0.7384850000000002 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.6325999999999999, |
| "alpha_median": 0.638, |
| "ci95_low": 0.6014795833333332, |
| "ci95_high": 0.6633079166666666 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.5557833333333332, |
| "alpha_median": 0.553, |
| "ci95_low": 0.5222575000000002, |
| "ci95_high": 0.5895004166666666 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.47598333333333337, |
| "alpha_median": 0.4825, |
| "ci95_low": 0.43973083333333335, |
| "ci95_high": 0.5133391666666666 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.4049833333333333, |
| "alpha_median": 0.39, |
| "ci95_low": 0.3712995833333333, |
| "ci95_high": 0.43846416666666665 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8569333333333334, |
| "alpha_median": 0.874, |
| "ci95_low": 0.8370483333333333, |
| "ci95_high": 0.8763833333333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7463333333333334, |
| "alpha_median": 0.763, |
| "ci95_low": 0.718665, |
| "ci95_high": 0.7722341666666669 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6522333333333332, |
| "alpha_median": 0.654, |
| "ci95_low": 0.6200466666666665, |
| "ci95_high": 0.6846175 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.5665999999999999, |
| "alpha_median": 0.5680000000000001, |
| "ci95_low": 0.5307308333333333, |
| "ci95_high": 0.6019720833333333 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.49448333333333333, |
| "alpha_median": 0.4755, |
| "ci95_low": 0.4554829166666667, |
| "ci95_high": 0.5327204166666667 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.42051666666666665, |
| "alpha_median": 0.397, |
| "ci95_low": 0.38084958333333335, |
| "ci95_high": 0.4608499999999999 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.3630166666666667, |
| "alpha_median": 0.3365, |
| "ci95_low": 0.3226812499999999, |
| "ci95_high": 0.4040516666666667 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 55, |
| "alpha_mean": 0.7442181818181818, |
| "alpha_median": 0.75, |
| "ci95_low": 0.7010240909090908, |
| "ci95_high": 0.7815731818181817 |
| }, |
| { |
| "position": 2, |
| "n": 55, |
| "alpha_mean": 0.5852181818181819, |
| "alpha_median": 0.561, |
| "ci95_low": 0.53427, |
| "ci95_high": 0.6413472727272729 |
| }, |
| { |
| "position": 3, |
| "n": 55, |
| "alpha_mean": 0.4758545454545455, |
| "alpha_median": 0.411, |
| "ci95_low": 0.41785954545454534, |
| "ci95_high": 0.5385318181818182 |
| }, |
| { |
| "position": 4, |
| "n": 55, |
| "alpha_mean": 0.3937818181818182, |
| "alpha_median": 0.301, |
| "ci95_low": 0.32965363636363637, |
| "ci95_high": 0.4654922727272726 |
| }, |
| { |
| "position": 5, |
| "n": 55, |
| "alpha_mean": 0.32007272727272723, |
| "alpha_median": 0.203, |
| "ci95_low": 0.2553986363636363, |
| "ci95_high": 0.3890754545454545 |
| }, |
| { |
| "position": 6, |
| "n": 55, |
| "alpha_mean": 0.2712, |
| "alpha_median": 0.159, |
| "ci95_low": 0.2051377272727273, |
| "ci95_high": 0.34276909090909086 |
| }, |
| { |
| "position": 7, |
| "n": 55, |
| "alpha_mean": 0.23603636363636366, |
| "alpha_median": 0.113, |
| "ci95_low": 0.16681272727272728, |
| "ci95_high": 0.3091281818181818 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 175, |
| "beta": 106.382523204685, |
| "intercept": 17.86586129833469, |
| "r2": 0.992550436179729, |
| "alpha_be": 0.14668501073981813, |
| "ci95": 0.011853202195414992 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 107.52488863148434, |
| "intercept": 17.432174324785418, |
| "r2": 0.998683017231229, |
| "alpha_be": 0.14262318740398908, |
| "ci95": 0.00848984843653783 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 107.19500825696319, |
| "intercept": 17.29974734812864, |
| "r2": 0.9960797533568224, |
| "alpha_be": 0.1418621964381463, |
| "ci95": 0.014793095909013997 |
| }, |
| "chat": { |
| "n": 55, |
| "beta": 104.72962189772807, |
| "intercept": 18.35244241291177, |
| "r2": 0.9864485826641572, |
| "alpha_be": 0.16171593279233015, |
| "ci95": 0.029348998495359956 |
| } |
| } |
| }, |
| "ksweep-qwen-q4-dflash-k10": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "dflash-f16", |
| "k": 10, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.000000000000143e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-qwen-q4-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 178, |
| "baseline_n_matched": 178, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 58 |
| }, |
| "baseline_excluded": 0, |
| "baseline_all": 51.49112359550562, |
| "baseline_by_domain": { |
| "math": 53.47833333333333, |
| "code": 51.10905000000001, |
| "chat": 49.83063793103449 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.014683333333333335, |
| "alpha_median": 0.01, |
| "ci95_low": 0.011265, |
| "ci95_high": 0.0186 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.0034000000000000002, |
| "alpha_median": 0.0, |
| "ci95_low": 0.001816666666666667, |
| "ci95_high": 0.0056 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.0017, |
| "alpha_median": 0.0, |
| "ci95_low": 0.00095, |
| "ci95_high": 0.002533749999999998 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.0011333333333333334, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0006666666666666666, |
| "ci95_high": 0.0016666666666666668 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.0006000000000000001, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0002666666666666667, |
| "ci95_high": 0.001 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.0003333333333333333, |
| "alpha_median": 0.0, |
| "ci95_low": 6.666666666666667e-05, |
| "ci95_high": 0.0006666666666666666 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.013800000000000003, |
| "alpha_median": 0.012, |
| "ci95_low": 0.011216250000000002, |
| "ci95_high": 0.01651666666666667 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.003916666666666667, |
| "alpha_median": 0.004, |
| "ci95_low": 0.002766666666666667, |
| "ci95_high": 0.005233333333333334 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.00155, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0008666666666666667, |
| "ci95_high": 0.0023833333333333337 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.0006166666666666667, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0002, |
| "ci95_high": 0.0011166666666666666 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.0003333333333333333, |
| "alpha_median": 0.0, |
| "ci95_low": 6.666666666666667e-05, |
| "ci95_high": 0.0006666666666666666 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.0002, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.00046666666666666666 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.021448275862068968, |
| "alpha_median": 0.016, |
| "ci95_low": 0.017085344827586207, |
| "ci95_high": 0.026397413793103446 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.004586206896551724, |
| "alpha_median": 0.004, |
| "ci95_low": 0.003327586206896552, |
| "ci95_high": 0.005983189655172413 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.0018965517241379313, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0011379310344827587, |
| "ci95_high": 0.0027241379310344828 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.0010000000000000002, |
| "alpha_median": 0.0, |
| "ci95_low": 0.00041379310344827585, |
| "ci95_high": 0.0016724137931034483 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.0005172413793103448, |
| "alpha_median": 0.0, |
| "ci95_low": 6.896551724137931e-05, |
| "ci95_high": 0.0011379310344827585 |
| }, |
| { |
| "position": 6, |
| "n": 58, |
| "alpha_mean": 0.0002931034482758621, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.000689655172413793 |
| }, |
| { |
| "position": 7, |
| "n": 58, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 8, |
| "n": 58, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 9, |
| "n": 58, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 10, |
| "n": 58, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 178, |
| "beta": 155.1157695563617, |
| "intercept": 26.66819869703642, |
| "r2": 0.8579197421864757, |
| "alpha_be": 0.16002837731756042, |
| "ci95": 0.06008432818710706 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 144.1451037644214, |
| "intercept": 26.594035482046554, |
| "r2": 0.7263764843539029, |
| "alpha_be": 0.18650857468751908, |
| "ci95": 0.15784748900271323 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 152.7967926107302, |
| "intercept": 26.696375780270486, |
| "r2": 0.9301975990488072, |
| "alpha_be": 0.15977216407888878, |
| "ci95": 0.07046106558142584 |
| }, |
| "chat": { |
| "n": 58, |
| "beta": 159.838825494361, |
| "intercept": 26.742363432294788, |
| "r2": 0.9804824586963088, |
| "alpha_be": 0.14444722317828992, |
| "ci95": 0.03692654962141036 |
| } |
| } |
| }, |
| "ksweep-qwen-q4-dflash-k5": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "dflash-f16", |
| "k": 5, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 4.99999999999997e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-qwen-q4-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 178, |
| "baseline_n_matched": 178, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 58 |
| }, |
| "baseline_excluded": 0, |
| "baseline_all": 51.49112359550562, |
| "baseline_by_domain": { |
| "math": 53.47833333333333, |
| "code": 51.10905000000001, |
| "chat": 49.83063793103449 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.013850000000000001, |
| "alpha_median": 0.012, |
| "ci95_low": 0.0105, |
| "ci95_high": 0.0178 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.003433333333333334, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0018658333333333333, |
| "ci95_high": 0.005633333333333334 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.0012666666666666668, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0007333333333333333, |
| "ci95_high": 0.0018666666666666666 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.0008, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0004, |
| "ci95_high": 0.0012000000000000001 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.0006000000000000001, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0002666666666666667, |
| "ci95_high": 0.001 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.012633333333333333, |
| "alpha_median": 0.012, |
| "ci95_low": 0.010132916666666669, |
| "ci95_high": 0.015167083333333331 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.0030166666666666675, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0020162500000000002, |
| "ci95_high": 0.004 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.0012666666666666668, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0007333333333333333, |
| "ci95_high": 0.0018000000000000002 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.0005333333333333334, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0002, |
| "ci95_high": 0.0009333333333333334 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.0002, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.00046666666666666666 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.020879310344827588, |
| "alpha_median": 0.016, |
| "ci95_low": 0.01656853448275862, |
| "ci95_high": 0.025655603448275863 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.003637931034482759, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0024310344827586207, |
| "ci95_high": 0.005051724137931035 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.0012413793103448277, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0006896551724137933, |
| "ci95_high": 0.0017931034482758623 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.0006206896551724138, |
| "alpha_median": 0.0, |
| "ci95_low": 0.00027586206896551725, |
| "ci95_high": 0.0010344827586206897 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.00020689655172413793, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0004827586206896552 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 178, |
| "beta": 142.46581957111744, |
| "intercept": 28.828018645675503, |
| "r2": 0.9628669806519348, |
| "alpha_be": 0.15907748973091004, |
| "ci95": 0.028993313293994443 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 140.20844229250247, |
| "intercept": 28.85900754573448, |
| "r2": 0.9953218488460952, |
| "alpha_be": 0.17559089442159326, |
| "ci95": 0.01763149923102116 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 139.53866272524186, |
| "intercept": 28.78510312523454, |
| "r2": 0.9052874615079485, |
| "alpha_be": 0.15998395311213756, |
| "ci95": 0.08319621564077606 |
| }, |
| "chat": { |
| "n": 58, |
| "beta": 142.81525573315085, |
| "intercept": 28.858762630748114, |
| "r2": 0.9682201355429803, |
| "alpha_be": 0.14684618385218, |
| "ci95": 0.047414082345922426 |
| } |
| } |
| }, |
| "ksweep-qwen-q4-dflash-k7": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "dflash-f16", |
| "k": 7, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.000000000000143e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-qwen-q4-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 178, |
| "baseline_n_matched": 178, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 58 |
| }, |
| "baseline_excluded": 0, |
| "baseline_all": 51.49112359550562, |
| "baseline_by_domain": { |
| "math": 53.47833333333333, |
| "code": 51.10905000000001, |
| "chat": 49.83063793103449 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.014683333333333335, |
| "alpha_median": 0.01, |
| "ci95_low": 0.011265, |
| "ci95_high": 0.0186 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.0034000000000000002, |
| "alpha_median": 0.0, |
| "ci95_low": 0.001816666666666667, |
| "ci95_high": 0.0056 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.0017, |
| "alpha_median": 0.0, |
| "ci95_low": 0.00095, |
| "ci95_high": 0.002533749999999998 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.0011333333333333334, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0006666666666666666, |
| "ci95_high": 0.0016666666666666668 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.0006000000000000001, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0002666666666666667, |
| "ci95_high": 0.001 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.0003333333333333333, |
| "alpha_median": 0.0, |
| "ci95_low": 6.666666666666667e-05, |
| "ci95_high": 0.0006666666666666666 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.013800000000000003, |
| "alpha_median": 0.012, |
| "ci95_low": 0.011216250000000002, |
| "ci95_high": 0.01651666666666667 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.003916666666666667, |
| "alpha_median": 0.004, |
| "ci95_low": 0.002766666666666667, |
| "ci95_high": 0.005233333333333334 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.00155, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0008666666666666667, |
| "ci95_high": 0.0023833333333333337 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.0006166666666666667, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0002, |
| "ci95_high": 0.0011166666666666666 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.0003333333333333333, |
| "alpha_median": 0.0, |
| "ci95_low": 6.666666666666667e-05, |
| "ci95_high": 0.0006666666666666666 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.0002, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.00046666666666666666 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.021448275862068968, |
| "alpha_median": 0.016, |
| "ci95_low": 0.017085344827586207, |
| "ci95_high": 0.026397413793103446 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.004586206896551724, |
| "alpha_median": 0.004, |
| "ci95_low": 0.003327586206896552, |
| "ci95_high": 0.005983189655172413 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.0018965517241379313, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0011379310344827587, |
| "ci95_high": 0.0027241379310344828 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.0010000000000000002, |
| "alpha_median": 0.0, |
| "ci95_low": 0.00041379310344827585, |
| "ci95_high": 0.0016724137931034483 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.0005172413793103448, |
| "alpha_median": 0.0, |
| "ci95_low": 6.896551724137931e-05, |
| "ci95_high": 0.0011379310344827585 |
| }, |
| { |
| "position": 6, |
| "n": 58, |
| "alpha_mean": 0.0002931034482758621, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.000689655172413793 |
| }, |
| { |
| "position": 7, |
| "n": 58, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 178, |
| "beta": 157.20818100115525, |
| "intercept": 26.731094438453738, |
| "r2": 0.9742909534426881, |
| "alpha_be": 0.15749835027268672, |
| "ci95": 0.023983953977647077 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 155.24845144897012, |
| "intercept": 26.766995739962496, |
| "r2": 0.9967171636491443, |
| "alpha_be": 0.17205542048289482, |
| "ci95": 0.014760591731619897 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 152.59434518310505, |
| "intercept": 26.704993927273883, |
| "r2": 0.9321570455788156, |
| "alpha_be": 0.15992765684365673, |
| "ci95": 0.06939197242113475 |
| }, |
| "chat": { |
| "n": 58, |
| "beta": 160.08544693486283, |
| "intercept": 26.73051794741944, |
| "r2": 0.9821992129025822, |
| "alpha_be": 0.1442986881438027, |
| "ci95": 0.035234347004426285 |
| } |
| } |
| }, |
| "ksweep-qwen-q4-dspark-p0-k10": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "dspark-p0", |
| "k": 10, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.0000000000105516e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-qwen-q4-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 178, |
| "baseline_n_matched": 178, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 58 |
| }, |
| "baseline_excluded": 0, |
| "baseline_all": 51.49112359550562, |
| "baseline_by_domain": { |
| "math": 53.47833333333333, |
| "code": 51.10905000000001, |
| "chat": 49.83063793103449 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8275166666666666, |
| "alpha_median": 0.8234999999999999, |
| "ci95_low": 0.80881625, |
| "ci95_high": 0.8464854166666665 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6882666666666667, |
| "alpha_median": 0.7064999999999999, |
| "ci95_low": 0.6566654166666668, |
| "ci95_high": 0.7224 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5771166666666667, |
| "alpha_median": 0.57, |
| "ci95_low": 0.5377991666666667, |
| "ci95_high": 0.6176891666666667 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.48430000000000006, |
| "alpha_median": 0.4585, |
| "ci95_low": 0.4371829166666667, |
| "ci95_high": 0.5336854166666667 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.4117, |
| "alpha_median": 0.4, |
| "ci95_low": 0.36243166666666665, |
| "ci95_high": 0.4655683333333333 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.35425, |
| "alpha_median": 0.339, |
| "ci95_low": 0.30452666666666667, |
| "ci95_high": 0.40827583333333334 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.30565, |
| "alpha_median": 0.295, |
| "ci95_low": 0.2567629166666667, |
| "ci95_high": 0.3597012499999999 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.7943666666666667, |
| "alpha_median": 0.808, |
| "ci95_low": 0.77531625, |
| "ci95_high": 0.8127087500000001 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6135166666666668, |
| "alpha_median": 0.6125, |
| "ci95_low": 0.58530875, |
| "ci95_high": 0.6420545833333332 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.46181666666666665, |
| "alpha_median": 0.4355, |
| "ci95_low": 0.4318120833333333, |
| "ci95_high": 0.4930520833333334 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.3523333333333333, |
| "alpha_median": 0.3255, |
| "ci95_low": 0.3216791666666667, |
| "ci95_high": 0.3854704166666667 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.26839999999999997, |
| "alpha_median": 0.2355, |
| "ci95_low": 0.23628, |
| "ci95_high": 0.3033833333333333 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.19771666666666665, |
| "alpha_median": 0.1725, |
| "ci95_low": 0.16911499999999996, |
| "ci95_high": 0.22833874999999998 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.15255000000000002, |
| "alpha_median": 0.119, |
| "ci95_low": 0.12411666666666668, |
| "ci95_high": 0.18401791666666664 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.7089310344827586, |
| "alpha_median": 0.7275, |
| "ci95_low": 0.665080172413793, |
| "ci95_high": 0.747226724137931 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.4922241379310345, |
| "alpha_median": 0.5095000000000001, |
| "ci95_low": 0.44306379310344834, |
| "ci95_high": 0.5365064655172413 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.35253448275862065, |
| "alpha_median": 0.3375, |
| "ci95_low": 0.30437112068965516, |
| "ci95_high": 0.39734525862068965 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.2588793103448276, |
| "alpha_median": 0.2205, |
| "ci95_low": 0.21575560344827588, |
| "ci95_high": 0.30172629310344823 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.19879310344827586, |
| "alpha_median": 0.1545, |
| "ci95_low": 0.15875646551724143, |
| "ci95_high": 0.24043620689655168 |
| }, |
| { |
| "position": 6, |
| "n": 58, |
| "alpha_mean": 0.15515517241379312, |
| "alpha_median": 0.0895, |
| "ci95_low": 0.11734439655172413, |
| "ci95_high": 0.19293146551724136 |
| }, |
| { |
| "position": 7, |
| "n": 58, |
| "alpha_mean": 0.12293103448275863, |
| "alpha_median": 0.065, |
| "ci95_low": 0.0872926724137931, |
| "ci95_high": 0.15955387931034481 |
| }, |
| { |
| "position": 8, |
| "n": 58, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 9, |
| "n": 58, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| }, |
| { |
| "position": 10, |
| "n": 58, |
| "alpha_mean": 0.0, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0, |
| "ci95_high": 0.0 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 178, |
| "beta": 152.84007000018624, |
| "intercept": 23.524191882241304, |
| "r2": 0.9993371112986554, |
| "alpha_be": 0.18298167302089194, |
| "ci95": 0.0035118194171447725 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 151.1887495856207, |
| "intercept": 24.38020425585159, |
| "r2": 0.9989825515615268, |
| "alpha_be": 0.19246226427055, |
| "ci95": 0.007384782940490939 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 153.79218465642955, |
| "intercept": 23.207569625410486, |
| "r2": 0.9989280150504728, |
| "alpha_be": 0.18142326566802597, |
| "ci95": 0.007804747017042869 |
| }, |
| "chat": { |
| "n": 58, |
| "beta": 154.14593034109316, |
| "intercept": 23.029808511584648, |
| "r2": 0.999626560816225, |
| "alpha_be": 0.1738666039391707, |
| "ci95": 0.0047728591746651755 |
| } |
| } |
| }, |
| "ksweep-qwen-q4-dspark-p0-k5": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "dspark-p0", |
| "k": 5, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.0000000000050004e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-qwen-q4-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 178, |
| "baseline_n_matched": 178, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 58 |
| }, |
| "baseline_excluded": 0, |
| "baseline_all": 51.49112359550562, |
| "baseline_by_domain": { |
| "math": 53.47833333333333, |
| "code": 51.10905000000001, |
| "chat": 49.83063793103449 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8351000000000001, |
| "alpha_median": 0.8434999999999999, |
| "ci95_low": 0.8158500000000001, |
| "ci95_high": 0.8545670833333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6908, |
| "alpha_median": 0.699, |
| "ci95_low": 0.6574162500000001, |
| "ci95_high": 0.7251333333333333 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5831, |
| "alpha_median": 0.5714999999999999, |
| "ci95_low": 0.5408058333333334, |
| "ci95_high": 0.6268512499999997 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.48841666666666667, |
| "alpha_median": 0.488, |
| "ci95_low": 0.44127791666666666, |
| "ci95_high": 0.5375004166666666 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.40971666666666656, |
| "alpha_median": 0.397, |
| "ci95_low": 0.3608645833333333, |
| "ci95_high": 0.4619025 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.7903666666666667, |
| "alpha_median": 0.792, |
| "ci95_low": 0.77183, |
| "ci95_high": 0.80858375 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6033166666666666, |
| "alpha_median": 0.5945, |
| "ci95_low": 0.5752324999999999, |
| "ci95_high": 0.6310170833333334 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.4645, |
| "alpha_median": 0.454, |
| "ci95_low": 0.4331312499999999, |
| "ci95_high": 0.4965354166666667 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.35390000000000005, |
| "alpha_median": 0.329, |
| "ci95_low": 0.32044125, |
| "ci95_high": 0.38973333333333327 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.26375, |
| "alpha_median": 0.23, |
| "ci95_low": 0.23050958333333335, |
| "ci95_high": 0.29978541666666664 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.7085689655172414, |
| "alpha_median": 0.7555000000000001, |
| "ci95_low": 0.664840948275862, |
| "ci95_high": 0.7459702586206896 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.4858793103448276, |
| "alpha_median": 0.513, |
| "ci95_low": 0.43451724137931036, |
| "ci95_high": 0.5313284482758621 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.352948275862069, |
| "alpha_median": 0.3545, |
| "ci95_low": 0.30419784482758616, |
| "ci95_high": 0.39955258620689643 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.26739655172413795, |
| "alpha_median": 0.2155, |
| "ci95_low": 0.2195293103448276, |
| "ci95_high": 0.3135099137931034 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.2047586206896552, |
| "alpha_median": 0.1445, |
| "ci95_low": 0.16067543103448276, |
| "ci95_high": 0.24851810344827574 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 178, |
| "beta": 126.64769809879108, |
| "intercept": 27.7752026738876, |
| "r2": 0.9921790944292123, |
| "alpha_be": 0.18725899702589544, |
| "ci95": 0.011883994782301592 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 125.79183572031826, |
| "intercept": 28.025430131666013, |
| "r2": 0.9884348158282333, |
| "alpha_be": 0.20234145607238396, |
| "ci95": 0.02443949316259995 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 128.71278207683835, |
| "intercept": 26.728004856515412, |
| "r2": 0.9817423471835494, |
| "alpha_be": 0.1894220974023366, |
| "ci95": 0.03179177306281013 |
| }, |
| "chat": { |
| "n": 58, |
| "beta": 128.4224399451065, |
| "intercept": 27.34948644798619, |
| "r2": 0.9952543165271275, |
| "alpha_be": 0.17505625568753985, |
| "ci95": 0.01680662933104417 |
| } |
| } |
| }, |
| "ksweep-qwen-q4-dspark-p0-k7": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "dspark-p0", |
| "k": 7, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.0000000000105516e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-qwen-q4-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 178, |
| "baseline_n_matched": 178, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 58 |
| }, |
| "baseline_excluded": 0, |
| "baseline_all": 51.49112359550562, |
| "baseline_by_domain": { |
| "math": 53.47833333333333, |
| "code": 51.10905000000001, |
| "chat": 49.83063793103449 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8275166666666666, |
| "alpha_median": 0.8234999999999999, |
| "ci95_low": 0.80881625, |
| "ci95_high": 0.8464854166666665 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6882666666666667, |
| "alpha_median": 0.7064999999999999, |
| "ci95_low": 0.6566654166666668, |
| "ci95_high": 0.7224 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5771166666666667, |
| "alpha_median": 0.57, |
| "ci95_low": 0.5377991666666667, |
| "ci95_high": 0.6176891666666667 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.48430000000000006, |
| "alpha_median": 0.4585, |
| "ci95_low": 0.4371829166666667, |
| "ci95_high": 0.5336854166666667 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.4117, |
| "alpha_median": 0.4, |
| "ci95_low": 0.36243166666666665, |
| "ci95_high": 0.4655683333333333 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.35425, |
| "alpha_median": 0.339, |
| "ci95_low": 0.30452666666666667, |
| "ci95_high": 0.40827583333333334 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.30565, |
| "alpha_median": 0.295, |
| "ci95_low": 0.2567629166666667, |
| "ci95_high": 0.3597012499999999 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.7943666666666667, |
| "alpha_median": 0.808, |
| "ci95_low": 0.77531625, |
| "ci95_high": 0.8127087500000001 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6135166666666668, |
| "alpha_median": 0.6125, |
| "ci95_low": 0.58530875, |
| "ci95_high": 0.6420545833333332 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.46181666666666665, |
| "alpha_median": 0.4355, |
| "ci95_low": 0.4318120833333333, |
| "ci95_high": 0.4930520833333334 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.3523333333333333, |
| "alpha_median": 0.3255, |
| "ci95_low": 0.3216791666666667, |
| "ci95_high": 0.3854704166666667 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.26839999999999997, |
| "alpha_median": 0.2355, |
| "ci95_low": 0.23628, |
| "ci95_high": 0.3033833333333333 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.19771666666666665, |
| "alpha_median": 0.1725, |
| "ci95_low": 0.16911499999999996, |
| "ci95_high": 0.22833874999999998 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.15255000000000002, |
| "alpha_median": 0.119, |
| "ci95_low": 0.12411666666666668, |
| "ci95_high": 0.18401791666666664 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.7089310344827586, |
| "alpha_median": 0.7275, |
| "ci95_low": 0.665080172413793, |
| "ci95_high": 0.747226724137931 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.4922241379310345, |
| "alpha_median": 0.5095000000000001, |
| "ci95_low": 0.44306379310344834, |
| "ci95_high": 0.5365064655172413 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.35253448275862065, |
| "alpha_median": 0.3375, |
| "ci95_low": 0.30437112068965516, |
| "ci95_high": 0.39734525862068965 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.2588793103448276, |
| "alpha_median": 0.2205, |
| "ci95_low": 0.21575560344827588, |
| "ci95_high": 0.30172629310344823 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.19879310344827586, |
| "alpha_median": 0.1545, |
| "ci95_low": 0.15875646551724143, |
| "ci95_high": 0.24043620689655168 |
| }, |
| { |
| "position": 6, |
| "n": 58, |
| "alpha_mean": 0.15515517241379312, |
| "alpha_median": 0.0895, |
| "ci95_low": 0.11734439655172413, |
| "ci95_high": 0.19293146551724136 |
| }, |
| { |
| "position": 7, |
| "n": 58, |
| "alpha_mean": 0.12293103448275863, |
| "alpha_median": 0.065, |
| "ci95_low": 0.0872926724137931, |
| "ci95_high": 0.15955387931034481 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 178, |
| "beta": 150.91904491850252, |
| "intercept": 23.989659108729462, |
| "r2": 0.9956257051108182, |
| "alpha_be": 0.18222660037125973, |
| "ci95": 0.009041124980654966 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 149.53095147695947, |
| "intercept": 24.162558865235876, |
| "r2": 0.9900120544340701, |
| "alpha_be": 0.1960515477132812, |
| "ci95": 0.02319392037908231 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 154.4545454230679, |
| "intercept": 22.800519103828318, |
| "r2": 0.997135142522458, |
| "alpha_be": 0.18328065916500888, |
| "ci95": 0.012759990054545468 |
| }, |
| "chat": { |
| "n": 58, |
| "beta": 154.71708196604047, |
| "intercept": 23.039200191852306, |
| "r2": 0.9995377240487928, |
| "alpha_be": 0.17316405789674053, |
| "ci95": 0.005311831991766195 |
| } |
| } |
| }, |
| "ksweep-qwen-q4-eagle3-k10": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "eagle3", |
| "k": 10, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.000000000002225e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-qwen-q4-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 178, |
| "baseline_n_matched": 178, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 58 |
| }, |
| "baseline_excluded": 0, |
| "baseline_all": 51.49112359550562, |
| "baseline_by_domain": { |
| "math": 53.47833333333333, |
| "code": 51.10905000000001, |
| "chat": 49.83063793103449 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6393000000000001, |
| "alpha_median": 0.636, |
| "ci95_low": 0.6221158333333334, |
| "ci95_high": 0.65715 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.41214999999999996, |
| "alpha_median": 0.413, |
| "ci95_low": 0.3911658333333333, |
| "ci95_high": 0.43366874999999994 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.2157, |
| "alpha_median": 0.2145, |
| "ci95_low": 0.20113291666666666, |
| "ci95_high": 0.23085124999999998 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.12154999999999999, |
| "alpha_median": 0.1165, |
| "ci95_low": 0.10949666666666666, |
| "ci95_high": 0.13506666666666664 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.07311666666666668, |
| "alpha_median": 0.07050000000000001, |
| "ci95_low": 0.06314958333333333, |
| "ci95_high": 0.08443333333333333 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.04493333333333333, |
| "alpha_median": 0.038, |
| "ci95_low": 0.037166250000000005, |
| "ci95_high": 0.05385083333333333 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.027366666666666664, |
| "alpha_median": 0.0225, |
| "ci95_low": 0.021782916666666666, |
| "ci95_high": 0.0334675 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.01655, |
| "alpha_median": 0.011, |
| "ci95_low": 0.012550000000000002, |
| "ci95_high": 0.020699999999999996 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.008683333333333333, |
| "alpha_median": 0.0085, |
| "ci95_low": 0.005449166666666666, |
| "ci95_high": 0.012416666666666666 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.005783333333333333, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0031833333333333336, |
| "ci95_high": 0.008583333333333333 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6601999999999999, |
| "alpha_median": 0.6415, |
| "ci95_low": 0.6419654166666666, |
| "ci95_high": 0.6793508333333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.41513333333333335, |
| "alpha_median": 0.399, |
| "ci95_low": 0.3911633333333333, |
| "ci95_high": 0.4415695833333333 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.21558333333333332, |
| "alpha_median": 0.174, |
| "ci95_low": 0.1876145833333333, |
| "ci95_high": 0.24768500000000002 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.12786666666666668, |
| "alpha_median": 0.0925, |
| "ci95_low": 0.10286625, |
| "ci95_high": 0.15703499999999995 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.08738333333333334, |
| "alpha_median": 0.046, |
| "ci95_low": 0.06396333333333334, |
| "ci95_high": 0.11490749999999998 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.057249999999999995, |
| "alpha_median": 0.02, |
| "ci95_low": 0.03816375, |
| "ci95_high": 0.07923541666666666 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.03891666666666666, |
| "alpha_median": 0.01, |
| "ci95_low": 0.022932499999999998, |
| "ci95_high": 0.05705333333333332 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.030533333333333336, |
| "alpha_median": 0.009, |
| "ci95_low": 0.01624916666666667, |
| "ci95_high": 0.047400833333333336 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.021716666666666665, |
| "alpha_median": 0.0, |
| "ci95_low": 0.010549166666666668, |
| "ci95_high": 0.03530249999999999 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.01635, |
| "alpha_median": 0.0, |
| "ci95_low": 0.008066666666666666, |
| "ci95_high": 0.025851666666666658 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.5735517241379311, |
| "alpha_median": 0.6014999999999999, |
| "ci95_low": 0.5235991379310345, |
| "ci95_high": 0.6212129310344826 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.34679310344827585, |
| "alpha_median": 0.3715, |
| "ci95_low": 0.2997573275862069, |
| "ci95_high": 0.39171379310344817 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.18879310344827585, |
| "alpha_median": 0.163, |
| "ci95_low": 0.1524030172413793, |
| "ci95_high": 0.22463879310344825 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.11891379310344827, |
| "alpha_median": 0.079, |
| "ci95_low": 0.08842974137931035, |
| "ci95_high": 0.15122887931034484 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.08306896551724137, |
| "alpha_median": 0.0405, |
| "ci95_low": 0.057982327586206896, |
| "ci95_high": 0.10953577586206896 |
| }, |
| { |
| "position": 6, |
| "n": 58, |
| "alpha_mean": 0.0649655172413793, |
| "alpha_median": 0.0285, |
| "ci95_low": 0.04182456896551724, |
| "ci95_high": 0.08976249999999997 |
| }, |
| { |
| "position": 7, |
| "n": 58, |
| "alpha_mean": 0.048999999999999995, |
| "alpha_median": 0.0095, |
| "ci95_low": 0.029137068965517245, |
| "ci95_high": 0.0712004310344827 |
| }, |
| { |
| "position": 8, |
| "n": 58, |
| "alpha_mean": 0.04074137931034483, |
| "alpha_median": 0.008, |
| "ci95_low": 0.022344827586206897, |
| "ci95_high": 0.06181077586206897 |
| }, |
| { |
| "position": 9, |
| "n": 58, |
| "alpha_mean": 0.034413793103448276, |
| "alpha_median": 0.0, |
| "ci95_low": 0.016790948275862073, |
| "ci95_high": 0.0545176724137931 |
| }, |
| { |
| "position": 10, |
| "n": 58, |
| "alpha_mean": 0.029206896551724138, |
| "alpha_median": 0.0, |
| "ci95_low": 0.013636637931034482, |
| "ci95_high": 0.047277586206896545 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 178, |
| "beta": 179.9698152641095, |
| "intercept": 18.174912990606554, |
| "r2": 0.9990778895392142, |
| "alpha_be": 0.18512110242490845, |
| "ci95": 0.004354675513391534 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 180.14654989804825, |
| "intercept": 18.16128445739837, |
| "r2": 0.9981746959379467, |
| "alpha_be": 0.196046212907892, |
| "ci95": 0.01066214417874208 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 183.48844434805207, |
| "intercept": 17.5300420163385, |
| "r2": 0.9993391781821137, |
| "alpha_be": 0.18300339349962988, |
| "ci95": 0.006413288845512137 |
| }, |
| "chat": { |
| "n": 58, |
| "beta": 178.32304557547923, |
| "intercept": 18.466067871088224, |
| "r2": 0.9993181610317233, |
| "alpha_be": 0.17588624038316183, |
| "ci95": 0.006655553072379677 |
| } |
| } |
| }, |
| "ksweep-qwen-q4-eagle3-k5": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "eagle3", |
| "k": 5, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.0000000000050004e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-qwen-q4-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 178, |
| "baseline_n_matched": 178, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 58 |
| }, |
| "baseline_excluded": 0, |
| "baseline_all": 51.49112359550562, |
| "baseline_by_domain": { |
| "math": 53.47833333333333, |
| "code": 51.10905000000001, |
| "chat": 49.83063793103449 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6475333333333333, |
| "alpha_median": 0.6315, |
| "ci95_low": 0.6281633333333333, |
| "ci95_high": 0.6681520833333335 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.4170999999999999, |
| "alpha_median": 0.405, |
| "ci95_low": 0.39436625, |
| "ci95_high": 0.44110166666666667 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.2237666666666667, |
| "alpha_median": 0.218, |
| "ci95_low": 0.20564458333333333, |
| "ci95_high": 0.24383749999999996 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.12868333333333334, |
| "alpha_median": 0.1205, |
| "ci95_low": 0.11228208333333332, |
| "ci95_high": 0.1467341666666667 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.0775, |
| "alpha_median": 0.0695, |
| "ci95_low": 0.06496583333333333, |
| "ci95_high": 0.09210125 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6593166666666668, |
| "alpha_median": 0.648, |
| "ci95_low": 0.6399995833333333, |
| "ci95_high": 0.6801691666666667 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.4212333333333333, |
| "alpha_median": 0.4105, |
| "ci95_low": 0.39419958333333327, |
| "ci95_high": 0.4513374999999999 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.21413333333333331, |
| "alpha_median": 0.176, |
| "ci95_low": 0.18536458333333336, |
| "ci95_high": 0.24663083333333327 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.1280666666666667, |
| "alpha_median": 0.088, |
| "ci95_low": 0.10104916666666666, |
| "ci95_high": 0.15938333333333332 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.0828, |
| "alpha_median": 0.048, |
| "ci95_low": 0.058109999999999995, |
| "ci95_high": 0.11205333333333332 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.5732241379310344, |
| "alpha_median": 0.597, |
| "ci95_low": 0.5165314655172414, |
| "ci95_high": 0.6251758620689656 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.3543620689655172, |
| "alpha_median": 0.3435, |
| "ci95_low": 0.3023431034482758, |
| "ci95_high": 0.4045181034482759 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.1993103448275862, |
| "alpha_median": 0.1605, |
| "ci95_low": 0.15927284482758625, |
| "ci95_high": 0.24270301724137924 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.1258103448275862, |
| "alpha_median": 0.092, |
| "ci95_low": 0.09251422413793102, |
| "ci95_high": 0.1614176724137931 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.08212068965517243, |
| "alpha_median": 0.0435, |
| "ci95_low": 0.05446120689655173, |
| "ci95_high": 0.11160344827586206 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 178, |
| "beta": 124.00546088925626, |
| "intercept": 25.168637372730156, |
| "r2": 0.9995948082589274, |
| "alpha_be": 0.21226876650442758, |
| "ci95": 0.0027914808962265377 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 123.4033847196296, |
| "intercept": 25.34221243118326, |
| "r2": 0.9993063214061196, |
| "alpha_be": 0.2280012089301672, |
| "ci95": 0.006316104226265306 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 124.51876657128227, |
| "intercept": 24.95940146575053, |
| "r2": 0.9993914722547149, |
| "alpha_be": 0.2100056823103833, |
| "ci95": 0.005947692775158475 |
| }, |
| "chat": { |
| "n": 58, |
| "beta": 124.00003816146088, |
| "intercept": 25.233496633487402, |
| "r2": 0.999762288937174, |
| "alpha_be": 0.1983639816749013, |
| "ci95": 0.003825103461249379 |
| } |
| } |
| }, |
| "ksweep-qwen-q4-eagle3-k7": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "eagle3", |
| "k": 7, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.000000000002225e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-qwen-q4-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 178, |
| "baseline_n_matched": 178, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 58 |
| }, |
| "baseline_excluded": 0, |
| "baseline_all": 51.49112359550562, |
| "baseline_by_domain": { |
| "math": 53.47833333333333, |
| "code": 51.10905000000001, |
| "chat": 49.83063793103449 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6443333333333333, |
| "alpha_median": 0.626, |
| "ci95_low": 0.6251470833333334, |
| "ci95_high": 0.6649041666666665 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.4131333333333334, |
| "alpha_median": 0.403, |
| "ci95_low": 0.3907324999999999, |
| "ci95_high": 0.4365841666666666 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.21961666666666668, |
| "alpha_median": 0.218, |
| "ci95_low": 0.20243291666666668, |
| "ci95_high": 0.2386175 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.12621666666666667, |
| "alpha_median": 0.118, |
| "ci95_low": 0.1104825, |
| "ci95_high": 0.14348999999999995 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.07546666666666667, |
| "alpha_median": 0.0675, |
| "ci95_low": 0.06334833333333333, |
| "ci95_high": 0.08971749999999999 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.045649999999999996, |
| "alpha_median": 0.036, |
| "ci95_low": 0.03564916666666666, |
| "ci95_high": 0.05713375 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.028, |
| "alpha_median": 0.02, |
| "ci95_low": 0.02039833333333333, |
| "ci95_high": 0.037367083333333335 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6557499999999999, |
| "alpha_median": 0.6445000000000001, |
| "ci95_low": 0.6375145833333333, |
| "ci95_high": 0.6747520833333334 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.41759999999999997, |
| "alpha_median": 0.4125, |
| "ci95_low": 0.39189124999999997, |
| "ci95_high": 0.4459733333333334 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.21246666666666666, |
| "alpha_median": 0.1815, |
| "ci95_low": 0.18428166666666662, |
| "ci95_high": 0.2451670833333333 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.12625000000000003, |
| "alpha_median": 0.0875, |
| "ci95_low": 0.09834916666666665, |
| "ci95_high": 0.15945666666666664 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.08131666666666669, |
| "alpha_median": 0.049, |
| "ci95_low": 0.05654833333333334, |
| "ci95_high": 0.11175416666666664 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.052649999999999995, |
| "alpha_median": 0.019, |
| "ci95_low": 0.03203333333333333, |
| "ci95_high": 0.07733583333333333 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.0377, |
| "alpha_median": 0.01, |
| "ci95_low": 0.020062916666666666, |
| "ci95_high": 0.05930083333333334 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.5708275862068966, |
| "alpha_median": 0.6045, |
| "ci95_low": 0.5146288793103448, |
| "ci95_high": 0.6221379310344828 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.3496551724137931, |
| "alpha_median": 0.3395, |
| "ci95_low": 0.29932672413793104, |
| "ci95_high": 0.39888405172413793 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.19015517241379312, |
| "alpha_median": 0.158, |
| "ci95_low": 0.15470474137931034, |
| "ci95_high": 0.22732758620689655 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.12032758620689656, |
| "alpha_median": 0.084, |
| "ci95_low": 0.09060258620689655, |
| "ci95_high": 0.15307241379310346 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.07534482758620689, |
| "alpha_median": 0.042499999999999996, |
| "ci95_low": 0.05094008620689655, |
| "ci95_high": 0.10100258620689655 |
| }, |
| { |
| "position": 6, |
| "n": 58, |
| "alpha_mean": 0.057896551724137935, |
| "alpha_median": 0.0205, |
| "ci95_low": 0.035170689655172414, |
| "ci95_high": 0.08256939655172414 |
| }, |
| { |
| "position": 7, |
| "n": 58, |
| "alpha_mean": 0.04268965517241379, |
| "alpha_median": 0.009, |
| "ci95_low": 0.022724137931034484, |
| "ci95_high": 0.06393362068965516 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 178, |
| "beta": 136.38572009224174, |
| "intercept": 19.785953970540383, |
| "r2": 0.9997089204363768, |
| "alpha_be": 0.23246692984809614, |
| "ci95": 0.002393383776401263 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 136.00101179865075, |
| "intercept": 19.85562650831181, |
| "r2": 0.9992706631099841, |
| "alpha_be": 0.24722394620710528, |
| "ci95": 0.00656820292046481 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 137.56683633503212, |
| "intercept": 19.51519942035037, |
| "r2": 0.9997118358385192, |
| "alpha_be": 0.22966182418199654, |
| "ci95": 0.004141243316511085 |
| }, |
| "chat": { |
| "n": 58, |
| "beta": 135.8092740235272, |
| "intercept": 19.921302976708613, |
| "r2": 0.9998608796338377, |
| "alpha_be": 0.2202304310171371, |
| "ci95": 0.0029523502222129163 |
| } |
| } |
| }, |
| "ksweep-qwen-q4-vanilla17b-k10": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "vanilla17b", |
| "k": 10, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.000000000002225e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-qwen-q4-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 178, |
| "baseline_n_matched": 178, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 58 |
| }, |
| "baseline_excluded": 0, |
| "baseline_all": 51.49112359550562, |
| "baseline_by_domain": { |
| "math": 53.47833333333333, |
| "code": 51.10905000000001, |
| "chat": 49.83063793103449 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8605833333333334, |
| "alpha_median": 0.864, |
| "ci95_low": 0.8454433333333332, |
| "ci95_high": 0.8753520833333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7415833333333333, |
| "alpha_median": 0.7255, |
| "ci95_low": 0.7151995833333333, |
| "ci95_high": 0.7684404166666667 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6566500000000001, |
| "alpha_median": 0.6505000000000001, |
| "ci95_low": 0.6250304166666666, |
| "ci95_high": 0.6895837499999998 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.5859333333333334, |
| "alpha_median": 0.5725, |
| "ci95_low": 0.5521279166666666, |
| "ci95_high": 0.6207170833333334 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.5292833333333333, |
| "alpha_median": 0.512, |
| "ci95_low": 0.49318166666666663, |
| "ci95_high": 0.5666537499999998 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.47714999999999996, |
| "alpha_median": 0.469, |
| "ci95_low": 0.44058083333333337, |
| "ci95_high": 0.51552375 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.4276666666666667, |
| "alpha_median": 0.42, |
| "ci95_low": 0.39024958333333337, |
| "ci95_high": 0.4667545833333333 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.3843833333333334, |
| "alpha_median": 0.366, |
| "ci95_low": 0.34676666666666667, |
| "ci95_high": 0.4235545833333333 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.3481, |
| "alpha_median": 0.341, |
| "ci95_low": 0.3101312500000001, |
| "ci95_high": 0.3868700000000001 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.32078333333333325, |
| "alpha_median": 0.314, |
| "ci95_low": 0.28234583333333335, |
| "ci95_high": 0.3583666666666666 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.82835, |
| "alpha_median": 0.835, |
| "ci95_low": 0.8096479166666668, |
| "ci95_high": 0.8476525 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6865166666666667, |
| "alpha_median": 0.6875, |
| "ci95_low": 0.6631316666666667, |
| "ci95_high": 0.7102224999999999 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5856499999999999, |
| "alpha_median": 0.578, |
| "ci95_low": 0.5595291666666666, |
| "ci95_high": 0.6141675 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.5046, |
| "alpha_median": 0.484, |
| "ci95_low": 0.47836416666666665, |
| "ci95_high": 0.5368516666666666 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.43585, |
| "alpha_median": 0.415, |
| "ci95_low": 0.4082495833333332, |
| "ci95_high": 0.4698133333333333 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.3830666666666667, |
| "alpha_median": 0.3645, |
| "ci95_low": 0.3531079166666666, |
| "ci95_high": 0.41880291666666664 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.34103333333333335, |
| "alpha_median": 0.3225, |
| "ci95_low": 0.3107825, |
| "ci95_high": 0.3766229166666667 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.30171666666666663, |
| "alpha_median": 0.278, |
| "ci95_low": 0.2721820833333333, |
| "ci95_high": 0.3373029166666667 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.2707166666666667, |
| "alpha_median": 0.242, |
| "ci95_low": 0.24014916666666664, |
| "ci95_high": 0.3073733333333334 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.2393666666666667, |
| "alpha_median": 0.2145, |
| "ci95_low": 0.21163, |
| "ci95_high": 0.27196833333333337 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.7706724137931034, |
| "alpha_median": 0.7444999999999999, |
| "ci95_low": 0.7471025862068965, |
| "ci95_high": 0.7944883620689656 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.5908793103448275, |
| "alpha_median": 0.588, |
| "ci95_low": 0.5608418103448276, |
| "ci95_high": 0.6219202586206896 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.47472413793103446, |
| "alpha_median": 0.478, |
| "ci95_low": 0.43965344827586206, |
| "ci95_high": 0.5126948275862069 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.38355172413793104, |
| "alpha_median": 0.362, |
| "ci95_low": 0.3471965517241379, |
| "ci95_high": 0.4236383620689655 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.3245862068965517, |
| "alpha_median": 0.2845, |
| "ci95_low": 0.28644482758620693, |
| "ci95_high": 0.3674366379310344 |
| }, |
| { |
| "position": 6, |
| "n": 58, |
| "alpha_mean": 0.27910344827586203, |
| "alpha_median": 0.231, |
| "ci95_low": 0.2389810344827586, |
| "ci95_high": 0.32389655172413795 |
| }, |
| { |
| "position": 7, |
| "n": 58, |
| "alpha_mean": 0.24193103448275866, |
| "alpha_median": 0.191, |
| "ci95_low": 0.20145646551724136, |
| "ci95_high": 0.2864323275862069 |
| }, |
| { |
| "position": 8, |
| "n": 58, |
| "alpha_mean": 0.21972413793103449, |
| "alpha_median": 0.16399999999999998, |
| "ci95_low": 0.17875862068965517, |
| "ci95_high": 0.266089224137931 |
| }, |
| { |
| "position": 9, |
| "n": 58, |
| "alpha_mean": 0.19453448275862065, |
| "alpha_median": 0.138, |
| "ci95_low": 0.15435689655172413, |
| "ci95_high": 0.23910603448275858 |
| }, |
| { |
| "position": 10, |
| "n": 58, |
| "alpha_mean": 0.17518965517241378, |
| "alpha_median": 0.11449999999999999, |
| "ci95_low": 0.1350336206896552, |
| "ci95_high": 0.21991681034482755 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 178, |
| "beta": 116.9362891623002, |
| "intercept": 13.17375042186088, |
| "r2": 0.9963018294654568, |
| "alpha_be": 0.32767734847873137, |
| "ci95": 0.007653783318616962 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 117.82313685422277, |
| "intercept": 13.034395764939275, |
| "r2": 0.9992943504470627, |
| "alpha_be": 0.34325972511183017, |
| "ci95": 0.005573941092610634 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 117.08214935162073, |
| "intercept": 12.639432519936767, |
| "r2": 0.9896060474900664, |
| "alpha_be": 0.32856945053623343, |
| "ci95": 0.02235528858074378 |
| }, |
| "chat": { |
| "n": 58, |
| "beta": 115.43701539092281, |
| "intercept": 13.859326934794522, |
| "r2": 0.9967365947791387, |
| "alpha_be": 0.3116098495307122, |
| "ci95": 0.013270427908675058 |
| } |
| } |
| }, |
| "ksweep-qwen-q4-vanilla17b-k5": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "vanilla17b", |
| "k": 5, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.0000000000050004e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-qwen-q4-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 178, |
| "baseline_n_matched": 178, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 58 |
| }, |
| "baseline_excluded": 0, |
| "baseline_all": 51.49112359550562, |
| "baseline_by_domain": { |
| "math": 53.47833333333333, |
| "code": 51.10905000000001, |
| "chat": 49.83063793103449 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8614333333333332, |
| "alpha_median": 0.8694999999999999, |
| "ci95_low": 0.8468833333333333, |
| "ci95_high": 0.8758516666666667 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7467333333333332, |
| "alpha_median": 0.737, |
| "ci95_low": 0.7221154166666666, |
| "ci95_high": 0.7710595833333332 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6641666666666667, |
| "alpha_median": 0.6555, |
| "ci95_low": 0.6353624999999999, |
| "ci95_high": 0.6928541666666668 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.5916833333333334, |
| "alpha_median": 0.5945, |
| "ci95_low": 0.5596133333333334, |
| "ci95_high": 0.6232516666666668 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.5365, |
| "alpha_median": 0.5255000000000001, |
| "ci95_low": 0.5021658333333333, |
| "ci95_high": 0.5705020833333334 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8306333333333333, |
| "alpha_median": 0.8225, |
| "ci95_low": 0.8160666666666666, |
| "ci95_high": 0.8457504166666666 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6994166666666667, |
| "alpha_median": 0.696, |
| "ci95_low": 0.6790833333333334, |
| "ci95_high": 0.7209337499999998 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5953000000000002, |
| "alpha_median": 0.585, |
| "ci95_low": 0.5718650000000002, |
| "ci95_high": 0.6201179166666667 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.5117333333333334, |
| "alpha_median": 0.492, |
| "ci95_low": 0.4881979166666667, |
| "ci95_high": 0.5381729166666667 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.44403333333333334, |
| "alpha_median": 0.4175, |
| "ci95_low": 0.4157829166666667, |
| "ci95_high": 0.4744208333333333 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.7753448275862069, |
| "alpha_median": 0.769, |
| "ci95_low": 0.7534068965517241, |
| "ci95_high": 0.7984706896551723 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.6073620689655171, |
| "alpha_median": 0.5805, |
| "ci95_low": 0.575498275862069, |
| "ci95_high": 0.6407241379310344 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.488948275862069, |
| "alpha_median": 0.46950000000000003, |
| "ci95_low": 0.4514625, |
| "ci95_high": 0.5298508620689655 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.40144827586206894, |
| "alpha_median": 0.363, |
| "ci95_low": 0.3599090517241379, |
| "ci95_high": 0.44630129310344824 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.33731034482758626, |
| "alpha_median": 0.283, |
| "ci95_low": 0.2937344827586207, |
| "ci95_high": 0.3847327586206897 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 178, |
| "beta": 89.12762701123346, |
| "intercept": 16.186078279775984, |
| "r2": 0.977793087527451, |
| "alpha_be": 0.3961178649048949, |
| "ci95": 0.016906936522748488 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 85.96841262566369, |
| "intercept": 19.23588158773496, |
| "r2": 0.9971865980783139, |
| "alpha_be": 0.39831434243995983, |
| "ci95": 0.009950751827095662 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 85.82487459287938, |
| "intercept": 18.39320437998536, |
| "r2": 0.961332243154551, |
| "alpha_be": 0.3811930489290686, |
| "ci95": 0.03943974692519082 |
| }, |
| "chat": { |
| "n": 58, |
| "beta": 83.92535023584126, |
| "intercept": 17.847296864028984, |
| "r2": 0.971428188831581, |
| "alpha_be": 0.38109273273365096, |
| "ci95": 0.035992084267018956 |
| } |
| } |
| }, |
| "ksweep-qwen-q4-vanilla17b-k7": { |
| "family": "ksweep", |
| "target": "qwen-q4", |
| "drafter": "vanilla17b", |
| "k": 7, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.0000000000105516e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-qwen-q4-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 178, |
| "baseline_n_matched": 178, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 58 |
| }, |
| "baseline_excluded": 0, |
| "baseline_all": 51.49112359550562, |
| "baseline_by_domain": { |
| "math": 53.47833333333333, |
| "code": 51.10905000000001, |
| "chat": 49.83063793103449 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8567666666666667, |
| "alpha_median": 0.857, |
| "ci95_low": 0.8415325, |
| "ci95_high": 0.8714508333333333 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.7372166666666667, |
| "alpha_median": 0.732, |
| "ci95_low": 0.7132141666666667, |
| "ci95_high": 0.7604666666666666 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.6550666666666667, |
| "alpha_median": 0.656, |
| "ci95_low": 0.6266645833333334, |
| "ci95_high": 0.6839679166666665 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.58005, |
| "alpha_median": 0.59, |
| "ci95_low": 0.54862375, |
| "ci95_high": 0.6120524999999999 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.5245666666666667, |
| "alpha_median": 0.521, |
| "ci95_low": 0.4935104166666666, |
| "ci95_high": 0.5572333333333331 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.47533333333333333, |
| "alpha_median": 0.4745, |
| "ci95_low": 0.4426595833333334, |
| "ci95_high": 0.5094366666666666 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.42833333333333334, |
| "alpha_median": 0.417, |
| "ci95_low": 0.39536541666666664, |
| "ci95_high": 0.4637670833333333 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.8221666666666666, |
| "alpha_median": 0.826, |
| "ci95_low": 0.8074325, |
| "ci95_high": 0.8374529166666667 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.6857833333333333, |
| "alpha_median": 0.673, |
| "ci95_low": 0.6650241666666667, |
| "ci95_high": 0.7074675 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.5782500000000002, |
| "alpha_median": 0.564, |
| "ci95_low": 0.5552654166666666, |
| "ci95_high": 0.6034895833333332 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.4988000000000001, |
| "alpha_median": 0.4785, |
| "ci95_low": 0.47428333333333333, |
| "ci95_high": 0.5256604166666666 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.4322833333333333, |
| "alpha_median": 0.415, |
| "ci95_low": 0.4041833333333334, |
| "ci95_high": 0.46378500000000006 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.3756166666666666, |
| "alpha_median": 0.358, |
| "ci95_low": 0.3435120833333334, |
| "ci95_high": 0.40943458333333327 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.3322, |
| "alpha_median": 0.305, |
| "ci95_low": 0.3008808333333333, |
| "ci95_high": 0.3676062500000001 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.7714827586206897, |
| "alpha_median": 0.7615000000000001, |
| "ci95_low": 0.750240948275862, |
| "ci95_high": 0.7938982758620688 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.5965, |
| "alpha_median": 0.5805, |
| "ci95_low": 0.5664987068965518, |
| "ci95_high": 0.6276793103448276 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.4778965517241379, |
| "alpha_median": 0.465, |
| "ci95_low": 0.4421551724137931, |
| "ci95_high": 0.5167913793103448 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.39296551724137935, |
| "alpha_median": 0.3585, |
| "ci95_low": 0.3511181034482759, |
| "ci95_high": 0.43686250000000004 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.3258103448275862, |
| "alpha_median": 0.268, |
| "ci95_low": 0.28316508620689657, |
| "ci95_high": 0.3719310344827586 |
| }, |
| { |
| "position": 6, |
| "n": 58, |
| "alpha_mean": 0.28493103448275864, |
| "alpha_median": 0.214, |
| "ci95_low": 0.24000862068965514, |
| "ci95_high": 0.3335034482758621 |
| }, |
| { |
| "position": 7, |
| "n": 58, |
| "alpha_mean": 0.2499655172413793, |
| "alpha_median": 0.1795, |
| "ci95_low": 0.2041969827586207, |
| "ci95_high": 0.30017543103448274 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 178, |
| "beta": 93.02021396622774, |
| "intercept": 14.34549676133501, |
| "r2": 0.9965214788429118, |
| "alpha_be": 0.3993285464561159, |
| "ci95": 0.006881675222664475 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 93.31148375702463, |
| "intercept": 14.462993336313076, |
| "r2": 0.9993480699244549, |
| "alpha_be": 0.41811938280408195, |
| "ci95": 0.004892051250056488 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 93.35757091710133, |
| "intercept": 13.791354462793384, |
| "r2": 0.989043007464966, |
| "alpha_be": 0.3997286472925008, |
| "ci95": 0.021289320742869745 |
| }, |
| "chat": { |
| "n": 58, |
| "beta": 91.96354516426943, |
| "intercept": 14.897885904910126, |
| "r2": 0.9975328281585035, |
| "alpha_be": 0.3798543429760771, |
| "ci95": 0.010832394076832803 |
| } |
| } |
| }, |
| "ksweep-qwen-q8-eagle3-k10": { |
| "family": "ksweep", |
| "target": "qwen-q8", |
| "drafter": "eagle3", |
| "k": 10, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.000000000002225e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-qwen-q8-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 178, |
| "baseline_n_matched": 178, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 58 |
| }, |
| "baseline_excluded": 0, |
| "baseline_all": 31.769084269662915, |
| "baseline_by_domain": { |
| "math": 30.842750000000002, |
| "code": 32.19831666666666, |
| "chat": 32.283327586206894 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6383833333333334, |
| "alpha_median": 0.631, |
| "ci95_low": 0.6204491666666665, |
| "ci95_high": 0.6561166666666665 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.4114333333333333, |
| "alpha_median": 0.406, |
| "ci95_low": 0.39191499999999996, |
| "ci95_high": 0.431985 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.2123, |
| "alpha_median": 0.214, |
| "ci95_low": 0.19686250000000002, |
| "ci95_high": 0.22998333333333332 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.11678333333333334, |
| "alpha_median": 0.114, |
| "ci95_low": 0.10509875, |
| "ci95_high": 0.12958583333333332 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.06493333333333333, |
| "alpha_median": 0.062, |
| "ci95_low": 0.05644958333333333, |
| "ci95_high": 0.07463541666666665 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.03548333333333334, |
| "alpha_median": 0.0315, |
| "ci95_low": 0.029499583333333326, |
| "ci95_high": 0.04180083333333333 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.019933333333333338, |
| "alpha_median": 0.019, |
| "ci95_low": 0.0161, |
| "ci95_high": 0.02403416666666666 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.011000000000000001, |
| "alpha_median": 0.01, |
| "ci95_low": 0.008366666666666666, |
| "ci95_high": 0.013767083333333333 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.006816666666666667, |
| "alpha_median": 0.004, |
| "ci95_low": 0.004932916666666666, |
| "ci95_high": 0.008833333333333334 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.0042833333333333334, |
| "alpha_median": 0.0, |
| "ci95_low": 0.002716666666666667, |
| "ci95_high": 0.005933333333333333 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.64665, |
| "alpha_median": 0.65, |
| "ci95_low": 0.6291, |
| "ci95_high": 0.6649175 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.40869999999999995, |
| "alpha_median": 0.401, |
| "ci95_low": 0.38830000000000003, |
| "ci95_high": 0.4312008333333334 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.19943333333333332, |
| "alpha_median": 0.1765, |
| "ci95_low": 0.17841625, |
| "ci95_high": 0.22465374999999999 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.11043333333333333, |
| "alpha_median": 0.092, |
| "ci95_low": 0.09008041666666666, |
| "ci95_high": 0.1367220833333333 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.06545, |
| "alpha_median": 0.045, |
| "ci95_low": 0.04749624999999999, |
| "ci95_high": 0.08988333333333334 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.04083333333333334, |
| "alpha_median": 0.02, |
| "ci95_low": 0.025680416666666664, |
| "ci95_high": 0.06138375000000001 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.028283333333333334, |
| "alpha_median": 0.01, |
| "ci95_low": 0.016983333333333336, |
| "ci95_high": 0.04260249999999999 |
| }, |
| { |
| "position": 8, |
| "n": 60, |
| "alpha_mean": 0.020483333333333336, |
| "alpha_median": 0.008, |
| "ci95_low": 0.011382916666666668, |
| "ci95_high": 0.03171791666666666 |
| }, |
| { |
| "position": 9, |
| "n": 60, |
| "alpha_mean": 0.012733333333333334, |
| "alpha_median": 0.0, |
| "ci95_low": 0.006415833333333332, |
| "ci95_high": 0.021366666666666662 |
| }, |
| { |
| "position": 10, |
| "n": 60, |
| "alpha_mean": 0.009200000000000002, |
| "alpha_median": 0.0, |
| "ci95_low": 0.0034825000000000004, |
| "ci95_high": 0.017300833333333328 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.577948275862069, |
| "alpha_median": 0.609, |
| "ci95_low": 0.5294887931034482, |
| "ci95_high": 0.623278448275862 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.3406379310344828, |
| "alpha_median": 0.345, |
| "ci95_low": 0.2972030172413792, |
| "ci95_high": 0.38113275862068957 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.17277586206896553, |
| "alpha_median": 0.1625, |
| "ci95_low": 0.1397866379310345, |
| "ci95_high": 0.2088625 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.11274137931034484, |
| "alpha_median": 0.0845, |
| "ci95_low": 0.08296508620689654, |
| "ci95_high": 0.14803965517241377 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.06896551724137931, |
| "alpha_median": 0.0455, |
| "ci95_low": 0.046757758620689656, |
| "ci95_high": 0.09586336206896551 |
| }, |
| { |
| "position": 6, |
| "n": 58, |
| "alpha_mean": 0.05379310344827586, |
| "alpha_median": 0.028, |
| "ci95_low": 0.03434439655172414, |
| "ci95_high": 0.07764525862068962 |
| }, |
| { |
| "position": 7, |
| "n": 58, |
| "alpha_mean": 0.0381206896551724, |
| "alpha_median": 0.0095, |
| "ci95_low": 0.02318965517241379, |
| "ci95_high": 0.056193965517241345 |
| }, |
| { |
| "position": 8, |
| "n": 58, |
| "alpha_mean": 0.030896551724137935, |
| "alpha_median": 0.009, |
| "ci95_low": 0.017016810344827587, |
| "ci95_high": 0.04772844827586205 |
| }, |
| { |
| "position": 9, |
| "n": 58, |
| "alpha_mean": 0.023724137931034485, |
| "alpha_median": 0.0, |
| "ci95_low": 0.012065517241379312, |
| "ci95_high": 0.03824999999999995 |
| }, |
| { |
| "position": 10, |
| "n": 58, |
| "alpha_mean": 0.01812068965517241, |
| "alpha_median": 0.0, |
| "ci95_low": 0.00794741379310345, |
| "ci95_high": 0.031227155172413777 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 178, |
| "beta": 149.50139451333408, |
| "intercept": 15.163527656104614, |
| "r2": 0.999363221682926, |
| "alpha_be": 0.1110729212099573, |
| "ci95": 0.0036662264954498746 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 148.7914088142256, |
| "intercept": 15.265950150965722, |
| "r2": 0.9980829640106651, |
| "alpha_be": 0.10468883904770863, |
| "ci95": 0.011096230647999271 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 151.30654692020363, |
| "intercept": 14.813219539185342, |
| "r2": 0.9992744437981491, |
| "alpha_be": 0.11489983402139177, |
| "ci95": 0.006809977694709568 |
| }, |
| "chat": { |
| "n": 58, |
| "beta": 148.95512228331646, |
| "intercept": 15.320449448641202, |
| "r2": 0.9996799543556998, |
| "alpha_be": 0.11387911927796524, |
| "ci95": 0.004608823700100185 |
| } |
| } |
| }, |
| "ksweep-qwen-q8-eagle3-k5": { |
| "family": "ksweep", |
| "target": "qwen-q8", |
| "drafter": "eagle3", |
| "k": 5, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 4.999999999999449e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-qwen-q8-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 178, |
| "baseline_n_matched": 178, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 58 |
| }, |
| "baseline_excluded": 0, |
| "baseline_all": 31.769084269662915, |
| "baseline_by_domain": { |
| "math": 30.842750000000002, |
| "code": 32.19831666666666, |
| "chat": 32.283327586206894 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.644, |
| "alpha_median": 0.629, |
| "ci95_low": 0.6263645833333332, |
| "ci95_high": 0.6616704166666665 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.41586666666666666, |
| "alpha_median": 0.422, |
| "ci95_low": 0.39656375, |
| "ci95_high": 0.4358345833333334 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.22018333333333331, |
| "alpha_median": 0.2165, |
| "ci95_low": 0.20504958333333334, |
| "ci95_high": 0.2355683333333333 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.12205, |
| "alpha_median": 0.1155, |
| "ci95_low": 0.11074958333333332, |
| "ci95_high": 0.13483625 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.06903333333333334, |
| "alpha_median": 0.063, |
| "ci95_low": 0.06015000000000001, |
| "ci95_high": 0.07895166666666667 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6542833333333333, |
| "alpha_median": 0.6465000000000001, |
| "ci95_low": 0.6373291666666667, |
| "ci95_high": 0.6723512500000001 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.41663333333333336, |
| "alpha_median": 0.412, |
| "ci95_low": 0.39533, |
| "ci95_high": 0.4391191666666666 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.20393333333333338, |
| "alpha_median": 0.1845, |
| "ci95_low": 0.18191583333333333, |
| "ci95_high": 0.22860499999999997 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.11448333333333333, |
| "alpha_median": 0.09, |
| "ci95_low": 0.09354500000000002, |
| "ci95_high": 0.13943791666666663 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.06634999999999999, |
| "alpha_median": 0.045, |
| "ci95_low": 0.048914166666666654, |
| "ci95_high": 0.08815166666666666 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.5883793103448276, |
| "alpha_median": 0.6265000000000001, |
| "ci95_low": 0.5389129310344828, |
| "ci95_high": 0.6331443965517242 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.3518275862068965, |
| "alpha_median": 0.37, |
| "ci95_low": 0.3077607758620689, |
| "ci95_high": 0.39405560344827584 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.18241379310344827, |
| "alpha_median": 0.174, |
| "ci95_low": 0.1496987068965517, |
| "ci95_high": 0.21997456896551718 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.11779310344827586, |
| "alpha_median": 0.0885, |
| "ci95_low": 0.09025387931034483, |
| "ci95_high": 0.15170948275862067 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.07225862068965518, |
| "alpha_median": 0.0445, |
| "ci95_low": 0.050567672413793105, |
| "ci95_high": 0.0976306034482758 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 178, |
| "beta": 99.7066125419974, |
| "intercept": 20.17060001572036, |
| "r2": 0.9992274909759372, |
| "alpha_be": 0.11632612881173914, |
| "ci95": 0.003972027492441261 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 99.07719826452487, |
| "intercept": 20.42238174387858, |
| "r2": 0.9986829293580858, |
| "alpha_be": 0.10517423220123989, |
| "ci95": 0.009054847988836469 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 101.20372921309948, |
| "intercept": 19.699544375558144, |
| "r2": 0.9987999302009661, |
| "alpha_be": 0.12350110404321661, |
| "ci95": 0.008598338069620702 |
| }, |
| "chat": { |
| "n": 58, |
| "beta": 99.22733195847222, |
| "intercept": 20.263225257552794, |
| "r2": 0.9995787696442888, |
| "alpha_be": 0.12113701025121437, |
| "ci95": 0.0052052490612202905 |
| } |
| } |
| }, |
| "ksweep-qwen-q8-eagle3-k7": { |
| "family": "ksweep", |
| "target": "qwen-q8", |
| "drafter": "eagle3", |
| "k": 7, |
| "n_excluded": 0, |
| "n_matched": 178, |
| "log_lines": 178, |
| "max_alpha_diff": 5.000000000002225e-05, |
| "ctx": 2048, |
| "prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_run": "baseline-ctx2048-qwen-q8-solo", |
| "baseline_ctx": 2048, |
| "baseline_prompt_set": "sha256:3f44c5fe062fd4299a07b15cb7ac52ea8c15e40ee362181cd6a5cbb6932eed7b", |
| "baseline_n": 178, |
| "baseline_n_matched": 178, |
| "baseline_n_by_domain": { |
| "math": 60, |
| "code": 60, |
| "chat": 58 |
| }, |
| "baseline_excluded": 0, |
| "baseline_all": 31.769084269662915, |
| "baseline_by_domain": { |
| "math": 30.842750000000002, |
| "code": 32.19831666666666, |
| "chat": 32.283327586206894 |
| }, |
| "per_domain_positions": { |
| "math": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6422333333333333, |
| "alpha_median": 0.6315, |
| "ci95_low": 0.6244462499999999, |
| "ci95_high": 0.6602345833333334 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.4123833333333334, |
| "alpha_median": 0.4095, |
| "ci95_low": 0.3926487499999999, |
| "ci95_high": 0.43237125 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.21725, |
| "alpha_median": 0.214, |
| "ci95_low": 0.20271458333333336, |
| "ci95_high": 0.2327575 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.11929999999999999, |
| "alpha_median": 0.1155, |
| "ci95_low": 0.10796541666666669, |
| "ci95_high": 0.13148624999999997 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.06716666666666665, |
| "alpha_median": 0.063, |
| "ci95_low": 0.05866583333333334, |
| "ci95_high": 0.0768675 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.03963333333333333, |
| "alpha_median": 0.035500000000000004, |
| "ci95_low": 0.033666666666666664, |
| "ci95_high": 0.04583499999999999 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.022533333333333332, |
| "alpha_median": 0.0195, |
| "ci95_low": 0.01825, |
| "ci95_high": 0.027017083333333334 |
| } |
| ], |
| "code": [ |
| { |
| "position": 1, |
| "n": 60, |
| "alpha_mean": 0.6501833333333332, |
| "alpha_median": 0.641, |
| "ci95_low": 0.6335320833333334, |
| "ci95_high": 0.6674170833333334 |
| }, |
| { |
| "position": 2, |
| "n": 60, |
| "alpha_mean": 0.4082166666666666, |
| "alpha_median": 0.404, |
| "ci95_low": 0.3865483333333332, |
| "ci95_high": 0.4309508333333333 |
| }, |
| { |
| "position": 3, |
| "n": 60, |
| "alpha_mean": 0.19706666666666664, |
| "alpha_median": 0.184, |
| "ci95_low": 0.17731624999999995, |
| "ci95_high": 0.22053708333333333 |
| }, |
| { |
| "position": 4, |
| "n": 60, |
| "alpha_mean": 0.10885000000000002, |
| "alpha_median": 0.088, |
| "ci95_low": 0.09053041666666665, |
| "ci95_high": 0.13218708333333332 |
| }, |
| { |
| "position": 5, |
| "n": 60, |
| "alpha_mean": 0.06378333333333333, |
| "alpha_median": 0.043, |
| "ci95_low": 0.0474325, |
| "ci95_high": 0.08616875 |
| }, |
| { |
| "position": 6, |
| "n": 60, |
| "alpha_mean": 0.03786666666666667, |
| "alpha_median": 0.02, |
| "ci95_low": 0.0248975, |
| "ci95_high": 0.05433458333333333 |
| }, |
| { |
| "position": 7, |
| "n": 60, |
| "alpha_mean": 0.027649999999999997, |
| "alpha_median": 0.009, |
| "ci95_low": 0.016564583333333334, |
| "ci95_high": 0.041801250000000005 |
| } |
| ], |
| "chat": [ |
| { |
| "position": 1, |
| "n": 58, |
| "alpha_mean": 0.5866896551724138, |
| "alpha_median": 0.62, |
| "ci95_low": 0.5381525862068965, |
| "ci95_high": 0.6317435344827586 |
| }, |
| { |
| "position": 2, |
| "n": 58, |
| "alpha_mean": 0.34374137931034476, |
| "alpha_median": 0.3615, |
| "ci95_low": 0.3013275862068966, |
| "ci95_high": 0.38341163793103433 |
| }, |
| { |
| "position": 3, |
| "n": 58, |
| "alpha_mean": 0.17375862068965517, |
| "alpha_median": 0.1685, |
| "ci95_low": 0.14348017241379313, |
| "ci95_high": 0.20646594827586204 |
| }, |
| { |
| "position": 4, |
| "n": 58, |
| "alpha_mean": 0.11463793103448276, |
| "alpha_median": 0.08549999999999999, |
| "ci95_low": 0.08758534482758619, |
| "ci95_high": 0.1466258620689655 |
| }, |
| { |
| "position": 5, |
| "n": 58, |
| "alpha_mean": 0.06991379310344828, |
| "alpha_median": 0.041499999999999995, |
| "ci95_low": 0.04782629310344828, |
| "ci95_high": 0.09610560344827586 |
| }, |
| { |
| "position": 6, |
| "n": 58, |
| "alpha_mean": 0.05153448275862069, |
| "alpha_median": 0.019, |
| "ci95_low": 0.031843534482758624, |
| "ci95_high": 0.07505689655172411 |
| }, |
| { |
| "position": 7, |
| "n": 58, |
| "alpha_mean": 0.03632758620689655, |
| "alpha_median": 0.0095, |
| "ci95_low": 0.02136206896551724, |
| "ci95_high": 0.05450474137931032 |
| } |
| ] |
| }, |
| "breakeven_pooled": { |
| "n": 178, |
| "beta": 124.64048221408584, |
| "intercept": 18.020365612671316, |
| "r2": 0.9992895367836464, |
| "alpha_be": 0.11030700790595813, |
| "ci95": 0.0038476324467990285 |
| }, |
| "breakeven_per_domain": { |
| "math": { |
| "n": 60, |
| "beta": 123.38545805979774, |
| "intercept": 18.2734774522408, |
| "r2": 0.9976469763893474, |
| "alpha_be": 0.10186996705614698, |
| "ci95": 0.012219163514663685 |
| }, |
| "code": { |
| "n": 60, |
| "beta": 125.76313992372975, |
| "intercept": 17.724757997176432, |
| "r2": 0.9990909026094454, |
| "alpha_be": 0.11508585646213873, |
| "ci95": 0.007570864014882201 |
| }, |
| "chat": { |
| "n": 58, |
| "beta": 124.54104706651054, |
| "intercept": 18.1188842661618, |
| "r2": 0.9996783226593118, |
| "alpha_be": 0.11373313179614301, |
| "ci95": 0.004592603267583699 |
| } |
| } |
| } |
| } |
| } |
