| # Self-contained HTML poster (base64 figures) -> poster/poster_embed.html | |
| import base64, json, os | |
| BASE = os.path.dirname(os.path.abspath(__file__)) | |
| os.makedirs(os.path.join(BASE, "poster"), exist_ok=True) | |
| def b64(p): | |
| return "data:image/png;base64," + base64.b64encode(open(os.path.join(BASE, p), "rb").read()).decode() | |
| J = lambda n: json.load(open(os.path.join(BASE, "outputs", n), encoding="utf-8")) | |
| c1, div, b2, rf = J("claim1_throughput_bias.json"), J("claim2_diversity.json"), J("claim2b_draft_length.json"), J("claim2b_roofline.json") | |
| o = div["overall"] | |
| html = f"""<!DOCTYPE html><html><head><meta charset="utf-8"><style> | |
| :root{{--ink:#14213d;--acc:#2a7fbf;--good:#2a9d3f;--warn:#bf6a2a;--bg:#f7f9fc;}} | |
| *{{box-sizing:border-box;font-family:-apple-system,Segoe UI,Roboto,sans-serif;}} | |
| body{{margin:0;background:#fff;color:var(--ink);}} | |
| .poster{{width:1120px;margin:0 auto;padding:26px 30px;}} | |
| h1{{font-size:26px;margin:0 0 2px;}} .sub{{color:#555;font-size:13px;margin:0 0 14px;}} | |
| .row{{display:flex;gap:16px;}} .col{{flex:1;background:var(--bg);border:1px solid #dce3ee;border-radius:10px;padding:14px 16px;}} | |
| h2{{font-size:15px;margin:0 0 8px;color:var(--acc);border-bottom:2px solid #dce3ee;padding-bottom:4px;}} | |
| .v{{font-weight:700;}} .full{{color:var(--good);}} .toy{{color:var(--warn);}} .inc{{color:#888;}} | |
| img{{width:100%;border:1px solid #dce3ee;border-radius:6px;margin-top:8px;background:#fff;}} | |
| table{{width:100%;border-collapse:collapse;font-size:12px;margin-top:6px;}} td,th{{border:1px solid #dce3ee;padding:4px 6px;text-align:left;}} | |
| .big{{font-size:20px;font-weight:800;}} .foot{{margin-top:14px;font-size:11px;color:#666;text-align:center;}} | |
| .chip{{display:inline-block;padding:2px 8px;border-radius:10px;font-size:11px;font-weight:700;color:#fff;}} | |
| </style></head><body><div class="poster"> | |
| <h1>Reproducing SPEED-Bench: A Unified & Diverse Benchmark for Speculative Decoding</h1> | |
| <p class="sub">ICML 2026 · arXiv:2604.09557 · OpenReview Rl2uQlCoQX · independent CPU-scale reproduction (distilgpt2 82M → gpt2 124M, real nvidia/SPEED-Bench) · agent: CLAUDE · cost $0</p> | |
| <div class="row"> | |
| <div class="col"><h2>Claim 1 — synthetic overestimates throughput <span class="chip inc" style="background:#888">INCONCLUSIVE (toy)</span></h2> | |
| <p>Greedy lossless SD acceptance length (AL → throughput): <span class="v">real {c1['real']['mean_AL']:.2f}</span> vs <span class="v">synthetic {c1['synthetic']['mean_AL']:.2f}</span> ({c1['overestimation_pct']:.1f}%, within seed spread 1.54–2.05). Direction does <b>not</b> reproduce with small models + uniform-random synthetic tokens; needs production scale.</p> | |
| <img src="{b64('figures/claim1_throughput_bias.png')}"></div> | |
| <div class="col"><h2>Claim 2a — low-diversity bias <span class="chip full" style="background:var(--good)">FULL</span></h2> | |
| <p>SPEED-Bench Algorithm 1 (greedy diverse selection) lowers intra-set similarity <span class="big full">{o['random_avg']:.3f}→{o['greedy_avg']:.3f}</span> vs random, in <span class="v">{o['n_better']}/{o['n_total']} categories</span> (deterministic, real MiniLM embeddings). Confirms the redundancy/bias premise.</p> | |
| <img src="{b64('figures/claim2a_diversity.png')}"></div> | |
| <div class="col"><h2>Claim 2b — batch-dependent draft length <span class="chip toy" style="background:var(--warn)">TOY</span></h2> | |
| <p>Roofline throughput model with <b>measured</b> α={b2['alpha']:.2f}: optimal draft length <span class="v">k* = 3→2→1</span> as batch grows (memory-bound→compute-bound). CPU hardware is compute-bound throughout (k*=1, no crossover) — mechanism reproduced, exact crossover is GPU-specific.</p> | |
| <img src="{b64('figures/claim2b_draft_length.png')}"></div> | |
| </div> | |
| <div class="row" style="margin-top:14px;"> | |
| <div class="col" style="flex:2"><h2>Outcome</h2> | |
| <table><tr><th>Official claim</th><th>Verdict</th><th>Evidence</th></tr> | |
| <tr><td>1 · synthetic overestimates throughput</td><td class="inc"><b>inconclusive (toy)</b></td><td>real 1.90 vs synthetic 1.84 AL, within noise</td></tr> | |
| <tr><td>2 · low-diversity bias</td><td class="full"><b>FULL</b></td><td>diversity selection 0.492→0.382, 11/11 categories</td></tr> | |
| <tr><td>2 · batch-dependent optimal draft length</td><td class="toy"><b>TOY</b></td><td>roofline (measured α) k*=3→1; CPU compute-bound null</td></tr></table></div> | |
| <div class="col"><h2>Method & honesty</h2> | |
| <p style="font-size:12px">Production speedup tables (70B–235B models, EAGLE3/MTP, prod engines) infeasible — <b>not attempted</b>. Greedy lossless SD, real dataset (11 cat × 80), fixed seeds, 5 unit tests pass. Diversity reproduces bit-identically. Results labelled honestly, not forced.</p></div> | |
| </div> | |
| <p class="foot">Reproduction bundle (code · outputs · figures · tests) published as Trackio logbook artifacts. Resumed from a stale sibling reservation via sanctioned lease takeover.</p> | |
| </div></body></html>""" | |
| open(os.path.join(BASE, "poster", "poster_embed.html"), "w", encoding="utf-8").write(html) | |
| print("wrote poster/poster_embed.html", len(html), "bytes") | |
Xet Storage Details
- Size:
- 5.17 kB
- Xet hash:
- 07dd5419a3bc47e37d90f258160de836565e765e954d81d9ec018fd3d30332d3
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.