| """Hero chart for the model card: accuracy vs mean output tokens for base Dory (1/2/4 loops), Dory-Beam-4B v2 at |
| 2 loops across its reasoning efforts (low, default, high) and Dory-Beam-4B-Accurate (2/4 loops). |
| |
| python3 scripts/make_chart.py # -> release/model/assets/accuracy_vs_tokens.png |
| |
| Emphasis form: base Dory in the de-emphasis gray, the two Dory-Beam models in categorical slots 1 and 2. One panel |
| per benchmark. (The v1 chart is scripts/make_chart_v1.py.) |
| """ |
|
|
| import json |
| from collections import defaultdict |
| from pathlib import Path |
|
|
| import matplotlib |
|
|
| matplotlib.use("Agg") |
| import matplotlib.pyplot as plt |
| from matplotlib.ticker import FuncFormatter |
|
|
| ROOT = Path(__file__).resolve().parent.parent |
| RES = ROOT / "data" / "eval" / "results" |
| OUT = ROOT / "release" / "model" / "assets" / "accuracy_vs_tokens.png" |
|
|
| SURFACE, INK, INK2, MUTED, GRID, AXIS = "#fcfcfb", "#0b0b0b", "#52514e", "#898781", "#e1e0d9", "#c3c2b7" |
| BLUE, ORANGE, BASE = "#2a78d6", "#eb6834", "#898781" |
| BENCHES = [("gpqa", "GPQA-Diamond"), ("math500", "MATH-500"), ("aime24", "AIME 2024"), |
| ("aime25", "AIME 2025"), ("amc23", "AMC 2023"), ("heldout", "Held-out (300)")] |
| |
| SERIES = [ |
| ("Dory1.0-4B (base) 路 1, 2, 4 loops", BASE, |
| [("1", "base_loops1-64k"), ("2", "base_loops2-64k"), ("4", "base_loops4-64k")]), |
| ("Dory-Beam-4B 路 reasoning effort low, default, high 路 2 loops", BLUE, |
| [("low", "rE-t050_loops2-low-64k"), ("default", "rE-t050_loops2-default-64k"), |
| ("high", "rE-t050_loops2-high-64k")]), |
| ("Dory-Beam-4B-Accurate 路 2, 4 loops", ORANGE, |
| [("2", "mix030_loops2-64k"), ("4", "mix030_loops4-64k")]), |
| ] |
|
|
|
|
| def load(tag): |
| d = defaultdict(list) |
| for line in open(RES / f"{tag}.jsonl"): |
| r = json.loads(line) |
| d[r["bench"]].append(r) |
| return {b: (100 * sum(x["correct"] for x in rs) / len(rs), sum(x["tokens"] for x in rs) / len(rs)) |
| for b, rs in d.items()} |
|
|
|
|
| def place_labels(fig, ax, labels, points): |
| """Point labels in muted ink. Each takes the first candidate spot (right, left, above, below; leftmost points |
| try left first) that overlaps neither an earlier label nor another marker.""" |
| renderer = fig.canvas.get_renderer() |
| to_px = ax.transData.transform |
| r = 6 * fig.dpi / 72 |
| markers = [(px - r, py - r, px + r, py + r) for px, py in map(to_px, points)] |
| placed = [] |
| hit = lambda a, b: a[0] < b[2] and b[0] < a[2] and a[1] < b[3] and b[1] < a[3] |
| for name, x, y, leftmost in labels: |
| spots = [((7, -3), "left"), ((-7, -3), "right"), ((0, 7), "center"), ((0, -14), "center")] |
| if leftmost: |
| spots[0], spots[1] = spots[1], spots[0] |
| own = to_px((x, y)) |
| best = None |
| for off, ha in spots: |
| t = ax.annotate(name, (x, y), xytext=off, textcoords="offset points", fontsize=8.5, color=MUTED, ha=ha) |
| bb = t.get_window_extent(renderer) |
| box = (bb.x0, bb.y0, bb.x1, bb.y1) |
| others = [m for m, p in zip(markers, points) if abs(to_px(p)[0] - own[0]) + abs(to_px(p)[1] - own[1]) > 1] |
| if not any(hit(box, o) for o in placed + others): |
| best = (t, box) |
| break |
| t.remove() |
| if best is None: |
| off, ha = spots[0] |
| t = ax.annotate(name, (x, y), xytext=off, textcoords="offset points", fontsize=8.5, color=MUTED, ha=ha) |
| bb = t.get_window_extent(renderer) |
| best = (t, (bb.x0, bb.y0, bb.x1, bb.y1)) |
| placed.append(best[1]) |
|
|
|
|
| def main(): |
| data = {tag: load(tag) for _, _, pts in SERIES for _, tag in pts} |
| plt.rcParams.update({"font.family": ["Helvetica Neue", "Helvetica", "Arial", "DejaVu Sans"], |
| "font.size": 10, "axes.edgecolor": AXIS, "axes.labelcolor": INK2, |
| "xtick.color": MUTED, "ytick.color": MUTED, "text.color": INK}) |
| fig, axes = plt.subplots(2, 3, figsize=(12, 7.6), facecolor=SURFACE) |
| fig.subplots_adjust(left=0.06, right=0.985, top=0.745, bottom=0.075, wspace=0.22, hspace=0.42) |
|
|
| for ax, (bench, title) in zip(axes.flat, BENCHES): |
| ax.set_facecolor(SURFACE) |
| for side in ("top", "right"): |
| ax.spines[side].set_visible(False) |
| for side in ("left", "bottom"): |
| ax.spines[side].set_linewidth(1) |
| ax.grid(True, color=GRID, linewidth=1, zorder=0) |
| ax.tick_params(length=0, pad=4) |
| accs, toks, labels = [], [], [] |
| for label, color, pts in SERIES: |
| ys = [data[tag][bench][0] for _, tag in pts] |
| xs = [data[tag][bench][1] / 1000 for _, tag in pts] |
| accs += ys |
| toks += xs |
| ax.plot(xs, ys, color=color, linewidth=2, solid_capstyle="round", solid_joinstyle="round", zorder=3) |
| ax.plot(xs, ys, "o", color=color, markersize=8, markeredgecolor=SURFACE, markeredgewidth=2, zorder=4, |
| label=label) |
| leftmost = min(range(len(xs)), key=lambda i: xs[i]) |
| for i, ((name, _), x, y) in enumerate(zip(pts, xs, ys)): |
| if name != "default": |
| labels.append((name, x, y, i == leftmost)) |
| lo, hi = min(accs), max(accs) |
| pad = max(2.0, (hi - lo) * 0.18) |
| ax.set_ylim(max(0, lo - pad), min(100.8, hi + pad)) |
| ax.set_xlim(0, max(toks) * 1.12) |
| ax.xaxis.set_major_formatter(FuncFormatter(lambda v, _: "0" if v == 0 else f"{v:g}k")) |
| ax.yaxis.set_major_formatter(FuncFormatter(lambda v, _: f"{v:g}%")) |
| ax.set_title(title, loc="left", fontsize=11.5, fontweight="semibold", color=INK, pad=8) |
| place_labels(fig, ax, labels, [(x, y) for _, _, pts in SERIES for x, y in |
| ((data[t][bench][1] / 1000, data[t][bench][0]) for _, t in pts)]) |
|
|
| for ax in axes[1]: |
| ax.set_xlabel("mean output tokens per answer", fontsize=9.5, color=INK2) |
| for ax in axes[:, 0]: |
| ax.set_ylabel("accuracy", fontsize=9.5, color=INK2) |
|
|
| fig.text(0.06, 0.955, "Dory-Beam-4B: Beam's reasoning efforts in a 4B looped model", |
| fontsize=17, fontweight="semibold", color=INK) |
| fig.text(0.06, 0.918, "Accuracy vs mean output tokens. Labels give the loop count or reasoning effort; the unlabeled " |
| "blue point is the default. vLLM, temperature 1.0, up to 64k tokens.", fontsize=10.5, color=INK2) |
| handles, labels = axes[0, 0].get_legend_handles_labels() |
| fig.legend(handles, labels, loc="upper left", bbox_to_anchor=(0.053, 0.905), ncol=2, frameon=False, |
| fontsize=10.5, handletextpad=0.4, columnspacing=1.6) |
| OUT.parent.mkdir(parents=True, exist_ok=True) |
| fig.savefig(OUT, dpi=200, facecolor=SURFACE) |
| print("wrote", OUT) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
