"""Hero chart for the model card: accuracy vs mean output tokens for base Dory (1/2/4 loops), Dory-Beam-4B v2 at 2 loops across its reasoning efforts (low, default, high) and Dory-Beam-4B-Accurate (2/4 loops). python3 scripts/make_chart.py # -> release/model/assets/accuracy_vs_tokens.png Emphasis form: base Dory in the de-emphasis gray, the two Dory-Beam models in categorical slots 1 and 2. One panel per benchmark. (The v1 chart is scripts/make_chart_v1.py.) """ import json from collections import defaultdict from pathlib import Path import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt # noqa: E402 from matplotlib.ticker import FuncFormatter # noqa: E402 ROOT = Path(__file__).resolve().parent.parent RES = ROOT / "data" / "eval" / "results" OUT = ROOT / "release" / "model" / "assets" / "accuracy_vs_tokens.png" SURFACE, INK, INK2, MUTED, GRID, AXIS = "#fcfcfb", "#0b0b0b", "#52514e", "#898781", "#e1e0d9", "#c3c2b7" BLUE, ORANGE, BASE = "#2a78d6", "#eb6834", "#898781" BENCHES = [("gpqa", "GPQA-Diamond"), ("math500", "MATH-500"), ("aime24", "AIME 2024"), ("aime25", "AIME 2025"), ("amc23", "AMC 2023"), ("heldout", "Held-out (300)")] # (legend label, color, [(point label, result file tag)]) SERIES = [ ("Dory1.0-4B (base) · 1, 2, 4 loops", BASE, [("1", "base_loops1-64k"), ("2", "base_loops2-64k"), ("4", "base_loops4-64k")]), ("Dory-Beam-4B · reasoning effort low, default, high · 2 loops", BLUE, [("low", "rE-t050_loops2-low-64k"), ("default", "rE-t050_loops2-default-64k"), ("high", "rE-t050_loops2-high-64k")]), ("Dory-Beam-4B-Accurate · 2, 4 loops", ORANGE, [("2", "mix030_loops2-64k"), ("4", "mix030_loops4-64k")]), ] def load(tag): d = defaultdict(list) for line in open(RES / f"{tag}.jsonl"): r = json.loads(line) d[r["bench"]].append(r) return {b: (100 * sum(x["correct"] for x in rs) / len(rs), sum(x["tokens"] for x in rs) / len(rs)) for b, rs in d.items()} def place_labels(fig, ax, labels, points): """Point labels in muted ink. Each takes the first candidate spot (right, left, above, below; leftmost points try left first) that overlaps neither an earlier label nor another marker.""" renderer = fig.canvas.get_renderer() to_px = ax.transData.transform r = 6 * fig.dpi / 72 # marker radius plus its surface ring, in pixels markers = [(px - r, py - r, px + r, py + r) for px, py in map(to_px, points)] placed = [] hit = lambda a, b: a[0] < b[2] and b[0] < a[2] and a[1] < b[3] and b[1] < a[3] for name, x, y, leftmost in labels: spots = [((7, -3), "left"), ((-7, -3), "right"), ((0, 7), "center"), ((0, -14), "center")] if leftmost: spots[0], spots[1] = spots[1], spots[0] own = to_px((x, y)) best = None for off, ha in spots: t = ax.annotate(name, (x, y), xytext=off, textcoords="offset points", fontsize=8.5, color=MUTED, ha=ha) bb = t.get_window_extent(renderer) box = (bb.x0, bb.y0, bb.x1, bb.y1) others = [m for m, p in zip(markers, points) if abs(to_px(p)[0] - own[0]) + abs(to_px(p)[1] - own[1]) > 1] if not any(hit(box, o) for o in placed + others): best = (t, box) break t.remove() if best is None: # nothing is free: fall back to the first spot off, ha = spots[0] t = ax.annotate(name, (x, y), xytext=off, textcoords="offset points", fontsize=8.5, color=MUTED, ha=ha) bb = t.get_window_extent(renderer) best = (t, (bb.x0, bb.y0, bb.x1, bb.y1)) placed.append(best[1]) def main(): data = {tag: load(tag) for _, _, pts in SERIES for _, tag in pts} plt.rcParams.update({"font.family": ["Helvetica Neue", "Helvetica", "Arial", "DejaVu Sans"], "font.size": 10, "axes.edgecolor": AXIS, "axes.labelcolor": INK2, "xtick.color": MUTED, "ytick.color": MUTED, "text.color": INK}) fig, axes = plt.subplots(2, 3, figsize=(12, 7.6), facecolor=SURFACE) fig.subplots_adjust(left=0.06, right=0.985, top=0.745, bottom=0.075, wspace=0.22, hspace=0.42) for ax, (bench, title) in zip(axes.flat, BENCHES): ax.set_facecolor(SURFACE) for side in ("top", "right"): ax.spines[side].set_visible(False) for side in ("left", "bottom"): ax.spines[side].set_linewidth(1) ax.grid(True, color=GRID, linewidth=1, zorder=0) ax.tick_params(length=0, pad=4) accs, toks, labels = [], [], [] for label, color, pts in SERIES: ys = [data[tag][bench][0] for _, tag in pts] xs = [data[tag][bench][1] / 1000 for _, tag in pts] accs += ys toks += xs ax.plot(xs, ys, color=color, linewidth=2, solid_capstyle="round", solid_joinstyle="round", zorder=3) ax.plot(xs, ys, "o", color=color, markersize=8, markeredgecolor=SURFACE, markeredgewidth=2, zorder=4, label=label) leftmost = min(range(len(xs)), key=lambda i: xs[i]) for i, ((name, _), x, y) in enumerate(zip(pts, xs, ys)): if name != "default": # the middle of the effort dial; low and high bracket it labels.append((name, x, y, i == leftmost)) lo, hi = min(accs), max(accs) pad = max(2.0, (hi - lo) * 0.18) ax.set_ylim(max(0, lo - pad), min(100.8, hi + pad)) ax.set_xlim(0, max(toks) * 1.12) ax.xaxis.set_major_formatter(FuncFormatter(lambda v, _: "0" if v == 0 else f"{v:g}k")) ax.yaxis.set_major_formatter(FuncFormatter(lambda v, _: f"{v:g}%")) ax.set_title(title, loc="left", fontsize=11.5, fontweight="semibold", color=INK, pad=8) place_labels(fig, ax, labels, [(x, y) for _, _, pts in SERIES for x, y in ((data[t][bench][1] / 1000, data[t][bench][0]) for _, t in pts)]) for ax in axes[1]: ax.set_xlabel("mean output tokens per answer", fontsize=9.5, color=INK2) for ax in axes[:, 0]: ax.set_ylabel("accuracy", fontsize=9.5, color=INK2) fig.text(0.06, 0.955, "Dory-Beam-4B: Beam's reasoning efforts in a 4B looped model", fontsize=17, fontweight="semibold", color=INK) fig.text(0.06, 0.918, "Accuracy vs mean output tokens. Labels give the loop count or reasoning effort; the unlabeled " "blue point is the default. vLLM, temperature 1.0, up to 64k tokens.", fontsize=10.5, color=INK2) handles, labels = axes[0, 0].get_legend_handles_labels() fig.legend(handles, labels, loc="upper left", bbox_to_anchor=(0.053, 0.905), ncol=2, frameon=False, fontsize=10.5, handletextpad=0.4, columnspacing=1.6) OUT.parent.mkdir(parents=True, exist_ok=True) fig.savefig(OUT, dpi=200, facecolor=SURFACE) print("wrote", OUT) if __name__ == "__main__": main()