File size: 6,980 Bytes
0052e99 f698686 0052e99 f698686 0052e99 f698686 0052e99 f698686 0052e99 f698686 0052e99 f698686 0052e99 f698686 0052e99 f698686 0052e99 f698686 0052e99 f698686 0052e99 f698686 0052e99 f698686 0052e99 f698686 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 | """Hero chart for the model card: accuracy vs mean output tokens for base Dory (1/2/4 loops), Dory-Beam-4B v2 at
2 loops across its reasoning efforts (low, default, high) and Dory-Beam-4B-Accurate (2/4 loops).
python3 scripts/make_chart.py # -> release/model/assets/accuracy_vs_tokens.png
Emphasis form: base Dory in the de-emphasis gray, the two Dory-Beam models in categorical slots 1 and 2. One panel
per benchmark. (The v1 chart is scripts/make_chart_v1.py.)
"""
import json
from collections import defaultdict
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
from matplotlib.ticker import FuncFormatter # noqa: E402
ROOT = Path(__file__).resolve().parent.parent
RES = ROOT / "data" / "eval" / "results"
OUT = ROOT / "release" / "model" / "assets" / "accuracy_vs_tokens.png"
SURFACE, INK, INK2, MUTED, GRID, AXIS = "#fcfcfb", "#0b0b0b", "#52514e", "#898781", "#e1e0d9", "#c3c2b7"
BLUE, ORANGE, BASE = "#2a78d6", "#eb6834", "#898781"
BENCHES = [("gpqa", "GPQA-Diamond"), ("math500", "MATH-500"), ("aime24", "AIME 2024"),
("aime25", "AIME 2025"), ("amc23", "AMC 2023"), ("heldout", "Held-out (300)")]
# (legend label, color, [(point label, result file tag)])
SERIES = [
("Dory1.0-4B (base) 路 1, 2, 4 loops", BASE,
[("1", "base_loops1-64k"), ("2", "base_loops2-64k"), ("4", "base_loops4-64k")]),
("Dory-Beam-4B 路 reasoning effort low, default, high 路 2 loops", BLUE,
[("low", "rE-t050_loops2-low-64k"), ("default", "rE-t050_loops2-default-64k"),
("high", "rE-t050_loops2-high-64k")]),
("Dory-Beam-4B-Accurate 路 2, 4 loops", ORANGE,
[("2", "mix030_loops2-64k"), ("4", "mix030_loops4-64k")]),
]
def load(tag):
d = defaultdict(list)
for line in open(RES / f"{tag}.jsonl"):
r = json.loads(line)
d[r["bench"]].append(r)
return {b: (100 * sum(x["correct"] for x in rs) / len(rs), sum(x["tokens"] for x in rs) / len(rs))
for b, rs in d.items()}
def place_labels(fig, ax, labels, points):
"""Point labels in muted ink. Each takes the first candidate spot (right, left, above, below; leftmost points
try left first) that overlaps neither an earlier label nor another marker."""
renderer = fig.canvas.get_renderer()
to_px = ax.transData.transform
r = 6 * fig.dpi / 72 # marker radius plus its surface ring, in pixels
markers = [(px - r, py - r, px + r, py + r) for px, py in map(to_px, points)]
placed = []
hit = lambda a, b: a[0] < b[2] and b[0] < a[2] and a[1] < b[3] and b[1] < a[3]
for name, x, y, leftmost in labels:
spots = [((7, -3), "left"), ((-7, -3), "right"), ((0, 7), "center"), ((0, -14), "center")]
if leftmost:
spots[0], spots[1] = spots[1], spots[0]
own = to_px((x, y))
best = None
for off, ha in spots:
t = ax.annotate(name, (x, y), xytext=off, textcoords="offset points", fontsize=8.5, color=MUTED, ha=ha)
bb = t.get_window_extent(renderer)
box = (bb.x0, bb.y0, bb.x1, bb.y1)
others = [m for m, p in zip(markers, points) if abs(to_px(p)[0] - own[0]) + abs(to_px(p)[1] - own[1]) > 1]
if not any(hit(box, o) for o in placed + others):
best = (t, box)
break
t.remove()
if best is None: # nothing is free: fall back to the first spot
off, ha = spots[0]
t = ax.annotate(name, (x, y), xytext=off, textcoords="offset points", fontsize=8.5, color=MUTED, ha=ha)
bb = t.get_window_extent(renderer)
best = (t, (bb.x0, bb.y0, bb.x1, bb.y1))
placed.append(best[1])
def main():
data = {tag: load(tag) for _, _, pts in SERIES for _, tag in pts}
plt.rcParams.update({"font.family": ["Helvetica Neue", "Helvetica", "Arial", "DejaVu Sans"],
"font.size": 10, "axes.edgecolor": AXIS, "axes.labelcolor": INK2,
"xtick.color": MUTED, "ytick.color": MUTED, "text.color": INK})
fig, axes = plt.subplots(2, 3, figsize=(12, 7.6), facecolor=SURFACE)
fig.subplots_adjust(left=0.06, right=0.985, top=0.745, bottom=0.075, wspace=0.22, hspace=0.42)
for ax, (bench, title) in zip(axes.flat, BENCHES):
ax.set_facecolor(SURFACE)
for side in ("top", "right"):
ax.spines[side].set_visible(False)
for side in ("left", "bottom"):
ax.spines[side].set_linewidth(1)
ax.grid(True, color=GRID, linewidth=1, zorder=0)
ax.tick_params(length=0, pad=4)
accs, toks, labels = [], [], []
for label, color, pts in SERIES:
ys = [data[tag][bench][0] for _, tag in pts]
xs = [data[tag][bench][1] / 1000 for _, tag in pts]
accs += ys
toks += xs
ax.plot(xs, ys, color=color, linewidth=2, solid_capstyle="round", solid_joinstyle="round", zorder=3)
ax.plot(xs, ys, "o", color=color, markersize=8, markeredgecolor=SURFACE, markeredgewidth=2, zorder=4,
label=label)
leftmost = min(range(len(xs)), key=lambda i: xs[i])
for i, ((name, _), x, y) in enumerate(zip(pts, xs, ys)):
if name != "default": # the middle of the effort dial; low and high bracket it
labels.append((name, x, y, i == leftmost))
lo, hi = min(accs), max(accs)
pad = max(2.0, (hi - lo) * 0.18)
ax.set_ylim(max(0, lo - pad), min(100.8, hi + pad))
ax.set_xlim(0, max(toks) * 1.12)
ax.xaxis.set_major_formatter(FuncFormatter(lambda v, _: "0" if v == 0 else f"{v:g}k"))
ax.yaxis.set_major_formatter(FuncFormatter(lambda v, _: f"{v:g}%"))
ax.set_title(title, loc="left", fontsize=11.5, fontweight="semibold", color=INK, pad=8)
place_labels(fig, ax, labels, [(x, y) for _, _, pts in SERIES for x, y in
((data[t][bench][1] / 1000, data[t][bench][0]) for _, t in pts)])
for ax in axes[1]:
ax.set_xlabel("mean output tokens per answer", fontsize=9.5, color=INK2)
for ax in axes[:, 0]:
ax.set_ylabel("accuracy", fontsize=9.5, color=INK2)
fig.text(0.06, 0.955, "Dory-Beam-4B: Beam's reasoning efforts in a 4B looped model",
fontsize=17, fontweight="semibold", color=INK)
fig.text(0.06, 0.918, "Accuracy vs mean output tokens. Labels give the loop count or reasoning effort; the unlabeled "
"blue point is the default. vLLM, temperature 1.0, up to 64k tokens.", fontsize=10.5, color=INK2)
handles, labels = axes[0, 0].get_legend_handles_labels()
fig.legend(handles, labels, loc="upper left", bbox_to_anchor=(0.053, 0.905), ncol=2, frameon=False,
fontsize=10.5, handletextpad=0.4, columnspacing=1.6)
OUT.parent.mkdir(parents=True, exist_ok=True)
fig.savefig(OUT, dpi=200, facecolor=SURFACE)
print("wrote", OUT)
if __name__ == "__main__":
main()
|