File size: 3,472 Bytes
e46c127 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | #!/usr/bin/env python
"""Radar: openjev NLI-4B zero-shot rerank vs Jev and Terra on the same spokes (all zero-shot).
Jev / Terra values are read off the reference chart (letter-only Terra), so they are approximate.
python radar.py --out assets/radar_openjev.png
"""
import argparse, json
import numpy as np
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
SPOKES = [("MMLU", "mmlu"), ("GPQA\nDiamond", "gpqa"), ("ARC-Easy", "arc_easy"), ("ARC-Challenge", "arc_challenge"),
("WinoGrande", "winogrande"), ("HellaSwag", "hellaswag"), ("GSM8K\n4 choices", "gsm8k_mc4"),
("GSM8K\n10 choices", "gsm8k_mc10"), ("Chess\n4 legal moves", "chess")]
# approximate, read off the "Jev vs. Terra" chart
JEV = [86, 50, 97, 92, 68, 86, 78, 67, 38]
TERRA = [84, 42, 97, 92, 52, 88, 86, 83, 45]
def load(paths):
r = {}
for p in paths:
try:
d = json.load(open(p))
except FileNotFoundError:
continue
d = d.get("results", d)
for m, v in d.items():
r.setdefault(m, {}).update(v)
return r
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--out", default="assets/radar_openjev.png")
args = ap.parse_args()
ev = load(["results/qwen4b_all.json", "results/qwen4b_extra_mc.json"])["ckpt/qwen3.5-4b-nli"]
zero = [100 * ev[k]["rerank_acc"] if k in ev else np.nan for _, k in SPOKES]
n = len(SPOKES)
ang = np.linspace(0, 2 * np.pi, n, endpoint=False)
close = lambda v: np.r_[v, v[:1]]
fig = plt.figure(figsize=(9.2, 10.4), facecolor="white")
ax = fig.add_subplot(111, polar=True)
fig.subplots_adjust(top=0.86, bottom=0.17)
ax.set_theta_offset(np.pi / 2); ax.set_theta_direction(-1)
ax.set_facecolor("#f4f5f7")
ax.set_ylim(0, 100); ax.set_yticks([20, 40, 60, 80, 100]); ax.set_yticklabels([f"{t}%" for t in [20, 40, 60, 80, 100]], color="#9aa0a6", fontsize=8)
ax.set_xticks(ang); ax.set_xticklabels([s for s, _ in SPOKES], fontsize=10.5, color="#222")
ax.grid(color="#d0d4d9", linewidth=0.8); ax.spines["polar"].set_color("#c8ccd1")
series = [("Jev", JEV, "#c4753a", "-"), ("Terra 路 letter only", TERRA, "#4c7fa8", "-"),
("openjev 路 NLI-4B zero-shot rerank", zero, "#4a4a4a", "-")]
for name, v, col, ls in series:
v = np.array(v, dtype=float)
ax.plot(close(ang), close(v), color=col, linewidth=2, linestyle=ls, marker="o", markersize=4, label=name)
ax.fill(close(ang), close(np.nan_to_num(v)), color=col, alpha=0.06)
ax.set_title("openjev vs. Jev vs. Terra", fontsize=17, fontweight="bold", pad=42)
fig.text(0.5, 0.905, "multiple-choice accuracy, common 0-100% scale 路 Jev/Terra read off their published chart",
ha="center", fontsize=9.5, color="#666")
ax.legend(loc="upper center", bbox_to_anchor=(0.5, -0.07), ncol=3, frameon=False, fontsize=9.5)
fig.text(0.5, 0.035, "openjev: Qwen3.5-4B fine-tuned as an NLI cross-encoder, zero-shot = argmax P(entailment) over the options, no task-specific training.\n"
"Chess is our synthetic 4-move legality set; GSM8K k-choice uses gold + numeric distractors. Polygon area is not an aggregate score.",
ha="center", fontsize=8, color="#777")
fig.savefig(args.out, dpi=170)
print("wrote", args.out)
for (s, k), z in zip(SPOKES, zero):
print(f"{s.replace(chr(10), ' '):22s} zero-shot {z:5.1f}")
if __name__ == "__main__":
main()
|