Buckets:
| """Our models on S1MB (System One Mosaic Benchmark, hotchpotch/S1MB), English v1: 137 typed benchmarks (Noul 59, Choice | |
| 57, Score 21), 14,009 cases, 26,269 judgments, dataset hotchpotch/S1MB-english-v1 @ 10bcb8c6. | |
| python s1mb_eval.py build # CPU: parse the dataset into our item format | |
| python s1mb_eval.py run --model <checkpoint> --name rl60 # GPU: fast readout on every judgment | |
| python s1mb_eval.py score # CPU: S1MB metrics, Task Avg, Borda vs the published models | |
| The S1MB evaluator itself is not executed here (third-party code); its data and published results are read as data and | |
| its SCORING.md rules are reimplemented (noul: balanced accuracy at p(true) >= 0.5, adjusted 2 BA - 1; choice: target | |
| mass at the argmax vs the best of uniform / fixed-ID / fixed-position baselines; score: MAE of the expected normalized | |
| value vs the median constant; clip to [0, 1] per benchmark, equal-weight task means, Task Avg = mean of the three; | |
| Borda over all benchmarks across the complete cohort). `score` first re-scores published models from their saved | |
| predictions and checks the result against their saved metrics. | |
| Our readout: one forward pass per judgment in our own typed prompt (the model's native interface), letter | |
| probabilities over the options; choice with more than 26 options runs as a tournament of <= 26-option readouts (the | |
| finalists' probabilities are the answer, 0 elsewhere; accuracy only uses the argmax). Temperatures per question type | |
| are fitted on our own calibration items (they change only the score task's expected values here). | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import glob | |
| import json | |
| import lzma | |
| import math | |
| import os | |
| import numpy as np | |
| from common import read_jsonl, softmax, write_jsonl | |
| D = "data/s1mb" | |
| QT = {"noul": "noul", "choice": "choice", "score": "score"} | |
| def _text(value) -> str: | |
| return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) | |
| def build() -> None: | |
| import pyarrow.parquet as pq | |
| cat = json.load(open(f"{D}/english-v1.json"))["benchmarks"] | |
| rows_out, counts = [], {} | |
| for bid in cat: | |
| b = json.load(open(f"{D}/benchmarks/{bid}.json")) | |
| sub = b["dataset"].split("/", 1)[1] | |
| files = sorted(glob.glob(f"{D}/dataset/data/{sub}/{b['split']}/*.parquet")) | |
| n_case = n_dec = 0 | |
| for f in files: | |
| for row in pq.read_table(f).to_pylist(): | |
| src = row["input"] | |
| targets = {t["decision_id"]: t for t in row["targets"]} | |
| qs = [] | |
| for dec in src["decisions"]: | |
| if dec["type"] != b["task"]: | |
| continue | |
| crit = {c["id"]: c for c in dec["criteria"]} | |
| opts = [] | |
| for c in dec["criteria"]: | |
| desc = json.loads(c["description_json"]) | |
| opts.append({"id": c["id"], "desc": None if desc is None else _text(desc), "value": c["value"]}) | |
| t = targets[dec["id"]] | |
| tp = dict(zip(t["ids"], t["probabilities"])) | |
| instr = _text(json.loads(dec["instructions_json"])) | |
| sysp = dec.get("system_prompt") or "" | |
| qs.append({"qid": dec["id"], "task": dec["type"], "criterion": (sysp + "\n\n" + instr) if sysp else instr, | |
| "options": opts, "target": [float(tp.get(o["id"], 0.0)) for o in opts]}) | |
| if not qs: | |
| continue | |
| n_case += 1 | |
| for q in qs: | |
| n_dec += 1 | |
| rows_out.append({"bench": bid, "task": b["task"], "case_id": row["case_id"], **q, | |
| "evidence": src["state_json"]}) | |
| counts[bid] = (n_case, n_dec, b["case_count"], b["decision_count"]) | |
| bad = {k: v for k, v in counts.items() if v[0] != v[2] or v[1] != v[3]} | |
| write_jsonl(f"{D}/items.jsonl", rows_out) | |
| print(json.dumps({"benchmarks": len(counts), "judgments": len(rows_out), "count_mismatch": bad, | |
| "max_options": max(len(r["options"]) for r in rows_out)}), flush=True) | |
| def to_item(r: dict) -> dict: | |
| """Our typed item: yes/no keeps true first (our noul format), choice / score keep the stored order.""" | |
| opts = r["options"] | |
| if r["task"] == "noul": | |
| by = {o["id"]: o for o in opts} | |
| opts = [by["true"], by["false"]] | |
| return {"id": f"{r['bench']}|{r['case_id']}|{r['qid']}", "qtype": QT[r["task"]], "evidence": r["evidence"], | |
| "criterion": r["criterion"], "options": [{"id": o["id"], "desc": o["desc"]} for o in opts]} | |
| def run(args) -> None: | |
| from transformers import AutoTokenizer | |
| from vllm_utils import make_llm, readout | |
| tok = AutoTokenizer.from_pretrained(args.model) | |
| rows = read_jsonl(f"{D}/items.jsonl") | |
| items = [to_item(r) for r in rows] | |
| lens = [len(x) for x in tok([json.dumps(x["evidence"]) for x in items], add_special_tokens=False)["input_ids"]] | |
| llm = make_llm(args.model, gpu_mem=args.gpu_mem) | |
| out = {} | |
| small = [i for i, x in enumerate(items) if len(x["options"]) <= 26] | |
| for i, lp in zip(small, readout(llm, tok, [items[i] for i in small], [list(range(len(items[i]["options"]))) for i in small])): | |
| out[i] = {"cands": list(range(len(items[i]["options"]))), "lp": lp.tolist()} | |
| big = [i for i, x in enumerate(items) if len(x["options"]) > 26] | |
| cand = {i: list(range(len(items[i]["options"]))) for i in big} | |
| while any(len(c) > 26 for c in cand.values()): # tournament rounds until every list fits 26 letters | |
| sub_items, sub_orders, owner = [], [], [] | |
| for i, c in cand.items(): | |
| if len(c) <= 26: | |
| continue | |
| n = math.ceil(len(c) / 26) | |
| size = math.ceil(len(c) / n) | |
| for k in range(n): | |
| chunk = c[k * size:(k + 1) * size] | |
| sub_items.append({**items[i], "options": [items[i]["options"][j] for j in chunk]}) | |
| sub_orders.append(list(range(len(chunk)))) | |
| owner.append((i, chunk, max(2, 26 // n))) | |
| keep = {} | |
| for (i, chunk, top), lp in zip(owner, readout(llm, tok, sub_items, sub_orders)): | |
| p = softmax(np.asarray(lp), 1.0) | |
| keep.setdefault(i, []).extend(chunk[j] for j in np.argsort(-p)[:top]) | |
| for i, ks in keep.items(): | |
| cand[i] = sorted(ks) | |
| if big: | |
| fin = list(cand) | |
| lps = readout(llm, tok, [{**items[i], "options": [items[i]["options"][j] for j in cand[i]]} for i in fin], | |
| [list(range(len(cand[i]))) for i in fin]) | |
| for i, lp in zip(fin, lps): | |
| out[i] = {"cands": cand[i], "lp": lp.tolist()} | |
| os.makedirs(f"{D}/runs", exist_ok=True) | |
| json.dump({"model": args.model, "name": args.name, "max_evidence_tokens": max(lens), | |
| "answers": [{"bench": rows[i]["bench"], "case_id": rows[i]["case_id"], "qid": rows[i]["qid"], **out[i]} for i in range(len(rows))]}, | |
| open(f"{D}/runs/{args.name}.json", "w")) | |
| print(json.dumps({"saved": f"{D}/runs/{args.name}.json", "judgments": len(rows), "tournament": len(big)}), flush=True) | |
| # ------------------------------------------------------------------ scoring (SCORING.md, baseline-adjusted-v1) | |
| def bench_metrics(task: str, qs: list[dict]) -> dict: | |
| """qs: [{'probs': {id: p}, 'target': {id: t}, 'order': [ids], 'values': {id: v}}] for one benchmark.""" | |
| if task == "noul": | |
| tp = fp = fn = tn = 0.0 | |
| for q in qs: | |
| t = q["target"].get("true", 0.0) | |
| if q["probs"]["true"] >= 0.5: | |
| tp += t; fp += 1 - t | |
| else: | |
| fn += t; tn += 1 - t | |
| if tp + fn <= 1e-12 or tn + fp <= 1e-12: | |
| return {"eligible": False} | |
| ba = (tp / (tp + fn) + tn / (tn + fp)) / 2 | |
| return {"eligible": True, "balanced_accuracy": ba, "skill": 2 * ba - 1} | |
| if task == "choice": | |
| acc = [] | |
| for q in qs: | |
| best = max(q["order"], key=lambda o: (q["probs"].get(o, 0.0), -q["order"].index(o))) | |
| acc.append(q["target"].get(best, 0.0)) | |
| acc = float(np.mean(acc)) | |
| uni = float(np.mean([sum(q["target"].values()) / len(q["order"]) for q in qs])) | |
| ids = {o for q in qs for o in q["order"]} | |
| fid = max(np.mean([q["target"].get(o, 0.0) if o in q["order"] else sum(q["target"].values()) / len(q["order"]) for q in qs]) for o in ids) | |
| npos = max(len(q["order"]) for q in qs) | |
| fpos = max(np.mean([q["target"].get(q["order"][p], 0.0) if p < len(q["order"]) else sum(q["target"].values()) / len(q["order"]) for q in qs]) for p in range(npos)) | |
| b = max(uni, float(fid), float(fpos)) | |
| if b >= 1 - 1e-12: | |
| return {"eligible": False} | |
| return {"eligible": True, "accuracy": acc, "baseline": b, "skill": (acc - b) / (1 - b)} | |
| pe, te = [], [] | |
| for q in qs: | |
| vs = [q["values"][o] for o in q["order"]] | |
| lo, hi = min(vs), max(vs) | |
| norm = {o: (q["values"][o] - lo) / (hi - lo) for o in q["order"]} | |
| pe.append(sum(q["probs"].get(o, 0.0) * norm[o] for o in q["order"])) | |
| te.append(sum(q["target"].get(o, 0.0) * norm[o] for o in q["order"])) | |
| pe, te = np.array(pe), np.array(te) | |
| mae = float(np.mean(np.abs(pe - te))) | |
| base = float(np.mean(np.abs(np.median(te) - te))) | |
| if base <= 1e-12: | |
| return {"eligible": False} | |
| return {"eligible": True, "mae": mae, "baseline_mae": base, "skill": 1 - mae / base} | |
| def task_avg(adj: dict, tasks: dict) -> dict: | |
| by = {} | |
| for b, s in adj.items(): | |
| if s is not None: | |
| by.setdefault(tasks[b], []).append(s) | |
| t = {k: float(np.mean(v)) for k, v in by.items()} | |
| return {**t, "task_avg": float(np.mean([t[k] for k in ("noul", "choice", "score")])) if len(t) == 3 else None} | |
| def our_scores(name: str, items: list[dict], temps: dict | None) -> dict: | |
| R = json.load(open(f"{D}/runs/{name}.json")) | |
| per = {} | |
| for r, a in zip(items, R["answers"]): | |
| assert (r["bench"], r["case_id"], r["qid"]) == (a["bench"], a["case_id"], a["qid"]) | |
| opts = r["options"] if r["task"] != "noul" else sorted(r["options"], key=lambda o: o["id"] != "true") | |
| ids = [o["id"] for o in opts] | |
| T = (temps or {}).get(QT[r["task"]], 1.0) | |
| p = softmax(np.asarray(a["lp"]), T) | |
| probs = {ids[j]: float(pj) for j, pj in zip(a["cands"], p)} | |
| per.setdefault(r["bench"], []).append({"probs": probs, "target": {o["id"]: t for o, t in zip(r["options"], r["target"])}, | |
| "order": [o["id"] for o in r["options"]], "values": {o["id"]: o["value"] for o in r["options"]}}) | |
| return per | |
| def score_all(args) -> None: | |
| from jevals_eval import temps_for_model | |
| items = read_jsonl(f"{D}/items.jsonl") | |
| cat = json.load(open(f"{D}/english-v1.json"))["benchmarks"] | |
| tasks = {b: json.load(open(f"{D}/benchmarks/{b}.json"))["task"] for b in cat} | |
| # published models: saved adjusted scores, and a re-score check from their predictions on two models | |
| pub = {} | |
| for mdir in sorted(glob.glob(f"{D}/published/*/")): | |
| name = os.path.basename(mdir.rstrip("/")) | |
| adj, ok = {}, True | |
| for b in cat: | |
| f = f"{mdir}{b}.json.xz" | |
| if not os.path.exists(f): | |
| ok = False; break | |
| d = json.loads(lzma.open(f).read()) | |
| if d.get("status") != "complete" or d.get("provenance") != "measured": | |
| ok = False; break | |
| adj[b] = d["metrics"].get("baseline_adjusted_score") | |
| if ok: | |
| pub[name] = adj | |
| checks = [] | |
| tgt = {} | |
| for r in items: | |
| tgt.setdefault((r["bench"], r["case_id"], r["qid"]), r) | |
| for name in list(pub)[:3]: | |
| diffs = [] | |
| for b in cat[::9]: | |
| d = json.loads(lzma.open(f"{D}/published/{name}/{b}.json.xz").read()) | |
| qs = [] | |
| for p in d["predictions"]: | |
| r = tgt[(b, p["case_id"], p["question_id"])] | |
| qs.append({"probs": p.get("probabilities") or {}, "target": {o["id"]: t for o, t in zip(r["options"], r["target"])}, | |
| "order": [o["id"] for o in r["options"]], "values": {o["id"]: o["value"] for o in r["options"]}}) | |
| m = bench_metrics(tasks[b], qs) | |
| mine = min(1.0, max(0.0, m["skill"])) if m["eligible"] else None | |
| theirs = d["metrics"].get("baseline_adjusted_score") | |
| if mine is not None and theirs is not None: | |
| diffs.append(abs(mine - theirs)) | |
| checks.append({"model": name, "benchmarks_checked": len(diffs), "max_abs_diff": max(diffs) if diffs else None}) | |
| print(json.dumps({"rescore_check": checks}), flush=True) | |
| ours = {} | |
| for f in sorted(glob.glob(f"{D}/runs/*.json")): | |
| name = os.path.basename(f)[:-5] | |
| from jevals_eval import PRED | |
| pf = PRED.get(name) | |
| temps = temps_for_model(pf) if pf and os.path.exists(pf) else None | |
| out = {} | |
| for mode, T in (("raw", None), ("calibrated", temps)): | |
| if mode == "calibrated" and T is None: | |
| continue | |
| per = our_scores(name, items, T) | |
| adj = {} | |
| for b in cat: | |
| m = bench_metrics(tasks[b], per[b]) | |
| adj[b] = min(1.0, max(0.0, m["skill"])) if m["eligible"] else None | |
| out[mode] = {"adjusted": adj, **task_avg(adj, tasks)} | |
| ours[name] = {"temps": temps, **out} | |
| # cohort: complete published models + ours (headline = calibrated where available); Borda over every benchmark | |
| cohort = {k: v for k, v in pub.items()} | |
| for k, o in ours.items(): | |
| cohort[f"OURS:{k}"] = (o.get("calibrated") or o["raw"])["adjusted"] | |
| elig = [b for b in cat if all(cohort[m].get(b) is not None for m in cohort)] | |
| N = len(cohort) | |
| borda = {m: 0.0 for m in cohort} | |
| for b in elig: | |
| vals = sorted(((cohort[m][b], m) for m in cohort), reverse=True) | |
| i = 0 | |
| while i < N: | |
| j = i | |
| while j + 1 < N and vals[j + 1][0] == vals[i][0]: | |
| j += 1 | |
| pts = np.mean([100 * (N - (r + 1)) / (N - 1) for r in range(i, j + 1)]) | |
| for r in range(i, j + 1): | |
| borda[vals[r][1]] += pts | |
| i = j + 1 | |
| borda = {m: s / len(elig) for m, s in borda.items()} | |
| table = [] | |
| for m, adj in cohort.items(): | |
| ta = task_avg(adj, tasks) | |
| table.append({"model": m, "borda": borda[m], **{k: (v * 100 if v is not None else None) for k, v in ta.items()}}) | |
| table.sort(key=lambda r: -r["borda"]) | |
| json.dump({"dataset": "hotchpotch/S1MB-english-v1@10bcb8c6", "benchmarks": len(cat), "borda_benchmarks": len(elig), | |
| "cohort": N, "rescore_check": checks, "ours": ours, "leaderboard": table}, open("results/s1mb.json", "w"), indent=1) | |
| for i, r in enumerate(table): | |
| if r["model"].startswith("OURS") or i < 25: | |
| print(f"{i + 1:4d} {r['model'][:48]:48s} Borda {r['borda']:6.2f} TaskAvg {r['task_avg'] or float('nan'):6.2f} " | |
| f"Noul {r.get('noul') or float('nan'):6.2f} Choice {r.get('choice') or float('nan'):6.2f} Score {r.get('score') or float('nan'):6.2f}") | |
| def main() -> None: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("cmd", choices=["build", "run", "score"]) | |
| ap.add_argument("--model") | |
| ap.add_argument("--name") | |
| ap.add_argument("--gpu-mem", type=float, default=0.90) | |
| args = ap.parse_args() | |
| if args.cmd == "run": | |
| os.environ.setdefault("MAX_BATCHED_TOKENS", "32768") | |
| if args.cmd == "build": | |
| build() | |
| elif args.cmd == "run": | |
| run(args) | |
| else: | |
| score_all(args) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 15.9 kB
- Xet hash:
- 1986bc17cd1c9258a1d89865c55ce37ed2c1ec6f018c875f9694109709bbc32f
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.