Buckets:
| """Compare candidate base models before committing: zero-shot readout on every eval item, and a thinking | |
| throughput probe (4 samples on 128 pool items) with chain lengths, parse rate, and readout after the chain.""" | |
| from __future__ import annotations | |
| import argparse | |
| import collections | |
| import json | |
| import os | |
| import random | |
| import time | |
| import numpy as np | |
| from transformers import AutoTokenizer | |
| import od_score | |
| from common import read_jsonl, softmax, split_chain | |
| from vllm_utils import make_llm, readout, think_sync | |
| def od_predictions(items, logps): | |
| preds = [] | |
| for it, lp in zip(items, logps): | |
| p = softmax(lp) | |
| preds.append({"id": it["od_id"], "probs": {lab: float(x) for lab, x in zip(it["od_labels"], p)}}) | |
| return preds | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--model", required=True) | |
| ap.add_argument("--n-think", type=int, default=128) | |
| ap.add_argument("--out", default="results/probe") | |
| args = ap.parse_args() | |
| os.makedirs(args.out, exist_ok=True) | |
| tag = args.model.split("/")[-1] | |
| tok = AutoTokenizer.from_pretrained(args.model) | |
| llm = make_llm(args.model) | |
| ev = read_jsonl("data/eval.jsonl") | |
| orders = [list(range(len(x["options"]))) for x in ev] | |
| t0 = time.time() | |
| lps = readout(llm, tok, ev, orders) | |
| t_read = time.time() - t0 | |
| res = {"model": args.model, "readout_seconds": t_read, "readout_items": len(ev)} | |
| acc = collections.defaultdict(list) | |
| for it, lp in zip(ev, lps): | |
| acc[it["set"]].append(int(np.argmax(lp)) == it["gold"]) | |
| res["readout_acc"] = {k: float(np.mean(v)) for k, v in acc.items()} | |
| od = [(it, lp) for it, lp in zip(ev, lps) if it["set"] == "opendecisions"] | |
| tasks = od_score.read_jsonl("data/raw/jgeuter__OpenDecisions/opendecisions.jsonl") | |
| s = od_score.score(tasks, od_predictions([x for x, _ in od], [y for _, y in od])) | |
| res["od_raw"] = {k: s[k] for k in ("capability", "intelligence", "calibration")} | |
| pool = read_jsonl("data/train_pool.jsonl")[:args.n_think] | |
| porders = [list(range(len(x["options"]))) for x in pool] | |
| t0 = time.time() | |
| samples = think_sync(llm, tok, pool, porders, n=4, seed=17) | |
| t_think = time.time() - t0 | |
| flat = [s for ss in samples for s in ss] | |
| toks = np.array([s["tokens"] for s in flat]) | |
| res["think"] = {"items": len(pool), "samples": len(flat), "seconds": t_think, | |
| "gen_tok_per_s": float(toks.sum() / t_think), "tokens_median": float(np.median(toks)), | |
| "tokens_p90": float(np.percentile(toks, 90)), "tokens_max": int(toks.max()), | |
| "parse_rate": float(np.mean([s["answer"] is not None for s in flat])), | |
| "looped": int(sum(s["looped"] for s in flat))} | |
| # accuracy: readout vs first thinking sample vs majority vote, on the same pool items | |
| rlps = readout(llm, tok, pool, porders) | |
| r_ok = [int(np.argmax(lp)) == it["gold"] for lp, it in zip(rlps, pool)] | |
| t_ok = [ss[0]["answer"] == it["gold"] for ss, it in zip(samples, pool)] | |
| maj_ok = [] | |
| for ss, it in zip(samples, pool): | |
| votes = collections.Counter(s["answer"] for s in ss if s["answer"] is not None) | |
| maj_ok.append(bool(votes) and votes.most_common(1)[0][0] == it["gold"]) | |
| # readout after the first chain ("decide after thinking") | |
| chains, keep = [], [] | |
| for i, ss in enumerate(samples): | |
| if ss[0]["closed"]: | |
| chains.append(split_chain(ss[0]["text"])[0]) | |
| keep.append(i) | |
| post = readout(llm, tok, [pool[i] for i in keep], [porders[i] for i in keep], think_chains=chains) | |
| post_ok = [int(np.argmax(lp)) == pool[i]["gold"] for lp, i in zip(post, keep)] | |
| post_agree = [int(np.argmax(lp)) == samples[i][0]["answer"] for lp, i in zip(post, keep)] | |
| by_set = collections.defaultdict(lambda: collections.defaultdict(list)) | |
| for i, it in enumerate(pool): | |
| by_set[it["set"]]["readout"].append(r_ok[i]) | |
| by_set[it["set"]]["think1"].append(t_ok[i]) | |
| by_set[it["set"]]["maj4"].append(maj_ok[i]) | |
| res["pool_acc"] = {s: {k: float(np.mean(v)) for k, v in d.items()} | {"n": len(d["readout"])} for s, d in by_set.items()} | |
| res["post_readout_acc"] = float(np.mean(post_ok)) | |
| res["post_readout_agrees_with_generated_letter"] = float(np.mean(post_agree)) | |
| print(json.dumps(res, indent=1)) | |
| with open(os.path.join(args.out, f"{tag}.json"), "w") as f: | |
| json.dump(res, f, indent=1) | |
| with open(os.path.join(args.out, f"{tag}_samples.jsonl"), "w") as f: | |
| for it, ss in zip(pool, samples): | |
| f.write(json.dumps({"id": it["id"], "set": it["set"], "gold": it["gold"], | |
| "samples": [{k: v for k, v in s.items() if k != "text"} | {"text": s["text"][-600:]} for s in ss]}) + "\n") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 4.81 kB
- Xet hash:
- 3c0c62101ed88d436cd425c90be09d8913e7e01c8d7a70c18d41ae39f264b0df
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.