Buckets:
| """How well does S1 already fit its own training set? (per-source CE, accuracy, share of low-confidence items)""" | |
| import sys, random, json, torch, numpy as np | |
| from collections import defaultdict | |
| sys.path.insert(0, "/workspace/think-decide/src") | |
| from common import DATA, Renderer, load_canonical, tokenizer_and_mapping | |
| from eval import build_model, run | |
| tok, mp, mid, mt = tokenizer_and_mapping() | |
| path = sys.argv[1] if len(sys.argv) > 1 else DATA + "/train.jsonl" | |
| items, _ = load_canonical(path, Renderer(tok, mid, mt, 0), 10**9) | |
| items = random.Random(0).sample(items, min(5000, len(items))) | |
| model = build_model("s1", 0, False, torch.device("cuda"), mp) | |
| pp, _, _ = run(model, items, 1, torch.device("cuda"), tok.pad_token_id, 65536, 32) | |
| by = defaultdict(list) | |
| for x, p in zip(items, pp): | |
| p = p[0]; t = np.array(x["target"]) | |
| by[x["source"]].append((-(t * np.log(np.maximum(p, 1e-12))).sum() + (t[t > 0] * np.log(t[t > 0])).sum(), | |
| float(p.argmax() == t.argmax()), float(p[t.argmax()] < 0.9))) | |
| print(path) | |
| allv = [v for vs in by.values() for v in vs] | |
| print("%-24s n=%5d KL %.4f acc %.3f p_gold<0.9 %.3f" % ("ALL", len(allv), *np.mean(allv, 0))) | |
| for s, vs in sorted(by.items()): | |
| print("%-24s n=%5d KL %.4f acc %.3f p_gold<0.9 %.3f" % (s, len(vs), *np.mean(vs, 0))) | |
Xet Storage Details
- Size:
- 1.31 kB
- Xet hash:
- cd1fd52437de6f951659a3676655d3d7de51a4165b1788afa12d09f5876fa244
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.