Buckets:
| """What does S1 predict at the 32 'Notes:' mask slots? (top tokens, entropy) -> is a soft-token feedback meaningful?""" | |
| import sys, random, torch | |
| sys.path.insert(0, "/workspace/think-decide/src") | |
| from common import DATA, Renderer, load_canonical, tokenizer_and_mapping | |
| from eval import build_model | |
| from model import collate | |
| tok, mp, mid, mt = tokenizer_and_mapping() | |
| items, _ = load_canonical(DATA + "/dev.jsonl", Renderer(tok, mid, mt, 32), 10**9) | |
| items = random.Random(0).sample(items, 8) | |
| dev = torch.device("cuda") | |
| model = build_model(sys.argv[1] if len(sys.argv) > 1 else "s1", 32, False, dev, mp) | |
| with torch.no_grad(): | |
| for x in items: | |
| b = collate([x], dev, tok.pad_token_id) | |
| e = model.bb.embed_tokens(b["ids"]) | |
| pos = torch.arange(b["ids"].shape[1], device=dev)[None] | |
| h = model.layers(e, b["attn"][:, None, None, :], model.bb.rotary_emb(e, pos)) | |
| z = model.base.lm_head(model.bb.norm(h[0, x["think"]])).float() | |
| z[:, mid] = -1e9 | |
| p = z.softmax(-1) | |
| ent = -(p * p.clamp_min(1e-12).log()).sum(-1) | |
| greedy = tok.decode(p.argmax(-1).tolist()) | |
| print("=" * 90) | |
| print("[%s] %s" % (x["source"], tok.decode(x["ids"][:x["pos"][0]])[-300:].replace("\n", " "))) | |
| print("greedy notes:", repr(greedy[:300])) | |
| print("entropy per slot (nats) first 8:", [round(v, 2) for v in ent[:8].tolist()], " mean %.2f" % ent.mean()) | |
| print("top5 slot0:", [(tok.decode([i]), round(v, 3)) for v, i in zip(*p[0].topk(5))]) | |
Xet Storage Details
- Size:
- 1.5 kB
- Xet hash:
- e276460b9e0174eef3f49fcaba02de4eeb5df08bfcb0f593fd1a21071021e1c3
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.