Buckets:
| """Check (1) spliced thought slots == rendering the Notes text through the chat template, (2) length stats.""" | |
| import json | |
| import sys | |
| from collections import Counter | |
| import numpy as np | |
| sys.path.insert(0, "/workspace/think-decide/src") | |
| from common import DATA, NOTES_TEXT, Renderer, load_canonical, load_core, tokenizer_and_mapping # noqa: E402 | |
| from bench_local import to_internal # noqa: E402 | |
| from laya.common import render_options, serialize_state # noqa: E402 | |
| tok, mapping, mask_id, mask_text = tokenizer_and_mapping() | |
| r0, r32 = Renderer(tok, mask_id, mask_text, 0), Renderer(tok, mask_id, mask_text, 32) | |
| def direct(state, qdef, T): | |
| """Independent re-implementation of decision_input with the Notes block in the user text.""" | |
| q = to_internal(qdef) | |
| options = render_options(q) | |
| user = "State:\n%s\n\nQuestion:\n%s\n\n" % (serialize_state(state), q["ins"]) | |
| if q["t"] == "noul": | |
| user += "Answer Yes or No.\nAnswer:" + mask_text | |
| else: | |
| user += "For each option, mark Yes if it answers the question, otherwise No.\n" | |
| user += "\n".join(o + ": " + mask_text for o in options) | |
| user += NOTES_TEXT + mask_text * T | |
| prompt = tok.apply_chat_template([{"role": "system", "content": "You are a helpful assistant."}, | |
| {"role": "user", "content": user}], tokenize=False, | |
| add_generation_prompt=False, enable_thinking=False) | |
| ids = [] | |
| for i, part in enumerate(prompt.split(mask_text)): | |
| if i: | |
| ids.append(mask_id) | |
| ids.extend(tok.encode(part, add_special_tokens=False)) | |
| return ids | |
| rows = [json.loads(l) for l, _ in zip(open(DATA + "/train.jsonl"), range(400))] | |
| bad = 0 | |
| for row in rows: | |
| (qid, q), = row["questions"].items() | |
| q = {k: q[k] for k in ("type", "instructions", "criteria") if k in q} | |
| a = r32(row["state"], q)[0] | |
| b = direct(row["state"], q, 32) | |
| bad += a != b | |
| print("splice mismatches on 400 train rows:", bad) | |
| core = [json.loads(l) for l in open("/workspace/data/core-v1/inputs.jsonl")] | |
| bad = sum(r32(r["state"], r["qdef"])[0] != direct(r["state"], r["qdef"], 32) for r in core[::20]) | |
| print("splice mismatches on %d core rows:" % len(core[::20]), bad) | |
| for name in ("train", "dev"): | |
| items, dropped = load_canonical(DATA + "/%s.jsonl" % name, r32, 10**9) | |
| L = np.array([len(x["ids"]) for x in items]) | |
| print(name, len(items), "len pct50/90/99/max", np.percentile(L, [50, 90, 99]).round(), L.max(), | |
| "over2048+32:", int((L > 2048 + 32).sum()), "mean", L.mean().round(1)) | |
| print(" prim", Counter(x["primitive"] for x in items), "K>8", sum(len(x["target"]) > 8 for x in items)) | |
| print(" sources", Counter(x["source"] for x in items).most_common(30)) | |
| print(" long by source", Counter(x["source"] for x in items if len(x["ids"]) > 2080)) | |
| items, uns = load_core(r32, 4096 + 32) | |
| L = np.array([len(x["ids"]) for x in items]) | |
| print("core ok", len(items), "unsupported", Counter(u["status"] for u in uns), Counter(u["suite"] for u in uns)) | |
| print("core len pct50/90/99/max", np.percentile(L, [50, 90, 99]).round(), L.max()) | |
| items0, uns0 = load_core(r0, 4096) | |
| print("core (no think, 4096) ok", len(items0), "unsupported", Counter(u["suite"] for u in uns0)) | |
Xet Storage Details
- Size:
- 3.26 kB
- Xet hash:
- b2c754f23d12158203d322be1257f521ce0a44a333e8ccb7ec4969be6e46d417
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.