#!/usr/bin/env python3 """Check an SFT dataset against the fixed ConspiracyBench training recipe before you submit it: pip install transformers python scripts/validate_sft_dataset.py path/to/dataset # folder with train/val/test.jsonl python scripts/validate_sft_dataset.py path/to/dataset --out stats.json Exit code 0 means every check passed. The stats it prints are the numbers DATA_CARD.md asks for. Row format, one JSON object per line: {"id": "ck-123-456", "messages": [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}], "meta": {"thread": 123}} Context budget (the same as the eval, see inspect_eval/modal_solver.py): the eval serves a 32,768-token window and keeps 8,192 of it for the reply, so for every assistant turn input (all tokens before the reply, chat template included) <= 24,576 label (the reply and its end-of-turn token) <= 8,192 input + label <= 32,768 (follows from the two above) Tokens are counted with the base model's tokenizer (Qwen/Qwen3.5-4B) and its chat template with thinking off, exactly as inspect_eval/modal_lora.py builds a training row. The model reads input and label as one sequence, so both count against the window. """ from __future__ import annotations import argparse import hashlib import json import re import statistics import sys from collections import Counter, defaultdict from pathlib import Path ROOT = Path(__file__).resolve().parent.parent CORPUS = ROOT / "questions" / "corpus.jsonl" CLAIM_KEYWORDS = ROOT / "completed_experiments" / "evals" / "xsft-control-ck" / "claim_keywords.json" SLURS_DIR = ROOT / "completed_experiments" / "evals" / "xsft-control-ck" / "data_prep" BASE = "Qwen/Qwen3.5-4B" BASE_REVISION = "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a" CONTEXT_TOKENS = 32768 LABEL_TOKENS = 8192 INPUT_TOKENS = CONTEXT_TOKENS - LABEL_TOKENS MIN_REPLY_CHARS, MAX_REPLY_CHARS = 20, 3000 NGRAM = 8 SPLITS = ("train", "val", "test") DUP_WARN_SHARE = 0.01 WORD = re.compile(r"[a-z0-9']+") KW_TOKEN = re.compile(r"[a-z0-9'-]+") PLACEHOLDERS = re.compile(r"<(?:URL|EMAIL|PHONE|HANDLE|IP)>") HTML_TAG = re.compile(r"]*>", re.I) HTML_ENTITY = re.compile(r"&(?:gt|lt|amp|quot|apos|nbsp|#\d+|#x[0-9a-f]+);", re.I) POST_LINK = re.compile(r">>>?/?\w*/?\d{3,}") URL = re.compile(r"(https?://\S+|www\.\S+)", re.I) EMAIL = re.compile(r"\b[\w.+-]+@[\w-]+\.[\w.-]+\b") PHONE = re.compile(r"(?") SPECIAL = re.compile(r"<\|[a-z_]+\|>") def split_of(thread) -> str: """The fixed thread split: md5 of the thread id, 5% test, 5% val, 90% train.""" h = int(hashlib.md5(str(thread).encode()).hexdigest(), 16) % 100 return "test" if h < 5 else "val" if h < 10 else "train" def ngrams(s: str, n: int = NGRAM) -> set[str]: w = WORD.findall(s.lower()) return {" ".join(w[i:i + n]) for i in range(len(w) - n + 1)} def corpus_grams(path: Path = CORPUS) -> set[str]: grams: set[str] = set() for item in map(json.loads, path.open(encoding="utf-8")): parts = [item.get("question", ""), item.get("ground_truth_answer", "")] for key in ("pressure_turns", "diverges_if", "does_not_diverge_if"): v = item.get(key) or [] parts += v if isinstance(v, list) else [str(v)] for p in parts: grams |= ngrams(str(p)) return grams def claim_phrases(path: Path = CLAIM_KEYWORDS) -> dict[str, list[tuple[str, ...]]]: d = json.loads(path.read_text(encoding="utf-8")) return {claim: [tuple(KW_TOKEN.findall(p.lower())) for p in v["phrases"]] for claim, v in d.items() if not claim.startswith("_")} def exposed_claims(text: str, phrases: dict[str, list[tuple[str, ...]]], lengths: set[int]) -> set[str]: toks = KW_TOKEN.findall(text.lower()) seqs = {tuple(toks[i:i + n]) for n in lengths for i in range(len(toks) - n + 1)} return {c for c, ps in phrases.items() if any(p in seqs for p in ps)} def load_slur_hits(): sys.path.insert(0, str(SLURS_DIR)) try: from slurs import hits # noqa: E402 finally: sys.path.pop(0) return hits def token_lengths(messages: list[dict], tok) -> list[tuple[int, int]]: """(input, label) token counts for every assistant turn, built the way modal_lora.py builds a row with thinking off: the prompt is the chat template up to the turn, the label is the reply plus <|im_end|>.""" out = [] for i, m in enumerate(messages): if m["role"] != "assistant": continue prompt = tok.apply_chat_template(messages[:i], tokenize=False, add_generation_prompt=True, enable_thinking=False) p = tok(prompt, add_special_tokens=False)["input_ids"] c = tok(m["content"] + "<|im_end|>\n", add_special_tokens=False)["input_ids"] out.append((len(p), len(c))) return out def schema_errors(row) -> list[str]: if not isinstance(row, dict): return ["row is not a JSON object"] errs = [] if not isinstance(row.get("id"), str) or not row["id"]: errs.append("missing string `id`") meta = row.get("meta") if not isinstance(meta, dict) or meta.get("thread") in (None, ""): errs.append("missing `meta.thread` (needed for the thread split)") msgs = row.get("messages") if not isinstance(msgs, list) or len(msgs) < 2: return errs + ["`messages` must be a list of at least 2 turns"] for i, m in enumerate(msgs): if not isinstance(m, dict) or not isinstance(m.get("content"), str) or not m["content"].strip(): errs.append(f"turn {i}: needs a non-empty string `content`") continue want = "user" if i % 2 == 0 else "assistant" if m.get("role") != want: errs.append(f"turn {i}: role {m.get('role')!r}, expected {want!r} (alternate user/assistant, no system)") if msgs[-1].get("role") != "assistant" if isinstance(msgs[-1], dict) else True: errs.append("last turn must be the assistant reply") return errs def text_errors(msgs: list[dict], slur_hits) -> list[str]: errs = [] for i, m in enumerate(msgs): t = m["content"] bare = PLACEHOLDERS.sub("", t) for name, rx in (("raw HTML tag", HTML_TAG), ("HTML entity", HTML_ENTITY), ("post link >>123", POST_LINK), ("URL (replace with )", URL), ("email (replace with )", EMAIL), ("phone number (replace with )", PHONE), ("think tag", THINK), ("chat special token", SPECIAL)): if rx.search(bare): errs.append(f"turn {i}: {name}") if slur_hits and slur_hits(t): errs.append(f"turn {i}: slur (data_prep/slurs.py)") reply = msgs[-1]["content"] if not MIN_REPLY_CHARS <= len(reply) <= MAX_REPLY_CHARS: errs.append(f"final reply is {len(reply)} chars (allowed {MIN_REPLY_CHARS}-{MAX_REPLY_CHARS})") return errs def pct(xs: list[int]) -> dict: if not xs: return {} s = sorted(xs) at = lambda q: s[min(len(s) - 1, int(q * len(s)))] # noqa: E731 return {"min": s[0], "p50": at(.5), "p90": at(.9), "p99": at(.99), "max": s[-1], "mean": round(statistics.fmean(s), 1)} def norm(t: str) -> str: return " ".join(t.lower().split()) def validate(folder: Path, tok, grams: set[str], phrases: dict, slur_hits=None) -> dict: errors: dict[str, list[str]] = defaultdict(list) warnings: list[str] = [] stats: dict = {"splits": {}} thread_splits: dict[str, set[str]] = defaultdict(set) seen_ids: set[str] = set() row_hash: dict[str, str] = {} lengths = {len(p) for ps in phrases.values() for p in ps} exposure: Counter = Counter() for split in SPLITS: path = folder / f"{split}.jsonl" if not path.exists(): errors["missing file"].append(f"{split}.jsonl") continue s = {"rows": 0, "sha256": hashlib.sha256(path.read_bytes()).hexdigest()} threads, reply_chars, all_asst_chars, turns = set(), [], 0, Counter() inp, lab, tot = [], [], [] over = Counter() reply_norms: Counter = Counter() overlap = 0 for n, line in enumerate(path.open(encoding="utf-8"), 1): where = f"{split}.jsonl:{n}" if not line.strip(): continue try: row = json.loads(line) except json.JSONDecodeError as e: errors["invalid JSON"].append(f"{where}: {e}") continue errs = schema_errors(row) if errs: errors["schema"] += [f"{where}: {e}" for e in errs] continue s["rows"] += 1 msgs = row["messages"] if row["id"] in seen_ids: errors["duplicate id"].append(f"{where}: {row['id']}") seen_ids.add(row["id"]) thread = str(row["meta"]["thread"]) threads.add(thread) thread_splits[thread].add(split) if split_of(thread) != split: errors["thread in the wrong split (md5 rule)"].append(f"{where}: thread {thread} belongs in {split_of(thread)}") h = hashlib.md5("\x00".join(m["content"] for m in msgs).encode()).hexdigest() if h in row_hash: errors["exact duplicate row"].append(f"{where}: same as {row_hash[h]}") row_hash.setdefault(h, where) errors["text"] += [f"{where}: {e}" for e in text_errors(msgs, slur_hits)] reply_chars.append(len(msgs[-1]["content"])) all_asst_chars += sum(len(m["content"]) for m in msgs if m["role"] == "assistant") reply_norms[norm(msgs[-1]["content"])] += 1 turns[len(msgs)] += 1 full = " ".join(m["content"] for m in msgs) if ngrams(full) & grams: overlap += 1 errors[f"{NGRAM}-word overlap with the corpus"].append(where) if split == "train": exposure.update(exposed_claims(full, phrases, lengths)) for k, (p, c) in enumerate(token_lengths(msgs, tok)): last = k == len(msgs) // 2 - 1 if last: inp.append(p) lab.append(c) tot.append(p + c) # input + label <= CONTEXT_TOKENS follows from the two limits; both are checked for name, bad in (("input", p > INPUT_TOKENS), ("label", c > LABEL_TOKENS)): if bad: over[name] += 1 errors[f"over the {name} token limit"].append(f"{where}: input {p}, label {c} (assistant turn {k})") dup_rows = sum(v - 1 for v in reply_norms.values() if v > 1) s.update({ "threads": len(threads), "turns_per_row": dict(sorted(turns.items())), "final_reply_chars_total": sum(reply_chars), "final_reply_chars": pct(reply_chars), "all_assistant_chars_total": all_asst_chars, "input_tokens": pct(inp), "label_tokens": pct(lab), "total_tokens": pct(tot), "label_tokens_total": sum(lab), "input_tokens_total": sum(inp), "rows_over_limits": dict(over), "duplicate_final_reply_rows": dup_rows, "duplicate_final_reply_share": round(dup_rows / max(s["rows"], 1), 4), "corpus_overlap_rows": overlap, }) if s["duplicate_final_reply_share"] > DUP_WARN_SHARE: warnings.append(f"{split}: {100 * s['duplicate_final_reply_share']:.1f}% of rows repeat another row's " "final reply (copypasta?); check near-duplicates") stats["splits"][split] = s for thread, ss in thread_splits.items(): if len(ss) > 1: errors["thread in more than one split"].append(f"thread {thread}: {sorted(ss)}") if not (folder / "DATA_CARD.md").exists(): warnings.append("no DATA_CARD.md next to the data") stats["claim_exposure_train"] = {"claims_with_any_row": len(exposure), "of": len(phrases), "rows_per_claim": dict(exposure.most_common())} stats["limits"] = {"tokenizer": BASE, "input_tokens": INPUT_TOKENS, "label_tokens": LABEL_TOKENS, "context_tokens": CONTEXT_TOKENS, "final_reply_chars": [MIN_REPLY_CHARS, MAX_REPLY_CHARS], "corpus_ngram": NGRAM} errors = {k: v for k, v in errors.items() if v} return {"ok": not errors, "errors": errors, "warnings": warnings, "stats": stats} def main() -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("folder", type=Path, help="folder with train.jsonl, val.jsonl, test.jsonl") ap.add_argument("--out", type=Path, help="also write the full report as JSON") ap.add_argument("--revision", default=BASE_REVISION, help="tokenizer revision (default: the pinned one)") ap.add_argument("--show", type=int, default=5, help="examples to print per error type") a = ap.parse_args() try: from transformers import AutoTokenizer except ImportError: sys.exit("Missing dependency: pip install transformers") tok = AutoTokenizer.from_pretrained(BASE, revision=a.revision) report = validate(a.folder, tok, corpus_grams(), claim_phrases(), load_slur_hits()) report["stats"]["limits"]["tokenizer_revision"] = a.revision if a.out: a.out.write_text(json.dumps(report, indent=1, ensure_ascii=False), encoding="utf-8") brief = {k: {kk: vv for kk, vv in v.items() if kk != "rows_per_claim"} if k == "claim_exposure_train" else v for k, v in report["stats"].items()} print(json.dumps(brief, indent=1)) for w in report["warnings"]: print(f"WARNING: {w}") for kind, items in report["errors"].items(): print(f"ERROR {kind}: {len(items)}") for it in items[:a.show]: print(f" {it}") print("PASS" if report["ok"] else "FAIL") return 0 if report["ok"] else 1 if __name__ == "__main__": sys.exit(main())