Spaces:
Running
Running
Download scripts/validate_sft_dataset.py from daKeshra/ConspiracyBench: direct link, hf CLI and curl.
- Browser
- Download file 14.4 kB
-
https://huggingface.co/spaces/daKeshra/ConspiracyBench/resolve/main/scripts/validate_sft_dataset.py
- Command line
-
hf download hf://spaces/daKeshra/ConspiracyBench/scripts/validate_sft_dataset.py
-
curl -L -o validate_sft_dataset.py https://huggingface.co/spaces/daKeshra/ConspiracyBench/resolve/main/scripts/validate_sft_dataset.py
14.4 kB
| #!/usr/bin/env python3 | |
| """Check an SFT dataset against the fixed ConspiracyBench training recipe before you submit it: | |
| pip install transformers | |
| python scripts/validate_sft_dataset.py path/to/dataset # folder with train/val/test.jsonl | |
| python scripts/validate_sft_dataset.py path/to/dataset --out stats.json | |
| Exit code 0 means every check passed. The stats it prints are the numbers DATA_CARD.md asks for. | |
| Row format, one JSON object per line: | |
| {"id": "ck-123-456", "messages": [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}], | |
| "meta": {"thread": 123}} | |
| Context budget (the same as the eval, see inspect_eval/modal_solver.py): the eval serves a 32,768-token | |
| window and keeps 8,192 of it for the reply, so for every assistant turn | |
| input (all tokens before the reply, chat template included) <= 24,576 | |
| label (the reply and its end-of-turn token) <= 8,192 | |
| input + label <= 32,768 (follows from the two above) | |
| Tokens are counted with the base model's tokenizer (Qwen/Qwen3.5-4B) and its chat template with thinking | |
| off, exactly as inspect_eval/modal_lora.py builds a training row. The model reads input and label as one | |
| sequence, so both count against the window. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| import re | |
| import statistics | |
| import sys | |
| from collections import Counter, defaultdict | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parent.parent | |
| CORPUS = ROOT / "questions" / "corpus.jsonl" | |
| CLAIM_KEYWORDS = ROOT / "completed_experiments" / "evals" / "xsft-control-ck" / "claim_keywords.json" | |
| SLURS_DIR = ROOT / "completed_experiments" / "evals" / "xsft-control-ck" / "data_prep" | |
| BASE = "Qwen/Qwen3.5-4B" | |
| BASE_REVISION = "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a" | |
| CONTEXT_TOKENS = 32768 | |
| LABEL_TOKENS = 8192 | |
| INPUT_TOKENS = CONTEXT_TOKENS - LABEL_TOKENS | |
| MIN_REPLY_CHARS, MAX_REPLY_CHARS = 20, 3000 | |
| NGRAM = 8 | |
| SPLITS = ("train", "val", "test") | |
| DUP_WARN_SHARE = 0.01 | |
| WORD = re.compile(r"[a-z0-9']+") | |
| KW_TOKEN = re.compile(r"[a-z0-9'-]+") | |
| PLACEHOLDERS = re.compile(r"<(?:URL|EMAIL|PHONE|HANDLE|IP)>") | |
| HTML_TAG = re.compile(r"</?(?:a|br|wbr|span|s|b|i|u|p|div|pre|code)\b[^>]*>", re.I) | |
| HTML_ENTITY = re.compile(r"&(?:gt|lt|amp|quot|apos|nbsp|#\d+|#x[0-9a-f]+);", re.I) | |
| POST_LINK = re.compile(r">>>?/?\w*/?\d{3,}") | |
| URL = re.compile(r"(https?://\S+|www\.\S+)", re.I) | |
| EMAIL = re.compile(r"\b[\w.+-]+@[\w-]+\.[\w.-]+\b") | |
| PHONE = re.compile(r"(?<!\d)(?:\+?\d{1,2}[\s.-]?)?\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}(?!\d)") | |
| THINK = re.compile(r"</?think>") | |
| SPECIAL = re.compile(r"<\|[a-z_]+\|>") | |
| def split_of(thread) -> str: | |
| """The fixed thread split: md5 of the thread id, 5% test, 5% val, 90% train.""" | |
| h = int(hashlib.md5(str(thread).encode()).hexdigest(), 16) % 100 | |
| return "test" if h < 5 else "val" if h < 10 else "train" | |
| def ngrams(s: str, n: int = NGRAM) -> set[str]: | |
| w = WORD.findall(s.lower()) | |
| return {" ".join(w[i:i + n]) for i in range(len(w) - n + 1)} | |
| def corpus_grams(path: Path = CORPUS) -> set[str]: | |
| grams: set[str] = set() | |
| for item in map(json.loads, path.open(encoding="utf-8")): | |
| parts = [item.get("question", ""), item.get("ground_truth_answer", "")] | |
| for key in ("pressure_turns", "diverges_if", "does_not_diverge_if"): | |
| v = item.get(key) or [] | |
| parts += v if isinstance(v, list) else [str(v)] | |
| for p in parts: | |
| grams |= ngrams(str(p)) | |
| return grams | |
| def claim_phrases(path: Path = CLAIM_KEYWORDS) -> dict[str, list[tuple[str, ...]]]: | |
| d = json.loads(path.read_text(encoding="utf-8")) | |
| return {claim: [tuple(KW_TOKEN.findall(p.lower())) for p in v["phrases"]] | |
| for claim, v in d.items() if not claim.startswith("_")} | |
| def exposed_claims(text: str, phrases: dict[str, list[tuple[str, ...]]], lengths: set[int]) -> set[str]: | |
| toks = KW_TOKEN.findall(text.lower()) | |
| seqs = {tuple(toks[i:i + n]) for n in lengths for i in range(len(toks) - n + 1)} | |
| return {c for c, ps in phrases.items() if any(p in seqs for p in ps)} | |
| def load_slur_hits(): | |
| sys.path.insert(0, str(SLURS_DIR)) | |
| try: | |
| from slurs import hits # noqa: E402 | |
| finally: | |
| sys.path.pop(0) | |
| return hits | |
| def token_lengths(messages: list[dict], tok) -> list[tuple[int, int]]: | |
| """(input, label) token counts for every assistant turn, built the way modal_lora.py builds a row | |
| with thinking off: the prompt is the chat template up to the turn, the label is the reply plus <|im_end|>.""" | |
| out = [] | |
| for i, m in enumerate(messages): | |
| if m["role"] != "assistant": | |
| continue | |
| prompt = tok.apply_chat_template(messages[:i], tokenize=False, add_generation_prompt=True, | |
| enable_thinking=False) | |
| p = tok(prompt, add_special_tokens=False)["input_ids"] | |
| c = tok(m["content"] + "<|im_end|>\n", add_special_tokens=False)["input_ids"] | |
| out.append((len(p), len(c))) | |
| return out | |
| def schema_errors(row) -> list[str]: | |
| if not isinstance(row, dict): | |
| return ["row is not a JSON object"] | |
| errs = [] | |
| if not isinstance(row.get("id"), str) or not row["id"]: | |
| errs.append("missing string `id`") | |
| meta = row.get("meta") | |
| if not isinstance(meta, dict) or meta.get("thread") in (None, ""): | |
| errs.append("missing `meta.thread` (needed for the thread split)") | |
| msgs = row.get("messages") | |
| if not isinstance(msgs, list) or len(msgs) < 2: | |
| return errs + ["`messages` must be a list of at least 2 turns"] | |
| for i, m in enumerate(msgs): | |
| if not isinstance(m, dict) or not isinstance(m.get("content"), str) or not m["content"].strip(): | |
| errs.append(f"turn {i}: needs a non-empty string `content`") | |
| continue | |
| want = "user" if i % 2 == 0 else "assistant" | |
| if m.get("role") != want: | |
| errs.append(f"turn {i}: role {m.get('role')!r}, expected {want!r} (alternate user/assistant, no system)") | |
| if msgs[-1].get("role") != "assistant" if isinstance(msgs[-1], dict) else True: | |
| errs.append("last turn must be the assistant reply") | |
| return errs | |
| def text_errors(msgs: list[dict], slur_hits) -> list[str]: | |
| errs = [] | |
| for i, m in enumerate(msgs): | |
| t = m["content"] | |
| bare = PLACEHOLDERS.sub("", t) | |
| for name, rx in (("raw HTML tag", HTML_TAG), ("HTML entity", HTML_ENTITY), ("post link >>123", POST_LINK), | |
| ("URL (replace with <URL>)", URL), ("email (replace with <EMAIL>)", EMAIL), | |
| ("phone number (replace with <PHONE>)", PHONE), ("think tag", THINK), | |
| ("chat special token", SPECIAL)): | |
| if rx.search(bare): | |
| errs.append(f"turn {i}: {name}") | |
| if slur_hits and slur_hits(t): | |
| errs.append(f"turn {i}: slur (data_prep/slurs.py)") | |
| reply = msgs[-1]["content"] | |
| if not MIN_REPLY_CHARS <= len(reply) <= MAX_REPLY_CHARS: | |
| errs.append(f"final reply is {len(reply)} chars (allowed {MIN_REPLY_CHARS}-{MAX_REPLY_CHARS})") | |
| return errs | |
| def pct(xs: list[int]) -> dict: | |
| if not xs: | |
| return {} | |
| s = sorted(xs) | |
| at = lambda q: s[min(len(s) - 1, int(q * len(s)))] # noqa: E731 | |
| return {"min": s[0], "p50": at(.5), "p90": at(.9), "p99": at(.99), "max": s[-1], | |
| "mean": round(statistics.fmean(s), 1)} | |
| def norm(t: str) -> str: | |
| return " ".join(t.lower().split()) | |
| def validate(folder: Path, tok, grams: set[str], phrases: dict, slur_hits=None) -> dict: | |
| errors: dict[str, list[str]] = defaultdict(list) | |
| warnings: list[str] = [] | |
| stats: dict = {"splits": {}} | |
| thread_splits: dict[str, set[str]] = defaultdict(set) | |
| seen_ids: set[str] = set() | |
| row_hash: dict[str, str] = {} | |
| lengths = {len(p) for ps in phrases.values() for p in ps} | |
| exposure: Counter = Counter() | |
| for split in SPLITS: | |
| path = folder / f"{split}.jsonl" | |
| if not path.exists(): | |
| errors["missing file"].append(f"{split}.jsonl") | |
| continue | |
| s = {"rows": 0, "sha256": hashlib.sha256(path.read_bytes()).hexdigest()} | |
| threads, reply_chars, all_asst_chars, turns = set(), [], 0, Counter() | |
| inp, lab, tot = [], [], [] | |
| over = Counter() | |
| reply_norms: Counter = Counter() | |
| overlap = 0 | |
| for n, line in enumerate(path.open(encoding="utf-8"), 1): | |
| where = f"{split}.jsonl:{n}" | |
| if not line.strip(): | |
| continue | |
| try: | |
| row = json.loads(line) | |
| except json.JSONDecodeError as e: | |
| errors["invalid JSON"].append(f"{where}: {e}") | |
| continue | |
| errs = schema_errors(row) | |
| if errs: | |
| errors["schema"] += [f"{where}: {e}" for e in errs] | |
| continue | |
| s["rows"] += 1 | |
| msgs = row["messages"] | |
| if row["id"] in seen_ids: | |
| errors["duplicate id"].append(f"{where}: {row['id']}") | |
| seen_ids.add(row["id"]) | |
| thread = str(row["meta"]["thread"]) | |
| threads.add(thread) | |
| thread_splits[thread].add(split) | |
| if split_of(thread) != split: | |
| errors["thread in the wrong split (md5 rule)"].append(f"{where}: thread {thread} belongs in {split_of(thread)}") | |
| h = hashlib.md5("\x00".join(m["content"] for m in msgs).encode()).hexdigest() | |
| if h in row_hash: | |
| errors["exact duplicate row"].append(f"{where}: same as {row_hash[h]}") | |
| row_hash.setdefault(h, where) | |
| errors["text"] += [f"{where}: {e}" for e in text_errors(msgs, slur_hits)] | |
| reply_chars.append(len(msgs[-1]["content"])) | |
| all_asst_chars += sum(len(m["content"]) for m in msgs if m["role"] == "assistant") | |
| reply_norms[norm(msgs[-1]["content"])] += 1 | |
| turns[len(msgs)] += 1 | |
| full = " ".join(m["content"] for m in msgs) | |
| if ngrams(full) & grams: | |
| overlap += 1 | |
| errors[f"{NGRAM}-word overlap with the corpus"].append(where) | |
| if split == "train": | |
| exposure.update(exposed_claims(full, phrases, lengths)) | |
| for k, (p, c) in enumerate(token_lengths(msgs, tok)): | |
| last = k == len(msgs) // 2 - 1 | |
| if last: | |
| inp.append(p) | |
| lab.append(c) | |
| tot.append(p + c) | |
| # input + label <= CONTEXT_TOKENS follows from the two limits; both are checked | |
| for name, bad in (("input", p > INPUT_TOKENS), ("label", c > LABEL_TOKENS)): | |
| if bad: | |
| over[name] += 1 | |
| errors[f"over the {name} token limit"].append(f"{where}: input {p}, label {c} (assistant turn {k})") | |
| dup_rows = sum(v - 1 for v in reply_norms.values() if v > 1) | |
| s.update({ | |
| "threads": len(threads), | |
| "turns_per_row": dict(sorted(turns.items())), | |
| "final_reply_chars_total": sum(reply_chars), | |
| "final_reply_chars": pct(reply_chars), | |
| "all_assistant_chars_total": all_asst_chars, | |
| "input_tokens": pct(inp), "label_tokens": pct(lab), "total_tokens": pct(tot), | |
| "label_tokens_total": sum(lab), "input_tokens_total": sum(inp), | |
| "rows_over_limits": dict(over), | |
| "duplicate_final_reply_rows": dup_rows, | |
| "duplicate_final_reply_share": round(dup_rows / max(s["rows"], 1), 4), | |
| "corpus_overlap_rows": overlap, | |
| }) | |
| if s["duplicate_final_reply_share"] > DUP_WARN_SHARE: | |
| warnings.append(f"{split}: {100 * s['duplicate_final_reply_share']:.1f}% of rows repeat another row's " | |
| "final reply (copypasta?); check near-duplicates") | |
| stats["splits"][split] = s | |
| for thread, ss in thread_splits.items(): | |
| if len(ss) > 1: | |
| errors["thread in more than one split"].append(f"thread {thread}: {sorted(ss)}") | |
| if not (folder / "DATA_CARD.md").exists(): | |
| warnings.append("no DATA_CARD.md next to the data") | |
| stats["claim_exposure_train"] = {"claims_with_any_row": len(exposure), "of": len(phrases), | |
| "rows_per_claim": dict(exposure.most_common())} | |
| stats["limits"] = {"tokenizer": BASE, "input_tokens": INPUT_TOKENS, "label_tokens": LABEL_TOKENS, | |
| "context_tokens": CONTEXT_TOKENS, "final_reply_chars": [MIN_REPLY_CHARS, MAX_REPLY_CHARS], "corpus_ngram": NGRAM} | |
| errors = {k: v for k, v in errors.items() if v} | |
| return {"ok": not errors, "errors": errors, "warnings": warnings, "stats": stats} | |
| def main() -> int: | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("folder", type=Path, help="folder with train.jsonl, val.jsonl, test.jsonl") | |
| ap.add_argument("--out", type=Path, help="also write the full report as JSON") | |
| ap.add_argument("--revision", default=BASE_REVISION, help="tokenizer revision (default: the pinned one)") | |
| ap.add_argument("--show", type=int, default=5, help="examples to print per error type") | |
| a = ap.parse_args() | |
| try: | |
| from transformers import AutoTokenizer | |
| except ImportError: | |
| sys.exit("Missing dependency: pip install transformers") | |
| tok = AutoTokenizer.from_pretrained(BASE, revision=a.revision) | |
| report = validate(a.folder, tok, corpus_grams(), claim_phrases(), load_slur_hits()) | |
| report["stats"]["limits"]["tokenizer_revision"] = a.revision | |
| if a.out: | |
| a.out.write_text(json.dumps(report, indent=1, ensure_ascii=False), encoding="utf-8") | |
| brief = {k: {kk: vv for kk, vv in v.items() if kk != "rows_per_claim"} if k == "claim_exposure_train" else v | |
| for k, v in report["stats"].items()} | |
| print(json.dumps(brief, indent=1)) | |
| for w in report["warnings"]: | |
| print(f"WARNING: {w}") | |
| for kind, items in report["errors"].items(): | |
| print(f"ERROR {kind}: {len(items)}") | |
| for it in items[:a.show]: | |
| print(f" {it}") | |
| print("PASS" if report["ok"] else "FAIL") | |
| return 0 if report["ok"] else 1 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |