ConspiracyBench / scripts /validate_sft_dataset.py
stationhouse's picture
Add datasets/ folder and SFT dataset validator (#462)
c333f79
Raw History Blame Contribute Delete
14.4 kB
#!/usr/bin/env python3
"""Check an SFT dataset against the fixed ConspiracyBench training recipe before you submit it:
pip install transformers
python scripts/validate_sft_dataset.py path/to/dataset # folder with train/val/test.jsonl
python scripts/validate_sft_dataset.py path/to/dataset --out stats.json
Exit code 0 means every check passed. The stats it prints are the numbers DATA_CARD.md asks for.
Row format, one JSON object per line:
{"id": "ck-123-456", "messages": [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}],
"meta": {"thread": 123}}
Context budget (the same as the eval, see inspect_eval/modal_solver.py): the eval serves a 32,768-token
window and keeps 8,192 of it for the reply, so for every assistant turn
input (all tokens before the reply, chat template included) <= 24,576
label (the reply and its end-of-turn token) <= 8,192
input + label <= 32,768 (follows from the two above)
Tokens are counted with the base model's tokenizer (Qwen/Qwen3.5-4B) and its chat template with thinking
off, exactly as inspect_eval/modal_lora.py builds a training row. The model reads input and label as one
sequence, so both count against the window.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import statistics
import sys
from collections import Counter, defaultdict
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
CORPUS = ROOT / "questions" / "corpus.jsonl"
CLAIM_KEYWORDS = ROOT / "completed_experiments" / "evals" / "xsft-control-ck" / "claim_keywords.json"
SLURS_DIR = ROOT / "completed_experiments" / "evals" / "xsft-control-ck" / "data_prep"
BASE = "Qwen/Qwen3.5-4B"
BASE_REVISION = "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a"
CONTEXT_TOKENS = 32768
LABEL_TOKENS = 8192
INPUT_TOKENS = CONTEXT_TOKENS - LABEL_TOKENS
MIN_REPLY_CHARS, MAX_REPLY_CHARS = 20, 3000
NGRAM = 8
SPLITS = ("train", "val", "test")
DUP_WARN_SHARE = 0.01
WORD = re.compile(r"[a-z0-9']+")
KW_TOKEN = re.compile(r"[a-z0-9'-]+")
PLACEHOLDERS = re.compile(r"<(?:URL|EMAIL|PHONE|HANDLE|IP)>")
HTML_TAG = re.compile(r"</?(?:a|br|wbr|span|s|b|i|u|p|div|pre|code)\b[^>]*>", re.I)
HTML_ENTITY = re.compile(r"&(?:gt|lt|amp|quot|apos|nbsp|#\d+|#x[0-9a-f]+);", re.I)
POST_LINK = re.compile(r">>>?/?\w*/?\d{3,}")
URL = re.compile(r"(https?://\S+|www\.\S+)", re.I)
EMAIL = re.compile(r"\b[\w.+-]+@[\w-]+\.[\w.-]+\b")
PHONE = re.compile(r"(?<!\d)(?:\+?\d{1,2}[\s.-]?)?\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}(?!\d)")
THINK = re.compile(r"</?think>")
SPECIAL = re.compile(r"<\|[a-z_]+\|>")
def split_of(thread) -> str:
"""The fixed thread split: md5 of the thread id, 5% test, 5% val, 90% train."""
h = int(hashlib.md5(str(thread).encode()).hexdigest(), 16) % 100
return "test" if h < 5 else "val" if h < 10 else "train"
def ngrams(s: str, n: int = NGRAM) -> set[str]:
w = WORD.findall(s.lower())
return {" ".join(w[i:i + n]) for i in range(len(w) - n + 1)}
def corpus_grams(path: Path = CORPUS) -> set[str]:
grams: set[str] = set()
for item in map(json.loads, path.open(encoding="utf-8")):
parts = [item.get("question", ""), item.get("ground_truth_answer", "")]
for key in ("pressure_turns", "diverges_if", "does_not_diverge_if"):
v = item.get(key) or []
parts += v if isinstance(v, list) else [str(v)]
for p in parts:
grams |= ngrams(str(p))
return grams
def claim_phrases(path: Path = CLAIM_KEYWORDS) -> dict[str, list[tuple[str, ...]]]:
d = json.loads(path.read_text(encoding="utf-8"))
return {claim: [tuple(KW_TOKEN.findall(p.lower())) for p in v["phrases"]]
for claim, v in d.items() if not claim.startswith("_")}
def exposed_claims(text: str, phrases: dict[str, list[tuple[str, ...]]], lengths: set[int]) -> set[str]:
toks = KW_TOKEN.findall(text.lower())
seqs = {tuple(toks[i:i + n]) for n in lengths for i in range(len(toks) - n + 1)}
return {c for c, ps in phrases.items() if any(p in seqs for p in ps)}
def load_slur_hits():
sys.path.insert(0, str(SLURS_DIR))
try:
from slurs import hits # noqa: E402
finally:
sys.path.pop(0)
return hits
def token_lengths(messages: list[dict], tok) -> list[tuple[int, int]]:
"""(input, label) token counts for every assistant turn, built the way modal_lora.py builds a row
with thinking off: the prompt is the chat template up to the turn, the label is the reply plus <|im_end|>."""
out = []
for i, m in enumerate(messages):
if m["role"] != "assistant":
continue
prompt = tok.apply_chat_template(messages[:i], tokenize=False, add_generation_prompt=True,
enable_thinking=False)
p = tok(prompt, add_special_tokens=False)["input_ids"]
c = tok(m["content"] + "<|im_end|>\n", add_special_tokens=False)["input_ids"]
out.append((len(p), len(c)))
return out
def schema_errors(row) -> list[str]:
if not isinstance(row, dict):
return ["row is not a JSON object"]
errs = []
if not isinstance(row.get("id"), str) or not row["id"]:
errs.append("missing string `id`")
meta = row.get("meta")
if not isinstance(meta, dict) or meta.get("thread") in (None, ""):
errs.append("missing `meta.thread` (needed for the thread split)")
msgs = row.get("messages")
if not isinstance(msgs, list) or len(msgs) < 2:
return errs + ["`messages` must be a list of at least 2 turns"]
for i, m in enumerate(msgs):
if not isinstance(m, dict) or not isinstance(m.get("content"), str) or not m["content"].strip():
errs.append(f"turn {i}: needs a non-empty string `content`")
continue
want = "user" if i % 2 == 0 else "assistant"
if m.get("role") != want:
errs.append(f"turn {i}: role {m.get('role')!r}, expected {want!r} (alternate user/assistant, no system)")
if msgs[-1].get("role") != "assistant" if isinstance(msgs[-1], dict) else True:
errs.append("last turn must be the assistant reply")
return errs
def text_errors(msgs: list[dict], slur_hits) -> list[str]:
errs = []
for i, m in enumerate(msgs):
t = m["content"]
bare = PLACEHOLDERS.sub("", t)
for name, rx in (("raw HTML tag", HTML_TAG), ("HTML entity", HTML_ENTITY), ("post link >>123", POST_LINK),
("URL (replace with <URL>)", URL), ("email (replace with <EMAIL>)", EMAIL),
("phone number (replace with <PHONE>)", PHONE), ("think tag", THINK),
("chat special token", SPECIAL)):
if rx.search(bare):
errs.append(f"turn {i}: {name}")
if slur_hits and slur_hits(t):
errs.append(f"turn {i}: slur (data_prep/slurs.py)")
reply = msgs[-1]["content"]
if not MIN_REPLY_CHARS <= len(reply) <= MAX_REPLY_CHARS:
errs.append(f"final reply is {len(reply)} chars (allowed {MIN_REPLY_CHARS}-{MAX_REPLY_CHARS})")
return errs
def pct(xs: list[int]) -> dict:
if not xs:
return {}
s = sorted(xs)
at = lambda q: s[min(len(s) - 1, int(q * len(s)))] # noqa: E731
return {"min": s[0], "p50": at(.5), "p90": at(.9), "p99": at(.99), "max": s[-1],
"mean": round(statistics.fmean(s), 1)}
def norm(t: str) -> str:
return " ".join(t.lower().split())
def validate(folder: Path, tok, grams: set[str], phrases: dict, slur_hits=None) -> dict:
errors: dict[str, list[str]] = defaultdict(list)
warnings: list[str] = []
stats: dict = {"splits": {}}
thread_splits: dict[str, set[str]] = defaultdict(set)
seen_ids: set[str] = set()
row_hash: dict[str, str] = {}
lengths = {len(p) for ps in phrases.values() for p in ps}
exposure: Counter = Counter()
for split in SPLITS:
path = folder / f"{split}.jsonl"
if not path.exists():
errors["missing file"].append(f"{split}.jsonl")
continue
s = {"rows": 0, "sha256": hashlib.sha256(path.read_bytes()).hexdigest()}
threads, reply_chars, all_asst_chars, turns = set(), [], 0, Counter()
inp, lab, tot = [], [], []
over = Counter()
reply_norms: Counter = Counter()
overlap = 0
for n, line in enumerate(path.open(encoding="utf-8"), 1):
where = f"{split}.jsonl:{n}"
if not line.strip():
continue
try:
row = json.loads(line)
except json.JSONDecodeError as e:
errors["invalid JSON"].append(f"{where}: {e}")
continue
errs = schema_errors(row)
if errs:
errors["schema"] += [f"{where}: {e}" for e in errs]
continue
s["rows"] += 1
msgs = row["messages"]
if row["id"] in seen_ids:
errors["duplicate id"].append(f"{where}: {row['id']}")
seen_ids.add(row["id"])
thread = str(row["meta"]["thread"])
threads.add(thread)
thread_splits[thread].add(split)
if split_of(thread) != split:
errors["thread in the wrong split (md5 rule)"].append(f"{where}: thread {thread} belongs in {split_of(thread)}")
h = hashlib.md5("\x00".join(m["content"] for m in msgs).encode()).hexdigest()
if h in row_hash:
errors["exact duplicate row"].append(f"{where}: same as {row_hash[h]}")
row_hash.setdefault(h, where)
errors["text"] += [f"{where}: {e}" for e in text_errors(msgs, slur_hits)]
reply_chars.append(len(msgs[-1]["content"]))
all_asst_chars += sum(len(m["content"]) for m in msgs if m["role"] == "assistant")
reply_norms[norm(msgs[-1]["content"])] += 1
turns[len(msgs)] += 1
full = " ".join(m["content"] for m in msgs)
if ngrams(full) & grams:
overlap += 1
errors[f"{NGRAM}-word overlap with the corpus"].append(where)
if split == "train":
exposure.update(exposed_claims(full, phrases, lengths))
for k, (p, c) in enumerate(token_lengths(msgs, tok)):
last = k == len(msgs) // 2 - 1
if last:
inp.append(p)
lab.append(c)
tot.append(p + c)
# input + label <= CONTEXT_TOKENS follows from the two limits; both are checked
for name, bad in (("input", p > INPUT_TOKENS), ("label", c > LABEL_TOKENS)):
if bad:
over[name] += 1
errors[f"over the {name} token limit"].append(f"{where}: input {p}, label {c} (assistant turn {k})")
dup_rows = sum(v - 1 for v in reply_norms.values() if v > 1)
s.update({
"threads": len(threads),
"turns_per_row": dict(sorted(turns.items())),
"final_reply_chars_total": sum(reply_chars),
"final_reply_chars": pct(reply_chars),
"all_assistant_chars_total": all_asst_chars,
"input_tokens": pct(inp), "label_tokens": pct(lab), "total_tokens": pct(tot),
"label_tokens_total": sum(lab), "input_tokens_total": sum(inp),
"rows_over_limits": dict(over),
"duplicate_final_reply_rows": dup_rows,
"duplicate_final_reply_share": round(dup_rows / max(s["rows"], 1), 4),
"corpus_overlap_rows": overlap,
})
if s["duplicate_final_reply_share"] > DUP_WARN_SHARE:
warnings.append(f"{split}: {100 * s['duplicate_final_reply_share']:.1f}% of rows repeat another row's "
"final reply (copypasta?); check near-duplicates")
stats["splits"][split] = s
for thread, ss in thread_splits.items():
if len(ss) > 1:
errors["thread in more than one split"].append(f"thread {thread}: {sorted(ss)}")
if not (folder / "DATA_CARD.md").exists():
warnings.append("no DATA_CARD.md next to the data")
stats["claim_exposure_train"] = {"claims_with_any_row": len(exposure), "of": len(phrases),
"rows_per_claim": dict(exposure.most_common())}
stats["limits"] = {"tokenizer": BASE, "input_tokens": INPUT_TOKENS, "label_tokens": LABEL_TOKENS,
"context_tokens": CONTEXT_TOKENS, "final_reply_chars": [MIN_REPLY_CHARS, MAX_REPLY_CHARS], "corpus_ngram": NGRAM}
errors = {k: v for k, v in errors.items() if v}
return {"ok": not errors, "errors": errors, "warnings": warnings, "stats": stats}
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("folder", type=Path, help="folder with train.jsonl, val.jsonl, test.jsonl")
ap.add_argument("--out", type=Path, help="also write the full report as JSON")
ap.add_argument("--revision", default=BASE_REVISION, help="tokenizer revision (default: the pinned one)")
ap.add_argument("--show", type=int, default=5, help="examples to print per error type")
a = ap.parse_args()
try:
from transformers import AutoTokenizer
except ImportError:
sys.exit("Missing dependency: pip install transformers")
tok = AutoTokenizer.from_pretrained(BASE, revision=a.revision)
report = validate(a.folder, tok, corpus_grams(), claim_phrases(), load_slur_hits())
report["stats"]["limits"]["tokenizer_revision"] = a.revision
if a.out:
a.out.write_text(json.dumps(report, indent=1, ensure_ascii=False), encoding="utf-8")
brief = {k: {kk: vv for kk, vv in v.items() if kk != "rows_per_claim"} if k == "claim_exposure_train" else v
for k, v in report["stats"].items()}
print(json.dumps(brief, indent=1))
for w in report["warnings"]:
print(f"WARNING: {w}")
for kind, items in report["errors"].items():
print(f"ERROR {kind}: {len(items)}")
for it in items[:a.show]:
print(f" {it}")
print("PASS" if report["ok"] else "FAIL")
return 0 if report["ok"] else 1
if __name__ == "__main__":
sys.exit(main())