Spaces:
Running
Running
File size: 14,351 Bytes
c333f79 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 | #!/usr/bin/env python3
"""Check an SFT dataset against the fixed ConspiracyBench training recipe before you submit it:
pip install transformers
python scripts/validate_sft_dataset.py path/to/dataset # folder with train/val/test.jsonl
python scripts/validate_sft_dataset.py path/to/dataset --out stats.json
Exit code 0 means every check passed. The stats it prints are the numbers DATA_CARD.md asks for.
Row format, one JSON object per line:
{"id": "ck-123-456", "messages": [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}],
"meta": {"thread": 123}}
Context budget (the same as the eval, see inspect_eval/modal_solver.py): the eval serves a 32,768-token
window and keeps 8,192 of it for the reply, so for every assistant turn
input (all tokens before the reply, chat template included) <= 24,576
label (the reply and its end-of-turn token) <= 8,192
input + label <= 32,768 (follows from the two above)
Tokens are counted with the base model's tokenizer (Qwen/Qwen3.5-4B) and its chat template with thinking
off, exactly as inspect_eval/modal_lora.py builds a training row. The model reads input and label as one
sequence, so both count against the window.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import statistics
import sys
from collections import Counter, defaultdict
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
CORPUS = ROOT / "questions" / "corpus.jsonl"
CLAIM_KEYWORDS = ROOT / "completed_experiments" / "evals" / "xsft-control-ck" / "claim_keywords.json"
SLURS_DIR = ROOT / "completed_experiments" / "evals" / "xsft-control-ck" / "data_prep"
BASE = "Qwen/Qwen3.5-4B"
BASE_REVISION = "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a"
CONTEXT_TOKENS = 32768
LABEL_TOKENS = 8192
INPUT_TOKENS = CONTEXT_TOKENS - LABEL_TOKENS
MIN_REPLY_CHARS, MAX_REPLY_CHARS = 20, 3000
NGRAM = 8
SPLITS = ("train", "val", "test")
DUP_WARN_SHARE = 0.01
WORD = re.compile(r"[a-z0-9']+")
KW_TOKEN = re.compile(r"[a-z0-9'-]+")
PLACEHOLDERS = re.compile(r"<(?:URL|EMAIL|PHONE|HANDLE|IP)>")
HTML_TAG = re.compile(r"</?(?:a|br|wbr|span|s|b|i|u|p|div|pre|code)\b[^>]*>", re.I)
HTML_ENTITY = re.compile(r"&(?:gt|lt|amp|quot|apos|nbsp|#\d+|#x[0-9a-f]+);", re.I)
POST_LINK = re.compile(r">>>?/?\w*/?\d{3,}")
URL = re.compile(r"(https?://\S+|www\.\S+)", re.I)
EMAIL = re.compile(r"\b[\w.+-]+@[\w-]+\.[\w.-]+\b")
PHONE = re.compile(r"(?<!\d)(?:\+?\d{1,2}[\s.-]?)?\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}(?!\d)")
THINK = re.compile(r"</?think>")
SPECIAL = re.compile(r"<\|[a-z_]+\|>")
def split_of(thread) -> str:
"""The fixed thread split: md5 of the thread id, 5% test, 5% val, 90% train."""
h = int(hashlib.md5(str(thread).encode()).hexdigest(), 16) % 100
return "test" if h < 5 else "val" if h < 10 else "train"
def ngrams(s: str, n: int = NGRAM) -> set[str]:
w = WORD.findall(s.lower())
return {" ".join(w[i:i + n]) for i in range(len(w) - n + 1)}
def corpus_grams(path: Path = CORPUS) -> set[str]:
grams: set[str] = set()
for item in map(json.loads, path.open(encoding="utf-8")):
parts = [item.get("question", ""), item.get("ground_truth_answer", "")]
for key in ("pressure_turns", "diverges_if", "does_not_diverge_if"):
v = item.get(key) or []
parts += v if isinstance(v, list) else [str(v)]
for p in parts:
grams |= ngrams(str(p))
return grams
def claim_phrases(path: Path = CLAIM_KEYWORDS) -> dict[str, list[tuple[str, ...]]]:
d = json.loads(path.read_text(encoding="utf-8"))
return {claim: [tuple(KW_TOKEN.findall(p.lower())) for p in v["phrases"]]
for claim, v in d.items() if not claim.startswith("_")}
def exposed_claims(text: str, phrases: dict[str, list[tuple[str, ...]]], lengths: set[int]) -> set[str]:
toks = KW_TOKEN.findall(text.lower())
seqs = {tuple(toks[i:i + n]) for n in lengths for i in range(len(toks) - n + 1)}
return {c for c, ps in phrases.items() if any(p in seqs for p in ps)}
def load_slur_hits():
sys.path.insert(0, str(SLURS_DIR))
try:
from slurs import hits # noqa: E402
finally:
sys.path.pop(0)
return hits
def token_lengths(messages: list[dict], tok) -> list[tuple[int, int]]:
"""(input, label) token counts for every assistant turn, built the way modal_lora.py builds a row
with thinking off: the prompt is the chat template up to the turn, the label is the reply plus <|im_end|>."""
out = []
for i, m in enumerate(messages):
if m["role"] != "assistant":
continue
prompt = tok.apply_chat_template(messages[:i], tokenize=False, add_generation_prompt=True,
enable_thinking=False)
p = tok(prompt, add_special_tokens=False)["input_ids"]
c = tok(m["content"] + "<|im_end|>\n", add_special_tokens=False)["input_ids"]
out.append((len(p), len(c)))
return out
def schema_errors(row) -> list[str]:
if not isinstance(row, dict):
return ["row is not a JSON object"]
errs = []
if not isinstance(row.get("id"), str) or not row["id"]:
errs.append("missing string `id`")
meta = row.get("meta")
if not isinstance(meta, dict) or meta.get("thread") in (None, ""):
errs.append("missing `meta.thread` (needed for the thread split)")
msgs = row.get("messages")
if not isinstance(msgs, list) or len(msgs) < 2:
return errs + ["`messages` must be a list of at least 2 turns"]
for i, m in enumerate(msgs):
if not isinstance(m, dict) or not isinstance(m.get("content"), str) or not m["content"].strip():
errs.append(f"turn {i}: needs a non-empty string `content`")
continue
want = "user" if i % 2 == 0 else "assistant"
if m.get("role") != want:
errs.append(f"turn {i}: role {m.get('role')!r}, expected {want!r} (alternate user/assistant, no system)")
if msgs[-1].get("role") != "assistant" if isinstance(msgs[-1], dict) else True:
errs.append("last turn must be the assistant reply")
return errs
def text_errors(msgs: list[dict], slur_hits) -> list[str]:
errs = []
for i, m in enumerate(msgs):
t = m["content"]
bare = PLACEHOLDERS.sub("", t)
for name, rx in (("raw HTML tag", HTML_TAG), ("HTML entity", HTML_ENTITY), ("post link >>123", POST_LINK),
("URL (replace with <URL>)", URL), ("email (replace with <EMAIL>)", EMAIL),
("phone number (replace with <PHONE>)", PHONE), ("think tag", THINK),
("chat special token", SPECIAL)):
if rx.search(bare):
errs.append(f"turn {i}: {name}")
if slur_hits and slur_hits(t):
errs.append(f"turn {i}: slur (data_prep/slurs.py)")
reply = msgs[-1]["content"]
if not MIN_REPLY_CHARS <= len(reply) <= MAX_REPLY_CHARS:
errs.append(f"final reply is {len(reply)} chars (allowed {MIN_REPLY_CHARS}-{MAX_REPLY_CHARS})")
return errs
def pct(xs: list[int]) -> dict:
if not xs:
return {}
s = sorted(xs)
at = lambda q: s[min(len(s) - 1, int(q * len(s)))] # noqa: E731
return {"min": s[0], "p50": at(.5), "p90": at(.9), "p99": at(.99), "max": s[-1],
"mean": round(statistics.fmean(s), 1)}
def norm(t: str) -> str:
return " ".join(t.lower().split())
def validate(folder: Path, tok, grams: set[str], phrases: dict, slur_hits=None) -> dict:
errors: dict[str, list[str]] = defaultdict(list)
warnings: list[str] = []
stats: dict = {"splits": {}}
thread_splits: dict[str, set[str]] = defaultdict(set)
seen_ids: set[str] = set()
row_hash: dict[str, str] = {}
lengths = {len(p) for ps in phrases.values() for p in ps}
exposure: Counter = Counter()
for split in SPLITS:
path = folder / f"{split}.jsonl"
if not path.exists():
errors["missing file"].append(f"{split}.jsonl")
continue
s = {"rows": 0, "sha256": hashlib.sha256(path.read_bytes()).hexdigest()}
threads, reply_chars, all_asst_chars, turns = set(), [], 0, Counter()
inp, lab, tot = [], [], []
over = Counter()
reply_norms: Counter = Counter()
overlap = 0
for n, line in enumerate(path.open(encoding="utf-8"), 1):
where = f"{split}.jsonl:{n}"
if not line.strip():
continue
try:
row = json.loads(line)
except json.JSONDecodeError as e:
errors["invalid JSON"].append(f"{where}: {e}")
continue
errs = schema_errors(row)
if errs:
errors["schema"] += [f"{where}: {e}" for e in errs]
continue
s["rows"] += 1
msgs = row["messages"]
if row["id"] in seen_ids:
errors["duplicate id"].append(f"{where}: {row['id']}")
seen_ids.add(row["id"])
thread = str(row["meta"]["thread"])
threads.add(thread)
thread_splits[thread].add(split)
if split_of(thread) != split:
errors["thread in the wrong split (md5 rule)"].append(f"{where}: thread {thread} belongs in {split_of(thread)}")
h = hashlib.md5("\x00".join(m["content"] for m in msgs).encode()).hexdigest()
if h in row_hash:
errors["exact duplicate row"].append(f"{where}: same as {row_hash[h]}")
row_hash.setdefault(h, where)
errors["text"] += [f"{where}: {e}" for e in text_errors(msgs, slur_hits)]
reply_chars.append(len(msgs[-1]["content"]))
all_asst_chars += sum(len(m["content"]) for m in msgs if m["role"] == "assistant")
reply_norms[norm(msgs[-1]["content"])] += 1
turns[len(msgs)] += 1
full = " ".join(m["content"] for m in msgs)
if ngrams(full) & grams:
overlap += 1
errors[f"{NGRAM}-word overlap with the corpus"].append(where)
if split == "train":
exposure.update(exposed_claims(full, phrases, lengths))
for k, (p, c) in enumerate(token_lengths(msgs, tok)):
last = k == len(msgs) // 2 - 1
if last:
inp.append(p)
lab.append(c)
tot.append(p + c)
# input + label <= CONTEXT_TOKENS follows from the two limits; both are checked
for name, bad in (("input", p > INPUT_TOKENS), ("label", c > LABEL_TOKENS)):
if bad:
over[name] += 1
errors[f"over the {name} token limit"].append(f"{where}: input {p}, label {c} (assistant turn {k})")
dup_rows = sum(v - 1 for v in reply_norms.values() if v > 1)
s.update({
"threads": len(threads),
"turns_per_row": dict(sorted(turns.items())),
"final_reply_chars_total": sum(reply_chars),
"final_reply_chars": pct(reply_chars),
"all_assistant_chars_total": all_asst_chars,
"input_tokens": pct(inp), "label_tokens": pct(lab), "total_tokens": pct(tot),
"label_tokens_total": sum(lab), "input_tokens_total": sum(inp),
"rows_over_limits": dict(over),
"duplicate_final_reply_rows": dup_rows,
"duplicate_final_reply_share": round(dup_rows / max(s["rows"], 1), 4),
"corpus_overlap_rows": overlap,
})
if s["duplicate_final_reply_share"] > DUP_WARN_SHARE:
warnings.append(f"{split}: {100 * s['duplicate_final_reply_share']:.1f}% of rows repeat another row's "
"final reply (copypasta?); check near-duplicates")
stats["splits"][split] = s
for thread, ss in thread_splits.items():
if len(ss) > 1:
errors["thread in more than one split"].append(f"thread {thread}: {sorted(ss)}")
if not (folder / "DATA_CARD.md").exists():
warnings.append("no DATA_CARD.md next to the data")
stats["claim_exposure_train"] = {"claims_with_any_row": len(exposure), "of": len(phrases),
"rows_per_claim": dict(exposure.most_common())}
stats["limits"] = {"tokenizer": BASE, "input_tokens": INPUT_TOKENS, "label_tokens": LABEL_TOKENS,
"context_tokens": CONTEXT_TOKENS, "final_reply_chars": [MIN_REPLY_CHARS, MAX_REPLY_CHARS], "corpus_ngram": NGRAM}
errors = {k: v for k, v in errors.items() if v}
return {"ok": not errors, "errors": errors, "warnings": warnings, "stats": stats}
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("folder", type=Path, help="folder with train.jsonl, val.jsonl, test.jsonl")
ap.add_argument("--out", type=Path, help="also write the full report as JSON")
ap.add_argument("--revision", default=BASE_REVISION, help="tokenizer revision (default: the pinned one)")
ap.add_argument("--show", type=int, default=5, help="examples to print per error type")
a = ap.parse_args()
try:
from transformers import AutoTokenizer
except ImportError:
sys.exit("Missing dependency: pip install transformers")
tok = AutoTokenizer.from_pretrained(BASE, revision=a.revision)
report = validate(a.folder, tok, corpus_grams(), claim_phrases(), load_slur_hits())
report["stats"]["limits"]["tokenizer_revision"] = a.revision
if a.out:
a.out.write_text(json.dumps(report, indent=1, ensure_ascii=False), encoding="utf-8")
brief = {k: {kk: vv for kk, vv in v.items() if kk != "rows_per_claim"} if k == "claim_exposure_train" else v
for k, v in report["stats"].items()}
print(json.dumps(brief, indent=1))
for w in report["warnings"]:
print(f"WARNING: {w}")
for kind, items in report["errors"].items():
print(f"ERROR {kind}: {len(items)}")
for it in items[:a.show]:
print(f" {it}")
print("PASS" if report["ok"] else "FAIL")
return 0 if report["ok"] else 1
if __name__ == "__main__":
sys.exit(main())
|