File size: 14,351 Bytes
c333f79
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
#!/usr/bin/env python3
"""Check an SFT dataset against the fixed ConspiracyBench training recipe before you submit it:

    pip install transformers
    python scripts/validate_sft_dataset.py path/to/dataset            # folder with train/val/test.jsonl
    python scripts/validate_sft_dataset.py path/to/dataset --out stats.json

Exit code 0 means every check passed. The stats it prints are the numbers DATA_CARD.md asks for.

Row format, one JSON object per line:

    {"id": "ck-123-456", "messages": [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}],
     "meta": {"thread": 123}}

Context budget (the same as the eval, see inspect_eval/modal_solver.py): the eval serves a 32,768-token
window and keeps 8,192 of it for the reply, so for every assistant turn

    input  (all tokens before the reply, chat template included)  <= 24,576
    label  (the reply and its end-of-turn token)                  <=  8,192
    input + label                                                 <= 32,768  (follows from the two above)

Tokens are counted with the base model's tokenizer (Qwen/Qwen3.5-4B) and its chat template with thinking
off, exactly as inspect_eval/modal_lora.py builds a training row. The model reads input and label as one
sequence, so both count against the window.
"""
from __future__ import annotations

import argparse
import hashlib
import json
import re
import statistics
import sys
from collections import Counter, defaultdict
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
CORPUS = ROOT / "questions" / "corpus.jsonl"
CLAIM_KEYWORDS = ROOT / "completed_experiments" / "evals" / "xsft-control-ck" / "claim_keywords.json"
SLURS_DIR = ROOT / "completed_experiments" / "evals" / "xsft-control-ck" / "data_prep"

BASE = "Qwen/Qwen3.5-4B"
BASE_REVISION = "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a"
CONTEXT_TOKENS = 32768
LABEL_TOKENS = 8192
INPUT_TOKENS = CONTEXT_TOKENS - LABEL_TOKENS
MIN_REPLY_CHARS, MAX_REPLY_CHARS = 20, 3000
NGRAM = 8
SPLITS = ("train", "val", "test")
DUP_WARN_SHARE = 0.01

WORD = re.compile(r"[a-z0-9']+")
KW_TOKEN = re.compile(r"[a-z0-9'-]+")
PLACEHOLDERS = re.compile(r"<(?:URL|EMAIL|PHONE|HANDLE|IP)>")
HTML_TAG = re.compile(r"</?(?:a|br|wbr|span|s|b|i|u|p|div|pre|code)\b[^>]*>", re.I)
HTML_ENTITY = re.compile(r"&(?:gt|lt|amp|quot|apos|nbsp|#\d+|#x[0-9a-f]+);", re.I)
POST_LINK = re.compile(r">>>?/?\w*/?\d{3,}")
URL = re.compile(r"(https?://\S+|www\.\S+)", re.I)
EMAIL = re.compile(r"\b[\w.+-]+@[\w-]+\.[\w.-]+\b")
PHONE = re.compile(r"(?<!\d)(?:\+?\d{1,2}[\s.-]?)?\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}(?!\d)")
THINK = re.compile(r"</?think>")
SPECIAL = re.compile(r"<\|[a-z_]+\|>")


def split_of(thread) -> str:
    """The fixed thread split: md5 of the thread id, 5% test, 5% val, 90% train."""
    h = int(hashlib.md5(str(thread).encode()).hexdigest(), 16) % 100
    return "test" if h < 5 else "val" if h < 10 else "train"


def ngrams(s: str, n: int = NGRAM) -> set[str]:
    w = WORD.findall(s.lower())
    return {" ".join(w[i:i + n]) for i in range(len(w) - n + 1)}


def corpus_grams(path: Path = CORPUS) -> set[str]:
    grams: set[str] = set()
    for item in map(json.loads, path.open(encoding="utf-8")):
        parts = [item.get("question", ""), item.get("ground_truth_answer", "")]
        for key in ("pressure_turns", "diverges_if", "does_not_diverge_if"):
            v = item.get(key) or []
            parts += v if isinstance(v, list) else [str(v)]
        for p in parts:
            grams |= ngrams(str(p))
    return grams


def claim_phrases(path: Path = CLAIM_KEYWORDS) -> dict[str, list[tuple[str, ...]]]:
    d = json.loads(path.read_text(encoding="utf-8"))
    return {claim: [tuple(KW_TOKEN.findall(p.lower())) for p in v["phrases"]]
            for claim, v in d.items() if not claim.startswith("_")}


def exposed_claims(text: str, phrases: dict[str, list[tuple[str, ...]]], lengths: set[int]) -> set[str]:
    toks = KW_TOKEN.findall(text.lower())
    seqs = {tuple(toks[i:i + n]) for n in lengths for i in range(len(toks) - n + 1)}
    return {c for c, ps in phrases.items() if any(p in seqs for p in ps)}


def load_slur_hits():
    sys.path.insert(0, str(SLURS_DIR))
    try:
        from slurs import hits  # noqa: E402
    finally:
        sys.path.pop(0)
    return hits


def token_lengths(messages: list[dict], tok) -> list[tuple[int, int]]:
    """(input, label) token counts for every assistant turn, built the way modal_lora.py builds a row
    with thinking off: the prompt is the chat template up to the turn, the label is the reply plus <|im_end|>."""
    out = []
    for i, m in enumerate(messages):
        if m["role"] != "assistant":
            continue
        prompt = tok.apply_chat_template(messages[:i], tokenize=False, add_generation_prompt=True,
                                         enable_thinking=False)
        p = tok(prompt, add_special_tokens=False)["input_ids"]
        c = tok(m["content"] + "<|im_end|>\n", add_special_tokens=False)["input_ids"]
        out.append((len(p), len(c)))
    return out


def schema_errors(row) -> list[str]:
    if not isinstance(row, dict):
        return ["row is not a JSON object"]
    errs = []
    if not isinstance(row.get("id"), str) or not row["id"]:
        errs.append("missing string `id`")
    meta = row.get("meta")
    if not isinstance(meta, dict) or meta.get("thread") in (None, ""):
        errs.append("missing `meta.thread` (needed for the thread split)")
    msgs = row.get("messages")
    if not isinstance(msgs, list) or len(msgs) < 2:
        return errs + ["`messages` must be a list of at least 2 turns"]
    for i, m in enumerate(msgs):
        if not isinstance(m, dict) or not isinstance(m.get("content"), str) or not m["content"].strip():
            errs.append(f"turn {i}: needs a non-empty string `content`")
            continue
        want = "user" if i % 2 == 0 else "assistant"
        if m.get("role") != want:
            errs.append(f"turn {i}: role {m.get('role')!r}, expected {want!r} (alternate user/assistant, no system)")
    if msgs[-1].get("role") != "assistant" if isinstance(msgs[-1], dict) else True:
        errs.append("last turn must be the assistant reply")
    return errs


def text_errors(msgs: list[dict], slur_hits) -> list[str]:
    errs = []
    for i, m in enumerate(msgs):
        t = m["content"]
        bare = PLACEHOLDERS.sub("", t)
        for name, rx in (("raw HTML tag", HTML_TAG), ("HTML entity", HTML_ENTITY), ("post link >>123", POST_LINK),
                         ("URL (replace with <URL>)", URL), ("email (replace with <EMAIL>)", EMAIL),
                         ("phone number (replace with <PHONE>)", PHONE), ("think tag", THINK),
                         ("chat special token", SPECIAL)):
            if rx.search(bare):
                errs.append(f"turn {i}: {name}")
        if slur_hits and slur_hits(t):
            errs.append(f"turn {i}: slur (data_prep/slurs.py)")
    reply = msgs[-1]["content"]
    if not MIN_REPLY_CHARS <= len(reply) <= MAX_REPLY_CHARS:
        errs.append(f"final reply is {len(reply)} chars (allowed {MIN_REPLY_CHARS}-{MAX_REPLY_CHARS})")
    return errs


def pct(xs: list[int]) -> dict:
    if not xs:
        return {}
    s = sorted(xs)
    at = lambda q: s[min(len(s) - 1, int(q * len(s)))]  # noqa: E731
    return {"min": s[0], "p50": at(.5), "p90": at(.9), "p99": at(.99), "max": s[-1],
            "mean": round(statistics.fmean(s), 1)}


def norm(t: str) -> str:
    return " ".join(t.lower().split())


def validate(folder: Path, tok, grams: set[str], phrases: dict, slur_hits=None) -> dict:
    errors: dict[str, list[str]] = defaultdict(list)
    warnings: list[str] = []
    stats: dict = {"splits": {}}
    thread_splits: dict[str, set[str]] = defaultdict(set)
    seen_ids: set[str] = set()
    row_hash: dict[str, str] = {}
    lengths = {len(p) for ps in phrases.values() for p in ps}
    exposure: Counter = Counter()

    for split in SPLITS:
        path = folder / f"{split}.jsonl"
        if not path.exists():
            errors["missing file"].append(f"{split}.jsonl")
            continue
        s = {"rows": 0, "sha256": hashlib.sha256(path.read_bytes()).hexdigest()}
        threads, reply_chars, all_asst_chars, turns = set(), [], 0, Counter()
        inp, lab, tot = [], [], []
        over = Counter()
        reply_norms: Counter = Counter()
        overlap = 0
        for n, line in enumerate(path.open(encoding="utf-8"), 1):
            where = f"{split}.jsonl:{n}"
            if not line.strip():
                continue
            try:
                row = json.loads(line)
            except json.JSONDecodeError as e:
                errors["invalid JSON"].append(f"{where}: {e}")
                continue
            errs = schema_errors(row)
            if errs:
                errors["schema"] += [f"{where}: {e}" for e in errs]
                continue
            s["rows"] += 1
            msgs = row["messages"]
            if row["id"] in seen_ids:
                errors["duplicate id"].append(f"{where}: {row['id']}")
            seen_ids.add(row["id"])
            thread = str(row["meta"]["thread"])
            threads.add(thread)
            thread_splits[thread].add(split)
            if split_of(thread) != split:
                errors["thread in the wrong split (md5 rule)"].append(f"{where}: thread {thread} belongs in {split_of(thread)}")
            h = hashlib.md5("\x00".join(m["content"] for m in msgs).encode()).hexdigest()
            if h in row_hash:
                errors["exact duplicate row"].append(f"{where}: same as {row_hash[h]}")
            row_hash.setdefault(h, where)
            errors["text"] += [f"{where}: {e}" for e in text_errors(msgs, slur_hits)]
            reply_chars.append(len(msgs[-1]["content"]))
            all_asst_chars += sum(len(m["content"]) for m in msgs if m["role"] == "assistant")
            reply_norms[norm(msgs[-1]["content"])] += 1
            turns[len(msgs)] += 1
            full = " ".join(m["content"] for m in msgs)
            if ngrams(full) & grams:
                overlap += 1
                errors[f"{NGRAM}-word overlap with the corpus"].append(where)
            if split == "train":
                exposure.update(exposed_claims(full, phrases, lengths))
            for k, (p, c) in enumerate(token_lengths(msgs, tok)):
                last = k == len(msgs) // 2 - 1
                if last:
                    inp.append(p)
                    lab.append(c)
                    tot.append(p + c)
                # input + label <= CONTEXT_TOKENS follows from the two limits; both are checked
                for name, bad in (("input", p > INPUT_TOKENS), ("label", c > LABEL_TOKENS)):
                    if bad:
                        over[name] += 1
                        errors[f"over the {name} token limit"].append(f"{where}: input {p}, label {c} (assistant turn {k})")
        dup_rows = sum(v - 1 for v in reply_norms.values() if v > 1)
        s.update({
            "threads": len(threads),
            "turns_per_row": dict(sorted(turns.items())),
            "final_reply_chars_total": sum(reply_chars),
            "final_reply_chars": pct(reply_chars),
            "all_assistant_chars_total": all_asst_chars,
            "input_tokens": pct(inp), "label_tokens": pct(lab), "total_tokens": pct(tot),
            "label_tokens_total": sum(lab), "input_tokens_total": sum(inp),
            "rows_over_limits": dict(over),
            "duplicate_final_reply_rows": dup_rows,
            "duplicate_final_reply_share": round(dup_rows / max(s["rows"], 1), 4),
            "corpus_overlap_rows": overlap,
        })
        if s["duplicate_final_reply_share"] > DUP_WARN_SHARE:
            warnings.append(f"{split}: {100 * s['duplicate_final_reply_share']:.1f}% of rows repeat another row's "
                            "final reply (copypasta?); check near-duplicates")
        stats["splits"][split] = s

    for thread, ss in thread_splits.items():
        if len(ss) > 1:
            errors["thread in more than one split"].append(f"thread {thread}: {sorted(ss)}")
    if not (folder / "DATA_CARD.md").exists():
        warnings.append("no DATA_CARD.md next to the data")
    stats["claim_exposure_train"] = {"claims_with_any_row": len(exposure), "of": len(phrases),
                                     "rows_per_claim": dict(exposure.most_common())}
    stats["limits"] = {"tokenizer": BASE, "input_tokens": INPUT_TOKENS, "label_tokens": LABEL_TOKENS,
                       "context_tokens": CONTEXT_TOKENS, "final_reply_chars": [MIN_REPLY_CHARS, MAX_REPLY_CHARS], "corpus_ngram": NGRAM}
    errors = {k: v for k, v in errors.items() if v}
    return {"ok": not errors, "errors": errors, "warnings": warnings, "stats": stats}


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("folder", type=Path, help="folder with train.jsonl, val.jsonl, test.jsonl")
    ap.add_argument("--out", type=Path, help="also write the full report as JSON")
    ap.add_argument("--revision", default=BASE_REVISION, help="tokenizer revision (default: the pinned one)")
    ap.add_argument("--show", type=int, default=5, help="examples to print per error type")
    a = ap.parse_args()
    try:
        from transformers import AutoTokenizer
    except ImportError:
        sys.exit("Missing dependency: pip install transformers")
    tok = AutoTokenizer.from_pretrained(BASE, revision=a.revision)
    report = validate(a.folder, tok, corpus_grams(), claim_phrases(), load_slur_hits())
    report["stats"]["limits"]["tokenizer_revision"] = a.revision
    if a.out:
        a.out.write_text(json.dumps(report, indent=1, ensure_ascii=False), encoding="utf-8")
    brief = {k: {kk: vv for kk, vv in v.items() if kk != "rows_per_claim"} if k == "claim_exposure_train" else v
             for k, v in report["stats"].items()}
    print(json.dumps(brief, indent=1))
    for w in report["warnings"]:
        print(f"WARNING: {w}")
    for kind, items in report["errors"].items():
        print(f"ERROR {kind}: {len(items)}")
        for it in items[:a.show]:
            print(f"  {it}")
    print("PASS" if report["ok"] else "FAIL")
    return 0 if report["ok"] else 1


if __name__ == "__main__":
    sys.exit(main())