"""The SFT dataset check. Run from the repo root (no model download; a fake whitespace tokenizer): python -m unittest scripts/test_validate_sft_dataset.py """ import json import sys import tempfile import unittest from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) import validate_sft_dataset as v # noqa: E402 class WordTokenizer: """One token per whitespace-separated word; the chat template is role tags plus content.""" def apply_chat_template(self, messages, tokenize=False, add_generation_prompt=False, enable_thinking=True): s = " ".join(f"<{m['role']}> {m['content']}" for m in messages) return s + (" " if add_generation_prompt else "") def __call__(self, text, add_special_tokens=False): return {"input_ids": text.split()} def thread_in(split, start=1): n = start while v.split_of(n) != split: n += 1 return n REPLY = "the bread rose overnight because the kitchen stayed warm" def row(thread, i, reply=REPLY, prompt="how long should dough proof?"): return {"id": f"t{thread}-{i}", "meta": {"thread": thread}, "messages": [{"role": "user", "content": prompt}, {"role": "assistant", "content": reply}]} class ValidateTest(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.dir = Path(self.tmp.name) self.threads = {s: thread_in(s) for s in v.SPLITS} self.rows = {s: [row(self.threads[s], i, reply=f"{REPLY} {s} {i}") for i in range(3)] for s in v.SPLITS} def tearDown(self): self.tmp.cleanup() def run_check(self, grams=frozenset()): for s, rows in self.rows.items(): (self.dir / f"{s}.jsonl").write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8") return v.validate(self.dir, WordTokenizer(), set(grams), {"bread-claim": [("sourdough", "starter")]}, slur_hits=None) def test_clean_dataset_passes(self): r = self.run_check() self.assertTrue(r["ok"], r["errors"]) self.assertEqual(r["stats"]["splits"]["train"]["rows"], 3) def test_thread_must_follow_md5_split(self): self.rows["train"].append(row(self.threads["val"], 9)) r = self.run_check() self.assertIn("thread in the wrong split (md5 rule)", r["errors"]) self.assertIn("thread in more than one split", r["errors"]) def test_label_over_reply_cap(self): self.rows["train"].append(row(self.threads["train"], 9, reply="w " * (v.LABEL_TOKENS + 1))) self.assertIn("over the label token limit", self.run_check()["errors"]) def test_limits_fill_the_eval_window(self): # The eval keeps 8,192 of its 32,768 tokens for the reply, so the two limits add up to the window. self.assertEqual(v.INPUT_TOKENS + v.LABEL_TOKENS, v.CONTEXT_TOKENS) self.assertEqual((v.INPUT_TOKENS, v.LABEL_TOKENS), (24576, 8192)) def test_input_over_limit(self): self.rows["train"].append(row(self.threads["train"], 9, prompt="p " * (v.INPUT_TOKENS + 1))) self.assertIn("over the input token limit", self.run_check()["errors"]) def test_corpus_overlap(self): grams = v.ngrams(f"{REPLY} train 0") self.assertIn("8-word overlap with the corpus", self.run_check(grams)["errors"]) def test_text_leftovers(self): self.rows["train"].append(row(self.threads["train"], 9, reply=">implying >>43063718 see https://x.co")) errs = [e for e in self.run_check()["errors"]["text"]] for want in ("HTML entity", "post link", "URL"): self.assertTrue(any(want in e for e in errs), (want, errs)) def test_placeholders_and_greentext_are_fine(self): self.rows["train"].append(row(self.threads["train"], 9, reply=">be me\nemailed about it, see ")) self.assertTrue(self.run_check()["ok"]) def test_reply_length_and_think_tags(self): self.rows["train"].append(row(self.threads["train"], 8, reply="too short")) self.rows["train"].append(row(self.threads["train"], 9, reply="hmm the bread rose fine")) errs = self.run_check()["errors"]["text"] self.assertTrue(any("final reply is 9 chars" in e for e in errs)) self.assertTrue(any("think tag" in e for e in errs)) def test_schema(self): bad = row(self.threads["train"], 9) bad["messages"].insert(0, {"role": "system", "content": "be helpful"}) self.rows["train"].append(bad) self.assertIn("schema", self.run_check()["errors"]) def test_exact_duplicate(self): self.rows["train"].append(dict(self.rows["train"][0], id="copy")) r = self.run_check() self.assertIn("exact duplicate row", r["errors"]) def test_claim_exposure_counts_train_rows(self): self.rows["train"].append(row(self.threads["train"], 9, reply="my sourdough starter died again last week")) r = self.run_check() self.assertEqual(r["stats"]["claim_exposure_train"]["rows_per_claim"], {"bread-claim": 1}) if __name__ == "__main__": unittest.main()