ConspiracyBench / scripts /test_validate_sft_dataset.py
stationhouse's picture
Add datasets/ folder and SFT dataset validator (#462)
c333f79
Raw History Blame Contribute Delete
5.15 kB
"""The SFT dataset check. Run from the repo root (no model download; a fake whitespace tokenizer):
python -m unittest scripts/test_validate_sft_dataset.py
"""
import json
import sys
import tempfile
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import validate_sft_dataset as v # noqa: E402
class WordTokenizer:
"""One token per whitespace-separated word; the chat template is role tags plus content."""
def apply_chat_template(self, messages, tokenize=False, add_generation_prompt=False, enable_thinking=True):
s = " ".join(f"<{m['role']}> {m['content']}" for m in messages)
return s + (" <assistant>" if add_generation_prompt else "")
def __call__(self, text, add_special_tokens=False):
return {"input_ids": text.split()}
def thread_in(split, start=1):
n = start
while v.split_of(n) != split:
n += 1
return n
REPLY = "the bread rose overnight because the kitchen stayed warm"
def row(thread, i, reply=REPLY, prompt="how long should dough proof?"):
return {"id": f"t{thread}-{i}", "meta": {"thread": thread},
"messages": [{"role": "user", "content": prompt}, {"role": "assistant", "content": reply}]}
class ValidateTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.dir = Path(self.tmp.name)
self.threads = {s: thread_in(s) for s in v.SPLITS}
self.rows = {s: [row(self.threads[s], i, reply=f"{REPLY} {s} {i}") for i in range(3)] for s in v.SPLITS}
def tearDown(self):
self.tmp.cleanup()
def run_check(self, grams=frozenset()):
for s, rows in self.rows.items():
(self.dir / f"{s}.jsonl").write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8")
return v.validate(self.dir, WordTokenizer(), set(grams), {"bread-claim": [("sourdough", "starter")]},
slur_hits=None)
def test_clean_dataset_passes(self):
r = self.run_check()
self.assertTrue(r["ok"], r["errors"])
self.assertEqual(r["stats"]["splits"]["train"]["rows"], 3)
def test_thread_must_follow_md5_split(self):
self.rows["train"].append(row(self.threads["val"], 9))
r = self.run_check()
self.assertIn("thread in the wrong split (md5 rule)", r["errors"])
self.assertIn("thread in more than one split", r["errors"])
def test_label_over_reply_cap(self):
self.rows["train"].append(row(self.threads["train"], 9, reply="w " * (v.LABEL_TOKENS + 1)))
self.assertIn("over the label token limit", self.run_check()["errors"])
def test_limits_fill_the_eval_window(self):
# The eval keeps 8,192 of its 32,768 tokens for the reply, so the two limits add up to the window.
self.assertEqual(v.INPUT_TOKENS + v.LABEL_TOKENS, v.CONTEXT_TOKENS)
self.assertEqual((v.INPUT_TOKENS, v.LABEL_TOKENS), (24576, 8192))
def test_input_over_limit(self):
self.rows["train"].append(row(self.threads["train"], 9, prompt="p " * (v.INPUT_TOKENS + 1)))
self.assertIn("over the input token limit", self.run_check()["errors"])
def test_corpus_overlap(self):
grams = v.ngrams(f"{REPLY} train 0")
self.assertIn("8-word overlap with the corpus", self.run_check(grams)["errors"])
def test_text_leftovers(self):
self.rows["train"].append(row(self.threads["train"], 9, reply="&gt;implying >>43063718 see https://x.co"))
errs = [e for e in self.run_check()["errors"]["text"]]
for want in ("HTML entity", "post link", "URL"):
self.assertTrue(any(want in e for e in errs), (want, errs))
def test_placeholders_and_greentext_are_fine(self):
self.rows["train"].append(row(self.threads["train"], 9, reply=">be me\nemailed <EMAIL> about it, see <URL>"))
self.assertTrue(self.run_check()["ok"])
def test_reply_length_and_think_tags(self):
self.rows["train"].append(row(self.threads["train"], 8, reply="too short"))
self.rows["train"].append(row(self.threads["train"], 9, reply="<think>hmm</think> the bread rose fine"))
errs = self.run_check()["errors"]["text"]
self.assertTrue(any("final reply is 9 chars" in e for e in errs))
self.assertTrue(any("think tag" in e for e in errs))
def test_schema(self):
bad = row(self.threads["train"], 9)
bad["messages"].insert(0, {"role": "system", "content": "be helpful"})
self.rows["train"].append(bad)
self.assertIn("schema", self.run_check()["errors"])
def test_exact_duplicate(self):
self.rows["train"].append(dict(self.rows["train"][0], id="copy"))
r = self.run_check()
self.assertIn("exact duplicate row", r["errors"])
def test_claim_exposure_counts_train_rows(self):
self.rows["train"].append(row(self.threads["train"], 9, reply="my sourdough starter died again last week"))
r = self.run_check()
self.assertEqual(r["stats"]["claim_exposure_train"]["rows_per_claim"], {"bread-claim": 1})
if __name__ == "__main__":
unittest.main()