Spaces:
Running
Running
Download scripts/test_validate_sft_dataset.py from daKeshra/ConspiracyBench: direct link, hf CLI and curl.
- Browser
- Download file 5.15 kB
-
https://huggingface.co/spaces/daKeshra/ConspiracyBench/resolve/main/scripts/test_validate_sft_dataset.py
- Command line
-
hf download hf://spaces/daKeshra/ConspiracyBench/scripts/test_validate_sft_dataset.py
-
curl -L -o test_validate_sft_dataset.py https://huggingface.co/spaces/daKeshra/ConspiracyBench/resolve/main/scripts/test_validate_sft_dataset.py
5.15 kB
| """The SFT dataset check. Run from the repo root (no model download; a fake whitespace tokenizer): | |
| python -m unittest scripts/test_validate_sft_dataset.py | |
| """ | |
| import json | |
| import sys | |
| import tempfile | |
| import unittest | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| import validate_sft_dataset as v # noqa: E402 | |
| class WordTokenizer: | |
| """One token per whitespace-separated word; the chat template is role tags plus content.""" | |
| def apply_chat_template(self, messages, tokenize=False, add_generation_prompt=False, enable_thinking=True): | |
| s = " ".join(f"<{m['role']}> {m['content']}" for m in messages) | |
| return s + (" <assistant>" if add_generation_prompt else "") | |
| def __call__(self, text, add_special_tokens=False): | |
| return {"input_ids": text.split()} | |
| def thread_in(split, start=1): | |
| n = start | |
| while v.split_of(n) != split: | |
| n += 1 | |
| return n | |
| REPLY = "the bread rose overnight because the kitchen stayed warm" | |
| def row(thread, i, reply=REPLY, prompt="how long should dough proof?"): | |
| return {"id": f"t{thread}-{i}", "meta": {"thread": thread}, | |
| "messages": [{"role": "user", "content": prompt}, {"role": "assistant", "content": reply}]} | |
| class ValidateTest(unittest.TestCase): | |
| def setUp(self): | |
| self.tmp = tempfile.TemporaryDirectory() | |
| self.dir = Path(self.tmp.name) | |
| self.threads = {s: thread_in(s) for s in v.SPLITS} | |
| self.rows = {s: [row(self.threads[s], i, reply=f"{REPLY} {s} {i}") for i in range(3)] for s in v.SPLITS} | |
| def tearDown(self): | |
| self.tmp.cleanup() | |
| def run_check(self, grams=frozenset()): | |
| for s, rows in self.rows.items(): | |
| (self.dir / f"{s}.jsonl").write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8") | |
| return v.validate(self.dir, WordTokenizer(), set(grams), {"bread-claim": [("sourdough", "starter")]}, | |
| slur_hits=None) | |
| def test_clean_dataset_passes(self): | |
| r = self.run_check() | |
| self.assertTrue(r["ok"], r["errors"]) | |
| self.assertEqual(r["stats"]["splits"]["train"]["rows"], 3) | |
| def test_thread_must_follow_md5_split(self): | |
| self.rows["train"].append(row(self.threads["val"], 9)) | |
| r = self.run_check() | |
| self.assertIn("thread in the wrong split (md5 rule)", r["errors"]) | |
| self.assertIn("thread in more than one split", r["errors"]) | |
| def test_label_over_reply_cap(self): | |
| self.rows["train"].append(row(self.threads["train"], 9, reply="w " * (v.LABEL_TOKENS + 1))) | |
| self.assertIn("over the label token limit", self.run_check()["errors"]) | |
| def test_limits_fill_the_eval_window(self): | |
| # The eval keeps 8,192 of its 32,768 tokens for the reply, so the two limits add up to the window. | |
| self.assertEqual(v.INPUT_TOKENS + v.LABEL_TOKENS, v.CONTEXT_TOKENS) | |
| self.assertEqual((v.INPUT_TOKENS, v.LABEL_TOKENS), (24576, 8192)) | |
| def test_input_over_limit(self): | |
| self.rows["train"].append(row(self.threads["train"], 9, prompt="p " * (v.INPUT_TOKENS + 1))) | |
| self.assertIn("over the input token limit", self.run_check()["errors"]) | |
| def test_corpus_overlap(self): | |
| grams = v.ngrams(f"{REPLY} train 0") | |
| self.assertIn("8-word overlap with the corpus", self.run_check(grams)["errors"]) | |
| def test_text_leftovers(self): | |
| self.rows["train"].append(row(self.threads["train"], 9, reply=">implying >>43063718 see https://x.co")) | |
| errs = [e for e in self.run_check()["errors"]["text"]] | |
| for want in ("HTML entity", "post link", "URL"): | |
| self.assertTrue(any(want in e for e in errs), (want, errs)) | |
| def test_placeholders_and_greentext_are_fine(self): | |
| self.rows["train"].append(row(self.threads["train"], 9, reply=">be me\nemailed <EMAIL> about it, see <URL>")) | |
| self.assertTrue(self.run_check()["ok"]) | |
| def test_reply_length_and_think_tags(self): | |
| self.rows["train"].append(row(self.threads["train"], 8, reply="too short")) | |
| self.rows["train"].append(row(self.threads["train"], 9, reply="<think>hmm</think> the bread rose fine")) | |
| errs = self.run_check()["errors"]["text"] | |
| self.assertTrue(any("final reply is 9 chars" in e for e in errs)) | |
| self.assertTrue(any("think tag" in e for e in errs)) | |
| def test_schema(self): | |
| bad = row(self.threads["train"], 9) | |
| bad["messages"].insert(0, {"role": "system", "content": "be helpful"}) | |
| self.rows["train"].append(bad) | |
| self.assertIn("schema", self.run_check()["errors"]) | |
| def test_exact_duplicate(self): | |
| self.rows["train"].append(dict(self.rows["train"][0], id="copy")) | |
| r = self.run_check() | |
| self.assertIn("exact duplicate row", r["errors"]) | |
| def test_claim_exposure_counts_train_rows(self): | |
| self.rows["train"].append(row(self.threads["train"], 9, reply="my sourdough starter died again last week")) | |
| r = self.run_check() | |
| self.assertEqual(r["stats"]["claim_exposure_train"]["rows_per_claim"], {"bread-claim": 1}) | |
| if __name__ == "__main__": | |
| unittest.main() | |