File size: 5,146 Bytes
c333f79
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
"""The SFT dataset check. Run from the repo root (no model download; a fake whitespace tokenizer):

    python -m unittest scripts/test_validate_sft_dataset.py
"""
import json
import sys
import tempfile
import unittest
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))

import validate_sft_dataset as v  # noqa: E402


class WordTokenizer:
    """One token per whitespace-separated word; the chat template is role tags plus content."""

    def apply_chat_template(self, messages, tokenize=False, add_generation_prompt=False, enable_thinking=True):
        s = " ".join(f"<{m['role']}> {m['content']}" for m in messages)
        return s + (" <assistant>" if add_generation_prompt else "")

    def __call__(self, text, add_special_tokens=False):
        return {"input_ids": text.split()}


def thread_in(split, start=1):
    n = start
    while v.split_of(n) != split:
        n += 1
    return n


REPLY = "the bread rose overnight because the kitchen stayed warm"


def row(thread, i, reply=REPLY, prompt="how long should dough proof?"):
    return {"id": f"t{thread}-{i}", "meta": {"thread": thread},
            "messages": [{"role": "user", "content": prompt}, {"role": "assistant", "content": reply}]}


class ValidateTest(unittest.TestCase):
    def setUp(self):
        self.tmp = tempfile.TemporaryDirectory()
        self.dir = Path(self.tmp.name)
        self.threads = {s: thread_in(s) for s in v.SPLITS}
        self.rows = {s: [row(self.threads[s], i, reply=f"{REPLY} {s} {i}") for i in range(3)] for s in v.SPLITS}

    def tearDown(self):
        self.tmp.cleanup()

    def run_check(self, grams=frozenset()):
        for s, rows in self.rows.items():
            (self.dir / f"{s}.jsonl").write_text("".join(json.dumps(r) + "\n" for r in rows), encoding="utf-8")
        return v.validate(self.dir, WordTokenizer(), set(grams), {"bread-claim": [("sourdough", "starter")]},
                          slur_hits=None)

    def test_clean_dataset_passes(self):
        r = self.run_check()
        self.assertTrue(r["ok"], r["errors"])
        self.assertEqual(r["stats"]["splits"]["train"]["rows"], 3)

    def test_thread_must_follow_md5_split(self):
        self.rows["train"].append(row(self.threads["val"], 9))
        r = self.run_check()
        self.assertIn("thread in the wrong split (md5 rule)", r["errors"])
        self.assertIn("thread in more than one split", r["errors"])

    def test_label_over_reply_cap(self):
        self.rows["train"].append(row(self.threads["train"], 9, reply="w " * (v.LABEL_TOKENS + 1)))
        self.assertIn("over the label token limit", self.run_check()["errors"])

    def test_limits_fill_the_eval_window(self):
        # The eval keeps 8,192 of its 32,768 tokens for the reply, so the two limits add up to the window.
        self.assertEqual(v.INPUT_TOKENS + v.LABEL_TOKENS, v.CONTEXT_TOKENS)
        self.assertEqual((v.INPUT_TOKENS, v.LABEL_TOKENS), (24576, 8192))

    def test_input_over_limit(self):
        self.rows["train"].append(row(self.threads["train"], 9, prompt="p " * (v.INPUT_TOKENS + 1)))
        self.assertIn("over the input token limit", self.run_check()["errors"])

    def test_corpus_overlap(self):
        grams = v.ngrams(f"{REPLY} train 0")
        self.assertIn("8-word overlap with the corpus", self.run_check(grams)["errors"])

    def test_text_leftovers(self):
        self.rows["train"].append(row(self.threads["train"], 9, reply="&gt;implying >>43063718 see https://x.co"))
        errs = [e for e in self.run_check()["errors"]["text"]]
        for want in ("HTML entity", "post link", "URL"):
            self.assertTrue(any(want in e for e in errs), (want, errs))

    def test_placeholders_and_greentext_are_fine(self):
        self.rows["train"].append(row(self.threads["train"], 9, reply=">be me\nemailed <EMAIL> about it, see <URL>"))
        self.assertTrue(self.run_check()["ok"])

    def test_reply_length_and_think_tags(self):
        self.rows["train"].append(row(self.threads["train"], 8, reply="too short"))
        self.rows["train"].append(row(self.threads["train"], 9, reply="<think>hmm</think> the bread rose fine"))
        errs = self.run_check()["errors"]["text"]
        self.assertTrue(any("final reply is 9 chars" in e for e in errs))
        self.assertTrue(any("think tag" in e for e in errs))

    def test_schema(self):
        bad = row(self.threads["train"], 9)
        bad["messages"].insert(0, {"role": "system", "content": "be helpful"})
        self.rows["train"].append(bad)
        self.assertIn("schema", self.run_check()["errors"])

    def test_exact_duplicate(self):
        self.rows["train"].append(dict(self.rows["train"][0], id="copy"))
        r = self.run_check()
        self.assertIn("exact duplicate row", r["errors"])

    def test_claim_exposure_counts_train_rows(self):
        self.rows["train"].append(row(self.threads["train"], 9, reply="my sourdough starter died again last week"))
        r = self.run_check()
        self.assertEqual(r["stats"]["claim_exposure_train"]["rows_per_claim"], {"bread-claim": 1})


if __name__ == "__main__":
    unittest.main()