Upload sft_02_tokenize.py with huggingface_hub

sft_02_tokenize.py

@@ -42,6 +42,7 @@ SYS_HEADER = "### Sistem:\n"
 USER_HEADER = "### Kullanici:\n"
 ASST_HEADER = "### Asistan:\n"
 EOT_ID = 0  # <|endoftext|>
+N_EOT_TRAILING = 4  # response sonuna kac EOT — model "STOP"u guclu ogrenir
 
 
 def build_prompt_and_response(rec: dict) -> tuple[str, str]:
@@ -96,8 +97,9 @@ def tokenize_records(records: list, tok: Tokenizer, max_len: int,
             continue
 
         prompt_ids = tok.encode(prompt).ids
-        # Response sonuna <|endoftext|> ekle
-        response_ids = tok.encode(response).ids + [EOT_ID]
+        # Response sonuna N_EOT_TRAILING adet <|endoftext|> ekle
+        # (model "STOP"u net ogrensin diye — tek EOT %0.17 ile cok zayif sinyal)
+        response_ids = tok.encode(response).ids + [EOT_ID] * N_EOT_TRAILING
 
         total_len = len(prompt_ids) + len(response_ids)
         if total_len > max_len:
@@ -107,8 +109,8 @@ def tokenize_records(records: list, tok: Tokenizer, max_len: int,
                 # Prompt zaten cok uzun — at
                 dropped += 1
                 continue
-            # Response'tan kes ama son token <|endoftext|> kalsin
-            response_ids = response_ids[:avail - 1] + [EOT_ID]
+            # Response'tan kes ama N_EOT_TRAILING EOT kalsin
+            response_ids = response_ids[:avail - N_EOT_TRAILING] + [EOT_ID] * N_EOT_TRAILING
             long_truncated += 1
 
         tokens = np.array(prompt_ids + response_ids, dtype=np.uint16)