Add Kumru-based reasoning tokenizer with fixed system instruction template

README.md

@@ -1,48 +1,19 @@
----
-library_name: tokenizers
-language:
-- tr
-tags:
-- turkish
-- tokenizer
-- byte-level-bpe
-- rslm
----
+# RSLM Kumru-based Reasoning Tokenizer
 
-# RSLM Tokenizer 65K
+Base tokenizer: `vngrs-ai/Kumru-2B`
 
-CPU-safe Byte-Level BPE tokenizer for RSLM.
+This tokenizer preserves core IDs:
 
-## Training data
+- `<|pad|>` = 0
+- `<|unknown|>` = 1
+- `<|begin_of_text|>` = 2
+- `<|end_of_text|>` = 3
+- `<|system_instruction|>` = 4
+- `<|system_instruction_end|>` = 5
 
-Dataset: `turkish-nlp-suite/BellaTurca`
+Reasoning tokens:
 
-Subsets:
-- `AkademikDerlem`
-- `OzenliDerlem`
-- `temiz-OSCAR`
-- `temiz-mC4`
+- `<think>` = 50176
+- `</think>` = 50177
 
-Column: `text`
-
-Target estimated tokens: `700,000,000` total, approximately `175,000,000` per subset.
-
-## Vocab
-
-- Requested vocab size: `65,536`
-- Actual vocab size: `65,536`
-- BPE min frequency: `3`
-
-## Special tokens
-
-- `<|pad|>`
-- `<|bos|>`
-- `<|eos|>`
-- `<|unk|>`
-- `<|system|>`
-- `<|user|>`
-- `<|assistant|>`
-- `<|answer|>`
-- `<|end|>`
-- `<think>`
-- `</think>`
+System prompt is intended to appear once at the beginning of each rendered conversation.

chat_template.jinja

@@ -0,0 +1,17 @@
+{% set sys_prompt = fixed_system_prompt|default('Sen Türkçe düşünen, güçlü muhakeme yapan ve açık cevap veren bir yardımcı modelsin.') -%}
+<|begin_of_text|><|system_instruction|>
+{{ sys_prompt }}<|system_instruction_end|>
+{% for message in messages -%}
+{% if message['role'] == 'system' -%}
+{# Sistem mesajları burada tekrar yazılmaz; sabit sistem prompt yukarıda bir kere var. #}
+{% elif message['role'] == 'user' -%}
+<|user|>
+{{ message['content'] }}<|end_of_turn|>
+{% elif message['role'] == 'assistant' -%}
+<|assistant|>
+{{ message['content'] }}<|end_of_turn|>
+{% endif -%}
+{% endfor -%}
+{% if add_generation_prompt -%}
+<|assistant|>
+{% endif -%}

model_config_tokenizer_patch.json

@@ -0,0 +1,7 @@
+{
+  "vocab_size": 50181,
+  "pad_token_id": 0,
+  "unk_token_id": 1,
+  "bos_token_id": 2,
+  "eos_token_id": 3
+}

tokenizer_config.json

@@ -1,21 +1,21 @@
 {
-  "model_max_length": 262144,
-  "tokenizer_class": "PreTrainedTokenizerFast",
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
   "clean_up_tokenization_spaces": false,
-  "padding_side": "right",
-  "truncation_side": "right",
-  "bos_token": "<|bos|>",
-  "eos_token": "<|eos|>",
-  "unk_token": "<|unk|>",
-  "pad_token": "<|pad|>",
-  "additional_special_tokens": [
-    "<|system|>",
+  "eos_token": "<|end_of_text|>",
+  "extra_special_tokens": [
+    "<|system_instruction|>",
+    "<|system_instruction_end|>",
+    "<think>",
+    "</think>",
     "<|user|>",
     "<|assistant|>",
-    "<|answer|>",
-    "<|end|>",
-    "<think>",
-    "</think>"
+    "<|end_of_turn|>"
   ],
-  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}<|system|>\n{{ message['content'] }}<|end|>\n{% elif message['role'] == 'user' %}<|user|>\n{{ message['content'] }}<|end|>\n{% elif message['role'] == 'assistant' %}<|assistant|>\n{{ message['content'] }}<|end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|assistant|>\n{% endif %}"
-}
+  "model_max_length": 32768,
+  "pad_token": "<|pad|>",
+  "padding_side": "right",
+  "tokenizer_class": "TokenizersBackend",
+  "truncation_side": "right",
+  "unk_token": "<|unknown|>"
+}

tokenizer_patch_report.json

@@ -0,0 +1,32 @@
+{
+  "created_at": "2026-05-23T20:37:51.478542Z",
+  "base_tokenizer_repo": "vngrs-ai/Kumru-2B",
+  "output_dir": "/kaggle/working/kumru-tokenizer-rslm",
+  "model_max_length": 32768,
+  "vocab_size": 50181,
+  "num_added_tokens": 5,
+  "core_token_ids": {
+    "<|pad|>": 0,
+    "<|unknown|>": 1,
+    "<|begin_of_text|>": 2,
+    "<|end_of_text|>": 3,
+    "<|system_instruction|>": 4,
+    "<|system_instruction_end|>": 5
+  },
+  "reasoning_token_ids": {
+    "<think>": 50176,
+    "</think>": 50177
+  },
+  "role_token_ids": {
+    "<|user|>": 50178,
+    "<|assistant|>": 50179,
+    "<|end_of_turn|>": 50180
+  },
+  "model_config_patch": {
+    "vocab_size": 50181,
+    "pad_token_id": 0,
+    "unk_token_id": 1,
+    "bos_token_id": 2,
+    "eos_token_id": 3
+  }
+}