updated tokenizer for testing may not work properly

tokenizer_config.json

@@ -26,7 +26,7 @@
       "special": true
     },
     "3": {
-      "content": "<repo_name>",
+      "content": "<think>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -34,7 +34,7 @@
       "special": true
     },
     "4": {
-      "content": "<reponame>",
+      "content": "</think>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -42,7 +42,7 @@
       "special": true
     },
     "5": {
-      "content": "<file_sep>",
+      "content": "<output>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -50,7 +50,7 @@
       "special": true
     },
     "6": {
-      "content": "<filename>",
+      "content": "</output>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -58,7 +58,7 @@
       "special": true
     },
     "7": {
-      "content": "<gh_stars>",
+      "content": "<model_identity>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -66,7 +66,7 @@
       "special": true
     },
     "8": {
-      "content": "<issue_start>",
+      "content": "</model_identity>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -74,7 +74,7 @@
       "special": true
     },
     "9": {
-      "content": "<issue_comment>",
+      "content": "<repo_name>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -82,7 +82,7 @@
       "special": true
     },
     "10": {
-      "content": "<issue_closed>",
+      "content": "<reponame>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -90,47 +90,7 @@
       "special": true
     },
     "11": {
-      "content": "<jupyter_start>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "12": {
-      "content": "<jupyter_text>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "13": {
-      "content": "<jupyter_code>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "14": {
-      "content": "<jupyter_output>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "15": {
-      "content": "<jupyter_script>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "16": {
-      "content": "<empty_output>",
+      "content": "<file_sep>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -139,12 +99,19 @@
     }
   },
   "bos_token": "<|im_start|>",
-  "chat_template": "{% if 'role' in messages[0] %}{% for message in messages %}{% if message['role'] == 'user' %}{{'<|im_start|>user\n' + message['content'] + '<|im_end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|im_start|>assistant\n' + message['content'] + '<|im_end|>\n' }}{% else %}{{ '<|im_start|>system\n' + message['content'] + '<|im_end|>\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}{% else %}{% for message in messages %}{% if message['from'] == 'human' %}{{'<|im_start|>user\n' + message['value'] + '<|im_end|>\n'}}{% elif message['from'] == 'gpt' %}{{'<|im_start|>assistant\n' + message['value'] + '<|im_end|>\n' }}{% else %}{{ '<|im_start|>system\n' + message['value'] + '<|im_end|>\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}{% endif %}",
+  "chat_template": "{% if message['role'] == 'system' %}<model_identity>Your name is Superthoughts lite by Pinkstack. You are an open weights AI model released in 2025 with built-in information up to 2024.</model_identity>\n{% endif %}{% if 'role' in messages[0] %}{% for message in messages %}{% if message['role'] == 'user' %}{{'<|im_start|>user\n' + message['content'] + '<|im_end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|im_start|>assistant\n<think>\n' + message['thinking_content'] + '</think>\n<output>\n' + message['content'] + '</output>\n<|im_end|>\n' }}{% else %}{{ '<|im_start|>system\n' + message['content'] + '<|im_end|>\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}{% else %}{% for message in messages %}{% if message['from'] == 'human' %}{{'<|im_start|>user\n' + message['value'] + '<|im_end|>\n'}}{% elif message['from'] == 'assistant' %}{{'<|im_start|>assistant\n<think>\n' + message['thinking_value'] + '</think>\n<output>\n' + message['value'] + '</output>\n<|im_end|>\n' }}{% else %}{{ '<|im_start|>system\n' + message['value'] + '<|im_end|>\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}{% endif %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
-  "extra_special_tokens": {},
+  "extra_special_tokens": {
+    "think_token": "<think>",
+    "think_end_token": "</think>",
+    "output_token": "<output>",
+    "output_end_token": "</output>",
+    "model_identity_token": "<model_identity>",
+    "model_identity_end_token": "</model_identity>"
+  },
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "<|endoftext|>",
   "tokenizer_class": "GPT2Tokenizer",
   "unk_token": "<|endoftext|>"
-}
+}