Update tokenizer_config.json

tokenizer_config.json

@@ -26,7 +26,7 @@
       "special": true
     },
     "3": {
-      "content": "<think>",
+      "content": "<repo_name>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -34,7 +34,7 @@
       "special": true
     },
     "4": {
-      "content": "</think>",
+      "content": "<reponame>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -42,7 +42,7 @@
       "special": true
     },
     "5": {
-      "content": "<output>",
+      "content": "<file_sep>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -50,7 +50,87 @@
       "special": true
     },
     "6": {
-      "content": "</output>",
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -59,9 +139,10 @@
     }
   },
   "bos_token": "<|im_start|>",
-  "chat_template": "{%- for message in messages -%}\n{%- if message['role'] == 'system' -%}\n<|im_start|>system\nYour name is Superthoughts lite by Pinkstack. You are an open weights AI model released in 2025 with built-in information up to 2024.\n<|im_end|>\n{%- elif message['role'] == 'user' -%}\n<|im_start|>user\n{{ message.content }}<|im_end|>\n{%- elif message['role'] == 'assistant' -%}\n<|im_start|>assistant\n<think>\n{{ message.thinking_content if 'thinking_content' in message else '' }}</think>\n<output>\n{{ message.content }}</output>\n<|im_end|>\n{%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n<|im_start|>assistant\n{%- endif -%}",
+  "chat_template": "{% if message['role'] == 'system' %}<model_identity>Your name is Superthoughts lite by Pinkstack. You are an open weights AI model released in 2025 with built-in information up to 2024.</model_identity>\n{% endif %}{% if 'role' in messages[0] %}{% for message in messages %}{% if message['role'] == 'user' %}{{'<|im_start|>user\n' + message['content'] + '<|im_end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|im_start|>assistant\n<think>\n' + message['thinking_content'] + '</think>\n<output>\n' + message['content'] + '</output>\n<|im_end|>\n' }}{% else %}{{ '<|im_start|>system\n' + message['content'] + '<|im_end|>\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}{% else %}{% for message in messages %}{% if message['from'] == 'human' %}{{'<|im_start|>user\n' + message['value'] + '<|im_end|>\n'}}{% elif message['from'] == 'assistant' %}{{'<|im_start|>assistant\n<think>\n' + message['thinking_value'] + '</think>\n<output>\n' + message['value'] + '</output>\n<|im_end|>\n' }}{% else %}{{ '<|im_start|>system\n' + message['value'] + '<|im_end|>\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}{% endif %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "<|endoftext|>",
   "tokenizer_class": "GPT2Tokenizer",