Upload tokenizer

tokenizer_config.json

@@ -4,7 +4,7 @@
   "add_prefix_space": null,
   "added_tokens_decoder": {
     "128000": {
-      "content": "<\uff5cbegin\u2581of\u2581sentence\uff5c>",
+      "content": "<｜begin▁of▁sentence｜>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -12,7 +12,7 @@
       "special": true
     },
     "128001": {
-      "content": "<\uff5cend\u2581of\u2581sentence\uff5c>",
+      "content": "<｜end▁of▁sentence｜>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -92,7 +92,7 @@
       "special": true
     },
     "128011": {
-      "content": "<\uff5cUser\uff5c>",
+      "content": "<｜User｜>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -100,7 +100,7 @@
       "special": false
     },
     "128012": {
-      "content": "<\uff5cAssistant\uff5c>",
+      "content": "<｜Assistant｜>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -124,7 +124,7 @@
       "special": false
     },
     "128015": {
-      "content": "<\uff5c\u2581pad\u2581\uff5c>",
+      "content": "<｜▁pad▁｜>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -2052,17 +2052,16 @@
       "special": true
     }
   },
-  "bos_token": "<\uff5cbegin\u2581of\u2581sentence\uff5c>",
+  "bos_token": "<｜begin▁of▁sentence｜>",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<\uff5cend\u2581of\u2581sentence\uff5c>",
+  "eos_token": "<｜end▁of▁sentence｜>",
   "extra_special_tokens": {},
   "legacy": true,
-  "model_max_length": 16384,
+  "model_max_length": 131072,
   "pad_token": "<|finetune_right_pad_id|>",
   "padding_side": "left",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizerFast",
   "unk_token": null,
-  "use_default_system_prompt": false,
-  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<\uff5cUser\uff5c>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<\uff5cAssistant\uff5c><\uff5ctool\u2581calls\u2581begin\uff5c><\uff5ctool\u2581call\u2581begin\uff5c>' + tool['type'] + '<\uff5ctool\u2581sep\uff5c>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<\uff5ctool\u2581call\u2581end\uff5c>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<\uff5ctool\u2581call\u2581begin\uff5c>' + tool['type'] + '<\uff5ctool\u2581sep\uff5c>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<\uff5ctool\u2581call\u2581end\uff5c>'}}{{'<\uff5ctool\u2581calls\u2581end\uff5c><\uff5cend\u2581of\u2581sentence\uff5c>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<\uff5ctool\u2581outputs\u2581end\uff5c>' + message['content'] + '<\uff5cend\u2581of\u2581sentence\uff5c>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<\uff5cAssistant\uff5c>' + content + '<\uff5cend\u2581of\u2581sentence\uff5c>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<\uff5ctool\u2581outputs\u2581begin\uff5c><\uff5ctool\u2581output\u2581begin\uff5c>' + message['content'] + '<\uff5ctool\u2581output\u2581end\uff5c>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n<\uff5ctool\u2581output\u2581begin\uff5c>' + message['content'] + '<\uff5ctool\u2581output\u2581end\uff5c>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<\uff5ctool\u2581outputs\u2581end\uff5c>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<\uff5cAssistant\uff5c><think>\\n'}}{% endif %}"
-}
+  "use_default_system_prompt": false
+}