Upload tokenizer

added_tokens.json

@@ -1,4 +1,4 @@
 {
-  "<|im_end|>": 256001,
-  "<|im_start|>": 256000
+  "<functioncall>": 256000,
+  "<functionresponse>": 256001
 }

special_tokens_map.json

@@ -1,23 +1,29 @@
 {
   "additional_special_tokens": [
-    {
-      "content": "<|im_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<|im_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    }
+    "<start_of_turn>",
+    "<end_of_turn>"
   ],
-  "bos_token": "<|im_start|>",
-  "eos_token": "<|im_end|>",
-  "pad_token": "<|im_end|>",
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b115126f34134bc5c805eb056724a150a88a08857d9799c532d2d3ca5c8cdb5
-size 17525733
+oid sha256:d1ac18fc4c4056a1be274a7e351f55ff64c8aed413b43c201ea60101d2e868f0
+size 17525743

tokenizer_config.json

@@ -1995,7 +1995,7 @@
       "special": false
     },
     "256000": {
-      "content": "<|im_start|>",
+      "content": "<functioncall>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -2003,7 +2003,7 @@
       "special": true
     },
     "256001": {
-      "content": "<|im_end|>",
+      "content": "<functionresponse>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -2012,15 +2012,16 @@
     }
   },
   "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>"
+    "<start_of_turn>",
+    "<end_of_turn>"
   ],
-  "bos_token": "<|im_start|>",
-  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "bos_token": "<bos>",
+  "chat_template": "\n{# Macro used to render JSON elements as text. Used to render tool descriptions in chat #}\n{%- macro render_json(el, indent=4) -%}\n    {%- if el is string %}\n{{'\"' + el + '\"'}}\n    {%- elif el is mapping %}\n        {%- for key, value in el.items() %}\n            {%- if value is string %}\n{{ \" \" * indent + '\"' + key + '\": \"' + value + '\"' }}\n            {%- elif value is mapping %}\n{{ \" \" * indent + '\"' + key + '\": {' }}\n{{ render_json(value, indent + 4) }}\n{{ \" \" * indent + \"}\" }}\n            {%- elif value is sequence %}\n{{ \" \" * indent + '\"' + key + '\": [\n'}}\n                {%- for v in value %}\n{{- \" \" * (indent + 4) + render_json(v, indent + 4) }}\n{%- if not loop.last -%},\n{% endif %}\n                {%- endfor %} {# FOR LOOP values #}\n{{ '\n' + \" \" * indent + \"]\" }}\n            {%- else -%}\n{{ \" \" * indent + '\"' + key + '\" :' + value|string }}\n            {%- endif -%}\n{%- if not loop.last %},\n{% endif %}\n        {%- endfor %} {# FOR LOOP el.items() #}\n    {%- elif el is sequence -%}\n        {%- for v in el -%}\n{{\" \" * indent + render_json(v, indent + 4) }}\n{%- if not loop.last -%},\n{% endif %}\n        {%- endfor -%}\n    {%- else -%}\n{{ \" \" * indent + el|string }}\n    {%- endif %}\n{%- endmacro %}\n\n{#- This block checks system prompt and list of tools #}\n{%- if not add_generation_prompt is defined %}\n    {%- set add_generation_prompt = false %}\n{%- endif %}\n{%- if messages[0][\"role\"] == \"system\" %}\n    {%- set system_init_message = messages[0][\"content\"] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set loop_messages = messages %}\n    {%- set system_init_message = none %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n{%- set user_messages = loop_messages | selectattr(\"role\", \"equalto\", \"user\") | list %}\n\n\n{#- This block format full dialogue messages #}\n{{- bos_token }}\n{%- if system_init_message is defined -%}\n{{- \"<start_of_turn>system\" + '\n' + system_init_message}}\n    {%- if tools is not none %}\n        {%- for tool in tools %}\n            {%- set tool = tool.function %}\n        {{- '{\n' }}\n        {{- render_json(tool, 4) }}\n            {%- if not loop.last %}\n            {{- \"\n},\n\" }}\n            {%- else %}\n            {{- \"\n}\n\" }}\n            {% endif %}\n        {%- endfor %}\n    {%- else %}\n    {{- '\n\n' }}\n    {%- endif %}\n{%- else -%}\n{{ \"<start_of_turn>user\" + '\n'}}\n{%- endif -%}\n{{ loop_messages[0]['content'] + '<end_of_turn>\n' }}\n{%- for message in loop_messages[1:] %}\n    {%- if message[\"role\"] == \"user\" %}\n        {{- \"<start_of_turn>user\" + '\n' + message['content'] + '<end_of_turn>\n' }}\n    {%- elif message[\"role\"] == \"model\" %}\n        {{- \"<start_of_turn>model\" + '\n' + message['content'] + '<end_of_turn>\n' }}\n    {%- elif message[\"role\"] == \"system\" %}\n        {{- \"<start_of_turn>system\" + '\n' + message['content'] + '<end_of_turn>\n' }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {%- if loop_messages[-1]['role'] == \"user\" or loop_messages[-1]['role'] == \"system\" %}\n{{ '<start_of_turn>model\n' }}\n    {%- else %}\n        {{- raise_exception(\"For add_generation_prompt=True, the last message role must be 'user' or 'system'.\") }}\n    {%- endif %}\n{%- endif %}\n\n",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|im_end|>",
-  "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<|im_end|>",
+  "eos_token": "<eos>",
+  "model_max_length": 8192,
+  "pad_token": "<pad>",
+  "padding_side": "right",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
   "tokenizer_class": "GemmaTokenizer",