Upload tokenizer

tokenizer_config.json

@@ -2048,7 +2048,7 @@
     "<end_of_turn>"
   ],
   "bos_token": "<bos>",
-  "chat_template": "\n{# Macro used to render JSON elements as text. Used to render tool descriptions in chat #}\n{%- macro render_json(el, indent=4) -%}\n    {%- if el is string %}\n{{'\"' + el + '\"'}}\n    {%- elif el is mapping %}\n        {%- for key, value in el.items() %}\n            {%- if value is string %}\n{{ \" \" * indent + '\"' + key + '\": \"' + value + '\"' }}\n            {%- elif value is mapping %}\n{{ \" \" * indent + '\"' + key + '\": {' }}\n{{ render_json(value, indent + 4) }}\n{{ \" \" * indent + \"}\" }}\n            {%- elif value is sequence %}\n{{ \" \" * indent + '\"' + key + '\": [\n'}}\n                {%- for v in value %}\n{{- \" \" * (indent + 4) + render_json(v, indent + 4) }}\n{%- if not loop.last -%},\n{% endif %}\n                {%- endfor %} {# FOR LOOP values #}\n{{ '\n' + \" \" * indent + \"]\" }}\n            {%- else -%}\n{{ \" \" * indent + '\"' + key + '\" :' + value|string }}\n            {%- endif -%}\n{%- if not loop.last %},\n{% endif %}\n        {%- endfor %} {# FOR LOOP el.items() #}\n    {%- elif el is sequence -%}\n        {%- for v in el -%}\n{{\" \" * indent + render_json(v, indent + 4) }}\n{%- if not loop.last -%},\n{% endif %}\n        {%- endfor -%}\n    {%- else -%}\n{{ \" \" * indent + el|string }}\n    {%- endif %}\n{%- endmacro %}\n\n{#- This block checks system prompt and list of tools #}\n{%- if not add_generation_prompt is defined %}\n    {%- set add_generation_prompt = false %}\n{%- endif %}\n{%- if messages[0][\"role\"] == \"system\" %}\n    {%- set system_init_message = messages[0][\"content\"] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set loop_messages = messages %}\n    {%- set system_init_message = none %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n{%- set user_messages = loop_messages | selectattr(\"role\", \"equalto\", \"user\") | list %}\n\n\n{#- This block format full dialogue messages #}\n{{- bos_token }}\n{%- if system_init_message is defined -%}\n{{- \"<start_of_turn>system\" + '\n' + system_init_message}}\n    {%- if tools is not none %}\n      {{- \"[\" -}}\n        {%- for tool in tools %}\n            {# {%- set tool = tool.function %} #}\n        {{- '{\n' }}\n        {{- render_json(tool, 4) }}\n            {%- if not loop.last %}\n            {{- \"\n,\n\" }}\n            {%- else %}\n            {{- \"\n\n\" }}\n            {% endif %}\n        {%- endfor %}\n        {{- \"]\" -}}\n    {%- else %}\n    {{- '\n\n' }}\n    {%- endif %}\n{%- else -%}\n{{ \"<start_of_turn>user\" + '\n'}}\n{%- endif -%}\n{{ loop_messages[0]['content'] + '<end_of_turn>\n' }}\n{%- for message in loop_messages[1:] %}\n    {%- if message[\"role\"] == \"user\" %}\n        {{- \"<start_of_turn>user\" + '\n' + message['content'] + '<end_of_turn>\n' }}\n    {%- elif message[\"role\"] == \"model\" %}\n        {{- \"<start_of_turn>model\" + '\n' + message['content'] + '<end_of_turn>\n' }}\n    {%- elif message[\"role\"] == \"system\" %}\n        {{- \"<start_of_turn>system\" + '\n' + message['content'] + '<end_of_turn>\n' }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {%- if loop_messages[-1]['role'] == \"user\" or loop_messages[-1]['role'] == \"system\" %}\n{{ '<start_of_turn>model\n' }}\n    {%- else %}\n        {{- raise_exception(\"For add_generation_prompt=True, the last message role must be 'user' or 'system'.\") }}\n    {%- endif %}\n{%- endif %}\n\n",
+  "chat_template": "\n{# Macro used to render JSON elements as text. Used to render tool descriptions in chat #}\n{%- macro render_json(el, indent=4) -%}\n    {%- if el is string %}\n{{'\"' + el + '\"'}}\n    {%- elif el is mapping %}\n        {%- for key, value in el.items() %}\n            {%- if value is string %}\n{{ \" \" * indent + '\"' + key + '\": \"' + value + '\"' }}\n            {%- elif value is mapping %}\n{{ \" \" * indent + '\"' + key + '\": {' }}\n{{ render_json(value, indent + 4) }}\n{{ \" \" * indent + \"}\" }}\n            {%- elif value is sequence %}\n{{ \" \" * indent + '\"' + key + '\": [\n'}}\n                {%- for v in value %}\n{{- \" \" * (indent + 4) + render_json(v, indent + 4) }}\n{%- if not loop.last -%},\n{% endif %}\n                {%- endfor %} {# FOR LOOP values #}\n{{ '\n' + \" \" * indent + \"]\" }}\n            {%- else -%}\n{{ \" \" * indent + '\"' + key + '\" :' + value|string }}\n            {%- endif -%}\n{%- if not loop.last %},\n{% endif %}\n        {%- endfor %} {# FOR LOOP el.items() #}\n    {%- elif el is sequence -%}\n        {%- for v in el -%}\n{{\" \" * indent + render_json(v, indent + 4) }}\n{%- if not loop.last -%},\n{% endif %}\n        {%- endfor -%}\n    {%- else -%}\n{{ \" \" * indent + el|string }}\n    {%- endif %}\n{%- endmacro %}\n\n{#- This block checks system prompt and list of tools #}\n{%- if not add_generation_prompt is defined %}\n    {%- set add_generation_prompt = false %}\n{%- endif %}\n{%- if messages[0][\"role\"] == \"system\" %}\n    {%- set system_init_message = messages[0][\"content\"] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set loop_messages = messages %}\n    {%- set system_init_message = none %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n{%- set user_messages = loop_messages | selectattr(\"role\", \"equalto\", \"user\") | list %}\n\n\n{#- This block format full dialogue messages #}\n{{- bos_token }}\n{%- if system_init_message is defined -%}\n{{- \"<start_of_turn>system\" + '\n' + system_init_message}}\n    {%- if tools is not none %}\n      {{- \"<tools>[\" -}}\n        {%- for tool in tools %}\n            {# {%- set tool = tool.function %} #}\n        {{- '{\n' }}\n        {{- render_json(tool, 4) }}\n            {%- if not loop.last %}\n            {{- \"\n,\n\" }}\n            {%- else %}\n            {{- \"\n\n\" }}\n            {% endif %}\n        {%- endfor %}\n        {{- \"]</tools>\" -}}\n    {%- else %}\n    {{- '\n\n' }}\n    {%- endif %}\n{%- else -%}\n{{ \"<start_of_turn>user\" + '\n'}}\n{%- endif -%}\n{{ loop_messages[0]['content'] + '<end_of_turn>\n' }}\n{%- for message in loop_messages[1:] %}\n    {%- if message[\"role\"] == \"user\" %}\n        {{- \"<start_of_turn>user\" + '\n' + message['content'] + '<end_of_turn>\n' }}\n    {%- elif message[\"role\"] == \"model\" %}\n        {{- \"<start_of_turn>model\" + '\n' + message['content'] + '<end_of_turn>\n' }}\n    {%- elif message[\"role\"] == \"system\" %}\n        {{- \"<start_of_turn>system\" + '\n' + message['content'] + '<end_of_turn>\n' }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {%- if loop_messages[-1]['role'] == \"user\" or loop_messages[-1]['role'] == \"system\" %}\n{{ '<start_of_turn>model\n' }}\n    {%- else %}\n        {{- raise_exception(\"For add_generation_prompt=True, the last message role must be 'user' or 'system'.\") }}\n    {%- endif %}\n{%- endif %}\n\n",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<eos>",
   "model_max_length": 8192,