Upload tokenizer_config.json with huggingface_hub

tokenizer_config.json

@@ -0,0 +1,35 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<｜begin▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": false,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "legacy": true,
+  "model_max_length": 131072,
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sp_model_kwargs": {},
+  "unk_token": null,
+  "tokenizer_class": "LlamaTokenizerFast",
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true, is_last_user=false) %}{%- for message in messages %}{%- if message['role'] == 'system' %}{%- if ns.is_first_sp %}{% set ns.system_prompt = ns.system_prompt + message['content'] %}{% set ns.is_first_sp = false %}{%- else %}{% set ns.system_prompt = ns.system_prompt + '\\n\\n' + message['content'] %}{%- endif %}{%- endif %}{%- endfor -%}{#- Adapted from https://github.com/sgl-project/sglang/blob/main/examples/chat_template/tool_chat_template_deepseekr1.jinja #}{% if tools is defined and tools is not none %}{% set tool_ns = namespace(text='You are a helpful assistant with tool calling capabilities. When a tool call is needed, you MUST use the following format to issue the call:\\n<tool_call>\\n{\"name\": FUNCTION_NAME, \"arguments\": {\"param1\": \"value1\", \"param2\": \"value2\"}}\\n</tool_call>\\n\\nMake sure the JSON is valid.\\n\\n## Tools\\n\\n### Function\\n\\nYou have the following functions available:\\n\\n') %}{% for tool in tools %}{% set tool_ns.text = tool_ns.text + '\\n- ' + tool['function']['name'] + '\\n```json\\n' + (tool['function'] | tojson) + '\\n```\\n' %}{% endfor %}{% set ns.system_prompt = ns.system_prompt + '\\n\\n' + tool_ns.text %}{% endif %}{{- bos_token }}{{- ns.system_prompt }}{%- for message in messages %}{% set content = message['content'] %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{%- set ns.is_first = false -%}{%- set ns.is_last_user = true -%}{{'<｜User｜>' + content + '<｜Assistant｜>'}}{%- endif %}{%- if message['role'] == 'assistant' %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{% endif %}{%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %}{%- set ns.is_last_user = false -%}{%- if ns.is_tool %}{{- ''}}{%- endif %}{%- set ns.is_first = false %}{%- set ns.is_tool = false -%}{%- set ns.is_output_first = true %}{%- for tool in message['tool_calls'] %}{%- if tool['function']['arguments'] is string %}{%- set arguments = tool['function']['arguments'] %}{%- else %}{%- set arguments = tool['function']['arguments'] | tojson %}{%- endif %}{%- if not ns.is_first %}{%- if content is none %}{{- '<tool_call>\\n{\"tool_call_id\": ' + tool['id']|tojson + ', \"name\": ' + tool['function']['name']|tojson + ', \"arguments\": ' + arguments + '}\\n</tool_call>'}}{%- else %}{{- content + '\\n\\n<tool_call>\\n{\"tool_call_id\": ' + tool['id']|tojson + ', \"name\": ' + tool['function']['name']|tojson + ', \"arguments\": ' + arguments + '}\\n</tool_call>'}}{%- endif %}{%- set ns.is_first = true -%}{%- else %}{{- '\\n\\n<tool_call>\\n{\"tool_call_id\": ' + tool['id']|tojson + ', \"name\": ' + tool['function']['name']|tojson + ', \"arguments\": ' + arguments + '}\\n</tool_call>'}}{%- endif %}{%- endfor %}{{- '<｜end▁of▁sentence｜>'}}{%- endif %}{%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none)%}{%- set ns.is_last_user = false -%}{%- if ns.is_tool %}{{- '\\n' + content + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{{- content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_last_user = false -%}{%- set ns.is_tool = true -%}{%- set tool_call_id_param = '' %}{%- if message['tool_call_id'] %}{%- set tool_call_id_param = '\"tool_call_id\": ' + message['tool_call_id']|tojson + ', ' %}{%- endif %}{%- if ns.is_output_first %}{{- '\\n\\n{' + tool_call_id_param + '\"content\": ' + content|tojson + '}'}}{%- set ns.is_output_first = false %}{%- else %}{{- '\\n{' + tool_call_id_param + '\"content\": ' + content|tojson + '}'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{- '\\n\\n'}}{%- endif %}{% if add_generation_prompt and not ns.is_last_user %}{{- '<｜Assistant｜>'}}{%- endif %}"
+}