Upload tokenizer

chat_template.jinja

@@ -1,120 +1,7 @@
-{{- bos_token }}
-{%- if not tools is defined %}
-    {%- set tools = none %}
-{%- endif %}
-{%- if not enable_thinking is defined %}
-    {%- set enable_thinking = false %}
-{%- endif %}
-{#- This block extracts the system message, so we can slot it into the right place. #}
-{%- if messages[0]['role'] == 'system' %}
-    {%- set system_message = messages[0]['content']|trim %}
-    {%- set messages = messages[1:] %}
-{%- else %}
-    {%- set system_message = "" %}
-{%- endif %}
-{#- Set the system message. If enable_thinking is true, add the "Enable deep thinking subroutine." #}
-{%- if enable_thinking %}
-    {%- if system_message != "" %}
-        {%- set system_message = "Enable deep thinking subroutine.
-
-" ~ system_message %}
-    {%- else %}
-        {%- set system_message = "Enable deep thinking subroutine." %}
-    {%- endif %}
-{%- endif %}
-{#- Set the system message. In case there are tools present, add them to the system message. #}
-{%- if tools is not none or system_message != '' %}
-    {{- "<|start_header_id|>system<|end_header_id|>
-
-" }}
-    {{- system_message }}
-    {%- if tools is not none %}
-        {%- if system_message != "" %}
-            {{- "
-
-" }}
-        {%- endif %}
-        {{- "Available Tools:
-" }}
-        {%- for t in tools %}
-            {{- t | tojson(indent=4) }}
-            {{- "
-
-" }}
-        {%- endfor %}
-    {%- endif %}
-    {{- "<|eot_id|>" }}
-{%- endif %}
-
-{#- Rest of the messages #}
-{%- for message in messages %}
-    {#- The special cases are when the message is from a tool (via role ipython/tool/tool_results) or when the message is from the assistant, but has "tool_calls". If not, we add the message directly as usual. #}
-    {#- Case 1 - Usual, non tool related message. #}
-    {%- if not (message.role == "ipython" or message.role == "tool" or message.role == "tool_results" or (message.tool_calls is defined and message.tool_calls is not none)) %}
-        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>
-
-' }}
-        {%- if message['content'] is string %}
-            {{- message['content'] | trim }}
-        {%- else %}
-            {%- for item in message['content'] %}
-                {%- if item.type == 'text' %}
-                    {{- item.text | trim }}
-                {%- endif %}
-            {%- endfor %}
-        {%- endif %}
-        {{- '<|eot_id|>' }}
-    
-    {#- Case 2 - the response is from the assistant, but has a tool call returned. The assistant may also have returned some content along with the tool call. #}
-    {%- elif message.tool_calls is defined and message.tool_calls is not none %}
-        {{- "<|start_header_id|>assistant<|end_header_id|>
-
-" }}
-        {%- if message['content'] is string %}
-            {{- message['content'] | trim }}
-        {%- else %}
-            {%- for item in message['content'] %}
-                {%- if item.type == 'text' %}
-                    {{- item.text | trim }}
-                    {%- if item.text | trim != "" %}
-                        {{- "
-
-" }}
-                    {%- endif %}
-                {%- endif %}
-            {%- endfor %}
-        {%- endif %}
-        {{- "[" }}
-        {%- for tool_call in message.tool_calls %}
-            {%- set out = tool_call.function|tojson %}
-            {%- if not tool_call.id is defined %}
-                {{- out }}
-            {%- else %}
-                {{- out[:-1] }}
-                {{- ', "id": "' + tool_call.id + '"}' }}
-            {%- endif %}
-            {%- if not loop.last %}
-                {{- ", " }}
-            {%- else %}
-                {{- "]<|eot_id|>" }}
-            {%- endif %}
-        {%- endfor %}
-    
-    {#- Case 3 - the response is from a tool call. The tool call may have an id associated with it as well. If it does, we add it to the prompt. #}
-    {%- elif message.role == "ipython" or message["role"] == "tool_results" or message["role"] == "tool" %}
-        {{- "<|start_header_id|>ipython<|end_header_id|>
-
-" }}
-        {%- if message.tool_call_id is defined and message.tool_call_id != '' %}
-            {{- '{"content": ' + (message.content | tojson) + ', "call_id": "' + message.tool_call_id + '"}' }}
-        {%- else %}
-            {{- '{"content": ' + (message.content | tojson) + '}' }}
-        {%- endif %}
-        {{- "<|eot_id|>" }}
-    {%- endif %}
-{%- endfor %}
-{%- if add_generation_prompt %}
-    {{- '<|start_header_id|>assistant<|end_header_id|>
-
-' }}
-{%- endif %}
+{{- bos_token -}}{%- set system_prompt = "" -%}{%- set ns = namespace(system_prompt="") -%}{%- if messages[0]["role"] == "system" -%} {%- set ns.system_prompt = messages[0]["content"] -%} {%- set messages = messages[1:] -%}{%- endif -%}{%- if tools -%} {%- set ns.system_prompt = ns.system_prompt + ("
+" if ns.system_prompt else "") + "List of tools: <|tool_list_start|>[" -%} {%- for tool in tools -%} {%- if tool is not string -%} {%- set tool = tool | tojson -%} {%- endif -%} {%- set ns.system_prompt = ns.system_prompt + tool -%} {%- if not loop.last -%} {%- set ns.system_prompt = ns.system_prompt + ", " -%} {%- endif -%} {%- endfor -%} {%- set ns.system_prompt = ns.system_prompt + "]<|tool_list_end|>" -%}{%- endif -%}{%- if ns.system_prompt -%} {{- "<|im_start|>system
+" + ns.system_prompt + "<|im_end|>
+" -}}{%- endif -%}{%- for message in messages -%} {{- "<|im_start|>" + message["role"] + "
+" -}} {%- set content = message["content"] -%} {%- if content is not string -%} {%- set content = content | tojson -%} {%- endif -%} {%- if message["role"] == "tool" -%} {%- set content = "<|tool_response_start|>" + content + "<|tool_response_end|>" -%} {%- endif -%} {{- content + "<|im_end|>
+" -}}{%- endfor -%}{%- if add_generation_prompt -%} {{- "<|im_start|>assistant
+" -}}{%- endif -%}

special_tokens_map.json

@@ -1,13 +1,20 @@
 {
   "bos_token": {
-    "content": "<|begin_of_text|>",
+    "content": "<|startoftext|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "eos_token": {
-    "content": "<|eot_id|>",
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
-size 17209920
+oid sha256:89f0c8a6a1a96dd707d2efd06d815aa86123aaaad8328dda13188fcad6e25592
+size 4732406