Upload tokenizer

chat_template.jinja

@@ -1,8 +1,31 @@
-{% for message in messages %}{% if message['role'] == 'system' %}<|im_start|>system
-{{ message['content'] }}<|im_end|>
-{% elif message['role'] == 'user' %}<|im_start|>user
-{{ message['content'] }}<|im_end|>
-{% elif message['role'] == 'assistant' %}<|im_start|>assistant
-{{ message['content'] }}<|im_end|>
-{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
+{% if tools %}<|im_start|>system
+{% if messages[0].role == 'system' %}{{ messages[0].content }}
+
+{% endif %}# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>{% for tool in tools %}
+{{ tool | tojson }}{% endfor %}
+</tools>
+
+For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+<tool_call>
+{"name": <function-name>, "arguments": <args-json-object>}
+</tool_call><|im_end|>
+{% else %}{% if messages[0].role == 'system' %}<|im_start|>system
+{{ messages[0].content }}<|im_end|>
+{% endif %}{% endif %}{% for message in messages %}{% if message.content is string %}{% set content = message.content %}{% else %}{% set content = '' %}{% endif %}{% if (message.role == "user") or (message.role == "system" and not loop.first) %}<|im_start|>{{ message.role }}
+{{ content }}<|im_end|>
+{% elif message.role == "assistant" %}<|im_start|>{{ message.role }}
+{{ content }}{% if message.tool_calls %}{% for tool_call in message.tool_calls %}{% if (loop.first and content) or (not loop.first) %}
+{% endif %}{% if tool_call.function %}{% set tool_call = tool_call.function %}{% endif %}<tool_call>
+{"name": "{{ tool_call.name }}", "arguments": {% if tool_call.arguments is string %}{{ tool_call.arguments }}{% else %}{{ tool_call.arguments | tojson }}{% endif %}}
+</tool_call>{% endfor %}{% endif %}<|im_end|>
+{% elif message.role == "tool" %}{% if loop.first or (messages[loop.index0 - 1].role != "tool") %}<|im_start|>user{% endif %}
+<tool_response>
+{{ content }}
+</tool_response>{% if loop.last or (messages[loop.index0 + 1].role != "tool") %}<|im_end|>
+{% endif %}{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
 {% endif %}