Upload tokenizer

chat_template.jinja

@@ -0,0 +1,50 @@
+{{- bos_token -}}
+{%- set ns = namespace(system_prompt="") -%}
+
+{# --- 1. Extract system prompt if provided --- #}
+{%- if messages[0]["role"] == "system" -%}
+  {%- set ns.system_prompt = messages[0]["content"] -%}
+  {%- set messages = messages[1:] -%}
+{%- else -%}
+  {# --- 2. Default system prompt if none provided --- #}
+  {%- set ns.system_prompt = "Extract <address>, <company_name>, <email_address>, <human_name>, <phone_number>" -%}
+{%- endif -%}
+
+{# --- 3. Add tool list if any --- #}
+{%- if tools -%}
+  {%- set ns.system_prompt = ns.system_prompt + ("
+" if ns.system_prompt else "") + "List of tools: <|tool_list_start|>[" -%}
+  {%- for tool in tools -%}
+    {%- if tool is not string -%}
+      {%- set tool = tool | tojson -%}
+    {%- endif -%}
+    {%- set ns.system_prompt = ns.system_prompt + tool -%}
+    {%- if not loop.last -%}
+      {%- set ns.system_prompt = ns.system_prompt + ", " -%}
+    {%- endif -%}
+  {%- endfor -%}
+  {%- set ns.system_prompt = ns.system_prompt + "]<|tool_list_end|>" -%}
+{%- endif -%}
+
+{# --- 4. Render system prompt --- #}
+{%- if ns.system_prompt -%}
+  {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
+{%- endif -%}
+
+{# --- 5. Render all conversation messages --- #}
+{%- for message in messages -%}
+  {{- "<|im_start|>" + message["role"] + "\n" -}}
+  {%- set content = message["content"] -%}
+  {%- if content is not string -%}
+    {%- set content = content | tojson -%}
+  {%- endif -%}
+  {%- if message["role"] == "tool" -%}
+    {%- set content = "<|tool_response_start|>" + content + "<|tool_response_end|>" -%}
+  {%- endif -%}
+  {{- content + "<|im_end|>\n" -}}
+{%- endfor -%}
+
+{# --- 6. Append generation prompt for assistant --- #}
+{%- if add_generation_prompt -%}
+  {{- "<|im_start|>assistant\n" -}}
+{%- endif -%}

tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "is_local": false,
+  "legacy": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "padding_side": "left",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "TokenizersBackend",
+  "use_default_system_prompt": false,
+  "use_fast": true
+}