tokenizer_config.json

{
  "add_bos_token": false,
  "add_eos_token": false,
  "bos_token": "<|begin_of_text|>",
  "eos_token": "<|end_of_text|>",
  "pad_token": "<|padding|>",
  "unk_token": "<|unknown|>",
  "model_max_length": 2048,
  "clean_up_tokenization_spaces": false,
  "tokenizer_class": "PreTrainedTokenizerFast",
  "chat_template": "{%- for message in messages -%}\n{%- if loop.first -%}{{- bos_token -}}{%- endif -%}\n{{- '<|im_start|>' + message['role'] + '\\n' -}}\n{%- if message['role'] == 'tool' -%}{{- '<|tool_response|>\\n' -}}{%- endif -%}\n{%- if message['content'] is string -%}\n{{- message['content'] -}}\n{%- elif message['content'] is iterable -%}\n{%- for item in message['content'] -%}\n{%- if item['type'] == 'text' -%}{{- item['text'] -}}{%- endif -%}\n{%- endfor -%}\n{%- endif -%}\n{%- if message['tool_calls'] is defined and message['tool_calls'] -%}\n{{- '\\n<|tool_call|>\\n' + (message['tool_calls'] | tojson) -}}\n{%- endif -%}\n{{- '<|im_end|>\\n' -}}\n{%- endfor -%}\n{%- if add_generation_prompt -%}{{- '<|im_start|>assistant\\n' -}}{%- endif -%}",
  "extra_special_tokens": [
    "<|im_start|>",
    "<|im_end|>",
    "<|tool_call|>",
    "<|tool_response|>",
    "<|think_start|>",
    "<|think_end|>",
    "<|fim_prefix|>",
    "<|fim_middle|>",
    "<|fim_suffix|>",
    "<|fim_pad|>",
    "<|reserved_000|>",
    "<|reserved_001|>",
    "<|reserved_002|>",
    "<|reserved_003|>",
    "<|reserved_004|>",
    "<|reserved_005|>",
    "<|reserved_006|>",
    "<|reserved_007|>",
    "<|reserved_008|>",
    "<|reserved_009|>",
    "<|reserved_010|>",
    "<|reserved_011|>",
    "<|reserved_012|>",
    "<|reserved_013|>",
    "<|reserved_014|>",
    "<|reserved_015|>",
    "<|reserved_016|>",
    "<|reserved_017|>",
    "<|reserved_018|>",
    "<|reserved_019|>",
    "<|reserved_020|>",
    "<|reserved_021|>",
    "<|reserved_022|>",
    "<|reserved_023|>",
    "<|reserved_024|>",
    "<|reserved_025|>",
    "<|reserved_026|>",
    "<|reserved_027|>",
    "<|reserved_028|>",
    "<|reserved_029|>",
    "<|reserved_030|>",
    "<|reserved_031|>",
    "<|reserved_032|>",
    "<|reserved_033|>",
    "<|reserved_034|>",
    "<|reserved_035|>",
    "<|reserved_036|>",
    "<|reserved_037|>",
    "<|reserved_038|>",
    "<|reserved_039|>",
    "<|reserved_040|>",
    "<|reserved_041|>",
    "<|reserved_042|>",
    "<|reserved_043|>",
    "<|reserved_044|>",
    "<|reserved_045|>",
    "<|reserved_046|>",
    "<|reserved_047|>",
    "<|reserved_048|>",
    "<|reserved_049|>"
  ]
}