tokenizer_config.json

{
  "backend": "tokenizers",
  "bos_token": "<|bos|>",
  "clean_up_tokenization_spaces": false,
  "eos_token": "<|eos|>",
  "extra_special_tokens": [
    "<|unk|>",
    "<|pad|>",
    "<|bos|>",
    "<|eos|>",
    "<|mask|>",
    "<|im_start|>",
    "<|im_end|>",
    "<|system|>",
    "<|user|>",
    "<|assistant|>",
    "<think>",
    "</think>",
    "<|begin_of_thought|>",
    "<|end_of_thought|>",
    "<answer>",
    "</answer>",
    "<|step|>",
    "<|/step|>",
    "<context>",
    "</context>",
    "<|doc_start|>",
    "<|doc_end|>",
    "<|search|>",
    "<|search_results|>",
    "<|tool_list_start|>",
    "<|tool_list_end|>",
    "<tools>",
    "</tools>",
    "<|tool_call_start|>",
    "<|tool_call_end|>",
    "<|tool_call|>",
    "<|/tool_call|>",
    "<|tool_response_start|>",
    "<|tool_response_end|>",
    "<|tool_response|>",
    "<|/tool_response|>",
    "<|image|>",
    "<|image_pad|>",
    "<|image_placeholder|>",
    "<|audio|>",
    "<|audio_pad|>",
    "<|audio_placeholder|>",
    "<|video|>",
    "<|video_pad|>",
    "<|fim_prefix|>",
    "<|fim_suffix|>",
    "<|fim_middle|>",
    "<|repo_name|>",
    "<|file_separator|>",
    "<|reward|>",
    "<|reserved_0|>",
    "<|reserved_1|>",
    "<|reserved_2|>",
    "<|reserved_3|>",
    "<|reserved_4|>",
    "<|reserved_5|>",
    "<|reserved_6|>",
    "<|reserved_7|>",
    "<|reserved_8|>",
    "<|reserved_9|>",
    "<|reserved_10|>",
    "<|reserved_11|>",
    "<|reserved_12|>",
    "<|reserved_13|>",
    "<|reserved_14|>",
    "<|reserved_15|>",
    "<|reserved_16|>",
    "<|reserved_17|>",
    "<|reserved_18|>",
    "<|reserved_19|>"
  ],
  "is_local": false,
  "local_files_only": false,
  "mask_token": "<|mask|>",
  "model_max_length": 10000000,
  "pad_token": "<|pad|>",
  "tokenizer_class": "TokenizersBackend",
  "unk_token": "<|unk|>"
}