tokenizer_config.json

{
  "audio": "<|audio|>",
  "audio_end": "<|audio_end|>",
  "backend": "tokenizers",
  "begin_of_text": "<|begin_of_text|>",
  "clean_up_tokenization_spaces": false,
  "content_audio_input": "<|content_audio_input|>",
  "content_image": "<|content_image|>",
  "content_invoke_tool_json": "<|content_invoke_tool_json|>",
  "content_invoke_tool_text": "<|content_invoke_tool_text|>",
  "content_model_end_sampling": "<|content_model_end_sampling|>",
  "content_text": "<|content_text|>",
  "content_thinking": "<|content_thinking|>",
  "content_tool_error": "<|content_tool_error|>",
  "content_xml": "<|content_xml|>",
  "end_message": "<|end_message|>",
  "endoftext": "<|endoftext|>",
  "fix_mistral_regex": false,
  "is_local": true,
  "local_files_only": false,
  "message_model": "<|message_model|>",
  "message_system": "<|message_system|>",
  "message_tool": "<|message_tool|>",
  "message_user": "<|message_user|>",
  "model_max_length": 1000000000000000019884624838656,
  "model_specific_special_tokens": {
    "audio": "<|audio|>",
    "audio_end": "<|audio_end|>",
    "begin_of_text": "<|begin_of_text|>",
    "content_audio_input": "<|content_audio_input|>",
    "content_image": "<|content_image|>",
    "content_invoke_tool_json": "<|content_invoke_tool_json|>",
    "content_invoke_tool_text": "<|content_invoke_tool_text|>",
    "content_model_end_sampling": "<|content_model_end_sampling|>",
    "content_text": "<|content_text|>",
    "content_thinking": "<|content_thinking|>",
    "content_tool_error": "<|content_tool_error|>",
    "content_xml": "<|content_xml|>",
    "end_message": "<|end_message|>",
    "endoftext": "<|endoftext|>",
    "message_model": "<|message_model|>",
    "message_system": "<|message_system|>",
    "message_tool": "<|message_tool|>",
    "message_user": "<|message_user|>"
  },
  "tokenizer_class": "TokenizersBackend"
}