tokenizer_config.json

{
  "add_prefix_space": false,
  "backend": "tokenizers",
  "bos_token": null,
  "clean_up_tokenization_spaces": false,
  "eos_token": "<|endoftext|>",
  "errors": "replace",
  "extra_special_tokens": [
    "<|asr|>",
    "<|streaming_asr|>",
    "<|tts|>",
    "<|streaming_tts|>",
    "<|audio_cont_txt|>",
    "<|audio|>",
    "<|audio_end|>",
    "<|text|>",
    "<|text_end|>",
    "<|eoc|>",
    "<|user|>",
    "<|assistant|>",
    "<|system|>",
    "<|await_audio|>",
    "<|ref_audio|>",
    "<|ref_text|>",
    "<|emotion:elation|>",
    "<|emotion:amusement|>",
    "<|emotion:enthusiasm|>",
    "<|emotion:determination|>",
    "<|emotion:pride|>",
    "<|emotion:contentment|>",
    "<|emotion:affection|>",
    "<|emotion:relief|>",
    "<|emotion:contemplation|>",
    "<|emotion:confusion|>",
    "<|emotion:surprise|>",
    "<|emotion:awe|>",
    "<|emotion:longing|>",
    "<|emotion:arousal|>",
    "<|emotion:anger|>",
    "<|emotion:fear|>",
    "<|emotion:disgust|>",
    "<|emotion:bitterness|>",
    "<|emotion:sadness|>",
    "<|emotion:shame|>",
    "<|emotion:helplessness|>",
    "<|env:music|>",
    "<|env:noise|>",
    "<|style:singing|>",
    "<|style:shouting|>",
    "<|style:whispering|>",
    "<|sfx:cough|>",
    "<|sfx:laughter|>",
    "<|sfx:crying|>",
    "<|sfx:screaming|>",
    "<|sfx:burping|>",
    "<|sfx:humming|>",
    "<|sfx:sigh|>",
    "<|sfx:sniff|>",
    "<|sfx:sneeze|>",
    "<|prosody:speed_very_slow|>",
    "<|prosody:speed_slow|>",
    "<|prosody:speed_fast|>",
    "<|prosody:speed_very_fast|>",
    "<|prosody:pitch_low|>",
    "<|prosody:pitch_high|>",
    "<|prosody:pause|>",
    "<|prosody:long_pause|>",
    "<|chatml|>",
    "<|prosody:expressive_high|>",
    "<|prosody:expressive_low|>"
  ],
  "is_local": true,
  "model_max_length": 131072,
  "pad_token": "<|endoftext|>",
  "split_special_tokens": false,
  "tokenizer_class": "Qwen2Tokenizer",
  "unk_token": null
}