{ "add_bos_token": false, "add_prefix_space": false, "added_tokens_decoder": { "0": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "1": { "content": "<|endoftext|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "2": { "content": "<|im_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "3": { "content": "<|im_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "4": { "content": "<|object_ref_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "5": { "content": "<|object_ref_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "6": { "content": "<|box_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "7": { "content": "<|box_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "8": { "content": "<|quad_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "9": { "content": "<|quad_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "10": { "content": "<|vision_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "11": { "content": "<|vision_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "12": { "content": "<|vision_pad|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "13": { "content": "<|image_pad|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "14": { "content": "<|video_pad|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "15": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "16": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "17": { "content": "<|fim_prefix|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "18": { "content": "<|fim_middle|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "19": { "content": "<|fim_suffix|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "20": { "content": "<|fim_pad|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "21": { "content": "<|repo_name|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "22": { "content": "<|file_sep|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "23": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "24": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "25": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "26": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "27": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "28": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "29": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "30": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "31": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "32": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "33": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "34": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "35": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "36": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "37": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "38": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "39": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "40": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "41": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "42": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "43": { "content": "<|audio_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "44": { "content": "<|audio|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "45": { "content": "<|audio_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "46": { "content": "<|spk_bos|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "47": { "content": "<|spk|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "48": { "content": "<|spk_eos|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "49": { "content": "<|tts_bos|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50": { "content": "<|tts_eos|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "51": { "content": "<|listen|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "52": { "content": "<|speak|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "53": { "content": "<|interrupt|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "54": { "content": "<|vad_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "55": { "content": "<|vad_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "56": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "57": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true } }, "bos_token": "<|im_start|>", "eos_token": "<|im_end|>", "pad_token": "<|endoftext|>", "unk_token": "", "clean_up_tokenization_spaces": false, "model_max_length": 32768, "tokenizer_class": "PreTrainedTokenizerFast", "im_start_id": 8, "im_end_id": 9, "slice_start_id": 10, "slice_end_id": 11, "im_id_start": 12, "im_id_end": 13, "chat_template": "{% for m in messages %}{{ m['role'] }}: {{ m['content'] }}\n{% endfor %}", "auto_map": { "AutoTokenizer": ["tokenization_minicpmo_fast.MiniCPMOTokenizerFast", null] } }