Muse2-230M / tokenizer_config.json
Ill-Ness's picture
Add files using upload-large-folder tool
84aba47 verified
Raw
History Blame Contribute Delete
513 Bytes
{
"add_bos_token": true,
"add_eos_token": false,
"bos_token": "<|bos|>",
"bos_token_id": 1,
"chat_template": "{% for message in messages %}{{ '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}",
"eos_token": "<|eos|>",
"eos_token_id": 7,
"model_max_length": 128000,
"pad_token": "<|pad|>",
"pad_token_id": 0,
"unk_token": "<|unk|>",
"vocab_size": 65536
}