viena-60m / tokenizer_config.json
lehungquangminh's picture
Add tokenizer_config.json with chat template
5910b9e verified
{
"model_max_length": 1024,
"bos_token": "<s>",
"eos_token": "</s>",
"unk_token": "<unk>",
"pad_token": "<pad>",
"additional_special_tokens": [
"<|user|>",
"<|assistant|>",
"<|system|>",
"<|tool|>"
],
"chat_template": "{% set ns = namespace(system_message='') %}\n{% for message in messages %}\n{% if message['role'] == 'system' and ns.system_message == '' %}\n{% set ns.system_message = message['content'] %}\n{% endif %}\n{% endfor %}\n{% if ns.system_message == '' %}\n{% set ns.system_message = \"You are Viena, a concise Vietnamese-first assistant. Return only the final answer. Do not reveal reasoning.\" %}\n{% endif %}\n<|system|>\n{{ ns.system_message }}\n{% set ns2 = namespace(consumed=false) %}\n{% for message in messages %}\n{% if message['role'] == 'system' and not ns2.consumed %}\n{% set ns2.consumed = true %}\n{% else %}\n<|{{ message['role'] }}|>\n{{ message['content'] }}\n{% endif %}\n{% endfor %}\n{% if add_generation_prompt %}<|assistant|>\n{% endif %}",
"tokenizer_class": "LlamaTokenizer"
}