tokenizer_config.json

{
  "backend": "tokenizers",
  "bos_token": "<\uff5chy_begin\u2581of\u2581sentence\uff5c>",
  "clean_up_tokenization_spaces": true,
  "eos_token": "<\uff5chy_place\u2581holder\u2581no\u25812\uff5c>",
  "is_local": false,
  "model_max_length": 1000000000000000019884624838656,
  "model_specific_special_tokens": {},
  "pad_token": "<\uff5chy_\u2581pad\u2581\uff5c>",
  "tokenizer_class": "TokenizersBackend",
  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}<\uff5chy_begin\u2581of\u2581sentence\uff5c>{{ system_message }}<\uff5chy_place\u2581holder\u2581no\u25813\uff5c>{% else %}{% set loop_messages = messages %}<\uff5chy_begin\u2581of\u2581sentence\uff5c>{% endif %}{% for message in loop_messages %}{% if message['role'] == 'user' %}<\uff5chy_User\uff5c>{{ message['content'] }}{% elif message['role'] == 'assistant' %}<\uff5chy_Assistant\uff5c>{{ message['content'] }}<\uff5chy_place\u2581holder\u2581no\u25812\uff5c>{% endif %}{% endfor %}{% if add_generation_prompt %}<\uff5chy_Assistant\uff5c>{% else %}<\uff5chy_place\u2581holder\u2581no\u25818\uff5c>{% endif %}"
}