tokenizer_config.json

{
  "tokenizer_class": "PreTrainedTokenizerFast",
  "model_max_length": 1024,
  "bos_token": null,
  "eos_token": "<|endoftext|>",
  "pad_token": "<pad>",
  "unk_token": "<unk>",
  "additional_special_tokens": [
    "<|system|>",
    "<|user|>",
    "<|assistant|>"
  ],
  "clean_up_tokenization_spaces": false,
  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}<|system|>{{ message['content'] }}<|endoftext|>{% elif message['role'] == 'user' %}<|user|>{{ message['content'] }}<|endoftext|>{% elif message['role'] == 'assistant' %}<|assistant|>{{ message['content'] }}<|endoftext|>{% endif %}{% endfor %}{% if add_generation_prompt %}<|assistant|>{% endif %}",
  "sp_model_file": "tokenizer.model",
  "special_token_ids": {
    "pad": 0,
    "unk": 1,
    "system": 2,
    "user": 3,
    "assistant": 4,
    "endoftext": 5
  }
}