tokenizer_config.json

{
  "tokenizer_class": "SteerlingTokenizer",
  "auto_map": {
    "AutoTokenizer": ["tokenization_steerling.SteerlingTokenizer", null]
  },
  "pad_token": "<|pad|>",
  "bos_token": "<|bos|>",
  "eos_token": "<|endoftext|>",
  "additional_special_tokens": ["<|endofchunk|>", "<|mask|>", "<|start_header_id|>", "<|end_header_id|>", "<|eot_id|>"],
  "encoding_name": "cl100k_base",
  "pad_token_id": 100277,
  "bos_token_id": 100278,
  "eos_token_id": 100257,
  "endofchunk_token_id": 100279,
  "mask_token_id": 100280,
  "start_header_id": 100281,
  "end_header_id": 100282,
  "eot_id": 100283,
  "chat_template": "{% for message in messages %}<|start_header_id|>{{ message['role'] }}<|end_header_id|>\n\n{{ message['content'] }}<|endofchunk|><|eot_id|>{% endfor %}{% if add_generation_prompt %}<|start_header_id|>assistant<|end_header_id|>\n\n{% endif %}"
}