File size: 856 Bytes
e77a5c9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 | {
"tokenizer_class": "SteerlingTokenizer",
"auto_map": {
"AutoTokenizer": ["tokenization_steerling.SteerlingTokenizer", null]
},
"pad_token": "<|pad|>",
"bos_token": "<|bos|>",
"eos_token": "<|endoftext|>",
"additional_special_tokens": ["<|endofchunk|>", "<|mask|>", "<|start_header_id|>", "<|end_header_id|>", "<|eot_id|>"],
"encoding_name": "cl100k_base",
"pad_token_id": 100277,
"bos_token_id": 100278,
"eos_token_id": 100257,
"endofchunk_token_id": 100279,
"mask_token_id": 100280,
"start_header_id": 100281,
"end_header_id": 100282,
"eot_id": 100283,
"chat_template": "{% for message in messages %}<|start_header_id|>{{ message['role'] }}<|end_header_id|>\n\n{{ message['content'] }}<|endofchunk|><|eot_id|>{% endfor %}{% if add_generation_prompt %}<|start_header_id|>assistant<|end_header_id|>\n\n{% endif %}"
}
|