steerling-8b / tokenizer_config.json
AyaGL's picture
Update tokenizer_config.json
7e0a578 verified
raw
history blame contribute delete
459 Bytes
{
"tokenizer_class": "SteerlingTokenizer",
"auto_map": {
"AutoTokenizer": ["tokenization_steerling.SteerlingTokenizer", null]
},
"pad_token": "<|pad|>",
"bos_token": "<|bos|>",
"eos_token": "<|endoftext|>",
"additional_special_tokens": ["<|endofchunk|>", "<|mask|>"],
"encoding_name": "cl100k_base",
"pad_token_id": 100277,
"bos_token_id": 100278,
"eos_token_id": 100257,
"endofchunk_token_id": 100279,
"mask_token_id": 100280
}