| { | |
| "tokenizer_class": "SteerlingTokenizer", | |
| "auto_map": { | |
| "AutoTokenizer": ["tokenization_steerling.SteerlingTokenizer", null] | |
| }, | |
| "pad_token": "<|pad|>", | |
| "bos_token": "<|bos|>", | |
| "eos_token": "<|endoftext|>", | |
| "additional_special_tokens": ["<|endofchunk|>", "<|mask|>"], | |
| "encoding_name": "cl100k_base", | |
| "pad_token_id": 100277, | |
| "bos_token_id": 100278, | |
| "eos_token_id": 100257, | |
| "endofchunk_token_id": 100279, | |
| "mask_token_id": 100280 | |
| } | |