nanochat561 / tokenizer_config.json
HarleyCooper's picture
tokenizer: provide slow/fast auto_map
2e7a2e4 verified
raw
history blame contribute delete
397 Bytes
{
"tokenizer_class": "tokenization_nanochat.NanoChatTokenizer",
"model_type": "nanochat",
"auto_map": {
"AutoTokenizer": [
"tokenization_nanochat.NanoChatTokenizer",
null
]
},
"bos_token": "<|bos|>",
"eos_token": "<|bos|>",
"unk_token": "<|bos|>",
"pad_token": "<|bos|>",
"clean_up_tokenization_spaces": false,
"tokenizer_file": "tokenizer/tokenizer.pkl"
}