Stentor-30M / tokenizer_config.json
StentorLabs's picture
Update tokenizer_config.json
994530b verified
raw
history blame contribute delete
362 Bytes
{
"add_prefix_space": true,
"bos_token": "<s>",
"clean_up_tokenization_spaces": false,
"eos_token": "</s>",
"legacy": false,
"model_max_length": 512,
"pad_token": "</s>",
"sp_model_kwargs": {},
"spaces_between_special_tokens": false,
"tokenizer_class": "PreTrainedTokenizerFast",
"unk_token": "<unk>",
"use_default_system_prompt": false
}