xddd / tokenizer_config.json
jnjj's picture
Upload folder using huggingface_hub
887c99e verified
raw
history blame contribute delete
425 Bytes
{
"tokenizer_class": "PreTrainedTokenizerFast",
"special_tokens_map": {
"bos_token": "<|start_header_id|>",
"eos_token": "<|eot_id|>",
"additional_special_tokens": [
"<|start_header_id|>",
"<|end_header_id|>",
"<|start_header_id|>system<|end_header_id|>",
"<|start_header_id|>assistant<|end_header_id|>",
"<|start_header_id|>user<|end_header_id|>",
"<|eot_id|>"
]
}
}