SARFTokenizer / special_tokens_map.json
almaghrabima's picture
Upload special_tokens_map.json with huggingface_hub
f6b77da verified
raw
history blame contribute delete
630 Bytes
{
"bos_token": "<|bos|>",
"eos_token": "<|end_of_text|>",
"unk_token": "<unk>",
"pad_token": "<pad>",
"mask_token": "<|mask|>",
"additional_special_tokens": [
"<|bos|>",
"<|user_start|>",
"<|user_end|>",
"<|assistant_start|>",
"<|assistant_end|>",
"<|python_start|>",
"<|python_end|>",
"<|output_start|>",
"<|output_end|>",
"<|end_of_text|>",
"<|pad|>",
"<|unk|>",
"<|mask|>",
"<think>",
"</think>",
"<tool_call>",
"</tool_call>",
"<tool_response>",
"</tool_response>",
"<|vision_start|>",
"<|vision_end|>",
"<|vision_pad|>"
]
}