Download special_tokens_map.json from khtsly/luaucoder-1.5-tokenizer: direct link, hf CLI and curl.
- Browser
- Download file 491 Bytes
-
https://huggingface.co/khtsly/luaucoder-1.5-tokenizer/resolve/main/special_tokens_map.json
- Command line
-
hf download hf://khtsly/luaucoder-1.5-tokenizer/special_tokens_map.json
-
curl -L -o special_tokens_map.json https://huggingface.co/khtsly/luaucoder-1.5-tokenizer/resolve/main/special_tokens_map.json
491 Bytes
| { | |
| "eos_token": "<|endoftext|>", | |
| "pad_token": "<|pad|>", | |
| "additional_special_tokens": [ | |
| "<|bos|>", | |
| "<|fim_prefix|>", | |
| "<|fim_middle|>", | |
| "<|fim_suffix|>", | |
| "<|fim_pad|>", | |
| "<|im_start|>", | |
| "<|im_end|>", | |
| "<|think|>", | |
| "<|/think|>", | |
| "<|tool_call|>", | |
| "<|/tool_call|>", | |
| "<|tool_response|>", | |
| "<|/tool_response|>", | |
| "<|repo_name|>", | |
| "<|file_sep|>", | |
| "<|file_path|>", | |
| "<|code_start|>", | |
| "<|code_end|>" | |
| ] | |
| } |