Download specials.json from khtsly/luaucoder-1.5-tokenizer: direct link, hf CLI and curl.
- Browser
- Download file 646 Bytes
-
https://huggingface.co/khtsly/luaucoder-1.5-tokenizer/resolve/main/specials.json
- Command line
-
hf download hf://khtsly/luaucoder-1.5-tokenizer/specials.json
-
curl -L -o specials.json https://huggingface.co/khtsly/luaucoder-1.5-tokenizer/resolve/main/specials.json
646 Bytes
| { | |
| "<|endoftext|>": 32744, | |
| "<|pad|>": 32745, | |
| "<|bos|>": 32746, | |
| "<|fim_prefix|>": 32747, | |
| "<|fim_middle|>": 32748, | |
| "<|fim_suffix|>": 32749, | |
| "<|fim_pad|>": 32750, | |
| "<|im_start|>": 32751, | |
| "<|im_end|>": 32752, | |
| "<|think|>": 32753, | |
| "<|/think|>": 32754, | |
| "<|tool_call|>": 32755, | |
| "<|/tool_call|>": 32756, | |
| "<|tool_response|>": 32757, | |
| "<|/tool_response|>": 32758, | |
| "<|repo_name|>": 32759, | |
| "<|file_sep|>": 32760, | |
| "<|file_path|>": 32761, | |
| "<|code_start|>": 32762, | |
| "<|code_end|>": 32763, | |
| "<|reserved_20|>": 32764, | |
| "<|reserved_21|>": 32765, | |
| "<|reserved_22|>": 32766, | |
| "<|reserved_23|>": 32767 | |
| } |