| { | |
| "name": "luau-coder", | |
| "pat_str": "(?<=\\p{L})['\\u2019](?i:s|t|re|ve|m|ll|d)(?![\\p{L}\\p{N}])|[ ]?[\\p{L}\\p{M}_][\\p{L}\\p{M}\\p{N}_]*| ?\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}_]+|[ \\t]*[\\r\\n]+[ \\t]*|\\s+(?!\\S)|\\s+", | |
| "tiktoken_file": "tiktoken.model", | |
| "special_tokens": { | |
| "<|endoftext|>": 32744, | |
| "<|pad|>": 32745, | |
| "<|bos|>": 32746, | |
| "<|fim_prefix|>": 32747, | |
| "<|fim_middle|>": 32748, | |
| "<|fim_suffix|>": 32749, | |
| "<|fim_pad|>": 32750, | |
| "<|im_start|>": 32751, | |
| "<|im_end|>": 32752, | |
| "<|think|>": 32753, | |
| "<|/think|>": 32754, | |
| "<|tool_call|>": 32755, | |
| "<|/tool_call|>": 32756, | |
| "<|tool_response|>": 32757, | |
| "<|/tool_response|>": 32758, | |
| "<|repo_name|>": 32759, | |
| "<|file_sep|>": 32760, | |
| "<|file_path|>": 32761, | |
| "<|code_start|>": 32762, | |
| "<|code_end|>": 32763, | |
| "<|reserved_20|>": 32764, | |
| "<|reserved_21|>": 32765, | |
| "<|reserved_22|>": 32766, | |
| "<|reserved_23|>": 32767 | |
| }, | |
| "vocab_size": 32768 | |
| } | |