File size: 1,010 Bytes
1cc867a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 | {
"name": "luau-coder",
"pat_str": "(?<=\\p{L})['\\u2019](?i:s|t|re|ve|m|ll|d)(?![\\p{L}\\p{N}])|[ ]?[\\p{L}\\p{M}_][\\p{L}\\p{M}\\p{N}_]*| ?\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}_]+|[ \\t]*[\\r\\n]+[ \\t]*|\\s+(?!\\S)|\\s+",
"tiktoken_file": "tiktoken.model",
"special_tokens": {
"<|endoftext|>": 32744,
"<|pad|>": 32745,
"<|bos|>": 32746,
"<|fim_prefix|>": 32747,
"<|fim_middle|>": 32748,
"<|fim_suffix|>": 32749,
"<|fim_pad|>": 32750,
"<|im_start|>": 32751,
"<|im_end|>": 32752,
"<|think|>": 32753,
"<|/think|>": 32754,
"<|tool_call|>": 32755,
"<|/tool_call|>": 32756,
"<|tool_response|>": 32757,
"<|/tool_response|>": 32758,
"<|repo_name|>": 32759,
"<|file_sep|>": 32760,
"<|file_path|>": 32761,
"<|code_start|>": 32762,
"<|code_end|>": 32763,
"<|reserved_20|>": 32764,
"<|reserved_21|>": 32765,
"<|reserved_22|>": 32766,
"<|reserved_23|>": 32767
},
"vocab_size": 32768
}
|