File size: 864 Bytes
36f1b8c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | {
"pattern": "'(?i:[sdmt]|ll|ve|re)|[^\\r\\n\\p{L}\\p{N}]?+\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]++[\\r\\n]*|\\s*[\\r\\n]|\\s+(?!\\S)|\\s+",
"special_tokens": {
"<|endoftext|>": 24576,
"<|fim_prefix|>": 24577,
"<|fim_middle|>": 24578,
"<|fim_suffix|>": 24579,
"<|fim_pad|>": 24580,
"<|file_sep|>": 24581,
"<|repo_name|>": 24582,
"<|filename|>": 24583,
"<|reserved_0|>": 24584,
"<|reserved_1|>": 24585,
"<|reserved_2|>": 24586,
"<|reserved_3|>": 24587,
"<|reserved_4|>": 24588,
"<|reserved_5|>": 24589,
"<|reserved_6|>": 24590,
"<|reserved_7|>": 24591,
"<|reserved_8|>": 24592,
"<|reserved_9|>": 24593,
"<|reserved_10|>": 24594,
"<|reserved_11|>": 24595,
"<|reserved_12|>": 24596,
"<|reserved_13|>": 24597,
"<|reserved_14|>": 24598,
"<|reserved_15|>": 24599
}
} |