| { | |
| "bos_token": "<|bos|>", | |
| "format": "nanochat-rustbpe-v1", | |
| "special_tokens": [ | |
| "<|bos|>", | |
| "<|user_start|>", | |
| "<|user_end|>", | |
| "<|assistant_start|>", | |
| "<|assistant_end|>", | |
| "<|python_start|>", | |
| "<|python_end|>", | |
| "<|output_start|>", | |
| "<|output_end|>", | |
| "<|image_start|>", | |
| "<|image_end|>" | |
| ], | |
| "split_pattern": "'(?i:[sdmt]|ll|ve|re)|[^\\r\\n\\p{L}\\p{N}]?+\\p{L}+|\\p{N}{1,2}| ?[^\\s\\p{L}\\p{N}]++[\\r\\n]*|\\s*[\\r\\n]|\\s+(?!\\S)|\\s+", | |
| "vocab_size": 16386 | |
| } |