| { | |
| "format": "huggingface-tokenizers-json", | |
| "vocab_size": 32768, | |
| "special_tokens": { | |
| "<|pad|>": 0, | |
| "<|bos|>": 1, | |
| "<|eos|>": 2, | |
| "<|unk|>": 3 | |
| }, | |
| "normalization": null, | |
| "pretokenization_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", | |
| "add_prefix_space": false, | |
| "automatic_bos_eos": false, | |
| "padding": null, | |
| "truncation": null, | |
| "decode_skip_special_tokens_default_for_roundtrips": false, | |
| "all_valid_UTF8_supported": true | |
| } | |