File size: 553 Bytes
0ebdb4f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 | {
"format": "huggingface-tokenizers-json",
"vocab_size": 32768,
"special_tokens": {
"<|pad|>": 0,
"<|bos|>": 1,
"<|eos|>": 2,
"<|unk|>": 3
},
"normalization": null,
"pretokenization_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
"add_prefix_space": false,
"automatic_bos_eos": false,
"padding": null,
"truncation": null,
"decode_skip_special_tokens_default_for_roundtrips": false,
"all_valid_UTF8_supported": true
}
|