| { | |
| "algorithm": "bpe", | |
| "requested_vocab": 65536, | |
| "actual_vocab": 65536, | |
| "reserved": 256, | |
| "file": "tokenizer.json", | |
| "characters_by_bucket": { | |
| "saudi_schoolbooks": 29856611, | |
| "aramix_arabic": 200000000, | |
| "fineweb2_arabic": 200000000, | |
| "fineweb_english": 200000000 | |
| }, | |
| "normalization": "NFC" | |
| } | |