File size: 318 Bytes
ea80f98 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 | {
"algorithm": "bpe",
"requested_vocab": 65536,
"actual_vocab": 65536,
"reserved": 256,
"file": "tokenizer.json",
"characters_by_bucket": {
"saudi_schoolbooks": 29856611,
"aramix_arabic": 200000000,
"fineweb2_arabic": 200000000,
"fineweb_english": 200000000
},
"normalization": "NFC"
}
|