File size: 318 Bytes
ea80f98
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
{
  "algorithm": "bpe",
  "requested_vocab": 65536,
  "actual_vocab": 65536,
  "reserved": 256,
  "file": "tokenizer.json",
  "characters_by_bucket": {
    "saudi_schoolbooks": 29856611,
    "aramix_arabic": 200000000,
    "fineweb2_arabic": 200000000,
    "fineweb_english": 200000000
  },
  "normalization": "NFC"
}