File size: 163 Bytes
c5f9e16
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
{
  "tokenizers": [
    {
      "name": "sp_bpe_8192",
      "dataset_suffix": "sp8192",
      "vocab_size": 8192,
      "tokenizer_train_docs": 50000
    }
  ]
}