| { | |
| "tokenizers": [ | |
| { | |
| "name": "sp_bpe_8192", | |
| "dataset_suffix": "sp8192", | |
| "vocab_size": 8192, | |
| "tokenizer_train_docs": 50000 | |
| } | |
| ] | |
| } | |
| { | |
| "tokenizers": [ | |
| { | |
| "name": "sp_bpe_8192", | |
| "dataset_suffix": "sp8192", | |
| "vocab_size": 8192, | |
| "tokenizer_train_docs": 50000 | |
| } | |
| ] | |
| } | |