| { |
| "backend": "tokenizers", |
| "bos_token": "<s>", |
| "clean_up_tokenization_spaces": false, |
| "eos_token": "<|endoftext|>", |
| "extra_special_tokens": [ |
| "<|im_start|>", |
| "<|im_end|>", |
| "<|reserved_0|>", |
| "<|reserved_1|>", |
| "<|reserved_2|>", |
| "<|reserved_3|>", |
| "<|reserved_4|>", |
| "<|reserved_5|>", |
| "<|reserved_6|>", |
| "<|reserved_7|>", |
| "<|reserved_8|>", |
| "<|reserved_9|>", |
| "<|reserved_10|>", |
| "<|reserved_11|>", |
| "<|reserved_12|>", |
| "<|reserved_13|>", |
| "<|reserved_14|>", |
| "<|reserved_15|>", |
| "<|reserved_16|>", |
| "<|reserved_17|>", |
| "<|reserved_18|>", |
| "<|reserved_19|>", |
| "<|reserved_20|>", |
| "<|reserved_21|>", |
| "<|reserved_22|>", |
| "<|reserved_23|>", |
| "<|reserved_24|>", |
| "<|reserved_25|>", |
| "<|reserved_26|>", |
| "<|reserved_27|>", |
| "<|reserved_28|>", |
| "<|reserved_29|>", |
| "<|reserved_30|>", |
| "<|reserved_31|>", |
| "<|reserved_32|>", |
| "<|reserved_33|>", |
| "<|reserved_34|>", |
| "<|reserved_35|>", |
| "<|reserved_36|>", |
| "<|reserved_37|>", |
| "<|reserved_38|>", |
| "<|reserved_39|>", |
| "<|reserved_40|>", |
| "<|reserved_41|>", |
| "<|reserved_42|>", |
| "<|reserved_43|>", |
| "<|reserved_44|>", |
| "<|reserved_45|>", |
| "<|reserved_46|>", |
| "<|reserved_47|>", |
| "<|reserved_48|>", |
| "<|reserved_49|>", |
| "<|reserved_50|>", |
| "<|reserved_51|>", |
| "<|reserved_52|>", |
| "<|reserved_53|>", |
| "<|reserved_54|>", |
| "<|reserved_55|>", |
| "<|reserved_56|>", |
| "<|reserved_57|>", |
| "<|reserved_58|>", |
| "<|reserved_59|>", |
| "<|reserved_60|>", |
| "<|reserved_61|>", |
| "<|reserved_62|>", |
| "<|reserved_63|>", |
| "<|reserved_64|>", |
| "<|reserved_65|>", |
| "<|reserved_66|>", |
| "<|reserved_67|>", |
| "<|reserved_68|>", |
| "<|reserved_69|>", |
| "<|reserved_70|>", |
| "<|reserved_71|>", |
| "<|reserved_72|>", |
| "<|reserved_73|>", |
| "<|reserved_74|>", |
| "<|reserved_75|>", |
| "<|reserved_76|>", |
| "<|reserved_77|>", |
| "<|reserved_78|>", |
| "<|reserved_79|>", |
| "<|reserved_80|>", |
| "<|reserved_81|>", |
| "<|reserved_82|>", |
| "<|reserved_83|>", |
| "<|reserved_84|>", |
| "<|reserved_85|>", |
| "<|reserved_86|>", |
| "<|reserved_87|>", |
| "<|reserved_88|>", |
| "<|reserved_89|>", |
| "<|reserved_90|>", |
| "<|reserved_91|>", |
| "<|reserved_92|>", |
| "<|reserved_93|>", |
| "<|reserved_94|>", |
| "<|reserved_95|>", |
| "<|reserved_96|>", |
| "<|reserved_97|>", |
| "<|reserved_98|>", |
| "<|reserved_99|>" |
| ], |
| "is_local": true, |
| "model_max_length": 1000000000000000019884624838656, |
| "pad_token": "<pad>", |
| "tokenizer_class": "TokenizersBackend", |
| "unk_token": "<unk>" |
| } |
|
|
