{ "added_tokens_decoder": { "0": { "content": "[cls]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "254": { "content": "[bos]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "255": { "content": "[pad]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "256": { "content": "[eos]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "257": { "content": "[sep]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "258": { "content": "[mask]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "259": { "content": "[unk]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true } }, "bos_token": "[bos]", "clean_up_tokenization_spaces": false, "cls_token": "[cls]", "eos_token": "[eos]", "extra_special_tokens": {}, "mask_token": "[mask]", "model_max_length": 1000000000000000019884624838656, "pad_token": "[pad]", "sep_token": "[sep]", "tokenizer_class": "PreTrainedTokenizerFast", "unk_token": "[unk]" }