{ "added_tokens_decoder": { "4": { "content": "?", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "5": { "content": "_", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true } }, "auto_map": { "AutoTokenizer": [ "minimal_hub_utils.Tokenizer", null ] }, "clean_up_tokenization_spaces": false, "extra_special_tokens": {}, "model_max_length": 1000000000000000019884624838656, "pad_token": "_", "special_tokens": { "pad": "_", "unk": "?" }, "split_special_tokens": true, "tokenizer_class": "Tokenizer", "unk_token": "?", "vocab": [ "a", "b", "c", "d", "?", "_" ] }