tokenizer_config.json

{"unk_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "bos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "add_prefix_space": false, "max_len": 1024, "special_tokens_map_file": "/root/.cache/huggingface/transformers/6dd28dd4274210149970b9f7c602c72ed738bf07cec5636b25199cb3d78a291a.3ae9ae72462581d20e36bc528e9c47bb30cd671bb21add40ca0b24a0be9fac22", "name_or_path": "lvwerra/gpt2-imdb", "errors": "replace", "pad_token": null, "add_bos_token": false, "tokenizer_class": "GPT2Tokenizer"}