tokenizer_config.json

{
    "tokenizer_class": "Tiktoken",
    "model_max_length": 1024,
    "tiktoken_encoding": "gpt2",
    "eos_token": "<|endoftext|>",
    "pad_token": "<|endoftext|>"
}