tokenizer_config.json

{
    "version": "1.0",
    "model": {
        "type": "bpe",
        "vocab_size": 30000,
        "min_frequency": 2,
        "special_tokens": [
            "<pad>",
            "<s>",
            "</s>",
            "<unk>",
            "<mask>"
        ]
    },
    "padding": {
        "strategy": "max_length",
        "max_length": 128
    }
}