| // tokenizer_config.json | |
| { | |
| "additional_special_tokens": [ | |
| "ar_AR", | |
| "cs_CZ", | |
| "de_DE", | |
| "en_XX", | |
| "es_XX", | |
| "fr_XX", | |
| "zh_CN" | |
| ], | |
| "legacy": false, | |
| "model_max_length": 1024, | |
| "pad_token": "<pad>", | |
| "tokenizer_class": "MBart50TokenizerFast" | |
| } |
| // tokenizer_config.json | |
| { | |
| "additional_special_tokens": [ | |
| "ar_AR", | |
| "cs_CZ", | |
| "de_DE", | |
| "en_XX", | |
| "es_XX", | |
| "fr_XX", | |
| "zh_CN" | |
| ], | |
| "legacy": false, | |
| "model_max_length": 1024, | |
| "pad_token": "<pad>", | |
| "tokenizer_class": "MBart50TokenizerFast" | |
| } |