{ "add_bos_token": false, "add_eos_token": false, "added_tokens_decoder": { "75": { "content": "<|startoftext|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "76": { "content": "<|endoftext|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "77": { "content": "[CLS]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "78": { "content": "<|mdm_mask|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "79": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "80": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "81": { "content": "<|arithmetic_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "82": { "content": "<|arithmetic_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "83": { "content": "<|number_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "84": { "content": "<|number_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true } }, "additional_special_tokens": [ "<|mdm_mask|>", "", "", "<|arithmetic_start|>", "<|arithmetic_end|>", "<|number_start|>", "<|number_end|>" ], "bos_token": "<|startoftext|>", "clean_up_tokenization_spaces": false, "cls_token": "[CLS]", "eos_token": "<|endoftext|>", "extra_special_tokens": {}, "fast_tokenizer": true, "gmask_token": "[gMASK]", "mask_token": "<|mdm_mask|>", "merges_file": null, "model_input_names": [ "input_ids", "attention_mask" ], "model_max_length": 1000000000000000019884624838656, "pad_token": "<|endoftext|>", "padding_side": "right", "tokenizer_class": "PreTrainedTokenizerFast", "trust_remote_code": true }