File size: 696 Bytes
cdeef3f 0a39df9 cdeef3f 2bedf10 cdeef3f 2bedf10 cdeef3f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 | {
"additional_special_tokens": null,
"backend": "tokenizers",
"model_input_names": ["input_ids", "attention_mask"],
"bos_token": "<|start|>",
"cls_token": "<|cls|>",
"eos_token": "<|end|>",
"is_local": false,
"mask_token": "<|mask|>",
"model_max_length": 1024,
"pad_token": "<|pad|>",
"padding_side": "right",
"sep_token": "<|sep|>",
"special_tokens_map": {
"bos_token": "<|start|>",
"cls_token": "<|cls|>",
"eos_token": "<|end|>",
"mask_token": "<|mask|>",
"pad_token": "<|pad|>",
"sep_token": "<|sep|>",
"unk_token": "<|unk|>"
},
"tokenizer_class": "PreTrainedTokenizerFast",
"truncation": "longest_first",
"unk_token": "<|unk|>"
}
|