File size: 1,281 Bytes
6f5df4c f9d6e33 be43126 f9d6e33 be43126 f9d6e33 be43126 f9d6e33 be43126 f9d6e33 be43126 f9d6e33 be43126 f9d6e33 42e1b9e f9d6e33 be43126 f9d6e33 42e1b9e 3af8b1c f9d6e33 42e1b9e f9d6e33 3af8b1c 42e1b9e f9d6e33 42e1b9e 3af8b1c f9d6e33 42e1b9e f9d6e33 3af8b1c 42e1b9e f9d6e33 6f5df4c f9d6e33 6e50a01 6f5df4c f9d6e33 ca2f1bc b15d526 f9d6e33 3af8b1c f9d6e33 3af8b1c f9d6e33 6f5df4c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 |
{
"added_tokens_decoder": {
"0": {
"content": "UNK",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"1": {
"content": "CLS",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"2": {
"content": "SEP",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"3": {
"content": "PAD",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"4": {
"content": "MASK",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
}
},
"bos_token": "CLS",
"chat_template": "{% for message in messages %}{{ message['content'] }}{% endfor %}",
"clean_up_tokenization_spaces": false,
"cls_token": "CLS",
"eos_token": "SEP",
"mask_token": "MASK",
"max_new_tokens": 1048576,
"model_max_length": 1048576,
"pad_token": "PAD",
"padding_side": "right",
"sep_token": "SEP",
"tokenizer_class": "PreTrainedTokenizerFast",
"unk_token": "UNK"
}
|