File size: 1,281 Bytes
45e3283 d46aef9 45e3283 b7fc258 45e3283 b7fc258 45e3283 d46aef9 45e3283 b7fc258 45e3283 b7fc258 45e3283 d46aef9 45e3283 b7fc258 45e3283 b7fc258 45e3283 d46aef9 45e3283 b7fc258 45e3283 b7fc258 45e3283 d46aef9 45e3283 b7fc258 45e3283 b7fc258 45e3283 d46aef9 45e3283 d46aef9 45e3283 d46aef9 45e3283 d46aef9 45e3283 d46aef9 45e3283 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 |
{
"added_tokens_decoder": {
"0": {
"content": "UNK",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"1": {
"content": "CLS",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"2": {
"content": "SEP",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"3": {
"content": "PAD",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"4": {
"content": "MASK",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
}
},
"bos_token": "CLS",
"chat_template": "{% for message in messages %}{{ message['content'] }}{% endfor %}",
"clean_up_tokenization_spaces": false,
"cls_token": "CLS",
"eos_token": "SEP",
"mask_token": "MASK",
"max_new_tokens": 1048576,
"model_max_length": 1048576,
"pad_token": "PAD",
"padding_side": "right",
"sep_token": "SEP",
"tokenizer_class": "PreTrainedTokenizerFast",
"unk_token": "UNK"
}
|