File size: 1,829 Bytes
6b20649 04afa92 6b20649 e03db1a 6b20649 e03db1a 6b20649 e03db1a 6b20649 e03db1a 00c08c9 e03db1a 00c08c9 2c3c16a 6b20649 e03db1a 00c08c9 04afa92 e03db1a 00c08c9 04afa92 e03db1a 6b20649 e03db1a 00c08c9 e03db1a 04afa92 2c3c16a e03db1a 00c08c9 6b20649 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 |
{
"add_bos_token": true,
"add_eos_token": false,
"add_prefix_space": true,
"added_tokens_decoder": {
"0": {
"content": "[PAD]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "[UNK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "[BOS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"3": {
"content": "[EOS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"4": {
"content": "[CLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"5": {
"content": "[SEP]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"6": {
"content": "[MASK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"bos_token": "[BOS]",
"bos_token_id": 2,
"clean_up_tokenization_spaces": false,
"cls_token": "[CLS]",
"cls_token_id": 4,
"eos_token": "[EOS]",
"eos_token_id": 3,
"legacy": true,
"mask_token": "[MASK]",
"mask_token_id": 6,
"model_max_length": 1000000000000000019884624838656,
"pad_token": "[PAD]",
"pad_token_id": 0,
"sep_token": "[SEP]",
"sep_token_id": 5,
"sp_model_kwargs": {},
"spaces_between_special_tokens": false,
"tokenizer_class": "HfTokenizerFromConfig",
"unk_token": "[UNK]",
"unk_token_id": 1
}
|