File size: 1,829 Bytes
c606dca fb5b719 c606dca 6a3891d c606dca 6a3891d c606dca 6a3891d c606dca 6a3891d 270b7ce 6a3891d 270b7ce df608df c606dca 6a3891d 270b7ce fb5b719 6a3891d 270b7ce fb5b719 6a3891d c606dca 6a3891d 270b7ce 6a3891d fb5b719 df608df 6a3891d 270b7ce c606dca |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 |
{
"add_bos_token": true,
"add_eos_token": false,
"add_prefix_space": true,
"added_tokens_decoder": {
"0": {
"content": "[PAD]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "[UNK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "[BOS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"3": {
"content": "[EOS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"4": {
"content": "[CLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"5": {
"content": "[SEP]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"6": {
"content": "[MASK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"bos_token": "[BOS]",
"bos_token_id": 2,
"clean_up_tokenization_spaces": false,
"cls_token": "[CLS]",
"cls_token_id": 4,
"eos_token": "[EOS]",
"eos_token_id": 3,
"legacy": true,
"mask_token": "[MASK]",
"mask_token_id": 6,
"model_max_length": 1000000000000000019884624838656,
"pad_token": "[PAD]",
"pad_token_id": 0,
"sep_token": "[SEP]",
"sep_token_id": 5,
"sp_model_kwargs": {},
"spaces_between_special_tokens": false,
"tokenizer_class": "HfTokenizerFromConfig",
"unk_token": "[UNK]",
"unk_token_id": 1
}
|