File size: 1,829 Bytes
98db34c c0a519c 98db34c eecea33 98db34c eecea33 98db34c eecea33 98db34c eecea33 5cf98d4 eecea33 5cf98d4 3f9ac81 98db34c eecea33 5cf98d4 c0a519c eecea33 5cf98d4 c0a519c eecea33 98db34c eecea33 5cf98d4 eecea33 c0a519c 3f9ac81 eecea33 5cf98d4 98db34c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 |
{
"add_bos_token": true,
"add_eos_token": false,
"add_prefix_space": true,
"added_tokens_decoder": {
"0": {
"content": "[PAD]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "[UNK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "[BOS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"3": {
"content": "[EOS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"4": {
"content": "[CLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"5": {
"content": "[SEP]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"6": {
"content": "[MASK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"bos_token": "[BOS]",
"bos_token_id": 2,
"clean_up_tokenization_spaces": false,
"cls_token": "[CLS]",
"cls_token_id": 4,
"eos_token": "[EOS]",
"eos_token_id": 3,
"legacy": true,
"mask_token": "[MASK]",
"mask_token_id": 6,
"model_max_length": 1000000000000000019884624838656,
"pad_token": "[PAD]",
"pad_token_id": 0,
"sep_token": "[SEP]",
"sep_token_id": 5,
"sp_model_kwargs": {},
"spaces_between_special_tokens": false,
"tokenizer_class": "HfTokenizerFromConfig",
"unk_token": "[UNK]",
"unk_token_id": 1
}
|