File size: 1,833 Bytes
94dde77 85ad770 94dde77 9235d93 94dde77 9235d93 94dde77 9235d93 94dde77 9235d93 94dde77 9235d93 94dde77 c9f9c75 94dde77 9235d93 85ad770 9235d93 85ad770 9235d93 94dde77 9235d93 85ad770 9235d93 85ad770 c9f9c75 9235d93 85ad770 94dde77 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 |
{
"add_bos_token": true,
"add_eos_token": false,
"add_prefix_space": true,
"added_tokens_decoder": {
"0": {
"content": "[PAD]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "[UNK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "[BOS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"3": {
"content": "[EOS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"4": {
"content": "[CLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"5": {
"content": "[SEP]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"6": {
"content": "[MASK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
}
},
"bos_token": "[BOS]",
"bos_token_id": 2,
"clean_up_tokenization_spaces": false,
"cls_token": "[CLS]",
"cls_token_id": 4,
"eos_token": "[EOS]",
"eos_token_id": 3,
"legacy": true,
"mask_token": "[MASK]",
"mask_token_id": 6,
"model_max_length": 1000000000000000019884624838656,
"pad_token": "[PAD]",
"pad_token_id": 0,
"sep_token": "[SEP]",
"sep_token_id": 5,
"sp_model_kwargs": {},
"spaces_between_special_tokens": false,
"tokenizer_class": "HfSentencePieceTokenizer",
"unk_token": "[UNK]",
"unk_token_id": 1
}
|