File size: 1,285 Bytes
e93046d a6849ac e93046d 5f99f18 a6849ac e93046d a6849ac e93046d 5f99f18 e93046d a6849ac e93046d 5f99f18 e93046d a6849ac e93046d 5f99f18 e93046d a6849ac e93046d 5f99f18 a6849ac e93046d a6849ac e93046d a6849ac e93046d a6849ac e93046d a6849ac e93046d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 |
{
"add_prefix_space": false,
"added_tokens_decoder": {
"11203": {
"content": "<s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
},
"11204": {
"content": "</s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
},
"11205": {
"content": "<unk>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
},
"11206": {
"content": "<pad>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
},
"11207": {
"content": "<mask>",
"lstrip": true,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
}
},
"bos_token": "<s>",
"clean_up_tokenization_spaces": false,
"cls_token": "<s>",
"eos_token": "</s>",
"errors": "replace",
"extra_special_tokens": {},
"mask_token": "<mask>",
"model_max_length": 1000000000000000019884624838656,
"pad_token": "<pad>",
"sep_token": "</s>",
"tokenizer_class": "LEDTokenizer",
"trim_offsets": true,
"unk_token": "<unk>"
}
|