File size: 2,241 Bytes
7c9afdb 2270930 7c9afdb 2270930 7c9afdb c908293 7c9afdb 2270930 9d15763 c908293 2270930 c908293 2270930 c908293 2270930 c908293 2270930 c908293 2270930 c908293 2270930 c908293 2270930 7c9afdb 2270930 c908293 2270930 c908293 2270930 c908293 2270930 683b9d3 2270930 7c9afdb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | {
"added_tokens_decoder": {
"0": {
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "<pad>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "</s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"3": {
"content": "<unk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"32769": {
"content": "<mask>",
"lstrip": true,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
},
"32770": {
"content": "<ent>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
},
"32771": {
"content": "<ent2>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
}
},
"additional_special_tokens": [
"<ent>",
"<ent2>",
"<ent>",
"<ent2>",
"<ent>",
"<ent2>"
],
"bos_token": "<s>",
"clean_up_tokenization_spaces": false,
"cls_token": "<s>",
"entity_mask2_token": "[MASK2]",
"entity_mask_token": "[MASK]",
"entity_pad_token": "[PAD]",
"entity_token_1": {
"__type": "AddedToken",
"content": "<ent>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"entity_token_2": {
"__type": "AddedToken",
"content": "<ent2>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"entity_unk_token": "[UNK]",
"eos_token": "</s>",
"extra_special_tokens": {},
"mask_token": "<mask>",
"max_entity_length": 32,
"max_mention_length": 30,
"model_max_length": 512,
"pad_token": "<pad>",
"sep_token": "</s>",
"sp_model_kwargs": {},
"task": null,
"tokenizer_class": "MLukeTokenizer",
"unk_token": "<unk>"
}
|