File size: 2,049 Bytes
e0bb631 85826bb e0bb631 85826bb e0bb631 85826bb e0bb631 85826bb e0bb631 85826bb e0bb631 85826bb e0bb631 85826bb e0bb631 85826bb e0bb631 85826bb e0bb631 85826bb e0bb631 85826bb e0bb631 85826bb e0bb631 85826bb e0bb631 85826bb e0bb631 85826bb e0bb631 85826bb e0bb631 85826bb e0bb631 85826bb e0bb631 85826bb e0bb631 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 | {
"added_tokens_decoder": {
"0": {
"content": "[PAD]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "[UNK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "[CLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"3": {
"content": "[SEP]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"4": {
"content": "[MASK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"5": {
"content": "[رابط]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"6": {
"content": "[بريد]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"7": {
"content": "[مستخدم]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
}
},
"clean_up_tokenization_spaces": true,
"cls_token": "[CLS]",
"do_basic_tokenize": true,
"do_lower_case": false,
"extra_special_tokens": {},
"mask_token": "[MASK]",
"max_len": 512,
"max_length": 128,
"model_max_length": 512,
"never_split": [
"[بريد]",
"[مستخدم]",
"[رابط]"
],
"pad_to_multiple_of": null,
"pad_token": "[PAD]",
"pad_token_type_id": 0,
"padding_side": "right",
"sep_token": "[SEP]",
"stride": 0,
"strip_accents": null,
"tokenize_chinese_chars": true,
"tokenizer_class": "ElectraTokenizer",
"truncation_side": "right",
"truncation_strategy": "longest_first",
"unk_token": "[UNK]"
}
|