File size: 2,240 Bytes
2398db6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 | {
"add_prefix_space": true,
"backend": "tokenizers",
"bos_token": "</s>",
"cls_token": "[CLS]",
"eos_token": "<s>",
"extra_special_tokens": [
"<pad>",
"</s>",
"<s>",
"<sep_0>",
"<sep_1>",
"<sep_2>",
"<sep_3>",
"<sep_4>",
"<no_saic_raw_sp>",
"<length_0>",
"<length_1>",
"<length_2>",
"<length_3>",
"<length_4>",
"<length_5>",
"<length_6>",
"<length_7>",
"<length_8>",
"<length_9>",
"<length_10>",
"<length_11>",
"<length_12>",
"<length_13>",
"<length_14>",
"<length_15>",
"<length_16>",
"<length_17>",
"<length_18>",
"<length_19>",
"<length_20>",
"<length_s>",
"<length_m>",
"<length_l>",
"<::::>",
"<space>",
"<unused_34>",
"<unused_35>",
"<unused_36>",
"<no_query_sp>",
"<no_passage_raw_sp>",
"<unused_39>",
"<unused_40>",
"<unused_41>",
"<unused_42>",
"<unused_43>",
"<unused_44>",
"<unused_45>",
"<unused_46>",
"<unused_47>",
"<unused_48>",
"<unused_49>",
"<paraphrasing_0>",
"<paraphrasing_1>",
"<paraphrasing_2>",
"<paraphrasing_3>",
"<paraphrasing_4>",
"<paraphrasing_5>",
"<paraphrasing_6>",
"<paraphrasing_7>",
"<paraphrasing_8>",
"<empty>",
"<row>",
"<cn>",
"<t>",
"<ans>",
"[CLS]",
"[SEP]",
"[MASK]",
"<unused_67>",
"<unused_68>",
"<unused_69>",
"<unused_70>",
"<unused_71>",
"<unused_72>",
"<unused_73>",
"<unused_74>",
"<unused_75>",
"<unused_76>",
"<unused_77>",
"<unused_78>",
"<unused_79>",
"<unused_80>",
"<unused_81>",
"<unused_82>",
"<unused_83>",
"<unused_84>",
"<unused_85>",
"<unused_86>",
"<unused_87>",
"<unused_88>",
"<unused_89>",
"<unused_90>",
"<unused_91>",
"<unused_92>",
"<unused_93>",
"<unused_94>",
"<unused_95>",
"<unused_96>",
"<unused_97>",
"<unused_98>"
],
"is_local": true,
"local_files_only": false,
"mask_token": "[MASK]",
"model_max_length": 256,
"pad_token": "<pad>",
"sep_token": "[SEP]",
"tokenizer_class": "BigBirdTokenizer",
"unk_id": 100,
"unk_token": "<unk>"
}
|