add tokenizer

special_tokens_map.json

@@ -1 +1 @@
-{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}
+{"mask_token": "[MASK]"}

tokenizer.json

@@ -5,7 +5,7 @@
   "added_tokens": [
     {
       "id": 0,
-      "content": "[PAD]",
+      "content": "[UNK]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -14,7 +14,7 @@
     },
     {
       "id": 1,
-      "content": "[UNK]",
+      "content": "[PAD]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -50,15 +50,17 @@
     }
   ],
   "normalizer": {
-    "type": "BertNormalizer",
-    "clean_text": true,
-    "handle_chinese_chars": true,
-    "strip_accents": null,
-    "lowercase": true
-  },
-  "pre_tokenizer": {
-    "type": "BertPreTokenizer"
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "NFD"
+      },
+      {
+        "type": "StripAccents"
+      }
+    ]
   },
+  "pre_tokenizer": null,
   "post_processor": {
     "type": "TemplateProcessing",
     "single": [
@@ -145,23 +147,506 @@
     "continuing_subword_prefix": "##",
     "max_input_chars_per_word": 100,
     "vocab": {
-      "[PAD]": 0,
-      "[UNK]": 1,
+      "[UNK]": 0,
+      "[PAD]": 1,
       "[CLS]": 2,
       "[SEP]": 3,
       "[MASK]": 4,
-      "1": 5,
-      "2": 6,
-      "3": 7,
-      "4": 8,
-      "5": 9,
-      "6": 10,
-      "##4": 11,
-      "##6": 12,
-      "##5": 13,
-      "##2": 14,
-      "##1": 15,
-      "##3": 16
+      " ": 5,
+      "1": 6,
+      "2": 7,
+      "3": 8,
+      "4": 9,
+      "5": 10,
+      "6": 11,
+      "##1": 12,
+      "## ": 13,
+      "##4": 14,
+      "##2": 15,
+      "##3": 16,
+      "##5": 17,
+      "##6": 18,
+      "##11": 19,
+      "##12": 20,
+      "##1 ": 21,
+      "##22": 22,
+      "##11 ": 23,
+      "##2 ": 24,
+      "##13": 25,
+      "##12 ": 26,
+      "##21 ": 27,
+      "##14": 28,
+      "##211": 29,
+      "##3 ": 30,
+      "##1111 ": 31,
+      "##22 ": 32,
+      "##212": 33,
+      "##4 ": 34,
+      "##13 ": 35,
+      "##31 ": 36,
+      "##1211 ": 37,
+      "##2111 ": 38,
+      "##1112 ": 39,
+      "##1121 ": 40,
+      "##23": 41,
+      "##311": 42,
+      "##14 ": 43,
+      "##15": 44,
+      "##24": 45,
+      "##1122 ": 46,
+      "##32 ": 47,
+      "##1221 ": 48,
+      "##2211 ": 49,
+      "##2121 ": 50,
+      "##41 ": 51,
+      "##2112 ": 52,
+      "##23 ": 53,
+      "##1212 ": 54,
+      "##32": 55,
+      "##16": 56,
+      "##411": 57,
+      "##213": 58,
+      "##5 ": 59,
+      "##121 ": 60,
+      "##42 ": 61,
+      "##2212 ": 62,
+      "##2221 ": 63,
+      "##1131 ": 64,
+      "##1311 ": 65,
+      "##1113 ": 66,
+      "##1222 ": 67,
+      "##2122 ": 68,
+      "##3111 ": 69,
+      "##24 ": 70,
+      "##33": 71,
+      "##42": 72,
+      "##6 ": 73,
+      "##214": 74,
+      "##221 ": 75,
+      "##122 ": 76,
+      "##2222 ": 77,
+      "##51 ": 78,
+      "##1114 ": 79,
+      "##33 ": 80,
+      "##15 ": 81,
+      "##25": 82,
+      "##1141 ": 83,
+      "##1213 ": 84,
+      "##1312 ": 85,
+      "##4111 ": 86,
+      "##1132 ": 87,
+      "##2131 ": 88,
+      "##1411 ": 89,
+      "##1321 ": 90,
+      "##2113 ": 91,
+      "##2311 ": 92,
+      "##3112 ": 93,
+      "##3211 ": 94,
+      "##1123 ": 95,
+      "##1231 ": 96,
+      "##3121 ": 97,
+      "##34": 98,
+      "##43": 99,
+      "##222 ": 100,
+      "##61 ": 101,
+      "##511": 102,
+      "##313": 103,
+      "##16 ": 104,
+      "##52 ": 105,
+      "##26": 106,
+      "##44": 107,
+      "##1142 ": 108,
+      "##1421 ": 109,
+      "##25 ": 110,
+      "##1111 1111 ": 111,
+      "##1214 ": 112,
+      "##4211 ": 113,
+      "##2213 ": 114,
+      "##4121 ": 115,
+      "##2114 ": 116,
+      "##1412 ": 117,
+      "##43 ": 118,
+      "##4112 ": 119,
+      "##34 ": 120,
+      "##3122 ": 121,
+      "##2132 ": 122,
+      "##2231 ": 123,
+      "##3212 ": 124,
+      "##2123 ": 125,
+      "##2411 ": 126,
+      "##2141 ": 127,
+      "##1232 ": 128,
+      "##1223 ": 129,
+      "##3221 ": 130,
+      "##2321 ": 131,
+      "##1322 ": 132,
+      "##1241 ": 133,
+      "##1124 ": 134,
+      "##2312 ": 135,
+      "##52": 136,
+      "##62 ": 137,
+      "##611": 138,
+      "##314": 139,
+      "##215": 140,
+      "##44 ": 141,
+      "##312": 142,
+      "##26 ": 143,
+      "##413": 144,
+      "##1422 ": 145,
+      "##2421 ": 146,
+      "##3113 ": 147,
+      "##4221 ": 148,
+      "##4212 ": 149,
+      "##1133 ": 150,
+      "##2142 ": 151,
+      "##2124 ": 152,
+      "##3222 ": 153,
+      "##4122 ": 154,
+      "##2223 ": 155,
+      "##2322 ": 156,
+      "##2214 ": 157,
+      "##3311 ": 158,
+      "##2241 ": 159,
+      "##2232 ": 160,
+      "##1242 ": 161,
+      "##5111 ": 162,
+      "##1224 ": 163,
+      "##1331 ": 164,
+      "##3131 ": 165,
+      "##62": 166,
+      "##1511 ": 167,
+      "##2412 ": 168,
+      "##1115 ": 169,
+      "##1313 ": 170,
+      "##35": 171,
+      "##1151 ": 172,
+      "##216": 173,
+      "##1111 1112 ": 174,
+      "##53": 175,
+      "##1111 1211 ": 176,
+      "##1111 1121 ": 177,
+      "##414": 178,
+      "##1111 2111 ": 179,
+      "##223 ": 180,
+      "##412": 181,
+      "##241 ": 182,
+      "##4222 ": 183,
+      "##53 ": 184,
+      "##232 ": 185,
+      "##2422 ": 186,
+      "##2242 ": 187,
+      "##1332 ": 188,
+      "##45": 189,
+      "##2224 ": 190,
+      "##3114 ": 191,
+      "##1161 ": 192,
+      "##1611 ": 193,
+      "##1323 ": 194,
+      "##1251 ": 195,
+      "##6111 ": 196,
+      "##1431 ": 197,
+      "##1211 1111 ": 198,
+      "##3213 ": 199,
+      "##2511 ": 200,
+      "##1116 ": 201,
+      "##1143 ": 202,
+      "##2133 ": 203,
+      "##2111 1111 ": 204,
+      "##3312 ": 205,
+      "##1134 ": 206,
+      "##1112 1111 ": 207,
+      "##4113 ": 208,
+      "##1314 ": 209,
+      "##5121 ": 210,
+      "##36": 211,
+      "##1125 ": 212,
+      "##3141 ": 213,
+      "##3321 ": 214,
+      "##5211 ": 215,
+      "##1413 ": 216,
+      "##1121 1111 ": 217,
+      "##1521 ": 218,
+      "##2331 ": 219,
+      "##3411 ": 220,
+      "##4131 ": 221,
+      "##3123 ": 222,
+      "##4311 ": 223,
+      "##1233 ": 224,
+      "##54": 225,
+      "##5112 ": 226,
+      "##1152 ": 227,
+      "##2115 ": 228,
+      "##35 ": 229,
+      "##3132 ": 230,
+      "##2151 ": 231,
+      "##1341 ": 232,
+      "##2313 ": 233,
+      "##1215 ": 234,
+      "##63": 235,
+      "##1512 ": 236,
+      "##46": 237,
+      "##224 ": 238,
+      "##3231 ": 239,
+      "##242 ": 240,
+      "##63 ": 241,
+      "##2111 1211 ": 242,
+      "##1252 ": 243,
+      "##2111 2111 ": 244,
+      "##2611 ": 245,
+      "##1121 1121 ": 246,
+      "##54 ": 247,
+      "##1441 ": 248,
+      "##2116 ": 249,
+      "##1112 1112 ": 250,
+      "##2161 ": 251,
+      "##2112 1111 ": 252,
+      "##5221 ": 253,
+      "##1621 ": 254,
+      "##2512 ": 255,
+      "##45 ": 256,
+      "##1144 ": 257,
+      "##1221 1111 ": 258,
+      "##64": 259,
+      "##131 ": 260,
+      "##2314 ": 261,
+      "##1261 ": 262,
+      "##1112 1211 ": 263,
+      "##1162 ": 264,
+      "##1126 ": 265,
+      "##1414 ": 266,
+      "##2152 ": 267,
+      "##151 ": 268,
+      "##36 ": 269,
+      "##3142 ": 270,
+      "##6211 ": 271,
+      "##3214 ": 272,
+      "##4114 ": 273,
+      "##1225 ": 274,
+      "##2413 ": 275,
+      "##3322 ": 276,
+      "##2121 1111 ": 277,
+      "##2125 ": 278,
+      "##2233 ": 279,
+      "##1112 1121 ": 280,
+      "##6121 ": 281,
+      "##4321 ": 282,
+      "##2251 ": 283,
+      "##2521 ": 284,
+      "##6112 ": 285,
+      "##1324 ": 286,
+      "##1211 1211 ": 287,
+      "##4213 ": 288,
+      "##2211 1111 ": 289,
+      "##1112 2111 ": 290,
+      "##2332 ": 291,
+      "##4411 ": 292,
+      "##1216 ": 293,
+      "##2143 ": 294,
+      "##1342 ": 295,
+      "##2134 ": 296,
+      "##3421 ": 297,
+      "##1612 ": 298,
+      "##4312 ": 299,
+      "##3223 ": 300,
+      "##2215 ": 301,
+      "##5212 ": 302,
+      "##1211 2111 ": 303,
+      "##4141 ": 304,
+      "##3124 ": 305,
+      "##1122 1111 ": 306,
+      "##4123 ": 307,
+      "##1522 ": 308,
+      "##1211 1121 ": 309,
+      "##1212 1111 ": 310,
+      "##1432 ": 311,
+      "##3232 ": 312,
+      "111": 313,
+      "##5122 ": 314,
+      "##1423 ": 315,
+      "##2431 ": 316,
+      "##2111 1121 ": 317,
+      "##3241 ": 318,
+      "##4132 ": 319,
+      "##4231 ": 320,
+      "##1234 ": 321,
+      "##1111 2112 ": 322,
+      "##1243 ": 323,
+      "##2111 1112 ": 324,
+      "##1111 2211 ": 325,
+      "##64 ": 326,
+      "##1111 2121 ": 327,
+      "##3412 ": 328,
+      "##1121 1112 ": 329,
+      "##2162 ": 330,
+      "##1211 1112 ": 331,
+      "##2323 ": 332,
+      "##2341 ": 333,
+      "##1121 2111 ": 334,
+      "##21": 335,
+      "##1111 1221 ": 336,
+      "##1111 1212 ": 337,
+      "##1111 1122 ": 338,
+      "##161 ": 339,
+      "##4421 ": 340,
+      "##46 ": 341,
+      "##1121 1211 ": 342,
+      "##233 ": 343,
+      "##1262 ": 344,
+      "##2261 ": 345,
+      "##2126 ": 346,
+      "##3313 ": 347,
+      "##1244 ": 348,
+      "##3331 ": 349,
+      "##2612 ": 350,
+      "##2225 ": 351,
+      "##512": 352,
+      "##6221 ": 353,
+      "##514": 354,
+      "##2252 ": 355,
+      "##152 ": 356,
+      "##2522 ": 357,
+      "##1211 1221 ": 358,
+      "##4412 ": 359,
+      "##6212 ": 360,
+      "##3133 ": 361,
+      "##1442 ": 362,
+      "##4214 ": 363,
+      "##2621 ": 364,
+      "##2112 1112 ": 365,
+      "##5222 ": 366,
+      "##3111 1111 ": 367,
+      "##4142 ": 368,
+      "##1226 ": 369,
+      "##2144 ": 370,
+      "##2216 ": 371,
+      "12": 372,
+      "##1333 ": 373,
+      "##4223 ": 374,
+      "##4241 ": 375,
+      "##132 ": 376,
+      "##4322 ": 377,
+      "##1131 1111 ": 378,
+      "##2211 2111 ": 379,
+      "##1211 1122 ": 380,
+      "##1424 ": 381,
+      "##6122 ": 382,
+      "##1311 1111 ": 383,
+      "##55": 384,
+      "##2342 ": 385,
+      "##2112 2111 ": 386,
+      "##1622 ": 387,
+      "##3224 ": 388,
+      "##1153 ": 389,
+      "##1531 ": 390,
+      "##3422 ": 391,
+      "##2441 ": 392,
+      "##1113 1111 ": 393,
+      "##4124 ": 394,
+      "##1121 2121 ": 395,
+      "##2432 ": 396,
+      "##1212 1211 ": 397,
+      "##1212 2111 ": 398,
+      "##1121 2211 ": 399,
+      "##2243 ": 400,
+      "##614": 401,
+      "##5311 ": 402,
+      "211": 403,
+      "##1221 1112 ": 404,
+      "##1212 1112 ": 405,
+      "##1122 1112 ": 406,
+      "##3242 ": 407,
+      "##612": 408,
+      "##2414 ": 409,
+      "##2211 1211 ": 410,
+      "##3115 ": 411,
+      "##2234 ": 412,
+      "##2121 2111 ": 413,
+      "##1122 1211 ": 414,
+      "##1122 2111 ": 415,
+      "##1221 2111 ": 416,
+      "##2324 ": 417,
+      "##2112 1211 ": 418,
+      "##1122 1121 ": 419,
+      "##1111 1131 ": 420,
+      "##2423 ": 421,
+      "##1351 ": 422,
+      "##1212 1121 ": 423,
+      "##2211 1112 ": 424,
+      "##1111": 425,
+      "##1221 1121 ": 426,
+      "##2121 1112 ": 427,
+      "##1222 1111 ": 428,
+      "##3511 ": 429,
+      "##2112 1121 ": 430,
+      "##4232 ": 431,
+      "##162 ": 432,
+      "##2122 1111 ": 433,
+      "##2226 ": 434,
+      "##55 ": 435,
+      "##2221 1111 ": 436,
+      "##2121 1121 ": 437,
+      "##324": 438,
+      "##1221 1211 ": 439,
+      "##5113 ": 440,
+      "##324 ": 441,
+      "##5131 ": 442,
+      "##225 ": 443,
+      "##1513 ": 444,
+      "##2121 1211 ": 445,
+      "##1135 ": 446,
+      "##6222 ": 447,
+      "##1111 2212 ": 448,
+      "##2211 1121 ": 449,
+      "##2262 ": 450,
+      "##2424 ": 451,
+      "##4224 ": 452,
+      "##3151 ": 453,
+      "##1111 1113 ": 454,
+      "##4422 ": 455,
+      "##4242 ": 456,
+      "##1315 ": 457,
+      "##65": 458,
+      "##2442 ": 459,
+      "##1311 1112 ": 460,
+      "##2111 2112 ": 461,
+      "##2622 ": 462,
+      "##1212 1221 ": 463,
+      "22": 464,
+      "##1111 3111 ": 465,
+      "##1111 2221 ": 466,
+      "##1211 2211 ": 467,
+      "##3314 ": 468,
+      "1111 ": 469,
+      "##2244 ": 470,
+      "##2111 2211 ": 471,
+      "##1211 2121 ": 472,
+      "##2111 1212 ": 473,
+      "##2111 1221 ": 474,
+      "##1112 2211 ": 475,
+      "##1112 1221 ": 476,
+      "##1111 1311 ": 477,
+      "##3413 ": 478,
+      "##1163 ": 479,
+      "##1111 1222 ": 480,
+      "##1211 2112 ": 481,
+      "##1112 2112 ": 482,
+      "##234 ": 483,
+      "##4511 ": 484,
+      "##3611 ": 485,
+      "##2111 1122 ": 486,
+      "##2111 2121 ": 487,
+      "##1121 1122 ": 488,
+      "##6311 ": 489,
+      "##1211 1212 ": 490,
+      "##66": 491,
+      "##1433 ": 492,
+      "##1121 1221 ": 493,
+      "##1112 1122 ": 494,
+      "##2333 ": 495,
+      "##2531 ": 496,
+      "##1121 1212 ": 497,
+      "##1111 2122 ": 498,
+      "##1112 1212 ": 499
     }
   }
 }

tokenizer_config.json

@@ -1 +1 @@
-{"do_lower_case": true, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "bert-base-uncased", "tokenizer_class": "BertTokenizer"}
+{"tokenizer_class": "PreTrainedTokenizerFast"}