add tokenizer

special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

tokenizer.json

@@ -0,0 +1,202 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "[CLS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "[SEP]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "BertNormalizer",
+    "clean_text": true,
+    "handle_chinese_chars": true,
+    "strip_accents": null,
+    "lowercase": true
+  },
+  "pre_tokenizer": {
+    "type": "WhitespaceSplit"
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "[CLS]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[SEP]",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "[CLS]": {
+        "id": "[CLS]",
+        "ids": [
+          2
+        ],
+        "tokens": [
+          "[CLS]"
+        ]
+      },
+      "[SEP]": {
+        "id": "[SEP]",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "[SEP]"
+        ]
+      }
+    }
+  },
+  "decoder": {
+    "type": "WordPiece",
+    "prefix": "##",
+    "cleanup": true
+  },
+  "model": {
+    "type": "WordPiece",
+    "unk_token": "[UNK]",
+    "continuing_subword_prefix": "##",
+    "max_input_chars_per_word": 100,
+    "vocab": {
+      "[UNK]": 0,
+      "[PAD]": 1,
+      "[CLS]": 2,
+      "[SEP]": 3,
+      "[MASK]": 4,
+      "\"": 5,
+      "#": 6,
+      "(": 7,
+      ")": 8,
+      "+": 9,
+      "-": 10,
+      ".": 11,
+      "/": 12,
+      "1": 13,
+      "2": 14,
+      "3": 15,
+      "4": 16,
+      "5": 17,
+      "6": 18,
+      "7": 19,
+      "8": 20,
+      "9": 21,
+      "=": 22,
+      "@": 23,
+      "A": 24,
+      "B": 25,
+      "C": 26,
+      "D": 27,
+      "E": 28,
+      "F": 29,
+      "G": 30,
+      "H": 31,
+      "I": 32,
+      "K": 33,
+      "L": 34,
+      "M": 35,
+      "N": 36,
+      "O": 37,
+      "P": 38,
+      "Q": 39,
+      "R": 40,
+      "S": 41,
+      "T": 42,
+      "V": 43,
+      "W": 44,
+      "X": 45,
+      "Y": 46,
+      "[": 47,
+      "\\": 48,
+      "]": 49,
+      "a": 50,
+      "c": 51,
+      "i": 52,
+      "l": 53,
+      "n": 54,
+      "o": 55,
+      "r": 56,
+      "s": 57,
+      "##O": 58,
+      "##\"": 59,
+      "##S": 60,
+      "##E": 61,
+      "##P": 62,
+      "##]": 63,
+      "##C": 64,
+      "##N": 65,
+      "##B": 66,
+      "##[": 67,
+      "##c": 68,
+      "##F": 69
+    }
+  }
+}

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": true, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "tokenizer_class": "BertTokenizer"}

vocab.txt

@@ -0,0 +1,70 @@
+[UNK]
+[PAD]
+[CLS]
+[SEP]
+[MASK]
+"
+#
+(
+)
++
+-
+.
+/
+1
+2
+3
+4
+5
+6
+7
+8
+9
+=
+@
+A
+B
+C
+D
+E
+F
+G
+H
+I
+K
+L
+M
+N
+O
+P
+Q
+R
+S
+T
+V
+W
+X
+Y
+[
+\
+]
+a
+c
+i
+l
+n
+o
+r
+s
+##O
+##"
+##S
+##E
+##P
+##]
+##C
+##N
+##B
+##[
+##c
+##F