add tokenizer

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer.json

@@ -0,0 +1,150 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "[CLS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "[SEP]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "NFD"
+      },
+      {
+        "type": "StripAccents"
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "BertPreTokenizer"
+  },
+  "post_processor": null,
+  "decoder": {
+    "type": "WordPiece",
+    "prefix": "##",
+    "cleanup": true
+  },
+  "model": {
+    "type": "WordPiece",
+    "unk_token": "[UNK]",
+    "continuing_subword_prefix": "##",
+    "max_input_chars_per_word": 100,
+    "vocab": {
+      "[UNK]": 0,
+      "[PAD]": 1,
+      "[CLS]": 2,
+      "[SEP]": 3,
+      "[MASK]": 4,
+      "A": 5,
+      "C": 6,
+      "G": 7,
+      "T": 8,
+      "##A": 9,
+      "##T": 10,
+      "##G": 11,
+      "##C": 12,
+      "AA": 13,
+      "GA": 14,
+      "TT": 15,
+      "AT": 16,
+      "##CT": 17,
+      "##CA": 18,
+      "CA": 19,
+      "GT": 20,
+      "AG": 21,
+      "GG": 22,
+      "GAA": 23,
+      "##CC": 24,
+      "AAA": 25,
+      "CT": 26,
+      "GAT": 27,
+      "AAT": 28,
+      "TA": 29,
+      "AAG": 30,
+      "ATT": 31,
+      "##CG": 32,
+      "CAA": 33,
+      "TTG": 34,
+      "TTA": 35,
+      "TTT": 36,
+      "AAC": 37,
+      "TCT": 38,
+      "GGT": 39,
+      "TG": 40,
+      "GTT": 41,
+      "AGA": 42,
+      "GCT": 43,
+      "GAC": 44,
+      "ACT": 45,
+      "GAG": 46,
+      "TCA": 47,
+      "ATG": 48,
+      "TAT": 49,
+      "TTC": 50,
+      "ATA": 51,
+      "ACA": 52,
+      "CCA": 53,
+      "ATC": 54,
+      "GCA": 55,
+      "AGT": 56,
+      "TAC": 57,
+      "TCC": 58,
+      "CAT": 59,
+      "CTA": 60,
+      "CCT": 61,
+      "CG": 62,
+      "CAG": 63,
+      "GCC": 64,
+      "ACC": 65,
+      "CTT": 66,
+      "GTA": 67,
+      "GTC": 68,
+      "GGA": 69
+    }
+  }
+}

tokenizer_config.json

@@ -0,0 +1,11 @@
+{
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

vocab.txt

@@ -0,0 +1,70 @@
+[UNK]
+[PAD]
+[CLS]
+[SEP]
+[MASK]
+A
+C
+G
+T
+##A
+##T
+##G
+##C
+AA
+GA
+TT
+AT
+##CT
+##CA
+CA
+GT
+AG
+GG
+GAA
+##CC
+AAA
+CT
+GAT
+AAT
+TA
+AAG
+ATT
+##CG
+CAA
+TTG
+TTA
+TTT
+AAC
+TCT
+GGT
+TG
+GTT
+AGA
+GCT
+GAC
+ACT
+GAG
+TCA
+ATG
+TAT
+TTC
+ATA
+ACA
+CCA
+ATC
+GCA
+AGT
+TAC
+TCC
+CAT
+CTA
+CCT
+CG
+CAG
+GCC
+ACC
+CTT
+GTA
+GTC
+GGA