add tokenizer

tokenizer.json

@@ -2,7 +2,53 @@
   "version": "1.0",
   "truncation": null,
   "padding": null,
-  "added_tokens": [],
+  "added_tokens": [
+    {
+      "id": 10,
+      "content": "[bos]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 32,
+      "content": "[eos]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 57,
+      "content": "[unk]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65,
+      "content": "[nop]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 88,
+      "content": "[mask]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
   "normalizer": null,
   "pre_tokenizer": {
     "type": "WhitespaceSplit"
@@ -15,90 +61,95 @@
     "continuing_subword_prefix": "##",
     "max_input_chars_per_word": 100,
     "vocab": {
-      "[As+1]": 0,
-      "[=SH0]": 1,
-      "[=SH1]": 2,
-      "[=Ring2]": 3,
-      "[=Ring1]": 4,
-      "[CH1]": 5,
-      "[S]": 6,
-      "[NH2+1]": 7,
-      "[B]": 8,
-      "[C-1]": 9,
-      "[#C]": 10,
-      "[=P]": 11,
-      "[As]": 12,
-      "[B-1]": 13,
-      "[bos]": 14,
-      "[O]": 15,
-      "[OH0]": 16,
-      "[I]": 17,
-      "[nop]": 18,
-      "[Cl]": 19,
-      "[SiH2]": 20,
-      "[Ring1]": 21,
-      "[Fe-4]": 22,
-      "[CH0]": 23,
-      "[Fe]": 24,
-      "[Fe+2]": 25,
-      "[CH1-1]": 26,
-      "[=Branch3]": 27,
-      "[#Branch1]": 28,
-      "[=Branch2]": 29,
-      "[NH0]": 30,
-      "[N-1]": 31,
-      "[C]": 32,
-      "[=NH2+1]": 33,
-      "[NH1-1]": 34,
-      "[#N+1]": 35,
-      "[SeH1]": 36,
-      "[Branch3]": 37,
-      "[SH1]": 38,
-      "[CH2-1]": 39,
-      "[SH0]": 40,
-      "[=Se]": 41,
-      "[NH1+1]": 42,
-      "[K]": 43,
-      "[Ring2]": 44,
-      "[#N]": 45,
-      "[O-1]": 46,
-      "[OH1+1]": 47,
-      "[#Branch2]": 48,
-      "[=C]": 49,
-      "[I+1]": 50,
-      "[Si]": 51,
-      "[F]": 52,
-      "[=N+1]": 53,
-      "[=OH1+1]": 54,
-      "[Branch2]": 55,
-      "[=O+1]": 56,
-      "[#S]": 57,
-      "[Na]": 58,
-      "[C+1]": 59,
-      "[=B]": 60,
-      "[S+1]": 61,
-      "[unk]": 62,
-      "[=Fe]": 63,
-      "[P]": 64,
-      "[=N]": 65,
-      "[SiH1]": 66,
+      "[Branch1]": 0,
+      "[=NH1+1]": 1,
+      "[Se]": 2,
+      "[#Branch2]": 3,
+      "[O-1]": 4,
+      "[SiH1]": 5,
+      "[SeH1]": 6,
+      "[CH2-1]": 7,
+      "[SH0]": 8,
+      "[PH1]": 9,
+      "[bos]": 10,
+      "[Si]": 11,
+      "[OH1+1]": 12,
+      "[Fe]": 13,
+      "[NH1]": 14,
+      "[Ring2]": 15,
+      "[=N]": 16,
+      "[=NH2+1]": 17,
+      "[B]": 18,
+      "[=SH1]": 19,
+      "[C]": 20,
+      "[=C]": 21,
+      "[NH1-1]": 22,
+      "[=O+1]": 23,
+      "[As]": 24,
+      "[#Branch1]": 25,
+      "[I]": 26,
+      "[=O]": 27,
+      "[B-1]": 28,
+      "[Fe-4]": 29,
+      "[=Ring1]": 30,
+      "[=S]": 31,
+      "[eos]": 32,
+      "[Cl]": 33,
+      "[=P]": 34,
+      "[=Fe]": 35,
+      "[NH1+1]": 36,
+      "[CH1]": 37,
+      "[#Ring1]": 38,
+      "[As+1]": 39,
+      "[Branch3]": 40,
+      "[O]": 41,
+      "[=OH1+1]": 42,
+      "[Branch2]": 43,
+      "[=As]": 44,
+      "[F]": 45,
+      "[P+1]": 46,
+      "[S]": 47,
+      "[#Ring2]": 48,
+      "[#N]": 49,
+      "[CH1+1]": 50,
+      "[OH0]": 51,
+      "[N]": 52,
+      "[I+1]": 53,
+      "[=Ring2]": 54,
+      "[C+1]": 55,
+      "[=B]": 56,
+      "[unk]": 57,
+      "[SiH2]": 58,
+      "[C-1]": 59,
+      "[=PH1]": 60,
+      "[#C]": 61,
+      "[SH1]": 62,
+      "[Fe-3]": 63,
+      "[Br]": 64,
+      "[nop]": 65,
+      "[CH1-1]": 66,
       "[NH3+1]": 67,
-      "[Fe-3]": 68,
-      "[CH1+1]": 69,
-      "[Branch1]": 70,
-      "[Fe+1]": 71,
-      "[=Branch1]": 72,
-      "[=S]": 73,
-      "[Se]": 74,
-      "[N]": 75,
-      "[=As]": 76,
-      "[#Ring2]": 77,
-      "[Br]": 78,
-      "[=O]": 79,
-      "[P+1]": 80,
-      "[N+1]": 81,
-      "[eos]": 82,
-      "[Se+1]": 83
+      "[=Branch1]": 68,
+      "[NH2+1]": 69,
+      "[P]": 70,
+      "[K]": 71,
+      "[N+1]": 72,
+      "[CH0]": 73,
+      "[=Se]": 74,
+      "[Fe+1]": 75,
+      "[Ring1]": 76,
+      "[S+1]": 77,
+      "[=Branch3]": 78,
+      "[Fe+2]": 79,
+      "[=S+1]": 80,
+      "[=N+1]": 81,
+      "[Na]": 82,
+      "[Se+1]": 83,
+      "[N-1]": 84,
+      "[NH0]": 85,
+      "[#S]": 86,
+      "[=Branch2]": 87,
+      "[mask]": 88
     }
   }
 }

tokenizer_config.json

@@ -1,8 +1,13 @@
 {
   "cls_token": "[bos]",
   "mask_token": "[mask]",
+  "model_max_length": 427,
+  "name_or_path": "tokenizer",
   "pad_token": "[nop]",
+  "padding_side": "right",
   "sep_token": "[eos]",
+  "special_tokens_map_file": "tokenizer/special_tokens_map.json",
   "tokenizer_class": "PreTrainedTokenizerFast",
+  "truncation_side": "right",
   "unk_token": "[unk]"
 }