Create tokenizer.json

tokenizer.json

@@ -0,0 +1,36 @@
+{
+    "model_type": "bert",
+    "vocab_size": 32000,
+    "type_vocab_size": 2,
+    "padding_id": 0,
+    "unk_id": 1,
+    "cls_id": 2,
+    "sep_id": 3,
+    "mask_id": 4,
+    "normalizer": null,
+    "pretokenizer": {
+        "type": "ByteLevel",
+        "add_prefix_space": true
+    },
+    "postprocessor": {
+        "type": "TemplateProcessing",
+        "single": {
+            "template": "{input} </s>",
+            "special_tokens": [
+                {
+                    "id": 2,
+                    "content": "</s>"
+                }
+            ]
+        },
+        "pair": {
+            "template": "{pair} </s>",
+            "special_tokens": [
+                {
+                    "id": 2,
+                    "content": "</s>"
+                }
+            ]
+        }
+    }
+}