Add tokenizer_config.json

tokenizer_config.json

@@ -0,0 +1,22 @@
+{
+  "vocab_target": 6000,
+  "max_merge_steps": 200000,
+  "special_tokens": [
+    "<PAD>",
+    "<UNK>",
+    "<BOS>",
+    "<EOS>"
+  ],
+  "placeholders": {
+    "URL": "<URL>",
+    "USER": "<USER>"
+  },
+  "protect_regex": {
+    "url": "https?://\\S+",
+    "user": "@[A-Za-z0-9_]+",
+    "time": "\\b\\d{1,2}:\\d{2}\\b",
+    "number": "\\b\\d+(?:\\.\\d+)?\\b",
+    "emote": "(?:(?:[A-Za-z]+){2,}[A-Z]{2,}|[A-Za-z]+(?:LOL|KEKW|LUL|OMEGA)\\b)"
+  },
+  "version": 1
+}