Upload tokenizer.json

tokenizer.json

@@ -184,7 +184,17 @@
       "special": true
     }
   ],
-  "normalizer": null,
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "NFKC"
+      },
+      {
+        "type": "Lowercase"
+      }
+    ]
+  },
   "pre_tokenizer": {
     "type": "ByteLevel",
     "add_prefix_space": false,
@@ -192,10 +202,19 @@
     "use_regex": true
   },
   "post_processor": {
-    "type": "ByteLevel",
-    "add_prefix_space": true,
-    "trim_offsets": false,
-    "use_regex": true
+    "type": "TemplateProcessing",
+    "single": "<|im_start|> $A <|im_end|>",
+    "pair": "<|im_start|> $A <|im_end|> <|im_start|> $B <|im_end|>",
+    "special_tokens": [
+      [
+        "<|im_start|>",
+        1
+      ],
+      [
+        "<|im_end|>",
+        2
+      ]
+    ]
   },
   "decoder": {
     "type": "ByteLevel",
@@ -207,10 +226,10 @@
     "type": "BPE",
     "dropout": null,
     "unk_token": null,
-    "continuing_subword_prefix": "",
+    "continuing_subword_prefix": "##",
     "end_of_word_suffix": "",
     "fuse_unk": false,
-    "byte_fallback": false,
+    "byte_fallback": true,
     "ignore_merges": false,
     "vocab": {
       "<|endoftext|>": 0,