Upload tokenizer

tokenizer.json

@@ -7370,14 +7370,62 @@
   "pre_tokenizer": {
     "type": "Metaspace",
     "replacement": "▁",
-    "prepend_scheme": "always",
+    "prepend_scheme": "first",
     "split": false
   },
   "post_processor": {
-    "type": "ByteLevel",
-    "add_prefix_space": true,
-    "trim_offsets": false,
-    "use_regex": true
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "<｜begin▁of▁sentence｜>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "<｜begin▁of▁sentence｜>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "<｜begin▁of▁sentence｜>",
+          "type_id": 1
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "<｜begin▁of▁sentence｜>": {
+        "id": "<｜begin▁of▁sentence｜>",
+        "ids": [
+          0
+        ],
+        "tokens": [
+          "<｜begin▁of▁sentence｜>"
+        ]
+      }
+    }
   },
   "decoder": {
     "type": "Sequence",

tokenizer_config.json

@@ -4,7 +4,7 @@
   "bos_token": "<｜begin▁of▁sentence｜>",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<｜end▁of▁sentence｜>",
-  "is_local": false,
+  "is_local": true,
   "model_max_length": 131072,
   "pad_token": "<｜end▁of▁sentence｜>",
   "sp_model_kwargs": {},