Upload tokenizer.json with huggingface_hub

tokenizer.json

@@ -10,7 +10,7 @@
     "strategy": {
       "Fixed": 4096
     },
-    "direction": "Left",
+    "direction": "Right",
     "pad_to_multiple_of": null,
     "pad_id": 32021,
     "pad_type_id": 0,
@@ -268,58 +268,10 @@
     ]
   },
   "post_processor": {
-    "type": "TemplateProcessing",
-    "single": [
-      {
-        "SpecialToken": {
-          "id": "<｜begin▁of▁sentence｜>",
-          "type_id": 0
-        }
-      },
-      {
-        "Sequence": {
-          "id": "A",
-          "type_id": 0
-        }
-      }
-    ],
-    "pair": [
-      {
-        "SpecialToken": {
-          "id": "<｜begin▁of▁sentence｜>",
-          "type_id": 0
-        }
-      },
-      {
-        "Sequence": {
-          "id": "A",
-          "type_id": 0
-        }
-      },
-      {
-        "SpecialToken": {
-          "id": "<｜begin▁of▁sentence｜>",
-          "type_id": 1
-        }
-      },
-      {
-        "Sequence": {
-          "id": "B",
-          "type_id": 1
-        }
-      }
-    ],
-    "special_tokens": {
-      "<｜begin▁of▁sentence｜>": {
-        "id": "<｜begin▁of▁sentence｜>",
-        "ids": [
-          32013
-        ],
-        "tokens": [
-          "<｜begin▁of▁sentence｜>"
-        ]
-      }
-    }
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": false,
+    "use_regex": true
   },
   "decoder": {
     "type": "ByteLevel",