Upload tokenizer.json with huggingface_hub

tokenizer.json

@@ -2,11 +2,18 @@
   "version": "1.0",
   "truncation": {
     "direction": "Right",
-    "max_length": 510,
+    "max_length": 512,
     "strategy": "LongestFirst",
     "stride": 0
   },
-  "padding": null,
+  "padding": {
+    "strategy": "BatchLongest",
+    "direction": "Left",
+    "pad_to_multiple_of": null,
+    "pad_id": 128000,
+    "pad_type_id": 0,
+    "pad_token": "[PAD]"
+  },
   "added_tokens": [
     {
       "id": 5139,
@@ -2343,69 +2350,73 @@
     ]
   },
   "post_processor": {
-    "type": "Sequence",
-    "processors": [
+    "type": "TemplateProcessing",
+    "single": [
       {
-        "type": "ByteLevel",
-        "add_prefix_space": true,
-        "trim_offsets": false,
-        "use_regex": true
+        "SpecialToken": {
+          "id": "<|begin_of_text|>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "<|end_of_text|>",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "<|begin_of_text|>",
+          "type_id": 0
+        }
       },
       {
-        "type": "TemplateProcessing",
-        "single": [
-          {
-            "SpecialToken": {
-              "id": "<|begin_of_text|>",
-              "type_id": 0
-            }
-          },
-          {
-            "Sequence": {
-              "id": "A",
-              "type_id": 0
-            }
-          }
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "<|end_of_text|>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "<|begin_of_text|>": {
+        "id": "<|begin_of_text|>",
+        "ids": [
+          101497
         ],
-        "pair": [
-          {
-            "SpecialToken": {
-              "id": "<|begin_of_text|>",
-              "type_id": 0
-            }
-          },
-          {
-            "Sequence": {
-              "id": "A",
-              "type_id": 0
-            }
-          },
-          {
-            "SpecialToken": {
-              "id": "<|begin_of_text|>",
-              "type_id": 1
-            }
-          },
-          {
-            "Sequence": {
-              "id": "B",
-              "type_id": 1
-            }
-          }
+        "tokens": [
+          "<|begin_of_text|>"
+        ]
+      },
+      "<|end_of_text|>": {
+        "id": "<|end_of_text|>",
+        "ids": [
+          57368
         ],
-        "special_tokens": {
-          "<|begin_of_text|>": {
-            "id": "<|begin_of_text|>",
-            "ids": [
-              128000
-            ],
-            "tokens": [
-              "<|begin_of_text|>"
-            ]
-          }
-        }
+        "tokens": [
+          "<|end_of_text|>"
+        ]
       }
-    ]
+    }
   },
   "decoder": {
     "type": "ByteLevel",