Upload tokenizer.json with huggingface_hub

tokenizer.json

@@ -31,43 +31,7 @@
       "special": true
     },
     {
-      "id": 32007,
-      "content": "▁<PRE>",
-      "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32008,
-      "content": "▁<SUF>",
-      "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32009,
-      "content": "▁<MID>",
-      "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32010,
-      "content": "▁<EOT>",
-      "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32016,
+      "id": 32000,
       "content": "<PAD>",
       "single_word": false,
       "lstrip": false,
@@ -32179,23 +32143,7 @@
       "왕": 31996,
       "收": 31997,
       "弘": 31998,
-      "给": 31999,
-      "▁<SU": 32000,
-      "▁<SUF": 32001,
-      "▁<PRE": 32002,
-      "▁<M": 32003,
-      "▁<MID": 32004,
-      "▁<E": 32005,
-      "▁<EOT": 32006,
-      "▁<PRE>": 32007,
-      "▁<SUF>": 32008,
-      "▁<MID>": 32009,
-      "▁<EOT>": 32010,
-      "▁<EOT><EOT>": 32011,
-      "▁<EOT><EOT><EOT>": 32012,
-      "▁<EOT><EOT><EOT><EOT>": 32013,
-      "▁<EOT><EOT><EOT><EOT><EOT>": 32014,
-      "▁<EOT><EOT><EOT><EOT><EOT><EOT>": 32015
+      "给": 31999
     },
     "merges": [
       "▁ t",
@@ -93446,18 +93394,7 @@
       "▁▁▁▁▁▁▁▁▁ ▁▁▁▁▁▁",
       "▁▁▁▁▁▁▁ ▁▁▁▁▁▁▁▁",
       "▁▁▁▁▁▁▁▁▁▁▁ ▁▁▁▁",
-      "▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
-      "▁< SU",
-      "▁<SU F",
-      "▁< PRE",
-      "▁< M",
-      "▁<M ID",
-      "▁< E",
-      "▁<E OT",
-      "▁<PRE >",
-      "▁<SUF >",
-      "▁<MID >",
-      "▁<EOT >"
+      "▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
     ]
   }
 }