Upload tokenizer.json with huggingface_hub

tokenizer.json

@@ -3,19 +3,40 @@
   "truncation": null,
   "padding": null,
   "added_tokens": [],
-  "normalizer": null,
-  "pre_tokenizer": null,
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "NFD"
+      },
+      {
+        "type": "Lowercase"
+      },
+      {
+        "type": "StripAccents"
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
   "post_processor": null,
   "decoder": null,
   "model": {
-    "type": "unigram",
-    "vocab": [
-      {
-        "id": 0,
-        "piece": "<unk>",
-        "score": 0.0
-      }
-    ],
-    "unk_id": 0
+    "type": "WordPiece",
+    "vocab": {
+      "[PAD]": 0,
+      "[UNK]": 1,
+      "a": 2,
+      "the": 3,
+      "cat": 4,
+      "dog": 5,
+      "house": 6,
+      "mountain": 7,
+      "landscape": 8,
+      "beautiful": 9,
+      "sunset": 10
+    },
+    "unk_token": "[UNK]"
   }
 }