Upload tokenizer

README.md

@@ -1,8 +1,8 @@
 ---
 license: mit
-base_model: facebook/mbart-large-50
 tags:
 - generated_from_trainer
+base_model: facebook/mbart-large-50
 model-index:
 - name: mBART_try_v1
   results: []

special_tokens_map.json

@@ -59,7 +59,7 @@
   "mask_token": {
     "content": "<mask>",
     "lstrip": true,
-    "normalized": false,
+    "normalized": true,
     "rstrip": false,
     "single_word": false
   },

tokenizer_config.json

@@ -233,15 +233,15 @@
       "special": true
     },
     "250026": {
-      "content": "<mask>",
-      "lstrip": true,
+      "content": "af_ZA",
+      "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
     "250027": {
-      "content": "af_ZA",
+      "content": "az_AZ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -249,7 +249,7 @@
       "special": true
     },
     "250028": {
-      "content": "az_AZ",
+      "content": "bn_IN",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -257,7 +257,7 @@
       "special": true
     },
     "250029": {
-      "content": "bn_IN",
+      "content": "fa_IR",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -265,7 +265,7 @@
       "special": true
     },
     "250030": {
-      "content": "fa_IR",
+      "content": "he_IL",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -273,7 +273,7 @@
       "special": true
     },
     "250031": {
-      "content": "he_IL",
+      "content": "hr_HR",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -281,7 +281,7 @@
       "special": true
     },
     "250032": {
-      "content": "hr_HR",
+      "content": "id_ID",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -289,7 +289,7 @@
       "special": true
     },
     "250033": {
-      "content": "id_ID",
+      "content": "ka_GE",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -297,7 +297,7 @@
       "special": true
     },
     "250034": {
-      "content": "ka_GE",
+      "content": "km_KH",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -305,7 +305,7 @@
       "special": true
     },
     "250035": {
-      "content": "km_KH",
+      "content": "mk_MK",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -313,7 +313,7 @@
       "special": true
     },
     "250036": {
-      "content": "mk_MK",
+      "content": "ml_IN",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -321,7 +321,7 @@
       "special": true
     },
     "250037": {
-      "content": "ml_IN",
+      "content": "mn_MN",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -329,7 +329,7 @@
       "special": true
     },
     "250038": {
-      "content": "mn_MN",
+      "content": "mr_IN",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -337,7 +337,7 @@
       "special": true
     },
     "250039": {
-      "content": "mr_IN",
+      "content": "pl_PL",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -345,7 +345,7 @@
       "special": true
     },
     "250040": {
-      "content": "pl_PL",
+      "content": "ps_AF",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -353,7 +353,7 @@
       "special": true
     },
     "250041": {
-      "content": "ps_AF",
+      "content": "pt_XX",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -361,7 +361,7 @@
       "special": true
     },
     "250042": {
-      "content": "pt_XX",
+      "content": "sv_SE",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -369,7 +369,7 @@
       "special": true
     },
     "250043": {
-      "content": "sv_SE",
+      "content": "sw_KE",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -377,7 +377,7 @@
       "special": true
     },
     "250044": {
-      "content": "sw_KE",
+      "content": "ta_IN",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -385,7 +385,7 @@
       "special": true
     },
     "250045": {
-      "content": "ta_IN",
+      "content": "te_IN",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -393,7 +393,7 @@
       "special": true
     },
     "250046": {
-      "content": "te_IN",
+      "content": "th_TH",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -401,7 +401,7 @@
       "special": true
     },
     "250047": {
-      "content": "th_TH",
+      "content": "tl_XX",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -409,7 +409,7 @@
       "special": true
     },
     "250048": {
-      "content": "tl_XX",
+      "content": "uk_UA",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -417,7 +417,7 @@
       "special": true
     },
     "250049": {
-      "content": "uk_UA",
+      "content": "ur_PK",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -425,7 +425,7 @@
       "special": true
     },
     "250050": {
-      "content": "ur_PK",
+      "content": "xh_ZA",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -433,7 +433,7 @@
       "special": true
     },
     "250051": {
-      "content": "xh_ZA",
+      "content": "gl_ES",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -441,7 +441,7 @@
       "special": true
     },
     "250052": {
-      "content": "gl_ES",
+      "content": "sl_SI",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -449,9 +449,9 @@
       "special": true
     },
     "250053": {
-      "content": "sl_SI",
-      "lstrip": false,
-      "normalized": false,
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
@@ -522,6 +522,6 @@
   "sp_model_kwargs": {},
   "src_lang": "en_XX",
   "tgt_lang": null,
-  "tokenizer_class": "MBartTokenizer",
+  "tokenizer_class": "MBart50Tokenizer",
   "unk_token": "<unk>"
 }