Upload tokenizer

special_tokens_map.json

@@ -1,107 +1,105 @@
 {
   "additional_special_tokens": [
-    "<mask_1>",
-    "<unk_2>",
-    "<unk_3>",
-    "<unk_4>",
-    "<unk_5>",
-    "<unk_6>",
-    "<unk_7>",
-    "<unk_8>",
-    "<unk_9>",
-    "<unk_10>",
-    "<unk_11>",
-    "<unk_12>",
-    "<unk_13>",
-    "<unk_14>",
-    "<unk_15>",
-    "<unk_16>",
-    "<unk_17>",
-    "<unk_18>",
-    "<unk_19>",
-    "<unk_20>",
-    "<unk_21>",
-    "<unk_22>",
-    "<unk_23>",
-    "<unk_24>",
-    "<unk_25>",
-    "<unk_26>",
-    "<unk_27>",
-    "<unk_28>",
-    "<unk_29>",
-    "<unk_30>",
-    "<unk_31>",
-    "<unk_32>",
-    "<unk_33>",
-    "<unk_34>",
-    "<unk_35>",
-    "<unk_36>",
-    "<unk_37>",
-    "<unk_38>",
-    "<unk_39>",
-    "<unk_40>",
-    "<unk_41>",
-    "<unk_42>",
-    "<unk_43>",
-    "<unk_44>",
-    "<unk_45>",
-    "<unk_46>",
-    "<unk_47>",
-    "<unk_48>",
-    "<unk_49>",
-    "<unk_50>",
-    "<unk_51>",
-    "<unk_52>",
-    "<unk_53>",
-    "<unk_54>",
-    "<unk_55>",
-    "<unk_56>",
-    "<unk_57>",
-    "<unk_58>",
-    "<unk_59>",
-    "<unk_60>",
-    "<unk_61>",
-    "<unk_62>",
-    "<unk_63>",
-    "<unk_64>",
-    "<unk_65>",
-    "<unk_66>",
-    "<unk_67>",
-    "<unk_68>",
-    "<unk_69>",
-    "<unk_70>",
-    "<unk_71>",
-    "<unk_72>",
-    "<unk_73>",
-    "<unk_74>",
-    "<unk_75>",
-    "<unk_76>",
-    "<unk_77>",
-    "<unk_78>",
-    "<unk_79>",
-    "<unk_80>",
-    "<unk_81>",
-    "<unk_82>",
-    "<unk_83>",
-    "<unk_84>",
-    "<unk_85>",
-    "<unk_86>",
-    "<unk_87>",
-    "<unk_88>",
-    "<unk_89>",
-    "<unk_90>",
-    "<unk_91>",
-    "<unk_92>",
-    "<unk_93>",
-    "<unk_94>",
-    "<unk_95>",
-    "<unk_96>",
-    "<unk_97>",
-    "<unk_98>",
-    "<unk_99>",
-    "<unk_100>",
-    "<unk_101>",
-    "<unk_102>"
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>"
   ],
   "eos_token": {
     "content": "</s>",
@@ -110,13 +108,6 @@
     "rstrip": false,
     "single_word": false
   },
-  "mask_token": {
-    "content": "<mask_2>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
   "pad_token": {
     "content": "<pad>",
     "lstrip": false,

spiece.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0015189ef36359283fec8b93cf6d9ce51bca37eb1101defc68a53b394913b96c
-size 1912529
+oid sha256:d60acb128cf7b7f2536e8f38a5b18a05535c9e14c7a355904270e15b0945ea86
+size 791656

tokenizer_config.json

@@ -1,4 +1,5 @@
 {
+  "add_prefix_space": true,
   "added_tokens_decoder": {
     "0": {
       "content": "<pad>",
@@ -17,831 +18,807 @@
       "special": true
     },
     "2": {
-      "content": "<mask_1>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "3": {
-      "content": "<mask_2>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "4": {
-      "content": "<unk_2>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "5": {
-      "content": "<unk_3>",
+      "content": "<unk>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "6": {
-      "content": "<unk_4>",
+    "32000": {
+      "content": "<extra_id_99>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "7": {
-      "content": "<unk_5>",
+    "32001": {
+      "content": "<extra_id_98>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "8": {
-      "content": "<unk_6>",
+    "32002": {
+      "content": "<extra_id_97>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "9": {
-      "content": "<unk_7>",
+    "32003": {
+      "content": "<extra_id_96>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "10": {
-      "content": "<unk_8>",
+    "32004": {
+      "content": "<extra_id_95>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "11": {
-      "content": "<unk_9>",
+    "32005": {
+      "content": "<extra_id_94>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "12": {
-      "content": "<unk_10>",
+    "32006": {
+      "content": "<extra_id_93>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "13": {
-      "content": "<unk_11>",
+    "32007": {
+      "content": "<extra_id_92>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "14": {
-      "content": "<unk_12>",
+    "32008": {
+      "content": "<extra_id_91>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "15": {
-      "content": "<unk_13>",
+    "32009": {
+      "content": "<extra_id_90>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "16": {
-      "content": "<unk_14>",
+    "32010": {
+      "content": "<extra_id_89>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "17": {
-      "content": "<unk_15>",
+    "32011": {
+      "content": "<extra_id_88>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "18": {
-      "content": "<unk_16>",
+    "32012": {
+      "content": "<extra_id_87>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "19": {
-      "content": "<unk_17>",
+    "32013": {
+      "content": "<extra_id_86>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "20": {
-      "content": "<unk_18>",
+    "32014": {
+      "content": "<extra_id_85>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "21": {
-      "content": "<unk_19>",
+    "32015": {
+      "content": "<extra_id_84>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "22": {
-      "content": "<unk_20>",
+    "32016": {
+      "content": "<extra_id_83>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "23": {
-      "content": "<unk_21>",
+    "32017": {
+      "content": "<extra_id_82>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "24": {
-      "content": "<unk_22>",
+    "32018": {
+      "content": "<extra_id_81>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "25": {
-      "content": "<unk_23>",
+    "32019": {
+      "content": "<extra_id_80>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "26": {
-      "content": "<unk_24>",
+    "32020": {
+      "content": "<extra_id_79>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "27": {
-      "content": "<unk_25>",
+    "32021": {
+      "content": "<extra_id_78>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "28": {
-      "content": "<unk_26>",
+    "32022": {
+      "content": "<extra_id_77>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "29": {
-      "content": "<unk_27>",
+    "32023": {
+      "content": "<extra_id_76>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "30": {
-      "content": "<unk_28>",
+    "32024": {
+      "content": "<extra_id_75>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "31": {
-      "content": "<unk_29>",
+    "32025": {
+      "content": "<extra_id_74>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "32": {
-      "content": "<unk_30>",
+    "32026": {
+      "content": "<extra_id_73>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "33": {
-      "content": "<unk_31>",
+    "32027": {
+      "content": "<extra_id_72>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "34": {
-      "content": "<unk_32>",
+    "32028": {
+      "content": "<extra_id_71>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "35": {
-      "content": "<unk_33>",
+    "32029": {
+      "content": "<extra_id_70>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "36": {
-      "content": "<unk_34>",
+    "32030": {
+      "content": "<extra_id_69>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "37": {
-      "content": "<unk_35>",
+    "32031": {
+      "content": "<extra_id_68>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "38": {
-      "content": "<unk_36>",
+    "32032": {
+      "content": "<extra_id_67>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "39": {
-      "content": "<unk_37>",
+    "32033": {
+      "content": "<extra_id_66>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "40": {
-      "content": "<unk_38>",
+    "32034": {
+      "content": "<extra_id_65>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "41": {
-      "content": "<unk_39>",
+    "32035": {
+      "content": "<extra_id_64>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "42": {
-      "content": "<unk_40>",
+    "32036": {
+      "content": "<extra_id_63>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "43": {
-      "content": "<unk_41>",
+    "32037": {
+      "content": "<extra_id_62>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "44": {
-      "content": "<unk_42>",
+    "32038": {
+      "content": "<extra_id_61>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "45": {
-      "content": "<unk_43>",
+    "32039": {
+      "content": "<extra_id_60>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "46": {
-      "content": "<unk_44>",
+    "32040": {
+      "content": "<extra_id_59>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "47": {
-      "content": "<unk_45>",
+    "32041": {
+      "content": "<extra_id_58>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "48": {
-      "content": "<unk_46>",
+    "32042": {
+      "content": "<extra_id_57>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "49": {
-      "content": "<unk_47>",
+    "32043": {
+      "content": "<extra_id_56>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "50": {
-      "content": "<unk_48>",
+    "32044": {
+      "content": "<extra_id_55>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "51": {
-      "content": "<unk_49>",
+    "32045": {
+      "content": "<extra_id_54>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "52": {
-      "content": "<unk_50>",
+    "32046": {
+      "content": "<extra_id_53>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "53": {
-      "content": "<unk_51>",
+    "32047": {
+      "content": "<extra_id_52>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "54": {
-      "content": "<unk_52>",
+    "32048": {
+      "content": "<extra_id_51>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "55": {
-      "content": "<unk_53>",
+    "32049": {
+      "content": "<extra_id_50>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "56": {
-      "content": "<unk_54>",
+    "32050": {
+      "content": "<extra_id_49>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "57": {
-      "content": "<unk_55>",
+    "32051": {
+      "content": "<extra_id_48>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "58": {
-      "content": "<unk_56>",
+    "32052": {
+      "content": "<extra_id_47>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "59": {
-      "content": "<unk_57>",
+    "32053": {
+      "content": "<extra_id_46>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "60": {
-      "content": "<unk_58>",
+    "32054": {
+      "content": "<extra_id_45>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "61": {
-      "content": "<unk_59>",
+    "32055": {
+      "content": "<extra_id_44>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "62": {
-      "content": "<unk_60>",
+    "32056": {
+      "content": "<extra_id_43>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "63": {
-      "content": "<unk_61>",
+    "32057": {
+      "content": "<extra_id_42>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "64": {
-      "content": "<unk_62>",
+    "32058": {
+      "content": "<extra_id_41>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "65": {
-      "content": "<unk_63>",
+    "32059": {
+      "content": "<extra_id_40>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "66": {
-      "content": "<unk_64>",
+    "32060": {
+      "content": "<extra_id_39>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "67": {
-      "content": "<unk_65>",
+    "32061": {
+      "content": "<extra_id_38>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "68": {
-      "content": "<unk_66>",
+    "32062": {
+      "content": "<extra_id_37>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "69": {
-      "content": "<unk_67>",
+    "32063": {
+      "content": "<extra_id_36>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "70": {
-      "content": "<unk_68>",
+    "32064": {
+      "content": "<extra_id_35>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "71": {
-      "content": "<unk_69>",
+    "32065": {
+      "content": "<extra_id_34>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "72": {
-      "content": "<unk_70>",
+    "32066": {
+      "content": "<extra_id_33>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "73": {
-      "content": "<unk_71>",
+    "32067": {
+      "content": "<extra_id_32>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "74": {
-      "content": "<unk_72>",
+    "32068": {
+      "content": "<extra_id_31>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "75": {
-      "content": "<unk_73>",
+    "32069": {
+      "content": "<extra_id_30>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "76": {
-      "content": "<unk_74>",
+    "32070": {
+      "content": "<extra_id_29>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "77": {
-      "content": "<unk_75>",
+    "32071": {
+      "content": "<extra_id_28>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "78": {
-      "content": "<unk_76>",
+    "32072": {
+      "content": "<extra_id_27>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "79": {
-      "content": "<unk_77>",
+    "32073": {
+      "content": "<extra_id_26>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "80": {
-      "content": "<unk_78>",
+    "32074": {
+      "content": "<extra_id_25>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "81": {
-      "content": "<unk_79>",
+    "32075": {
+      "content": "<extra_id_24>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "82": {
-      "content": "<unk_80>",
+    "32076": {
+      "content": "<extra_id_23>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "83": {
-      "content": "<unk_81>",
+    "32077": {
+      "content": "<extra_id_22>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "84": {
-      "content": "<unk_82>",
+    "32078": {
+      "content": "<extra_id_21>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "85": {
-      "content": "<unk_83>",
+    "32079": {
+      "content": "<extra_id_20>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "86": {
-      "content": "<unk_84>",
+    "32080": {
+      "content": "<extra_id_19>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "87": {
-      "content": "<unk_85>",
+    "32081": {
+      "content": "<extra_id_18>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "88": {
-      "content": "<unk_86>",
+    "32082": {
+      "content": "<extra_id_17>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "89": {
-      "content": "<unk_87>",
+    "32083": {
+      "content": "<extra_id_16>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "90": {
-      "content": "<unk_88>",
+    "32084": {
+      "content": "<extra_id_15>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "91": {
-      "content": "<unk_89>",
+    "32085": {
+      "content": "<extra_id_14>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "92": {
-      "content": "<unk_90>",
+    "32086": {
+      "content": "<extra_id_13>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "93": {
-      "content": "<unk_91>",
+    "32087": {
+      "content": "<extra_id_12>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "94": {
-      "content": "<unk_92>",
+    "32088": {
+      "content": "<extra_id_11>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "95": {
-      "content": "<unk_93>",
+    "32089": {
+      "content": "<extra_id_10>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "96": {
-      "content": "<unk_94>",
+    "32090": {
+      "content": "<extra_id_9>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "97": {
-      "content": "<unk_95>",
+    "32091": {
+      "content": "<extra_id_8>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "98": {
-      "content": "<unk_96>",
+    "32092": {
+      "content": "<extra_id_7>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "99": {
-      "content": "<unk_97>",
+    "32093": {
+      "content": "<extra_id_6>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "100": {
-      "content": "<unk_98>",
+    "32094": {
+      "content": "<extra_id_5>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "101": {
-      "content": "<unk_99>",
+    "32095": {
+      "content": "<extra_id_4>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "102": {
-      "content": "<unk_100>",
+    "32096": {
+      "content": "<extra_id_3>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "103": {
-      "content": "<unk_101>",
+    "32097": {
+      "content": "<extra_id_2>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "104": {
-      "content": "<unk_102>",
+    "32098": {
+      "content": "<extra_id_1>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "105": {
-      "content": "<unk>",
+    "32099": {
+      "content": "<extra_id_0>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -850,126 +827,115 @@
     }
   },
   "additional_special_tokens": [
-    "<mask_1>",
-    "<unk_2>",
-    "<unk_3>",
-    "<unk_4>",
-    "<unk_5>",
-    "<unk_6>",
-    "<unk_7>",
-    "<unk_8>",
-    "<unk_9>",
-    "<unk_10>",
-    "<unk_11>",
-    "<unk_12>",
-    "<unk_13>",
-    "<unk_14>",
-    "<unk_15>",
-    "<unk_16>",
-    "<unk_17>",
-    "<unk_18>",
-    "<unk_19>",
-    "<unk_20>",
-    "<unk_21>",
-    "<unk_22>",
-    "<unk_23>",
-    "<unk_24>",
-    "<unk_25>",
-    "<unk_26>",
-    "<unk_27>",
-    "<unk_28>",
-    "<unk_29>",
-    "<unk_30>",
-    "<unk_31>",
-    "<unk_32>",
-    "<unk_33>",
-    "<unk_34>",
-    "<unk_35>",
-    "<unk_36>",
-    "<unk_37>",
-    "<unk_38>",
-    "<unk_39>",
-    "<unk_40>",
-    "<unk_41>",
-    "<unk_42>",
-    "<unk_43>",
-    "<unk_44>",
-    "<unk_45>",
-    "<unk_46>",
-    "<unk_47>",
-    "<unk_48>",
-    "<unk_49>",
-    "<unk_50>",
-    "<unk_51>",
-    "<unk_52>",
-    "<unk_53>",
-    "<unk_54>",
-    "<unk_55>",
-    "<unk_56>",
-    "<unk_57>",
-    "<unk_58>",
-    "<unk_59>",
-    "<unk_60>",
-    "<unk_61>",
-    "<unk_62>",
-    "<unk_63>",
-    "<unk_64>",
-    "<unk_65>",
-    "<unk_66>",
-    "<unk_67>",
-    "<unk_68>",
-    "<unk_69>",
-    "<unk_70>",
-    "<unk_71>",
-    "<unk_72>",
-    "<unk_73>",
-    "<unk_74>",
-    "<unk_75>",
-    "<unk_76>",
-    "<unk_77>",
-    "<unk_78>",
-    "<unk_79>",
-    "<unk_80>",
-    "<unk_81>",
-    "<unk_82>",
-    "<unk_83>",
-    "<unk_84>",
-    "<unk_85>",
-    "<unk_86>",
-    "<unk_87>",
-    "<unk_88>",
-    "<unk_89>",
-    "<unk_90>",
-    "<unk_91>",
-    "<unk_92>",
-    "<unk_93>",
-    "<unk_94>",
-    "<unk_95>",
-    "<unk_96>",
-    "<unk_97>",
-    "<unk_98>",
-    "<unk_99>",
-    "<unk_100>",
-    "<unk_101>",
-    "<unk_102>"
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>"
   ],
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
+  "extra_ids": 100,
   "extra_special_tokens": {},
-  "full_tokenizer_file": null,
-  "mask_token": "<mask_2>",
-  "mask_token_sent": "<mask_1>",
-  "max_length": 128,
-  "model_max_length": 1024,
-  "offset": 103,
-  "pad_to_multiple_of": null,
+  "legacy": true,
+  "model_max_length": 512,
   "pad_token": "<pad>",
-  "pad_token_type_id": 0,
-  "padding_side": "right",
   "sp_model_kwargs": {},
-  "stride": 0,
-  "tokenizer_class": "PegasusTokenizer",
-  "truncation_side": "right",
-  "truncation_strategy": "longest_first",
+  "tokenizer_class": "T5Tokenizer",
   "unk_token": "<unk>"
 }