Upload tokenizer

added_tokens.json

@@ -0,0 +1,288 @@
+{
+  "<|af|>": 50591,
+  "<|am|>": 50598,
+  "<|ar|>": 50536,
+  "<|as|>": 50614,
+  "<|az|>": 50568,
+  "<|ba|>": 50619,
+  "<|be|>": 50594,
+  "<|bg|>": 50556,
+  "<|bn|>": 50566,
+  "<|bo|>": 50611,
+  "<|br|>": 50573,
+  "<|bs|>": 50579,
+  "<|ca|>": 50534,
+  "<|cs|>": 50547,
+  "<|cy|>": 50561,
+  "<|da|>": 50549,
+  "<|de|>": 50525,
+  "<|el|>": 50545,
+  "<|endoftext|>": 50628,
+  "<|en|>": 50523,
+  "<|es|>": 50526,
+  "<|et|>": 50571,
+  "<|eu|>": 50574,
+  "<|fa|>": 50564,
+  "<|fi|>": 50541,
+  "<|fo|>": 50602,
+  "<|fr|>": 50529,
+  "<|gl|>": 50583,
+  "<|gu|>": 50597,
+  "<|haw|>": 50616,
+  "<|ha|>": 50618,
+  "<|he|>": 50543,
+  "<|hi|>": 50540,
+  "<|hr|>": 50555,
+  "<|ht|>": 50603,
+  "<|hu|>": 50550,
+  "<|hy|>": 50576,
+  "<|id|>": 50539,
+  "<|is|>": 50575,
+  "<|it|>": 50538,
+  "<|ja|>": 50530,
+  "<|jw|>": 50620,
+  "<|ka|>": 50593,
+  "<|kk|>": 50580,
+  "<|km|>": 50587,
+  "<|kn|>": 50570,
+  "<|ko|>": 50528,
+  "<|la|>": 50558,
+  "<|lb|>": 50609,
+  "<|ln|>": 50617,
+  "<|lo|>": 50600,
+  "<|lt|>": 50557,
+  "<|lv|>": 50565,
+  "<|mg|>": 50613,
+  "<|mi|>": 50559,
+  "<|mk|>": 50572,
+  "<|ml|>": 50560,
+  "<|mn|>": 50578,
+  "<|mr|>": 50584,
+  "<|ms|>": 50546,
+  "<|mt|>": 50607,
+  "<|my|>": 50610,
+  "<|ne|>": 50577,
+  "<|nl|>": 50535,
+  "<|nn|>": 50606,
+  "<|nocaptions|>": 50626,
+  "<|notimestamps|>": 50627,
+  "<|no|>": 50552,
+  "<|oc|>": 50592,
+  "<|pa|>": 50585,
+  "<|pl|>": 50533,
+  "<|ps|>": 50604,
+  "<|pt|>": 50531,
+  "<|ro|>": 50548,
+  "<|ru|>": 50527,
+  "<|sa|>": 50608,
+  "<|sd|>": 50596,
+  "<|si|>": 50586,
+  "<|sk|>": 50562,
+  "<|sl|>": 50569,
+  "<|sn|>": 50588,
+  "<|so|>": 50590,
+  "<|sq|>": 50581,
+  "<|sr|>": 50567,
+  "<|startoflm|>": 50624,
+  "<|startofprev|>": 50625,
+  "<|startoftranscript|>": 50522,
+  "<|su|>": 50621,
+  "<|sv|>": 50537,
+  "<|sw|>": 50582,
+  "<|ta|>": 50551,
+  "<|te|>": 50563,
+  "<|tg|>": 50595,
+  "<|th|>": 50553,
+  "<|tk|>": 50605,
+  "<|tl|>": 50612,
+  "<|transcribe|>": 50623,
+  "<|translate|>": 50622,
+  "<|tr|>": 50532,
+  "<|tt|>": 50615,
+  "<|uk|>": 50544,
+  "<|ur|>": 50554,
+  "<|uz|>": 50601,
+  "<|vi|>": 50542,
+  "<|yi|>": 50599,
+  "<|yo|>": 50589,
+  "<|zh|>": 50524,
+  "<|¡|>": 50629,
+  "<|¢|>": 50630,
+  "<|£|>": 50631,
+  "<|¤|>": 50632,
+  "<|¥|>": 50633,
+  "<|¦|>": 50634,
+  "<|§|>": 50635,
+  "<|¨|>": 50636,
+  "<|©|>": 50637,
+  "<|ª|>": 50638,
+  "<|«|>": 50639,
+  "<|¬|>": 50640,
+  "<|®|>": 50641,
+  "<|¯|>": 50642,
+  "<|°|>": 50643,
+  "<|±|>": 50644,
+  "<|²|>": 50645,
+  "<|³|>": 50646,
+  "<|´|>": 50647,
+  "<|µ|>": 50648,
+  "<|¶|>": 50649,
+  "<|·|>": 50650,
+  "<|¸|>": 50651,
+  "<|¹|>": 50652,
+  "<|º|>": 50653,
+  "<|»|>": 50654,
+  "<|¼|>": 50655,
+  "<|½|>": 50656,
+  "<|¾|>": 50657,
+  "<|¿|>": 50658,
+  "<|À|>": 50659,
+  "<|Á|>": 50660,
+  "<|Â|>": 50661,
+  "<|Ã|>": 50662,
+  "<|Ä|>": 50663,
+  "<|Å|>": 50664,
+  "<|Æ|>": 50665,
+  "<|Ç|>": 50666,
+  "<|È|>": 50667,
+  "<|É|>": 50668,
+  "<|Ê|>": 50669,
+  "<|Ë|>": 50670,
+  "<|Ì|>": 50671,
+  "<|Í|>": 50672,
+  "<|Î|>": 50673,
+  "<|Ï|>": 50674,
+  "<|Ð|>": 50675,
+  "<|Ñ|>": 50676,
+  "<|Ò|>": 50677,
+  "<|Ó|>": 50678,
+  "<|Ô|>": 50679,
+  "<|Õ|>": 50680,
+  "<|Ö|>": 50681,
+  "<|×|>": 50682,
+  "<|Ø|>": 50683,
+  "<|ا|>": 50793,
+  "<|اÙĨ|>": 50801,
+  "<|ت|>": 50800,
+  "<|د|>": 50803,
+  "<|ر|>": 50797,
+  "<|س|>": 50807,
+  "<|Ù|>": 50684,
+  "<|Ùħ|>": 50804,
+  "<|ÙĨ|>": 50796,
+  "<|ÙĪ|>": 50795,
+  "<|Ú|>": 50685,
+  "<|Ú©|>": 50798,
+  "<|Û|>": 50686,
+  "<|ÛĨ|>": 50806,
+  "<|ÛĮ|>": 50792,
+  "<|Ûİ|>": 50802,
+  "<|Ûķ|>": 50791,
+  "<|Ü|>": 50687,
+  "<|Ý|>": 50688,
+  "<|Þ|>": 50689,
+  "<|ß|>": 50690,
+  "<|à|>": 50691,
+  "<|á|>": 50692,
+  "<|â|>": 50693,
+  "<|ã|>": 50694,
+  "<|ä|>": 50695,
+  "<|å|>": 50696,
+  "<|æ|>": 50697,
+  "<|ç|>": 50698,
+  "<|è|>": 50699,
+  "<|é|>": 50700,
+  "<|ê|>": 50701,
+  "<|ë|>": 50702,
+  "<|ì|>": 50703,
+  "<|í|>": 50704,
+  "<|î|>": 50705,
+  "<|ï|>": 50706,
+  "<|ð|>": 50707,
+  "<|ñ|>": 50708,
+  "<|ò|>": 50709,
+  "<|ó|>": 50710,
+  "<|ô|>": 50711,
+  "<|õ|>": 50712,
+  "<|ö|>": 50713,
+  "<|÷|>": 50714,
+  "<|ø|>": 50715,
+  "<|ù|>": 50716,
+  "<|ú|>": 50717,
+  "<|û|>": 50718,
+  "<|ü|>": 50719,
+  "<|ý|>": 50720,
+  "<|þ|>": 50721,
+  "<|ÿ|>": 50722,
+  "<|Ā|>": 50723,
+  "<|ā|>": 50724,
+  "<|Ă|>": 50725,
+  "<|ă|>": 50726,
+  "<|Ą|>": 50727,
+  "<|ą|>": 50728,
+  "<|Ć|>": 50729,
+  "<|ć|>": 50730,
+  "<|Ĉ|>": 50731,
+  "<|ĉ|>": 50732,
+  "<|Ċ|>": 50733,
+  "<|ċ|>": 50734,
+  "<|Č|>": 50735,
+  "<|č|>": 50736,
+  "<|Ď|>": 50737,
+  "<|ď|>": 50738,
+  "<|Đ|>": 50739,
+  "<|đ|>": 50740,
+  "<|Ē|>": 50741,
+  "<|ē|>": 50742,
+  "<|Ĕ|>": 50743,
+  "<|ĕ|>": 50744,
+  "<|Ė|>": 50745,
+  "<|ė|>": 50746,
+  "<|Ę|>": 50747,
+  "<|ę|>": 50748,
+  "<|Ě|>": 50749,
+  "<|ě|>": 50750,
+  "<|Ĝ|>": 50751,
+  "<|ĝ|>": 50752,
+  "<|Ğ|>": 50753,
+  "<|ğ|>": 50754,
+  "<|Ġ|>": 50755,
+  "<|ĠØ|>": 50794,
+  "<|Ġب|>": 50805,
+  "<|ĠÙ|>": 50799,
+  "<|ġ|>": 50756,
+  "<|Ģ|>": 50757,
+  "<|ģ|>": 50758,
+  "<|Ĥ|>": 50759,
+  "<|ĥ|>": 50760,
+  "<|Ħ|>": 50761,
+  "<|ħ|>": 50762,
+  "<|Ĩ|>": 50763,
+  "<|ĩ|>": 50764,
+  "<|Ī|>": 50765,
+  "<|ī|>": 50766,
+  "<|Ĭ|>": 50767,
+  "<|ĭ|>": 50768,
+  "<|Į|>": 50769,
+  "<|į|>": 50770,
+  "<|İ|>": 50771,
+  "<|ı|>": 50772,
+  "<|IJ|>": 50773,
+  "<|ij|>": 50774,
+  "<|Ĵ|>": 50775,
+  "<|ĵ|>": 50776,
+  "<|Ķ|>": 50777,
+  "<|ķ|>": 50778,
+  "<|ĸ|>": 50779,
+  "<|Ĺ|>": 50780,
+  "<|ĺ|>": 50781,
+  "<|Ļ|>": 50782,
+  "<|ļ|>": 50783,
+  "<|Ľ|>": 50784,
+  "<|ľ|>": 50785,
+  "<|Ŀ|>": 50786,
+  "<|ŀ|>": 50787,
+  "<|Ł|>": 50788,
+  "<|ł|>": 50789,
+  "<|Ń|>": 50790
+}

special_tokens_map.json

@@ -0,0 +1,219 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|startoftranscript|>",
+    "<|¡|>",
+    "<|¢|>",
+    "<|£|>",
+    "<|¤|>",
+    "<|¥|>",
+    "<|¦|>",
+    "<|§|>",
+    "<|¨|>",
+    "<|©|>",
+    "<|ª|>",
+    "<|«|>",
+    "<|¬|>",
+    "<|®|>",
+    "<|¯|>",
+    "<|°|>",
+    "<|±|>",
+    "<|²|>",
+    "<|³|>",
+    "<|´|>",
+    "<|µ|>",
+    "<|¶|>",
+    "<|·|>",
+    "<|¸|>",
+    "<|¹|>",
+    "<|º|>",
+    "<|»|>",
+    "<|¼|>",
+    "<|½|>",
+    "<|¾|>",
+    "<|¿|>",
+    "<|À|>",
+    "<|Á|>",
+    "<|Â|>",
+    "<|Ã|>",
+    "<|Ä|>",
+    "<|Å|>",
+    "<|Æ|>",
+    "<|Ç|>",
+    "<|È|>",
+    "<|É|>",
+    "<|Ê|>",
+    "<|Ë|>",
+    "<|Ì|>",
+    "<|Í|>",
+    "<|Î|>",
+    "<|Ï|>",
+    "<|Ð|>",
+    "<|Ñ|>",
+    "<|Ò|>",
+    "<|Ó|>",
+    "<|Ô|>",
+    "<|Õ|>",
+    "<|Ö|>",
+    "<|×|>",
+    "<|Ø|>",
+    "<|Ù|>",
+    "<|Ú|>",
+    "<|Û|>",
+    "<|Ü|>",
+    "<|Ý|>",
+    "<|Þ|>",
+    "<|ß|>",
+    "<|à|>",
+    "<|á|>",
+    "<|â|>",
+    "<|ã|>",
+    "<|ä|>",
+    "<|å|>",
+    "<|æ|>",
+    "<|ç|>",
+    "<|è|>",
+    "<|é|>",
+    "<|ê|>",
+    "<|ë|>",
+    "<|ì|>",
+    "<|í|>",
+    "<|î|>",
+    "<|ï|>",
+    "<|ð|>",
+    "<|ñ|>",
+    "<|ò|>",
+    "<|ó|>",
+    "<|ô|>",
+    "<|õ|>",
+    "<|ö|>",
+    "<|÷|>",
+    "<|ø|>",
+    "<|ù|>",
+    "<|ú|>",
+    "<|û|>",
+    "<|ü|>",
+    "<|ý|>",
+    "<|þ|>",
+    "<|ÿ|>",
+    "<|Ā|>",
+    "<|ā|>",
+    "<|Ă|>",
+    "<|ă|>",
+    "<|Ą|>",
+    "<|ą|>",
+    "<|Ć|>",
+    "<|ć|>",
+    "<|Ĉ|>",
+    "<|ĉ|>",
+    "<|Ċ|>",
+    "<|ċ|>",
+    "<|Č|>",
+    "<|č|>",
+    "<|Ď|>",
+    "<|ď|>",
+    "<|Đ|>",
+    "<|đ|>",
+    "<|Ē|>",
+    "<|ē|>",
+    "<|Ĕ|>",
+    "<|ĕ|>",
+    "<|Ė|>",
+    "<|ė|>",
+    "<|Ę|>",
+    "<|ę|>",
+    "<|Ě|>",
+    "<|ě|>",
+    "<|Ĝ|>",
+    "<|ĝ|>",
+    "<|Ğ|>",
+    "<|ğ|>",
+    "<|Ġ|>",
+    "<|ġ|>",
+    "<|Ģ|>",
+    "<|ģ|>",
+    "<|Ĥ|>",
+    "<|ĥ|>",
+    "<|Ħ|>",
+    "<|ħ|>",
+    "<|Ĩ|>",
+    "<|ĩ|>",
+    "<|Ī|>",
+    "<|ī|>",
+    "<|Ĭ|>",
+    "<|ĭ|>",
+    "<|Į|>",
+    "<|į|>",
+    "<|İ|>",
+    "<|ı|>",
+    "<|IJ|>",
+    "<|ij|>",
+    "<|Ĵ|>",
+    "<|ĵ|>",
+    "<|Ķ|>",
+    "<|ķ|>",
+    "<|ĸ|>",
+    "<|Ĺ|>",
+    "<|ĺ|>",
+    "<|Ļ|>",
+    "<|ļ|>",
+    "<|Ľ|>",
+    "<|ľ|>",
+    "<|Ŀ|>",
+    "<|ŀ|>",
+    "<|Ł|>",
+    "<|ł|>",
+    "<|Ń|>",
+    "<|Ûķ|>",
+    "<|ÛĮ|>",
+    "<|ا|>",
+    "<|ĠØ|>",
+    "<|ÙĪ|>",
+    "<|ÙĨ|>",
+    "<|ر|>",
+    "<|Ú©|>",
+    "<|ĠÙ|>",
+    "<|ت|>",
+    "<|اÙĨ|>",
+    "<|Ûİ|>",
+    "<|د|>",
+    "<|Ùħ|>",
+    "<|Ġب|>",
+    "<|ÛĨ|>",
+    "<|س|>",
+    "<|translate|>",
+    "<|transcribe|>",
+    "<|startoflm|>",
+    "<|startofprev|>",
+    "<|nocaptions|>",
+    "<|notimestamps|>"
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,2493 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50522": {
+      "content": "<|startoftranscript|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50523": {
+      "content": "<|en|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50524": {
+      "content": "<|zh|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50525": {
+      "content": "<|de|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50526": {
+      "content": "<|es|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50527": {
+      "content": "<|ru|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50528": {
+      "content": "<|ko|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50529": {
+      "content": "<|fr|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50530": {
+      "content": "<|ja|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50531": {
+      "content": "<|pt|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50532": {
+      "content": "<|tr|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50533": {
+      "content": "<|pl|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50534": {
+      "content": "<|ca|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50535": {
+      "content": "<|nl|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50536": {
+      "content": "<|ar|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50537": {
+      "content": "<|sv|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50538": {
+      "content": "<|it|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50539": {
+      "content": "<|id|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50540": {
+      "content": "<|hi|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50541": {
+      "content": "<|fi|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50542": {
+      "content": "<|vi|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50543": {
+      "content": "<|he|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50544": {
+      "content": "<|uk|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50545": {
+      "content": "<|el|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50546": {
+      "content": "<|ms|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50547": {
+      "content": "<|cs|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50548": {
+      "content": "<|ro|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50549": {
+      "content": "<|da|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50550": {
+      "content": "<|hu|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50551": {
+      "content": "<|ta|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50552": {
+      "content": "<|no|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50553": {
+      "content": "<|th|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50554": {
+      "content": "<|ur|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50555": {
+      "content": "<|hr|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50556": {
+      "content": "<|bg|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50557": {
+      "content": "<|lt|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50558": {
+      "content": "<|la|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50559": {
+      "content": "<|mi|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50560": {
+      "content": "<|ml|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50561": {
+      "content": "<|cy|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50562": {
+      "content": "<|sk|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50563": {
+      "content": "<|te|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50564": {
+      "content": "<|fa|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50565": {
+      "content": "<|lv|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50566": {
+      "content": "<|bn|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50567": {
+      "content": "<|sr|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50568": {
+      "content": "<|az|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50569": {
+      "content": "<|sl|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50570": {
+      "content": "<|kn|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50571": {
+      "content": "<|et|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50572": {
+      "content": "<|mk|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50573": {
+      "content": "<|br|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50574": {
+      "content": "<|eu|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50575": {
+      "content": "<|is|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50576": {
+      "content": "<|hy|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50577": {
+      "content": "<|ne|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50578": {
+      "content": "<|mn|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50579": {
+      "content": "<|bs|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50580": {
+      "content": "<|kk|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50581": {
+      "content": "<|sq|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50582": {
+      "content": "<|sw|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50583": {
+      "content": "<|gl|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50584": {
+      "content": "<|mr|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50585": {
+      "content": "<|pa|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50586": {
+      "content": "<|si|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50587": {
+      "content": "<|km|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50588": {
+      "content": "<|sn|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50589": {
+      "content": "<|yo|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50590": {
+      "content": "<|so|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50591": {
+      "content": "<|af|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50592": {
+      "content": "<|oc|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50593": {
+      "content": "<|ka|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50594": {
+      "content": "<|be|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50595": {
+      "content": "<|tg|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50596": {
+      "content": "<|sd|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50597": {
+      "content": "<|gu|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50598": {
+      "content": "<|am|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50599": {
+      "content": "<|yi|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50600": {
+      "content": "<|lo|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50601": {
+      "content": "<|uz|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50602": {
+      "content": "<|fo|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50603": {
+      "content": "<|ht|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50604": {
+      "content": "<|ps|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50605": {
+      "content": "<|tk|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50606": {
+      "content": "<|nn|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50607": {
+      "content": "<|mt|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50608": {
+      "content": "<|sa|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50609": {
+      "content": "<|lb|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50610": {
+      "content": "<|my|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50611": {
+      "content": "<|bo|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50612": {
+      "content": "<|tl|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50613": {
+      "content": "<|mg|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50614": {
+      "content": "<|as|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50615": {
+      "content": "<|tt|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50616": {
+      "content": "<|haw|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50617": {
+      "content": "<|ln|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50618": {
+      "content": "<|ha|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50619": {
+      "content": "<|ba|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50620": {
+      "content": "<|jw|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50621": {
+      "content": "<|su|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50622": {
+      "content": "<|translate|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50623": {
+      "content": "<|transcribe|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50624": {
+      "content": "<|startoflm|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50625": {
+      "content": "<|startofprev|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50626": {
+      "content": "<|nocaptions|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50627": {
+      "content": "<|notimestamps|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50628": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50629": {
+      "content": "<|¡|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50630": {
+      "content": "<|¢|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50631": {
+      "content": "<|£|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50632": {
+      "content": "<|¤|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50633": {
+      "content": "<|¥|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50634": {
+      "content": "<|¦|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50635": {
+      "content": "<|§|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50636": {
+      "content": "<|¨|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50637": {
+      "content": "<|©|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50638": {
+      "content": "<|ª|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50639": {
+      "content": "<|«|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50640": {
+      "content": "<|¬|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50641": {
+      "content": "<|®|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50642": {
+      "content": "<|¯|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50643": {
+      "content": "<|°|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50644": {
+      "content": "<|±|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50645": {
+      "content": "<|²|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50646": {
+      "content": "<|³|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50647": {
+      "content": "<|´|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50648": {
+      "content": "<|µ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50649": {
+      "content": "<|¶|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50650": {
+      "content": "<|·|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50651": {
+      "content": "<|¸|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50652": {
+      "content": "<|¹|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50653": {
+      "content": "<|º|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50654": {
+      "content": "<|»|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50655": {
+      "content": "<|¼|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50656": {
+      "content": "<|½|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50657": {
+      "content": "<|¾|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50658": {
+      "content": "<|¿|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50659": {
+      "content": "<|À|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50660": {
+      "content": "<|Á|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50661": {
+      "content": "<|Â|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50662": {
+      "content": "<|Ã|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50663": {
+      "content": "<|Ä|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50664": {
+      "content": "<|Å|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50665": {
+      "content": "<|Æ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50666": {
+      "content": "<|Ç|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50667": {
+      "content": "<|È|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50668": {
+      "content": "<|É|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50669": {
+      "content": "<|Ê|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50670": {
+      "content": "<|Ë|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50671": {
+      "content": "<|Ì|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50672": {
+      "content": "<|Í|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50673": {
+      "content": "<|Î|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50674": {
+      "content": "<|Ï|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50675": {
+      "content": "<|Ð|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50676": {
+      "content": "<|Ñ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50677": {
+      "content": "<|Ò|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50678": {
+      "content": "<|Ó|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50679": {
+      "content": "<|Ô|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50680": {
+      "content": "<|Õ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50681": {
+      "content": "<|Ö|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50682": {
+      "content": "<|×|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50683": {
+      "content": "<|Ø|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50684": {
+      "content": "<|Ù|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50685": {
+      "content": "<|Ú|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50686": {
+      "content": "<|Û|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50687": {
+      "content": "<|Ü|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50688": {
+      "content": "<|Ý|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50689": {
+      "content": "<|Þ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50690": {
+      "content": "<|ß|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50691": {
+      "content": "<|à|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50692": {
+      "content": "<|á|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50693": {
+      "content": "<|â|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50694": {
+      "content": "<|ã|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50695": {
+      "content": "<|ä|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50696": {
+      "content": "<|å|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50697": {
+      "content": "<|æ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50698": {
+      "content": "<|ç|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50699": {
+      "content": "<|è|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50700": {
+      "content": "<|é|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50701": {
+      "content": "<|ê|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50702": {
+      "content": "<|ë|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50703": {
+      "content": "<|ì|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50704": {
+      "content": "<|í|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50705": {
+      "content": "<|î|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50706": {
+      "content": "<|ï|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50707": {
+      "content": "<|ð|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50708": {
+      "content": "<|ñ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50709": {
+      "content": "<|ò|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50710": {
+      "content": "<|ó|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50711": {
+      "content": "<|ô|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50712": {
+      "content": "<|õ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50713": {
+      "content": "<|ö|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50714": {
+      "content": "<|÷|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50715": {
+      "content": "<|ø|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50716": {
+      "content": "<|ù|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50717": {
+      "content": "<|ú|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50718": {
+      "content": "<|û|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50719": {
+      "content": "<|ü|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50720": {
+      "content": "<|ý|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50721": {
+      "content": "<|þ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50722": {
+      "content": "<|ÿ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50723": {
+      "content": "<|Ā|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50724": {
+      "content": "<|ā|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50725": {
+      "content": "<|Ă|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50726": {
+      "content": "<|ă|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50727": {
+      "content": "<|Ą|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50728": {
+      "content": "<|ą|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50729": {
+      "content": "<|Ć|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50730": {
+      "content": "<|ć|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50731": {
+      "content": "<|Ĉ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50732": {
+      "content": "<|ĉ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50733": {
+      "content": "<|Ċ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50734": {
+      "content": "<|ċ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50735": {
+      "content": "<|Č|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50736": {
+      "content": "<|č|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50737": {
+      "content": "<|Ď|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50738": {
+      "content": "<|ď|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50739": {
+      "content": "<|Đ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50740": {
+      "content": "<|đ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50741": {
+      "content": "<|Ē|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50742": {
+      "content": "<|ē|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50743": {
+      "content": "<|Ĕ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50744": {
+      "content": "<|ĕ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50745": {
+      "content": "<|Ė|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50746": {
+      "content": "<|ė|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50747": {
+      "content": "<|Ę|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50748": {
+      "content": "<|ę|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50749": {
+      "content": "<|Ě|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50750": {
+      "content": "<|ě|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50751": {
+      "content": "<|Ĝ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50752": {
+      "content": "<|ĝ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50753": {
+      "content": "<|Ğ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50754": {
+      "content": "<|ğ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50755": {
+      "content": "<|Ġ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50756": {
+      "content": "<|ġ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50757": {
+      "content": "<|Ģ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50758": {
+      "content": "<|ģ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50759": {
+      "content": "<|Ĥ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50760": {
+      "content": "<|ĥ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50761": {
+      "content": "<|Ħ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50762": {
+      "content": "<|ħ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50763": {
+      "content": "<|Ĩ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50764": {
+      "content": "<|ĩ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50765": {
+      "content": "<|Ī|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50766": {
+      "content": "<|ī|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50767": {
+      "content": "<|Ĭ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50768": {
+      "content": "<|ĭ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50769": {
+      "content": "<|Į|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50770": {
+      "content": "<|į|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50771": {
+      "content": "<|İ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50772": {
+      "content": "<|ı|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50773": {
+      "content": "<|IJ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50774": {
+      "content": "<|ij|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50775": {
+      "content": "<|Ĵ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50776": {
+      "content": "<|ĵ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50777": {
+      "content": "<|Ķ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50778": {
+      "content": "<|ķ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50779": {
+      "content": "<|ĸ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50780": {
+      "content": "<|Ĺ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50781": {
+      "content": "<|ĺ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50782": {
+      "content": "<|Ļ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50783": {
+      "content": "<|ļ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50784": {
+      "content": "<|Ľ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50785": {
+      "content": "<|ľ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50786": {
+      "content": "<|Ŀ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50787": {
+      "content": "<|ŀ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50788": {
+      "content": "<|Ł|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50789": {
+      "content": "<|ł|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50790": {
+      "content": "<|Ń|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50791": {
+      "content": "<|Ûķ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50792": {
+      "content": "<|ÛĮ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50793": {
+      "content": "<|ا|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50794": {
+      "content": "<|ĠØ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50795": {
+      "content": "<|ÙĪ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50796": {
+      "content": "<|ÙĨ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50797": {
+      "content": "<|ر|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50798": {
+      "content": "<|Ú©|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50799": {
+      "content": "<|ĠÙ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50800": {
+      "content": "<|ت|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50801": {
+      "content": "<|اÙĨ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50802": {
+      "content": "<|Ûİ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50803": {
+      "content": "<|د|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50804": {
+      "content": "<|Ùħ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50805": {
+      "content": "<|Ġب|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50806": {
+      "content": "<|ÛĨ|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50807": {
+      "content": "<|س|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|startoftranscript|>",
+    "<|¡|>",
+    "<|¢|>",
+    "<|£|>",
+    "<|¤|>",
+    "<|¥|>",
+    "<|¦|>",
+    "<|§|>",
+    "<|¨|>",
+    "<|©|>",
+    "<|ª|>",
+    "<|«|>",
+    "<|¬|>",
+    "<|®|>",
+    "<|¯|>",
+    "<|°|>",
+    "<|±|>",
+    "<|²|>",
+    "<|³|>",
+    "<|´|>",
+    "<|µ|>",
+    "<|¶|>",
+    "<|·|>",
+    "<|¸|>",
+    "<|¹|>",
+    "<|º|>",
+    "<|»|>",
+    "<|¼|>",
+    "<|½|>",
+    "<|¾|>",
+    "<|¿|>",
+    "<|À|>",
+    "<|Á|>",
+    "<|Â|>",
+    "<|Ã|>",
+    "<|Ä|>",
+    "<|Å|>",
+    "<|Æ|>",
+    "<|Ç|>",
+    "<|È|>",
+    "<|É|>",
+    "<|Ê|>",
+    "<|Ë|>",
+    "<|Ì|>",
+    "<|Í|>",
+    "<|Î|>",
+    "<|Ï|>",
+    "<|Ð|>",
+    "<|Ñ|>",
+    "<|Ò|>",
+    "<|Ó|>",
+    "<|Ô|>",
+    "<|Õ|>",
+    "<|Ö|>",
+    "<|×|>",
+    "<|Ø|>",
+    "<|Ù|>",
+    "<|Ú|>",
+    "<|Û|>",
+    "<|Ü|>",
+    "<|Ý|>",
+    "<|Þ|>",
+    "<|ß|>",
+    "<|à|>",
+    "<|á|>",
+    "<|â|>",
+    "<|ã|>",
+    "<|ä|>",
+    "<|å|>",
+    "<|æ|>",
+    "<|ç|>",
+    "<|è|>",
+    "<|é|>",
+    "<|ê|>",
+    "<|ë|>",
+    "<|ì|>",
+    "<|í|>",
+    "<|î|>",
+    "<|ï|>",
+    "<|ð|>",
+    "<|ñ|>",
+    "<|ò|>",
+    "<|ó|>",
+    "<|ô|>",
+    "<|õ|>",
+    "<|ö|>",
+    "<|÷|>",
+    "<|ø|>",
+    "<|ù|>",
+    "<|ú|>",
+    "<|û|>",
+    "<|ü|>",
+    "<|ý|>",
+    "<|þ|>",
+    "<|ÿ|>",
+    "<|Ā|>",
+    "<|ā|>",
+    "<|Ă|>",
+    "<|ă|>",
+    "<|Ą|>",
+    "<|ą|>",
+    "<|Ć|>",
+    "<|ć|>",
+    "<|Ĉ|>",
+    "<|ĉ|>",
+    "<|Ċ|>",
+    "<|ċ|>",
+    "<|Č|>",
+    "<|č|>",
+    "<|Ď|>",
+    "<|ď|>",
+    "<|Đ|>",
+    "<|đ|>",
+    "<|Ē|>",
+    "<|ē|>",
+    "<|Ĕ|>",
+    "<|ĕ|>",
+    "<|Ė|>",
+    "<|ė|>",
+    "<|Ę|>",
+    "<|ę|>",
+    "<|Ě|>",
+    "<|ě|>",
+    "<|Ĝ|>",
+    "<|ĝ|>",
+    "<|Ğ|>",
+    "<|ğ|>",
+    "<|Ġ|>",
+    "<|ġ|>",
+    "<|Ģ|>",
+    "<|ģ|>",
+    "<|Ĥ|>",
+    "<|ĥ|>",
+    "<|Ħ|>",
+    "<|ħ|>",
+    "<|Ĩ|>",
+    "<|ĩ|>",
+    "<|Ī|>",
+    "<|ī|>",
+    "<|Ĭ|>",
+    "<|ĭ|>",
+    "<|Į|>",
+    "<|į|>",
+    "<|İ|>",
+    "<|ı|>",
+    "<|IJ|>",
+    "<|ij|>",
+    "<|Ĵ|>",
+    "<|ĵ|>",
+    "<|Ķ|>",
+    "<|ķ|>",
+    "<|ĸ|>",
+    "<|Ĺ|>",
+    "<|ĺ|>",
+    "<|Ļ|>",
+    "<|ļ|>",
+    "<|Ľ|>",
+    "<|ľ|>",
+    "<|Ŀ|>",
+    "<|ŀ|>",
+    "<|Ł|>",
+    "<|ł|>",
+    "<|Ń|>",
+    "<|Ûķ|>",
+    "<|ÛĮ|>",
+    "<|ا|>",
+    "<|ĠØ|>",
+    "<|ÙĪ|>",
+    "<|ÙĨ|>",
+    "<|ر|>",
+    "<|Ú©|>",
+    "<|ĠÙ|>",
+    "<|ت|>",
+    "<|اÙĨ|>",
+    "<|Ûİ|>",
+    "<|د|>",
+    "<|Ùħ|>",
+    "<|Ġب|>",
+    "<|ÛĨ|>",
+    "<|س|>",
+    "<|translate|>",
+    "<|transcribe|>",
+    "<|startoflm|>",
+    "<|startofprev|>",
+    "<|nocaptions|>",
+    "<|notimestamps|>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "WhisperProcessor",
+  "return_attention_mask": false,
+  "tokenizer_class": "WhisperTokenizer",
+  "unk_token": "<|endoftext|>"
+}