Upload tokenizer

tokenizer.json

@@ -70,225 +70,282 @@
       "<pad>": 2,
       "<unk>": 3,
       "<mask>": 4,
-      ".": 5,
-      "A": 6,
-      "B": 7,
-      "E": 8,
-      "F": 9,
-      "I": 10,
-      "L": 11,
-      "M": 12,
-      "O": 13,
-      "R": 14,
-      "S": 15,
-      "T": 16,
-      "a": 17,
-      "c": 18,
-      "d": 19,
-      "e": 20,
-      "f": 21,
-      "g": 22,
-      "h": 23,
-      "i": 24,
-      "j": 25,
-      "l": 26,
-      "m": 27,
-      "n": 28,
-      "o": 29,
-      "p": 30,
-      "r": 31,
-      "s": 32,
-      "t": 33,
-      "u": 34,
-      "v": 35,
-      "x": 36,
-      "y": 37,
-      "è": 38,
-      "in": 39,
-      "de": 40,
-      "an": 41,
-      "en": 42,
-      "ra": 43,
-      "te": 44,
-      "ing": 45,
-      "ce": 46,
-      "ch": 47,
-      "is": 48,
-      "le": 49,
-      "ode": 50,
-      "ro": 51,
-      "ti": 52,
-      "ar": 53,
-      "lo": 54,
-      "or": 55,
-      "pro": 56,
-      "tu": 57,
-      "ua": 58,
-      "OR": 59,
-      "atu": 60,
-      "ci": 61,
-      "ge": 62,
-      "gen": 63,
-      "gua": 64,
-      "lan": 65,
-      "men": 66,
-      "mode": 67,
-      "no": 68,
-      "ning": 69,
-      "natu": 70,
-      "on": 71,
-      "rs": 72,
-      "ss": 73,
-      "un": 74,
-      "and": 75,
-      "ral": 76,
-      "ish": 77,
-      "lear": 78,
-      "tion": 79,
-      "proce": 80,
-      "ORB": 81,
-      "ment": 82,
-      "natural": 83,
-      "learning": 84,
-      "AI": 85,
-      "El": 86,
-      "En": 87,
-      "Fr": 88,
-      "Le": 89,
-      "Mode": 90,
-      "Sp": 91,
-      "Th": 92,
-      "al": 93,
-      "ach": 94,
-      "age": 95,
-      "ass": 96,
-      "ation": 97,
-      "cl": 98,
-      "code": 99,
-      "ction": 100,
-      "du": 101,
-      "dè": 102,
-      "ep": 103,
-      "es": 104,
-      "era": 105,
-      "elo": 106,
-      "etu": 107,
-      "ers": 108,
-      "fi": 109,
-      "fro": 110,
-      "for": 111,
-      "fun": 112,
-      "gl": 113,
-      "gy": 114,
-      "gra": 115,
-      "gage": 116,
-      "im": 117,
-      "it": 118,
-      "ite": 119,
-      "igen": 120,
-      "je": 121,
-      "ll": 122,
-      "len": 123,
-      "ling": 124,
-      "lti": 125,
-      "mm": 126,
-      "mo": 127,
-      "mu": 128,
-      "mach": 129,
-      "mers": 130,
-      "ou": 131,
-      "pu": 132,
-      "por": 133,
-      "pment": 134,
-      "pou": 135,
-      "rn": 136,
-      "ran": 137,
-      "retu": 138,
-      "sa": 139,
-      "san": 140,
-      "sci": 141,
-      "sfor": 142,
-      "tra": 143,
-      "tor": 144,
-      "tand": 145,
-      "tran": 146,
-      "vation": 147,
-      "velo": 148,
-      "xt": 149,
-      "ine": 150,
-      "inte": 151,
-      "inno": 152,
-      "init": 153,
-      "def": 154,
-      "ders": 155,
-      "deep": 156,
-      "develo": 157,
-      "anish": 158,
-      "ence": 159,
-      "ench": 160,
-      "tes": 161,
-      "tech": 162,
-      "text": 163,
-      "issan": 164,
-      "tifi": 165,
-      "artifi": 166,
-      "logy": 167,
-      "progra": 168,
-      "ual": 169,
-      "cial": 170,
-      "genera": 171,
-      "guage": 172,
-      "guaje": 173,
-      "langage": 174,
-      "language": 175,
-      "model": 176,
-      "modelo": 177,
-      "nology": 178,
-      "ssing": 179,
-      "unders": 180,
-      "procesa": 181,
-      "processing": 182,
-      "Engl": 183,
-      "French": 184,
-      "Model": 185,
-      "Spanish": 186,
-      "This": 187,
-      "class": 188,
-      "dèle": 189,
-      "est": 190,
-      "from": 191,
-      "function": 192,
-      "impor": 193,
-      "itement": 194,
-      "igence": 195,
-      "lligence": 196,
-      "lenguaje": 197,
-      "lingual": 198,
-      "ltilingual": 199,
-      "mming": 200,
-      "modèle": 201,
-      "multilingual": 202,
-      "machine": 203,
-      "puissan": 204,
-      "pour": 205,
-      "return": 206,
-      "science": 207,
-      "sformers": 208,
-      "traitement": 209,
-      "torch": 210,
-      "tands": 211,
-      "transformers": 212,
-      "intelligence": 213,
-      "innovation": 214,
-      "development": 215,
-      "technology": 216,
-      "artificial": 217,
-      "programming": 218,
-      "generates": 219,
-      "understands": 220,
-      "English": 221,
-      "import": 222,
-      "puissant": 223
+      "'": 5,
+      "-": 6,
+      ".": 7,
+      "0": 8,
+      "1": 9,
+      "A": 10,
+      "B": 11,
+      "E": 12,
+      "F": 13,
+      "I": 14,
+      "L": 15,
+      "M": 16,
+      "O": 17,
+      "R": 18,
+      "S": 19,
+      "T": 20,
+      "a": 21,
+      "b": 22,
+      "c": 23,
+      "d": 24,
+      "e": 25,
+      "f": 26,
+      "g": 27,
+      "h": 28,
+      "i": 29,
+      "j": 30,
+      "k": 31,
+      "l": 32,
+      "m": 33,
+      "n": 34,
+      "o": 35,
+      "p": 36,
+      "r": 37,
+      "s": 38,
+      "t": 39,
+      "u": 40,
+      "v": 41,
+      "w": 42,
+      "x": 43,
+      "y": 44,
+      "è": 45,
+      "in": 46,
+      "de": 47,
+      "or": 48,
+      "te": 49,
+      "an": 50,
+      "en": 51,
+      "es": 52,
+      "op": 53,
+      "ra": 54,
+      "is": 55,
+      "le": 56,
+      "pr": 57,
+      "ing": 58,
+      "ar": 59,
+      "ch": 60,
+      "gop": 61,
+      "io": 62,
+      "lo": 63,
+      "ode": 64,
+      "re": 65,
+      "tu": 66,
+      "pro": 67,
+      "gopu": 68,
+      "cre": 69,
+      "me": 70,
+      "no": 71,
+      "ua": 72,
+      "ur": 73,
+      "inc": 74,
+      "orb": 75,
+      "ang": 76,
+      "ion": 77,
+      "AI": 78,
+      "OR": 79,
+      "ate": 80,
+      "atu": 81,
+      "ce": 82,
+      "ci": 83,
+      "ces": 84,
+      "flo": 85,
+      "ge": 86,
+      "gen": 87,
+      "gra": 88,
+      "ite": 89,
+      "lang": 90,
+      "ma": 91,
+      "mp": 92,
+      "men": 93,
+      "mode": 94,
+      "mme": 95,
+      "ning": 96,
+      "natu": 97,
+      "pu": 98,
+      "rs": 99,
+      "ti": 100,
+      "tion": 101,
+      "un": 102,
+      "v1": 103,
+      "and": 104,
+      "est": 105,
+      "ral": 106,
+      "ish": 107,
+      "lear": 108,
+      "proces": 109,
+      "progra": 110,
+      "create": 111,
+      "ORB": 112,
+      "flow": 113,
+      "ment": 114,
+      "natural": 115,
+      "learning": 116,
+      "El": 117,
+      "En": 118,
+      "Fr": 119,
+      "Le": 120,
+      "Mode": 121,
+      "Mang": 122,
+      "Sp": 123,
+      "Th": 124,
+      "al": 125,
+      "as": 126,
+      "age": 127,
+      "ation": 128,
+      "cl": 129,
+      "cor": 130,
+      "cio": 131,
+      "code": 132,
+      "ction": 133,
+      "du": 134,
+      "dè": 135,
+      "el": 136,
+      "ep": 137,
+      "er": 138,
+      "et": 139,
+      "era": 140,
+      "fi": 141,
+      "fr": 142,
+      "for": 143,
+      "fun": 144,
+      "gl": 145,
+      "gy": 146,
+      "gua": 147,
+      "it": 148,
+      "ior": 149,
+      "itu": 150,
+      "igen": 151,
+      "imp": 152,
+      "icio": 153,
+      "je": 154,
+      "ka": 155,
+      "kflow": 156,
+      "ll": 157,
+      "len": 158,
+      "les": 159,
+      "ling": 160,
+      "lti": 161,
+      "mm": 162,
+      "mo": 163,
+      "mu": 164,
+      "of": 165,
+      "om": 166,
+      "our": 167,
+      "par": 168,
+      "pour": 169,
+      "rn": 170,
+      "ran": 171,
+      "san": 172,
+      "ses": 173,
+      "sing": 174,
+      "sion": 175,
+      "sci": 176,
+      "sfor": 177,
+      "tor": 178,
+      "tra": 179,
+      "tand": 180,
+      "tran": 181,
+      "vation": 182,
+      "vel": 183,
+      "wor": 184,
+      "xt": 185,
+      "ine": 186,
+      "inte": 187,
+      "inno": 188,
+      "init": 189,
+      "def": 190,
+      "ders": 191,
+      "deep": 192,
+      "devel": 193,
+      "ort": 194,
+      "tes": 195,
+      "tech": 196,
+      "temp": 197,
+      "text": 198,
+      "anish": 199,
+      "ench": 200,
+      "enAI": 201,
+      "ence": 202,
+      "opment": 203,
+      "openAI": 204,
+      "issan": 205,
+      "prior": 206,
+      "arti": 207,
+      "chine": 208,
+      "logy": 209,
+      "retu": 210,
+      "cree": 211,
+      "mers": 212,
+      "nomme": 213,
+      "nology": 214,
+      "ual": 215,
+      "uage": 216,
+      "uricio": 217,
+      "cial": 218,
+      "genera": 219,
+      "ites": 220,
+      "itement": 221,
+      "langage": 222,
+      "language": 223,
+      "machine": 224,
+      "mauricio": 225,
+      "model": 226,
+      "modelo": 227,
+      "pus": 228,
+      "puissan": 229,
+      "unders": 230,
+      "procesa": 231,
+      "processing": 232,
+      "programme": 233,
+      "programm": 234,
+      "createur": 235,
+      "flower": 236,
+      "Engl": 237,
+      "French": 238,
+      "Model": 239,
+      "Mangitu": 240,
+      "Spanish": 241,
+      "This": 242,
+      "ass": 243,
+      "class": 244,
+      "corpus": 245,
+      "dèle": 246,
+      "ficial": 247,
+      "from": 248,
+      "function": 249,
+      "guaje": 250,
+      "igence": 251,
+      "import": 252,
+      "lligence": 253,
+      "lenguaje": 254,
+      "lingual": 255,
+      "ltilingual": 256,
+      "modèle": 257,
+      "multilingual": 258,
+      "session": 259,
+      "science": 260,
+      "sformers": 261,
+      "torch": 262,
+      "traitement": 263,
+      "tands": 264,
+      "transformers": 265,
+      "workflow": 266,
+      "intelligence": 267,
+      "innovation": 268,
+      "development": 269,
+      "technology": 270,
+      "temps": 271,
+      "priorites": 272,
+      "artificial": 273,
+      "return": 274,
+      "generates": 275,
+      "puissant": 276,
+      "understands": 277,
+      "programming": 278,
+      "English": 279,
+      "Mangituka": 280
     },
     "merges": [
       [
@@ -299,6 +356,14 @@
         "d",
         "e"
       ],
+      [
+        "o",
+        "r"
+      ],
+      [
+        "t",
+        "e"
+      ],
       [
         "a",
         "n"
@@ -307,33 +372,53 @@
         "e",
         "n"
       ],
+      [
+        "e",
+        "s"
+      ],
+      [
+        "o",
+        "p"
+      ],
       [
         "r",
         "a"
       ],
       [
-        "t",
+        "i",
+        "s"
+      ],
+      [
+        "l",
         "e"
       ],
+      [
+        "p",
+        "r"
+      ],
       [
         "in",
         "g"
       ],
       [
-        "c",
-        "e"
+        "a",
+        "r"
       ],
       [
         "c",
         "h"
       ],
+      [
+        "g",
+        "op"
+      ],
       [
         "i",
-        "s"
+        "o"
       ],
       [
         "l",
-        "e"
+        "o"
       ],
       [
         "o",
@@ -341,48 +426,88 @@
       ],
       [
         "r",
-        "o"
+        "e"
       ],
       [
         "t",
-        "i"
+        "u"
       ],
       [
-        "a",
-        "r"
+        "pr",
+        "o"
       ],
       [
-        "l",
-        "o"
+        "gop",
+        "u"
       ],
       [
-        "o",
-        "r"
+        "c",
+        "re"
       ],
       [
-        "p",
-        "ro"
+        "m",
+        "e"
       ],
       [
-        "t",
-        "u"
+        "n",
+        "o"
       ],
       [
         "u",
         "a"
       ],
+      [
+        "u",
+        "r"
+      ],
+      [
+        "in",
+        "c"
+      ],
+      [
+        "or",
+        "b"
+      ],
+      [
+        "an",
+        "g"
+      ],
+      [
+        "io",
+        "n"
+      ],
+      [
+        "A",
+        "I"
+      ],
       [
         "O",
         "R"
       ],
+      [
+        "a",
+        "te"
+      ],
       [
         "a",
         "tu"
       ],
+      [
+        "c",
+        "e"
+      ],
       [
         "c",
         "i"
       ],
+      [
+        "c",
+        "es"
+      ],
+      [
+        "f",
+        "lo"
+      ],
       [
         "g",
         "e"
@@ -393,11 +518,23 @@
       ],
       [
         "g",
-        "ua"
+        "ra"
+      ],
+      [
+        "i",
+        "te"
       ],
       [
         "l",
-        "an"
+        "ang"
+      ],
+      [
+        "m",
+        "a"
+      ],
+      [
+        "m",
+        "p"
       ],
       [
         "m",
@@ -408,8 +545,8 @@
         "ode"
       ],
       [
-        "n",
-        "o"
+        "m",
+        "me"
       ],
       [
         "n",
@@ -420,25 +557,37 @@
         "atu"
       ],
       [
-        "o",
-        "n"
+        "p",
+        "u"
       ],
       [
         "r",
         "s"
       ],
       [
-        "s",
-        "s"
+        "t",
+        "i"
+      ],
+      [
+        "t",
+        "ion"
       ],
       [
         "u",
         "n"
       ],
+      [
+        "v",
+        "1"
+      ],
       [
         "an",
         "d"
       ],
+      [
+        "es",
+        "t"
+      ],
       [
         "ra",
         "l"
@@ -452,17 +601,25 @@
         "ar"
       ],
       [
-        "ti",
-        "on"
+        "pro",
+        "ces"
       ],
       [
         "pro",
-        "ce"
+        "gra"
+      ],
+      [
+        "cre",
+        "ate"
       ],
       [
         "OR",
         "B"
       ],
+      [
+        "flo",
+        "w"
+      ],
       [
         "men",
         "t"
@@ -475,10 +632,6 @@
         "lear",
         "ning"
       ],
-      [
-        "A",
-        "I"
-      ],
       [
         "E",
         "l"
@@ -499,6 +652,10 @@
         "M",
         "ode"
       ],
+      [
+        "M",
+        "ang"
+      ],
       [
         "S",
         "p"
@@ -513,16 +670,12 @@
       ],
       [
         "a",
-        "ch"
+        "s"
       ],
       [
         "a",
         "ge"
       ],
-      [
-        "a",
-        "ss"
-      ],
       [
         "a",
         "tion"
@@ -531,6 +684,14 @@
         "c",
         "l"
       ],
+      [
+        "c",
+        "or"
+      ],
+      [
+        "c",
+        "io"
+      ],
       [
         "c",
         "ode"
@@ -549,27 +710,23 @@
       ],
       [
         "e",
-        "p"
-      ],
-      [
-        "e",
-        "s"
+        "l"
       ],
       [
         "e",
-        "ra"
+        "p"
       ],
       [
         "e",
-        "lo"
+        "r"
       ],
       [
         "e",
-        "tu"
+        "t"
       ],
       [
         "e",
-        "rs"
+        "ra"
       ],
       [
         "f",
@@ -577,7 +734,7 @@
       ],
       [
         "f",
-        "ro"
+        "r"
       ],
       [
         "f",
@@ -597,32 +754,44 @@
       ],
       [
         "g",
-        "ra"
+        "ua"
       ],
       [
-        "g",
-        "age"
+        "i",
+        "t"
       ],
       [
         "i",
-        "m"
+        "or"
       ],
       [
         "i",
-        "t"
+        "tu"
       ],
       [
         "i",
-        "te"
+        "gen"
       ],
       [
         "i",
-        "gen"
+        "mp"
+      ],
+      [
+        "i",
+        "cio"
       ],
       [
         "j",
         "e"
       ],
+      [
+        "k",
+        "a"
+      ],
+      [
+        "k",
+        "flow"
+      ],
       [
         "l",
         "l"
@@ -631,6 +800,10 @@
         "l",
         "en"
       ],
+      [
+        "l",
+        "es"
+      ],
       [
         "l",
         "ing"
@@ -651,33 +824,25 @@
         "m",
         "u"
       ],
-      [
-        "m",
-        "ach"
-      ],
-      [
-        "m",
-        "ers"
-      ],
       [
         "o",
-        "u"
+        "f"
       ],
       [
-        "p",
-        "u"
+        "o",
+        "m"
       ],
       [
-        "p",
-        "or"
+        "o",
+        "ur"
       ],
       [
         "p",
-        "ment"
+        "ar"
       ],
       [
         "p",
-        "ou"
+        "our"
       ],
       [
         "r",
@@ -688,16 +853,20 @@
         "an"
       ],
       [
-        "r",
-        "etu"
+        "s",
+        "an"
       ],
       [
         "s",
-        "a"
+        "es"
       ],
       [
         "s",
-        "an"
+        "ing"
+      ],
+      [
+        "s",
+        "ion"
       ],
       [
         "s",
@@ -709,11 +878,11 @@
       ],
       [
         "t",
-        "ra"
+        "or"
       ],
       [
         "t",
-        "or"
+        "ra"
       ],
       [
         "t",
@@ -729,7 +898,11 @@
       ],
       [
         "v",
-        "elo"
+        "el"
+      ],
+      [
+        "w",
+        "or"
       ],
       [
         "x",
@@ -765,56 +938,104 @@
       ],
       [
         "de",
-        "velo"
+        "vel"
       ],
       [
-        "an",
-        "ish"
+        "or",
+        "t"
       ],
       [
-        "en",
-        "ce"
+        "te",
+        "s"
       ],
       [
-        "en",
+        "te",
         "ch"
       ],
       [
         "te",
-        "s"
+        "mp"
       ],
       [
         "te",
+        "xt"
+      ],
+      [
+        "an",
+        "ish"
+      ],
+      [
+        "en",
         "ch"
       ],
       [
-        "te",
-        "xt"
+        "en",
+        "AI"
+      ],
+      [
+        "en",
+        "ce"
+      ],
+      [
+        "op",
+        "ment"
+      ],
+      [
+        "op",
+        "enAI"
       ],
       [
         "is",
         "san"
       ],
       [
-        "ti",
-        "fi"
+        "pr",
+        "ior"
       ],
       [
         "ar",
-        "tifi"
+        "ti"
+      ],
+      [
+        "ch",
+        "ine"
       ],
       [
         "lo",
         "gy"
       ],
       [
-        "pro",
-        "gra"
+        "re",
+        "tu"
+      ],
+      [
+        "cre",
+        "e"
+      ],
+      [
+        "me",
+        "rs"
+      ],
+      [
+        "no",
+        "mme"
+      ],
+      [
+        "no",
+        "logy"
       ],
       [
         "ua",
         "l"
       ],
+      [
+        "ua",
+        "ge"
+      ],
+      [
+        "ur",
+        "icio"
+      ],
       [
         "ci",
         "al"
@@ -824,20 +1045,28 @@
         "era"
       ],
       [
-        "gua",
-        "ge"
+        "ite",
+        "s"
       ],
       [
-        "gua",
-        "je"
+        "ite",
+        "ment"
       ],
       [
-        "lan",
-        "gage"
+        "lang",
+        "age"
+      ],
+      [
+        "lang",
+        "uage"
       ],
       [
-        "lan",
-        "guage"
+        "ma",
+        "chine"
+      ],
+      [
+        "ma",
+        "uricio"
       ],
       [
         "mode",
@@ -848,24 +1077,40 @@
         "lo"
       ],
       [
-        "no",
-        "logy"
+        "pu",
+        "s"
       ],
       [
-        "ss",
-        "ing"
+        "pu",
+        "issan"
       ],
       [
         "un",
         "ders"
       ],
       [
-        "proce",
-        "sa"
+        "proces",
+        "a"
+      ],
+      [
+        "proces",
+        "sing"
       ],
       [
-        "proce",
-        "ssing"
+        "progra",
+        "mme"
+      ],
+      [
+        "progra",
+        "mm"
+      ],
+      [
+        "create",
+        "ur"
+      ],
+      [
+        "flow",
+        "er"
       ],
       [
         "En",
@@ -879,6 +1124,10 @@
         "Mode",
         "l"
       ],
+      [
+        "Mang",
+        "itu"
+      ],
       [
         "Sp",
         "anish"
@@ -887,38 +1136,46 @@
         "Th",
         "is"
       ],
+      [
+        "as",
+        "s"
+      ],
       [
         "cl",
         "ass"
       ],
+      [
+        "cor",
+        "pus"
+      ],
       [
         "dè",
         "le"
       ],
       [
-        "es",
-        "t"
+        "fi",
+        "cial"
       ],
       [
-        "fro",
-        "m"
+        "fr",
+        "om"
       ],
       [
         "fun",
         "ction"
       ],
       [
-        "im",
-        "por"
-      ],
-      [
-        "ite",
-        "ment"
+        "gua",
+        "je"
       ],
       [
         "igen",
         "ce"
       ],
+      [
+        "imp",
+        "ort"
+      ],
       [
         "ll",
         "igence"
@@ -935,10 +1192,6 @@
         "lti",
         "lingual"
       ],
-      [
-        "mm",
-        "ing"
-      ],
       [
         "mo",
         "dèle"
@@ -948,20 +1201,8 @@
         "ltilingual"
       ],
       [
-        "mach",
-        "ine"
-      ],
-      [
-        "pu",
-        "issan"
-      ],
-      [
-        "pou",
-        "r"
-      ],
-      [
-        "retu",
-        "rn"
+        "ses",
+        "sion"
       ],
       [
         "sci",
@@ -971,14 +1212,14 @@
         "sfor",
         "mers"
       ],
-      [
-        "tra",
-        "itement"
-      ],
       [
         "tor",
         "ch"
       ],
+      [
+        "tra",
+        "itement"
+      ],
       [
         "tand",
         "s"
@@ -987,6 +1228,10 @@
         "tran",
         "sformers"
       ],
+      [
+        "wor",
+        "kflow"
+      ],
       [
         "inte",
         "lligence"
@@ -996,40 +1241,52 @@
         "vation"
       ],
       [
-        "develo",
-        "pment"
+        "devel",
+        "opment"
       ],
       [
         "tech",
         "nology"
       ],
       [
-        "artifi",
-        "cial"
+        "temp",
+        "s"
       ],
       [
-        "progra",
-        "mming"
+        "prior",
+        "ites"
+      ],
+      [
+        "arti",
+        "ficial"
+      ],
+      [
+        "retu",
+        "rn"
       ],
       [
         "genera",
         "tes"
       ],
+      [
+        "puissan",
+        "t"
+      ],
       [
         "unders",
         "tands"
       ],
       [
-        "Engl",
-        "ish"
+        "programm",
+        "ing"
       ],
       [
-        "impor",
-        "t"
+        "Engl",
+        "ish"
       ],
       [
-        "puissan",
-        "t"
+        "Mangitu",
+        "ka"
       ]
     ]
   }