Upload tokenizer

Browse files

Files changed (3) hide show

special_tokens_map.json +5 -0
tokenizer.json +753 -0
tokenizer_config.json +75 -0

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,5 @@

+{
+  "eos_token": "[EOS]",
+  "pad_token": "[PAD]",
+  "unk_token": "[UNK]"
+}

tokenizer.json ADDED Viewed

	@@ -0,0 +1,753 @@

+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "[CLS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "[SEP]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 5,
+      "content": "[SOS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 6,
+      "content": "[EOS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 7,
+      "content": "[SPACE]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": null,
+  "decoder": null,
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "[PAD]": 0,
+      "[UNK]": 1,
+      "[CLS]": 2,
+      "[SEP]": 3,
+      "[MASK]": 4,
+      "[SOS]": 5,
+      "[EOS]": 6,
+      "[SPACE]": 7,
+      "’": 8,
+      "Ng": 9,
+      "a": 10,
+      "e": 11,
+      "ng": 12,
+      "o": 13,
+      "\u0000": 14,
+      "\u0001": 15,
+      "\u0002": 16,
+      "\u0003": 17,
+      "\u0004": 18,
+      "\u0005": 19,
+      "\u0006": 20,
+      "\u0007": 21,
+      "\b": 22,
+      "\u000e": 23,
+      "\u000f": 24,
+      "\u0010": 25,
+      "\u0011": 26,
+      "\u0012": 27,
+      "\u0013": 28,
+      "\u0014": 29,
+      "\u0015": 30,
+      "\u0016": 31,
+      "\u0017": 32,
+      "\u0018": 33,
+      "\u0019": 34,
+      "\u001a": 35,
+      "\u001b": 36,
+      "\u001c": 37,
+      "\u001d": 38,
+      "\u001e": 39,
+      "\u001f": 40,
+      "!": 41,
+      "\"": 42,
+      "#": 43,
+      "$": 44,
+      "%": 45,
+      "&": 46,
+      "'": 47,
+      "(": 48,
+      ")": 49,
+      "*": 50,
+      "+": 51,
+      ",": 52,
+      "-": 53,
+      ".": 54,
+      "/": 55,
+      "0": 56,
+      "1": 57,
+      "2": 58,
+      "3": 59,
+      "4": 60,
+      "5": 61,
+      "6": 62,
+      "7": 63,
+      "8": 64,
+      "9": 65,
+      ":": 66,
+      ";": 67,
+      "<": 68,
+      "=": 69,
+      ">": 70,
+      "?": 71,
+      "@": 72,
+      "A": 73,
+      "B": 74,
+      "Ba": 75,
+      "Be": 76,
+      "Bi": 77,
+      "Bo": 78,
+      "Bu": 79,
+      "Bwa": 80,
+      "Bwe": 81,
+      "Bwi": 82,
+      "C": 83,
+      "Cha": 84,
+      "Che": 85,
+      "Chi": 86,
+      "Cho": 87,
+      "Chu": 88,
+      "Chwa": 89,
+      "Chwe": 90,
+      "Chwi": 91,
+      "D": 92,
+      "Da": 93,
+      "De": 94,
+      "Dha": 95,
+      "Dhe": 96,
+      "Dhi": 97,
+      "Dho": 98,
+      "Dhu": 99,
+      "Di": 100,
+      "Do": 101,
+      "Du": 102,
+      "E": 103,
+      "F": 104,
+      "Fa": 105,
+      "Fe": 106,
+      "Fi": 107,
+      "Fo": 108,
+      "Fu": 109,
+      "G": 110,
+      "Ga": 111,
+      "Ge": 112,
+      "Gha": 113,
+      "Ghe": 114,
+      "Ghi": 115,
+      "Gho": 116,
+      "Ghu": 117,
+      "Gi": 118,
+      "Go": 119,
+      "Gu": 120,
+      "Gwa": 121,
+      "Gwe": 122,
+      "Gwi": 123,
+      "H": 124,
+      "Ha": 125,
+      "He": 126,
+      "Hi": 127,
+      "Ho": 128,
+      "Hu": 129,
+      "I": 130,
+      "J": 131,
+      "Ja": 132,
+      "Je": 133,
+      "Ji": 134,
+      "Jo": 135,
+      "Ju": 136,
+      "Jwa": 137,
+      "Jwe": 138,
+      "Jwi": 139,
+      "K": 140,
+      "Ka": 141,
+      "Ke": 142,
+      "Kha": 143,
+      "Khe": 144,
+      "Kho": 145,
+      "Khu": 146,
+      "Ki": 147,
+      "Ko": 148,
+      "Ku": 149,
+      "Kwa": 150,
+      "Kwe": 151,
+      "Kwi": 152,
+      "L": 153,
+      "La": 154,
+      "Le": 155,
+      "Li": 156,
+      "Lo": 157,
+      "Lu": 158,
+      "Lwa": 159,
+      "Lwe": 160,
+      "Lwi": 161,
+      "M": 162,
+      "Ma": 163,
+      "Mba": 164,
+      "Mbe": 165,
+      "Mbi": 166,
+      "Mbo": 167,
+      "Mbu": 168,
+      "Mbwa": 169,
+      "Mbwe": 170,
+      "Mbwi": 171,
+      "Me": 172,
+      "Mi": 173,
+      "Mo": 174,
+      "Mu": 175,
+      "Mwa": 176,
+      "Mwe": 177,
+      "Mwi": 178,
+      "N": 179,
+      "Na": 180,
+      "Nda": 181,
+      "Nde": 182,
+      "Ndi": 183,
+      "Ndo": 184,
+      "Ndu": 185,
+      "Ndwa": 186,
+      "Ndwe": 187,
+      "Ndwi": 188,
+      "Ne": 189,
+      "Nga": 190,
+      "Nge": 191,
+      "Ngi": 192,
+      "Ngo": 193,
+      "Ngu": 194,
+      "Ngwa": 195,
+      "Ngwe": 196,
+      "Ngwi": 197,
+      "Ni": 198,
+      "Nja": 199,
+      "Nje": 200,
+      "Nji": 201,
+      "Njo": 202,
+      "Nju": 203,
+      "Njwa": 204,
+      "Njwe": 205,
+      "Njwi": 206,
+      "No": 207,
+      "Nu": 208,
+      "Nya": 209,
+      "Nye": 210,
+      "Nyi": 211,
+      "Nyo": 212,
+      "Nyu": 213,
+      "Nywa": 214,
+      "Nywe": 215,
+      "Nza": 216,
+      "Nze": 217,
+      "Nzi": 218,
+      "Nzo": 219,
+      "Nzu": 220,
+      "O": 221,
+      "P": 222,
+      "Pa": 223,
+      "Pe": 224,
+      "Pi": 225,
+      "Po": 226,
+      "Pu": 227,
+      "Pwa": 228,
+      "Pwe": 229,
+      "Pwi": 230,
+      "Pwo": 231,
+      "Q": 232,
+      "R": 233,
+      "Ra": 234,
+      "Re": 235,
+      "Ri": 236,
+      "Ro": 237,
+      "Ru": 238,
+      "S": 239,
+      "Sa": 240,
+      "Se": 241,
+      "Sha": 242,
+      "She": 243,
+      "Shi": 244,
+      "Sho": 245,
+      "Shu": 246,
+      "Shwa": 247,
+      "Shwe": 248,
+      "Shwi": 249,
+      "Si": 250,
+      "So": 251,
+      "Su": 252,
+      "Swa": 253,
+      "Swe": 254,
+      "Swi": 255,
+      "T": 256,
+      "Ta": 257,
+      "Te": 258,
+      "Tha": 259,
+      "The": 260,
+      "Thi": 261,
+      "Tho": 262,
+      "Thu": 263,
+      "Ti": 264,
+      "To": 265,
+      "Twa": 266,
+      "Twe": 267,
+      "Twi": 268,
+      "U": 269,
+      "V": 270,
+      "Va": 271,
+      "Ve": 272,
+      "Vi": 273,
+      "Vo": 274,
+      "Vu": 275,
+      "Vya": 276,
+      "Vye": 277,
+      "Vyo": 278,
+      "W": 279,
+      "Wa": 280,
+      "We": 281,
+      "Wi": 282,
+      "Wo": 283,
+      "Wu": 284,
+      "X": 285,
+      "Y": 286,
+      "Ya": 287,
+      "Ye": 288,
+      "Yi": 289,
+      "Yo": 290,
+      "Yu": 291,
+      "Z": 292,
+      "Za": 293,
+      "Ze": 294,
+      "Zi": 295,
+      "Zo": 296,
+      "Zu": 297,
+      "Zwa": 298,
+      "Zwe": 299,
+      "Zwi": 300,
+      "[": 301,
+      "\\": 302,
+      "]": 303,
+      "^": 304,
+      "_": 305,
+      "`": 306,
+      "b": 307,
+      "ba": 308,
+      "be": 309,
+      "bi": 310,
+      "bo": 311,
+      "bu": 312,
+      "bwa": 313,
+      "bwe": 314,
+      "bwi": 315,
+      "c": 316,
+      "cha": 317,
+      "che": 318,
+      "chi": 319,
+      "cho": 320,
+      "chu": 321,
+      "chwa": 322,
+      "chwe": 323,
+      "chwi": 324,
+      "d": 325,
+      "da": 326,
+      "de": 327,
+      "dha": 328,
+      "dhe": 329,
+      "dhi": 330,
+      "dho": 331,
+      "dhu": 332,
+      "di": 333,
+      "do": 334,
+      "du": 335,
+      "f": 336,
+      "fa": 337,
+      "fe": 338,
+      "fi": 339,
+      "fo": 340,
+      "fu": 341,
+      "g": 342,
+      "ga": 343,
+      "ge": 344,
+      "gha": 345,
+      "ghe": 346,
+      "ghi": 347,
+      "gho": 348,
+      "ghu": 349,
+      "gi": 350,
+      "go": 351,
+      "gu": 352,
+      "gwa": 353,
+      "gwe": 354,
+      "gwi": 355,
+      "h": 356,
+      "ha": 357,
+      "he": 358,
+      "hi": 359,
+      "ho": 360,
+      "hu": 361,
+      "i": 362,
+      "j": 363,
+      "ja": 364,
+      "je": 365,
+      "ji": 366,
+      "jo": 367,
+      "ju": 368,
+      "jwa": 369,
+      "jwe": 370,
+      "jwi": 371,
+      "k": 372,
+      "ka": 373,
+      "ke": 374,
+      "kha": 375,
+      "khe": 376,
+      "kho": 377,
+      "khu": 378,
+      "ki": 379,
+      "ko": 380,
+      "ku": 381,
+      "kwa": 382,
+      "kwe": 383,
+      "kwi": 384,
+      "l": 385,
+      "la": 386,
+      "le": 387,
+      "li": 388,
+      "lo": 389,
+      "lu": 390,
+      "lwa": 391,
+      "lwe": 392,
+      "lwi": 393,
+      "m": 394,
+      "ma": 395,
+      "mba": 396,
+      "mbe": 397,
+      "mbi": 398,
+      "mbo": 399,
+      "mbu": 400,
+      "mbwa": 401,
+      "mbwe": 402,
+      "mbwi": 403,
+      "me": 404,
+      "mi": 405,
+      "mo": 406,
+      "mu": 407,
+      "mwa": 408,
+      "mwe": 409,
+      "mwi": 410,
+      "n": 411,
+      "na": 412,
+      "nda": 413,
+      "nde": 414,
+      "ndi": 415,
+      "ndo": 416,
+      "ndu": 417,
+      "ndwa": 418,
+      "ndwe": 419,
+      "ndwi": 420,
+      "ne": 421,
+      "nga": 422,
+      "nge": 423,
+      "ngi": 424,
+      "ngo": 425,
+      "ngu": 426,
+      "ngwa": 427,
+      "ngwe": 428,
+      "ngwi": 429,
+      "ni": 430,
+      "nja": 431,
+      "nje": 432,
+      "nji": 433,
+      "njo": 434,
+      "nju": 435,
+      "njwa": 436,
+      "njwe": 437,
+      "njwi": 438,
+      "no": 439,
+      "nu": 440,
+      "nya": 441,
+      "nye": 442,
+      "nyi": 443,
+      "nyo": 444,
+      "nyu": 445,
+      "nywa": 446,
+      "nywe": 447,
+      "nza": 448,
+      "nze": 449,
+      "nzi": 450,
+      "nzo": 451,
+      "nzu": 452,
+      "p": 453,
+      "pa": 454,
+      "pe": 455,
+      "pi": 456,
+      "po": 457,
+      "pu": 458,
+      "pwa": 459,
+      "pwe": 460,
+      "pwi": 461,
+      "pwo": 462,
+      "q": 463,
+      "r": 464,
+      "ra": 465,
+      "re": 466,
+      "ri": 467,
+      "ro": 468,
+      "ru": 469,
+      "s": 470,
+      "sa": 471,
+      "se": 472,
+      "sha": 473,
+      "she": 474,
+      "shi": 475,
+      "sho": 476,
+      "shu": 477,
+      "shwa": 478,
+      "shwe": 479,
+      "shwi": 480,
+      "si": 481,
+      "so": 482,
+      "su": 483,
+      "swa": 484,
+      "swe": 485,
+      "swi": 486,
+      "t": 487,
+      "ta": 488,
+      "te": 489,
+      "tha": 490,
+      "the": 491,
+      "thi": 492,
+      "tho": 493,
+      "thu": 494,
+      "ti": 495,
+      "to": 496,
+      "twa": 497,
+      "twe": 498,
+      "twi": 499,
+      "u": 500,
+      "v": 501,
+      "va": 502,
+      "ve": 503,
+      "vi": 504,
+      "vo": 505,
+      "vu": 506,
+      "vya": 507,
+      "vye": 508,
+      "vyo": 509,
+      "w": 510,
+      "wa": 511,
+      "we": 512,
+      "wi": 513,
+      "wo": 514,
+      "wu": 515,
+      "x": 516,
+      "y": 517,
+      "ya": 518,
+      "ye": 519,
+      "yi": 520,
+      "yo": 521,
+      "yu": 522,
+      "z": 523,
+      "za": 524,
+      "ze": 525,
+      "zi": 526,
+      "zo": 527,
+      "zu": 528,
+      "zwa": 529,
+      "zwe": 530,
+      "zwi": 531,
+      "{": 532,
+      "|": 533,
+      "}": 534,
+      "~": 535,
+      "": 536,
+      "": 537,
+      "": 538,
+      "": 539,
+      "": 540,
+      "": 541,
+      "": 542,
+      "": 543,
+      "": 544,
+      "": 545,
+      "": 546,
+      "": 547,
+      "": 548,
+      "": 549,
+      "": 550,
+      "": 551,
+      "": 552,
+      "": 553,
+      "": 554,
+      "": 555,
+      "": 556,
+      "": 557,
+      "": 558,
+      "": 559,
+      "": 560,
+      "": 561,
+      "": 562,
+      "": 563,
+      "": 564,
+      "": 565,
+      "": 566,
+      "": 567,
+      "¡": 568,
+      "¢": 569,
+      "£": 570,
+      "¤": 571,
+      "¥": 572,
+      "¦": 573,
+      "§": 574,
+      "¨": 575,
+      "©": 576,
+      "ª": 577,
+      "«": 578,
+      "¬": 579,
+      "": 580,
+      "®": 581,
+      "¯": 582,
+      "°": 583,
+      "±": 584,
+      "²": 585,
+      "³": 586,
+      "´": 587,
+      "µ": 588,
+      "¶": 589,
+      "·": 590,
+      "¸": 591,
+      "¹": 592,
+      "º": 593,
+      "»": 594,
+      "¼": 595,
+      "½": 596,
+      "¾": 597,
+      "¿": 598,
+      "À": 599,
+      "Á": 600,
+      "Â": 601,
+      "Ã": 602,
+      "Ä": 603,
+      "Å": 604,
+      "Æ": 605,
+      "Ç": 606,
+      "È": 607,
+      "É": 608,
+      "Ê": 609,
+      "Ë": 610,
+      "Ì": 611,
+      "Í": 612,
+      "Î": 613,
+      "Ï": 614,
+      "Ð": 615,
+      "Ñ": 616,
+      "Ò": 617,
+      "Ó": 618,
+      "Ô": 619,
+      "Õ": 620,
+      "Ö": 621,
+      "×": 622,
+      "Ø": 623,
+      "Ù": 624,
+      "Ú": 625,
+      "Û": 626,
+      "Ü": 627,
+      "Ý": 628,
+      "Þ": 629,
+      "ß": 630,
+      "à": 631,
+      "á": 632,
+      "â": 633,
+      "ã": 634,
+      "ä": 635,
+      "å": 636,
+      "æ": 637,
+      "ç": 638,
+      "è": 639,
+      "é": 640,
+      "ê": 641,
+      "ë": 642,
+      "ì": 643,
+      "í": 644,
+      "î": 645,
+      "ï": 646,
+      "ð": 647,
+      "ñ": 648,
+      "ò": 649,
+      "ó": 650,
+      "ô": 651,
+      "õ": 652,
+      "ö": 653,
+      "÷": 654,
+      "ø": 655,
+      "ù": 656,
+      "ú": 657,
+      "û": 658,
+      "ü": 659,
+      "ý": 660,
+      "þ": 661
+    },
+    "unk_token": "[UNK]"
+  }
+}

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,75 @@

+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "[SOS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "[EOS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "[SPACE]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "[EOS]",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "tokenizer_class": "KiswahiliSilabiTokenizer",
+  "unk_token": "[UNK]"
+}