Hugging Face's logo

zaydzuhri
/
tasklets_tokenizer_600

Model card Files
xet
Community
tasklets_tokenizer_600 / tokenizer.json
zaydzuhri's picture
zaydzuhri
Add tokenizer files
165a2dc verified
raw
history blame contribute delete
12.5 kB
 {
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "_",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "[PAD]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "[UNK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "[BOS]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 4,
"content": "[EOS]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "NFD"
},
{
"type": "StripAccents"
},
{
"type": "Lowercase"
}
]
},
"pre_tokenizer": {
"type": "Whitespace"
},
"post_processor": null,
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {
"_": 0,
"[PAD]": 1,
"[UNK]": 2,
"[BOS]": 3,
"[EOS]": 4,
"a": 5,
"b": 6,
"c": 7,
"d": 8,
"e": 9,
"f": 10,
"g": 11,
"h": 12,
"i": 13,
"j": 14,
"k": 15,
"l": 16,
"m": 17,
"n": 18,
"o": 19,
"p": 20,
"q": 21,
"r": 22,
"s": 23,
"t": 24,
"u": 25,
"v": 26,
"w": 27,
"x": 28,
"y": 29,
"z": 30,
"0": 31,
"1": 32,
"2": 33,
"3": 34,
"4": 35,
"5": 36,
"6": 37,
"7": 38,
"8": 39,
"9": 40,
"10": 41,
"11": 42,
"12": 43,
"13": 44,
"14": 45,
"15": 46,
"16": 47,
"17": 48,
"18": 49,
"19": 50,
"20": 51,
"21": 52,
"22": 53,
"23": 54,
"24": 55,
"25": 56,
"26": 57,
"27": 58,
"28": 59,
"29": 60,
"30": 61,
"31": 62,
"32": 63,
"33": 64,
"34": 65,
"35": 66,
"36": 67,
"37": 68,
"38": 69,
"39": 70,
"40": 71,
"41": 72,
"42": 73,
"43": 74,
"44": 75,
"45": 76,
"46": 77,
"47": 78,
"48": 79,
"49": 80,
"50": 81,
"51": 82,
"52": 83,
"53": 84,
"54": 85,
"55": 86,
"56": 87,
"57": 88,
"58": 89,
"59": 90,
"60": 91,
"61": 92,
"62": 93,
"63": 94,
"64": 95,
"65": 96,
"66": 97,
"67": 98,
"68": 99,
"69": 100,
"70": 101,
"71": 102,
"72": 103,
"73": 104,
"74": 105,
"75": 106,
"76": 107,
"77": 108,
"78": 109,
"79": 110,
"80": 111,
"81": 112,
"82": 113,
"83": 114,
"84": 115,
"85": 116,
"86": 117,
"87": 118,
"88": 119,
"89": 120,
"90": 121,
"91": 122,
"92": 123,
"93": 124,
"94": 125,
"95": 126,
"96": 127,
"97": 128,
"98": 129,
"99": 130,
"100": 131,
"101": 132,
"102": 133,
"103": 134,
"104": 135,
"105": 136,
"106": 137,
"107": 138,
"108": 139,
"109": 140,
"110": 141,
"111": 142,
"112": 143,
"113": 144,
"114": 145,
"115": 146,
"116": 147,
"117": 148,
"118": 149,
"119": 150,
"120": 151,
"121": 152,
"122": 153,
"123": 154,
"124": 155,
"125": 156,
"126": 157,
"127": 158,
"128": 159,
"129": 160,
"130": 161,
"131": 162,
"132": 163,
"133": 164,
"134": 165,
"135": 166,
"136": 167,
"137": 168,
"138": 169,
"139": 170,
"140": 171,
"141": 172,
"142": 173,
"143": 174,
"144": 175,
"145": 176,
"146": 177,
"147": 178,
"148": 179,
"149": 180,
"150": 181,
"151": 182,
"152": 183,
"153": 184,
"154": 185,
"155": 186,
"156": 187,
"157": 188,
"158": 189,
"159": 190,
"160": 191,
"161": 192,
"162": 193,
"163": 194,
"164": 195,
"165": 196,
"166": 197,
"167": 198,
"168": 199,
"169": 200,
"170": 201,
"171": 202,
"172": 203,
"173": 204,
"174": 205,
"175": 206,
"176": 207,
"177": 208,
"178": 209,
"179": 210,
"180": 211,
"181": 212,
"182": 213,
"183": 214,
"184": 215,
"185": 216,
"186": 217,
"187": 218,
"188": 219,
"189": 220,
"190": 221,
"191": 222,
"192": 223,
"193": 224,
"194": 225,
"195": 226,
"196": 227,
"197": 228,
"198": 229,
"199": 230,
"200": 231,
"201": 232,
"202": 233,
"203": 234,
"204": 235,
"205": 236,
"206": 237,
"207": 238,
"208": 239,
"209": 240,
"210": 241,
"211": 242,
"212": 243,
"213": 244,
"214": 245,
"215": 246,
"216": 247,
"217": 248,
"218": 249,
"219": 250,
"220": 251,
"221": 252,
"222": 253,
"223": 254,
"224": 255,
"225": 256,
"226": 257,
"227": 258,
"228": 259,
"229": 260,
"230": 261,
"231": 262,
"232": 263,
"233": 264,
"234": 265,
"235": 266,
"236": 267,
"237": 268,
"238": 269,
"239": 270,
"240": 271,
"241": 272,
"242": 273,
"243": 274,
"244": 275,
"245": 276,
"246": 277,
"247": 278,
"248": 279,
"249": 280,
"250": 281,
"251": 282,
"252": 283,
"253": 284,
"254": 285,
"255": 286,
"256": 287,
"257": 288,
"258": 289,
"259": 290,
"260": 291,
"261": 292,
"262": 293,
"263": 294,
"264": 295,
"265": 296,
"266": 297,
"267": 298,
"268": 299,
"269": 300,
"270": 301,
"271": 302,
"272": 303,
"273": 304,
"274": 305,
"275": 306,
"276": 307,
"277": 308,
"278": 309,
"279": 310,
"280": 311,
"281": 312,
"282": 313,
"283": 314,
"284": 315,
"285": 316,
"286": 317,
"287": 318,
"288": 319,
"289": 320,
"290": 321,
"291": 322,
"292": 323,
"293": 324,
"294": 325,
"295": 326,
"296": 327,
"297": 328,
"298": 329,
"299": 330,
"300": 331,
"301": 332,
"302": 333,
"303": 334,
"304": 335,
"305": 336,
"306": 337,
"307": 338,
"308": 339,
"309": 340,
"310": 341,
"311": 342,
"312": 343,
"313": 344,
"314": 345,
"315": 346,
"316": 347,
"317": 348,
"318": 349,
"319": 350,
"320": 351,
"321": 352,
"322": 353,
"323": 354,
"324": 355,
"325": 356,
"326": 357,
"327": 358,
"328": 359,
"329": 360,
"330": 361,
"331": 362,
"332": 363,
"333": 364,
"334": 365,
"335": 366,
"336": 367,
"337": 368,
"338": 369,
"339": 370,
"340": 371,
"341": 372,
"342": 373,
"343": 374,
"344": 375,
"345": 376,
"346": 377,
"347": 378,
"348": 379,
"349": 380,
"350": 381,
"351": 382,
"352": 383,
"353": 384,
"354": 385,
"355": 386,
"356": 387,
"357": 388,
"358": 389,
"359": 390,
"360": 391,
"361": 392,
"362": 393,
"363": 394,
"364": 395,
"365": 396,
"366": 397,
"367": 398,
"368": 399,
"369": 400,
"370": 401,
"371": 402,
"372": 403,
"373": 404,
"374": 405,
"375": 406,
"376": 407,
"377": 408,
"378": 409,
"379": 410,
"380": 411,
"381": 412,
"382": 413,
"383": 414,
"384": 415,
"385": 416,
"386": 417,
"387": 418,
"388": 419,
"389": 420,
"390": 421,
"391": 422,
"392": 423,
"393": 424,
"394": 425,
"395": 426,
"396": 427,
"397": 428,
"398": 429,
"399": 430,
"400": 431,
"401": 432,
"402": 433,
"403": 434,
"404": 435,
"405": 436,
"406": 437,
"407": 438,
"408": 439,
"409": 440,
"410": 441,
"411": 442,
"412": 443,
"413": 444,
"414": 445,
"415": 446,
"416": 447,
"417": 448,
"418": 449,
"419": 450,
"420": 451,
"421": 452,
"422": 453,
"423": 454,
"424": 455,
"425": 456,
"426": 457,
"427": 458,
"428": 459,
"429": 460,
"430": 461,
"431": 462,
"432": 463,
"433": 464,
"434": 465,
"435": 466,
"436": 467,
"437": 468,
"438": 469,
"439": 470,
"440": 471,
"441": 472,
"442": 473,
"443": 474,
"444": 475,
"445": 476,
"446": 477,
"447": 478,
"448": 479,
"449": 480,
"450": 481,
"451": 482,
"452": 483,
"453": 484,
"454": 485,
"455": 486,
"456": 487,
"457": 488,
"458": 489,
"459": 490,
"460": 491,
"461": 492,
"462": 493,
"463": 494,
"464": 495,
"465": 496,
"466": 497,
"467": 498,
"468": 499,
"469": 500,
"470": 501,
"471": 502,
"472": 503,
"473": 504,
"474": 505,
"475": 506,
"476": 507,
"477": 508,
"478": 509,
"479": 510,
"480": 511,
"481": 512,
"482": 513,
"483": 514,
"484": 515,
"485": 516,
"486": 517,
"487": 518,
"488": 519,
"489": 520,
"490": 521,
"491": 522,
"492": 523,
"493": 524,
"494": 525,
"495": 526,
"496": 527,
"497": 528,
"498": 529,
"499": 530,
"500": 531,
"501": 532,
"502": 533,
"503": 534,
"504": 535,
"505": 536,
"506": 537,
"507": 538,
"508": 539,
"509": 540,
"510": 541,
"511": 542,
"512": 543,
"513": 544,
"514": 545,
"515": 546,
"516": 547,
"517": 548,
"518": 549,
"519": 550,
"520": 551,
"521": 552,
"522": 553,
"523": 554,
"524": 555,
"525": 556,
"526": 557,
"527": 558,
"528": 559,
"529": 560,
"530": 561,
"531": 562,
"532": 563,
"533": 564,
"534": 565,
"535": 566,
"536": 567,
"537": 568,
"538": 569,
"539": 570,
"540": 571,
"541": 572,
"542": 573,
"543": 574,
"544": 575,
"545": 576,
"546": 577,
"547": 578,
"548": 579,
"549": 580,
"550": 581,
"551": 582,
"552": 583,
"553": 584,
"554": 585,
"555": 586,
"556": 587,
"557": 588,
"558": 589,
"559": 590,
"560": 591,
"561": 592,
"562": 593,
"563": 594,
"564": 595,
"565": 596,
"566": 597,
"567": 598,
"568": 599,
"569": 600,
"570": 601,
"571": 602,
"572": 603,
"573": 604,
"574": 605,
"575": 606,
"576": 607,
"577": 608,
"578": 609,
"579": 610,
"580": 611,
"581": 612,
"582": 613,
"583": 614,
"584": 615,
"585": 616,
"586": 617,
"587": 618,
"588": 619,
"589": 620,
"590": 621,
"591": 622,
"592": 623,
"593": 624,
"594": 625,
"595": 626,
"596": 627,
"597": 628,
"598": 629,
"599": 630,
"600": 631,
"|": 632,
"?": 633
},
"unk_token": "[UNK]"
}
}