turkish-bpe-tokenizer / tokenizer.json
sedayzc's picture
Upload Turkish Byte-Level BPE tokenizer files
0ec8913 verified
Raw
History Blame Contribute Delete
22.1 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<unk>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<pad>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<bos>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "<eos>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": true,
"use_regex": true
},
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
}
],
"pair": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"Sequence": {
"id": "B",
"type_id": 1
}
}
],
"special_tokens": {}
},
"decoder": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true,
"use_regex": true
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": "<unk>",
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"<unk>": 0,
"<pad>": 1,
"<bos>": 2,
"<eos>": 3,
"!": 4,
"\"": 5,
"#": 6,
"$": 7,
"%": 8,
"&": 9,
"'": 10,
"(": 11,
")": 12,
"*": 13,
"+": 14,
",": 15,
"-": 16,
".": 17,
"/": 18,
"0": 19,
"1": 20,
"2": 21,
"3": 22,
"4": 23,
"5": 24,
"6": 25,
"7": 26,
"8": 27,
"9": 28,
":": 29,
";": 30,
"<": 31,
"=": 32,
">": 33,
"?": 34,
"@": 35,
"A": 36,
"B": 37,
"C": 38,
"D": 39,
"E": 40,
"F": 41,
"G": 42,
"H": 43,
"I": 44,
"J": 45,
"K": 46,
"L": 47,
"M": 48,
"N": 49,
"O": 50,
"P": 51,
"Q": 52,
"R": 53,
"S": 54,
"T": 55,
"U": 56,
"V": 57,
"W": 58,
"X": 59,
"Y": 60,
"Z": 61,
"[": 62,
"\\": 63,
"]": 64,
"^": 65,
"_": 66,
"`": 67,
"a": 68,
"b": 69,
"c": 70,
"d": 71,
"e": 72,
"f": 73,
"g": 74,
"h": 75,
"i": 76,
"j": 77,
"k": 78,
"l": 79,
"m": 80,
"n": 81,
"o": 82,
"p": 83,
"q": 84,
"r": 85,
"s": 86,
"t": 87,
"u": 88,
"v": 89,
"w": 90,
"x": 91,
"y": 92,
"z": 93,
"{": 94,
"|": 95,
"}": 96,
"~": 97,
"¡": 98,
"¢": 99,
"£": 100,
"¤": 101,
"¥": 102,
"¦": 103,
"§": 104,
"¨": 105,
"©": 106,
"ª": 107,
"«": 108,
"¬": 109,
"®": 110,
"¯": 111,
"°": 112,
"±": 113,
"²": 114,
"³": 115,
"´": 116,
"µ": 117,
"¶": 118,
"·": 119,
"¸": 120,
"¹": 121,
"º": 122,
"»": 123,
"¼": 124,
"½": 125,
"¾": 126,
"¿": 127,
"À": 128,
"Á": 129,
"Â": 130,
"Ã": 131,
"Ä": 132,
"Å": 133,
"Æ": 134,
"Ç": 135,
"È": 136,
"É": 137,
"Ê": 138,
"Ë": 139,
"Ì": 140,
"Í": 141,
"Î": 142,
"Ï": 143,
"Ð": 144,
"Ñ": 145,
"Ò": 146,
"Ó": 147,
"Ô": 148,
"Õ": 149,
"Ö": 150,
"×": 151,
"Ø": 152,
"Ù": 153,
"Ú": 154,
"Û": 155,
"Ü": 156,
"Ý": 157,
"Þ": 158,
"ß": 159,
"à": 160,
"á": 161,
"â": 162,
"ã": 163,
"ä": 164,
"å": 165,
"æ": 166,
"ç": 167,
"è": 168,
"é": 169,
"ê": 170,
"ë": 171,
"ì": 172,
"í": 173,
"î": 174,
"ï": 175,
"ð": 176,
"ñ": 177,
"ò": 178,
"ó": 179,
"ô": 180,
"õ": 181,
"ö": 182,
"÷": 183,
"ø": 184,
"ù": 185,
"ú": 186,
"û": 187,
"ü": 188,
"ý": 189,
"þ": 190,
"ÿ": 191,
"Ā": 192,
"ā": 193,
"Ă": 194,
"ă": 195,
"Ą": 196,
"ą": 197,
"Ć": 198,
"ć": 199,
"Ĉ": 200,
"ĉ": 201,
"Ċ": 202,
"ċ": 203,
"Č": 204,
"č": 205,
"Ď": 206,
"ď": 207,
"Đ": 208,
"đ": 209,
"Ē": 210,
"ē": 211,
"Ĕ": 212,
"ĕ": 213,
"Ė": 214,
"ė": 215,
"Ę": 216,
"ę": 217,
"Ě": 218,
"ě": 219,
"Ĝ": 220,
"ĝ": 221,
"Ğ": 222,
"ğ": 223,
"Ġ": 224,
"ġ": 225,
"Ģ": 226,
"ģ": 227,
"Ĥ": 228,
"ĥ": 229,
"Ħ": 230,
"ħ": 231,
"Ĩ": 232,
"ĩ": 233,
"Ī": 234,
"ī": 235,
"Ĭ": 236,
"ĭ": 237,
"Į": 238,
"į": 239,
"İ": 240,
"ı": 241,
"IJ": 242,
"ij": 243,
"Ĵ": 244,
"ĵ": 245,
"Ķ": 246,
"ķ": 247,
"ĸ": 248,
"Ĺ": 249,
"ĺ": 250,
"Ļ": 251,
"ļ": 252,
"Ľ": 253,
"ľ": 254,
"Ŀ": 255,
"ŀ": 256,
"Ł": 257,
"ł": 258,
"Ń": 259,
"ı": 260,
"ü": 261,
"ar": 262,
"ın": 263,
"in": 264,
"an": 265,
"de": 266,
"er": 267,
"Ġb": 268,
"ÄŁ": 269,
"ÅŁ": 270,
"Ġk": 271,
"ç": 272,
"Ġg": 273,
"Ġs": 274,
"Ġy": 275,
"lar": 276,
"ö": 277,
"ir": 278,
"da": 279,
"un": 280,
"on": 281,
"den": 282,
"ak": 283,
"di": 284,
"du": 285,
"or": 286,
"iz": 287,
"dı": 288,
"en": 289,
"ler": 290,
"ay": 291,
"es": 292,
"tı": 293,
"Ġt": 294,
"Ġbir": 295,
"al": 296,
"la": 297,
"ün": 298,
"Ġden": 299,
"Ġdeniz": 300,
"os": 301,
"âĢ": 302,
"Ġh": 303,
"le": 304,
"Ġi": 305,
"Ġv": 306,
"ını": 307,
"Ġge": 308,
"yor": 309,
"Ġa": 310,
"li": 311,
"ek": 312,
"lm": 313,
"Ġgö": 314,
"Ġç": 315,
"âĢĻ": 316,
"ah": 317,
"aÅŁ": 318,
"ab": 319,
"Ġo": 320,
"as": 321,
"yordu": 322,
"ap": 323,
"et": 324,
"ki": 325,
"Ä±ÅŁ": 326,
"inde": 327,
"ların": 328,
"ĠA": 329,
"Ġan": 330,
"mi": 331,
"at": 332,
"ti": 333,
"ında": 334,
"ini": 335,
"Ġve": 336,
"Ġd": 337,
"ıl": 338,
"ır": 339,
"dıģ": 340,
"üz": 341,
"ları": 342,
"aÄŁ": 343,
"Ġde": 344,
"Ġö": 345,
"eri": 346,
"dan": 347,
"aya": 348,
"id": 349,
"ĠP": 350,
"Ġf": 351,
"ık": 352,
"Le": 353,
"dr": 354,
"eid": 355,
"il": 356,
"ĠLe": 357,
"ıy": 358,
"andr": 359,
"Ġsö": 360,
"unu": 361,
"leri": 362,
"oseid": 363,
"ĠPoseid": 364,
"ĠLeandr": 365,
"ĠPoseidon": 366,
"ĠLeandros": 367,
"ul": 368,
"yle": 369,
"Ġon": 370,
"ız": 371,
"ce": 372,
"it": 373,
"lk": 374,
"ma": 375,
"me": 376,
"ol": 377,
"tar": 378,
"Ġu": 379,
"Ġda": 380,
"ĠB": 381,
"Ġin": 382,
"se": 383,
"te": 384,
"tu": 385,
"Ġe": 386,
"üç": 387,
"lerin": 388,
"Ġgör": 389,
"ahtar": 390,
"Ġar": 391,
"mÄ±ÅŁ": 392,
"Ġp": 393,
"ük": 394,
"Ġkar": 395,
"Ġses": 396,
"akat": 397,
"Ġiç": 398,
"apın": 399,
"Ġanahtar": 400,
"dıģını": 401,
"ca": 402,
"rı": 403,
"ye": 404,
"ĠN": 405,
"ĠS": 406,
"dü": 407,
"ĠH": 408,
"ĠT": 409,
"Ġdu": 410,
"Ġal": 411,
"ına": 412,
"ÄŁu": 413,
"Ġkor": 414,
"Ġyer": 415,
"Ġyal": 416,
"Ġgemi": 417,
"lı": 418,
"man": 419,
"us": 420,
"Ġkay": 421,
"Ġsi": 422,
"Ġtan": 423,
"Ġdenizin": 424,
"Ġsöyle": 425,
"siz": 426,
"uÅŁ": 427,
"â": 428,
"Ġay": 429,
"ür": 430,
"ine": 431,
"diÄŁ": 432,
"laÅŁ": 433,
"Ġuz": 434,
"lan": 435,
"nız": 436,
"san": 437,
"inin": 438,
"Ġbu": 439,
"ĠbaÅŁ": 440,
"Ġgü": 441,
"ĠtaÅŁ": 442,
"Ġgeç": 443,
"ĠdeÄŁ": 444,
"Ġyalnız": 445,
"Ġtanrı": 446,
"Ġyalnızca": 447,
"ku": 448,
"Ġet": 449,
"Ġkab": 450,
"ön": 451,
"erion": 452,
"Ġinsan": 453,
"Ġpar": 454,
"ĠNerion": 455,
"uk": 456,
"Ġes": 457,
"Ġüç": 458,
"Ġbü": 459,
"ÄŁunu": 460,
"Ġken": 461,
"Ġkıy": 462,
"Ġson": 463,
"ladı": 464,
"ĠdeÄŁil": 465,
"Ġeski": 466,
"Ze": 467,
"ade": 468,
"atı": 469,
"ci": 470,
"cak": 471,
"ev": 472,
"lg": 473,
"lü": 474,
"ne": 475,
"ru": 476,
"th": 477,
"yah": 478,
"ĠO": 479,
"Ġz": 480,
"ĠZe": 481,
"Ġyıl": 482,
"larını": 483,
"Ġise": 484,
"Ġaç": 485,
"Ġada": 486,
"lik": 487,
"Ġçık": 488,
"ası": 489,
"Ġfakat": 490,
"mÄ±ÅŁtı": 491,
"Ġalt": 492,
"ĠZeus": 493,
"aç": 494,
"ky": 495,
"mes": 496,
"minin": 497,
"rin": 498,
"Ġm": 499,
"ģı": 500,
"Ġkap": 501,
"Ġger": 502,
"ena": 503,
"üny": 504,
"ünü": 505,
"Ġol": 506,
"Ġolm": 507,
"ĠAth": 508,
"Ġdüny": 509,
"Ġkayb": 510,
"Ġsöyledi": 511
},
"merges": [
[
"Ä",
"±"
],
[
"Ã",
"¼"
],
[
"a",
"r"
],
[
"ı",
"n"
],
[
"i",
"n"
],
[
"a",
"n"
],
[
"d",
"e"
],
[
"e",
"r"
],
[
"Ġ",
"b"
],
[
"Ä",
"Ł"
],
[
"Å",
"Ł"
],
[
"Ġ",
"k"
],
[
"Ã",
"§"
],
[
"Ġ",
"g"
],
[
"Ġ",
"s"
],
[
"Ġ",
"y"
],
[
"l",
"ar"
],
[
"Ã",
"¶"
],
[
"i",
"r"
],
[
"d",
"a"
],
[
"u",
"n"
],
[
"o",
"n"
],
[
"de",
"n"
],
[
"a",
"k"
],
[
"d",
"i"
],
[
"d",
"u"
],
[
"o",
"r"
],
[
"i",
"z"
],
[
"d",
"ı"
],
[
"e",
"n"
],
[
"l",
"er"
],
[
"a",
"y"
],
[
"e",
"s"
],
[
"t",
"ı"
],
[
"Ġ",
"t"
],
[
"Ġb",
"ir"
],
[
"a",
"l"
],
[
"l",
"a"
],
[
"ü",
"n"
],
[
"Ġ",
"den"
],
[
"Ġden",
"iz"
],
[
"o",
"s"
],
[
"â",
"Ģ"
],
[
"Ġ",
"h"
],
[
"l",
"e"
],
[
"Ġ",
"i"
],
[
"Ġ",
"v"
],
[
"ın",
"ı"
],
[
"Ġg",
"e"
],
[
"y",
"or"
],
[
"Ġ",
"a"
],
[
"l",
"i"
],
[
"e",
"k"
],
[
"l",
"m"
],
[
"Ġg",
"ö"
],
[
"Ġ",
"ç"
],
[
"âĢ",
"Ļ"
],
[
"a",
"h"
],
[
"a",
"ÅŁ"
],
[
"a",
"b"
],
[
"Ġ",
"o"
],
[
"a",
"s"
],
[
"yor",
"du"
],
[
"a",
"p"
],
[
"e",
"t"
],
[
"k",
"i"
],
[
"ı",
"ÅŁ"
],
[
"in",
"de"
],
[
"lar",
"ın"
],
[
"Ġ",
"A"
],
[
"Ġ",
"an"
],
[
"m",
"i"
],
[
"a",
"t"
],
[
"t",
"i"
],
[
"ın",
"da"
],
[
"in",
"i"
],
[
"Ġv",
"e"
],
[
"Ġ",
"d"
],
[
"ı",
"l"
],
[
"ı",
"r"
],
[
"dı",
"ÄŁ"
],
[
"ü",
"z"
],
[
"lar",
"ı"
],
[
"a",
"ÄŁ"
],
[
"Ġ",
"de"
],
[
"Ġ",
"ö"
],
[
"er",
"i"
],
[
"d",
"an"
],
[
"ay",
"a"
],
[
"i",
"d"
],
[
"Ġ",
"P"
],
[
"Ġ",
"f"
],
[
"ı",
"k"
],
[
"L",
"e"
],
[
"d",
"r"
],
[
"e",
"id"
],
[
"i",
"l"
],
[
"Ġ",
"Le"
],
[
"ı",
"y"
],
[
"an",
"dr"
],
[
"Ġs",
"ö"
],
[
"un",
"u"
],
[
"ler",
"i"
],
[
"os",
"eid"
],
[
"ĠP",
"oseid"
],
[
"ĠLe",
"andr"
],
[
"ĠPoseid",
"on"
],
[
"ĠLeandr",
"os"
],
[
"u",
"l"
],
[
"y",
"le"
],
[
"Ġ",
"on"
],
[
"ı",
"z"
],
[
"c",
"e"
],
[
"i",
"t"
],
[
"l",
"k"
],
[
"m",
"a"
],
[
"m",
"e"
],
[
"o",
"l"
],
[
"t",
"ar"
],
[
"Ġ",
"u"
],
[
"Ġ",
"da"
],
[
"Ġ",
"B"
],
[
"Ġ",
"in"
],
[
"s",
"e"
],
[
"t",
"e"
],
[
"t",
"u"
],
[
"Ġ",
"e"
],
[
"ü",
"ç"
],
[
"ler",
"in"
],
[
"Ġgö",
"r"
],
[
"ah",
"tar"
],
[
"Ġ",
"ar"
],
[
"m",
"Ä±ÅŁ"
],
[
"Ġ",
"p"
],
[
"ü",
"k"
],
[
"Ġk",
"ar"
],
[
"Ġs",
"es"
],
[
"ak",
"at"
],
[
"Ġi",
"ç"
],
[
"ap",
"ın"
],
[
"Ġan",
"ahtar"
],
[
"dıģ",
"ını"
],
[
"c",
"a"
],
[
"r",
"ı"
],
[
"y",
"e"
],
[
"Ġ",
"N"
],
[
"Ġ",
"S"
],
[
"d",
"ü"
],
[
"Ġ",
"H"
],
[
"Ġ",
"T"
],
[
"Ġ",
"du"
],
[
"Ġ",
"al"
],
[
"ın",
"a"
],
[
"ÄŁ",
"u"
],
[
"Ġk",
"or"
],
[
"Ġy",
"er"
],
[
"Ġy",
"al"
],
[
"Ġge",
"mi"
],
[
"l",
"ı"
],
[
"m",
"an"
],
[
"u",
"s"
],
[
"Ġk",
"ay"
],
[
"Ġs",
"i"
],
[
"Ġt",
"an"
],
[
"Ġdeniz",
"in"
],
[
"Ġsö",
"yle"
],
[
"s",
"iz"
],
[
"u",
"ÅŁ"
],
[
"Ã",
"¢"
],
[
"Ġ",
"ay"
],
[
"ü",
"r"
],
[
"in",
"e"
],
[
"di",
"ÄŁ"
],
[
"la",
"ÅŁ"
],
[
"Ġu",
"z"
],
[
"l",
"an"
],
[
"n",
"ız"
],
[
"s",
"an"
],
[
"in",
"in"
],
[
"Ġb",
"u"
],
[
"Ġb",
"aÅŁ"
],
[
"Ġg",
"ü"
],
[
"Ġt",
"aÅŁ"
],
[
"Ġge",
"ç"
],
[
"Ġde",
"ÄŁ"
],
[
"Ġyal",
"nız"
],
[
"Ġtan",
"rı"
],
[
"Ġyalnız",
"ca"
],
[
"k",
"u"
],
[
"Ġ",
"et"
],
[
"Ġk",
"ab"
],
[
"ö",
"n"
],
[
"eri",
"on"
],
[
"Ġin",
"san"
],
[
"Ġp",
"ar"
],
[
"ĠN",
"erion"
],
[
"u",
"k"
],
[
"Ġ",
"es"
],
[
"Ġ",
"üç"
],
[
"Ġb",
"ü"
],
[
"ÄŁ",
"unu"
],
[
"Ġk",
"en"
],
[
"Ġk",
"ıy"
],
[
"Ġs",
"on"
],
[
"la",
"dı"
],
[
"ĠdeÄŁ",
"il"
],
[
"Ġes",
"ki"
],
[
"Z",
"e"
],
[
"a",
"de"
],
[
"a",
"tı"
],
[
"c",
"i"
],
[
"c",
"ak"
],
[
"e",
"v"
],
[
"l",
"g"
],
[
"l",
"ü"
],
[
"n",
"e"
],
[
"r",
"u"
],
[
"t",
"h"
],
[
"y",
"ah"
],
[
"Ġ",
"O"
],
[
"Ġ",
"z"
],
[
"Ġ",
"Ze"
],
[
"Ġy",
"ıl"
],
[
"lar",
"ını"
],
[
"Ġi",
"se"
],
[
"Ġa",
"ç"
],
[
"Ġa",
"da"
],
[
"li",
"k"
],
[
"Ġç",
"ık"
],
[
"as",
"ı"
],
[
"Ġf",
"akat"
],
[
"mÄ±ÅŁ",
"tı"
],
[
"Ġal",
"t"
],
[
"ĠZe",
"us"
],
[
"a",
"ç"
],
[
"k",
"y"
],
[
"m",
"es"
],
[
"m",
"inin"
],
[
"r",
"in"
],
[
"Ġ",
"m"
],
[
"ÄŁ",
"ı"
],
[
"Ġk",
"ap"
],
[
"Ġg",
"er"
],
[
"en",
"a"
],
[
"ün",
"y"
],
[
"ün",
"ü"
],
[
"Ġo",
"l"
],
[
"Ġo",
"lm"
],
[
"ĠA",
"th"
],
[
"Ġd",
"üny"
],
[
"Ġkay",
"b"
],
[
"Ġsöyle",
"di"
]
]
}
}