Hugging Face's logo

hf-internal-testing
/
tiny-processor-owlv2

owlv2
Model card Files
xet
 Community
tiny-processor-owlv2 / tokenizer.json
hf-transformers-bot's picture
hf-transformers-bot
Add tiny processor fixture for testing
7019acd verified
Raw
History Blame Contribute Delete
11.6 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 500,
"special": true,
"content": "<|startoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true
},
{
"id": 501,
"special": true,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false
}
],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "NFC"
},
{
"type": "Replace",
"pattern": {
"Regex": "\\s+"
},
"content": " "
},
{
"type": "Lowercase"
}
]
},
"pre_tokenizer": {
"type": "Sequence",
"pretokenizers": [
{
"type": "Split",
"pattern": {
"Regex": "<\\|startoftext\\|>|<\\|endoftext\\|>|'s|'t|'re|'ve|'m|'ll|'d|[\\p{L}]+|[\\p{N}]|[^\\s\\p{L}\\p{N}]+"
},
"behavior": "Removed",
"invert": true
},
{
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": true
}
]
},
"post_processor": {
"type": "RobertaProcessing",
"sep": [
"<|endoftext|>",
49407
],
"cls": [
"<|startoftext|>",
49406
],
"trim_offsets": false,
"add_prefix_space": false
},
"decoder": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": "<|endoftext|>",
"continuing_subword_prefix": "",
"end_of_word_suffix": "</w>",
"fuse_unk": false,
"vocab": {
"!": 0,
"\"": 1,
"#": 2,
"$": 3,
"%": 4,
"&": 5,
"'": 6,
"(": 7,
")": 8,
"*": 9,
"+": 10,
",": 11,
"-": 12,
".": 13,
"/": 14,
"0": 15,
"1": 16,
"2": 17,
"3": 18,
"4": 19,
"5": 20,
"6": 21,
"7": 22,
"8": 23,
"9": 24,
":": 25,
";": 26,
"<": 27,
"=": 28,
">": 29,
"?": 30,
"@": 31,
"A": 32,
"B": 33,
"C": 34,
"D": 35,
"E": 36,
"F": 37,
"G": 38,
"H": 39,
"I": 40,
"J": 41,
"K": 42,
"L": 43,
"M": 44,
"N": 45,
"O": 46,
"P": 47,
"Q": 48,
"R": 49,
"S": 50,
"T": 51,
"U": 52,
"V": 53,
"W": 54,
"X": 55,
"Y": 56,
"Z": 57,
"[": 58,
"\\": 59,
"]": 60,
"^": 61,
"_": 62,
"`": 63,
"a": 64,
"b": 65,
"c": 66,
"d": 67,
"e": 68,
"f": 69,
"g": 70,
"h": 71,
"i": 72,
"j": 73,
"k": 74,
"l": 75,
"m": 76,
"n": 77,
"o": 78,
"p": 79,
"q": 80,
"r": 81,
"s": 82,
"t": 83,
"u": 84,
"v": 85,
"w": 86,
"x": 87,
"y": 88,
"z": 89,
"{": 90,
"|": 91,
"}": 92,
"~": 93,
"¡": 94,
"¢": 95,
"£": 96,
"¤": 97,
"¥": 98,
"¦": 99,
"§": 100,
"¨": 101,
"©": 102,
"ª": 103,
"«": 104,
"¬": 105,
"®": 106,
"¯": 107,
"°": 108,
"±": 109,
"²": 110,
"³": 111,
"´": 112,
"µ": 113,
"¶": 114,
"·": 115,
"¸": 116,
"¹": 117,
"º": 118,
"»": 119,
"¼": 120,
"½": 121,
"¾": 122,
"¿": 123,
"À": 124,
"Á": 125,
"Â": 126,
"Ã": 127,
"Ä": 128,
"Å": 129,
"Æ": 130,
"Ç": 131,
"È": 132,
"É": 133,
"Ê": 134,
"Ë": 135,
"Ì": 136,
"Í": 137,
"Î": 138,
"Ï": 139,
"Ð": 140,
"Ñ": 141,
"Ò": 142,
"Ó": 143,
"Ô": 144,
"Õ": 145,
"Ö": 146,
"×": 147,
"Ø": 148,
"Ù": 149,
"Ú": 150,
"Û": 151,
"Ü": 152,
"Ý": 153,
"Þ": 154,
"ß": 155,
"à": 156,
"á": 157,
"â": 158,
"ã": 159,
"ä": 160,
"å": 161,
"æ": 162,
"ç": 163,
"è": 164,
"é": 165,
"ê": 166,
"ë": 167,
"ì": 168,
"í": 169,
"î": 170,
"ï": 171,
"ð": 172,
"ñ": 173,
"ò": 174,
"ó": 175,
"ô": 176,
"õ": 177,
"ö": 178,
"÷": 179,
"ø": 180,
"ù": 181,
"ú": 182,
"û": 183,
"ü": 184,
"ý": 185,
"þ": 186,
"ÿ": 187,
"Ā": 188,
"ā": 189,
"Ă": 190,
"ă": 191,
"Ą": 192,
"ą": 193,
"Ć": 194,
"ć": 195,
"Ĉ": 196,
"ĉ": 197,
"Ċ": 198,
"ċ": 199,
"Č": 200,
"č": 201,
"Ď": 202,
"ď": 203,
"Đ": 204,
"đ": 205,
"Ē": 206,
"ē": 207,
"Ĕ": 208,
"ĕ": 209,
"Ė": 210,
"ė": 211,
"Ę": 212,
"ę": 213,
"Ě": 214,
"ě": 215,
"Ĝ": 216,
"ĝ": 217,
"Ğ": 218,
"ğ": 219,
"Ġ": 220,
"ġ": 221,
"Ģ": 222,
"ģ": 223,
"Ĥ": 224,
"ĥ": 225,
"Ħ": 226,
"ħ": 227,
"Ĩ": 228,
"ĩ": 229,
"Ī": 230,
"ī": 231,
"Ĭ": 232,
"ĭ": 233,
"Į": 234,
"į": 235,
"İ": 236,
"ı": 237,
"IJ": 238,
"ij": 239,
"Ĵ": 240,
"ĵ": 241,
"Ķ": 242,
"ķ": 243,
"ĸ": 244,
"Ĺ": 245,
"ĺ": 246,
"Ļ": 247,
"ļ": 248,
"Ľ": 249,
"ľ": 250,
"Ŀ": 251,
"ŀ": 252,
"Ł": 253,
"ł": 254,
"Ń": 255,
"!</w>": 256,
"\"</w>": 257,
"#</w>": 258,
"$</w>": 259,
"%</w>": 260,
"&</w>": 261,
"'</w>": 262,
"(</w>": 263,
")</w>": 264,
"*</w>": 265,
"+</w>": 266,
",</w>": 267,
"-</w>": 268,
".</w>": 269,
"/</w>": 270,
"0</w>": 271,
"1</w>": 272,
"2</w>": 273,
"3</w>": 274,
"4</w>": 275,
"5</w>": 276,
"6</w>": 277,
"7</w>": 278,
"8</w>": 279,
"9</w>": 280,
":</w>": 281,
";</w>": 282,
"<</w>": 283,
"=</w>": 284,
"></w>": 285,
"?</w>": 286,
"@</w>": 287,
"A</w>": 288,
"B</w>": 289,
"C</w>": 290,
"D</w>": 291,
"E</w>": 292,
"F</w>": 293,
"G</w>": 294,
"H</w>": 295,
"I</w>": 296,
"J</w>": 297,
"K</w>": 298,
"L</w>": 299,
"M</w>": 300,
"N</w>": 301,
"O</w>": 302,
"P</w>": 303,
"Q</w>": 304,
"R</w>": 305,
"S</w>": 306,
"T</w>": 307,
"U</w>": 308,
"V</w>": 309,
"W</w>": 310,
"X</w>": 311,
"Y</w>": 312,
"Z</w>": 313,
"[</w>": 314,
"\\</w>": 315,
"]</w>": 316,
"^</w>": 317,
"_</w>": 318,
"`</w>": 319,
"a</w>": 320,
"b</w>": 321,
"c</w>": 322,
"d</w>": 323,
"e</w>": 324,
"f</w>": 325,
"g</w>": 326,
"h</w>": 327,
"i</w>": 328,
"j</w>": 329,
"k</w>": 330,
"l</w>": 331,
"m</w>": 332,
"n</w>": 333,
"o</w>": 334,
"p</w>": 335,
"q</w>": 336,
"r</w>": 337,
"s</w>": 338,
"t</w>": 339,
"u</w>": 340,
"v</w>": 341,
"w</w>": 342,
"x</w>": 343,
"y</w>": 344,
"z</w>": 345,
"{</w>": 346,
"|</w>": 347,
"}</w>": 348,
"~</w>": 349,
"¡</w>": 350,
"¢</w>": 351,
"£</w>": 352,
"¤</w>": 353,
"¥</w>": 354,
"¦</w>": 355,
"§</w>": 356,
"¨</w>": 357,
"©</w>": 358,
"ª</w>": 359,
"«</w>": 360,
"¬</w>": 361,
"®</w>": 362,
"¯</w>": 363,
"°</w>": 364,
"±</w>": 365,
"²</w>": 366,
"³</w>": 367,
"´</w>": 368,
"µ</w>": 369,
"¶</w>": 370,
"·</w>": 371,
"¸</w>": 372,
"¹</w>": 373,
"º</w>": 374,
"»</w>": 375,
"¼</w>": 376,
"½</w>": 377,
"¾</w>": 378,
"¿</w>": 379,
"À</w>": 380,
"Á</w>": 381,
"Â</w>": 382,
"Ã</w>": 383,
"Ä</w>": 384,
"Å</w>": 385,
"Æ</w>": 386,
"Ç</w>": 387,
"È</w>": 388,
"É</w>": 389,
"Ê</w>": 390,
"Ë</w>": 391,
"Ì</w>": 392,
"Í</w>": 393,
"Î</w>": 394,
"Ï</w>": 395,
"Ð</w>": 396,
"Ñ</w>": 397,
"Ò</w>": 398,
"Ó</w>": 399,
"Ô</w>": 400,
"Õ</w>": 401,
"Ö</w>": 402,
"×</w>": 403,
"Ø</w>": 404,
"Ù</w>": 405,
"Ú</w>": 406,
"Û</w>": 407,
"Ü</w>": 408,
"Ý</w>": 409,
"Þ</w>": 410,
"ß</w>": 411,
"à</w>": 412,
"á</w>": 413,
"â</w>": 414,
"ã</w>": 415,
"ä</w>": 416,
"å</w>": 417,
"æ</w>": 418,
"ç</w>": 419,
"è</w>": 420,
"é</w>": 421,
"ê</w>": 422,
"ë</w>": 423,
"ì</w>": 424,
"í</w>": 425,
"î</w>": 426,
"ï</w>": 427,
"ð</w>": 428,
"ñ</w>": 429,
"ò</w>": 430,
"ó</w>": 431,
"ô</w>": 432,
"õ</w>": 433,
"ö</w>": 434,
"÷</w>": 435,
"ø</w>": 436,
"ù</w>": 437,
"ú</w>": 438,
"û</w>": 439,
"ü</w>": 440,
"ý</w>": 441,
"þ</w>": 442,
"ÿ</w>": 443,
"Ā</w>": 444,
"ā</w>": 445,
"Ă</w>": 446,
"ă</w>": 447,
"Ą</w>": 448,
"ą</w>": 449,
"Ć</w>": 450,
"ć</w>": 451,
"Ĉ</w>": 452,
"ĉ</w>": 453,
"Ċ</w>": 454,
"ċ</w>": 455,
"Č</w>": 456,
"č</w>": 457,
"Ď</w>": 458,
"ď</w>": 459,
"Đ</w>": 460,
"đ</w>": 461,
"Ē</w>": 462,
"ē</w>": 463,
"Ĕ</w>": 464,
"ĕ</w>": 465,
"Ė</w>": 466,
"ė</w>": 467,
"Ę</w>": 468,
"ę</w>": 469,
"Ě</w>": 470,
"ě</w>": 471,
"Ĝ</w>": 472,
"ĝ</w>": 473,
"Ğ</w>": 474,
"ğ</w>": 475,
"Ġ</w>": 476,
"ġ</w>": 477,
"Ģ</w>": 478,
"ģ</w>": 479,
"Ĥ</w>": 480,
"ĥ</w>": 481,
"Ħ</w>": 482,
"ħ</w>": 483,
"Ĩ</w>": 484,
"ĩ</w>": 485,
"Ī</w>": 486,
"ī</w>": 487,
"Ĭ</w>": 488,
"ĭ</w>": 489,
"Į</w>": 490,
"į</w>": 491,
"İ</w>": 492,
"ı</w>": 493,
"IJ</w>": 494,
"ij</w>": 495,
"Ĵ</w>": 496,
"ĵ</w>": 497,
"Ķ</w>": 498,
"ķ</w>": 499,
"<|startoftext|>": 500,
"<|endoftext|>": 501
},
"merges": []
}
}