Hugging Face's logo

hf-internal-testing
/
tiny-processor-llava

llava
Model card Files
xet
 Community
tiny-processor-llava / tokenizer.json
hf-transformers-bot's picture
hf-transformers-bot
Add tiny processor fixture for testing
172197e verified
Raw
History Blame Contribute Delete
15.7 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<unk>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "</s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 501,
"content": "<image>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 502,
"content": "<pad>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "Prepend",
"prepend": "▁"
},
{
"type": "Replace",
"pattern": {
"String": " "
},
"content": "▁"
}
]
},
"pre_tokenizer": null,
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"SpecialToken": {
"id": "<s>",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
}
],
"pair": [
{
"SpecialToken": {
"id": "<s>",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "<s>",
"type_id": 1
}
},
{
"Sequence": {
"id": "B",
"type_id": 1
}
}
],
"special_tokens": {
"<s>": {
"id": "<s>",
"ids": [
1
],
"tokens": [
"<s>"
]
}
}
},
"decoder": {
"type": "Sequence",
"decoders": [
{
"type": "Replace",
"pattern": {
"String": "▁"
},
"content": " "
},
{
"type": "ByteFallback"
},
{
"type": "Fuse"
},
{
"type": "Strip",
"content": " ",
"start": 1,
"stop": 0
}
]
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": "<unk>",
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": true,
"byte_fallback": true,
"ignore_merges": false,
"vocab": {
"<unk>": 0,
"<s>": 1,
"</s>": 2,
"<0x00>": 3,
"<0x01>": 4,
"<0x02>": 5,
"<0x03>": 6,
"<0x04>": 7,
"<0x05>": 8,
"<0x06>": 9,
"<0x07>": 10,
"<0x08>": 11,
"<0x09>": 12,
"<0x0A>": 13,
"<0x0B>": 14,
"<0x0C>": 15,
"<0x0D>": 16,
"<0x0E>": 17,
"<0x0F>": 18,
"<0x10>": 19,
"<0x11>": 20,
"<0x12>": 21,
"<0x13>": 22,
"<0x14>": 23,
"<0x15>": 24,
"<0x16>": 25,
"<0x17>": 26,
"<0x18>": 27,
"<0x19>": 28,
"<0x1A>": 29,
"<0x1B>": 30,
"<0x1C>": 31,
"<0x1D>": 32,
"<0x1E>": 33,
"<0x1F>": 34,
"<0x20>": 35,
"<0x21>": 36,
"<0x22>": 37,
"<0x23>": 38,
"<0x24>": 39,
"<0x25>": 40,
"<0x26>": 41,
"<0x27>": 42,
"<0x28>": 43,
"<0x29>": 44,
"<0x2A>": 45,
"<0x2B>": 46,
"<0x2C>": 47,
"<0x2D>": 48,
"<0x2E>": 49,
"<0x2F>": 50,
"<0x30>": 51,
"<0x31>": 52,
"<0x32>": 53,
"<0x33>": 54,
"<0x34>": 55,
"<0x35>": 56,
"<0x36>": 57,
"<0x37>": 58,
"<0x38>": 59,
"<0x39>": 60,
"<0x3A>": 61,
"<0x3B>": 62,
"<0x3C>": 63,
"<0x3D>": 64,
"<0x3E>": 65,
"<0x3F>": 66,
"<0x40>": 67,
"<0x41>": 68,
"<0x42>": 69,
"<0x43>": 70,
"<0x44>": 71,
"<0x45>": 72,
"<0x46>": 73,
"<0x47>": 74,
"<0x48>": 75,
"<0x49>": 76,
"<0x4A>": 77,
"<0x4B>": 78,
"<0x4C>": 79,
"<0x4D>": 80,
"<0x4E>": 81,
"<0x4F>": 82,
"<0x50>": 83,
"<0x51>": 84,
"<0x52>": 85,
"<0x53>": 86,
"<0x54>": 87,
"<0x55>": 88,
"<0x56>": 89,
"<0x57>": 90,
"<0x58>": 91,
"<0x59>": 92,
"<0x5A>": 93,
"<0x5B>": 94,
"<0x5C>": 95,
"<0x5D>": 96,
"<0x5E>": 97,
"<0x5F>": 98,
"<0x60>": 99,
"<0x61>": 100,
"<0x62>": 101,
"<0x63>": 102,
"<0x64>": 103,
"<0x65>": 104,
"<0x66>": 105,
"<0x67>": 106,
"<0x68>": 107,
"<0x69>": 108,
"<0x6A>": 109,
"<0x6B>": 110,
"<0x6C>": 111,
"<0x6D>": 112,
"<0x6E>": 113,
"<0x6F>": 114,
"<0x70>": 115,
"<0x71>": 116,
"<0x72>": 117,
"<0x73>": 118,
"<0x74>": 119,
"<0x75>": 120,
"<0x76>": 121,
"<0x77>": 122,
"<0x78>": 123,
"<0x79>": 124,
"<0x7A>": 125,
"<0x7B>": 126,
"<0x7C>": 127,
"<0x7D>": 128,
"<0x7E>": 129,
"<0x7F>": 130,
"<0x80>": 131,
"<0x81>": 132,
"<0x82>": 133,
"<0x83>": 134,
"<0x84>": 135,
"<0x85>": 136,
"<0x86>": 137,
"<0x87>": 138,
"<0x88>": 139,
"<0x89>": 140,
"<0x8A>": 141,
"<0x8B>": 142,
"<0x8C>": 143,
"<0x8D>": 144,
"<0x8E>": 145,
"<0x8F>": 146,
"<0x90>": 147,
"<0x91>": 148,
"<0x92>": 149,
"<0x93>": 150,
"<0x94>": 151,
"<0x95>": 152,
"<0x96>": 153,
"<0x97>": 154,
"<0x98>": 155,
"<0x99>": 156,
"<0x9A>": 157,
"<0x9B>": 158,
"<0x9C>": 159,
"<0x9D>": 160,
"<0x9E>": 161,
"<0x9F>": 162,
"<0xA0>": 163,
"<0xA1>": 164,
"<0xA2>": 165,
"<0xA3>": 166,
"<0xA4>": 167,
"<0xA5>": 168,
"<0xA6>": 169,
"<0xA7>": 170,
"<0xA8>": 171,
"<0xA9>": 172,
"<0xAA>": 173,
"<0xAB>": 174,
"<0xAC>": 175,
"<0xAD>": 176,
"<0xAE>": 177,
"<0xAF>": 178,
"<0xB0>": 179,
"<0xB1>": 180,
"<0xB2>": 181,
"<0xB3>": 182,
"<0xB4>": 183,
"<0xB5>": 184,
"<0xB6>": 185,
"<0xB7>": 186,
"<0xB8>": 187,
"<0xB9>": 188,
"<0xBA>": 189,
"<0xBB>": 190,
"<0xBC>": 191,
"<0xBD>": 192,
"<0xBE>": 193,
"<0xBF>": 194,
"<0xC0>": 195,
"<0xC1>": 196,
"<0xC2>": 197,
"<0xC3>": 198,
"<0xC4>": 199,
"<0xC5>": 200,
"<0xC6>": 201,
"<0xC7>": 202,
"<0xC8>": 203,
"<0xC9>": 204,
"<0xCA>": 205,
"<0xCB>": 206,
"<0xCC>": 207,
"<0xCD>": 208,
"<0xCE>": 209,
"<0xCF>": 210,
"<0xD0>": 211,
"<0xD1>": 212,
"<0xD2>": 213,
"<0xD3>": 214,
"<0xD4>": 215,
"<0xD5>": 216,
"<0xD6>": 217,
"<0xD7>": 218,
"<0xD8>": 219,
"<0xD9>": 220,
"<0xDA>": 221,
"<0xDB>": 222,
"<0xDC>": 223,
"<0xDD>": 224,
"<0xDE>": 225,
"<0xDF>": 226,
"<0xE0>": 227,
"<0xE1>": 228,
"<0xE2>": 229,
"<0xE3>": 230,
"<0xE4>": 231,
"<0xE5>": 232,
"<0xE6>": 233,
"<0xE7>": 234,
"<0xE8>": 235,
"<0xE9>": 236,
"<0xEA>": 237,
"<0xEB>": 238,
"<0xEC>": 239,
"<0xED>": 240,
"<0xEE>": 241,
"<0xEF>": 242,
"<0xF0>": 243,
"<0xF1>": 244,
"<0xF2>": 245,
"<0xF3>": 246,
"<0xF4>": 247,
"<0xF5>": 248,
"<0xF6>": 249,
"<0xF7>": 250,
"<0xF8>": 251,
"<0xF9>": 252,
"<0xFA>": 253,
"<0xFB>": 254,
"<0xFC>": 255,
"<0xFD>": 256,
"<0xFE>": 257,
"<0xFF>": 258,
"▁▁": 259,
"▁t": 260,
"er": 261,
"in": 262,
"▁a": 263,
"en": 264,
"on": 265,
"▁th": 266,
"es": 267,
"▁▁▁▁": 268,
"▁s": 269,
"▁d": 270,
"at": 271,
"or": 272,
"an": 273,
"▁c": 274,
"is": 275,
"re": 276,
"it": 277,
"▁the": 278,
"ar": 279,
"le": 280,
"▁w": 281,
"▁p": 282,
"ou": 283,
"al": 284,
"▁f": 285,
"▁m": 286,
"ed": 287,
"▁o": 288,
"▁b": 289,
"om": 290,
"ion": 291,
"ing": 292,
"ic": 293,
"as": 294,
"el": 295,
"ent": 296,
"▁in": 297,
"▁h": 298,
"nd": 299,
"et": 300,
"▁l": 301,
"▁n": 302,
"st": 303,
"▁to": 304,
"ch": 305,
"▁I": 306,
"ro": 307,
"▁▁▁▁▁▁▁▁": 308,
"il": 309,
"▁of": 310,
"de": 311,
"ct": 312,
"▁(": 313,
"am": 314,
"▁C": 315,
"▁de": 316,
"▁S": 317,
"▁u": 318,
"▁A": 319,
"▁\\": 320,
"▁e": 321,
"▁and": 322,
"▁T": 323,
"ol": 324,
"▁v": 325,
"im": 326,
"ot": 327,
"ad": 328,
"ut": 329,
"▁g": 330,
"em": 331,
"ur": 332,
"id": 333,
"▁*": 334,
"ig": 335,
"ra": 336,
"▁re": 337,
"▁is": 338,
"qu": 339,
"ow": 340,
"▁M": 341,
"est": 342,
"▁y": 343,
"se": 344,
"ve": 345,
"ce": 346,
"ie": 347,
"un": 348,
"▁P": 349,
"▁B": 350,
"ag": 351,
"ul": 352,
"▁=": 353,
"he": 354,
"end": 355,
"ode": 356,
"ter": 357,
"ment": 358,
"os": 359,
"▁D": 360,
"if": 361,
"ation": 362,
"▁for": 363,
"▁r": 364,
"▁L": 365,
"▁you": 366,
"▁be": 367,
"ly": 368,
"ver": 369,
"ab": 370,
"te": 371,
"▁it": 372,
"▁on": 373,
"ri": 374,
"us": 375,
"▁\"": 376,
"▁wh": 377,
"▁con": 378,
"▁H": 379,
"▁st": 380,
"ir": 381,
"▁E": 382,
"▁F": 383,
"ck": 384,
"▁an": 385,
"th": 386,
"eg": 387,
"ay": 388,
"ith": 389,
"▁R": 390,
"ist": 391,
"and": 392,
"▁that": 393,
"▁al": 394,
"▁$": 395,
"▁#": 396,
"od": 397,
"um": 398,
"▁W": 399,
"ht": 400,
"code": 401,
"▁G": 402,
"ate": 403,
"ess": 404,
"▁N": 405,
"ere": 406,
"pp": 407,
"▁as": 408,
"▁se": 409,
"▁pro": 410,
"▁with": 411,
"pe": 412,
"▁k": 413,
"ers": 414,
"pt": 415,
");": 416,
"lo": 417,
"▁▁▁▁▁": 418,
"▁com": 419,
"ame": 420,
"▁`": 421,
"▁Com": 422,
"ia": 423,
"ant": 424,
"▁la": 425,
"▁{": 426,
"▁en": 427,
"ction": 428,
"▁ex": 429,
"ld": 430,
"ub": 431,
"▁j": 432,
"la": 433,
"ue": 434,
"▁J": 435,
"ich": 436,
"▁do": 437,
"▁O": 438,
"▁qu": 439,
"iv": 440,
"ort": 441,
"art": 442,
"▁un": 443,
"▁##": 444,
"▁this": 445,
"ke": 446,
"▁ha": 447,
"▁-": 448,
"out": 449,
"▁The": 450,
"▁not": 451,
"▁ne": 452,
"ill": 453,
"▁le": 454,
"ci": 455,
"rom": 456,
"ine": 457,
"//": 458,
"op": 459,
"egin": 460,
"▁Comment": 461,
"▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁": 462,
"begin": 463,
"ст": 464,
"ass": 465,
"iz": 466,
").": 467,
"og": 468,
"▁п": 469,
"▁or": 470,
"▁was": 471,
"▁at": 472,
"our": 473,
"▁i": 474,
"ain": 475,
"▁K": 476,
"Π½Π°": 477,
"▁V": 478,
"ge": 479,
"▁su": 480,
"ap": 481,
"age": 482,
"ould": 483,
"ne": 484,
"av": 485,
"xt": 486,
"ore": 487,
"ile": 488,
"--": 489,
"▁в": 490,
"▁by": 491,
"li": 492,
"ath": 493,
"Ρ€Π°": 494,
"ber": 495,
"ach": 496,
"all": 497,
"▁Th": 498,
"ult": 499,
"▁": 500
},
"merges": [
[
"▁",
"th"
],
[
"▁t",
"he"
],
[
"▁",
"in"
],
[
"▁",
"de"
],
[
"▁a",
"nd"
],
[
"▁",
"and"
],
[
"▁",
"re"
],
[
"▁",
"is"
],
[
"at",
"ion"
],
[
"▁f",
"or"
],
[
"▁y",
"ou"
],
[
"▁",
"it"
],
[
"▁",
"on"
],
[
"▁c",
"on"
],
[
"▁",
"st"
],
[
"▁",
"an"
],
[
"▁th",
"at"
],
[
"▁",
"al"
],
[
"▁",
"as"
],
[
"▁",
"se"
],
[
"▁p",
"ro"
],
[
"▁w",
"ith"
],
[
"▁c",
"om"
],
[
"▁C",
"om"
],
[
"▁",
"la"
],
[
"▁",
"en"
],
[
"ct",
"ion"
],
[
"▁",
"qu"
],
[
"▁",
"un"
],
[
"▁th",
"is"
],
[
"▁T",
"he"
],
[
"▁n",
"ot"
],
[
"▁",
"ne"
],
[
"▁",
"le"
],
[
"eg",
"in"
],
[
"▁Com",
"ment"
],
[
"▁",
"or"
],
[
"▁w",
"as"
],
[
"▁",
"at"
],
[
"ou",
"ld"
],
[
"▁",
"▁"
],
[
"▁▁",
"▁▁"
],
[
"▁▁▁▁",
"▁▁▁▁"
],
[
"▁▁▁▁",
"▁"
],
[
"▁",
"▁▁▁▁"
],
[
"▁▁▁▁▁▁▁▁",
"▁▁▁▁▁▁▁▁"
]
]
}
}