hf-transformers-bot's picture
Add tiny processor fixture for testing
234292a verified
Raw
History Blame Contribute Delete
26.3 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 500,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 501,
"content": "<|im_start|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 502,
"content": "<|im_end|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 503,
"content": "<|AUDIO|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 504,
"content": "<|audio_bos|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 505,
"content": "<|audio_eos|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 506,
"content": "<|box_end|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 507,
"content": "<|quad_start|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 508,
"content": "<|quad_end|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 509,
"content": "<|vision_bos|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 510,
"content": "<|vision_eos|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 511,
"content": "<|vision_pad|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 512,
"content": "<|IMAGE|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 513,
"content": "<|VIDEO|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 514,
"content": "<tool_call>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": false
},
{
"id": 515,
"content": "</tool_call>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": false
},
{
"id": 516,
"content": "<|fim_prefix|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": false
},
{
"id": 517,
"content": "<|fim_middle|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": false
},
{
"id": 518,
"content": "<|fim_suffix|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": false
},
{
"id": 519,
"content": "<|fim_pad|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": false
},
{
"id": 520,
"content": "<|repo_name|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": false
},
{
"id": 521,
"content": "<|file_sep|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": false
}
],
"normalizer": {
"type": "NFC"
},
"pre_tokenizer": {
"type": "Sequence",
"pretokenizers": [
{
"type": "Split",
"pattern": {
"Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
},
"behavior": "Isolated",
"invert": false
},
{
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": false,
"use_regex": false
}
]
},
"post_processor": {
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": false,
"use_regex": false
},
"decoder": {
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": false,
"use_regex": false
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": "",
"end_of_word_suffix": "",
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"!": 0,
"\"": 1,
"#": 2,
"$": 3,
"%": 4,
"&": 5,
"'": 6,
"(": 7,
")": 8,
"*": 9,
"+": 10,
",": 11,
"-": 12,
".": 13,
"/": 14,
"0": 15,
"1": 16,
"2": 17,
"3": 18,
"4": 19,
"5": 20,
"6": 21,
"7": 22,
"8": 23,
"9": 24,
":": 25,
";": 26,
"<": 27,
"=": 28,
">": 29,
"?": 30,
"@": 31,
"A": 32,
"B": 33,
"C": 34,
"D": 35,
"E": 36,
"F": 37,
"G": 38,
"H": 39,
"I": 40,
"J": 41,
"K": 42,
"L": 43,
"M": 44,
"N": 45,
"O": 46,
"P": 47,
"Q": 48,
"R": 49,
"S": 50,
"T": 51,
"U": 52,
"V": 53,
"W": 54,
"X": 55,
"Y": 56,
"Z": 57,
"[": 58,
"\\": 59,
"]": 60,
"^": 61,
"_": 62,
"`": 63,
"a": 64,
"b": 65,
"c": 66,
"d": 67,
"e": 68,
"f": 69,
"g": 70,
"h": 71,
"i": 72,
"j": 73,
"k": 74,
"l": 75,
"m": 76,
"n": 77,
"o": 78,
"p": 79,
"q": 80,
"r": 81,
"s": 82,
"t": 83,
"u": 84,
"v": 85,
"w": 86,
"x": 87,
"y": 88,
"z": 89,
"{": 90,
"|": 91,
"}": 92,
"~": 93,
"¡": 94,
"¢": 95,
"£": 96,
"¤": 97,
"¥": 98,
"¦": 99,
"§": 100,
"¨": 101,
"©": 102,
"ª": 103,
"«": 104,
"¬": 105,
"®": 106,
"¯": 107,
"°": 108,
"±": 109,
"²": 110,
"³": 111,
"´": 112,
"µ": 113,
"¶": 114,
"·": 115,
"¸": 116,
"¹": 117,
"º": 118,
"»": 119,
"¼": 120,
"½": 121,
"¾": 122,
"¿": 123,
"À": 124,
"Á": 125,
"Â": 126,
"Ã": 127,
"Ä": 128,
"Å": 129,
"Æ": 130,
"Ç": 131,
"È": 132,
"É": 133,
"Ê": 134,
"Ë": 135,
"Ì": 136,
"Í": 137,
"Î": 138,
"Ï": 139,
"Ð": 140,
"Ñ": 141,
"Ò": 142,
"Ó": 143,
"Ô": 144,
"Õ": 145,
"Ö": 146,
"×": 147,
"Ø": 148,
"Ù": 149,
"Ú": 150,
"Û": 151,
"Ü": 152,
"Ý": 153,
"Þ": 154,
"ß": 155,
"à": 156,
"á": 157,
"â": 158,
"ã": 159,
"ä": 160,
"å": 161,
"æ": 162,
"ç": 163,
"è": 164,
"é": 165,
"ê": 166,
"ë": 167,
"ì": 168,
"í": 169,
"î": 170,
"ï": 171,
"ð": 172,
"ñ": 173,
"ò": 174,
"ó": 175,
"ô": 176,
"õ": 177,
"ö": 178,
"÷": 179,
"ø": 180,
"ù": 181,
"ú": 182,
"û": 183,
"ü": 184,
"ý": 185,
"þ": 186,
"ÿ": 187,
"Ā": 188,
"ā": 189,
"Ă": 190,
"ă": 191,
"Ą": 192,
"ą": 193,
"Ć": 194,
"ć": 195,
"Ĉ": 196,
"ĉ": 197,
"Ċ": 198,
"ċ": 199,
"Č": 200,
"č": 201,
"Ď": 202,
"ď": 203,
"Đ": 204,
"đ": 205,
"Ē": 206,
"ē": 207,
"Ĕ": 208,
"ĕ": 209,
"Ė": 210,
"ė": 211,
"Ę": 212,
"ę": 213,
"Ě": 214,
"ě": 215,
"Ĝ": 216,
"ĝ": 217,
"Ğ": 218,
"ğ": 219,
"Ġ": 220,
"ġ": 221,
"Ģ": 222,
"ģ": 223,
"Ĥ": 224,
"ĥ": 225,
"Ħ": 226,
"ħ": 227,
"Ĩ": 228,
"ĩ": 229,
"Ī": 230,
"ī": 231,
"Ĭ": 232,
"ĭ": 233,
"Į": 234,
"į": 235,
"İ": 236,
"ı": 237,
"IJ": 238,
"ij": 239,
"Ĵ": 240,
"ĵ": 241,
"Ķ": 242,
"ķ": 243,
"ĸ": 244,
"Ĺ": 245,
"ĺ": 246,
"Ļ": 247,
"ļ": 248,
"Ľ": 249,
"ľ": 250,
"Ŀ": 251,
"ŀ": 252,
"Ł": 253,
"ł": 254,
"Ń": 255,
"ĠĠ": 256,
"ĠĠĠĠ": 257,
"in": 258,
"Ġt": 259,
"ĠĠĠĠĠĠĠĠ": 260,
"er": 261,
"ĠĠĠ": 262,
"on": 263,
"Ġa": 264,
"re": 265,
"at": 266,
"st": 267,
"en": 268,
"or": 269,
"Ġth": 270,
"ĊĊ": 271,
"Ġc": 272,
"le": 273,
"Ġs": 274,
"it": 275,
"an": 276,
"ar": 277,
"al": 278,
"Ġthe": 279,
";Ċ": 280,
"Ġp": 281,
"Ġf": 282,
"ou": 283,
"Ġ=": 284,
"is": 285,
"ĠĠĠĠĠĠĠ": 286,
"ing": 287,
"es": 288,
"Ġw": 289,
"ion": 290,
"ed": 291,
"ic": 292,
"Ġb": 293,
"Ġd": 294,
"et": 295,
"Ġm": 296,
"Ġo": 297,
"ĉĉ": 298,
"ro": 299,
"as": 300,
"el": 301,
"ct": 302,
"nd": 303,
"Ġin": 304,
"Ġh": 305,
"ent": 306,
"id": 307,
"Ġn": 308,
"am": 309,
"ĠĠĠĠĠĠĠĠĠĠĠ": 310,
"Ġto": 311,
"Ġre": 312,
"--": 313,
"Ġ{": 314,
"Ġof": 315,
"om": 316,
");Ċ": 317,
"im": 318,
"čĊ": 319,
"Ġ(": 320,
"il": 321,
"//": 322,
"Ġand": 323,
"ur": 324,
"se": 325,
"Ġl": 326,
"ex": 327,
"ĠS": 328,
"ad": 329,
"Ġ\"": 330,
"ch": 331,
"ut": 332,
"if": 333,
"**": 334,
"Ġ}": 335,
"em": 336,
"ol": 337,
"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 338,
"th": 339,
")Ċ": 340,
"Ġ{Ċ": 341,
"Ġg": 342,
"ig": 343,
"iv": 344,
",Ċ": 345,
"ce": 346,
"od": 347,
"Ġv": 348,
"ate": 349,
"ĠT": 350,
"ag": 351,
"ay": 352,
"Ġ*": 353,
"ot": 354,
"us": 355,
"ĠC": 356,
"Ġst": 357,
"ĠI": 358,
"un": 359,
"ul": 360,
"ue": 361,
"ĠA": 362,
"ow": 363,
"Ġ'": 364,
"ew": 365,
"Ġ<": 366,
"ation": 367,
"()": 368,
"Ġfor": 369,
"ab": 370,
"ort": 371,
"um": 372,
"ame": 373,
"Ġis": 374,
"pe": 375,
"tr": 376,
"ck": 377,
"âĢ": 378,
"Ġy": 379,
"ist": 380,
"----": 381,
".ĊĊ": 382,
"he": 383,
"Ġe": 384,
"lo": 385,
"ĠM": 386,
"Ġbe": 387,
"ers": 388,
"Ġon": 389,
"Ġcon": 390,
"ap": 391,
"ub": 392,
"ĠP": 393,
"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 394,
"ass": 395,
"int": 396,
">Ċ": 397,
"ly": 398,
"urn": 399,
"Ġ$": 400,
";ĊĊ": 401,
"av": 402,
"port": 403,
"ir": 404,
"->": 405,
"nt": 406,
"ction": 407,
"end": 408,
"Ġde": 409,
"ith": 410,
"out": 411,
"turn": 412,
"our": 413,
"ĠĠĠĠĠ": 414,
"lic": 415,
"res": 416,
"pt": 417,
"==": 418,
"Ġthis": 419,
"Ġwh": 420,
"Ġif": 421,
"ĠD": 422,
"ver": 423,
"age": 424,
"ĠB": 425,
"ht": 426,
"ext": 427,
"=\"": 428,
"Ġthat": 429,
"****": 430,
"ĠR": 431,
"Ġit": 432,
"ess": 433,
"ĠF": 434,
"Ġr": 435,
"os": 436,
"and": 437,
"Ġas": 438,
"ect": 439,
"ke": 440,
"rom": 441,
"Ġ//": 442,
"con": 443,
"ĠL": 444,
"(\"": 445,
"qu": 446,
"lass": 447,
"Ġwith": 448,
"iz": 449,
"de": 450,
"ĠN": 451,
"Ġal": 452,
"op": 453,
"up": 454,
"get": 455,
"Ġ}Ċ": 456,
"ile": 457,
"Ġan": 458,
"ata": 459,
"ore": 460,
"ri": 461,
"Ġpro": 462,
";čĊ": 463,
"ĉĉĉĉ": 464,
"ter": 465,
"ain": 466,
"ĠW": 467,
"ĠE": 468,
"Ġcom": 469,
"Ġreturn": 470,
"art": 471,
"ĠH": 472,
"ack": 473,
"import": 474,
"ublic": 475,
"Ġor": 476,
"est": 477,
"ment": 478,
"ĠG": 479,
"able": 480,
"Ġ-": 481,
"ine": 482,
"ill": 483,
"ind": 484,
"ere": 485,
"::": 486,
"ity": 487,
"Ġ+": 488,
"Ġtr": 489,
"elf": 490,
"ight": 491,
"('": 492,
"orm": 493,
"ult": 494,
"str": 495,
"..": 496,
"\",": 497,
"Ġyou": 498,
"ype": 499
},
"merges": [
[
"Ġ",
"Ġ"
],
[
"ĠĠ",
"ĠĠ"
],
[
"i",
"n"
],
[
"Ġ",
"t"
],
[
"ĠĠĠĠ",
"ĠĠĠĠ"
],
[
"e",
"r"
],
[
"ĠĠ",
"Ġ"
],
[
"o",
"n"
],
[
"Ġ",
"a"
],
[
"r",
"e"
],
[
"a",
"t"
],
[
"s",
"t"
],
[
"e",
"n"
],
[
"o",
"r"
],
[
"Ġt",
"h"
],
[
"Ċ",
"Ċ"
],
[
"Ġ",
"c"
],
[
"l",
"e"
],
[
"Ġ",
"s"
],
[
"i",
"t"
],
[
"a",
"n"
],
[
"a",
"r"
],
[
"a",
"l"
],
[
"Ġth",
"e"
],
[
";",
"Ċ"
],
[
"Ġ",
"p"
],
[
"Ġ",
"f"
],
[
"o",
"u"
],
[
"Ġ",
"="
],
[
"i",
"s"
],
[
"ĠĠĠĠ",
"ĠĠĠ"
],
[
"in",
"g"
],
[
"e",
"s"
],
[
"Ġ",
"w"
],
[
"i",
"on"
],
[
"e",
"d"
],
[
"i",
"c"
],
[
"Ġ",
"b"
],
[
"Ġ",
"d"
],
[
"e",
"t"
],
[
"Ġ",
"m"
],
[
"Ġ",
"o"
],
[
"ĉ",
"ĉ"
],
[
"r",
"o"
],
[
"a",
"s"
],
[
"e",
"l"
],
[
"c",
"t"
],
[
"n",
"d"
],
[
"Ġ",
"in"
],
[
"Ġ",
"h"
],
[
"en",
"t"
],
[
"i",
"d"
],
[
"Ġ",
"n"
],
[
"a",
"m"
],
[
"ĠĠĠĠĠĠĠĠ",
"ĠĠĠ"
],
[
"Ġt",
"o"
],
[
"Ġ",
"re"
],
[
"-",
"-"
],
[
"Ġ",
"{"
],
[
"Ġo",
"f"
],
[
"o",
"m"
],
[
")",
";Ċ"
],
[
"i",
"m"
],
[
"č",
"Ċ"
],
[
"Ġ",
"("
],
[
"i",
"l"
],
[
"/",
"/"
],
[
"Ġa",
"nd"
],
[
"u",
"r"
],
[
"s",
"e"
],
[
"Ġ",
"l"
],
[
"e",
"x"
],
[
"Ġ",
"S"
],
[
"a",
"d"
],
[
"Ġ",
"\""
],
[
"c",
"h"
],
[
"u",
"t"
],
[
"i",
"f"
],
[
"*",
"*"
],
[
"Ġ",
"}"
],
[
"e",
"m"
],
[
"o",
"l"
],
[
"ĠĠĠĠĠĠĠĠ",
"ĠĠĠĠĠĠĠĠ"
],
[
"t",
"h"
],
[
")",
"Ċ"
],
[
"Ġ{",
"Ċ"
],
[
"Ġ",
"g"
],
[
"i",
"g"
],
[
"i",
"v"
],
[
",",
"Ċ"
],
[
"c",
"e"
],
[
"o",
"d"
],
[
"Ġ",
"v"
],
[
"at",
"e"
],
[
"Ġ",
"T"
],
[
"a",
"g"
],
[
"a",
"y"
],
[
"Ġ",
"*"
],
[
"o",
"t"
],
[
"u",
"s"
],
[
"Ġ",
"C"
],
[
"Ġ",
"st"
],
[
"Ġ",
"I"
],
[
"u",
"n"
],
[
"u",
"l"
],
[
"u",
"e"
],
[
"Ġ",
"A"
],
[
"o",
"w"
],
[
"Ġ",
"'"
],
[
"e",
"w"
],
[
"Ġ",
"<"
],
[
"at",
"ion"
],
[
"(",
")"
],
[
"Ġf",
"or"
],
[
"a",
"b"
],
[
"or",
"t"
],
[
"u",
"m"
],
[
"am",
"e"
],
[
"Ġ",
"is"
],
[
"p",
"e"
],
[
"t",
"r"
],
[
"c",
"k"
],
[
"â",
"Ģ"
],
[
"Ġ",
"y"
],
[
"i",
"st"
],
[
"--",
"--"
],
[
".",
"ĊĊ"
],
[
"h",
"e"
],
[
"Ġ",
"e"
],
[
"l",
"o"
],
[
"Ġ",
"M"
],
[
"Ġb",
"e"
],
[
"er",
"s"
],
[
"Ġ",
"on"
],
[
"Ġc",
"on"
],
[
"a",
"p"
],
[
"u",
"b"
],
[
"Ġ",
"P"
],
[
"ĠĠĠĠĠĠĠĠ",
"ĠĠĠĠĠĠĠ"
],
[
"as",
"s"
],
[
"in",
"t"
],
[
">",
"Ċ"
],
[
"l",
"y"
],
[
"ur",
"n"
],
[
"Ġ",
"$"
],
[
";",
"ĊĊ"
],
[
"a",
"v"
],
[
"p",
"ort"
],
[
"i",
"r"
],
[
"-",
">"
],
[
"n",
"t"
],
[
"ct",
"ion"
],
[
"en",
"d"
],
[
"Ġd",
"e"
],
[
"it",
"h"
],
[
"ou",
"t"
],
[
"t",
"urn"
],
[
"ou",
"r"
],
[
"ĠĠĠĠ",
"Ġ"
],
[
"l",
"ic"
],
[
"re",
"s"
],
[
"p",
"t"
],
[
"=",
"="
],
[
"Ġth",
"is"
],
[
"Ġw",
"h"
],
[
"Ġ",
"if"
],
[
"Ġ",
"D"
],
[
"v",
"er"
],
[
"ag",
"e"
],
[
"Ġ",
"B"
],
[
"h",
"t"
],
[
"ex",
"t"
],
[
"=",
"\""
],
[
"Ġth",
"at"
],
[
"**",
"**"
],
[
"Ġ",
"R"
],
[
"Ġ",
"it"
],
[
"es",
"s"
],
[
"Ġ",
"F"
],
[
"Ġ",
"r"
],
[
"o",
"s"
],
[
"an",
"d"
],
[
"Ġa",
"s"
],
[
"e",
"ct"
],
[
"k",
"e"
],
[
"ro",
"m"
],
[
"Ġ",
"//"
],
[
"c",
"on"
],
[
"Ġ",
"L"
],
[
"(",
"\""
],
[
"q",
"u"
],
[
"l",
"ass"
],
[
"Ġw",
"ith"
],
[
"i",
"z"
],
[
"d",
"e"
],
[
"Ġ",
"N"
],
[
"Ġa",
"l"
],
[
"o",
"p"
],
[
"u",
"p"
],
[
"g",
"et"
],
[
"Ġ}",
"Ċ"
],
[
"i",
"le"
],
[
"Ġa",
"n"
],
[
"at",
"a"
],
[
"o",
"re"
],
[
"r",
"i"
],
[
"Ġp",
"ro"
],
[
";",
"čĊ"
],
[
"ĉĉ",
"ĉĉ"
],
[
"t",
"er"
],
[
"a",
"in"
],
[
"Ġ",
"W"
],
[
"Ġ",
"E"
],
[
"Ġc",
"om"
],
[
"Ġre",
"turn"
],
[
"ar",
"t"
],
[
"Ġ",
"H"
],
[
"a",
"ck"
],
[
"im",
"port"
],
[
"ub",
"lic"
],
[
"Ġ",
"or"
],
[
"e",
"st"
],
[
"m",
"ent"
],
[
"Ġ",
"G"
],
[
"ab",
"le"
],
[
"Ġ",
"-"
],
[
"in",
"e"
],
[
"il",
"l"
],
[
"in",
"d"
],
[
"er",
"e"
],
[
":",
":"
],
[
"it",
"y"
],
[
"Ġ",
"+"
],
[
"Ġt",
"r"
],
[
"el",
"f"
],
[
"ig",
"ht"
],
[
"(",
"'"
],
[
"or",
"m"
],
[
"ul",
"t"
],
[
"st",
"r"
],
[
".",
"."
],
[
"\"",
","
],
[
"Ġy",
"ou"
],
[
"y",
"pe"
]
]
}
}