Add tiny processor fixture for testing

d41ca4c verified 3 days ago

14.7 kB

	{
	"version": "1.0",
	"truncation": null,
	"padding": null,
	"added_tokens": [
	{
	"id": 500,
	"content": "<\|endoftext\|>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 501,
	"content": "<\|im_start\|>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 502,
	"content": "<\|im_end\|>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	}
	],
	"normalizer": {
	"type": "NFC"
	},
	"pre_tokenizer": {
	"type": "Sequence",
	"pretokenizers": [
	{
	"type": "Split",
	"pattern": {
	"Regex": "(?i:'s\|'t\|'re\|'ve\|'m\|'ll\|'d)\|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+\|\\p{N}\| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]\|\\s[\\r\\n]+\|\\s+(?!\\S)\|\\s+"
	},
	"behavior": "Isolated",
	"invert": false
	},
	{
	"type": "ByteLevel",
	"add_prefix_space": false,
	"trim_offsets": false,
	"use_regex": false
	}
	]
	},
	"post_processor": {
	"type": "ByteLevel",
	"add_prefix_space": false,
	"trim_offsets": false,
	"use_regex": false
	},
	"decoder": {
	"type": "ByteLevel",
	"add_prefix_space": false,
	"trim_offsets": false,
	"use_regex": false
	},
	"model": {
	"type": "BPE",
	"dropout": null,
	"unk_token": null,
	"continuing_subword_prefix": "",
	"end_of_word_suffix": "",
	"fuse_unk": false,
	"byte_fallback": false,
	"vocab": {
	"!": 0,
	"\"": 1,
	"#": 2,
	"$": 3,
	"%": 4,
	"&": 5,
	"'": 6,
	"(": 7,
	")": 8,
	"*": 9,
	"+": 10,
	",": 11,
	"-": 12,
	".": 13,
	"/": 14,
	"0": 15,
	"1": 16,
	"2": 17,
	"3": 18,
	"4": 19,
	"5": 20,
	"6": 21,
	"7": 22,
	"8": 23,
	"9": 24,
	":": 25,
	";": 26,
	"<": 27,
	"=": 28,
	">": 29,
	"?": 30,
	"@": 31,
	"A": 32,
	"B": 33,
	"C": 34,
	"D": 35,
	"E": 36,
	"F": 37,
	"G": 38,
	"H": 39,
	"I": 40,
	"J": 41,
	"K": 42,
	"L": 43,
	"M": 44,
	"N": 45,
	"O": 46,
	"P": 47,
	"Q": 48,
	"R": 49,
	"S": 50,
	"T": 51,
	"U": 52,
	"V": 53,
	"W": 54,
	"X": 55,
	"Y": 56,
	"Z": 57,
	"[": 58,
	"\\": 59,
	"]": 60,
	"^": 61,
	"_": 62,
	"`": 63,
	"a": 64,
	"b": 65,
	"c": 66,
	"d": 67,
	"e": 68,
	"f": 69,
	"g": 70,
	"h": 71,
	"i": 72,
	"j": 73,
	"k": 74,
	"l": 75,
	"m": 76,
	"n": 77,
	"o": 78,
	"p": 79,
	"q": 80,
	"r": 81,
	"s": 82,
	"t": 83,
	"u": 84,
	"v": 85,
	"w": 86,
	"x": 87,
	"y": 88,
	"z": 89,
	"{": 90,
	"\|": 91,
	"}": 92,
	"~": 93,
	"¡": 94,
	"¢": 95,
	"£": 96,
	"¤": 97,
	"¥": 98,
	"¦": 99,
	"§": 100,
	"¨": 101,
	"©": 102,
	"ª": 103,
	"«": 104,
	"¬": 105,
	"®": 106,
	"¯": 107,
	"°": 108,
	"±": 109,
	"²": 110,
	"³": 111,
	"´": 112,
	"µ": 113,
	"¶": 114,
	"·": 115,
	"¸": 116,
	"¹": 117,
	"º": 118,
	"»": 119,
	"¼": 120,
	"½": 121,
	"¾": 122,
	"¿": 123,
	"À": 124,
	"Á": 125,
	"Â": 126,
	"Ã": 127,
	"Ä": 128,
	"Å": 129,
	"Æ": 130,
	"Ç": 131,
	"È": 132,
	"É": 133,
	"Ê": 134,
	"Ë": 135,
	"Ì": 136,
	"Í": 137,
	"Î": 138,
	"Ï": 139,
	"Ð": 140,
	"Ñ": 141,
	"Ò": 142,
	"Ó": 143,
	"Ô": 144,
	"Õ": 145,
	"Ö": 146,
	"×": 147,
	"Ø": 148,
	"Ù": 149,
	"Ú": 150,
	"Û": 151,
	"Ü": 152,
	"Ý": 153,
	"Þ": 154,
	"ß": 155,
	"à": 156,
	"á": 157,
	"â": 158,
	"ã": 159,
	"ä": 160,
	"å": 161,
	"æ": 162,
	"ç": 163,
	"è": 164,
	"é": 165,
	"ê": 166,
	"ë": 167,
	"ì": 168,
	"í": 169,
	"î": 170,
	"ï": 171,
	"ð": 172,
	"ñ": 173,
	"ò": 174,
	"ó": 175,
	"ô": 176,
	"õ": 177,
	"ö": 178,
	"÷": 179,
	"ø": 180,
	"ù": 181,
	"ú": 182,
	"û": 183,
	"ü": 184,
	"ý": 185,
	"þ": 186,
	"ÿ": 187,
	"Ā": 188,
	"ā": 189,
	"Ă": 190,
	"ă": 191,
	"Ą": 192,
	"ą": 193,
	"Ć": 194,
	"ć": 195,
	"Ĉ": 196,
	"ĉ": 197,
	"Ċ": 198,
	"ċ": 199,
	"Č": 200,
	"č": 201,
	"Ď": 202,
	"ď": 203,
	"Đ": 204,
	"đ": 205,
	"Ē": 206,
	"ē": 207,
	"Ĕ": 208,
	"ĕ": 209,
	"Ė": 210,
	"ė": 211,
	"Ę": 212,
	"ę": 213,
	"Ě": 214,
	"ě": 215,
	"Ĝ": 216,
	"ĝ": 217,
	"Ğ": 218,
	"ğ": 219,
	"Ġ": 220,
	"ġ": 221,
	"Ģ": 222,
	"ģ": 223,
	"Ĥ": 224,
	"ĥ": 225,
	"Ħ": 226,
	"ħ": 227,
	"Ĩ": 228,
	"ĩ": 229,
	"Ī": 230,
	"ī": 231,
	"Ĭ": 232,
	"ĭ": 233,
	"Į": 234,
	"į": 235,
	"İ": 236,
	"ı": 237,
	"Ĳ": 238,
	"ĳ": 239,
	"Ĵ": 240,
	"ĵ": 241,
	"Ķ": 242,
	"ķ": 243,
	"ĸ": 244,
	"Ĺ": 245,
	"ĺ": 246,
	"Ļ": 247,
	"ļ": 248,
	"Ľ": 249,
	"ľ": 250,
	"Ŀ": 251,
	"ŀ": 252,
	"Ł": 253,
	"ł": 254,
	"Ń": 255,
	"ĠĠ": 256,
	"ĠĠĠĠ": 257,
	"in": 258,
	"Ġt": 259,
	"ĠĠĠĠĠĠĠĠ": 260,
	"er": 261,
	"ĠĠĠ": 262,
	"on": 263,
	"Ġa": 264,
	"re": 265,
	"at": 266,
	"st": 267,
	"en": 268,
	"or": 269,
	"Ġth": 270,
	"ĊĊ": 271,
	"Ġc": 272,
	"le": 273,
	"Ġs": 274,
	"it": 275,
	"an": 276,
	"ar": 277,
	"al": 278,
	"Ġthe": 279,
	";Ċ": 280,
	"Ġp": 281,
	"Ġf": 282,
	"ou": 283,
	"Ġ=": 284,
	"is": 285,
	"ĠĠĠĠĠĠĠ": 286,
	"ing": 287,
	"es": 288,
	"Ġw": 289,
	"ion": 290,
	"ed": 291,
	"ic": 292,
	"Ġb": 293,
	"Ġd": 294,
	"et": 295,
	"Ġm": 296,
	"Ġo": 297,
	"ĉĉ": 298,
	"ro": 299,
	"as": 300,
	"el": 301,
	"ct": 302,
	"nd": 303,
	"Ġin": 304,
	"Ġh": 305,
	"ent": 306,
	"id": 307,
	"Ġn": 308,
	"am": 309,
	"ĠĠĠĠĠĠĠĠĠĠĠ": 310,
	"Ġto": 311,
	"Ġre": 312,
	"--": 313,
	"Ġ{": 314,
	"Ġof": 315,
	"om": 316,
	");Ċ": 317,
	"im": 318,
	"čĊ": 319,
	"Ġ(": 320,
	"il": 321,
	"//": 322,
	"Ġand": 323,
	"ur": 324,
	"se": 325,
	"Ġl": 326,
	"ex": 327,
	"ĠS": 328,
	"ad": 329,
	"Ġ\"": 330,
	"ch": 331,
	"ut": 332,
	"if": 333,
	"**": 334,
	"Ġ}": 335,
	"em": 336,
	"ol": 337,
	"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 338,
	"th": 339,
	")Ċ": 340,
	"Ġ{Ċ": 341,
	"Ġg": 342,
	"ig": 343,
	"iv": 344,
	",Ċ": 345,
	"ce": 346,
	"od": 347,
	"Ġv": 348,
	"ate": 349,
	"ĠT": 350,
	"ag": 351,
	"ay": 352,
	"Ġ*": 353,
	"ot": 354,
	"us": 355,
	"ĠC": 356,
	"Ġst": 357,
	"ĠI": 358,
	"un": 359,
	"ul": 360,
	"ue": 361,
	"ĠA": 362,
	"ow": 363,
	"Ġ'": 364,
	"ew": 365,
	"Ġ<": 366,
	"ation": 367,
	"()": 368,
	"Ġfor": 369,
	"ab": 370,
	"ort": 371,
	"um": 372,
	"ame": 373,
	"Ġis": 374,
	"pe": 375,
	"tr": 376,
	"ck": 377,
	"âĢ": 378,
	"Ġy": 379,
	"ist": 380,
	"----": 381,
	".ĊĊ": 382,
	"he": 383,
	"Ġe": 384,
	"lo": 385,
	"ĠM": 386,
	"Ġbe": 387,
	"ers": 388,
	"Ġon": 389,
	"Ġcon": 390,
	"ap": 391,
	"ub": 392,
	"ĠP": 393,
	"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 394,
	"ass": 395,
	"int": 396,
	">Ċ": 397,
	"ly": 398,
	"urn": 399,
	"Ġ$": 400,
	";ĊĊ": 401,
	"av": 402,
	"port": 403,
	"ir": 404,
	"->": 405,
	"nt": 406,
	"ction": 407,
	"end": 408,
	"Ġde": 409,
	"ith": 410,
	"out": 411,
	"turn": 412,
	"our": 413,
	"ĠĠĠĠĠ": 414,
	"lic": 415,
	"res": 416,
	"pt": 417,
	"==": 418,
	"Ġthis": 419,
	"Ġwh": 420,
	"Ġif": 421,
	"ĠD": 422,
	"ver": 423,
	"age": 424,
	"ĠB": 425,
	"ht": 426,
	"ext": 427,
	"=\"": 428,
	"Ġthat": 429,
	"****": 430,
	"ĠR": 431,
	"Ġit": 432,
	"ess": 433,
	"ĠF": 434,
	"Ġr": 435,
	"os": 436,
	"and": 437,
	"Ġas": 438,
	"ect": 439,
	"ke": 440,
	"rom": 441,
	"Ġ//": 442,
	"con": 443,
	"ĠL": 444,
	"(\"": 445,
	"qu": 446,
	"lass": 447,
	"Ġwith": 448,
	"iz": 449,
	"de": 450,
	"ĠN": 451,
	"Ġal": 452,
	"op": 453,
	"up": 454,
	"get": 455,
	"Ġ}Ċ": 456,
	"ile": 457,
	"Ġan": 458,
	"ata": 459,
	"ore": 460,
	"ri": 461,
	"Ġpro": 462,
	";čĊ": 463,
	"ĉĉĉĉ": 464,
	"ter": 465,
	"ain": 466,
	"ĠW": 467,
	"ĠE": 468,
	"Ġcom": 469,
	"Ġreturn": 470,
	"art": 471,
	"ĠH": 472,
	"ack": 473,
	"import": 474,
	"ublic": 475,
	"Ġor": 476,
	"est": 477,
	"ment": 478,
	"ĠG": 479,
	"able": 480,
	"Ġ-": 481,
	"ine": 482,
	"ill": 483,
	"ind": 484,
	"ere": 485,
	"::": 486,
	"ity": 487,
	"Ġ+": 488,
	"Ġtr": 489,
	"elf": 490,
	"ight": 491,
	"('": 492,
	"orm": 493,
	"ult": 494,
	"str": 495,
	"..": 496,
	"\",": 497,
	"Ġyou": 498,
	"ype": 499
	},
	"merges": [
	"Ġ Ġ",
	"ĠĠ ĠĠ",
	"i n",
	"Ġ t",
	"ĠĠĠĠ ĠĠĠĠ",
	"e r",
	"ĠĠ Ġ",
	"o n",
	"Ġ a",
	"r e",
	"a t",
	"s t",
	"e n",
	"o r",
	"Ġt h",
	"Ċ Ċ",
	"Ġ c",
	"l e",
	"Ġ s",
	"i t",
	"a n",
	"a r",
	"a l",
	"Ġth e",
	"; Ċ",
	"Ġ p",
	"Ġ f",
	"o u",
	"Ġ =",
	"i s",
	"ĠĠĠĠ ĠĠĠ",
	"in g",
	"e s",
	"Ġ w",
	"i on",
	"e d",
	"i c",
	"Ġ b",
	"Ġ d",
	"e t",
	"Ġ m",
	"Ġ o",
	"ĉ ĉ",
	"r o",
	"a s",
	"e l",
	"c t",
	"n d",
	"Ġ in",
	"Ġ h",
	"en t",
	"i d",
	"Ġ n",
	"a m",
	"ĠĠĠĠĠĠĠĠ ĠĠĠ",
	"Ġt o",
	"Ġ re",
	"- -",
	"Ġ {",
	"Ġo f",
	"o m",
	") ;Ċ",
	"i m",
	"č Ċ",
	"Ġ (",
	"i l",
	"/ /",
	"Ġa nd",
	"u r",
	"s e",
	"Ġ l",
	"e x",
	"Ġ S",
	"a d",
	"Ġ \"",
	"c h",
	"u t",
	"i f",
	"* *",
	"Ġ }",
	"e m",
	"o l",
	"ĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠ",
	"t h",
	") Ċ",
	"Ġ{ Ċ",
	"Ġ g",
	"i g",
	"i v",
	", Ċ",
	"c e",
	"o d",
	"Ġ v",
	"at e",
	"Ġ T",
	"a g",
	"a y",
	"Ġ *",
	"o t",
	"u s",
	"Ġ C",
	"Ġ st",
	"Ġ I",
	"u n",
	"u l",
	"u e",
	"Ġ A",
	"o w",
	"Ġ '",
	"e w",
	"Ġ <",
	"at ion",
	"( )",
	"Ġf or",
	"a b",
	"or t",
	"u m",
	"am e",
	"Ġ is",
	"p e",
	"t r",
	"c k",
	"â Ģ",
	"Ġ y",
	"i st",
	"-- --",
	". ĊĊ",
	"h e",
	"Ġ e",
	"l o",
	"Ġ M",
	"Ġb e",
	"er s",
	"Ġ on",
	"Ġc on",
	"a p",
	"u b",
	"Ġ P",
	"ĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠ",
	"as s",
	"in t",
	"> Ċ",
	"l y",
	"ur n",
	"Ġ $",
	"; ĊĊ",
	"a v",
	"p ort",
	"i r",
	"- >",
	"n t",
	"ct ion",
	"en d",
	"Ġd e",
	"it h",
	"ou t",
	"t urn",
	"ou r",
	"ĠĠĠĠ Ġ",
	"l ic",
	"re s",
	"p t",
	"= =",
	"Ġth is",
	"Ġw h",
	"Ġ if",
	"Ġ D",
	"v er",
	"ag e",
	"Ġ B",
	"h t",
	"ex t",
	"= \"",
	"Ġth at",
	" ",
	"Ġ R",
	"Ġ it",
	"es s",
	"Ġ F",
	"Ġ r",
	"o s",
	"an d",
	"Ġa s",
	"e ct",
	"k e",
	"ro m",
	"Ġ //",
	"c on",
	"Ġ L",
	"( \"",
	"q u",
	"l ass",
	"Ġw ith",
	"i z",
	"d e",
	"Ġ N",
	"Ġa l",
	"o p",
	"u p",
	"g et",
	"Ġ} Ċ",
	"i le",
	"Ġa n",
	"at a",
	"o re",
	"r i",
	"Ġp ro",
	"; čĊ",
	"ĉĉ ĉĉ",
	"t er",
	"a in",
	"Ġ W",
	"Ġ E",
	"Ġc om",
	"Ġre turn",
	"ar t",
	"Ġ H",
	"a ck",
	"im port",
	"ub lic",
	"Ġ or",
	"e st",
	"m ent",
	"Ġ G",
	"ab le",
	"Ġ -",
	"in e",
	"il l",
	"in d",
	"er e",
	": :",
	"it y",
	"Ġ +",
	"Ġt r",
	"el f",
	"ig ht",
	"( '",
	"or m",
	"ul t",
	"st r",
	". .",
	"\" ,",
	"Ġy ou",
	"y pe"
	]
	}
	}