Add tiny processor fixture for testing

e08a6cd verified 3 days ago

11.6 kB

	{
	"version": "1.0",
	"truncation": null,
	"padding": null,
	"added_tokens": [
	{
	"id": 500,
	"special": true,
	"content": "<\|startoftext\|>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": true
	},
	{
	"id": 501,
	"special": true,
	"content": "<\|endoftext\|>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false
	}
	],
	"normalizer": {
	"type": "Sequence",
	"normalizers": [
	{
	"type": "NFC"
	},
	{
	"type": "Replace",
	"pattern": {
	"Regex": "\\s+"
	},
	"content": " "
	},
	{
	"type": "Lowercase"
	}
	]
	},
	"pre_tokenizer": {
	"type": "Sequence",
	"pretokenizers": [
	{
	"type": "Split",
	"pattern": {
	"Regex": "<\\\|startoftext\\\|>\|<\\\|endoftext\\\|>\|'s\|'t\|'re\|'ve\|'m\|'ll\|'d\|[\\p{L}]+\|[\\p{N}]\|[^\\s\\p{L}\\p{N}]+"
	},
	"behavior": "Removed",
	"invert": true
	},
	{
	"type": "ByteLevel",
	"add_prefix_space": false,
	"trim_offsets": true
	}
	]
	},
	"post_processor": {
	"type": "RobertaProcessing",
	"sep": [
	"<\|endoftext\|>",
	49407
	],
	"cls": [
	"<\|startoftext\|>",
	49406
	],
	"trim_offsets": false,
	"add_prefix_space": false
	},
	"decoder": {
	"type": "ByteLevel",
	"add_prefix_space": true,
	"trim_offsets": true
	},
	"model": {
	"type": "BPE",
	"dropout": null,
	"unk_token": "<\|endoftext\|>",
	"continuing_subword_prefix": "",
	"end_of_word_suffix": "</w>",
	"fuse_unk": false,
	"vocab": {
	"!": 0,
	"\"": 1,
	"#": 2,
	"$": 3,
	"%": 4,
	"&": 5,
	"'": 6,
	"(": 7,
	")": 8,
	"*": 9,
	"+": 10,
	",": 11,
	"-": 12,
	".": 13,
	"/": 14,
	"0": 15,
	"1": 16,
	"2": 17,
	"3": 18,
	"4": 19,
	"5": 20,
	"6": 21,
	"7": 22,
	"8": 23,
	"9": 24,
	":": 25,
	";": 26,
	"<": 27,
	"=": 28,
	">": 29,
	"?": 30,
	"@": 31,
	"A": 32,
	"B": 33,
	"C": 34,
	"D": 35,
	"E": 36,
	"F": 37,
	"G": 38,
	"H": 39,
	"I": 40,
	"J": 41,
	"K": 42,
	"L": 43,
	"M": 44,
	"N": 45,
	"O": 46,
	"P": 47,
	"Q": 48,
	"R": 49,
	"S": 50,
	"T": 51,
	"U": 52,
	"V": 53,
	"W": 54,
	"X": 55,
	"Y": 56,
	"Z": 57,
	"[": 58,
	"\\": 59,
	"]": 60,
	"^": 61,
	"_": 62,
	"`": 63,
	"a": 64,
	"b": 65,
	"c": 66,
	"d": 67,
	"e": 68,
	"f": 69,
	"g": 70,
	"h": 71,
	"i": 72,
	"j": 73,
	"k": 74,
	"l": 75,
	"m": 76,
	"n": 77,
	"o": 78,
	"p": 79,
	"q": 80,
	"r": 81,
	"s": 82,
	"t": 83,
	"u": 84,
	"v": 85,
	"w": 86,
	"x": 87,
	"y": 88,
	"z": 89,
	"{": 90,
	"\|": 91,
	"}": 92,
	"~": 93,
	"¡": 94,
	"¢": 95,
	"£": 96,
	"¤": 97,
	"¥": 98,
	"¦": 99,
	"§": 100,
	"¨": 101,
	"©": 102,
	"ª": 103,
	"«": 104,
	"¬": 105,
	"®": 106,
	"¯": 107,
	"°": 108,
	"±": 109,
	"²": 110,
	"³": 111,
	"´": 112,
	"µ": 113,
	"¶": 114,
	"·": 115,
	"¸": 116,
	"¹": 117,
	"º": 118,
	"»": 119,
	"¼": 120,
	"½": 121,
	"¾": 122,
	"¿": 123,
	"À": 124,
	"Á": 125,
	"Â": 126,
	"Ã": 127,
	"Ä": 128,
	"Å": 129,
	"Æ": 130,
	"Ç": 131,
	"È": 132,
	"É": 133,
	"Ê": 134,
	"Ë": 135,
	"Ì": 136,
	"Í": 137,
	"Î": 138,
	"Ï": 139,
	"Ð": 140,
	"Ñ": 141,
	"Ò": 142,
	"Ó": 143,
	"Ô": 144,
	"Õ": 145,
	"Ö": 146,
	"×": 147,
	"Ø": 148,
	"Ù": 149,
	"Ú": 150,
	"Û": 151,
	"Ü": 152,
	"Ý": 153,
	"Þ": 154,
	"ß": 155,
	"à": 156,
	"á": 157,
	"â": 158,
	"ã": 159,
	"ä": 160,
	"å": 161,
	"æ": 162,
	"ç": 163,
	"è": 164,
	"é": 165,
	"ê": 166,
	"ë": 167,
	"ì": 168,
	"í": 169,
	"î": 170,
	"ï": 171,
	"ð": 172,
	"ñ": 173,
	"ò": 174,
	"ó": 175,
	"ô": 176,
	"õ": 177,
	"ö": 178,
	"÷": 179,
	"ø": 180,
	"ù": 181,
	"ú": 182,
	"û": 183,
	"ü": 184,
	"ý": 185,
	"þ": 186,
	"ÿ": 187,
	"Ā": 188,
	"ā": 189,
	"Ă": 190,
	"ă": 191,
	"Ą": 192,
	"ą": 193,
	"Ć": 194,
	"ć": 195,
	"Ĉ": 196,
	"ĉ": 197,
	"Ċ": 198,
	"ċ": 199,
	"Č": 200,
	"č": 201,
	"Ď": 202,
	"ď": 203,
	"Đ": 204,
	"đ": 205,
	"Ē": 206,
	"ē": 207,
	"Ĕ": 208,
	"ĕ": 209,
	"Ė": 210,
	"ė": 211,
	"Ę": 212,
	"ę": 213,
	"Ě": 214,
	"ě": 215,
	"Ĝ": 216,
	"ĝ": 217,
	"Ğ": 218,
	"ğ": 219,
	"Ġ": 220,
	"ġ": 221,
	"Ģ": 222,
	"ģ": 223,
	"Ĥ": 224,
	"ĥ": 225,
	"Ħ": 226,
	"ħ": 227,
	"Ĩ": 228,
	"ĩ": 229,
	"Ī": 230,
	"ī": 231,
	"Ĭ": 232,
	"ĭ": 233,
	"Į": 234,
	"į": 235,
	"İ": 236,
	"ı": 237,
	"Ĳ": 238,
	"ĳ": 239,
	"Ĵ": 240,
	"ĵ": 241,
	"Ķ": 242,
	"ķ": 243,
	"ĸ": 244,
	"Ĺ": 245,
	"ĺ": 246,
	"Ļ": 247,
	"ļ": 248,
	"Ľ": 249,
	"ľ": 250,
	"Ŀ": 251,
	"ŀ": 252,
	"Ł": 253,
	"ł": 254,
	"Ń": 255,
	"!</w>": 256,
	"\"</w>": 257,
	"#</w>": 258,
	"$</w>": 259,
	"%</w>": 260,
	"&</w>": 261,
	"'</w>": 262,
	"(</w>": 263,
	")</w>": 264,
	"*</w>": 265,
	"+</w>": 266,
	",</w>": 267,
	"-</w>": 268,
	".</w>": 269,
	"/</w>": 270,
	"0</w>": 271,
	"1</w>": 272,
	"2</w>": 273,
	"3</w>": 274,
	"4</w>": 275,
	"5</w>": 276,
	"6</w>": 277,
	"7</w>": 278,
	"8</w>": 279,
	"9</w>": 280,
	":</w>": 281,
	";</w>": 282,
	"<</w>": 283,
	"=</w>": 284,
	"></w>": 285,
	"?</w>": 286,
	"@</w>": 287,
	"A</w>": 288,
	"B</w>": 289,
	"C</w>": 290,
	"D</w>": 291,
	"E</w>": 292,
	"F</w>": 293,
	"G</w>": 294,
	"H</w>": 295,
	"I</w>": 296,
	"J</w>": 297,
	"K</w>": 298,
	"L</w>": 299,
	"M</w>": 300,
	"N</w>": 301,
	"O</w>": 302,
	"P</w>": 303,
	"Q</w>": 304,
	"R</w>": 305,
	"S</w>": 306,
	"T</w>": 307,
	"U</w>": 308,
	"V</w>": 309,
	"W</w>": 310,
	"X</w>": 311,
	"Y</w>": 312,
	"Z</w>": 313,
	"[</w>": 314,
	"\\</w>": 315,
	"]</w>": 316,
	"^</w>": 317,
	"_</w>": 318,
	"`</w>": 319,
	"a</w>": 320,
	"b</w>": 321,
	"c</w>": 322,
	"d</w>": 323,
	"e</w>": 324,
	"f</w>": 325,
	"g</w>": 326,
	"h</w>": 327,
	"i</w>": 328,
	"j</w>": 329,
	"k</w>": 330,
	"l</w>": 331,
	"m</w>": 332,
	"n</w>": 333,
	"o</w>": 334,
	"p</w>": 335,
	"q</w>": 336,
	"r</w>": 337,
	"s</w>": 338,
	"t</w>": 339,
	"u</w>": 340,
	"v</w>": 341,
	"w</w>": 342,
	"x</w>": 343,
	"y</w>": 344,
	"z</w>": 345,
	"{</w>": 346,
	"\|</w>": 347,
	"}</w>": 348,
	"~</w>": 349,
	"¡</w>": 350,
	"¢</w>": 351,
	"£</w>": 352,
	"¤</w>": 353,
	"¥</w>": 354,
	"¦</w>": 355,
	"§</w>": 356,
	"¨</w>": 357,
	"©</w>": 358,
	"ª</w>": 359,
	"«</w>": 360,
	"¬</w>": 361,
	"®</w>": 362,
	"¯</w>": 363,
	"°</w>": 364,
	"±</w>": 365,
	"²</w>": 366,
	"³</w>": 367,
	"´</w>": 368,
	"µ</w>": 369,
	"¶</w>": 370,
	"·</w>": 371,
	"¸</w>": 372,
	"¹</w>": 373,
	"º</w>": 374,
	"»</w>": 375,
	"¼</w>": 376,
	"½</w>": 377,
	"¾</w>": 378,
	"¿</w>": 379,
	"À</w>": 380,
	"Á</w>": 381,
	"Â</w>": 382,
	"Ã</w>": 383,
	"Ä</w>": 384,
	"Å</w>": 385,
	"Æ</w>": 386,
	"Ç</w>": 387,
	"È</w>": 388,
	"É</w>": 389,
	"Ê</w>": 390,
	"Ë</w>": 391,
	"Ì</w>": 392,
	"Í</w>": 393,
	"Î</w>": 394,
	"Ï</w>": 395,
	"Ð</w>": 396,
	"Ñ</w>": 397,
	"Ò</w>": 398,
	"Ó</w>": 399,
	"Ô</w>": 400,
	"Õ</w>": 401,
	"Ö</w>": 402,
	"×</w>": 403,
	"Ø</w>": 404,
	"Ù</w>": 405,
	"Ú</w>": 406,
	"Û</w>": 407,
	"Ü</w>": 408,
	"Ý</w>": 409,
	"Þ</w>": 410,
	"ß</w>": 411,
	"à</w>": 412,
	"á</w>": 413,
	"â</w>": 414,
	"ã</w>": 415,
	"ä</w>": 416,
	"å</w>": 417,
	"æ</w>": 418,
	"ç</w>": 419,
	"è</w>": 420,
	"é</w>": 421,
	"ê</w>": 422,
	"ë</w>": 423,
	"ì</w>": 424,
	"í</w>": 425,
	"î</w>": 426,
	"ï</w>": 427,
	"ð</w>": 428,
	"ñ</w>": 429,
	"ò</w>": 430,
	"ó</w>": 431,
	"ô</w>": 432,
	"õ</w>": 433,
	"ö</w>": 434,
	"÷</w>": 435,
	"ø</w>": 436,
	"ù</w>": 437,
	"ú</w>": 438,
	"û</w>": 439,
	"ü</w>": 440,
	"ý</w>": 441,
	"þ</w>": 442,
	"ÿ</w>": 443,
	"Ā</w>": 444,
	"ā</w>": 445,
	"Ă</w>": 446,
	"ă</w>": 447,
	"Ą</w>": 448,
	"ą</w>": 449,
	"Ć</w>": 450,
	"ć</w>": 451,
	"Ĉ</w>": 452,
	"ĉ</w>": 453,
	"Ċ</w>": 454,
	"ċ</w>": 455,
	"Č</w>": 456,
	"č</w>": 457,
	"Ď</w>": 458,
	"ď</w>": 459,
	"Đ</w>": 460,
	"đ</w>": 461,
	"Ē</w>": 462,
	"ē</w>": 463,
	"Ĕ</w>": 464,
	"ĕ</w>": 465,
	"Ė</w>": 466,
	"ė</w>": 467,
	"Ę</w>": 468,
	"ę</w>": 469,
	"Ě</w>": 470,
	"ě</w>": 471,
	"Ĝ</w>": 472,
	"ĝ</w>": 473,
	"Ğ</w>": 474,
	"ğ</w>": 475,
	"Ġ</w>": 476,
	"ġ</w>": 477,
	"Ģ</w>": 478,
	"ģ</w>": 479,
	"Ĥ</w>": 480,
	"ĥ</w>": 481,
	"Ħ</w>": 482,
	"ħ</w>": 483,
	"Ĩ</w>": 484,
	"ĩ</w>": 485,
	"Ī</w>": 486,
	"ī</w>": 487,
	"Ĭ</w>": 488,
	"ĭ</w>": 489,
	"Į</w>": 490,
	"į</w>": 491,
	"İ</w>": 492,
	"ı</w>": 493,
	"Ĳ</w>": 494,
	"ĳ</w>": 495,
	"Ĵ</w>": 496,
	"ĵ</w>": 497,
	"Ķ</w>": 498,
	"ķ</w>": 499,
	"<\|startoftext\|>": 500,
	"<\|endoftext\|>": 501
	},
	"merges": []
	}
	}