Upload folder using huggingface_hub

196375b verified 21 days ago

12.5 kB

	{
	"version": "1.0",
	"truncation": null,
	"padding": null,
	"added_tokens": [
	{
	"id": 0,
	"content": "<s>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 1,
	"content": "</s>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 2,
	"content": "<pad>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	}
	],
	"normalizer": {
	"type": "NFC"
	},
	"pre_tokenizer": {
	"type": "Sequence",
	"pretokenizers": [
	{
	"type": "Split",
	"pattern": {
	"Regex": "[+=]\|[^\\S\\r\\n]*[\\n\\r]+\|[^\\S\\r\\n]+"
	},
	"behavior": "Isolated",
	"invert": false
	},
	{
	"type": "Split",
	"pattern": {
	"Regex": "\\p{N}{1,3}"
	},
	"behavior": "Isolated",
	"invert": false
	},
	{
	"type": "ByteLevel",
	"add_prefix_space": false,
	"trim_offsets": true,
	"use_regex": false
	}
	]
	},
	"post_processor": null,
	"decoder": {
	"type": "ByteLevel",
	"add_prefix_space": true,
	"trim_offsets": true,
	"use_regex": true
	},
	"model": {
	"type": "BPE",
	"dropout": null,
	"unk_token": null,
	"continuing_subword_prefix": null,
	"end_of_word_suffix": null,
	"fuse_unk": false,
	"byte_fallback": false,
	"ignore_merges": false,
	"vocab": {
	"<s>": 0,
	"</s>": 1,
	"<pad>": 2,
	"!": 3,
	"\"": 4,
	"#": 5,
	"$": 6,
	"%": 7,
	"&": 8,
	"'": 9,
	"(": 10,
	")": 11,
	"*": 12,
	"+": 13,
	",": 14,
	"-": 15,
	".": 16,
	"/": 17,
	"0": 18,
	"1": 19,
	"2": 20,
	"3": 21,
	"4": 22,
	"5": 23,
	"6": 24,
	"7": 25,
	"8": 26,
	"9": 27,
	":": 28,
	";": 29,
	"<": 30,
	"=": 31,
	">": 32,
	"?": 33,
	"@": 34,
	"A": 35,
	"B": 36,
	"C": 37,
	"D": 38,
	"E": 39,
	"F": 40,
	"G": 41,
	"H": 42,
	"I": 43,
	"J": 44,
	"K": 45,
	"L": 46,
	"M": 47,
	"N": 48,
	"O": 49,
	"P": 50,
	"Q": 51,
	"R": 52,
	"S": 53,
	"T": 54,
	"U": 55,
	"V": 56,
	"W": 57,
	"X": 58,
	"Y": 59,
	"Z": 60,
	"[": 61,
	"\\": 62,
	"]": 63,
	"^": 64,
	"_": 65,
	"`": 66,
	"a": 67,
	"b": 68,
	"c": 69,
	"d": 70,
	"e": 71,
	"f": 72,
	"g": 73,
	"h": 74,
	"i": 75,
	"j": 76,
	"k": 77,
	"l": 78,
	"m": 79,
	"n": 80,
	"o": 81,
	"p": 82,
	"q": 83,
	"r": 84,
	"s": 85,
	"t": 86,
	"u": 87,
	"v": 88,
	"w": 89,
	"x": 90,
	"y": 91,
	"z": 92,
	"{": 93,
	"\|": 94,
	"}": 95,
	"~": 96,
	"¡": 97,
	"¢": 98,
	"£": 99,
	"¤": 100,
	"¥": 101,
	"¦": 102,
	"§": 103,
	"¨": 104,
	"©": 105,
	"ª": 106,
	"«": 107,
	"¬": 108,
	"®": 109,
	"¯": 110,
	"°": 111,
	"±": 112,
	"²": 113,
	"³": 114,
	"´": 115,
	"µ": 116,
	"¶": 117,
	"·": 118,
	"¸": 119,
	"¹": 120,
	"º": 121,
	"»": 122,
	"¼": 123,
	"½": 124,
	"¾": 125,
	"¿": 126,
	"À": 127,
	"Á": 128,
	"Â": 129,
	"Ã": 130,
	"Ä": 131,
	"Å": 132,
	"Æ": 133,
	"Ç": 134,
	"È": 135,
	"É": 136,
	"Ê": 137,
	"Ë": 138,
	"Ì": 139,
	"Í": 140,
	"Î": 141,
	"Ï": 142,
	"Ð": 143,
	"Ñ": 144,
	"Ò": 145,
	"Ó": 146,
	"Ô": 147,
	"Õ": 148,
	"Ö": 149,
	"×": 150,
	"Ø": 151,
	"Ù": 152,
	"Ú": 153,
	"Û": 154,
	"Ü": 155,
	"Ý": 156,
	"Þ": 157,
	"ß": 158,
	"à": 159,
	"á": 160,
	"â": 161,
	"ã": 162,
	"ä": 163,
	"å": 164,
	"æ": 165,
	"ç": 166,
	"è": 167,
	"é": 168,
	"ê": 169,
	"ë": 170,
	"ì": 171,
	"í": 172,
	"î": 173,
	"ï": 174,
	"ð": 175,
	"ñ": 176,
	"ò": 177,
	"ó": 178,
	"ô": 179,
	"õ": 180,
	"ö": 181,
	"÷": 182,
	"ø": 183,
	"ù": 184,
	"ú": 185,
	"û": 186,
	"ü": 187,
	"ý": 188,
	"þ": 189,
	"ÿ": 190,
	"Ā": 191,
	"ā": 192,
	"Ă": 193,
	"ă": 194,
	"Ą": 195,
	"ą": 196,
	"Ć": 197,
	"ć": 198,
	"Ĉ": 199,
	"ĉ": 200,
	"Ċ": 201,
	"ċ": 202,
	"Č": 203,
	"č": 204,
	"Ď": 205,
	"ď": 206,
	"Đ": 207,
	"đ": 208,
	"Ē": 209,
	"ē": 210,
	"Ĕ": 211,
	"ĕ": 212,
	"Ė": 213,
	"ė": 214,
	"Ę": 215,
	"ę": 216,
	"Ě": 217,
	"ě": 218,
	"Ĝ": 219,
	"ĝ": 220,
	"Ğ": 221,
	"ğ": 222,
	"Ġ": 223,
	"ġ": 224,
	"Ģ": 225,
	"ģ": 226,
	"Ĥ": 227,
	"ĥ": 228,
	"Ħ": 229,
	"ħ": 230,
	"Ĩ": 231,
	"ĩ": 232,
	"Ī": 233,
	"ī": 234,
	"Ĭ": 235,
	"ĭ": 236,
	"Į": 237,
	"į": 238,
	"İ": 239,
	"ı": 240,
	"Ĳ": 241,
	"ĳ": 242,
	"Ĵ": 243,
	"ĵ": 244,
	"Ķ": 245,
	"ķ": 246,
	"ĸ": 247,
	"Ĺ": 248,
	"ĺ": 249,
	"Ļ": 250,
	"ļ": 251,
	"Ľ": 252,
	"ľ": 253,
	"Ŀ": 254,
	"ŀ": 255,
	"Ł": 256,
	"ł": 257,
	"Ń": 258,
	"ÙĪ": 259,
	"Ø§": 260,
	"Ø©": 261,
	"Ùħ": 262,
	"ÙĨ": 263,
	"Ø§Ø": 264,
	"Ø³": 265,
	"Ø¹": 266,
	"¦Ø©": 267,
	"ÙĪÙĨ": 268,
	"ÙħØ§Ø": 269,
	"ÙħØ§Ø¦Ø©": 270,
	"Øª": 271,
	"¨Ø¹": 272,
	"Ø«": 273,
	"Ø¨Ø¹": 274,
	"Ø±": 275,
	"Ø§ÙĨ": 276,
	"Ø£": 277,
	"ÙĦ": 278,
	"Ø§Ø«": 279,
	"®Ùħ": 280,
	"Ø®Ùħ": 281,
	"ÙħØ§ÙĨ": 282,
	"Ø³Ø¹": 283,
	"Ø³Øª": 284,
	"Ø³Ø¨Ø¹": 285,
	"ØªØ³Ø¹": 286,
	"Ø«ÙĦ": 287,
	"Ø«ÙħØ§ÙĨ": 288,
	"Ø±Ø¨Ø¹": 289,
	"Ø£Ø±Ø¨Ø¹": 290,
	"Ø®ÙħØ³": 291,
	"Ø«ÙĦØ§Ø«": 292,
	"´Ø±": 293,
	"Ø´Ø±": 294,
	"Ø¹Ø´Ø±": 295,
	"Ùģ": 296,
	"Ø£ÙĦ": 297,
	"Ø£ÙĦÙģ": 298,
	"¦Øª": 299,
	"Ø¯": 300,
	"Ø¦Øª": 301,
	"ÙĬ": 302,
	"ŃØ¯": 303,
	"ÙħØ¦Øª": 304,
	"Ø§Ø«ÙĨ": 305,
	"Ø³ØªØ©": 306,
	"Ø³ØªÙĪÙĨ": 307,
	"Ø³ØªÙħØ§Ø¦Ø©": 308,
	"Ø³Ø¨Ø¹Ø©": 309,
	"Ø³Ø¨Ø¹ÙĪÙĨ": 310,
	"Ø³Ø¨Ø¹ÙħØ§Ø¦Ø©": 311,
	"ØªØ³Ø¹Ø©": 312,
	"ØªØ³Ø¹ÙĪÙĨ": 313,
	"ØªØ³Ø¹ÙħØ§Ø¦Ø©": 314,
	"Ø«ÙħØ§ÙĨÙĪÙĨ": 315,
	"Ø«ÙħØ§ÙĨÙħØ§Ø¦Ø©": 316,
	"Ø«ÙħØ§ÙĨÙĬ": 317,
	"Ø£Ø±Ø¨Ø¹Ø©": 318,
	"Ø£Ø±Ø¨Ø¹ÙĪÙĨ": 319,
	"Ø£Ø±Ø¨Ø¹ÙħØ§Ø¦Ø©": 320,
	"Ø®ÙħØ³Ø©": 321,
	"Ø®ÙħØ³ÙĪÙĨ": 322,
	"Ø®ÙħØ³ÙħØ§Ø¦Ø©": 323,
	"Ø«ÙĦØ§Ø«Ø©": 324,
	"Ø«ÙĦØ§Ø«ÙĪÙĨ": 325,
	"Ø«ÙĦØ§Ø«ÙħØ§Ø¦Ø©": 326,
	"Ø¹Ø´Ø±ÙĪÙĨ": 327,
	"Ø«ÙħØ§ÙĨÙĬØ©": 328,
	"ÙħØ¦ØªØ§ÙĨ": 329,
	"ÙĪØ§Ø": 330,
	"Ø§Ø«ÙĨØ§ÙĨ": 331,
	"ÙĪØ§ØŃØ¯": 332,
	"ØŃØ¯": 333,
	"Ø£ØŃØ¯": 334,
	"Ø¹Ø´Ø±Ø©": 335,
	"Ø§Ø«ÙĨØ§": 336,
	"ÙħØ¦ØªØ§": 337,
	"µÙģ": 338,
	"ØµÙģ": 339,
	"ØµÙģØ±": 340
	},
	"merges": [
	[
	"Ù",
	"Ī"
	],
	[
	"Ø",
	"§"
	],
	[
	"Ø",
	"©"
	],
	[
	"Ù",
	"ħ"
	],
	[
	"Ù",
	"Ĩ"
	],
	[
	"Ø§",
	"Ø"
	],
	[
	"Ø",
	"³"
	],
	[
	"Ø",
	"¹"
	],
	[
	"¦",
	"Ø©"
	],
	[
	"ÙĪ",
	"ÙĨ"
	],
	[
	"Ùħ",
	"Ø§Ø"
	],
	[
	"ÙħØ§Ø",
	"¦Ø©"
	],
	[
	"Ø",
	"ª"
	],
	[
	"¨",
	"Ø¹"
	],
	[
	"Ø",
	"«"
	],
	[
	"Ø",
	"¨Ø¹"
	],
	[
	"Ø",
	"±"
	],
	[
	"Ø§",
	"ÙĨ"
	],
	[
	"Ø",
	"£"
	],
	[
	"Ù",
	"Ħ"
	],
	[
	"Ø§Ø",
	"«"
	],
	[
	"®",
	"Ùħ"
	],
	[
	"Ø",
	"®Ùħ"
	],
	[
	"Ùħ",
	"Ø§ÙĨ"
	],
	[
	"Ø³",
	"Ø¹"
	],
	[
	"Ø³",
	"Øª"
	],
	[
	"Ø³",
	"Ø¨Ø¹"
	],
	[
	"Øª",
	"Ø³Ø¹"
	],
	[
	"Ø«",
	"ÙĦ"
	],
	[
	"Ø«",
	"ÙħØ§ÙĨ"
	],
	[
	"Ø±",
	"Ø¨Ø¹"
	],
	[
	"Ø£",
	"Ø±Ø¨Ø¹"
	],
	[
	"Ø®Ùħ",
	"Ø³"
	],
	[
	"Ø«ÙĦ",
	"Ø§Ø«"
	],
	[
	"´",
	"Ø±"
	],
	[
	"Ø",
	"´Ø±"
	],
	[
	"Ø¹",
	"Ø´Ø±"
	],
	[
	"Ù",
	"ģ"
	],
	[
	"Ø£",
	"ÙĦ"
	],
	[
	"Ø£ÙĦ",
	"Ùģ"
	],
	[
	"¦",
	"Øª"
	],
	[
	"Ø",
	"¯"
	],
	[
	"Ø",
	"¦Øª"
	],
	[
	"Ù",
	"Ĭ"
	],
	[
	"Ń",
	"Ø¯"
	],
	[
	"Ùħ",
	"Ø¦Øª"
	],
	[
	"Ø§Ø«",
	"ÙĨ"
	],
	[
	"Ø³Øª",
	"Ø©"
	],
	[
	"Ø³Øª",
	"ÙĪÙĨ"
	],
	[
	"Ø³Øª",
	"ÙħØ§Ø¦Ø©"
	],
	[
	"Ø³Ø¨Ø¹",
	"Ø©"
	],
	[
	"Ø³Ø¨Ø¹",
	"ÙĪÙĨ"
	],
	[
	"Ø³Ø¨Ø¹",
	"ÙħØ§Ø¦Ø©"
	],
	[
	"ØªØ³Ø¹",
	"Ø©"
	],
	[
	"ØªØ³Ø¹",
	"ÙĪÙĨ"
	],
	[
	"ØªØ³Ø¹",
	"ÙħØ§Ø¦Ø©"
	],
	[
	"Ø«ÙħØ§ÙĨ",
	"ÙĪÙĨ"
	],
	[
	"Ø«ÙħØ§ÙĨ",
	"ÙħØ§Ø¦Ø©"
	],
	[
	"Ø«ÙħØ§ÙĨ",
	"ÙĬ"
	],
	[
	"Ø£Ø±Ø¨Ø¹",
	"Ø©"
	],
	[
	"Ø£Ø±Ø¨Ø¹",
	"ÙĪÙĨ"
	],
	[
	"Ø£Ø±Ø¨Ø¹",
	"ÙħØ§Ø¦Ø©"
	],
	[
	"Ø®ÙħØ³",
	"Ø©"
	],
	[
	"Ø®ÙħØ³",
	"ÙĪÙĨ"
	],
	[
	"Ø®ÙħØ³",
	"ÙħØ§Ø¦Ø©"
	],
	[
	"Ø«ÙĦØ§Ø«",
	"Ø©"
	],
	[
	"Ø«ÙĦØ§Ø«",
	"ÙĪÙĨ"
	],
	[
	"Ø«ÙĦØ§Ø«",
	"ÙħØ§Ø¦Ø©"
	],
	[
	"Ø¹Ø´Ø±",
	"ÙĪÙĨ"
	],
	[
	"Ø«ÙħØ§ÙĨÙĬ",
	"Ø©"
	],
	[
	"ÙħØ¦Øª",
	"Ø§ÙĨ"
	],
	[
	"ÙĪ",
	"Ø§Ø"
	],
	[
	"Ø§Ø«ÙĨ",
	"Ø§ÙĨ"
	],
	[
	"ÙĪØ§Ø",
	"ŃØ¯"
	],
	[
	"Ø",
	"ŃØ¯"
	],
	[
	"Ø£",
	"ØŃØ¯"
	],
	[
	"Ø¹Ø´Ø±",
	"Ø©"
	],
	[
	"Ø§Ø«ÙĨ",
	"Ø§"
	],
	[
	"ÙħØ¦Øª",
	"Ø§"
	],
	[
	"µ",
	"Ùģ"
	],
	[
	"Ø",
	"µÙģ"
	],
	[
	"ØµÙģ",
	"Ø±"
	]
	]
	}
	}