{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": {
"type": "NFC"
},
"pre_tokenizer": {
"type": "Sequence",
"pretokenizers": [
{
"type": "Split",
"pattern": {
"Regex": "[+=]|[^\\S\\r\\n]*[\\n\\r]+|[^\\S\\r\\n]+"
},
"behavior": "Isolated",
"invert": false
},
{
"type": "Split",
"pattern": {
"Regex": "\\p{N}{1,3}"
},
"behavior": "Isolated",
"invert": false
},
{
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": true,
"use_regex": false
}
]
},
"post_processor": null,
"decoder": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true,
"use_regex": true
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"": 0,
"": 1,
"": 2,
"!": 3,
"\"": 4,
"#": 5,
"$": 6,
"%": 7,
"&": 8,
"'": 9,
"(": 10,
")": 11,
"*": 12,
"+": 13,
",": 14,
"-": 15,
".": 16,
"/": 17,
"0": 18,
"1": 19,
"2": 20,
"3": 21,
"4": 22,
"5": 23,
"6": 24,
"7": 25,
"8": 26,
"9": 27,
":": 28,
";": 29,
"<": 30,
"=": 31,
">": 32,
"?": 33,
"@": 34,
"A": 35,
"B": 36,
"C": 37,
"D": 38,
"E": 39,
"F": 40,
"G": 41,
"H": 42,
"I": 43,
"J": 44,
"K": 45,
"L": 46,
"M": 47,
"N": 48,
"O": 49,
"P": 50,
"Q": 51,
"R": 52,
"S": 53,
"T": 54,
"U": 55,
"V": 56,
"W": 57,
"X": 58,
"Y": 59,
"Z": 60,
"[": 61,
"\\": 62,
"]": 63,
"^": 64,
"_": 65,
"`": 66,
"a": 67,
"b": 68,
"c": 69,
"d": 70,
"e": 71,
"f": 72,
"g": 73,
"h": 74,
"i": 75,
"j": 76,
"k": 77,
"l": 78,
"m": 79,
"n": 80,
"o": 81,
"p": 82,
"q": 83,
"r": 84,
"s": 85,
"t": 86,
"u": 87,
"v": 88,
"w": 89,
"x": 90,
"y": 91,
"z": 92,
"{": 93,
"|": 94,
"}": 95,
"~": 96,
"¡": 97,
"¢": 98,
"£": 99,
"¤": 100,
"¥": 101,
"¦": 102,
"§": 103,
"¨": 104,
"©": 105,
"ª": 106,
"«": 107,
"¬": 108,
"®": 109,
"¯": 110,
"°": 111,
"±": 112,
"²": 113,
"³": 114,
"´": 115,
"µ": 116,
"¶": 117,
"·": 118,
"¸": 119,
"¹": 120,
"º": 121,
"»": 122,
"¼": 123,
"½": 124,
"¾": 125,
"¿": 126,
"À": 127,
"Á": 128,
"Â": 129,
"Ã": 130,
"Ä": 131,
"Å": 132,
"Æ": 133,
"Ç": 134,
"È": 135,
"É": 136,
"Ê": 137,
"Ë": 138,
"Ì": 139,
"Í": 140,
"Î": 141,
"Ï": 142,
"Ð": 143,
"Ñ": 144,
"Ò": 145,
"Ó": 146,
"Ô": 147,
"Õ": 148,
"Ö": 149,
"×": 150,
"Ø": 151,
"Ù": 152,
"Ú": 153,
"Û": 154,
"Ü": 155,
"Ý": 156,
"Þ": 157,
"ß": 158,
"à": 159,
"á": 160,
"â": 161,
"ã": 162,
"ä": 163,
"å": 164,
"æ": 165,
"ç": 166,
"è": 167,
"é": 168,
"ê": 169,
"ë": 170,
"ì": 171,
"í": 172,
"î": 173,
"ï": 174,
"ð": 175,
"ñ": 176,
"ò": 177,
"ó": 178,
"ô": 179,
"õ": 180,
"ö": 181,
"÷": 182,
"ø": 183,
"ù": 184,
"ú": 185,
"û": 186,
"ü": 187,
"ý": 188,
"þ": 189,
"ÿ": 190,
"Ā": 191,
"ā": 192,
"Ă": 193,
"ă": 194,
"Ą": 195,
"ą": 196,
"Ć": 197,
"ć": 198,
"Ĉ": 199,
"ĉ": 200,
"Ċ": 201,
"ċ": 202,
"Č": 203,
"č": 204,
"Ď": 205,
"ď": 206,
"Đ": 207,
"đ": 208,
"Ē": 209,
"ē": 210,
"Ĕ": 211,
"ĕ": 212,
"Ė": 213,
"ė": 214,
"Ę": 215,
"ę": 216,
"Ě": 217,
"ě": 218,
"Ĝ": 219,
"ĝ": 220,
"Ğ": 221,
"ğ": 222,
"Ġ": 223,
"ġ": 224,
"Ģ": 225,
"ģ": 226,
"Ĥ": 227,
"ĥ": 228,
"Ħ": 229,
"ħ": 230,
"Ĩ": 231,
"ĩ": 232,
"Ī": 233,
"ī": 234,
"Ĭ": 235,
"ĭ": 236,
"Į": 237,
"į": 238,
"İ": 239,
"ı": 240,
"IJ": 241,
"ij": 242,
"Ĵ": 243,
"ĵ": 244,
"Ķ": 245,
"ķ": 246,
"ĸ": 247,
"Ĺ": 248,
"ĺ": 249,
"Ļ": 250,
"ļ": 251,
"Ľ": 252,
"ľ": 253,
"Ŀ": 254,
"ŀ": 255,
"Ł": 256,
"ł": 257,
"Ń": 258,
"åį": 259,
"çĻ": 260,
"åįģ": 261,
"çĻ¾": 262,
"ä¸": 263,
"äº": 264,
"åħ": 265,
"¹Ŀ": 266,
"ä¹Ŀ": 267,
"åĽ": 268,
"ä¸ĥ": 269,
"ä¸ī": 270,
"äºĮ": 271,
"äºĶ": 272,
"åħ«": 273,
"åħŃ": 274,
"åĽĽ": 275,
"åįĥ": 276,
"ä¸Ģ": 277,
"ä¹Ŀåįģ": 278,
"ä¹ĿçĻ¾": 279,
"ä¸ĥåįģ": 280,
"ä¸ĥçĻ¾": 281,
"ä¸īåįģ": 282,
"ä¸īçĻ¾": 283,
"äºĮåįģ": 284,
"äºĮçĻ¾": 285,
"äºĶåįģ": 286,
"äºĶçĻ¾": 287,
"åħ«åįģ": 288,
"åħ«çĻ¾": 289,
"åħŃåįģ": 290,
"åħŃçĻ¾": 291,
"åĽĽåįģ": 292,
"åĽĽçĻ¾": 293,
"åįĥçĻ¾": 294,
"åįĥäºĮçĻ¾": 295,
"åįĥä¸īçĻ¾": 296,
"åįĥåĽĽçĻ¾": 297,
"åįĥäºĶçĻ¾": 298,
"åįĥåħŃçĻ¾": 299,
"åįģä¹Ŀ": 300,
"åįģä¸ĥ": 301,
"åįģä¸ī": 302,
"åįģäºĮ": 303,
"åįģäºĶ": 304,
"åįģåħ«": 305,
"åįģåħŃ": 306,
"åįģåĽĽ": 307,
"åįģä¸Ģ": 308,
"ä¹Ŀåįģä¹Ŀ": 309,
"ä¹Ŀåįģä¸ĥ": 310,
"ä¹Ŀåįģä¸ī": 311,
"ä¹ĿåįģäºĮ": 312,
"ä¹ĿåįģäºĶ": 313,
"ä¹Ŀåįģåħ«": 314,
"ä¹ĿåįģåħŃ": 315,
"ä¹ĿåįģåĽĽ": 316,
"ä¹Ŀåįģä¸Ģ": 317,
"ä¹ĿçĻ¾ä¸ĥåįģ": 318,
"ä¹ĿçĻ¾ä¸īåįģ": 319,
"ä¹ĿçĻ¾äºĮåįģ": 320,
"ä¹ĿçĻ¾äºĶåįģ": 321,
"ä¹ĿçĻ¾åħ«åįģ": 322,
"ä¹ĿçĻ¾åħŃåįģ": 323,
"ä¹ĿçĻ¾åĽĽåįģ": 324,
"ä¸ĥçĻ¾ä¸ĥåįģ": 325,
"ä¸ĥçĻ¾ä¸īåįģ": 326,
"ä¸ĥçĻ¾äºĮåįģ": 327,
"ä¸ĥçĻ¾äºĶåįģ": 328,
"ä¸ĥçĻ¾åħ«åįģ": 329,
"ä¸ĥçĻ¾åħŃåįģ": 330,
"ä¸ĥçĻ¾åĽĽåįģ": 331,
"åħ«çĻ¾ä¸ĥåįģ": 332,
"åħ«çĻ¾ä¸īåįģ": 333,
"åħ«çĻ¾äºĮåįģ": 334,
"åħ«çĻ¾äºĶåįģ": 335,
"åħ«çĻ¾åħ«åįģ": 336,
"åħ«çĻ¾åħŃåįģ": 337,
"åħ«çĻ¾åĽĽåįģ": 338,
"åħŃçĻ¾åħ«åįģ": 339,
"åħŃçĻ¾ä¸ĥåįģ": 340,
"åħŃçĻ¾åħŃåįģ": 341,
"åħŃçĻ¾äºĶåįģ": 342,
"åħŃçĻ¾åĽĽåįģ": 343,
"åħŃçĻ¾ä¸īåįģ": 344,
"åħŃçĻ¾äºĮåįģ": 345,
"äºĶçĻ¾åħ«åįģ": 346,
"äºĶçĻ¾ä¸ĥåįģ": 347,
"äºĶçĻ¾åħŃåįģ": 348,
"äºĶçĻ¾äºĶåįģ": 349,
"äºĶçĻ¾åĽĽåįģ": 350,
"äºĶçĻ¾ä¸īåįģ": 351,
"äºĶçĻ¾äºĮåįģ": 352,
"åĽĽçĻ¾åħ«åįģ": 353,
"åĽĽçĻ¾ä¸ĥåįģ": 354,
"åĽĽçĻ¾åħŃåįģ": 355,
"åĽĽçĻ¾äºĶåįģ": 356,
"åĽĽçĻ¾åĽĽåįģ": 357,
"åĽĽçĻ¾ä¸īåįģ": 358,
"åĽĽçĻ¾äºĮåįģ": 359
},
"merges": [
[
"å",
"į"
],
[
"ç",
"Ļ"
],
[
"åį",
"ģ"
],
[
"çĻ",
"¾"
],
[
"ä",
"¸"
],
[
"ä",
"º"
],
[
"å",
"ħ"
],
[
"¹",
"Ŀ"
],
[
"ä",
"¹Ŀ"
],
[
"å",
"Ľ"
],
[
"ä¸",
"ĥ"
],
[
"ä¸",
"ī"
],
[
"äº",
"Į"
],
[
"äº",
"Ķ"
],
[
"åħ",
"«"
],
[
"åħ",
"Ń"
],
[
"åĽ",
"Ľ"
],
[
"åį",
"ĥ"
],
[
"ä¸",
"Ģ"
],
[
"ä¹Ŀ",
"åįģ"
],
[
"ä¹Ŀ",
"çĻ¾"
],
[
"ä¸ĥ",
"åįģ"
],
[
"ä¸ĥ",
"çĻ¾"
],
[
"ä¸ī",
"åįģ"
],
[
"ä¸ī",
"çĻ¾"
],
[
"äºĮ",
"åįģ"
],
[
"äºĮ",
"çĻ¾"
],
[
"äºĶ",
"åįģ"
],
[
"äºĶ",
"çĻ¾"
],
[
"åħ«",
"åįģ"
],
[
"åħ«",
"çĻ¾"
],
[
"åħŃ",
"åįģ"
],
[
"åħŃ",
"çĻ¾"
],
[
"åĽĽ",
"åįģ"
],
[
"åĽĽ",
"çĻ¾"
],
[
"åįĥ",
"çĻ¾"
],
[
"åįĥ",
"äºĮçĻ¾"
],
[
"åįĥ",
"ä¸īçĻ¾"
],
[
"åįĥ",
"åĽĽçĻ¾"
],
[
"åįĥ",
"äºĶçĻ¾"
],
[
"åįĥ",
"åħŃçĻ¾"
],
[
"åįģ",
"ä¹Ŀ"
],
[
"åįģ",
"ä¸ĥ"
],
[
"åįģ",
"ä¸ī"
],
[
"åįģ",
"äºĮ"
],
[
"åįģ",
"äºĶ"
],
[
"åįģ",
"åħ«"
],
[
"åįģ",
"åħŃ"
],
[
"åįģ",
"åĽĽ"
],
[
"åįģ",
"ä¸Ģ"
],
[
"ä¹Ŀåįģ",
"ä¹Ŀ"
],
[
"ä¹Ŀåįģ",
"ä¸ĥ"
],
[
"ä¹Ŀåįģ",
"ä¸ī"
],
[
"ä¹Ŀåįģ",
"äºĮ"
],
[
"ä¹Ŀåįģ",
"äºĶ"
],
[
"ä¹Ŀåįģ",
"åħ«"
],
[
"ä¹Ŀåįģ",
"åħŃ"
],
[
"ä¹Ŀåįģ",
"åĽĽ"
],
[
"ä¹Ŀåįģ",
"ä¸Ģ"
],
[
"ä¹ĿçĻ¾",
"ä¸ĥåįģ"
],
[
"ä¹ĿçĻ¾",
"ä¸īåįģ"
],
[
"ä¹ĿçĻ¾",
"äºĮåįģ"
],
[
"ä¹ĿçĻ¾",
"äºĶåįģ"
],
[
"ä¹ĿçĻ¾",
"åħ«åįģ"
],
[
"ä¹ĿçĻ¾",
"åħŃåįģ"
],
[
"ä¹ĿçĻ¾",
"åĽĽåįģ"
],
[
"ä¸ĥçĻ¾",
"ä¸ĥåįģ"
],
[
"ä¸ĥçĻ¾",
"ä¸īåįģ"
],
[
"ä¸ĥçĻ¾",
"äºĮåįģ"
],
[
"ä¸ĥçĻ¾",
"äºĶåįģ"
],
[
"ä¸ĥçĻ¾",
"åħ«åįģ"
],
[
"ä¸ĥçĻ¾",
"åħŃåįģ"
],
[
"ä¸ĥçĻ¾",
"åĽĽåįģ"
],
[
"åħ«çĻ¾",
"ä¸ĥåįģ"
],
[
"åħ«çĻ¾",
"ä¸īåįģ"
],
[
"åħ«çĻ¾",
"äºĮåįģ"
],
[
"åħ«çĻ¾",
"äºĶåįģ"
],
[
"åħ«çĻ¾",
"åħ«åįģ"
],
[
"åħ«çĻ¾",
"åħŃåįģ"
],
[
"åħ«çĻ¾",
"åĽĽåįģ"
],
[
"åħŃçĻ¾",
"åħ«åįģ"
],
[
"åħŃçĻ¾",
"ä¸ĥåįģ"
],
[
"åħŃçĻ¾",
"åħŃåįģ"
],
[
"åħŃçĻ¾",
"äºĶåįģ"
],
[
"åħŃçĻ¾",
"åĽĽåįģ"
],
[
"åħŃçĻ¾",
"ä¸īåįģ"
],
[
"åħŃçĻ¾",
"äºĮåįģ"
],
[
"äºĶçĻ¾",
"åħ«åįģ"
],
[
"äºĶçĻ¾",
"ä¸ĥåįģ"
],
[
"äºĶçĻ¾",
"åħŃåįģ"
],
[
"äºĶçĻ¾",
"äºĶåįģ"
],
[
"äºĶçĻ¾",
"åĽĽåįģ"
],
[
"äºĶçĻ¾",
"ä¸īåįģ"
],
[
"äºĶçĻ¾",
"äºĮåįģ"
],
[
"åĽĽçĻ¾",
"åħ«åįģ"
],
[
"åĽĽçĻ¾",
"ä¸ĥåįģ"
],
[
"åĽĽçĻ¾",
"åħŃåįģ"
],
[
"åĽĽçĻ¾",
"äºĶåįģ"
],
[
"åĽĽçĻ¾",
"åĽĽåįģ"
],
[
"åĽĽçĻ¾",
"ä¸īåįģ"
],
[
"åĽĽçĻ¾",
"äºĮåįģ"
]
]
}
}