Transformers
Safetensors
Japanese
rwkv7
text-embed
WariHima's picture
Upload 60 files
356943f verified
Raw History Blame Contribute Delete
12.8 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<pad>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<unk>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<bos>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "<eos>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": null,
"post_processor": null,
"decoder": null,
"model": {
"type": "BPE",
"dropout": null,
"unk_token": "<unk>",
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": true,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"<pad>": 0,
"<unk>": 1,
"<bos>": 2,
"<eos>": 3,
"\t": 4,
"\n": 5,
" ": 6,
"!": 7,
"\"": 8,
"#": 9,
"$": 10,
"%": 11,
"&": 12,
"'": 13,
"(": 14,
")": 15,
"*": 16,
"+": 17,
",": 18,
"-": 19,
".": 20,
"/": 21,
"0": 22,
"1": 23,
"2": 24,
"3": 25,
"4": 26,
"5": 27,
"6": 28,
"7": 29,
"8": 30,
"9": 31,
":": 32,
";": 33,
"<": 34,
"=": 35,
">": 36,
"?": 37,
"@": 38,
"A": 39,
"B": 40,
"C": 41,
"D": 42,
"E": 43,
"F": 44,
"G": 45,
"H": 46,
"I": 47,
"J": 48,
"K": 49,
"L": 50,
"M": 51,
"N": 52,
"O": 53,
"P": 54,
"Q": 55,
"R": 56,
"S": 57,
"T": 58,
"U": 59,
"V": 60,
"W": 61,
"X": 62,
"Y": 63,
"Z": 64,
"[": 65,
"\\": 66,
"]": 67,
"^": 68,
"_": 69,
"`": 70,
"a": 71,
"b": 72,
"c": 73,
"d": 74,
"e": 75,
"f": 76,
"g": 77,
"h": 78,
"i": 79,
"j": 80,
"k": 81,
"l": 82,
"m": 83,
"n": 84,
"o": 85,
"p": 86,
"q": 87,
"r": 88,
"s": 89,
"t": 90,
"u": 91,
"v": 92,
"w": 93,
"x": 94,
"y": 95,
"z": 96,
"{": 97,
"|": 98,
"}": 99,
"~": 100,
"¡": 101,
"¢": 102,
"£": 103,
"¤": 104,
"¥": 105,
"¦": 106,
"§": 107,
"¨": 108,
"©": 109,
"ª": 110,
"«": 111,
"¬": 112,
"®": 113,
"¯": 114,
"°": 115,
"±": 116,
"²": 117,
"³": 118,
"´": 119,
"µ": 120,
"¶": 121,
"·": 122,
"¸": 123,
"¹": 124,
"º": 125,
"»": 126,
"¼": 127,
"½": 128,
"¾": 129,
"¿": 130,
"À": 131,
"Á": 132,
"Â": 133,
"Ã": 134,
"Ä": 135,
"Å": 136,
"Æ": 137,
"Ç": 138,
"È": 139,
"É": 140,
"Ê": 141,
"Ë": 142,
"Ì": 143,
"Í": 144,
"Î": 145,
"Ï": 146,
"Ð": 147,
"Ñ": 148,
"Ò": 149,
"Ó": 150,
"Ô": 151,
"Õ": 152,
"Ö": 153,
"×": 154,
"Ø": 155,
"Ù": 156,
"Ú": 157,
"Û": 158,
"Ü": 159,
"Ý": 160,
"Þ": 161,
"ß": 162,
"à": 163,
"á": 164,
"â": 165,
"ã": 166,
"ä": 167,
"å": 168,
"æ": 169,
"ç": 170,
"è": 171,
"é": 172,
"ê": 173,
"ë": 174,
"ì": 175,
"í": 176,
"î": 177,
"ï": 178,
"ð": 179,
"ñ": 180,
"ò": 181,
"ó": 182,
"ô": 183,
"õ": 184,
"ö": 185,
"÷": 186,
"ø": 187,
"ù": 188,
"ú": 189,
"û": 190,
"ü": 191,
"ý": 192,
"þ": 193,
"ÿ": 194,
"Ā": 195,
"ā": 196,
"Ă": 197,
"ă": 198,
"Ą": 199,
"ą": 200,
"Ć": 201,
"ć": 202,
"Ĉ": 203,
"ĉ": 204,
"Ċ": 205,
"ċ": 206,
"Č": 207,
"č": 208,
"Ď": 209,
"ď": 210,
"Đ": 211,
"đ": 212,
"Ē": 213,
"ē": 214,
"Ĕ": 215,
"ĕ": 216,
"Ė": 217,
"ė": 218,
"Ę": 219,
"ę": 220,
"Ě": 221,
"ě": 222,
"Ĝ": 223,
"ĝ": 224,
"Ğ": 225,
"ğ": 226,
"Ġ": 227,
"ġ": 228,
"Ģ": 229,
"ģ": 230,
"Ĥ": 231,
"ĥ": 232,
"Ħ": 233,
"ħ": 234,
"Ĩ": 235,
"ĩ": 236,
"Ī": 237,
"ī": 238,
"Ĭ": 239,
"ĭ": 240,
"Į": 241,
"į": 242,
"İ": 243,
"ı": 244,
"IJ": 245,
"ij": 246,
"Ĵ": 247,
"ĵ": 248,
"Ķ": 249,
"ķ": 250,
"ĸ": 251,
"Ĺ": 252,
"ĺ": 253,
"Ļ": 254,
"ļ": 255,
"Ľ": 256,
"ľ": 257,
"Ŀ": 258,
"ŀ": 259,
"Ł": 260,
"ł": 261,
"Ń": 262,
"ぁ": 263,
"あ": 264,
"ぃ": 265,
"い": 266,
"ぅ": 267,
"う": 268,
"ぇ": 269,
"え": 270,
"ぉ": 271,
"お": 272,
"か": 273,
"が": 274,
"き": 275,
"ぎ": 276,
"く": 277,
"ぐ": 278,
"け": 279,
"げ": 280,
"こ": 281,
"ご": 282,
"さ": 283,
"ざ": 284,
"し": 285,
"じ": 286,
"す": 287,
"ず": 288,
"せ": 289,
"ぜ": 290,
"そ": 291,
"ぞ": 292,
"た": 293,
"だ": 294,
"ち": 295,
"ぢ": 296,
"っ": 297,
"つ": 298,
"づ": 299,
"て": 300,
"で": 301,
"と": 302,
"ど": 303,
"な": 304,
"に": 305,
"ぬ": 306,
"ね": 307,
"の": 308,
"は": 309,
"ば": 310,
"ぱ": 311,
"ひ": 312,
"び": 313,
"ぴ": 314,
"ふ": 315,
"ぶ": 316,
"ぷ": 317,
"へ": 318,
"べ": 319,
"ぺ": 320,
"ほ": 321,
"ぼ": 322,
"ぽ": 323,
"ま": 324,
"み": 325,
"む": 326,
"め": 327,
"も": 328,
"ゃ": 329,
"や": 330,
"ゅ": 331,
"ゆ": 332,
"ょ": 333,
"よ": 334,
"ら": 335,
"り": 336,
"る": 337,
"れ": 338,
"ろ": 339,
"ゎ": 340,
"わ": 341,
"ゐ": 342,
"ゑ": 343,
"を": 344,
"ん": 345,
"ゔ": 346,
"ゕ": 347,
"゚": 348,
"゛": 349,
"゜": 350,
"ゝ": 351,
"ゞ": 352,
"ゟ": 353,
"゠": 354,
"ァ": 355,
"ア": 356,
"ィ": 357,
"イ": 358,
"ゥ": 359,
"ウ": 360,
"ェ": 361,
"エ": 362,
"ォ": 363,
"オ": 364,
"カ": 365,
"ガ": 366,
"キ": 367,
"ギ": 368,
"ク": 369,
"グ": 370,
"ケ": 371,
"ゲ": 372,
"コ": 373,
"ゴ": 374,
"サ": 375,
"ザ": 376,
"シ": 377,
"ジ": 378,
"ス": 379,
"ズ": 380,
"セ": 381,
"ゼ": 382,
"ソ": 383,
"ゾ": 384,
"タ": 385,
"ダ": 386,
"チ": 387,
"ヂ": 388,
"ッ": 389,
"ツ": 390,
"ヅ": 391,
"テ": 392,
"デ": 393,
"ト": 394,
"ド": 395,
"ナ": 396,
"ニ": 397,
"ヌ": 398,
"ネ": 399,
"ノ": 400,
"ハ": 401,
"バ": 402,
"パ": 403,
"ヒ": 404,
"ビ": 405,
"ピ": 406,
"フ": 407,
"ブ": 408,
"プ": 409,
"ヘ": 410,
"ベ": 411,
"ペ": 412,
"ホ": 413,
"ボ": 414,
"ポ": 415,
"マ": 416,
"ミ": 417,
"ム": 418,
"メ": 419,
"モ": 420,
"ャ": 421,
"ヤ": 422,
"ュ": 423,
"ユ": 424,
"ョ": 425,
"ヨ": 426,
"ラ": 427,
"リ": 428,
"ル": 429,
"レ": 430,
"ロ": 431,
"ヮ": 432,
"ワ": 433,
"ヰ": 434,
"ヱ": 435,
"ヲ": 436,
"ン": 437,
"ヴ": 438,
"ヵ": 439,
"ヶ": 440,
"ヷ": 441,
"ヸ": 442,
"ヹ": 443,
"ヺ": 444,
"・": 445,
"ー": 446,
"ヽ": 447,
"ヾ": 448,
"ãĢ": 449,
"ãĢģ": 450,
"ï¼": 451,
"ãĢĤ": 452,
"å¹": 453,
"å¹´": 454,
"\n ": 455,
"ä¸": 456,
"æľ": 457,
"ï¼Ī": 458,
"ï¼ī": 459,
"çĶ": 460,
"å¤": 461,
"äº": 462,
"å®": 463,
"æĹ": 464,
"åħ": 465,
"るãĢĤ": 466,
"æĹ¥": 467,
"åIJ": 468,
"- ": 469,
"19": 470,
" - ": 471,
"åı": 472,
"åĪ": 473,
"ä½": 474,
"ä»": 475,
"æľĪ": 476,
"åŃ": 477,
"åĽ": 478,
"éĸ": 479,
"æĸ": 480,
"éĢ": 481,
"して": 482,
"20": 483,
"éģ": 484,
"è¡": 485,
"たãĢĤ": 486,
"され": 487,
"のå": 488,
"æĿ": 489,
"è¨": 490,
"æĪ": 491,
"ãĢį": 492,
"ãĢĮ": 493,
"のæ": 494,
"大": 495,
"åľ": 496,
"する": 497,
"\n\n": 498,
"ï¼ī\n ": 499,
"ç·": 500,
"てい": 501,
"åĮ": 502,
"éĥ": 503,
"è»": 504,
"å¾": 505,
"であ": 506,
"åĽ½": 507,
"èª": 508,
"æľ¬": 509,
"çµ": 510,
"éĩ": 511
},
"merges": [
[
"ã",
"Ģ"
],
[
"ãĢ",
"ģ"
],
[
"ï",
"¼"
],
[
"ãĢ",
"Ĥ"
],
[
"å",
"¹"
],
[
"å¹",
"´"
],
[
"\n",
" "
],
[
"ä",
"¸"
],
[
"æ",
"ľ"
],
[
"ï¼",
"Ī"
],
[
"ï¼",
"ī"
],
[
"ç",
"Ķ"
],
[
"å",
"¤"
],
[
"ä",
"º"
],
[
"å",
"®"
],
[
"æ",
"Ĺ"
],
[
"å",
"ħ"
],
[
"る",
"ãĢĤ"
],
[
"æĹ",
"¥"
],
[
"å",
"IJ"
],
[
"-",
" "
],
[
"1",
"9"
],
[
" ",
"- "
],
[
"å",
"ı"
],
[
"å",
"Ī"
],
[
"ä",
"½"
],
[
"ä",
"»"
],
[
"æľ",
"Ī"
],
[
"å",
"Ń"
],
[
"å",
"Ľ"
],
[
"é",
"ĸ"
],
[
"æ",
"ĸ"
],
[
"é",
"Ģ"
],
[
"し",
"て"
],
[
"2",
"0"
],
[
"é",
"ģ"
],
[
"è",
"¡"
],
[
"た",
"ãĢĤ"
],
[
"さ",
"れ"
],
[
"の",
"å"
],
[
"æ",
"Ŀ"
],
[
"è",
"¨"
],
[
"æ",
"Ī"
],
[
"ãĢ",
"į"
],
[
"ãĢ",
"Į"
],
[
"の",
"æ"
],
[
"å¤",
"§"
],
[
"å",
"ľ"
],
[
"す",
"る"
],
[
"\n",
"\n"
],
[
"ï¼ī",
"\n "
],
[
"ç",
"·"
],
[
"て",
"い"
],
[
"å",
"Į"
],
[
"é",
"ĥ"
],
[
"è",
"»"
],
[
"å",
"¾"
],
[
"で",
"あ"
],
[
"åĽ",
"½"
],
[
"è",
"ª"
],
[
"æľ",
"¬"
],
[
"ç",
"µ"
],
[
"é",
"ĩ"
]
]
}
}