single-floppy-346k-base / tokenizer.json
NILKNARFGonzo's picture
Upload 5 files
718d60b verified
Raw
History Blame Contribute Delete
24.4 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "[BOS]",
"special": true
},
{
"id": 1,
"content": "[EOS]",
"special": true
},
{
"id": 2,
"content": "[PAD]",
"special": true
},
{
"id": 3,
"content": "[UNK]",
"special": true
}
],
"normalizer": null,
"pre_tokenizer": null,
"post_processor": null,
"decoder": null,
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"[BOS]": 0,
"[EOS]": 1,
"[PAD]": 2,
"[UNK]": 3,
"\t": 4,
"\n": 5,
" ": 6,
"!": 7,
"\"": 8,
"#": 9,
"$": 10,
"%": 11,
"&": 12,
"'": 13,
"(": 14,
")": 15,
"*": 16,
"+": 17,
",": 18,
"-": 19,
".": 20,
"/": 21,
"0": 22,
"1": 23,
"2": 24,
"3": 25,
"4": 26,
"5": 27,
"6": 28,
"7": 29,
"8": 30,
"9": 31,
":": 32,
";": 33,
"<": 34,
"=": 35,
">": 36,
"?": 37,
"@": 38,
"A": 39,
"B": 40,
"C": 41,
"D": 42,
"E": 43,
"F": 44,
"G": 45,
"H": 46,
"I": 47,
"J": 48,
"K": 49,
"L": 50,
"M": 51,
"N": 52,
"O": 53,
"P": 54,
"Q": 55,
"R": 56,
"S": 57,
"T": 58,
"U": 59,
"V": 60,
"W": 61,
"X": 62,
"Y": 63,
"Z": 64,
"[": 65,
"\\": 66,
"]": 67,
"^": 68,
"_": 69,
"`": 70,
"a": 71,
"b": 72,
"c": 73,
"d": 74,
"e": 75,
"f": 76,
"g": 77,
"h": 78,
"i": 79,
"j": 80,
"k": 81,
"l": 82,
"m": 83,
"n": 84,
"o": 85,
"p": 86,
"q": 87,
"r": 88,
"s": 89,
"t": 90,
"u": 91,
"v": 92,
"w": 93,
"x": 94,
"y": 95,
"z": 96,
"{": 97,
"|": 98,
"}": 99,
"\u00a0": 100,
"\u00a7": 101,
"\u00b0": 102,
"\u00b2": 103,
"\u00b3": 104,
"\u00c9": 105,
"\u00d1": 106,
"\u00d7": 107,
"\u00e0": 108,
"\u00e2": 109,
"\u00e7": 110,
"\u00e8": 111,
"\u00e9": 112,
"\u00ea": 113,
"\u00eb": 114,
"\u00ed": 115,
"\u00f6": 116,
"\u00f7": 117,
"\u00f8": 118,
"\u00f9": 119,
"\u00fa": 120,
"\u00fc": 121,
"\u0301": 122,
"\u0393": 123,
"\u0394": 124,
"\u039b": 125,
"\u03a3": 126,
"\u03b1": 127,
"\u03bb": 128,
"\u03c0": 129,
"\u0410": 130,
"\u0414": 131,
"\u0430": 132,
"\u0432": 133,
"\u0434": 134,
"\u0435": 135,
"\u0438": 136,
"\u043a": 137,
"\u043b": 138,
"\u043c": 139,
"\u043d": 140,
"\u043e": 141,
"\u0440": 142,
"\u0441": 143,
"\u0442": 144,
"\u0447": 145,
"\u044b": 146,
"\u044c": 147,
"\u2013": 148,
"\u2014": 149,
"\u2018": 150,
"\u2019": 151,
"\u201c": 152,
"\u201d": 153,
"\u2026": 154,
"\u20ac": 155,
"\u2192": 156,
"\u221a": 157,
"\u2229": 158,
"\u222a": 159,
"\u222b": 160,
"\u222c": 161,
"\u2248": 162,
"\u2764": 163,
"\uc678": 164,
"\ud654": 165,
"\ufe0f": 166,
"e ": 167,
" t": 168,
"in": 169,
"s ": 170,
"an": 171,
"er": 172,
" th": 173,
"on": 174,
"d ": 175,
"at": 176,
"en": 177,
" ": 178,
"or": 179,
", ": 180,
" the ": 181,
"t ": 182,
"al": 183,
"es": 184,
"ing": 185,
"ar": 186,
"y ": 187,
"ou": 188,
". ": 189,
"st": 190,
"it": 191,
"ion": 192,
"of": 193,
"re": 194,
"o ": 195,
"and ": 196,
"ic": 197,
"a ": 198,
"\n\n": 199,
"th": 200,
"ing ": 201,
" ": 202,
"ro": 203,
"ac": 204,
"of ": 205,
"le": 206,
"is ": 207,
" to ": 208,
"al ": 209,
"ed ": 210,
"er ": 211,
"el": 212,
"ec": 213,
"ent": 214,
"is": 215,
".\n\n": 216,
"or ": 217,
"es ": 218,
"im": 219,
"Th": 220,
"om": 221,
"an ": 222,
"h ": 223,
"as": 224,
"il": 225,
"ation": 226,
"##": 227,
"ur": 228,
"ig": 229,
"em": 230,
"iv": 231,
"ow": 232,
"ol": 233,
"ed": 234,
"in ": 235,
"ch": 236,
"us": 237,
"pro": 238,
"e th": 239,
"et": 240,
"ad": 241,
"ir": 242,
"con": 243,
"un": 244,
"tion": 245,
"pl": 246,
"if": 247,
"ul": 248,
"all": 249,
"am": 250,
"id": 251,
"at ": 252,
"um": 253,
"ex": 254,
"ist": 255,
". Th": 256,
"wh": 257,
"ab": 258,
": ": 259,
"ag": 260,
"uc": 261,
"ld ": 262,
"av": 263,
"can ": 264,
"as ": 265,
"int": 266,
" ": 267,
"'s ": 268,
"wit": 269,
":\n": 270,
"res": 271,
"for": 272,
"and": 273,
"ould ": 274,
"s, ": 275,
"per": 276,
"os": 277,
"# ": 278,
"ap": 279,
"com": 280,
"en ": 281,
"ev": 282,
"igh": 283,
"ant": 284,
"### ": 285,
"you": 286,
"enc": 287,
"ent ": 288,
"ot": 289,
"e t": 290,
" the": 291,
"for ": 292,
"oc": 293,
"e s": 294,
"of the ": 295,
"ut": 296,
"od": 297,
"qu": 298,
"ma": 299,
"ep": 300,
"ation ": 301,
"ay": 302,
"e, ": 303,
"act": 304,
"e the ": 305,
" that ": 306,
"n ": 307,
"it ": 308,
"ov": 309,
"ic ": 310,
"ne": 311,
"op": 312,
"str": 313,
"e of ": 314,
"be ": 315,
"anc": 316,
"sh": 317,
"ir ": 318,
"ect": 319,
"ess": 320,
"on ": 321,
"ef": 322,
"with ": 323,
"ly ": 324,
". I": 325,
"oun": 326,
"ate ": 327,
"in the ": 328,
"= ": 329,
"sist": 330,
"le ": 331,
"cl": 332,
"k ": 333,
"pr": 334,
"gr": 335,
"s and ": 336,
"iz": 337,
" to": 338,
"m ": 339,
". This ": 340,
"are ": 341,
"tion ": 342,
"por": 343,
"could ": 344,
"tr": 345,
"our": 346,
"ing the ": 347,
"es, ": 348,
"As": 349,
"oth": 350,
"be": 351,
", the ": 352,
"y, ": 353,
"wor": 354,
"ers": 355,
"ib": 356,
"uman": 357,
"ot ": 358,
"su": 359,
"ive ": 360,
"ial ": 361,
"ity ": 362,
") ": 363,
"ment": 364,
" their ": 365,
"**": 366,
" tr": 367,
"mor": 368,
"Assist": 369,
"Human": 370,
"ens": 371,
"Assistant": 372,
"### Human": 373,
"### Assistant": 374,
"### Human:\n": 375,
"### Assistant:\n": 376,
"fro": 377,
"ind": 378,
"sp": 379,
"og": 380,
"cre": 381,
"hav": 382,
"ud": 383,
"s of ": 384,
"ip": 385,
"art": 386,
"comp": 387,
"00": 388,
"ang": 389,
"ight ": 390,
"\n ": 391,
"fl": 392,
"l ": 393,
"end": 394,
"ain": 395,
"with": 396,
"als": 397,
"The ": 398,
"duc": 399,
"you ": 400,
"not ": 401,
"ere ": 402,
"a s": 403,
"ally ": 404,
"ow ": 405,
"he": 406,
"ition": 407,
"est": 408,
"ical ": 409,
"eng": 410,
"ers ": 411,
"ese ": 412,
"we": 413,
"dis": 414,
"li": 415,
"e to ": 416,
"ar ": 417,
"w ": 418,
"by ": 419,
"se": 420,
"ies ": 421,
". The ": 422,
"form": 423,
"der": 424,
", and ": 425,
"up": 426,
"ak": 427,
"fer": 428,
"would ": 429,
"rec": 430,
"ed to ": 431,
"e and ": 432,
"your": 433,
"ign": 434,
"me": 435,
"fe": 436,
"ity": 437,
"port": 438,
"ong": 439,
"bu": 440,
"ight": 441,
"ph": 442,
"s.\n\n": 443,
"ter": 444,
"sc": 445,
"prov": 446,
"your ": 447,
"ess ": 448,
"ich ": 449,
"ass": 450,
"0 ": 451,
"more ": 452,
"out": 453,
"on the ": 454,
"oo": 455,
" to the ": 456,
"also ": 457,
"ienc": 458,
"have ": 459,
"ast": 460,
"out ": 461,
"af": 462,
"from ": 463,
"over": 464,
"oug": 465,
" and ": 466,
"bl": 467,
"cont": 468,
". H": 469,
"ust": 470,
"ath": 471,
".\n": 472,
"which ": 473,
"cr": 474,
"and the ": 475,
" this ": 476,
"man": 477,
"ay ": 478,
"tur": 479,
"s to ": 480,
"ack": 481,
"ever": 482,
"his ": 483,
"pos": 484,
"\n\n### Assistant:\n": 485,
"pre": 486,
"lik": 487,
"pot": 488,
". A": 489,
"\n ": 490,
"car": 491,
"fr": 492,
"hel": 493,
"inter": 494,
"- ": 495,
" they ": 496,
"ure": 497,
"sy": 498,
"might ": 499,
"potent": 500,
"de": 501
},
"merges": [
[
"e",
" "
],
[
" ",
"t"
],
[
"i",
"n"
],
[
"s",
" "
],
[
"a",
"n"
],
[
"e",
"r"
],
[
" t",
"h"
],
[
"o",
"n"
],
[
"d",
" "
],
[
"a",
"t"
],
[
"e",
"n"
],
[
" ",
" "
],
[
"o",
"r"
],
[
",",
" "
],
[
" th",
"e "
],
[
"t",
" "
],
[
"a",
"l"
],
[
"e",
"s"
],
[
"in",
"g"
],
[
"a",
"r"
],
[
"y",
" "
],
[
"o",
"u"
],
[
".",
" "
],
[
"s",
"t"
],
[
"i",
"t"
],
[
"i",
"on"
],
[
"o",
"f"
],
[
"r",
"e"
],
[
"o",
" "
],
[
"an",
"d "
],
[
"i",
"c"
],
[
"a",
" "
],
[
"\n",
"\n"
],
[
"t",
"h"
],
[
"ing",
" "
],
[
" ",
" "
],
[
"r",
"o"
],
[
"a",
"c"
],
[
"of",
" "
],
[
"l",
"e"
],
[
"i",
"s "
],
[
" t",
"o "
],
[
"al",
" "
],
[
"e",
"d "
],
[
"er",
" "
],
[
"e",
"l"
],
[
"e",
"c"
],
[
"en",
"t"
],
[
"i",
"s"
],
[
".",
"\n\n"
],
[
"or",
" "
],
[
"e",
"s "
],
[
"i",
"m"
],
[
"T",
"h"
],
[
"o",
"m"
],
[
"an",
" "
],
[
"h",
" "
],
[
"a",
"s"
],
[
"i",
"l"
],
[
"at",
"ion"
],
[
"#",
"#"
],
[
"u",
"r"
],
[
"i",
"g"
],
[
"e",
"m"
],
[
"i",
"v"
],
[
"o",
"w"
],
[
"o",
"l"
],
[
"e",
"d"
],
[
"in",
" "
],
[
"c",
"h"
],
[
"u",
"s"
],
[
"p",
"ro"
],
[
"e ",
"th"
],
[
"e",
"t"
],
[
"a",
"d"
],
[
"i",
"r"
],
[
"c",
"on"
],
[
"u",
"n"
],
[
"t",
"ion"
],
[
"p",
"l"
],
[
"i",
"f"
],
[
"u",
"l"
],
[
"al",
"l"
],
[
"a",
"m"
],
[
"i",
"d"
],
[
"at",
" "
],
[
"u",
"m"
],
[
"e",
"x"
],
[
"i",
"st"
],
[
". ",
"Th"
],
[
"w",
"h"
],
[
"a",
"b"
],
[
":",
" "
],
[
"a",
"g"
],
[
"u",
"c"
],
[
"l",
"d "
],
[
"a",
"v"
],
[
"c",
"an "
],
[
"a",
"s "
],
[
"in",
"t"
],
[
" ",
" "
],
[
"'",
"s "
],
[
"w",
"it"
],
[
":",
"\n"
],
[
"r",
"es"
],
[
"f",
"or"
],
[
"an",
"d"
],
[
"ou",
"ld "
],
[
"s",
", "
],
[
"p",
"er"
],
[
"o",
"s"
],
[
"#",
" "
],
[
"a",
"p"
],
[
"c",
"om"
],
[
"en",
" "
],
[
"e",
"v"
],
[
"ig",
"h"
],
[
"an",
"t"
],
[
"##",
"# "
],
[
"y",
"ou"
],
[
"en",
"c"
],
[
"en",
"t "
],
[
"o",
"t"
],
[
"e ",
"t"
],
[
" th",
"e"
],
[
"f",
"or "
],
[
"o",
"c"
],
[
"e ",
"s"
],
[
"of",
" the "
],
[
"u",
"t"
],
[
"o",
"d"
],
[
"q",
"u"
],
[
"m",
"a"
],
[
"e",
"p"
],
[
"ation",
" "
],
[
"a",
"y"
],
[
"e",
", "
],
[
"ac",
"t"
],
[
"e th",
"e "
],
[
" th",
"at "
],
[
"n",
" "
],
[
"i",
"t "
],
[
"o",
"v"
],
[
"ic",
" "
],
[
"n",
"e"
],
[
"o",
"p"
],
[
"st",
"r"
],
[
"e ",
"of "
],
[
"b",
"e "
],
[
"an",
"c"
],
[
"s",
"h"
],
[
"ir",
" "
],
[
"ec",
"t"
],
[
"es",
"s"
],
[
"on",
" "
],
[
"e",
"f"
],
[
"wit",
"h "
],
[
"l",
"y "
],
[
". ",
"I"
],
[
"ou",
"n"
],
[
"at",
"e "
],
[
"in",
" the "
],
[
"=",
" "
],
[
"s",
"ist"
],
[
"l",
"e "
],
[
"c",
"l"
],
[
"k",
" "
],
[
"p",
"r"
],
[
"g",
"r"
],
[
"s ",
"and "
],
[
"i",
"z"
],
[
" t",
"o"
],
[
"m",
" "
],
[
". Th",
"is "
],
[
"ar",
"e "
],
[
"tion",
" "
],
[
"p",
"or"
],
[
"c",
"ould "
],
[
"t",
"r"
],
[
"ou",
"r"
],
[
"ing",
" the "
],
[
"es",
", "
],
[
"A",
"s"
],
[
"o",
"th"
],
[
"b",
"e"
],
[
",",
" the "
],
[
"y",
", "
],
[
"w",
"or"
],
[
"er",
"s"
],
[
"i",
"b"
],
[
"um",
"an"
],
[
"o",
"t "
],
[
"s",
"u"
],
[
"iv",
"e "
],
[
"i",
"al "
],
[
"it",
"y "
],
[
")",
" "
],
[
"m",
"ent"
],
[
" the",
"ir "
],
[
"*",
"*"
],
[
" t",
"r"
],
[
"m",
"or"
],
[
"As",
"sist"
],
[
"H",
"uman"
],
[
"en",
"s"
],
[
"Assist",
"ant"
],
[
"### ",
"Human"
],
[
"### ",
"Assistant"
],
[
"### Human",
":\n"
],
[
"### Assistant",
":\n"
],
[
"f",
"ro"
],
[
"in",
"d"
],
[
"s",
"p"
],
[
"o",
"g"
],
[
"c",
"re"
],
[
"h",
"av"
],
[
"u",
"d"
],
[
"s ",
"of "
],
[
"i",
"p"
],
[
"ar",
"t"
],
[
"com",
"p"
],
[
"0",
"0"
],
[
"an",
"g"
],
[
"igh",
"t "
],
[
"\n",
" "
],
[
"f",
"l"
],
[
"l",
" "
],
[
"en",
"d"
],
[
"a",
"in"
],
[
"wit",
"h"
],
[
"al",
"s"
],
[
"Th",
"e "
],
[
"d",
"uc"
],
[
"you",
" "
],
[
"n",
"ot "
],
[
"er",
"e "
],
[
"a ",
"s"
],
[
"all",
"y "
],
[
"ow",
" "
],
[
"h",
"e"
],
[
"it",
"ion"
],
[
"es",
"t"
],
[
"ic",
"al "
],
[
"en",
"g"
],
[
"er",
"s "
],
[
"es",
"e "
],
[
"w",
"e"
],
[
"d",
"is"
],
[
"l",
"i"
],
[
"e t",
"o "
],
[
"ar",
" "
],
[
"w",
" "
],
[
"b",
"y "
],
[
"s",
"e"
],
[
"i",
"es "
],
[
". Th",
"e "
],
[
"for",
"m"
],
[
"d",
"er"
],
[
", ",
"and "
],
[
"u",
"p"
],
[
"a",
"k"
],
[
"f",
"er"
],
[
"w",
"ould "
],
[
"re",
"c"
],
[
"ed",
" to "
],
[
"e ",
"and "
],
[
"you",
"r"
],
[
"ig",
"n"
],
[
"m",
"e"
],
[
"f",
"e"
],
[
"it",
"y"
],
[
"por",
"t"
],
[
"on",
"g"
],
[
"b",
"u"
],
[
"igh",
"t"
],
[
"p",
"h"
],
[
"s",
".\n\n"
],
[
"t",
"er"
],
[
"s",
"c"
],
[
"pro",
"v"
],
[
"your",
" "
],
[
"es",
"s "
],
[
"ic",
"h "
],
[
"as",
"s"
],
[
"0",
" "
],
[
"mor",
"e "
],
[
"ou",
"t"
],
[
"on",
" the "
],
[
"o",
"o"
],
[
" to",
" the "
],
[
"als",
"o "
],
[
"i",
"enc"
],
[
"hav",
"e "
],
[
"a",
"st"
],
[
"ou",
"t "
],
[
"a",
"f"
],
[
"fro",
"m "
],
[
"ov",
"er"
],
[
"ou",
"g"
],
[
" ",
"and "
],
[
"b",
"l"
],
[
"con",
"t"
],
[
". ",
"H"
],
[
"u",
"st"
],
[
"at",
"h"
],
[
".",
"\n"
],
[
"wh",
"ich "
],
[
"c",
"r"
],
[
"and",
" the "
],
[
" th",
"is "
],
[
"m",
"an"
],
[
"a",
"y "
],
[
"t",
"ur"
],
[
"s",
" to "
],
[
"ac",
"k"
],
[
"ev",
"er"
],
[
"h",
"is "
],
[
"p",
"os"
],
[
"\n\n",
"### Assistant:\n"
],
[
"p",
"re"
],
[
"li",
"k"
],
[
"p",
"ot"
],
[
". ",
"A"
],
[
"\n",
" "
],
[
"c",
"ar"
],
[
"f",
"r"
],
[
"h",
"el"
],
[
"int",
"er"
],
[
"-",
" "
],
[
" the",
"y "
],
[
"u",
"re"
],
[
"s",
"y"
],
[
"m",
"ight "
],
[
"pot",
"ent"
],
[
"d",
"e"
]
]
}
}