Tiny-LLM-A / final_model_complete /tokenizer.json

Upload complete artifacts: final_model_complete

c0cecf6 verified 2 months ago

12.4 kB

	{
	"version": "1.0",
	"truncation": null,
	"padding": null,
	"added_tokens": [
	{
	"id": 0,
	"content": "<\|endoftext\|>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 1,
	"content": "<\|im_start\|>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 2,
	"content": "<\|im_end\|>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	}
	],
	"normalizer": null,
	"pre_tokenizer": {
	"type": "ByteLevel",
	"add_prefix_space": false,
	"trim_offsets": true,
	"use_regex": true
	},
	"post_processor": {
	"type": "ByteLevel",
	"add_prefix_space": true,
	"trim_offsets": false,
	"use_regex": true
	},
	"decoder": {
	"type": "ByteLevel",
	"add_prefix_space": true,
	"trim_offsets": true,
	"use_regex": true
	},
	"model": {
	"type": "BPE",
	"dropout": null,
	"unk_token": null,
	"continuing_subword_prefix": null,
	"end_of_word_suffix": null,
	"fuse_unk": false,
	"byte_fallback": false,
	"ignore_merges": false,
	"vocab": {
	"<\|endoftext\|>": 0,
	"<\|im_start\|>": 1,
	"<\|im_end\|>": 2,
	"!": 3,
	"\"": 4,
	"#": 5,
	"$": 6,
	"%": 7,
	"&": 8,
	"'": 9,
	"(": 10,
	")": 11,
	"*": 12,
	"+": 13,
	",": 14,
	"-": 15,
	".": 16,
	"/": 17,
	"0": 18,
	"1": 19,
	"2": 20,
	"3": 21,
	"4": 22,
	"5": 23,
	"6": 24,
	"7": 25,
	"8": 26,
	"9": 27,
	":": 28,
	";": 29,
	"<": 30,
	"=": 31,
	">": 32,
	"?": 33,
	"@": 34,
	"A": 35,
	"B": 36,
	"C": 37,
	"D": 38,
	"E": 39,
	"F": 40,
	"G": 41,
	"H": 42,
	"I": 43,
	"J": 44,
	"K": 45,
	"L": 46,
	"M": 47,
	"N": 48,
	"O": 49,
	"P": 50,
	"Q": 51,
	"R": 52,
	"S": 53,
	"T": 54,
	"U": 55,
	"V": 56,
	"W": 57,
	"X": 58,
	"Y": 59,
	"Z": 60,
	"[": 61,
	"\\": 62,
	"]": 63,
	"^": 64,
	"_": 65,
	"`": 66,
	"a": 67,
	"b": 68,
	"c": 69,
	"d": 70,
	"e": 71,
	"f": 72,
	"g": 73,
	"h": 74,
	"i": 75,
	"j": 76,
	"k": 77,
	"l": 78,
	"m": 79,
	"n": 80,
	"o": 81,
	"p": 82,
	"q": 83,
	"r": 84,
	"s": 85,
	"t": 86,
	"u": 87,
	"v": 88,
	"w": 89,
	"x": 90,
	"y": 91,
	"z": 92,
	"{": 93,
	"\|": 94,
	"}": 95,
	"~": 96,
	"Ċ": 97,
	"Ġ": 98,
	"Ġt": 99,
	"Ġa": 100,
	"in": 101,
	"on": 102,
	"he": 103,
	"ti": 104,
	"re": 105,
	"Ġthe": 106,
	"er": 107,
	"Ġs": 108,
	"Ġo": 109,
	"en": 110,
	"al": 111,
	"Ġc": 112,
	"or": 113,
	"at": 114,
	"tion": 115,
	"es": 116,
	"Ġof": 117,
	"is": 118,
	"Ġp": 119,
	"an": 120,
	"ed": 121,
	"Ġf": 122,
	"ar": 123,
	"it": 124,
	"ro": 125,
	"ing": 126,
	"Ġan": 127,
	"Ġin": 128,
	"Ġd": 129,
	"Ġw": 130,
	"Ġm": 131,
	"ic": 132,
	"le": 133,
	"Ġb": 134,
	"Ġand": 135,
	"ation": 136,
	"as": 137,
	"Ġe": 138,
	"el": 139,
	"Ġto": 140,
	"et": 141,
	"ent": 142,
	"Ġre": 143,
	"st": 144,
	"ct": 145,
	"Ġth": 146,
	"om": 147,
	"im": 148,
	"ul": 149,
	"ra": 150,
	"Ġn": 151,
	"ve": 152,
	"ly": 153,
	"od": 154,
	"ur": 155,
	"un": 156,
	"si": 157,
	"ce": 158,
	"Ġis": 159,
	"Ġl": 160,
	"Ġfor": 161,
	"us": 162,
	"ith": 163,
	"ĠT": 164,
	"ol": 165,
	"Ġg": 166,
	"Ġst": 167,
	"Ġpro": 168,
	"ch": 169,
	"ow": 170,
	"tic": 171,
	"Ġcon": 172,
	"qu": 173,
	"Ġh": 174,
	"per": 175,
	"Ġon": 176,
	"ig": 177,
	"am": 178,
	"res": 179,
	"Ġwith": 180,
	"Ġthat": 181,
	"ĠW": 182,
	"ver": 183,
	"um": 184,
	"Ġ$": 185,
	"il": 186,
	"Ġex": 187,
	"ut": 188,
	"se": 189,
	"ot": 190,
	"ate": 191,
	"ec": 192,
	"id": 193,
	"em": 194,
	"ity": 195,
	"Ġwe": 196,
	"pl": 197,
	"ĠWe": 198,
	"if": 199,
	"Ġ(": 200,
	"ab": 201,
	"ge": 202,
	"ĠThe": 203,
	"ri": 204,
	"ter": 205,
	"the": 206,
	"Ġal": 207,
	"Ġas": 208,
	"Ġbe": 209,
	"Ġare": 210,
	"ect": 211,
	"omp": 212,
	"ction": 213,
	"ations": 214,
	"ure": 215,
	"ĠI": 216,
	"Ġmod": 217,
	"Ġv": 218,
	"ir": 219,
	"os": 220,
	"Ġsu": 221,
	"Ġby": 222,
	"ts": 223,
	"Ġsp": 224,
	"pro": 225,
	"und": 226,
	"iz": 227,
	"ac": 228,
	"ical": 229,
	"for": 230,
	"ran": 231,
	"ant": 232,
	"Ġcomp": 233,
	"di": 234,
	"ain": 235,
	"Ġwh": 236,
	"ener": 237,
	"ren": 238,
	"rom": 239,
	"Ġmodel": 240,
	"Ġus": 241,
	"ĠA": 242,
	"Ġthis": 243,
	"ist": 244,
	"tive": 245,
	"Ġres": 246,
	"sion": 247,
	"ag": 248,
	"Ġde": 249,
	"du": 250,
	"Ġsh": 251,
	"ess": 252,
	"Ġat": 253,
	"ĠS": 254,
	"oc": 255
	},
	"merges": [
	[
	"Ġ",
	"t"
	],
	[
	"Ġ",
	"a"
	],
	[
	"i",
	"n"
	],
	[
	"o",
	"n"
	],
	[
	"h",
	"e"
	],
	[
	"t",
	"i"
	],
	[
	"r",
	"e"
	],
	[
	"Ġt",
	"he"
	],
	[
	"e",
	"r"
	],
	[
	"Ġ",
	"s"
	],
	[
	"Ġ",
	"o"
	],
	[
	"e",
	"n"
	],
	[
	"a",
	"l"
	],
	[
	"Ġ",
	"c"
	],
	[
	"o",
	"r"
	],
	[
	"a",
	"t"
	],
	[
	"ti",
	"on"
	],
	[
	"e",
	"s"
	],
	[
	"Ġo",
	"f"
	],
	[
	"i",
	"s"
	],
	[
	"Ġ",
	"p"
	],
	[
	"a",
	"n"
	],
	[
	"e",
	"d"
	],
	[
	"Ġ",
	"f"
	],
	[
	"a",
	"r"
	],
	[
	"i",
	"t"
	],
	[
	"r",
	"o"
	],
	[
	"in",
	"g"
	],
	[
	"Ġa",
	"n"
	],
	[
	"Ġ",
	"in"
	],
	[
	"Ġ",
	"d"
	],
	[
	"Ġ",
	"w"
	],
	[
	"Ġ",
	"m"
	],
	[
	"i",
	"c"
	],
	[
	"l",
	"e"
	],
	[
	"Ġ",
	"b"
	],
	[
	"Ġan",
	"d"
	],
	[
	"a",
	"tion"
	],
	[
	"a",
	"s"
	],
	[
	"Ġ",
	"e"
	],
	[
	"e",
	"l"
	],
	[
	"Ġt",
	"o"
	],
	[
	"e",
	"t"
	],
	[
	"en",
	"t"
	],
	[
	"Ġ",
	"re"
	],
	[
	"s",
	"t"
	],
	[
	"c",
	"t"
	],
	[
	"Ġt",
	"h"
	],
	[
	"o",
	"m"
	],
	[
	"i",
	"m"
	],
	[
	"u",
	"l"
	],
	[
	"r",
	"a"
	],
	[
	"Ġ",
	"n"
	],
	[
	"v",
	"e"
	],
	[
	"l",
	"y"
	],
	[
	"o",
	"d"
	],
	[
	"u",
	"r"
	],
	[
	"u",
	"n"
	],
	[
	"s",
	"i"
	],
	[
	"c",
	"e"
	],
	[
	"Ġ",
	"is"
	],
	[
	"Ġ",
	"l"
	],
	[
	"Ġf",
	"or"
	],
	[
	"u",
	"s"
	],
	[
	"it",
	"h"
	],
	[
	"Ġ",
	"T"
	],
	[
	"o",
	"l"
	],
	[
	"Ġ",
	"g"
	],
	[
	"Ġs",
	"t"
	],
	[
	"Ġp",
	"ro"
	],
	[
	"c",
	"h"
	],
	[
	"o",
	"w"
	],
	[
	"ti",
	"c"
	],
	[
	"Ġc",
	"on"
	],
	[
	"q",
	"u"
	],
	[
	"Ġ",
	"h"
	],
	[
	"p",
	"er"
	],
	[
	"Ġ",
	"on"
	],
	[
	"i",
	"g"
	],
	[
	"a",
	"m"
	],
	[
	"re",
	"s"
	],
	[
	"Ġw",
	"ith"
	],
	[
	"Ġth",
	"at"
	],
	[
	"Ġ",
	"W"
	],
	[
	"v",
	"er"
	],
	[
	"u",
	"m"
	],
	[
	"Ġ",
	"$"
	],
	[
	"i",
	"l"
	],
	[
	"Ġe",
	"x"
	],
	[
	"u",
	"t"
	],
	[
	"s",
	"e"
	],
	[
	"o",
	"t"
	],
	[
	"at",
	"e"
	],
	[
	"e",
	"c"
	],
	[
	"i",
	"d"
	],
	[
	"e",
	"m"
	],
	[
	"it",
	"y"
	],
	[
	"Ġw",
	"e"
	],
	[
	"p",
	"l"
	],
	[
	"ĠW",
	"e"
	],
	[
	"i",
	"f"
	],
	[
	"Ġ",
	"("
	],
	[
	"a",
	"b"
	],
	[
	"g",
	"e"
	],
	[
	"ĠT",
	"he"
	],
	[
	"r",
	"i"
	],
	[
	"t",
	"er"
	],
	[
	"t",
	"he"
	],
	[
	"Ġa",
	"l"
	],
	[
	"Ġa",
	"s"
	],
	[
	"Ġb",
	"e"
	],
	[
	"Ġa",
	"re"
	],
	[
	"e",
	"ct"
	],
	[
	"om",
	"p"
	],
	[
	"c",
	"tion"
	],
	[
	"ation",
	"s"
	],
	[
	"u",
	"re"
	],
	[
	"Ġ",
	"I"
	],
	[
	"Ġm",
	"od"
	],
	[
	"Ġ",
	"v"
	],
	[
	"i",
	"r"
	],
	[
	"o",
	"s"
	],
	[
	"Ġs",
	"u"
	],
	[
	"Ġb",
	"y"
	],
	[
	"t",
	"s"
	],
	[
	"Ġs",
	"p"
	],
	[
	"p",
	"ro"
	],
	[
	"un",
	"d"
	],
	[
	"i",
	"z"
	],
	[
	"a",
	"c"
	],
	[
	"ic",
	"al"
	],
	[
	"f",
	"or"
	],
	[
	"r",
	"an"
	],
	[
	"an",
	"t"
	],
	[
	"Ġc",
	"omp"
	],
	[
	"d",
	"i"
	],
	[
	"a",
	"in"
	],
	[
	"Ġw",
	"h"
	],
	[
	"en",
	"er"
	],
	[
	"re",
	"n"
	],
	[
	"ro",
	"m"
	],
	[
	"Ġmod",
	"el"
	],
	[
	"Ġ",
	"us"
	],
	[
	"Ġ",
	"A"
	],
	[
	"Ġth",
	"is"
	],
	[
	"is",
	"t"
	],
	[
	"ti",
	"ve"
	],
	[
	"Ġre",
	"s"
	],
	[
	"si",
	"on"
	],
	[
	"a",
	"g"
	],
	[
	"Ġd",
	"e"
	],
	[
	"d",
	"u"
	],
	[
	"Ġs",
	"h"
	],
	[
	"es",
	"s"
	],
	[
	"Ġa",
	"t"
	],
	[
	"Ġ",
	"S"
	],
	[
	"o",
	"c"
	]
	]
	}
	}