Upload folder using huggingface_hub

dad87ca verified 11 days ago

9.42 kB

	{
	"version": "1.0",
	"truncation": null,
	"padding": null,
	"added_tokens": [
	{
	"id": 0,
	"content": "<\|endoftext\|>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 1,
	"content": "<\|unk\|>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	}
	],
	"normalizer": null,
	"pre_tokenizer": {
	"type": "ByteLevel",
	"add_prefix_space": false,
	"trim_offsets": true,
	"use_regex": true
	},
	"post_processor": null,
	"decoder": {
	"type": "ByteLevel",
	"add_prefix_space": true,
	"trim_offsets": true,
	"use_regex": true
	},
	"model": {
	"type": "BPE",
	"dropout": null,
	"unk_token": "<\|unk\|>",
	"continuing_subword_prefix": null,
	"end_of_word_suffix": null,
	"fuse_unk": false,
	"byte_fallback": false,
	"ignore_merges": false,
	"vocab": {
	"<\|endoftext\|>": 0,
	"<\|unk\|>": 1,
	"!": 2,
	".": 3,
	"H": 4,
	"I": 5,
	"O": 6,
	"T": 7,
	"a": 8,
	"b": 9,
	"c": 10,
	"d": 11,
	"e": 12,
	"f": 13,
	"g": 14,
	"h": 15,
	"i": 16,
	"j": 17,
	"k": 18,
	"l": 19,
	"m": 20,
	"n": 21,
	"o": 22,
	"p": 23,
	"q": 24,
	"r": 25,
	"s": 26,
	"t": 27,
	"u": 28,
	"v": 29,
	"w": 30,
	"x": 31,
	"y": 32,
	"z": 33,
	"Ċ": 34,
	"Ġ": 35,
	"Ġt": 36,
	"Ġa": 37,
	"el": 38,
	"er": 39,
	"Th": 40,
	"del": 41,
	"es": 42,
	"fo": 43,
	"in": 44,
	"is": 45,
	"la": 46,
	"mo": 47,
	"Ġla": 48,
	"Ġmo": 49,
	"Ġtes": 50,
	"Ġmodel": 51,
	"Ġtest": 52,
	"al": 53,
	"as": 54,
	"ed": 55,
	"on": 56,
	"ra": 57,
	"wo": 58,
	"Ġb": 59,
	"Ġo": 60,
	"Ġu": 61,
	"Ġfo": 62,
	"Ġis": 63,
	"Ġth": 64,
	"Ġtin": 65,
	"The": 66,
	"Ġtiny": 67,
	"Hel": 68,
	"In": 69,
	"On": 70,
	"ag": 71,
	"ce": 72,
	"ck": 73,
	"do": 74,
	"eg": 75,
	"eed": 76,
	"gu": 77,
	"has": 78,
	"id": 79,
	"im": 80,
	"ion": 81,
	"ick": 82,
	"ju": 83,
	"ld": 84,
	"lo": 85,
	"mp": 86,
	"mer": 87,
	"mal": 88,
	"nd": 89,
	"ns": 90,
	"need": 91,
	"ngu": 92,
	"ow": 93,
	"pon": 94,
	"qu": 95,
	"rld": 96,
	"rmer": 97,
	"row": 98,
	"sed": 99,
	"smal": 100,
	"teg": 101,
	"tion": 102,
	"ut": 103,
	"ver": 104,
	"val": 105,
	"was": 106,
	"yer": 107,
	"zy": 108,
	"ĠTh": 109,
	"Ġwo": 110,
	"Ġdo": 111,
	"Ġhas": 112,
	"Ġju": 113,
	"Ġneed": 114,
	"Ġqu": 115,
	"Ġsmal": 116,
	"Ġval": 117,
	"Ġwas": 118,
	"Ġtra": 119,
	"Ġtwo": 120,
	"Ġtim": 121,
	"Ġand": 122,
	"ere": 123,
	"former": 124,
	"ing": 125,
	"Ġlangu": 126,
	"Ġlayer": 127,
	"Ġlazy": 128,
	"Ġtests": 129,
	"Ġtesting": 130,
	"ration": 131,
	"Ġbrow": 132,
	"Ġbut": 133,
	"Ġof": 134,
	"Ġover": 135,
	"Ġupon": 136,
	"Ġused": 137,
	"Ġfor": 138,
	"Ġfox": 139,
	"Ġthe": 140,
	"Ġthere": 141,
	"Hello": 142,
	"Integ": 143,
	"Once": 144,
	"age": 145,
	"mps": 146,
	"nsformer": 147,
	"ĠThis": 148,
	"Ġworld": 149,
	"Ġdog": 150,
	"Ġjumps": 151,
	"Ġquick": 152,
	"Ġsmall": 153,
	"Ġvalid": 154,
	"Ġtransformer": 155,
	"Ġtime": 156,
	"Ġlanguage": 157,
	"Ġlayers": 158,
	"Ġbrown": 159,
	"Integration": 160
	},
	"merges": [
	[
	"Ġ",
	"t"
	],
	[
	"Ġ",
	"a"
	],
	[
	"e",
	"l"
	],
	[
	"e",
	"r"
	],
	[
	"T",
	"h"
	],
	[
	"d",
	"el"
	],
	[
	"e",
	"s"
	],
	[
	"f",
	"o"
	],
	[
	"i",
	"n"
	],
	[
	"i",
	"s"
	],
	[
	"l",
	"a"
	],
	[
	"m",
	"o"
	],
	[
	"Ġ",
	"la"
	],
	[
	"Ġ",
	"mo"
	],
	[
	"Ġt",
	"es"
	],
	[
	"Ġmo",
	"del"
	],
	[
	"Ġtes",
	"t"
	],
	[
	"a",
	"l"
	],
	[
	"a",
	"s"
	],
	[
	"e",
	"d"
	],
	[
	"o",
	"n"
	],
	[
	"r",
	"a"
	],
	[
	"w",
	"o"
	],
	[
	"Ġ",
	"b"
	],
	[
	"Ġ",
	"o"
	],
	[
	"Ġ",
	"u"
	],
	[
	"Ġ",
	"fo"
	],
	[
	"Ġ",
	"is"
	],
	[
	"Ġt",
	"h"
	],
	[
	"Ġt",
	"in"
	],
	[
	"Th",
	"e"
	],
	[
	"Ġtin",
	"y"
	],
	[
	"H",
	"el"
	],
	[
	"I",
	"n"
	],
	[
	"O",
	"n"
	],
	[
	"a",
	"g"
	],
	[
	"c",
	"e"
	],
	[
	"c",
	"k"
	],
	[
	"d",
	"o"
	],
	[
	"e",
	"g"
	],
	[
	"e",
	"ed"
	],
	[
	"g",
	"u"
	],
	[
	"h",
	"as"
	],
	[
	"i",
	"d"
	],
	[
	"i",
	"m"
	],
	[
	"i",
	"on"
	],
	[
	"i",
	"ck"
	],
	[
	"j",
	"u"
	],
	[
	"l",
	"d"
	],
	[
	"l",
	"o"
	],
	[
	"m",
	"p"
	],
	[
	"m",
	"er"
	],
	[
	"m",
	"al"
	],
	[
	"n",
	"d"
	],
	[
	"n",
	"s"
	],
	[
	"n",
	"eed"
	],
	[
	"n",
	"gu"
	],
	[
	"o",
	"w"
	],
	[
	"p",
	"on"
	],
	[
	"q",
	"u"
	],
	[
	"r",
	"ld"
	],
	[
	"r",
	"mer"
	],
	[
	"r",
	"ow"
	],
	[
	"s",
	"ed"
	],
	[
	"s",
	"mal"
	],
	[
	"t",
	"eg"
	],
	[
	"t",
	"ion"
	],
	[
	"u",
	"t"
	],
	[
	"v",
	"er"
	],
	[
	"v",
	"al"
	],
	[
	"w",
	"as"
	],
	[
	"y",
	"er"
	],
	[
	"z",
	"y"
	],
	[
	"Ġ",
	"Th"
	],
	[
	"Ġ",
	"wo"
	],
	[
	"Ġ",
	"do"
	],
	[
	"Ġ",
	"has"
	],
	[
	"Ġ",
	"ju"
	],
	[
	"Ġ",
	"need"
	],
	[
	"Ġ",
	"qu"
	],
	[
	"Ġ",
	"smal"
	],
	[
	"Ġ",
	"val"
	],
	[
	"Ġ",
	"was"
	],
	[
	"Ġt",
	"ra"
	],
	[
	"Ġt",
	"wo"
	],
	[
	"Ġt",
	"im"
	],
	[
	"Ġa",
	"nd"
	],
	[
	"er",
	"e"
	],
	[
	"fo",
	"rmer"
	],
	[
	"in",
	"g"
	],
	[
	"Ġla",
	"ngu"
	],
	[
	"Ġla",
	"yer"
	],
	[
	"Ġla",
	"zy"
	],
	[
	"Ġtest",
	"s"
	],
	[
	"Ġtest",
	"ing"
	],
	[
	"ra",
	"tion"
	],
	[
	"Ġb",
	"row"
	],
	[
	"Ġb",
	"ut"
	],
	[
	"Ġo",
	"f"
	],
	[
	"Ġo",
	"ver"
	],
	[
	"Ġu",
	"pon"
	],
	[
	"Ġu",
	"sed"
	],
	[
	"Ġfo",
	"r"
	],
	[
	"Ġfo",
	"x"
	],
	[
	"Ġth",
	"e"
	],
	[
	"Ġth",
	"ere"
	],
	[
	"Hel",
	"lo"
	],
	[
	"In",
	"teg"
	],
	[
	"On",
	"ce"
	],
	[
	"ag",
	"e"
	],
	[
	"mp",
	"s"
	],
	[
	"ns",
	"former"
	],
	[
	"ĠTh",
	"is"
	],
	[
	"Ġwo",
	"rld"
	],
	[
	"Ġdo",
	"g"
	],
	[
	"Ġju",
	"mps"
	],
	[
	"Ġqu",
	"ick"
	],
	[
	"Ġsmal",
	"l"
	],
	[
	"Ġval",
	"id"
	],
	[
	"Ġtra",
	"nsformer"
	],
	[
	"Ġtim",
	"e"
	],
	[
	"Ġlangu",
	"age"
	],
	[
	"Ġlayer",
	"s"
	],
	[
	"Ġbrow",
	"n"
	],
	[
	"Integ",
	"ration"
	]
	]
	}
	}