Hugging Face's logo

Mauricio-100
/
orb-v1.0

Text Generation
Transformers
Safetensors
gpt2
text-generation-inference
Model card Files
xet
Community
orb-v1.0 / tokenizer.json
Mauricio-100's picture
Mauricio-100
Upload tokenizer
45cf72e verified
raw
history blame contribute delete
14.6 kB
 {
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "</s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<pad>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "<unk>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 4,
"content": "<mask>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "Whitespace"
},
"post_processor": null,
"decoder": null,
"model": {
"type": "BPE",
"dropout": null,
"unk_token": "<unk>",
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"<s>": 0,
"</s>": 1,
"<pad>": 2,
"<unk>": 3,
"<mask>": 4,
"'": 5,
".": 6,
"D": 7,
"E": 8,
"F": 9,
"L": 10,
"P": 11,
"S": 12,
"T": 13,
"W": 14,
"a": 15,
"b": 16,
"c": 17,
"d": 18,
"e": 19,
"f": 20,
"g": 21,
"h": 22,
"i": 23,
"k": 24,
"l": 25,
"m": 26,
"n": 27,
"o": 28,
"p": 29,
"q": 30,
"r": 31,
"s": 32,
"t": 33,
"u": 34,
"v": 35,
"x": 36,
"y": 37,
"à": 38,
"è": 39,
"é": 40,
"es": 41,
"le": 42,
"la": 43,
"at": 44,
"in": 45,
"on": 46,
"re": 47,
"ou": 48,
"de": 49,
"he": 50,
"is": 51,
"ti": 52,
"our": 53,
"el": 54,
"Le": 55,
"an": 56,
"ap": 57,
"ar": 58,
"en": 59,
"il": 60,
"the": 61,
"est": 62,
"lan": 63,
"The": 64,
"ai": 65,
"al": 66,
"au": 67,
"ale": 68,
"ce": 69,
"ch": 70,
"ci": 71,
"cap": 72,
"des": 73,
"don": 74,
"er": 75,
"eu": 76,
"ge": 77,
"it": 78,
"li": 79,
"lu": 80,
"or": 81,
"ole": 82,
"pr": 83,
"pour": 84,
"sen": 85,
"tour": 86,
"un": 87,
"essen": 88,
"elle": 89,
"capit": 90,
"oleil": 91,
"essenti": 92,
"Deu": 93,
"En": 94,
"Fr": 95,
"La": 96,
"Les": 97,
"Lon": 98,
"Py": 99,
"Par": 100,
"Soleil": 101,
"Ter": 102,
"Wat": 103,
"am": 104,
"ach": 105,
"age": 106,
"br": 107,
"bes": 108,
"cat": 109,
"cti": 110,
"di": 111,
"du": 112,
"din": 113,
"dre": 114,
"dan": 115,
"eat": 116,
"eau": 117,
"fe": 118,
"fi": 119,
"fai": 120,
"for": 121,
"gr": 122,
"gen": 123,
"glan": 124,
"gale": 125,
"gage": 126,
"hin": 127,
"hon": 128,
"ie": 129,
"iè": 130,
"ion": 131,
"ire": 132,
"ky": 133,
"les": 134,
"lis": 135,
"mat": 136,
"mou": 137,
"man": 138,
"mach": 139,
"miè": 140,
"ne": 141,
"né": 142,
"nin": 143,
"ndre": 144,
"of": 145,
"op": 146,
"oin": 147,
"ogr": 148,
"pre": 149,
"plan": 150,
"plu": 151,
"pop": 152,
"qu": 153,
"rs": 154,
"se": 155,
"sour": 156,
"sun": 157,
"soleil": 158,
"shin": 159,
"sky": 160,
"tes": 161,
"tel": 162,
"thon": 163,
"ux": 164,
"ula": 165,
"uti": 166,
"vie": 167,
"édi": 168,
"égale": 169,
"lear": 170,
"atre": 171,
"ateu": 172,
"ine": 173,
"intel": 174,
"ons": 175,
"ont": 176,
"deux": 177,
"tifi": 178,
"ance": 179,
"appre": 180,
"artifi": 181,
"ille": 182,
"langage": 183,
"aide": 184,
"autour": 185,
"chat": 186,
"ciel": 187,
"cielle": 188,
"donné": 189,
"life": 190,
"ligen": 191,
"lumiè": 192,
"ordin": 193,
"progr": 194,
"prédi": 195,
"tourne": 196,
"capital": 197,
"capitale": 198,
"essential": 199,
"essentielle": 200,
"Deux": 201,
"Englan": 202,
"France": 203,
"London": 204,
"Python": 205,
"Paris": 206,
"Terre": 207,
"Water": 208,
"ammat": 209,
"brille": 210,
"besoin": 211,
"ctions": 212,
"dans": 213,
"eats": 214,
"faire": 215,
"lise": 216,
"mouse": 217,
"mange": 218,
"machine": 219,
"ning": 220,
"plantes": 221,
"plus": 222,
"popula": 223,
"quatre": 224,
"souris": 225,
"shines": 226,
"utilise": 227,
"learning": 228,
"ateurs": 229,
"intelligen": 230,
"apprendre": 231,
"artificielle": 232,
"données": 233,
"lumière": 234,
"ordinateurs": 235,
"programmat": 236,
"prédictions": 237,
"England": 238,
"populaire": 239,
"intelligence": 240,
"programmation": 241
},
"merges": [
[
"e",
"s"
],
[
"l",
"e"
],
[
"l",
"a"
],
[
"a",
"t"
],
[
"i",
"n"
],
[
"o",
"n"
],
[
"r",
"e"
],
[
"o",
"u"
],
[
"d",
"e"
],
[
"h",
"e"
],
[
"i",
"s"
],
[
"t",
"i"
],
[
"ou",
"r"
],
[
"e",
"l"
],
[
"L",
"e"
],
[
"a",
"n"
],
[
"a",
"p"
],
[
"a",
"r"
],
[
"e",
"n"
],
[
"i",
"l"
],
[
"t",
"he"
],
[
"es",
"t"
],
[
"la",
"n"
],
[
"T",
"he"
],
[
"a",
"i"
],
[
"a",
"l"
],
[
"a",
"u"
],
[
"a",
"le"
],
[
"c",
"e"
],
[
"c",
"h"
],
[
"c",
"i"
],
[
"c",
"ap"
],
[
"d",
"es"
],
[
"d",
"on"
],
[
"e",
"r"
],
[
"e",
"u"
],
[
"g",
"e"
],
[
"i",
"t"
],
[
"l",
"i"
],
[
"l",
"u"
],
[
"o",
"r"
],
[
"o",
"le"
],
[
"p",
"r"
],
[
"p",
"our"
],
[
"s",
"en"
],
[
"t",
"our"
],
[
"u",
"n"
],
[
"es",
"sen"
],
[
"el",
"le"
],
[
"cap",
"it"
],
[
"ole",
"il"
],
[
"essen",
"ti"
],
[
"D",
"eu"
],
[
"E",
"n"
],
[
"F",
"r"
],
[
"L",
"a"
],
[
"L",
"es"
],
[
"L",
"on"
],
[
"P",
"y"
],
[
"P",
"ar"
],
[
"S",
"oleil"
],
[
"T",
"er"
],
[
"W",
"at"
],
[
"a",
"m"
],
[
"a",
"ch"
],
[
"a",
"ge"
],
[
"b",
"r"
],
[
"b",
"es"
],
[
"c",
"at"
],
[
"c",
"ti"
],
[
"d",
"i"
],
[
"d",
"u"
],
[
"d",
"in"
],
[
"d",
"re"
],
[
"d",
"an"
],
[
"e",
"at"
],
[
"e",
"au"
],
[
"f",
"e"
],
[
"f",
"i"
],
[
"f",
"ai"
],
[
"f",
"or"
],
[
"g",
"r"
],
[
"g",
"en"
],
[
"g",
"lan"
],
[
"g",
"ale"
],
[
"g",
"age"
],
[
"h",
"in"
],
[
"h",
"on"
],
[
"i",
"e"
],
[
"i",
"è"
],
[
"i",
"on"
],
[
"i",
"re"
],
[
"k",
"y"
],
[
"l",
"es"
],
[
"l",
"is"
],
[
"m",
"at"
],
[
"m",
"ou"
],
[
"m",
"an"
],
[
"m",
"ach"
],
[
"m",
"iè"
],
[
"n",
"e"
],
[
"n",
"é"
],
[
"n",
"in"
],
[
"n",
"dre"
],
[
"o",
"f"
],
[
"o",
"p"
],
[
"o",
"in"
],
[
"o",
"gr"
],
[
"p",
"re"
],
[
"p",
"lan"
],
[
"p",
"lu"
],
[
"p",
"op"
],
[
"q",
"u"
],
[
"r",
"s"
],
[
"s",
"e"
],
[
"s",
"our"
],
[
"s",
"un"
],
[
"s",
"oleil"
],
[
"s",
"hin"
],
[
"s",
"ky"
],
[
"t",
"es"
],
[
"t",
"el"
],
[
"t",
"hon"
],
[
"u",
"x"
],
[
"u",
"la"
],
[
"u",
"ti"
],
[
"v",
"ie"
],
[
"é",
"di"
],
[
"é",
"gale"
],
[
"le",
"ar"
],
[
"at",
"re"
],
[
"at",
"eu"
],
[
"in",
"e"
],
[
"in",
"tel"
],
[
"on",
"s"
],
[
"on",
"t"
],
[
"de",
"ux"
],
[
"ti",
"fi"
],
[
"an",
"ce"
],
[
"ap",
"pre"
],
[
"ar",
"tifi"
],
[
"il",
"le"
],
[
"lan",
"gage"
],
[
"ai",
"de"
],
[
"au",
"tour"
],
[
"ch",
"at"
],
[
"ci",
"el"
],
[
"ci",
"elle"
],
[
"don",
"né"
],
[
"li",
"fe"
],
[
"li",
"gen"
],
[
"lu",
"miè"
],
[
"or",
"din"
],
[
"pr",
"ogr"
],
[
"pr",
"édi"
],
[
"tour",
"ne"
],
[
"capit",
"al"
],
[
"capit",
"ale"
],
[
"essenti",
"al"
],
[
"essenti",
"elle"
],
[
"Deu",
"x"
],
[
"En",
"glan"
],
[
"Fr",
"ance"
],
[
"Lon",
"don"
],
[
"Py",
"thon"
],
[
"Par",
"is"
],
[
"Ter",
"re"
],
[
"Wat",
"er"
],
[
"am",
"mat"
],
[
"br",
"ille"
],
[
"bes",
"oin"
],
[
"cti",
"ons"
],
[
"dan",
"s"
],
[
"eat",
"s"
],
[
"fai",
"re"
],
[
"lis",
"e"
],
[
"mou",
"se"
],
[
"man",
"ge"
],
[
"mach",
"ine"
],
[
"nin",
"g"
],
[
"plan",
"tes"
],
[
"plu",
"s"
],
[
"pop",
"ula"
],
[
"qu",
"atre"
],
[
"sour",
"is"
],
[
"shin",
"es"
],
[
"uti",
"lise"
],
[
"lear",
"ning"
],
[
"ateu",
"rs"
],
[
"intel",
"ligen"
],
[
"appre",
"ndre"
],
[
"artifi",
"cielle"
],
[
"donné",
"es"
],
[
"lumiè",
"re"
],
[
"ordin",
"ateurs"
],
[
"progr",
"ammat"
],
[
"prédi",
"ctions"
],
[
"Englan",
"d"
],
[
"popula",
"ire"
],
[
"intelligen",
"ce"
],
[
"programmat",
"ion"
]
]
}
}