Hugging Face's logo

Finisha-F-scratch
/
Tiny-Rosa

Text Generation
Transformers
Safetensors
gpt2
art
conlangs
kiara
text-generation-inference
Model card Files
xet
Community
Tiny-Rosa / tokenizer.json
Clemylia's picture
Clemylia
Ajout du tokenizer associé au modèle final
aab0028 verified
raw
history blame contribute delete
11.9 kB
 {
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "[UNK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "[CLS]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "[SEP]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "[PAD]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 4,
"content": "[MASK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "Whitespace"
},
"post_processor": null,
"decoder": null,
"model": {
"type": "BPE",
"dropout": null,
"unk_token": "[UNK]",
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"[UNK]": 0,
"[CLS]": 1,
"[SEP]": 2,
"[PAD]": 3,
"[MASK]": 4,
"!": 5,
",": 6,
".": 7,
"A": 8,
"E": 9,
"F": 10,
"G": 11,
"H": 12,
"J": 13,
"K": 14,
"L": 15,
"M": 16,
"N": 17,
"O": 18,
"P": 19,
"R": 20,
"S": 21,
"T": 22,
"a": 23,
"d": 24,
"e": 25,
"f": 26,
"g": 27,
"h": 28,
"i": 29,
"j": 30,
"k": 31,
"l": 32,
"m": 33,
"n": 34,
"o": 35,
"p": 36,
"r": 37,
"s": 38,
"t": 39,
"u": 40,
"w": 41,
"x": 42,
"y": 43,
"z": 44,
"na": 45,
"ra": 46,
"as": 47,
"mo": 48,
"it": 49,
"ita": 50,
"ol": 51,
"ia": 52,
"ina": 53,
"fe": 54,
"op": 55,
"ki": 56,
"sa": 57,
"ola": 58,
"rita": 59,
"li": 60,
"az": 61,
"er": 62,
"lola": 63,
"itana": 64,
"opa": 65,
"nu": 66,
"nus": 67,
"titana": 68,
"popa": 69,
"mona": 70,
"fenus": 71,
"liia": 72,
"ni": 73,
"asna": 74,
"momo": 75,
"aze": 76,
"iara": 77,
"fera": 78,
"nina": 79,
"momorita": 80,
"aski": 81,
"ma": 82,
"we": 83,
"kiara": 84,
"ro": 85,
"wen": 86,
"kol": 87,
"ll": 88,
"llia": 89,
"era": 90,
"ga": 91,
"kira": 92,
"sallia": 93,
"ka": 94,
"olga": 95,
"ona": 96,
"wx": 97,
"yna": 98,
"erina": 99,
"wxa": 100,
"gwen": 101,
"ie": 102,
"rosa": 103,
"As": 104,
"hi": 105,
"hyna": 106,
"aska": 107,
"ashi": 108,
"mwxa": 109,
"uni": 110,
"gwenna": 111,
"asni": 112,
"erie": 113,
"kina": 114,
"eera": 115,
"opma": 116,
"kolina": 117,
"in": 118,
"jona": 119,
"lopma": 120,
"azin": 121,
"azinn": 122,
"una": 123,
"Lola": 124,
"jeera": 125,
"ja": 126,
"kolma": 127,
"Mo": 128,
"da": 129,
"de": 130,
"lisa": 131,
"delisa": 132,
"juna": 133,
"suni": 134,
"asnina": 135,
"Er": 136,
"se": 137,
"Fera": 138,
"lazinn": 139,
"Suni": 140,
"erra": 141,
"momodelisa": 142,
"Rita": 143,
"Aski": 144,
"Momo": 145,
"Kiara": 146,
"Gwen": 147,
"Kina": 148,
"Nina": 149,
"Ro": 150,
"rose": 151,
"Lazinn": 152,
"Erie": 153,
"Jona": 154,
"Sa": 155,
"Asni": 156,
"Juna": 157,
"Mwxa": 158,
"lli": 159,
"Aska": 160,
"Mora": 161,
"Rose": 162,
"Kol": 163,
"Li": 164,
"Mona": 165,
"Asna": 166,
"Erra": 167,
"Liia": 168,
"Fe": 169,
"Jeera": 170,
"Ki": 171,
"Popa": 172,
"mora": 173,
"Momodelisa": 174,
"Gwenna": 175,
"Rosa": 176,
"Sallia": 177,
"Salli": 178,
"Kolma": 179,
"Fenus": 180,
"Kira": 181,
"Az": 182,
"Hyna": 183,
"Titana": 184,
"Asnina": 185,
"Aze": 186,
"Ol": 187,
"le": 188,
"salli": 189,
"Ashi": 190,
"Erina": 191,
"Olga": 192,
"..": 193,
"Ja": 194,
"Je": 195,
"Lopma": 196,
"ar": 197,
"gwe": 198,
"par": 199,
"Kolina": 200,
"gwena": 201,
"parle": 202
},
"merges": [
[
"n",
"a"
],
[
"r",
"a"
],
[
"a",
"s"
],
[
"m",
"o"
],
[
"i",
"t"
],
[
"it",
"a"
],
[
"o",
"l"
],
[
"i",
"a"
],
[
"i",
"na"
],
[
"f",
"e"
],
[
"o",
"p"
],
[
"k",
"i"
],
[
"s",
"a"
],
[
"ol",
"a"
],
[
"r",
"ita"
],
[
"l",
"i"
],
[
"a",
"z"
],
[
"e",
"r"
],
[
"l",
"ola"
],
[
"ita",
"na"
],
[
"op",
"a"
],
[
"n",
"u"
],
[
"nu",
"s"
],
[
"t",
"itana"
],
[
"p",
"opa"
],
[
"mo",
"na"
],
[
"fe",
"nus"
],
[
"li",
"ia"
],
[
"n",
"i"
],
[
"as",
"na"
],
[
"mo",
"mo"
],
[
"az",
"e"
],
[
"ia",
"ra"
],
[
"fe",
"ra"
],
[
"n",
"ina"
],
[
"momo",
"rita"
],
[
"as",
"ki"
],
[
"m",
"a"
],
[
"w",
"e"
],
[
"k",
"iara"
],
[
"r",
"o"
],
[
"we",
"n"
],
[
"k",
"ol"
],
[
"l",
"l"
],
[
"ll",
"ia"
],
[
"e",
"ra"
],
[
"g",
"a"
],
[
"ki",
"ra"
],
[
"sa",
"llia"
],
[
"k",
"a"
],
[
"ol",
"ga"
],
[
"o",
"na"
],
[
"w",
"x"
],
[
"y",
"na"
],
[
"er",
"ina"
],
[
"wx",
"a"
],
[
"g",
"wen"
],
[
"i",
"e"
],
[
"ro",
"sa"
],
[
"A",
"s"
],
[
"h",
"i"
],
[
"h",
"yna"
],
[
"as",
"ka"
],
[
"as",
"hi"
],
[
"m",
"wxa"
],
[
"u",
"ni"
],
[
"gwen",
"na"
],
[
"as",
"ni"
],
[
"er",
"ie"
],
[
"k",
"ina"
],
[
"e",
"era"
],
[
"op",
"ma"
],
[
"kol",
"ina"
],
[
"i",
"n"
],
[
"j",
"ona"
],
[
"l",
"opma"
],
[
"az",
"in"
],
[
"azin",
"n"
],
[
"u",
"na"
],
[
"L",
"ola"
],
[
"j",
"eera"
],
[
"j",
"a"
],
[
"kol",
"ma"
],
[
"M",
"o"
],
[
"d",
"a"
],
[
"d",
"e"
],
[
"li",
"sa"
],
[
"de",
"lisa"
],
[
"j",
"una"
],
[
"s",
"uni"
],
[
"as",
"nina"
],
[
"E",
"r"
],
[
"s",
"e"
],
[
"F",
"era"
],
[
"l",
"azinn"
],
[
"S",
"uni"
],
[
"er",
"ra"
],
[
"momo",
"delisa"
],
[
"R",
"ita"
],
[
"As",
"ki"
],
[
"Mo",
"mo"
],
[
"K",
"iara"
],
[
"G",
"wen"
],
[
"K",
"ina"
],
[
"N",
"ina"
],
[
"R",
"o"
],
[
"ro",
"se"
],
[
"L",
"azinn"
],
[
"Er",
"ie"
],
[
"J",
"ona"
],
[
"S",
"a"
],
[
"As",
"ni"
],
[
"J",
"una"
],
[
"M",
"wxa"
],
[
"l",
"li"
],
[
"As",
"ka"
],
[
"Mo",
"ra"
],
[
"Ro",
"se"
],
[
"K",
"ol"
],
[
"L",
"i"
],
[
"M",
"ona"
],
[
"As",
"na"
],
[
"Er",
"ra"
],
[
"Li",
"ia"
],
[
"F",
"e"
],
[
"J",
"eera"
],
[
"K",
"i"
],
[
"P",
"opa"
],
[
"mo",
"ra"
],
[
"Momo",
"delisa"
],
[
"Gwen",
"na"
],
[
"Ro",
"sa"
],
[
"Sa",
"llia"
],
[
"Sa",
"lli"
],
[
"Kol",
"ma"
],
[
"Fe",
"nus"
],
[
"Ki",
"ra"
],
[
"A",
"z"
],
[
"H",
"yna"
],
[
"T",
"itana"
],
[
"As",
"nina"
],
[
"Az",
"e"
],
[
"O",
"l"
],
[
"l",
"e"
],
[
"sa",
"lli"
],
[
"As",
"hi"
],
[
"Er",
"ina"
],
[
"Ol",
"ga"
],
[
".",
"."
],
[
"J",
"a"
],
[
"J",
"e"
],
[
"L",
"opma"
],
[
"a",
"r"
],
[
"g",
"we"
],
[
"p",
"ar"
],
[
"Kol",
"ina"
],
[
"gwe",
"na"
],
[
"par",
"le"
]
]
}
}