Rugolo-200m / tokenizer.json
hidude562's picture
Upload 6 files
c6c6153 verified
{
"char_to_id": {
"-": 154,
"0": 155,
"1": 156,
"2": 157,
"3": 158,
"4": 159,
"5": 160,
"6": 161,
"7": 162,
"8": 163,
"9": 164,
"b": 165,
"c": 166,
"d": 167,
"f": 168,
"g": 169,
"h": 170,
"j": 171,
"k": 172,
"l": 173,
"m": 174,
"n": 175,
"q": 176,
"s": 177,
"t": 178,
"v": 179,
"w": 180,
"x": 181,
"y": 182,
"z": 183
},
"number_to_id": {
"0": 184,
"120": 185,
"42": 186,
"240": 187,
"60": 188,
"5": 189,
"10": 190,
"36": 191,
"62": 192,
"64": 193,
"69": 194,
"40": 195,
"1": 196,
"2": 197,
"54": 198,
"35": 199,
"55": 200,
"70": 201,
"57": 202,
"67": 203,
"59": 204,
"4": 205,
"38": 206,
"65": 207,
"50": 208,
"80": 209,
"8": 210,
"100": 211,
"20": 212,
"66": 213,
"72": 214,
"52": 215,
"46": 216,
"56": 217,
"30": 218,
"480": 219,
"63": 220,
"61": 221,
"44": 222,
"115": 223,
"58": 224,
"45": 225,
"235": 226,
"71": 227,
"74": 228,
"48": 229,
"68": 230,
"76": 231,
"51": 232,
"160": 233,
"6": 234,
"3": 235,
"53": 236,
"75": 237,
"15": 238,
"116": 239,
"43": 240,
"39": 241,
"82": 242,
"118": 243,
"37": 244,
"236": 245,
"33": 246,
"73": 247,
"12": 248,
"7": 249,
"110": 250,
"41": 251,
"47": 252,
"77": 253,
"28": 254,
"90": 255,
"25": 256,
"78": 257,
"79": 258,
"220": 259,
"31": 260,
"81": 261,
"49": 262,
"85": 263,
"32": 264,
"112": 265,
"96": 266,
"16": 267,
"95": 268,
"230": 269,
"192": 270,
"84": 271,
"105": 272,
"34": 273,
"18": 274,
"88": 275,
"200": 276,
"180": 277,
"108": 278,
"238": 279,
"140": 280,
"119": 281,
"232": 282,
"960": 283
},
"timestamp_to_id": {
"<|0.52|>": 4,
"<|1.56|>": 5,
"<|2.08|>": 6,
"<|0.92|>": 7,
"<|1.82|>": 8,
"<|1.70|>": 9,
"<|1.04|>": 10,
"<|2.02|>": 11,
"<|1.24|>": 12,
"<|0.40|>": 13,
"<|1.44|>": 14,
"<|2.04|>": 15,
"<|0.82|>": 16,
"<|0.14|>": 17,
"<|2.20|>": 18,
"<|2.48|>": 19,
"<|2.18|>": 20,
"<|0.98|>": 21,
"<|2.54|>": 22,
"<|2.22|>": 23,
"<|0.66|>": 24,
"<|0.26|>": 25,
"<|0.56|>": 26,
"<|1.92|>": 27,
"<|0.78|>": 28,
"<|1.12|>": 29,
"<|2.42|>": 30,
"<|1.30|>": 31,
"<|2.32|>": 32,
"<|0.74|>": 33,
"<|0.06|>": 34,
"<|1.96|>": 35,
"<|0.84|>": 36,
"<|0.94|>": 37,
"<|2.14|>": 38,
"<|1.54|>": 39,
"<|2.16|>": 40,
"<|0.58|>": 41,
"<|1.76|>": 42,
"<|1.50|>": 43,
"<|1.18|>": 44,
"<|1.90|>": 45,
"<|1.22|>": 46,
"<|2.34|>": 47,
"<|0.76|>": 48,
"<|1.26|>": 49,
"<|2.66|>": 50,
"<|2.44|>": 51,
"<|0.20|>": 52,
"<|1.62|>": 53,
"<|2.46|>": 54,
"<|1.80|>": 55,
"<|1.42|>": 56,
"<|1.46|>": 57,
"<|0.02|>": 58,
"<|1.64|>": 59,
"<|2.30|>": 60,
"<|1.72|>": 61,
"<|0.48|>": 62,
"<|0.46|>": 63,
"<|1.08|>": 64,
"<|1.14|>": 65,
"<|1.52|>": 66,
"<|1.00|>": 67,
"<|0.42|>": 68,
"<|2.40|>": 69,
"<|2.62|>": 70,
"<|1.40|>": 71,
"<|0.30|>": 72,
"<|0.22|>": 73,
"<|1.88|>": 74,
"<|1.84|>": 75,
"<|1.78|>": 76,
"<|2.36|>": 77,
"<|2.74|>": 78,
"<|0.24|>": 79,
"<|1.98|>": 80,
"<|2.64|>": 81,
"<|0.28|>": 82,
"<|1.02|>": 83,
"<|0.62|>": 84,
"<|0.08|>": 85,
"<|2.52|>": 86,
"<|0.96|>": 87,
"<|1.58|>": 88,
"<|1.28|>": 89,
"<|0.88|>": 90,
"<|0.70|>": 91,
"<|1.74|>": 92,
"<|0.04|>": 93,
"<|2.24|>": 94,
"<|1.10|>": 95,
"<|2.06|>": 96,
"<|0.32|>": 97,
"<|0.60|>": 98,
"<|1.36|>": 99,
"<|1.86|>": 100,
"<|2.58|>": 101,
"<|2.60|>": 102,
"<|1.68|>": 103,
"<|2.56|>": 104,
"<|2.26|>": 105,
"<|2.00|>": 106,
"<|0.54|>": 107,
"<|0.34|>": 108,
"<|0.18|>": 109,
"<|0.36|>": 110,
"<|0.44|>": 111,
"<|1.16|>": 112,
"<|1.20|>": 113,
"<|0.80|>": 114,
"<|2.10|>": 115,
"<|1.32|>": 116,
"<|1.48|>": 117,
"<|1.38|>": 118,
"<|2.28|>": 119,
"<|2.12|>": 120,
"<|2.68|>": 121,
"<|2.82|>": 122,
"<|0.16|>": 123,
"<|2.38|>": 124,
"<|1.34|>": 125,
"<|0.50|>": 126,
"<|0.72|>": 127,
"<|0.68|>": 128,
"<|2.78|>": 129,
"<|1.60|>": 130,
"<|2.76|>": 131,
"<|0.10|>": 132,
"<|1.66|>": 133,
"<|0.86|>": 134,
"<|1.06|>": 135,
"<|0.12|>": 136,
"<|0.38|>": 137,
"<|2.70|>": 138,
"<|0.00|>": 139,
"<|0.90|>": 140,
"<|1.94|>": 141,
"<|2.80|>": 142,
"<|0.64|>": 143,
"<|2.50|>": 144,
"<|2.72|>": 145,
"<|2.88|>": 146,
"<|2.84|>": 147,
"<|2.90|>": 148,
"<|2.92|>": 149,
"<|2.86|>": 150,
"<|2.96|>": 151,
"<|2.94|>": 152,
"<|2.98|>": 153
},
"special_tokens": {
" ": 0,
"<|startoftranscript|>": 1,
"<|endoftranscript|>": 2,
"<|unk|>": 3
},
"vocab_size": 284,
"config": {
"top_k_numbers": 100,
"pad_token": " ",
"bos_token": "<|startoftranscript|>",
"eos_token": "<|endoftranscript|>",
"unk_token": "<|unk|>"
}
}