Upload tokenizer.json with huggingface_hub

tokenizer.json

@@ -0,0 +1,476 @@
+{
+  "model_name": "OxideLLM_TK_SSM_V1",
+  "iteration": 1200,
+  "vocab_size": 228,
+  "char2idx": {
+    "<|pad|>": 0,
+    "<|user|>": 1,
+    "<|assistant|>": 2,
+    "<|end|>": 3,
+    "\n": 4,
+    " ": 5,
+    "!": 6,
+    "\"": 7,
+    "#": 8,
+    "$": 9,
+    "%": 10,
+    "&": 11,
+    "'": 12,
+    "(": 13,
+    ")": 14,
+    "*": 15,
+    "+": 16,
+    ",": 17,
+    "-": 18,
+    ".": 19,
+    "/": 20,
+    "0": 21,
+    "1": 22,
+    "2": 23,
+    "3": 24,
+    "4": 25,
+    "5": 26,
+    "6": 27,
+    "7": 28,
+    "8": 29,
+    "9": 30,
+    ":": 31,
+    ";": 32,
+    "<": 33,
+    "=": 34,
+    ">": 35,
+    "?": 36,
+    "A": 37,
+    "B": 38,
+    "C": 39,
+    "D": 40,
+    "E": 41,
+    "F": 42,
+    "G": 43,
+    "H": 44,
+    "I": 45,
+    "J": 46,
+    "K": 47,
+    "L": 48,
+    "M": 49,
+    "N": 50,
+    "O": 51,
+    "P": 52,
+    "Q": 53,
+    "R": 54,
+    "S": 55,
+    "T": 56,
+    "U": 57,
+    "V": 58,
+    "W": 59,
+    "X": 60,
+    "Y": 61,
+    "Z": 62,
+    "[": 63,
+    "]": 64,
+    "^": 65,
+    "_": 66,
+    "a": 67,
+    "b": 68,
+    "c": 69,
+    "d": 70,
+    "e": 71,
+    "f": 72,
+    "g": 73,
+    "h": 74,
+    "i": 75,
+    "j": 76,
+    "k": 77,
+    "l": 78,
+    "m": 79,
+    "n": 80,
+    "o": 81,
+    "p": 82,
+    "q": 83,
+    "r": 84,
+    "s": 85,
+    "t": 86,
+    "u": 87,
+    "v": 88,
+    "w": 89,
+    "x": 90,
+    "y": 91,
+    "z": 92,
+    "|": 93,
+    "}": 94,
+    "¡": 95,
+    "£": 96,
+    "§": 97,
+    "©": 98,
+    "ª": 99,
+    "«": 100,
+    "°": 101,
+    "´": 102,
+    "·": 103,
+    "º": 104,
+    "»": 105,
+    "½": 106,
+    "¾": 107,
+    "¿": 108,
+    "Á": 109,
+    "Â": 110,
+    "Æ": 111,
+    "Ç": 112,
+    "È": 113,
+    "É": 114,
+    "Ê": 115,
+    "Í": 116,
+    "Î": 117,
+    "Ñ": 118,
+    "Ó": 119,
+    "Ú": 120,
+    "Ü": 121,
+    "à": 122,
+    "á": 123,
+    "â": 124,
+    "ä": 125,
+    "æ": 126,
+    "ç": 127,
+    "è": 128,
+    "é": 129,
+    "ê": 130,
+    "ë": 131,
+    "ì": 132,
+    "í": 133,
+    "î": 134,
+    "ï": 135,
+    "ñ": 136,
+    "ò": 137,
+    "ó": 138,
+    "ô": 139,
+    "ö": 140,
+    "ù": 141,
+    "ú": 142,
+    "û": 143,
+    "ü": 144,
+    "ā": 145,
+    "ē": 146,
+    "ě": 147,
+    "ī": 148,
+    "ō": 149,
+    "Œ": 150,
+    "œ": 151,
+    "̃": 152,
+    "Δ": 153,
+    "Ο": 154,
+    "Π": 155,
+    "Τ": 156,
+    "Φ": 157,
+    "ά": 158,
+    "έ": 159,
+    "ί": 160,
+    "α": 161,
+    "β": 162,
+    "γ": 163,
+    "δ": 164,
+    "ε": 165,
+    "η": 166,
+    "θ": 167,
+    "ι": 168,
+    "κ": 169,
+    "λ": 170,
+    "μ": 171,
+    "ν": 172,
+    "ξ": 173,
+    "ο": 174,
+    "π": 175,
+    "ρ": 176,
+    "ς": 177,
+    "σ": 178,
+    "τ": 179,
+    "υ": 180,
+    "χ": 181,
+    "ω": 182,
+    "ό": 183,
+    "ύ": 184,
+    "ώ": 185,
+    "ἀ": 186,
+    "ἄ": 187,
+    "ἐ": 188,
+    "ἔ": 189,
+    "ἠ": 190,
+    "ἡ": 191,
+    "ἤ": 192,
+    "ἦ": 193,
+    "ἰ": 194,
+    "ἵ": 195,
+    "ἶ": 196,
+    "ὁ": 197,
+    "ὄ": 198,
+    "ὅ": 199,
+    "ὐ": 200,
+    "ὑ": 201,
+    "ὰ": 202,
+    "ὲ": 203,
+    "ὴ": 204,
+    "ὶ": 205,
+    "ὸ": 206,
+    "ὺ": 207,
+    "ᾶ": 208,
+    "᾽": 209,
+    "ῆ": 210,
+    "ῖ": 211,
+    "ῦ": 212,
+    "ῶ": 213,
+    " ": 214,
+    "–": 215,
+    "—": 216,
+    "‘": 217,
+    "’": 218,
+    "“": 219,
+    "”": 220,
+    "•": 221,
+    "′": 222,
+    "™": 223,
+    "⅓": 224,
+    "○": 225,
+    "●": 226,
+    "": 227
+  },
+  "idx2char": {
+    "0": "<|pad|>",
+    "1": "<|user|>",
+    "2": "<|assistant|>",
+    "3": "<|end|>",
+    "4": "\n",
+    "5": " ",
+    "6": "!",
+    "7": "\"",
+    "8": "#",
+    "9": "$",
+    "10": "%",
+    "11": "&",
+    "12": "'",
+    "13": "(",
+    "14": ")",
+    "15": "*",
+    "16": "+",
+    "17": ",",
+    "18": "-",
+    "19": ".",
+    "20": "/",
+    "21": "0",
+    "22": "1",
+    "23": "2",
+    "24": "3",
+    "25": "4",
+    "26": "5",
+    "27": "6",
+    "28": "7",
+    "29": "8",
+    "30": "9",
+    "31": ":",
+    "32": ";",
+    "33": "<",
+    "34": "=",
+    "35": ">",
+    "36": "?",
+    "37": "A",
+    "38": "B",
+    "39": "C",
+    "40": "D",
+    "41": "E",
+    "42": "F",
+    "43": "G",
+    "44": "H",
+    "45": "I",
+    "46": "J",
+    "47": "K",
+    "48": "L",
+    "49": "M",
+    "50": "N",
+    "51": "O",
+    "52": "P",
+    "53": "Q",
+    "54": "R",
+    "55": "S",
+    "56": "T",
+    "57": "U",
+    "58": "V",
+    "59": "W",
+    "60": "X",
+    "61": "Y",
+    "62": "Z",
+    "63": "[",
+    "64": "]",
+    "65": "^",
+    "66": "_",
+    "67": "a",
+    "68": "b",
+    "69": "c",
+    "70": "d",
+    "71": "e",
+    "72": "f",
+    "73": "g",
+    "74": "h",
+    "75": "i",
+    "76": "j",
+    "77": "k",
+    "78": "l",
+    "79": "m",
+    "80": "n",
+    "81": "o",
+    "82": "p",
+    "83": "q",
+    "84": "r",
+    "85": "s",
+    "86": "t",
+    "87": "u",
+    "88": "v",
+    "89": "w",
+    "90": "x",
+    "91": "y",
+    "92": "z",
+    "93": "|",
+    "94": "}",
+    "95": "¡",
+    "96": "£",
+    "97": "§",
+    "98": "©",
+    "99": "ª",
+    "100": "«",
+    "101": "°",
+    "102": "´",
+    "103": "·",
+    "104": "º",
+    "105": "»",
+    "106": "½",
+    "107": "¾",
+    "108": "¿",
+    "109": "Á",
+    "110": "Â",
+    "111": "Æ",
+    "112": "Ç",
+    "113": "È",
+    "114": "É",
+    "115": "Ê",
+    "116": "Í",
+    "117": "Î",
+    "118": "Ñ",
+    "119": "Ó",
+    "120": "Ú",
+    "121": "Ü",
+    "122": "à",
+    "123": "á",
+    "124": "â",
+    "125": "ä",
+    "126": "æ",
+    "127": "ç",
+    "128": "è",
+    "129": "é",
+    "130": "ê",
+    "131": "ë",
+    "132": "ì",
+    "133": "í",
+    "134": "î",
+    "135": "ï",
+    "136": "ñ",
+    "137": "ò",
+    "138": "ó",
+    "139": "ô",
+    "140": "ö",
+    "141": "ù",
+    "142": "ú",
+    "143": "û",
+    "144": "ü",
+    "145": "ā",
+    "146": "ē",
+    "147": "ě",
+    "148": "ī",
+    "149": "ō",
+    "150": "Œ",
+    "151": "œ",
+    "152": "̃",
+    "153": "Δ",
+    "154": "Ο",
+    "155": "Π",
+    "156": "Τ",
+    "157": "Φ",
+    "158": "ά",
+    "159": "έ",
+    "160": "ί",
+    "161": "α",
+    "162": "β",
+    "163": "γ",
+    "164": "δ",
+    "165": "ε",
+    "166": "η",
+    "167": "θ",
+    "168": "ι",
+    "169": "κ",
+    "170": "λ",
+    "171": "μ",
+    "172": "ν",
+    "173": "ξ",
+    "174": "ο",
+    "175": "π",
+    "176": "ρ",
+    "177": "ς",
+    "178": "σ",
+    "179": "τ",
+    "180": "υ",
+    "181": "χ",
+    "182": "ω",
+    "183": "ό",
+    "184": "ύ",
+    "185": "ώ",
+    "186": "ἀ",
+    "187": "ἄ",
+    "188": "ἐ",
+    "189": "ἔ",
+    "190": "ἠ",
+    "191": "ἡ",
+    "192": "ἤ",
+    "193": "ἦ",
+    "194": "ἰ",
+    "195": "ἵ",
+    "196": "ἶ",
+    "197": "ὁ",
+    "198": "ὄ",
+    "199": "ὅ",
+    "200": "ὐ",
+    "201": "ὑ",
+    "202": "ὰ",
+    "203": "ὲ",
+    "204": "ὴ",
+    "205": "ὶ",
+    "206": "ὸ",
+    "207": "ὺ",
+    "208": "ᾶ",
+    "209": "᾽",
+    "210": "ῆ",
+    "211": "ῖ",
+    "212": "ῦ",
+    "213": "ῶ",
+    "214": " ",
+    "215": "–",
+    "216": "—",
+    "217": "‘",
+    "218": "’",
+    "219": "“",
+    "220": "”",
+    "221": "•",
+    "222": "′",
+    "223": "™",
+    "224": "⅓",
+    "225": "○",
+    "226": "●",
+    "227": ""
+  },
+  "special_tokens": [
+    "<|pad|>",
+    "<|user|>",
+    "<|assistant|>",
+    "<|end|>"
+  ],
+  "config": {
+    "dim": 128,
+    "state_dim": 16,
+    "n_layers": 4
+  }
+}