Upload tokenizer_config.json with huggingface_hub

tokenizer_config.json

@@ -0,0 +1,60 @@
+{
+  "tokenizer_class": "AksaraTokenizer",
+  "model_type": "aksarallm",
+  "vocab_size": 32768,
+  "special_tokens": {
+    "[PAD]": 0,
+    "[EOS]": 1,
+    "[BOS]": 2,
+    "[UNK]": 3,
+    "[SEP]": 4,
+    "[MASK]": 5,
+    "[SYSTEM]": 6,
+    "[USER]": 7,
+    "[ASST]": 8,
+    "[INST]": 9,
+    "[/INST]": 10,
+    "[LANG_ID]": 11,
+    "[LANG_JV]": 12,
+    "[LANG_SU]": 13,
+    "[LANG_BL]": 14,
+    "[LANG_BT]": 15,
+    "[LANG_BG]": 16,
+    "[LANG_MN]": 17,
+    "[LANG_MD]": 18,
+    "[LANG_AC]": 19,
+    "[LANG_BJ]": 20,
+    "[LANG_EN]": 21,
+    "[TURN]": 22,
+    "[THINK]": 23,
+    "[/THINK]": 24,
+    "[CODE]": 25,
+    "[/CODE]": 26,
+    "[CITE]": 27,
+    "[NEWLINE]": 28
+  },
+  "pad_token": "[PAD]",
+  "eos_token": "[EOS]",
+  "bos_token": "[BOS]",
+  "unk_token": "[UNK]",
+  "sep_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "max_length": 512,
+  "pre_tokenizer": "ByteLevel",
+  "model": "BPE",
+  "version": "1.0.0",
+  "language": [
+    "id",
+    "jv",
+    "su",
+    "bal",
+    "btk",
+    "bug",
+    "min",
+    "mad",
+    "ace",
+    "bjn",
+    "en"
+  ],
+  "description": "Custom BPE tokenizer for AksaraLLM, optimized for Indonesian and local languages"
+}