Initial release: UnitronX Tokenizer 32k v1

meta.json

@@ -0,0 +1,25 @@
+{
+  "name": "UnitronX-Tokenizer-32k-v1",
+  "saved_at": "2025-11-06T21:44:44Z",
+  "vocab_size": 32000,
+  "special_tokens": [
+    "<unk>",
+    "<s>",
+    "</s>",
+    "<pad>",
+    "<mask>",
+    "<eot>",
+    "<URL>",
+    "<EMAIL>",
+    "<PATH>",
+    "<HEX>",
+    "<UUID>",
+    "<HANDLE>",
+    "<HASHTAG>",
+    "<NUM_2D>",
+    "<NUM_4D>",
+    "<NUM_N>"
+  ],
+  "byte_fallback": true,
+  "sha256": "b69c786fa3cee01dd1f60835cd1b5081efd197c6ea84214fc2ba0d605c229df0"
+}

special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "unk_token": "<unk>",
+  "pad_token": "<pad>",
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "additional_special_tokens": [
+    "<eot>",
+    "<URL>",
+    "<EMAIL>",
+    "<PATH>",
+    "<HEX>",
+    "<UUID>",
+    "<HANDLE>",
+    "<HASHTAG>",
+    "<NUM_2D>",
+    "<NUM_4D>",
+    "<NUM_N>"
+  ]
+}

tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+  "model_max_length": 4096,
+  "padding_side": "right",
+  "truncation_side": "right",
+  "add_prefix_space": true,
+  "name_or_path": "UnitronX-Tokenizer-32k-v1",
+  "bos_token": "<s>",
+  "eos_token": "</s>"
+}