updated upload of JiRackTokenizer with improvement for Robotics and Routing tags

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdadf56c9b1ccc78da4b2c777797f9a7e1ba74a453622432956913997c5a0dcd
+size 32190336

tokenizer_config.json

@@ -0,0 +1,163 @@
+{
+  "audio_token": "<|audio|>",
+  "backend": "tokenizers",
+  "boa_token": "<|audio>",
+  "boi_token": "<|image>",
+  "bos_token": "<bos>",
+  "eoa_token": "<audio|>",
+  "eoc_token": "<channel|>",
+  "eoi_token": "<image|>",
+  "eos_token": "<eos>",
+  "eot_token": "<turn|>",
+  "escape_token": "<|\"|>",
+  "etc_token": "<tool_call|>",
+  "etd_token": "<tool|>",
+  "etr_token": "<tool_response|>",
+  "extra_special_tokens": [
+    "<|image|>",
+    "<|video|>",
+    "<|sound|>",
+    "<|voice|>",
+    "<|listening|>",
+    "<|vision|>",
+    "<|mood_happy|>",
+    "<|mood_sad|>",
+    "<|mood_angry|>",
+    "<|mood_neutral|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<|action_start|>",
+    "<|action_end|>",
+    "<|trajectory_start|>",
+    "<|trajectory_end|>",
+    "<|joint_start|>",
+    "<|joint_end|>",
+    "<|sensor_start|>",
+    "<|sensor_end|>",
+    "<|command_start|>",
+    "<|command_end|>",
+    "<|state_start|>",
+    "<|state_end|>",
+    "<|pose|>",
+    "<|velocity|>",
+    "<|force|>",
+    "<|torque|>",
+    "<|gripper|>",
+    "<|navigation|>",
+    "<|obstacle|>",
+    "<|task_start|>",
+    "<|task_end|>",
+    "<|plan_start|>",
+    "<|plan_end|>",
+    "<|behavior_start|>",
+    "<|behavior_end|>",
+    "<|skill_start|>",
+    "<|skill_end|>",
+    "<|motor|>",
+    "<|servo|>",
+    "<|imu|>",
+    "<|lidar|>",
+    "<|camera|>",
+    "<|depth|>",
+    "<|waypoint|>",
+    "<|path|>",
+    "<|collision|>",
+    "<|grasp|>",
+    "<|release|>",
+    "<|homing|>",
+    "<|emergency_stop|>",
+    "<|calibration|>",
+    "<|manipulation|>",
+    "<|locomotion|>",
+    "<|feedback|>",
+    "<|control_loop|>",
+    "__SCIENCE__",
+    "__CODING__",
+    "__STOCK_EXCHANGE__",
+    "__MEDICINE__",
+    "__GOVERNMENT__",
+    "__NEWS__",
+    "__GENERAL__",
+    "__MATERIAL_SCIENCE__",
+    "__ELECTRONICS__",
+    "__MICROELECTRONICS__",
+    "__ENGINEERING__",
+    "__ROBOTICS__",
+    "__ENERGY__",
+    "__AUTOMOTIVE__",
+    "__AVIATION__",
+    "__MATH__",
+    "__PYTHON__",
+    "__C__",
+    "__CPP__",
+    "__C_SHARP__",
+    "__JAVA__",
+    "__JAVASCRIPT__",
+    "__TYPESCRIPT__",
+    "__RUST__",
+    "__GO__",
+    "__RUBY__",
+    "__PHP__",
+    "__SWIFT__",
+    "__KOTLIN__",
+    "__BASH__",
+    "__SQL__",
+    "__ASSEMBLY__",
+    "__PHILOSOPHY__",
+    "__LITERATURE__",
+    "__SOCIOLOGY__",
+    "__PSYCHOLOGY__",
+    "__POLITICAL_SCIENCE__",
+    "__CULTURAL_STUDIES__",
+    "__ETHNOGRAPHY__",
+    "__HUMAN_RIGHTS__",
+    "__COMPLIANCE__",
+    "__MILITARY__",
+    "__BANKING__",
+    "__OIL_INDUSTRY__",
+    "__LIGHT_INDUSTRY__",
+    "__NATURE__",
+    "__OCEAN__",
+    "__SPORT__",
+    "__CULINARY__",
+    "__TRAVEL__",
+    "__HOBBY__"
+  ],
+  "image_token": "<|image|>",
+  "is_local": false,
+  "local_files_only": false,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "model_specific_special_tokens": {
+    "audio_token": "<|audio|>",
+    "boa_token": "<|audio>",
+    "boi_token": "<|image>",
+    "eoa_token": "<audio|>",
+    "eoc_token": "<channel|>",
+    "eoi_token": "<image|>",
+    "eot_token": "<turn|>",
+    "escape_token": "<|\"|>",
+    "etc_token": "<tool_call|>",
+    "etd_token": "<tool|>",
+    "etr_token": "<tool_response|>",
+    "image_token": "<|image|>",
+    "soc_token": "<|channel>",
+    "sot_token": "<|turn>",
+    "stc_token": "<|tool_call>",
+    "std_token": "<|tool>",
+    "str_token": "<|tool_response>",
+    "think_token": "<|think|>"
+  },
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "processor_class": "Gemma4Processor",
+  "soc_token": "<|channel>",
+  "sot_token": "<|turn>",
+  "stc_token": "<|tool_call>",
+  "std_token": "<|tool>",
+  "str_token": "<|tool_response>",
+  "think_token": "<|think|>",
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>"
+}