updated upload of JiRackTokenizer with improvement for Robotics and Routing tags

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2dc31d9d029ae5874f26a5195d178e2295a59ed0f870e698d52cf0a8b4202e5
+size 11443288

tokenizer_config.json

@@ -0,0 +1,126 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|image|>",
+    "<|video|>",
+    "<|sound|>",
+    "<|voice|>",
+    "<|listening|>",
+    "<|vision|>",
+    "<|mood_happy|>",
+    "<|mood_sad|>",
+    "<|mood_angry|>",
+    "<|mood_neutral|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<|action_start|>",
+    "<|action_end|>",
+    "<|trajectory_start|>",
+    "<|trajectory_end|>",
+    "<|joint_start|>",
+    "<|joint_end|>",
+    "<|sensor_start|>",
+    "<|sensor_end|>",
+    "<|command_start|>",
+    "<|command_end|>",
+    "<|state_start|>",
+    "<|state_end|>",
+    "<|pose|>",
+    "<|velocity|>",
+    "<|force|>",
+    "<|torque|>",
+    "<|gripper|>",
+    "<|navigation|>",
+    "<|obstacle|>",
+    "<|task_start|>",
+    "<|task_end|>",
+    "<|plan_start|>",
+    "<|plan_end|>",
+    "<|behavior_start|>",
+    "<|behavior_end|>",
+    "<|skill_start|>",
+    "<|skill_end|>",
+    "<|motor|>",
+    "<|servo|>",
+    "<|imu|>",
+    "<|lidar|>",
+    "<|camera|>",
+    "<|depth|>",
+    "<|waypoint|>",
+    "<|path|>",
+    "<|collision|>",
+    "<|grasp|>",
+    "<|release|>",
+    "<|homing|>",
+    "<|emergency_stop|>",
+    "<|calibration|>",
+    "<|manipulation|>",
+    "<|locomotion|>",
+    "<|feedback|>",
+    "<|control_loop|>",
+    "__SCIENCE__",
+    "__CODING__",
+    "__STOCK_EXCHANGE__",
+    "__MEDICINE__",
+    "__GOVERNMENT__",
+    "__NEWS__",
+    "__GENERAL__",
+    "__MATERIAL_SCIENCE__",
+    "__ELECTRONICS__",
+    "__MICROELECTRONICS__",
+    "__ENGINEERING__",
+    "__ROBOTICS__",
+    "__ENERGY__",
+    "__AUTOMOTIVE__",
+    "__AVIATION__",
+    "__MATH__",
+    "__PYTHON__",
+    "__C__",
+    "__CPP__",
+    "__C_SHARP__",
+    "__JAVA__",
+    "__JAVASCRIPT__",
+    "__TYPESCRIPT__",
+    "__RUST__",
+    "__GO__",
+    "__RUBY__",
+    "__PHP__",
+    "__SWIFT__",
+    "__KOTLIN__",
+    "__BASH__",
+    "__SQL__",
+    "__ASSEMBLY__",
+    "__PHILOSOPHY__",
+    "__LITERATURE__",
+    "__SOCIOLOGY__",
+    "__PSYCHOLOGY__",
+    "__POLITICAL_SCIENCE__",
+    "__CULTURAL_STUDIES__",
+    "__ETHNOGRAPHY__",
+    "__HUMAN_RIGHTS__",
+    "__COMPLIANCE__",
+    "__MILITARY__",
+    "__BANKING__",
+    "__OIL_INDUSTRY__",
+    "__LIGHT_INDUSTRY__",
+    "__NATURE__",
+    "__OCEAN__",
+    "__SPORT__",
+    "__CULINARY__",
+    "__TRAVEL__",
+    "__HOBBY__"
+  ],
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}