Upload tokenizer

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd13b042f774e9202de232415686d7285860998a84fdba40afa5c2bcaadbef4c
-size 10810609
+oid sha256:7a29c2a38c962b724abf8c63045663798107925b1d9d1ab814d2b16a23f4a8c9
+size 11418677

tokenizer_config.json

@@ -8,7 +8,7 @@
       "single_word": false,
       "special": true
     },
-    "3462": {
+    "3802": {
       "content": "user",
       "lstrip": false,
       "normalized": true,
@@ -16,7 +16,7 @@
       "single_word": false,
       "special": false
     },
-    "5319": {
+    "4535": {
       "content": "system",
       "lstrip": false,
       "normalized": true,
@@ -24,6 +24,14 @@
       "single_word": false,
       "special": false
     },
+    "116061": {
+      "content": "assistant",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
     "127900": {
       "content": "<think>",
       "lstrip": false,
@@ -57,14 +65,6 @@
       "special": false
     },
     "127904": {
-      "content": "assistant",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "127905": {
       "content": "BiBo",
       "lstrip": false,
       "normalized": true,
@@ -72,7 +72,7 @@
       "single_word": false,
       "special": false
     },
-    "127906": {
+    "127905": {
       "content": "aloobun",
       "lstrip": false,
       "normalized": true,
@@ -80,7 +80,7 @@
       "single_word": false,
       "special": false
     },
-    "127907": {
+    "127906": {
       "content": "LowIQGenAI",
       "lstrip": false,
       "normalized": true,
@@ -88,7 +88,7 @@
       "single_word": false,
       "special": false
     },
-    "127908": {
+    "127907": {
       "content": "IndieTechie",
       "lstrip": false,
       "normalized": true,
@@ -96,7 +96,7 @@
       "single_word": false,
       "special": false
     },
-    "127909": {
+    "127908": {
       "content": "Swamy",
       "lstrip": false,
       "normalized": true,
@@ -104,7 +104,7 @@
       "single_word": false,
       "special": false
     },
-    "127910": {
+    "127909": {
       "content": "AdarshXs",
       "lstrip": false,
       "normalized": true,
@@ -113,7 +113,6 @@
       "special": false
     }
   },
-  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "extra_special_tokens": {},