Add converted tokenizer (no trust_remote_code needed)

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 model.safetensors.index.json filter=lfs diff=lfs merge=lfs -text
 banner.png filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

chat_template.jinja

@@ -0,0 +1,37 @@
+{% if tools -%}
+    {{ '<|im_system|>tool_declare<|im_middle|>' -}}
+    {{- tools | tojson -}}
+    {{ '<|im_end|>' -}}
+{%- endif -%}
+
+{%- for message in messages -%}
+  {%- if loop.first and messages[0]['role'] != 'system' -%}
+    {{ '<|im_system|>system<|im_middle|>You are a helpful assistant<|im_end|>' }}
+  {%- endif -%}
+  {%- if message['role'] == 'system' -%}
+    {{ '<|im_system|>system<|im_middle|>' }}
+  {%- elif message['role'] == 'user' -%}
+    {{ '<|im_user|>user<|im_middle|>' }}
+  {%- elif message['role'] == 'assistant' -%}
+    {{ '<|im_assistant|>assistant<|im_middle|>' }}
+  {%- elif message['role'] == 'tool' -%}
+    {{ '<|im_system|>tool<|im_middle|>' }}
+  {%- endif -%}
+
+  {%- if message['content'] is string -%}
+    {{- message['content'] + '<|im_end|>' -}}
+  {%- else -%}
+    {%- for content in message['content'] -%}
+      {%- if content['type'] == 'image' or 'image' in content or 'image_url' in content -%}
+        {{ '<|media_start|>image<|media_content|><|media_pad|><|media_end|>' }}
+      {%- else -%}
+        {{ content['text'] }}
+      {%- endif -%}
+    {%- endfor -%}
+    {{ '<|im_end|>' }}
+  {%- endif -%}
+{%- endfor -%}
+
+{%- if add_generation_prompt -%}
+  {{ '<|im_assistant|>assistant<|im_middle|>' }}
+{%- endif -%}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6ec426a05b6cfb107053944445ca6ca642fef207704fe4d5112053b17803334
+size 19544371

tokenizer_config.json

@@ -1,165 +1,17 @@
 {
-  "added_tokens_decoder": {
-    "163584": {
-      "content": "[BOS]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "163585": {
-      "content": "[EOS]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "163586": {
-      "content": "<|im_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "163587": {
-      "content": "<|im_user|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "163588": {
-      "content": "<|im_assistant|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "163590": {
-      "content": "<|start_header_id|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "163591": {
-      "content": "<|end_header_id|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "163593": {
-      "content": "[EOT]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "163594": {
-      "content": "<|im_system|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "163595": {
-      "content": "<|tool_calls_section_begin|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "163596": {
-      "content": "<|tool_calls_section_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "163597": {
-      "content": "<|tool_call_begin|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "163598": {
-      "content": "<|tool_call_argument_begin|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "163599": {
-      "content": "<|tool_call_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "163601": {
-      "content": "<|im_middle|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "163838": {
-      "content": "[UNK]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "163839": {
-      "content": "[PAD]",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_kimi.TikTokenTokenizer",
+      null
+    ]
   },
-  "additional_special_tokens": [
-    "<|im_end|>",
-    "<|im_user|>",
-    "<|im_assistant|>",
-    "<|start_header_id|>",
-    "<|end_header_id|>",
-    "[EOT]",
-    "<|im_system|>",
-    "<|im_middle|>"
-  ],
+  "backend": "tokenizers",
   "bos_token": "[BOS]",
   "clean_up_tokenization_spaces": false,
   "eos_token": "[EOS]",
-  "extra_special_tokens": {},
-  "chat_template": "{% if tools -%}\n    {{ '<|im_system|>tool_declare<|im_middle|>' -}}\n    {{- tools | tojson -}}\n    {{ '<|im_end|>' -}}\n{%- endif -%}\n\n{%- for message in messages -%}\n  {%- if loop.first and messages[0]['role'] != 'system' -%}\n    {{ '<|im_system|>system<|im_middle|>You are a helpful assistant<|im_end|>' }}\n  {%- endif -%}\n  {%- if message['role'] == 'system' -%}\n    {{ '<|im_system|>system<|im_middle|>' }}\n  {%- elif message['role'] == 'user' -%}\n    {{ '<|im_user|>user<|im_middle|>' }}\n  {%- elif message['role'] == 'assistant' -%}\n    {{ '<|im_assistant|>assistant<|im_middle|>' }}\n  {%- elif message['role'] == 'tool' -%}\n    {{ '<|im_system|>tool<|im_middle|>' }}\n  {%- endif -%}\n\n  {%- if message['content'] is string -%}\n    {{- message['content'] + '<|im_end|>' -}}\n  {%- else -%}\n    {%- for content in message['content'] -%}\n      {%- if content['type'] == 'image' or 'image' in content or 'image_url' in content -%}\n        {{ '<|media_start|>image<|media_content|><|media_pad|><|media_end|>' }}\n      {%- else -%}\n        {{ content['text'] }}\n      {%- endif -%}\n    {%- endfor -%}\n    {{ '<|im_end|>' }}\n  {%- endif -%}\n{%- endfor -%}\n\n{%- if add_generation_prompt -%}\n  {{ '<|im_assistant|>assistant<|im_middle|>' }}\n{%- endif -%}",
+  "is_local": false,
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "[PAD]",
-  "tokenizer_class": "TikTokenTokenizer",
-  "unk_token": "[UNK]",
-  "auto_map": {
-    "AutoTokenizer": [
-      "tokenization_kimi.TikTokenTokenizer",
-      null
-    ]
-  }
-}
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "[UNK]"
+}