Upload 2 files

tokenization_kimi.py

@@ -16,7 +16,7 @@ from shutil import copyfile
 from tiktoken.load import load_tiktoken_bpe
 from tokenizers import AddedToken, pre_tokenizers, Regex
 from transformers.tokenization_utils import PreTrainedTokenizer
-from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode
+from transformers.convert_slow_tokenizer import bytes_to_unicode
 from typing import Any
 
 

tokenizer_config.json

@@ -72,6 +72,46 @@
       "single_word": false,
       "special": true
     },
+    "163595": {
+      "content": "<|tool_calls_section_begin|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "163596": {
+      "content": "<|tool_calls_section_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "163597": {
+      "content": "<|tool_call_begin|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "163598": {
+      "content": "<|tool_call_argument_begin|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "163599": {
+      "content": "<|tool_call_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
     "163601": {
       "content": "<|im_middle|>",
       "lstrip": false,
@@ -107,12 +147,6 @@
     "<|im_system|>",
     "<|im_middle|>"
   ],
-  "auto_map": {
-    "AutoTokenizer": [
-      "tokenization_kimi.TikTokenTokenizer",
-      null
-    ]
-  },
   "bos_token": "[BOS]",
   "clean_up_tokenization_spaces": false,
   "eos_token": "[EOS]",
@@ -120,5 +154,11 @@
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "[PAD]",
   "tokenizer_class": "TikTokenTokenizer",
-  "unk_token": "[UNK]"
-}
+  "unk_token": "[UNK]",
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_kimi.TikTokenTokenizer",
+      null
+    ]
+  }
+}