KIMI-K2

tokenization_kimi.py

@@ -16,7 +16,7 @@ from shutil import copyfile
 from tiktoken.load import load_tiktoken_bpe
 from tokenizers import AddedToken, pre_tokenizers, Regex
 from transformers.tokenization_utils import PreTrainedTokenizer
-from transformers.convert_slow_tokenizer import bytes_to_unicode
+from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode
 from typing import Any