noamw-aai commited on
Commit
f7f8a77
·
verified ·
1 Parent(s): 6126819

Update tokenization_kimi.py

Browse files
Files changed (1) hide show
  1. tokenization_kimi.py +6 -2
tokenization_kimi.py CHANGED
@@ -15,8 +15,12 @@ from typing import (
15
  from shutil import copyfile
16
  from tiktoken.load import load_tiktoken_bpe
17
  from tokenizers import AddedToken, pre_tokenizers, Regex
18
- from transformers.tokenization_utils import PreTrainedTokenizer
19
- from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode
 
 
 
 
20
  from typing import Any
21
 
22
 
 
15
  from shutil import copyfile
16
  from tiktoken.load import load_tiktoken_bpe
17
  from tokenizers import AddedToken, pre_tokenizers, Regex
18
+ try:
19
+ from transformers.tokenization_utils import PreTrainedTokenizer
20
+ from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode
21
+ except ModuleNotFoundError:
22
+ from transformers import PreTrainedTokenizer
23
+ from transformers.convert_slow_tokenizer import bytes_to_unicode
24
  from typing import Any
25
 
26