Remove unnecessary deprecated import
Browse files- tokenization_kimi.py +1 -5
tokenization_kimi.py
CHANGED
|
@@ -9,11 +9,7 @@ import tiktoken
|
|
| 9 |
from tiktoken.load import load_tiktoken_bpe
|
| 10 |
from tokenizers import AddedToken
|
| 11 |
|
| 12 |
-
|
| 13 |
-
from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode
|
| 14 |
-
except:
|
| 15 |
-
from transformers.convert_slow_tokenizer import bytes_to_unicode
|
| 16 |
-
|
| 17 |
from transformers.tokenization_utils import PreTrainedTokenizer
|
| 18 |
|
| 19 |
from .tool_declaration_ts import encode_tools_to_typescript_style
|
|
|
|
| 9 |
from tiktoken.load import load_tiktoken_bpe
|
| 10 |
from tokenizers import AddedToken
|
| 11 |
|
| 12 |
+
from transformers.convert_slow_tokenizer import bytes_to_unicode
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
from transformers.tokenization_utils import PreTrainedTokenizer
|
| 14 |
|
| 15 |
from .tool_declaration_ts import encode_tools_to_typescript_style
|