Update tokenization_kimi.py
Browse files- tokenization_kimi.py +6 -2
tokenization_kimi.py
CHANGED
|
@@ -15,8 +15,12 @@ from typing import (
|
|
| 15 |
from shutil import copyfile
|
| 16 |
from tiktoken.load import load_tiktoken_bpe
|
| 17 |
from tokenizers import AddedToken, pre_tokenizers, Regex
|
| 18 |
-
|
| 19 |
-
from transformers.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
from typing import Any
|
| 21 |
|
| 22 |
|
|
|
|
| 15 |
from shutil import copyfile
|
| 16 |
from tiktoken.load import load_tiktoken_bpe
|
| 17 |
from tokenizers import AddedToken, pre_tokenizers, Regex
|
| 18 |
+
try:
|
| 19 |
+
from transformers.tokenization_utils import PreTrainedTokenizer
|
| 20 |
+
from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode
|
| 21 |
+
except ModuleNotFoundError:
|
| 22 |
+
from transformers import PreTrainedTokenizer
|
| 23 |
+
from transformers.convert_slow_tokenizer import bytes_to_unicode
|
| 24 |
from typing import Any
|
| 25 |
|
| 26 |
|