Update tokenization_chatglm.py
Browse files- tokenization_chatglm.py +1 -1
tokenization_chatglm.py
CHANGED
|
@@ -4,7 +4,7 @@ from typing import List, Optional, Union, Dict
|
|
| 4 |
from sentencepiece import SentencePieceProcessor
|
| 5 |
from transformers import PreTrainedTokenizer
|
| 6 |
from transformers.utils import logging, PaddingStrategy
|
| 7 |
-
from transformers.tokenization_utils_base import EncodedInput, BatchEncoding
|
| 8 |
|
| 9 |
|
| 10 |
class SPTokenizer:
|
|
|
|
| 4 |
from sentencepiece import SentencePieceProcessor
|
| 5 |
from transformers import PreTrainedTokenizer
|
| 6 |
from transformers.utils import logging, PaddingStrategy
|
| 7 |
+
from transformers.tokenization_utils_base import EncodedInput, BatchEncoding, AddedToken
|
| 8 |
|
| 9 |
|
| 10 |
class SPTokenizer:
|