Update tokenization_cpmbee.py
Browse files- tokenization_cpmbee.py +3 -3
tokenization_cpmbee.py
CHANGED
|
@@ -20,9 +20,9 @@ from typing import Any, Dict, List, Optional, Tuple, Union
|
|
| 20 |
import numpy as np
|
| 21 |
from typing_extensions import TypedDict
|
| 22 |
|
| 23 |
-
from
|
| 24 |
-
from
|
| 25 |
-
from
|
| 26 |
|
| 27 |
|
| 28 |
logger = logging.get_logger(__name__)
|
|
|
|
| 20 |
import numpy as np
|
| 21 |
from typing_extensions import TypedDict
|
| 22 |
|
| 23 |
+
from transformers.tokenization_utils import PaddingStrategy, PreTrainedTokenizer, TensorType
|
| 24 |
+
from transformers.tokenization_utils_base import AddedToken, BatchEncoding, TextInput, TruncationStrategy
|
| 25 |
+
from transformers.utils import logging
|
| 26 |
|
| 27 |
|
| 28 |
logger = logging.get_logger(__name__)
|