Upload 5 files
Browse files- processing_kosmos2.py +5 -5
- tokenization_kosmos2.py +2 -2
- tokenization_kosmos2_fast.py +3 -3
processing_kosmos2.py
CHANGED
|
@@ -21,11 +21,11 @@ from typing import List, Optional, Tuple, Union
|
|
| 21 |
|
| 22 |
import numpy as np
|
| 23 |
|
| 24 |
-
from .
|
| 25 |
-
from .
|
| 26 |
-
from .
|
| 27 |
-
from .
|
| 28 |
-
from .
|
| 29 |
|
| 30 |
|
| 31 |
if is_torch_available():
|
|
|
|
| 21 |
|
| 22 |
import numpy as np
|
| 23 |
|
| 24 |
+
from transformers.image_processing_utils import BatchFeature
|
| 25 |
+
from transformers.image_utils import ImageInput, is_batched
|
| 26 |
+
from transformers.processing_utils import ProcessorMixin
|
| 27 |
+
from transformers.tokenization_utils_base import PaddingStrategy, TextInput, TruncationStrategy
|
| 28 |
+
from transformers.utils import TensorType, is_tf_available, is_torch_available
|
| 29 |
|
| 30 |
|
| 31 |
if is_torch_available():
|
tokenization_kosmos2.py
CHANGED
|
@@ -21,8 +21,8 @@ from typing import Any, Dict, List, Optional, Tuple
|
|
| 21 |
|
| 22 |
import sentencepiece as spm
|
| 23 |
|
| 24 |
-
from .
|
| 25 |
-
from .
|
| 26 |
|
| 27 |
|
| 28 |
logger = logging.get_logger(__name__)
|
|
|
|
| 21 |
|
| 22 |
import sentencepiece as spm
|
| 23 |
|
| 24 |
+
from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
|
| 25 |
+
from transformers.utils import logging
|
| 26 |
|
| 27 |
|
| 28 |
logger = logging.get_logger(__name__)
|
tokenization_kosmos2_fast.py
CHANGED
|
@@ -19,9 +19,9 @@ import os
|
|
| 19 |
from shutil import copyfile
|
| 20 |
from typing import List, Optional, Tuple
|
| 21 |
|
| 22 |
-
from .
|
| 23 |
-
from .
|
| 24 |
-
from .
|
| 25 |
|
| 26 |
|
| 27 |
if is_sentencepiece_available():
|
|
|
|
| 19 |
from shutil import copyfile
|
| 20 |
from typing import List, Optional, Tuple
|
| 21 |
|
| 22 |
+
from transformers.tokenization_utils import AddedToken
|
| 23 |
+
from transformers.tokenization_utils_fast import PreTrainedTokenizerFast
|
| 24 |
+
from transformers.utils import is_sentencepiece_available, logging
|
| 25 |
|
| 26 |
|
| 27 |
if is_sentencepiece_available():
|