XS
Browse files- tokenizeConfig.py +0 -1
tokenizeConfig.py
CHANGED
|
@@ -8,7 +8,6 @@ from transformers.utils import PaddingStrategy
|
|
| 8 |
class OBITokenizer(PreTrainedTokenizer):
|
| 9 |
def __init__(
|
| 10 |
self,
|
| 11 |
-
vocab_file,
|
| 12 |
unk_token="<unk>",
|
| 13 |
bos_token="<s>",
|
| 14 |
eos_token="</s>",
|
|
|
|
| 8 |
class OBITokenizer(PreTrainedTokenizer):
|
| 9 |
def __init__(
|
| 10 |
self,
|
|
|
|
| 11 |
unk_token="<unk>",
|
| 12 |
bos_token="<s>",
|
| 13 |
eos_token="</s>",
|