ss
Browse files- tokenizeConfig.py +1 -3
tokenizeConfig.py
CHANGED
|
@@ -50,9 +50,7 @@ class OBITokenizer(PreTrainedTokenizer):
|
|
| 50 |
self.eos_token = "[SEP]"
|
| 51 |
self.pad_token = "[PAD]"
|
| 52 |
|
| 53 |
-
|
| 54 |
-
self.tokenizer.get_vocab().add_special_tokens([self.cls_token, self.sep_token, self.unk_token, self.mask_token])
|
| 55 |
-
|
| 56 |
def _pad(
|
| 57 |
self,
|
| 58 |
encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
|
|
|
|
| 50 |
self.eos_token = "[SEP]"
|
| 51 |
self.pad_token = "[PAD]"
|
| 52 |
|
| 53 |
+
|
|
|
|
|
|
|
| 54 |
def _pad(
|
| 55 |
self,
|
| 56 |
encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
|