Upload tokenization_splinter.py
Browse files- tokenization_splinter.py +3 -0
tokenization_splinter.py
CHANGED
|
@@ -214,3 +214,6 @@ class SplinterBertTokenizerFast(BertTokenizerFast):
|
|
| 214 |
PreTokenizer.custom(SplinterPreTokenizer(self.splinter))
|
| 215 |
])
|
| 216 |
self._tokenizer.decoder = Decoder.custom(SplinterDecoder(self.splinter))
|
|
|
|
|
|
|
|
|
|
|
|
| 214 |
PreTokenizer.custom(SplinterPreTokenizer(self.splinter))
|
| 215 |
])
|
| 216 |
self._tokenizer.decoder = Decoder.custom(SplinterDecoder(self.splinter))
|
| 217 |
+
|
| 218 |
+
def save(self, *args, **kwargs):
|
| 219 |
+
print('Cannot save SplinterBertTokenizerFast, please copy the files directly from the repository')
|