Support sentencepiece 0.2.2 in text tokenizer
Browse files- text_encoder.py +0 -2
text_encoder.py
CHANGED
|
@@ -29,8 +29,6 @@ class Tokenizer(object):
|
|
| 29 |
|
| 30 |
def __init__(self, tokenizer_path: str):
|
| 31 |
self.sp = spm.SentencePieceProcessor(model_file=tokenizer_path)
|
| 32 |
-
# Match tensorflow_text.SentencepieceTokenizer(add_bos=False, add_eos=False)
|
| 33 |
-
self.sp.SetEncodeExtraOptions("")
|
| 34 |
# Explicitly disable BOS/EOS to match the reference Colab implementation.
|
| 35 |
self._add_bos = False
|
| 36 |
self._add_eos = False
|
|
|
|
| 29 |
|
| 30 |
def __init__(self, tokenizer_path: str):
|
| 31 |
self.sp = spm.SentencePieceProcessor(model_file=tokenizer_path)
|
|
|
|
|
|
|
| 32 |
# Explicitly disable BOS/EOS to match the reference Colab implementation.
|
| 33 |
self._add_bos = False
|
| 34 |
self._add_eos = False
|