Update tokenization_chatglm.py
Browse files- tokenization_chatglm.py +3 -3
tokenization_chatglm.py
CHANGED
|
@@ -70,9 +70,9 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
|
|
| 70 |
|
| 71 |
self.vocab_file = vocab_file
|
| 72 |
self.tokenizer = SPTokenizer(vocab_file)
|
| 73 |
-
kwargs.pop("eos_token")
|
| 74 |
-
kwargs.pop("pad_token")
|
| 75 |
-
kwargs.pop("unk_token")
|
| 76 |
self.special_tokens = {
|
| 77 |
"<bos>": self.tokenizer.bos_id,
|
| 78 |
"<eos>": self.tokenizer.eos_id,
|
|
|
|
| 70 |
|
| 71 |
self.vocab_file = vocab_file
|
| 72 |
self.tokenizer = SPTokenizer(vocab_file)
|
| 73 |
+
kwargs.pop("eos_token", None)
|
| 74 |
+
kwargs.pop("pad_token", None)
|
| 75 |
+
kwargs.pop("unk_token", None)
|
| 76 |
self.special_tokens = {
|
| 77 |
"<bos>": self.tokenizer.bos_id,
|
| 78 |
"<eos>": self.tokenizer.eos_id,
|