Fix incorrect unk_id assignment
#16
by
codecho
- opened
- tokenization_kimia.py +1 -1
tokenization_kimia.py
CHANGED
|
@@ -127,7 +127,7 @@ class TikTokenTokenizer(PreTrainedTokenizer):
|
|
| 127 |
self.pad_id: int = self.special_tokens[self.pad_token]
|
| 128 |
|
| 129 |
self.unk_token: str = special_tokens[-2]
|
| 130 |
-
self.unk_id: int = self.special_tokens[self.
|
| 131 |
|
| 132 |
self.stop_tokens = {
|
| 133 |
self.special_tokens["[EOS]"],
|
|
|
|
| 127 |
self.pad_id: int = self.special_tokens[self.pad_token]
|
| 128 |
|
| 129 |
self.unk_token: str = special_tokens[-2]
|
| 130 |
+
self.unk_id: int = self.special_tokens[self.unk_token]
|
| 131 |
|
| 132 |
self.stop_tokens = {
|
| 133 |
self.special_tokens["[EOS]"],
|