Update tokenization_rwkv_world.py
Browse files
tokenization_rwkv_world.py
CHANGED
|
@@ -244,13 +244,13 @@ class RWKVWorldTokenizer(PreTrainedTokenizer):
|
|
| 244 |
**kwargs
|
| 245 |
):
|
| 246 |
self.add_bos_token = False
|
|
|
|
|
|
|
|
|
|
| 247 |
super().__init__(
|
| 248 |
errors=errors,
|
| 249 |
**kwargs,
|
| 250 |
)
|
| 251 |
-
|
| 252 |
-
with open(vocab_file, encoding="utf-8") as vocab_handle:
|
| 253 |
-
self.encoder = json.load(vocab_handle)
|
| 254 |
self.decoder = {v: k for k, v in self.encoder.items()}
|
| 255 |
self.trie = DATrie(self.all_special_ids)
|
| 256 |
for k, v in self.encoder.items():
|
|
|
|
| 244 |
**kwargs
|
| 245 |
):
|
| 246 |
self.add_bos_token = False
|
| 247 |
+
|
| 248 |
+
with open(vocab_file, encoding="utf-8") as vocab_handle:
|
| 249 |
+
self.encoder = json.load(vocab_handle)
|
| 250 |
super().__init__(
|
| 251 |
errors=errors,
|
| 252 |
**kwargs,
|
| 253 |
)
|
|
|
|
|
|
|
|
|
|
| 254 |
self.decoder = {v: k for k, v in self.encoder.items()}
|
| 255 |
self.trie = DATrie(self.all_special_ids)
|
| 256 |
for k, v in self.encoder.items():
|