Update tokenization_chatglm.py
#7
by gameofdimension - opened
- tokenization_chatglm.py +2 -0
tokenization_chatglm.py
CHANGED
|
@@ -72,6 +72,8 @@ class SPTokenizer:
|
|
| 72 |
return self.index_special_tokens[index]
|
| 73 |
if index in [self.eos_id, self.bos_id, self.pad_id] or index < 0:
|
| 74 |
return ""
|
|
|
|
|
|
|
| 75 |
return self.sp_model.IdToPiece(index)
|
| 76 |
|
| 77 |
|
|
|
|
| 72 |
return self.index_special_tokens[index]
|
| 73 |
if index in [self.eos_id, self.bos_id, self.pad_id] or index < 0:
|
| 74 |
return ""
|
| 75 |
+
if index >= self.sp_model.vocab_size():
|
| 76 |
+
return ""
|
| 77 |
return self.sp_model.IdToPiece(index)
|
| 78 |
|
| 79 |
|