add none additional token
Browse files- tokenization_evabyte.py +1 -0
tokenization_evabyte.py
CHANGED
|
@@ -63,6 +63,7 @@ class EvaByteTokenizer(PreTrainedTokenizer):
|
|
| 63 |
" extra_ids tokens"
|
| 64 |
)
|
| 65 |
|
|
|
|
| 66 |
#### override some reserved tokens to support chat template
|
| 67 |
for i, token in enumerate(additional_special_tokens):
|
| 68 |
if token == "<extra_id_5>":
|
|
|
|
| 63 |
" extra_ids tokens"
|
| 64 |
)
|
| 65 |
|
| 66 |
+
additional_special_tokens = additional_special_tokens or[]
|
| 67 |
#### override some reserved tokens to support chat template
|
| 68 |
for i, token in enumerate(additional_special_tokens):
|
| 69 |
if token == "<extra_id_5>":
|