Upload tokenization_codegen25.py

tokenization_codegen25.py

@@ -166,7 +166,7 @@ class CodeGen25Tokenizer(PreTrainedTokenizer):
 
     def _convert_id_to_token(self, index):
         """Converts an index (integer) in a token (str) using the vocab."""
-        return self.encoder.decode_single_token_bytes(index).decode("utf-8")
+        return self.encoder.decode_single_token_bytes(index).decode("utf-8", errors="ignore")
 
     def _decode(self, token_ids: List[int], skip_special_tokens: bool = False, **kwargs):
         if skip_special_tokens: