fix out of vocab token (#5)

- fix out of vocab token (a6c5212a2fb00470a8e6407f7c0def425709d024)


Co-authored-by: RangiLyu <RangiLyu@users.noreply.huggingface.co>

tokenization_interns1.py

@@ -893,7 +893,9 @@ class InternS1Tokenizer(Qwen2Tokenizer):
 
     def convert_tokens_to_string(self, tokens):
         """Converts a sequence of tokens (string) in a single string."""
-        text = "".join(tokens)
+        text = ""
+        for token in tokens:
+            text += token if token else ""
         text = text.replace(
             "▁", "Ġ"
         )  # This discrepancy stems from differing whitespace treatment in SentencePiece versus BPE tokenization.