Update tokenizer.py

tokenizer.py

@@ -10,7 +10,7 @@ logger = logging.get_logger(__name__)
 
 from huggingface_hub import hf_hub_download
 import json
-import os
+from transformers import AddedToken
 
 def load_json(path, repo_id=None):
     if repo_id:
@@ -128,16 +128,14 @@ class STLTokenizer(PreTrainedTokenizer):
                 tokens.append(self.unk_token)
                 i += 1
         return tokens
-
-    def convert_tokens_to_ids(self, tokens: List[str]) -> List[int]:
-        """
-        Converts a list of tokens into a list of token IDs.
-        Args:
-            tokens (List[str]): A list of tokens to be converted into IDs.
-        Returns:
-            List[int]: A list of corresponding token IDs.
-        """
-        return [self.vocab.get(token, self.vocab[self.unk_token]) for token in tokens]
+        
+    def convert_tokens_to_ids(self, tokens: Union[List[str], str, AddedToken]) -> List[int]:
+        # Se è un singolo token non iterabile, lo metti in lista
+        if isinstance(tokens, (str, AddedToken)):
+            tokens = [tokens]
+        # Converti token a stringa se sono AddedToken
+        tokens_str = [str(token) if isinstance(token, AddedToken) else token for token in tokens]
+        return [self.vocab.get(token, self.vocab[self.unk_token]) for token in tokens_str]
 
     def convert_ids_to_tokens(self, ids: List[int]) -> List[str]:
         """