Update fireredasr2s/fireredasr2/tokenizer/llm_tokenizer.py

fireredasr2s/fireredasr2/tokenizer/llm_tokenizer.py

@@ -75,7 +75,7 @@ class LlmTokenizerWrapper:
         max_len_texts = max([len(text) for text in texts])
         if tokenizer.padding_side == "right":
             texts = [
-                list(text) + [tokenizer.pad_token_id] * (max_len_texts - len(text))
+                text + [tokenizer.pad_token_id] * (max_len_texts - len(text))
                 for text in texts
             ]
         else:
@@ -83,6 +83,7 @@ class LlmTokenizerWrapper:
                 [tokenizer.pad_token_id] * (max_len_texts - len(text)) + text
                 for text in texts
             ]
+        print(texts)
         input_ids = torch.tensor(texts, dtype=torch.int)
 
         target_ids = input_ids.clone()