Update fireredasr2s/fireredasr2/tokenizer/llm_tokenizer.py

fireredasr2s/fireredasr2/tokenizer/llm_tokenizer.py

@@ -72,10 +72,11 @@ class LlmTokenizerWrapper:
             )
 
         # Padding texts
+        print(1, texts)
         max_len_texts = max([len(text) for text in texts])
         if tokenizer.padding_side == "right":
             texts = [
-                text + [tokenizer.pad_token_id] * (max_len_texts - len(text))
+                list(text) + [tokenizer.pad_token_id] * (max_len_texts - len(text))
                 for text in texts
             ]
         else:
@@ -83,7 +84,7 @@ class LlmTokenizerWrapper:
                 [tokenizer.pad_token_id] * (max_len_texts - len(text)) + text
                 for text in texts
             ]
-        print(texts)
+        print(2, texts)
         input_ids = torch.tensor(texts, dtype=torch.int)
 
         target_ids = input_ids.clone()