Upload folder using huggingface_hub

run_transformers_training.py

@@ -400,8 +400,13 @@ def load_dataset_with_mapping(dataset_config):
                 record = {}
                 record["prompt_number"] = dataset[i].get("prompt_number", "N/A")
                 record["article_id"] = dataset[i].get("article_id", "N/A")
-                if "conversations" in dataset[i]:
-                    record["conversations_length"] = len(dataset[i]["conversations"])
+                # Safely get conversations length with None check
+                conversations = dataset[i].get("conversations")
+                if conversations is not None and isinstance(conversations, list):
+                    record["conversations_length"] = len(conversations)
+                else:
+                    record["conversations_length"] = 0
+                    logger.warning(f"Invalid conversations for sample {i}: {type(conversations)}")
                 sample_records.append(record)
                 
             logger.info(f"Sample records: {sample_records}")