Upload folder using huggingface_hub

run_transformers_training.py

@@ -980,6 +980,28 @@ def main():
             
             # Set up training arguments
             log_info("Setting up training arguments")
+            
+            # Validate FSDP config before using it
+            fsdp_args = None
+            if fsdp_config is not None and is_distributed and multi_gpu_strategy == "fsdp":
+                try:
+                    # Convert FSDP config to proper format expected by TrainingArguments
+                    fsdp_args = {
+                        "fsdp_transformer_layer_cls_to_wrap": fsdp_config.get("fsdp_transformer_layer_cls_to_wrap", []),
+                        "fsdp_offload_params": fsdp_config.get("fsdp_offload_params", False),
+                        "fsdp_backward_prefetch": fsdp_config.get("fsdp_backward_prefetch", "BACKWARD_PRE"),
+                        "fsdp_min_num_params": fsdp_config.get("fsdp_min_num_params", 1e6),
+                        "fsdp_sharding_strategy": fsdp_config.get("fsdp_sharding_strategy", 1),
+                    }
+                    log_info("FSDP config validated and prepared")
+                except Exception as e:
+                    log_info(f"Error preparing FSDP config: {str(e)}, disabling FSDP")
+                    fsdp_args = None
+
+            # Check if we're running in a Space
+            is_space = bool(os.environ.get("SPACE_ID"))
+            
+            # Create training arguments with validated FSDP config
             training_args = TrainingArguments(
                 output_dir=transformers_config.get("output_dir", "./results") or transformers_config.get("checkpointing", {}).get("output_dir", "./results"),
                 num_train_epochs=transformers_config.get("training", {}).get("num_train_epochs", 3),
@@ -998,7 +1020,8 @@ def main():
                 max_grad_norm=transformers_config.get("training", {}).get("max_grad_norm", 1.0),
                 push_to_hub=transformers_config.get("huggingface_hub", {}).get("push_to_hub", False),
                 hub_model_id=transformers_config.get("huggingface_hub", {}).get("hub_model_id", None),
-                hub_token=os.environ.get("HF_TOKEN", None),
+                # Don't set hub_token when running in a Space - it will use Space secrets automatically
+                hub_token=None if is_space else os.environ.get("HF_TOKEN", None),
                 report_to="tensorboard",
                 remove_unused_columns=False,  # Keep all columns
                 gradient_checkpointing=transformers_config.get("training", {}).get("gradient_checkpointing", True),
@@ -1008,12 +1031,18 @@ def main():
                 dataloader_drop_last=False,  # Process all examples
                 dataloader_num_workers=dataloader_workers,
                 no_cuda=False if CUDA_AVAILABLE else True,  # Use CUDA if available
-                # Only add FSDP if we're in distributed mode with FSDP strategy
-                fsdp=fsdp_config if is_distributed and multi_gpu_strategy == "fsdp" else None,
+                fsdp=fsdp_args,  # Use validated FSDP config
             )
+
+            log_info("Training arguments created successfully")
+            
+            # Validate dataset before creating sampler
+            if dataset is None:
+                raise ValueError("Dataset is None - cannot create sampler")
             
             # Create sequential sampler to maintain original dataset order
             sequential_sampler = torch.utils.data.SequentialSampler(dataset)
+            log_info("Sequential sampler created")
             
             # Initialize trainer first
             log_info("Initializing Trainer")