Upload train_qwen3_8b_hf.py with huggingface_hub

train_qwen3_8b_hf.py

@@ -53,7 +53,7 @@ eval_dataset = dataset_split["test"]
 print(f"   Train: {len(train_dataset)} examples")
 print(f"   Eval: {len(eval_dataset)} examples")
 
-# Training configuration
+# Training configuration - optimized for memory on A10G
 config = SFTConfig(
     # Hub settings
     output_dir="qwen3-8b-vyvo-copilot",
@@ -62,26 +62,27 @@ config = SFTConfig(
     hub_strategy="every_save",
     hub_private_repo=False,
 
-    # Training parameters - optimized for 8B model with LoRA
+    # Training parameters - reduced for memory
     num_train_epochs=3,
-    per_device_train_batch_size=2,
-    gradient_accumulation_steps=8,  # Effective batch size = 16
+    per_device_train_batch_size=1,  # Reduced from 2
+    gradient_accumulation_steps=16,  # Increased to maintain effective batch size
     learning_rate=2e-4,
-    max_length=2048,  # Good context for multi-turn conversations
+    max_length=1024,  # Reduced from 2048 to save memory
 
     # Memory optimization
     gradient_checkpointing=True,
+    gradient_checkpointing_kwargs={"use_reentrant": False},
     bf16=True,
+    optim="adamw_8bit",  # Use 8-bit optimizer to save memory
 
     # Logging & checkpointing
     logging_steps=10,
     save_strategy="steps",
     save_steps=200,
-    save_total_limit=3,
+    save_total_limit=2,
 
-    # Evaluation
-    eval_strategy="steps",
-    eval_steps=200,
+    # Evaluation - skip eval during training to save memory
+    eval_strategy="no",
 
     # Optimization
     warmup_ratio=0.05,
@@ -109,7 +110,6 @@ print("🎯 Initializing trainer with Qwen/Qwen3-8B...")
 trainer = SFTTrainer(
     model="Qwen/Qwen3-8B",
     train_dataset=train_dataset,
-    eval_dataset=eval_dataset,
     args=config,
     peft_config=peft_config,
 )