Upload scripts/train_alizee_v2_stage1_sft.py with huggingface_hub

scripts/train_alizee_v2_stage1_sft.py

@@ -219,22 +219,26 @@ def format_coding_sample(example):
 
     return {"messages": messages, "source": "coding"}
 
-# Format datasets
+# Format datasets with better memory handling
 print("\n🔄 Formatting datasets...")
-print("   Formatting reasoning samples...")
+print("   Formatting reasoning samples (this may take a few minutes)...")
+
+# Use lower parallelism to avoid OOM and add batched processing
 reasoning_formatted = ocr_full.map(
     format_reasoning_sample,
     remove_columns=ocr_full.column_names,
-    num_proc=8,
-    desc="Formatting reasoning"
+    num_proc=4,  # Reduced from 8 to avoid memory issues
+    desc="Formatting reasoning",
+    load_from_cache_file=False,  # Don't cache to save disk
 )
 
 print("   Formatting coding samples...")
 coding_formatted = coding_ds_final.map(
     format_coding_sample,
     remove_columns=coding_ds_final.column_names,
-    num_proc=4,
-    desc="Formatting coding"
+    num_proc=2,  # Reduced parallelism
+    desc="Formatting coding",
+    load_from_cache_file=False,
 )
 
 # Combine and shuffle