Upload scripts/train_alizee_v2_stage1_sft.py with huggingface_hub
Browse files
scripts/train_alizee_v2_stage1_sft.py
CHANGED
|
@@ -311,7 +311,6 @@ training_config = SFTConfig(
|
|
| 311 |
|
| 312 |
# Memory-efficient tokenization (reduce parallel processes to save RAM)
|
| 313 |
dataset_num_proc=1, # Single process to avoid OOM during tokenization
|
| 314 |
-
dataset_batch_size=100, # Smaller batches during tokenization
|
| 315 |
)
|
| 316 |
|
| 317 |
# Initialize trainer
|
|
|
|
| 311 |
|
| 312 |
# Memory-efficient tokenization (reduce parallel processes to save RAM)
|
| 313 |
dataset_num_proc=1, # Single process to avoid OOM during tokenization
|
|
|
|
| 314 |
)
|
| 315 |
|
| 316 |
# Initialize trainer
|