Upload train_infrastructure_model.py with huggingface_hub

train_infrastructure_model.py

@@ -103,16 +103,17 @@ peft_config = LoraConfig(
 
 # Training configuration
 print("Configuring training...")
-training_args = SFTConfig(
+from transformers import TrainingArguments
+
+training_args = TrainingArguments(
     output_dir=OUTPUT_MODEL,
 
     # Training parameters
     num_train_epochs=3,
-    per_device_train_batch_size=4,
-    per_device_eval_batch_size=4,
-    gradient_accumulation_steps=4,
+    per_device_train_batch_size=2,
+    per_device_eval_batch_size=2,
+    gradient_accumulation_steps=8,
     gradient_checkpointing=True,
-    max_seq_length=2048,
 
     # Optimization
     learning_rate=2e-4,
@@ -123,7 +124,7 @@ training_args = SFTConfig(
 
     # Evaluation and logging
     eval_strategy="steps",
-    eval_steps=50,
+    eval_steps=100,
     logging_steps=10,
     save_strategy="steps",
     save_steps=200,
@@ -142,12 +143,9 @@ training_args = SFTConfig(
     # Performance
     bf16=True,
     max_grad_norm=0.3,
-    group_by_length=True,
 
     # Misc
     seed=42,
-    dataset_text_field="text",
-    packing=False,
 )
 
 # Initialize trainer
@@ -158,6 +156,9 @@ trainer = SFTTrainer(
     eval_dataset=eval_dataset,
     peft_config=peft_config,
     args=training_args,
+    dataset_text_field="text",
+    max_seq_length=2048,
+    packing=False,
 )
 
 # Train