Update README.md

README.md

@@ -6,9 +6,74 @@ tags:
 
 # Model Card for Model ID
 
-<!-- Provide a quick summary of what the model is/does. -->
-
-
+Map: 100%
+ 2920/2920 [00:01<00:00, 1602.09 examples/s]
+ [365/365 4:25:54]
+Test Loss: 1.0123
+
+Step	Training Loss	Validation Loss
+250	0.983800	0.957103
+500	0.937900	0.954966
+750	0.862300	0.968044
+1000	0.800900	0.986456
+1250	0.712600	1.017532
+1500	0.652100	1.035168
+1750	0.600500	1.051357
+2000	0.412800	1.152156
+2250	0.386200	1.168790
+2500	0.377300	1.185837
+2750	0.346600	1.223637
+3000	0.351300	1.254214
+3250	0.321700	1.273642
+3500	0.329900	1.280087
+
+train_dataset_transformed = train_dataset_transformed.shuffle(seed=3407)
+
+trainer = SFTTrainer(
+    model=model,
+    tokenizer=tokenizer,
+    train_dataset=train_dataset_transformed,
+    eval_dataset=val_dataset_transformed,
+    max_seq_length=max_seq_length,
+    dataset_num_proc=2,
+    packing=False,
+    args=TrainingArguments(
+        per_device_train_batch_size=8,  # Increased batch size
+        gradient_accumulation_steps=1,  # Reduced from 4
+        warmup_ratio=0.05,  # Better than fixed 5 steps for 20K samples
+        num_train_epochs=2,  # Compromise between 1 and 3
+        learning_rate=1.5e-4,  # Try between 1e-4 and 2e-4
+        fp16=not is_bfloat16_supported(),
+        bf16=is_bfloat16_supported(),
+        logging_steps=50,
+        optim="adamw_8bit",
+        weight_decay=0.02,  # Increased regularization
+        lr_scheduler_type="cosine_with_restarts",
+        seed=3407,
+        output_dir="outputs",
+        evaluation_strategy="steps",
+        eval_steps=250,  # More frequent validation
+        save_strategy="steps",
+        save_steps=250,
+        load_best_model_at_end=True,
+        metric_for_best_model="eval_loss",  # Changed from "loss"
+        greater_is_better=False,
+    ),
+)
+
+
+# another revise 
+model = FastLanguageModel.get_peft_model(
+    model,
+    r = 32,  # Reduced from 64 for better generalization
+    target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
+                    "gate_proj", "up_proj", "down_proj"],
+    lora_alpha = 16,  # Reduced from 32 (alpha = r/2 is common)
+    lora_dropout = 0.1,  # Slight regularization
+    bias = "none",
+    use_gradient_checkpointing = "unsloth",
+    random_state = 3407,
+)
 
 ## Model Details