Training in progress, epoch 1

README.md

@@ -4,8 +4,8 @@ library_name: transformers
 model_name: medgemma-27b-it-dr7
 tags:
 - generated_from_trainer
-- trl
 - sft
+- trl
 licence: license
 ---
 
@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/yoon307-kaist/medgemma-27b-it-dr7-Project/runs/xum7do4o) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/yoon307-kaist/medgemma-27b-it-dr7-Project/runs/ysbi2fv0) 
 
 
 This model was trained with SFT.

adapter_config.json

@@ -28,16 +28,16 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "o_proj",
-    "q_proj",
+    "out_proj",
+    "up_proj",
     "fc1",
     "v_proj",
-    "out_proj",
-    "fc2",
-    "gate_proj",
     "down_proj",
-    "up_proj",
-    "k_proj"
+    "fc2",
+    "o_proj",
+    "q_proj",
+    "k_proj",
+    "gate_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3c0250ec29e4f943185fe8811b4cafef65b141075f8b3d1d0f1ff7e9de07fba3
+oid sha256:d8e6259a4f22e06219271b585070fb37ea36fa4894096a0d2235347ea650bb05
 size 6616400680

runs/Jul22_21-50-05_meedgxh100a/events.out.tfevents.1753235406.meedgxh100a.3618383.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:deeffefabc11641cb5525834045fda992d597129b2f4c99d661233dc53cd89b5
+size 9917

train_medgemma_ft_copy.py

@@ -303,7 +303,7 @@ if __name__ == '__main__':
 
     training_args = SFTConfig(
         output_dir=exp_name,
-        num_train_epochs= 5,                       # Number of training epochs
+        num_train_epochs= 15,                       # Number of training epochs
         per_device_train_batch_size=2,                           # Batch size per device during training
         per_device_eval_batch_size=4,                            # Batch size per device during evaluation
         gradient_accumulation_steps=8,                           # Number of steps before performing a backward/update pass
@@ -313,7 +313,7 @@ if __name__ == '__main__':
         save_strategy="epoch",                                   # Save checkpoint every epoch
         eval_strategy="steps",                                   # Evaluate every `eval_steps`
         eval_steps=10000,                                           # Number of steps between evaluations
-        learning_rate=2e-5,                             # Learning rate based on QLoRA paper
+        learning_rate=1e-4,                             # Learning rate based on QLoRA paper
         bf16=True,                                               # Use bfloat16 precision
         max_grad_norm=0.3,                                       # Max gradient norm based on QLoRA paper
         warmup_ratio=0.03,                                       # Warmup ratio based on QLoRA paper

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f169d34baac7f3033f31c79056229210efe2de0c8a6f3d7543ae292a836d1f64
+oid sha256:d419952f7a391ce7541d8747f554ed112a42a866627d0740f7b0550c01abdd71
 size 5816