Upload folder using huggingface_hub

adapter_config.json

@@ -25,13 +25,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "up_proj",
+    "down_proj",
+    "k_proj",
     "v_proj",
+    "q_proj",
     "o_proj",
-    "down_proj",
     "gate_proj",
-    "q_proj",
-    "k_proj"
+    "up_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e325566f131c65e9ade91ef4d6b36bf75f4d6d3f099460bf3dccbd2564d075c3
+oid sha256:3d8d37d3bd75fc01d14a55529984572995053bef53b73a30d53f4b080d82a0f2
 size 664584480

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bb20f5cc62e14285d3b8a72eaca1ae71faf3b293ee954757b7f91023a0fca6d9
+oid sha256:622750c88c69e639da993c5b73cd1aa1da12f5e11cd900058b9e9a499b2fffdf
 size 1329377575

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e1599b40990af505c591b8a948a922af153eca9aff68fc7776ee92dc8b2dd2b0
+oid sha256:d85b518b1add175fef95a2cd66cf2e301b338be446b49c058da16fb22164c09d
 size 14645

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8db2bbbb37be651c165fdb7fb553d8117ad58dbacc21093c0131c68d882ce6e4
+oid sha256:f36b49862996c84ec71df64aef02c8b667bafc5196ea03b019509144d50963b4
 size 1465

trainer_state.json

@@ -1,76 +1,160 @@
 {
-  "best_global_step": 60,
-  "best_metric": 0.21963942050933838,
-  "best_model_checkpoint": "/content/models/gemma_jigsaw_lmh/checkpoint-60",
-  "epoch": 1.1764705882352942,
+  "best_global_step": 140,
+  "best_metric": 0.22836367785930634,
+  "best_model_checkpoint": "/content/models/gemma_jigsaw_lmh/checkpoint-140",
+  "epoch": 2.7450980392156863,
   "eval_steps": 20,
-  "global_step": 60,
+  "global_step": 140,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "entropy": 2.3684401620518076,
+      "entropy": 2.560735213756561,
       "epoch": 0.39215686274509803,
-      "grad_norm": 5.825000286102295,
+      "grad_norm": 6.186038494110107,
       "learning_rate": 8.758169934640524e-06,
-      "loss": 0.2165,
-      "mean_token_accuracy": 0.9179518394397966,
-      "num_tokens": 581990.0,
+      "loss": 0.5541,
+      "mean_token_accuracy": 0.7453125,
+      "num_tokens": 67412.0,
       "step": 20
     },
     {
       "epoch": 0.39215686274509803,
-      "eval_entropy": 2.3978513204134426,
-      "eval_loss": 0.22738918662071228,
-      "eval_mean_token_accuracy": 0.892045456629533,
-      "eval_num_tokens": 581990.0,
-      "eval_runtime": 2.7348,
-      "eval_samples_per_second": 74.228,
-      "eval_steps_per_second": 4.754,
+      "eval_entropy": 2.491304580981915,
+      "eval_loss": 0.3785918951034546,
+      "eval_mean_token_accuracy": 0.7578671345343957,
+      "eval_num_tokens": 67412.0,
+      "eval_runtime": 2.6932,
+      "eval_samples_per_second": 75.376,
+      "eval_steps_per_second": 4.827,
       "step": 20
     },
     {
-      "entropy": 2.366195046901703,
+      "entropy": 2.419952464103699,
       "epoch": 0.7843137254901961,
-      "grad_norm": 15.021644592285156,
+      "grad_norm": 2.4226012229919434,
       "learning_rate": 7.450980392156863e-06,
-      "loss": 0.188,
-      "mean_token_accuracy": 0.92265625,
-      "num_tokens": 649386.0,
+      "loss": 0.3293,
+      "mean_token_accuracy": 0.81640625,
+      "num_tokens": 134808.0,
       "step": 40
     },
     {
       "epoch": 0.7843137254901961,
-      "eval_entropy": 2.3892184037428637,
-      "eval_loss": 0.28083646297454834,
-      "eval_mean_token_accuracy": 0.8824300720141485,
-      "eval_num_tokens": 649386.0,
-      "eval_runtime": 2.7229,
-      "eval_samples_per_second": 74.553,
-      "eval_steps_per_second": 4.774,
+      "eval_entropy": 2.4435020043299747,
+      "eval_loss": 0.2946617007255554,
+      "eval_mean_token_accuracy": 0.8513986009817857,
+      "eval_num_tokens": 134808.0,
+      "eval_runtime": 2.6642,
+      "eval_samples_per_second": 76.195,
+      "eval_steps_per_second": 4.879,
       "step": 40
     },
     {
-      "entropy": 2.4170044481754305,
+      "entropy": 2.423317462205887,
       "epoch": 1.1764705882352942,
-      "grad_norm": 12.809877395629883,
+      "grad_norm": 10.875091552734375,
       "learning_rate": 6.143790849673204e-06,
-      "loss": 0.182,
-      "mean_token_accuracy": 0.9083705350756646,
-      "num_tokens": 715624.0,
+      "loss": 0.2871,
+      "mean_token_accuracy": 0.8487723216414451,
+      "num_tokens": 201046.0,
       "step": 60
     },
     {
       "epoch": 1.1764705882352942,
-      "eval_entropy": 2.436244304363544,
-      "eval_loss": 0.21963942050933838,
-      "eval_mean_token_accuracy": 0.8944493027833792,
-      "eval_num_tokens": 715624.0,
-      "eval_runtime": 2.7838,
-      "eval_samples_per_second": 72.923,
-      "eval_steps_per_second": 4.67,
+      "eval_entropy": 2.448130937723013,
+      "eval_loss": 0.2594100534915924,
+      "eval_mean_token_accuracy": 0.8621066441902747,
+      "eval_num_tokens": 201046.0,
+      "eval_runtime": 2.6633,
+      "eval_samples_per_second": 76.221,
+      "eval_steps_per_second": 4.881,
       "step": 60
+    },
+    {
+      "entropy": 2.3511528968811035,
+      "epoch": 1.5686274509803921,
+      "grad_norm": 5.017323970794678,
+      "learning_rate": 4.836601307189543e-06,
+      "loss": 0.2503,
+      "mean_token_accuracy": 0.87109375,
+      "num_tokens": 269334.0,
+      "step": 80
+    },
+    {
+      "epoch": 1.5686274509803921,
+      "eval_entropy": 2.3352334682758036,
+      "eval_loss": 0.2594275176525116,
+      "eval_mean_token_accuracy": 0.8824300720141485,
+      "eval_num_tokens": 269334.0,
+      "eval_runtime": 2.6613,
+      "eval_samples_per_second": 76.278,
+      "eval_steps_per_second": 4.885,
+      "step": 80
+    },
+    {
+      "entropy": 2.3079891920089723,
+      "epoch": 1.9607843137254903,
+      "grad_norm": 7.2991743087768555,
+      "learning_rate": 3.529411764705883e-06,
+      "loss": 0.247,
+      "mean_token_accuracy": 0.87890625,
+      "num_tokens": 337424.0,
+      "step": 100
+    },
+    {
+      "epoch": 1.9607843137254903,
+      "eval_entropy": 2.3540270145122824,
+      "eval_loss": 0.23124322295188904,
+      "eval_mean_token_accuracy": 0.8861451057287363,
+      "eval_num_tokens": 337424.0,
+      "eval_runtime": 2.7035,
+      "eval_samples_per_second": 75.087,
+      "eval_steps_per_second": 4.809,
+      "step": 100
+    },
+    {
+      "entropy": 2.3495707869529725,
+      "epoch": 2.3529411764705883,
+      "grad_norm": 5.988176345825195,
+      "learning_rate": 2.222222222222222e-06,
+      "loss": 0.2009,
+      "mean_token_accuracy": 0.9171875,
+      "num_tokens": 404955.0,
+      "step": 120
+    },
+    {
+      "epoch": 2.3529411764705883,
+      "eval_entropy": 2.4085900966937723,
+      "eval_loss": 0.23444519937038422,
+      "eval_mean_token_accuracy": 0.8957604903441209,
+      "eval_num_tokens": 404955.0,
+      "eval_runtime": 2.6365,
+      "eval_samples_per_second": 76.995,
+      "eval_steps_per_second": 4.931,
+      "step": 120
+    },
+    {
+      "entropy": 2.3811947822570803,
+      "epoch": 2.7450980392156863,
+      "grad_norm": 9.767471313476562,
+      "learning_rate": 9.150326797385621e-07,
+      "loss": 0.1973,
+      "mean_token_accuracy": 0.91328125,
+      "num_tokens": 472246.0,
+      "step": 140
+    },
+    {
+      "epoch": 2.7450980392156863,
+      "eval_entropy": 2.4008009983943057,
+      "eval_loss": 0.22836367785930634,
+      "eval_mean_token_accuracy": 0.9040646873987638,
+      "eval_num_tokens": 472246.0,
+      "eval_runtime": 2.645,
+      "eval_samples_per_second": 76.748,
+      "eval_steps_per_second": 4.915,
+      "step": 140
     }
   ],
   "logging_steps": 20,
@@ -90,7 +174,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 4282364704969728.0,
+  "total_flos": 1.0222031863807488e+16,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6666f3ffe597916194c55d1e8f4a39a253059f0511383c7a71cd741dc4d25435
+oid sha256:ba3e30758f814af90a14f019df18e1d2888c00c0bbdb4d0137148582f3a9dada
 size 6353