Training in progress, step 1200

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:05f7efbf6b466535444076e687258cb18a535bcb077227f52f4e089319560ffc
+oid sha256:0fa281a3450d8c802377bbf9c31fb17bc5dd54ed62421d168899bda5e799396d
 size 319977229

{checkpoint-1600 → checkpoint-1000/adapter_model/adapter_model}/adapter_config.json

@@ -14,13 +14,13 @@
   "r": 64,
   "revision": null,
   "target_modules": [
-    "v_proj",
+    "o_proj",
+    "gate_proj",
     "q_proj",
+    "v_proj",
     "up_proj",
     "k_proj",
-    "down_proj",
-    "o_proj",
-    "gate_proj"
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

{checkpoint-1600 → checkpoint-1000/adapter_model/adapter_model}/adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:661c11102b2c8e2770a9e6544a76b860defa1c672e6f4c9bea4d9a5838e30270
+oid sha256:05f7efbf6b466535444076e687258cb18a535bcb077227f52f4e089319560ffc
 size 319977229

{checkpoint-1600/adapter_model/adapter_model → checkpoint-1200}/adapter_config.json

@@ -14,13 +14,13 @@
   "r": 64,
   "revision": null,
   "target_modules": [
-    "v_proj",
+    "o_proj",
+    "gate_proj",
     "q_proj",
+    "v_proj",
     "up_proj",
     "k_proj",
-    "down_proj",
-    "o_proj",
-    "gate_proj"
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

{checkpoint-1600/adapter_model/adapter_model → checkpoint-1200}/adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:661c11102b2c8e2770a9e6544a76b860defa1c672e6f4c9bea4d9a5838e30270
+oid sha256:0fa281a3450d8c802377bbf9c31fb17bc5dd54ed62421d168899bda5e799396d
 size 319977229

{checkpoint-1600 → checkpoint-1200}/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9ea6a9f8d74a1230904722e63b90c94507b5204e3a90b2323f850826f8c83af0
+oid sha256:5a4f6131a3dc044c7d7b3d4021384f3235a623adf298af2ab625a0be18fc3351
 size 1279539973

{checkpoint-1600 → checkpoint-1200}/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f85d39e532078df894cbe0b7a8747d6b80fba787b530da7aac7473a2de3d754
+oid sha256:f582e10f242274f63599f54a56042a5295303141e63a7bf3490a0d2609f24067
 size 14511

{checkpoint-1600 → checkpoint-1200}/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fcd2587f362188ac4728d4fa6edf8d2b0b6d72db365d49f7b847d4d79e3da09f
+oid sha256:3d8d6be7898f87772ccbc5c732e900fe63a643c4595ce6af3d6bc6f811ba4b65
 size 627

{checkpoint-1600 → checkpoint-1200}/trainer_state.json

@@ -1,8 +1,8 @@
 {
-  "best_metric": 0.7887665629386902,
-  "best_model_checkpoint": "experts/expert-0/checkpoint-1600",
-  "epoch": 0.5740425150237689,
-  "global_step": 1600,
+  "best_metric": 0.80042564868927,
+  "best_model_checkpoint": "experts/expert-0/checkpoint-1200",
+  "epoch": 0.43053188626782674,
+  "global_step": 1200,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -16,109 +16,109 @@
     {
       "epoch": 0.01,
       "learning_rate": 0.0002,
-      "loss": 0.8925,
+      "loss": 0.8926,
       "step": 20
     },
     {
       "epoch": 0.01,
       "learning_rate": 0.0002,
-      "loss": 0.9028,
+      "loss": 0.9029,
       "step": 30
     },
     {
       "epoch": 0.01,
       "learning_rate": 0.0002,
-      "loss": 0.8606,
+      "loss": 0.8605,
       "step": 40
     },
     {
       "epoch": 0.02,
       "learning_rate": 0.0002,
-      "loss": 0.9295,
+      "loss": 0.9303,
       "step": 50
     },
     {
       "epoch": 0.02,
       "learning_rate": 0.0002,
-      "loss": 0.9049,
+      "loss": 0.905,
       "step": 60
     },
     {
       "epoch": 0.03,
       "learning_rate": 0.0002,
-      "loss": 0.8361,
+      "loss": 0.8362,
       "step": 70
     },
     {
       "epoch": 0.03,
       "learning_rate": 0.0002,
-      "loss": 0.8492,
+      "loss": 0.8497,
       "step": 80
     },
     {
       "epoch": 0.03,
       "learning_rate": 0.0002,
-      "loss": 0.8251,
+      "loss": 0.8248,
       "step": 90
     },
     {
       "epoch": 0.04,
       "learning_rate": 0.0002,
-      "loss": 0.8576,
+      "loss": 0.8572,
       "step": 100
     },
     {
       "epoch": 0.04,
       "learning_rate": 0.0002,
-      "loss": 0.7906,
+      "loss": 0.7907,
       "step": 110
     },
     {
       "epoch": 0.04,
       "learning_rate": 0.0002,
-      "loss": 0.8599,
+      "loss": 0.8598,
       "step": 120
     },
     {
       "epoch": 0.05,
       "learning_rate": 0.0002,
-      "loss": 0.8079,
+      "loss": 0.8078,
       "step": 130
     },
     {
       "epoch": 0.05,
       "learning_rate": 0.0002,
-      "loss": 0.8672,
+      "loss": 0.8671,
       "step": 140
     },
     {
       "epoch": 0.05,
       "learning_rate": 0.0002,
-      "loss": 0.8222,
+      "loss": 0.8225,
       "step": 150
     },
     {
       "epoch": 0.06,
       "learning_rate": 0.0002,
-      "loss": 0.8248,
+      "loss": 0.8246,
       "step": 160
     },
     {
       "epoch": 0.06,
       "learning_rate": 0.0002,
-      "loss": 0.9017,
+      "loss": 0.9022,
       "step": 170
     },
     {
       "epoch": 0.06,
       "learning_rate": 0.0002,
-      "loss": 0.8596,
+      "loss": 0.8598,
       "step": 180
     },
     {
       "epoch": 0.07,
       "learning_rate": 0.0002,
-      "loss": 0.8714,
+      "loss": 0.8717,
       "step": 190
     },
     {
@@ -129,23 +129,23 @@
     },
     {
       "epoch": 0.07,
-      "eval_loss": 0.8343863487243652,
-      "eval_runtime": 86.193,
-      "eval_samples_per_second": 11.602,
-      "eval_steps_per_second": 5.801,
+      "eval_loss": 0.8350953459739685,
+      "eval_runtime": 85.2141,
+      "eval_samples_per_second": 11.735,
+      "eval_steps_per_second": 5.868,
       "step": 200
     },
     {
       "epoch": 0.07,
-      "mmlu_eval_accuracy": 0.48061337229293305,
+      "mmlu_eval_accuracy": 0.4813944126598768,
       "mmlu_eval_accuracy_abstract_algebra": 0.2727272727272727,
       "mmlu_eval_accuracy_anatomy": 0.6428571428571429,
       "mmlu_eval_accuracy_astronomy": 0.4375,
       "mmlu_eval_accuracy_business_ethics": 0.5454545454545454,
-      "mmlu_eval_accuracy_clinical_knowledge": 0.4482758620689655,
+      "mmlu_eval_accuracy_clinical_knowledge": 0.4827586206896552,
       "mmlu_eval_accuracy_college_biology": 0.4375,
       "mmlu_eval_accuracy_college_chemistry": 0.25,
-      "mmlu_eval_accuracy_college_computer_science": 0.2727272727272727,
+      "mmlu_eval_accuracy_college_computer_science": 0.18181818181818182,
       "mmlu_eval_accuracy_college_mathematics": 0.2727272727272727,
       "mmlu_eval_accuracy_college_medicine": 0.36363636363636365,
       "mmlu_eval_accuracy_college_physics": 0.45454545454545453,
@@ -157,18 +157,18 @@
       "mmlu_eval_accuracy_formal_logic": 0.14285714285714285,
       "mmlu_eval_accuracy_global_facts": 0.5,
       "mmlu_eval_accuracy_high_school_biology": 0.375,
-      "mmlu_eval_accuracy_high_school_chemistry": 0.18181818181818182,
+      "mmlu_eval_accuracy_high_school_chemistry": 0.22727272727272727,
       "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
       "mmlu_eval_accuracy_high_school_european_history": 0.5555555555555556,
       "mmlu_eval_accuracy_high_school_geography": 0.9090909090909091,
-      "mmlu_eval_accuracy_high_school_government_and_politics": 0.5714285714285714,
-      "mmlu_eval_accuracy_high_school_macroeconomics": 0.46511627906976744,
-      "mmlu_eval_accuracy_high_school_mathematics": 0.20689655172413793,
-      "mmlu_eval_accuracy_high_school_microeconomics": 0.46153846153846156,
-      "mmlu_eval_accuracy_high_school_physics": 0.11764705882352941,
+      "mmlu_eval_accuracy_high_school_government_and_politics": 0.5238095238095238,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.4883720930232558,
+      "mmlu_eval_accuracy_high_school_mathematics": 0.2413793103448276,
+      "mmlu_eval_accuracy_high_school_microeconomics": 0.38461538461538464,
+      "mmlu_eval_accuracy_high_school_physics": 0.17647058823529413,
       "mmlu_eval_accuracy_high_school_psychology": 0.8833333333333333,
-      "mmlu_eval_accuracy_high_school_statistics": 0.30434782608695654,
-      "mmlu_eval_accuracy_high_school_us_history": 0.6363636363636364,
+      "mmlu_eval_accuracy_high_school_statistics": 0.34782608695652173,
+      "mmlu_eval_accuracy_high_school_us_history": 0.6818181818181818,
       "mmlu_eval_accuracy_high_school_world_history": 0.6923076923076923,
       "mmlu_eval_accuracy_human_aging": 0.6521739130434783,
       "mmlu_eval_accuracy_human_sexuality": 0.3333333333333333,
@@ -181,57 +181,57 @@
       "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
       "mmlu_eval_accuracy_miscellaneous": 0.6744186046511628,
       "mmlu_eval_accuracy_moral_disputes": 0.5,
-      "mmlu_eval_accuracy_moral_scenarios": 0.22,
-      "mmlu_eval_accuracy_nutrition": 0.6060606060606061,
+      "mmlu_eval_accuracy_moral_scenarios": 0.21,
+      "mmlu_eval_accuracy_nutrition": 0.5757575757575758,
       "mmlu_eval_accuracy_philosophy": 0.4411764705882353,
       "mmlu_eval_accuracy_prehistory": 0.4857142857142857,
-      "mmlu_eval_accuracy_professional_accounting": 0.3548387096774194,
+      "mmlu_eval_accuracy_professional_accounting": 0.3870967741935484,
       "mmlu_eval_accuracy_professional_law": 0.3176470588235294,
       "mmlu_eval_accuracy_professional_medicine": 0.5483870967741935,
-      "mmlu_eval_accuracy_professional_psychology": 0.5072463768115942,
+      "mmlu_eval_accuracy_professional_psychology": 0.4927536231884058,
       "mmlu_eval_accuracy_public_relations": 0.6666666666666666,
       "mmlu_eval_accuracy_security_studies": 0.4444444444444444,
       "mmlu_eval_accuracy_sociology": 0.7272727272727273,
       "mmlu_eval_accuracy_us_foreign_policy": 0.6363636363636364,
-      "mmlu_eval_accuracy_virology": 0.5555555555555556,
-      "mmlu_eval_accuracy_world_religions": 0.6842105263157895,
-      "mmlu_loss": 1.4117222026029392,
+      "mmlu_eval_accuracy_virology": 0.5,
+      "mmlu_eval_accuracy_world_religions": 0.7368421052631579,
+      "mmlu_loss": 1.4071910156437997,
       "step": 200
     },
     {
       "epoch": 0.08,
       "learning_rate": 0.0002,
-      "loss": 0.8838,
+      "loss": 0.8834,
       "step": 210
     },
     {
       "epoch": 0.08,
       "learning_rate": 0.0002,
-      "loss": 0.8882,
+      "loss": 0.8887,
       "step": 220
     },
     {
       "epoch": 0.08,
       "learning_rate": 0.0002,
-      "loss": 0.7978,
+      "loss": 0.7981,
       "step": 230
     },
     {
       "epoch": 0.09,
       "learning_rate": 0.0002,
-      "loss": 0.8739,
+      "loss": 0.8734,
       "step": 240
     },
     {
       "epoch": 0.09,
       "learning_rate": 0.0002,
-      "loss": 0.8501,
+      "loss": 0.8505,
       "step": 250
     },
     {
       "epoch": 0.09,
       "learning_rate": 0.0002,
-      "loss": 0.7981,
+      "loss": 0.7984,
       "step": 260
     },
     {
@@ -243,43 +243,43 @@
     {
       "epoch": 0.1,
       "learning_rate": 0.0002,
-      "loss": 0.7935,
+      "loss": 0.7936,
       "step": 280
     },
     {
       "epoch": 0.1,
       "learning_rate": 0.0002,
-      "loss": 0.826,
+      "loss": 0.8271,
       "step": 290
     },
     {
       "epoch": 0.11,
       "learning_rate": 0.0002,
-      "loss": 0.8809,
+      "loss": 0.8804,
       "step": 300
     },
     {
       "epoch": 0.11,
       "learning_rate": 0.0002,
-      "loss": 0.8352,
+      "loss": 0.8345,
       "step": 310
     },
     {
       "epoch": 0.11,
       "learning_rate": 0.0002,
-      "loss": 0.9132,
+      "loss": 0.9121,
       "step": 320
     },
     {
       "epoch": 0.12,
       "learning_rate": 0.0002,
-      "loss": 0.823,
+      "loss": 0.8226,
       "step": 330
     },
     {
       "epoch": 0.12,
       "learning_rate": 0.0002,
-      "loss": 0.7908,
+      "loss": 0.791,
       "step": 340
     },
     {
@@ -291,53 +291,53 @@
     {
       "epoch": 0.13,
       "learning_rate": 0.0002,
-      "loss": 0.8627,
+      "loss": 0.8631,
       "step": 360
     },
     {
       "epoch": 0.13,
       "learning_rate": 0.0002,
-      "loss": 0.8301,
+      "loss": 0.8304,
       "step": 370
     },
     {
       "epoch": 0.14,
       "learning_rate": 0.0002,
-      "loss": 0.8413,
+      "loss": 0.8412,
       "step": 380
     },
     {
       "epoch": 0.14,
       "learning_rate": 0.0002,
-      "loss": 0.8481,
+      "loss": 0.8477,
       "step": 390
     },
     {
       "epoch": 0.14,
       "learning_rate": 0.0002,
-      "loss": 0.7947,
+      "loss": 0.7948,
       "step": 400
     },
     {
       "epoch": 0.14,
-      "eval_loss": 0.8168367147445679,
-      "eval_runtime": 86.2462,
-      "eval_samples_per_second": 11.595,
-      "eval_steps_per_second": 5.797,
+      "eval_loss": 0.8169942498207092,
+      "eval_runtime": 84.8824,
+      "eval_samples_per_second": 11.781,
+      "eval_steps_per_second": 5.891,
       "step": 400
     },
     {
       "epoch": 0.14,
-      "mmlu_eval_accuracy": 0.4878461855478828,
+      "mmlu_eval_accuracy": 0.4838679988012413,
       "mmlu_eval_accuracy_abstract_algebra": 0.2727272727272727,
-      "mmlu_eval_accuracy_anatomy": 0.7142857142857143,
+      "mmlu_eval_accuracy_anatomy": 0.6428571428571429,
       "mmlu_eval_accuracy_astronomy": 0.4375,
       "mmlu_eval_accuracy_business_ethics": 0.5454545454545454,
-      "mmlu_eval_accuracy_clinical_knowledge": 0.5517241379310345,
-      "mmlu_eval_accuracy_college_biology": 0.3125,
+      "mmlu_eval_accuracy_clinical_knowledge": 0.5862068965517241,
+      "mmlu_eval_accuracy_college_biology": 0.375,
       "mmlu_eval_accuracy_college_chemistry": 0.25,
       "mmlu_eval_accuracy_college_computer_science": 0.2727272727272727,
-      "mmlu_eval_accuracy_college_mathematics": 0.18181818181818182,
+      "mmlu_eval_accuracy_college_mathematics": 0.2727272727272727,
       "mmlu_eval_accuracy_college_medicine": 0.45454545454545453,
       "mmlu_eval_accuracy_college_physics": 0.45454545454545453,
       "mmlu_eval_accuracy_computer_security": 0.45454545454545453,
@@ -348,36 +348,36 @@
       "mmlu_eval_accuracy_formal_logic": 0.14285714285714285,
       "mmlu_eval_accuracy_global_facts": 0.4,
       "mmlu_eval_accuracy_high_school_biology": 0.375,
-      "mmlu_eval_accuracy_high_school_chemistry": 0.18181818181818182,
+      "mmlu_eval_accuracy_high_school_chemistry": 0.22727272727272727,
       "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
-      "mmlu_eval_accuracy_high_school_european_history": 0.6111111111111112,
+      "mmlu_eval_accuracy_high_school_european_history": 0.5555555555555556,
       "mmlu_eval_accuracy_high_school_geography": 0.9090909090909091,
-      "mmlu_eval_accuracy_high_school_government_and_politics": 0.47619047619047616,
-      "mmlu_eval_accuracy_high_school_macroeconomics": 0.46511627906976744,
+      "mmlu_eval_accuracy_high_school_government_and_politics": 0.5238095238095238,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.4883720930232558,
       "mmlu_eval_accuracy_high_school_mathematics": 0.2413793103448276,
       "mmlu_eval_accuracy_high_school_microeconomics": 0.34615384615384615,
-      "mmlu_eval_accuracy_high_school_physics": 0.23529411764705882,
-      "mmlu_eval_accuracy_high_school_psychology": 0.8833333333333333,
+      "mmlu_eval_accuracy_high_school_physics": 0.17647058823529413,
+      "mmlu_eval_accuracy_high_school_psychology": 0.8666666666666667,
       "mmlu_eval_accuracy_high_school_statistics": 0.391304347826087,
       "mmlu_eval_accuracy_high_school_us_history": 0.6363636363636364,
       "mmlu_eval_accuracy_high_school_world_history": 0.7692307692307693,
-      "mmlu_eval_accuracy_human_aging": 0.6956521739130435,
+      "mmlu_eval_accuracy_human_aging": 0.6521739130434783,
       "mmlu_eval_accuracy_human_sexuality": 0.3333333333333333,
       "mmlu_eval_accuracy_international_law": 0.8461538461538461,
-      "mmlu_eval_accuracy_jurisprudence": 0.36363636363636365,
+      "mmlu_eval_accuracy_jurisprudence": 0.2727272727272727,
       "mmlu_eval_accuracy_logical_fallacies": 0.6666666666666666,
-      "mmlu_eval_accuracy_machine_learning": 0.2727272727272727,
+      "mmlu_eval_accuracy_machine_learning": 0.18181818181818182,
       "mmlu_eval_accuracy_management": 0.6363636363636364,
       "mmlu_eval_accuracy_marketing": 0.84,
       "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
-      "mmlu_eval_accuracy_miscellaneous": 0.686046511627907,
-      "mmlu_eval_accuracy_moral_disputes": 0.47368421052631576,
-      "mmlu_eval_accuracy_moral_scenarios": 0.25,
-      "mmlu_eval_accuracy_nutrition": 0.6363636363636364,
-      "mmlu_eval_accuracy_philosophy": 0.47058823529411764,
-      "mmlu_eval_accuracy_prehistory": 0.4857142857142857,
-      "mmlu_eval_accuracy_professional_accounting": 0.3870967741935484,
-      "mmlu_eval_accuracy_professional_law": 0.3235294117647059,
+      "mmlu_eval_accuracy_miscellaneous": 0.6744186046511628,
+      "mmlu_eval_accuracy_moral_disputes": 0.42105263157894735,
+      "mmlu_eval_accuracy_moral_scenarios": 0.24,
+      "mmlu_eval_accuracy_nutrition": 0.6060606060606061,
+      "mmlu_eval_accuracy_philosophy": 0.4411764705882353,
+      "mmlu_eval_accuracy_prehistory": 0.5428571428571428,
+      "mmlu_eval_accuracy_professional_accounting": 0.3548387096774194,
+      "mmlu_eval_accuracy_professional_law": 0.32941176470588235,
       "mmlu_eval_accuracy_professional_medicine": 0.5161290322580645,
       "mmlu_eval_accuracy_professional_psychology": 0.5072463768115942,
       "mmlu_eval_accuracy_public_relations": 0.5833333333333334,
@@ -386,73 +386,73 @@
       "mmlu_eval_accuracy_us_foreign_policy": 0.6363636363636364,
       "mmlu_eval_accuracy_virology": 0.5555555555555556,
       "mmlu_eval_accuracy_world_religions": 0.7368421052631579,
-      "mmlu_loss": 1.2934161317644168,
+      "mmlu_loss": 1.2756395328682957,
       "step": 400
     },
     {
       "epoch": 0.15,
       "learning_rate": 0.0002,
-      "loss": 0.8409,
+      "loss": 0.8417,
       "step": 410
     },
     {
       "epoch": 0.15,
       "learning_rate": 0.0002,
-      "loss": 0.85,
+      "loss": 0.8508,
       "step": 420
     },
     {
       "epoch": 0.15,
       "learning_rate": 0.0002,
-      "loss": 0.8232,
+      "loss": 0.8227,
       "step": 430
     },
     {
       "epoch": 0.16,
       "learning_rate": 0.0002,
-      "loss": 0.9175,
+      "loss": 0.9174,
       "step": 440
     },
     {
       "epoch": 0.16,
       "learning_rate": 0.0002,
-      "loss": 0.8452,
+      "loss": 0.8457,
       "step": 450
     },
     {
       "epoch": 0.17,
       "learning_rate": 0.0002,
-      "loss": 0.8765,
+      "loss": 0.8773,
       "step": 460
     },
     {
       "epoch": 0.17,
       "learning_rate": 0.0002,
-      "loss": 0.8552,
+      "loss": 0.855,
       "step": 470
     },
     {
       "epoch": 0.17,
       "learning_rate": 0.0002,
-      "loss": 0.8562,
+      "loss": 0.8574,
       "step": 480
     },
     {
       "epoch": 0.18,
       "learning_rate": 0.0002,
-      "loss": 0.7962,
+      "loss": 0.7959,
       "step": 490
     },
     {
       "epoch": 0.18,
       "learning_rate": 0.0002,
-      "loss": 0.8113,
+      "loss": 0.8112,
       "step": 500
     },
     {
       "epoch": 0.18,
       "learning_rate": 0.0002,
-      "loss": 0.7964,
+      "loss": 0.7963,
       "step": 510
     },
     {
@@ -464,144 +464,144 @@
     {
       "epoch": 0.19,
       "learning_rate": 0.0002,
-      "loss": 0.8194,
+      "loss": 0.8192,
       "step": 530
     },
     {
       "epoch": 0.19,
       "learning_rate": 0.0002,
-      "loss": 0.8203,
+      "loss": 0.8217,
       "step": 540
     },
     {
       "epoch": 0.2,
       "learning_rate": 0.0002,
-      "loss": 0.8183,
+      "loss": 0.8181,
       "step": 550
     },
     {
       "epoch": 0.2,
       "learning_rate": 0.0002,
-      "loss": 0.8796,
+      "loss": 0.881,
       "step": 560
     },
     {
       "epoch": 0.2,
       "learning_rate": 0.0002,
-      "loss": 0.834,
+      "loss": 0.8344,
       "step": 570
     },
     {
       "epoch": 0.21,
       "learning_rate": 0.0002,
-      "loss": 0.8418,
+      "loss": 0.8425,
       "step": 580
     },
     {
       "epoch": 0.21,
       "learning_rate": 0.0002,
-      "loss": 0.7957,
+      "loss": 0.7964,
       "step": 590
     },
     {
       "epoch": 0.22,
       "learning_rate": 0.0002,
-      "loss": 0.7757,
+      "loss": 0.7759,
       "step": 600
     },
     {
       "epoch": 0.22,
-      "eval_loss": 0.8131685853004456,
-      "eval_runtime": 86.1888,
-      "eval_samples_per_second": 11.602,
-      "eval_steps_per_second": 5.801,
+      "eval_loss": 0.813377320766449,
+      "eval_runtime": 84.9789,
+      "eval_samples_per_second": 11.768,
+      "eval_steps_per_second": 5.884,
       "step": 600
     },
     {
       "epoch": 0.22,
-      "mmlu_eval_accuracy": 0.4782432986263529,
+      "mmlu_eval_accuracy": 0.480348815935918,
       "mmlu_eval_accuracy_abstract_algebra": 0.2727272727272727,
       "mmlu_eval_accuracy_anatomy": 0.7142857142857143,
       "mmlu_eval_accuracy_astronomy": 0.4375,
       "mmlu_eval_accuracy_business_ethics": 0.6363636363636364,
-      "mmlu_eval_accuracy_clinical_knowledge": 0.4827586206896552,
-      "mmlu_eval_accuracy_college_biology": 0.375,
+      "mmlu_eval_accuracy_clinical_knowledge": 0.5172413793103449,
+      "mmlu_eval_accuracy_college_biology": 0.4375,
       "mmlu_eval_accuracy_college_chemistry": 0.25,
       "mmlu_eval_accuracy_college_computer_science": 0.18181818181818182,
       "mmlu_eval_accuracy_college_mathematics": 0.2727272727272727,
-      "mmlu_eval_accuracy_college_medicine": 0.36363636363636365,
+      "mmlu_eval_accuracy_college_medicine": 0.4090909090909091,
       "mmlu_eval_accuracy_college_physics": 0.45454545454545453,
       "mmlu_eval_accuracy_computer_security": 0.45454545454545453,
-      "mmlu_eval_accuracy_conceptual_physics": 0.4230769230769231,
-      "mmlu_eval_accuracy_econometrics": 0.25,
+      "mmlu_eval_accuracy_conceptual_physics": 0.38461538461538464,
+      "mmlu_eval_accuracy_econometrics": 0.16666666666666666,
       "mmlu_eval_accuracy_electrical_engineering": 0.3125,
-      "mmlu_eval_accuracy_elementary_mathematics": 0.3170731707317073,
-      "mmlu_eval_accuracy_formal_logic": 0.21428571428571427,
+      "mmlu_eval_accuracy_elementary_mathematics": 0.34146341463414637,
+      "mmlu_eval_accuracy_formal_logic": 0.14285714285714285,
       "mmlu_eval_accuracy_global_facts": 0.4,
-      "mmlu_eval_accuracy_high_school_biology": 0.34375,
-      "mmlu_eval_accuracy_high_school_chemistry": 0.2727272727272727,
+      "mmlu_eval_accuracy_high_school_biology": 0.40625,
+      "mmlu_eval_accuracy_high_school_chemistry": 0.3181818181818182,
       "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
-      "mmlu_eval_accuracy_high_school_european_history": 0.5,
+      "mmlu_eval_accuracy_high_school_european_history": 0.5555555555555556,
       "mmlu_eval_accuracy_high_school_geography": 0.9090909090909091,
-      "mmlu_eval_accuracy_high_school_government_and_politics": 0.47619047619047616,
-      "mmlu_eval_accuracy_high_school_macroeconomics": 0.4418604651162791,
+      "mmlu_eval_accuracy_high_school_government_and_politics": 0.5238095238095238,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.4883720930232558,
       "mmlu_eval_accuracy_high_school_mathematics": 0.20689655172413793,
       "mmlu_eval_accuracy_high_school_microeconomics": 0.2692307692307692,
       "mmlu_eval_accuracy_high_school_physics": 0.11764705882352941,
       "mmlu_eval_accuracy_high_school_psychology": 0.8333333333333334,
-      "mmlu_eval_accuracy_high_school_statistics": 0.391304347826087,
-      "mmlu_eval_accuracy_high_school_us_history": 0.6363636363636364,
-      "mmlu_eval_accuracy_high_school_world_history": 0.7692307692307693,
+      "mmlu_eval_accuracy_high_school_statistics": 0.34782608695652173,
+      "mmlu_eval_accuracy_high_school_us_history": 0.5909090909090909,
+      "mmlu_eval_accuracy_high_school_world_history": 0.7307692307692307,
       "mmlu_eval_accuracy_human_aging": 0.6521739130434783,
       "mmlu_eval_accuracy_human_sexuality": 0.3333333333333333,
       "mmlu_eval_accuracy_international_law": 0.8461538461538461,
-      "mmlu_eval_accuracy_jurisprudence": 0.45454545454545453,
+      "mmlu_eval_accuracy_jurisprudence": 0.2727272727272727,
       "mmlu_eval_accuracy_logical_fallacies": 0.6666666666666666,
-      "mmlu_eval_accuracy_machine_learning": 0.18181818181818182,
-      "mmlu_eval_accuracy_management": 0.5454545454545454,
-      "mmlu_eval_accuracy_marketing": 0.84,
+      "mmlu_eval_accuracy_machine_learning": 0.2727272727272727,
+      "mmlu_eval_accuracy_management": 0.6363636363636364,
+      "mmlu_eval_accuracy_marketing": 0.8,
       "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
-      "mmlu_eval_accuracy_miscellaneous": 0.686046511627907,
+      "mmlu_eval_accuracy_miscellaneous": 0.6627906976744186,
       "mmlu_eval_accuracy_moral_disputes": 0.4473684210526316,
-      "mmlu_eval_accuracy_moral_scenarios": 0.25,
+      "mmlu_eval_accuracy_moral_scenarios": 0.24,
       "mmlu_eval_accuracy_nutrition": 0.6666666666666666,
       "mmlu_eval_accuracy_philosophy": 0.47058823529411764,
-      "mmlu_eval_accuracy_prehistory": 0.4,
+      "mmlu_eval_accuracy_prehistory": 0.45714285714285713,
       "mmlu_eval_accuracy_professional_accounting": 0.3548387096774194,
-      "mmlu_eval_accuracy_professional_law": 0.3235294117647059,
-      "mmlu_eval_accuracy_professional_medicine": 0.5483870967741935,
-      "mmlu_eval_accuracy_professional_psychology": 0.463768115942029,
+      "mmlu_eval_accuracy_professional_law": 0.3352941176470588,
+      "mmlu_eval_accuracy_professional_medicine": 0.5161290322580645,
+      "mmlu_eval_accuracy_professional_psychology": 0.4782608695652174,
       "mmlu_eval_accuracy_public_relations": 0.6666666666666666,
       "mmlu_eval_accuracy_security_studies": 0.4444444444444444,
       "mmlu_eval_accuracy_sociology": 0.6818181818181818,
-      "mmlu_eval_accuracy_us_foreign_policy": 0.5454545454545454,
+      "mmlu_eval_accuracy_us_foreign_policy": 0.6363636363636364,
       "mmlu_eval_accuracy_virology": 0.5555555555555556,
-      "mmlu_eval_accuracy_world_religions": 0.7894736842105263,
-      "mmlu_loss": 1.2197232340941542,
+      "mmlu_eval_accuracy_world_religions": 0.7368421052631579,
+      "mmlu_loss": 1.222284234145914,
       "step": 600
     },
     {
       "epoch": 0.22,
       "learning_rate": 0.0002,
-      "loss": 0.794,
+      "loss": 0.7938,
       "step": 610
     },
     {
       "epoch": 0.22,
       "learning_rate": 0.0002,
-      "loss": 0.782,
+      "loss": 0.7819,
       "step": 620
     },
     {
       "epoch": 0.23,
       "learning_rate": 0.0002,
-      "loss": 0.8509,
+      "loss": 0.8524,
       "step": 630
     },
     {
       "epoch": 0.23,
       "learning_rate": 0.0002,
-      "loss": 0.832,
+      "loss": 0.8319,
       "step": 640
     },
     {
@@ -619,292 +619,292 @@
     {
       "epoch": 0.24,
       "learning_rate": 0.0002,
-      "loss": 0.8392,
+      "loss": 0.8398,
       "step": 670
     },
     {
       "epoch": 0.24,
       "learning_rate": 0.0002,
-      "loss": 0.7434,
+      "loss": 0.7441,
       "step": 680
     },
     {
       "epoch": 0.25,
       "learning_rate": 0.0002,
-      "loss": 0.8529,
+      "loss": 0.8534,
       "step": 690
     },
     {
       "epoch": 0.25,
       "learning_rate": 0.0002,
-      "loss": 0.7871,
+      "loss": 0.7876,
       "step": 700
     },
     {
       "epoch": 0.25,
       "learning_rate": 0.0002,
-      "loss": 0.8532,
+      "loss": 0.8528,
       "step": 710
     },
     {
       "epoch": 0.26,
       "learning_rate": 0.0002,
-      "loss": 0.7918,
+      "loss": 0.7934,
       "step": 720
     },
     {
       "epoch": 0.26,
       "learning_rate": 0.0002,
-      "loss": 0.7966,
+      "loss": 0.7964,
       "step": 730
     },
     {
       "epoch": 0.27,
       "learning_rate": 0.0002,
-      "loss": 0.8118,
+      "loss": 0.8117,
       "step": 740
     },
     {
       "epoch": 0.27,
       "learning_rate": 0.0002,
-      "loss": 0.8539,
+      "loss": 0.8532,
       "step": 750
     },
     {
       "epoch": 0.27,
       "learning_rate": 0.0002,
-      "loss": 0.838,
+      "loss": 0.8373,
       "step": 760
     },
     {
       "epoch": 0.28,
       "learning_rate": 0.0002,
-      "loss": 0.831,
+      "loss": 0.8319,
       "step": 770
     },
     {
       "epoch": 0.28,
       "learning_rate": 0.0002,
-      "loss": 0.8491,
+      "loss": 0.8481,
       "step": 780
     },
     {
       "epoch": 0.28,
       "learning_rate": 0.0002,
-      "loss": 0.8056,
+      "loss": 0.8067,
       "step": 790
     },
     {
       "epoch": 0.29,
       "learning_rate": 0.0002,
-      "loss": 0.8124,
+      "loss": 0.8122,
       "step": 800
     },
     {
       "epoch": 0.29,
-      "eval_loss": 0.8050150871276855,
-      "eval_runtime": 86.2046,
-      "eval_samples_per_second": 11.6,
-      "eval_steps_per_second": 5.8,
+      "eval_loss": 0.8047721982002258,
+      "eval_runtime": 84.844,
+      "eval_samples_per_second": 11.786,
+      "eval_steps_per_second": 5.893,
       "step": 800
     },
     {
       "epoch": 0.29,
-      "mmlu_eval_accuracy": 0.5103451530390745,
-      "mmlu_eval_accuracy_abstract_algebra": 0.2727272727272727,
-      "mmlu_eval_accuracy_anatomy": 0.7142857142857143,
+      "mmlu_eval_accuracy": 0.5012897300189987,
+      "mmlu_eval_accuracy_abstract_algebra": 0.36363636363636365,
+      "mmlu_eval_accuracy_anatomy": 0.6428571428571429,
       "mmlu_eval_accuracy_astronomy": 0.4375,
-      "mmlu_eval_accuracy_business_ethics": 0.5454545454545454,
-      "mmlu_eval_accuracy_clinical_knowledge": 0.5172413793103449,
+      "mmlu_eval_accuracy_business_ethics": 0.6363636363636364,
+      "mmlu_eval_accuracy_clinical_knowledge": 0.5517241379310345,
       "mmlu_eval_accuracy_college_biology": 0.5,
       "mmlu_eval_accuracy_college_chemistry": 0.25,
       "mmlu_eval_accuracy_college_computer_science": 0.36363636363636365,
-      "mmlu_eval_accuracy_college_mathematics": 0.2727272727272727,
-      "mmlu_eval_accuracy_college_medicine": 0.45454545454545453,
-      "mmlu_eval_accuracy_college_physics": 0.45454545454545453,
-      "mmlu_eval_accuracy_computer_security": 0.6363636363636364,
+      "mmlu_eval_accuracy_college_mathematics": 0.18181818181818182,
+      "mmlu_eval_accuracy_college_medicine": 0.4090909090909091,
+      "mmlu_eval_accuracy_college_physics": 0.36363636363636365,
+      "mmlu_eval_accuracy_computer_security": 0.45454545454545453,
       "mmlu_eval_accuracy_conceptual_physics": 0.38461538461538464,
       "mmlu_eval_accuracy_econometrics": 0.16666666666666666,
       "mmlu_eval_accuracy_electrical_engineering": 0.3125,
       "mmlu_eval_accuracy_elementary_mathematics": 0.4146341463414634,
       "mmlu_eval_accuracy_formal_logic": 0.21428571428571427,
       "mmlu_eval_accuracy_global_facts": 0.5,
-      "mmlu_eval_accuracy_high_school_biology": 0.46875,
-      "mmlu_eval_accuracy_high_school_chemistry": 0.22727272727272727,
+      "mmlu_eval_accuracy_high_school_biology": 0.5,
+      "mmlu_eval_accuracy_high_school_chemistry": 0.3181818181818182,
       "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
-      "mmlu_eval_accuracy_high_school_european_history": 0.6666666666666666,
+      "mmlu_eval_accuracy_high_school_european_history": 0.6111111111111112,
       "mmlu_eval_accuracy_high_school_geography": 0.9090909090909091,
       "mmlu_eval_accuracy_high_school_government_and_politics": 0.5714285714285714,
-      "mmlu_eval_accuracy_high_school_macroeconomics": 0.4883720930232558,
-      "mmlu_eval_accuracy_high_school_mathematics": 0.2413793103448276,
-      "mmlu_eval_accuracy_high_school_microeconomics": 0.38461538461538464,
-      "mmlu_eval_accuracy_high_school_physics": 0.29411764705882354,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.46511627906976744,
+      "mmlu_eval_accuracy_high_school_mathematics": 0.27586206896551724,
+      "mmlu_eval_accuracy_high_school_microeconomics": 0.34615384615384615,
+      "mmlu_eval_accuracy_high_school_physics": 0.23529411764705882,
       "mmlu_eval_accuracy_high_school_psychology": 0.8833333333333333,
       "mmlu_eval_accuracy_high_school_statistics": 0.34782608695652173,
-      "mmlu_eval_accuracy_high_school_us_history": 0.6818181818181818,
+      "mmlu_eval_accuracy_high_school_us_history": 0.7272727272727273,
       "mmlu_eval_accuracy_high_school_world_history": 0.7307692307692307,
       "mmlu_eval_accuracy_human_aging": 0.6956521739130435,
       "mmlu_eval_accuracy_human_sexuality": 0.3333333333333333,
       "mmlu_eval_accuracy_international_law": 0.8461538461538461,
-      "mmlu_eval_accuracy_jurisprudence": 0.36363636363636365,
+      "mmlu_eval_accuracy_jurisprudence": 0.2727272727272727,
       "mmlu_eval_accuracy_logical_fallacies": 0.6666666666666666,
       "mmlu_eval_accuracy_machine_learning": 0.18181818181818182,
-      "mmlu_eval_accuracy_management": 0.6363636363636364,
-      "mmlu_eval_accuracy_marketing": 0.8,
+      "mmlu_eval_accuracy_management": 0.7272727272727273,
+      "mmlu_eval_accuracy_marketing": 0.84,
       "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
-      "mmlu_eval_accuracy_miscellaneous": 0.6627906976744186,
-      "mmlu_eval_accuracy_moral_disputes": 0.5,
+      "mmlu_eval_accuracy_miscellaneous": 0.6511627906976745,
+      "mmlu_eval_accuracy_moral_disputes": 0.4473684210526316,
       "mmlu_eval_accuracy_moral_scenarios": 0.23,
-      "mmlu_eval_accuracy_nutrition": 0.7575757575757576,
+      "mmlu_eval_accuracy_nutrition": 0.6666666666666666,
       "mmlu_eval_accuracy_philosophy": 0.5,
-      "mmlu_eval_accuracy_prehistory": 0.42857142857142855,
-      "mmlu_eval_accuracy_professional_accounting": 0.3225806451612903,
+      "mmlu_eval_accuracy_prehistory": 0.45714285714285713,
+      "mmlu_eval_accuracy_professional_accounting": 0.3870967741935484,
       "mmlu_eval_accuracy_professional_law": 0.32941176470588235,
       "mmlu_eval_accuracy_professional_medicine": 0.5483870967741935,
-      "mmlu_eval_accuracy_professional_psychology": 0.5217391304347826,
-      "mmlu_eval_accuracy_public_relations": 0.6666666666666666,
+      "mmlu_eval_accuracy_professional_psychology": 0.4927536231884058,
+      "mmlu_eval_accuracy_public_relations": 0.5833333333333334,
       "mmlu_eval_accuracy_security_studies": 0.48148148148148145,
       "mmlu_eval_accuracy_sociology": 0.6818181818181818,
-      "mmlu_eval_accuracy_us_foreign_policy": 0.8181818181818182,
+      "mmlu_eval_accuracy_us_foreign_policy": 0.7272727272727273,
       "mmlu_eval_accuracy_virology": 0.5555555555555556,
-      "mmlu_eval_accuracy_world_religions": 0.7894736842105263,
-      "mmlu_loss": 1.2832923761469266,
+      "mmlu_eval_accuracy_world_religions": 0.7368421052631579,
+      "mmlu_loss": 1.2707059693865614,
       "step": 800
     },
     {
       "epoch": 0.29,
       "learning_rate": 0.0002,
-      "loss": 0.8273,
+      "loss": 0.827,
       "step": 810
     },
     {
       "epoch": 0.29,
       "learning_rate": 0.0002,
-      "loss": 0.7679,
+      "loss": 0.7675,
       "step": 820
     },
     {
       "epoch": 0.3,
       "learning_rate": 0.0002,
-      "loss": 0.8327,
+      "loss": 0.8325,
       "step": 830
     },
     {
       "epoch": 0.3,
       "learning_rate": 0.0002,
-      "loss": 0.7997,
+      "loss": 0.7988,
       "step": 840
     },
     {
       "epoch": 0.3,
       "learning_rate": 0.0002,
-      "loss": 0.772,
+      "loss": 0.7721,
       "step": 850
     },
     {
       "epoch": 0.31,
       "learning_rate": 0.0002,
-      "loss": 0.8724,
+      "loss": 0.8722,
       "step": 860
     },
     {
       "epoch": 0.31,
       "learning_rate": 0.0002,
-      "loss": 0.8366,
+      "loss": 0.8378,
       "step": 870
     },
     {
       "epoch": 0.32,
       "learning_rate": 0.0002,
-      "loss": 0.8401,
+      "loss": 0.8396,
       "step": 880
     },
     {
       "epoch": 0.32,
       "learning_rate": 0.0002,
-      "loss": 0.8067,
+      "loss": 0.807,
       "step": 890
     },
     {
       "epoch": 0.32,
       "learning_rate": 0.0002,
-      "loss": 0.7934,
+      "loss": 0.7931,
       "step": 900
     },
     {
       "epoch": 0.33,
       "learning_rate": 0.0002,
-      "loss": 0.7952,
+      "loss": 0.7958,
       "step": 910
     },
     {
       "epoch": 0.33,
       "learning_rate": 0.0002,
-      "loss": 0.8114,
+      "loss": 0.8116,
       "step": 920
     },
     {
       "epoch": 0.33,
       "learning_rate": 0.0002,
-      "loss": 0.7641,
+      "loss": 0.7646,
       "step": 930
     },
     {
       "epoch": 0.34,
       "learning_rate": 0.0002,
-      "loss": 0.8185,
+      "loss": 0.8183,
       "step": 940
     },
     {
       "epoch": 0.34,
       "learning_rate": 0.0002,
-      "loss": 0.8528,
+      "loss": 0.8519,
       "step": 950
     },
     {
       "epoch": 0.34,
       "learning_rate": 0.0002,
-      "loss": 0.8234,
+      "loss": 0.8223,
       "step": 960
     },
     {
       "epoch": 0.35,
       "learning_rate": 0.0002,
-      "loss": 0.8706,
+      "loss": 0.8704,
       "step": 970
     },
     {
       "epoch": 0.35,
       "learning_rate": 0.0002,
-      "loss": 0.8514,
+      "loss": 0.8513,
       "step": 980
     },
     {
       "epoch": 0.36,
       "learning_rate": 0.0002,
-      "loss": 0.8284,
+      "loss": 0.8278,
       "step": 990
     },
     {
       "epoch": 0.36,
       "learning_rate": 0.0002,
-      "loss": 0.769,
+      "loss": 0.7686,
       "step": 1000
     },
     {
       "epoch": 0.36,
-      "eval_loss": 0.8012195229530334,
-      "eval_runtime": 86.5898,
-      "eval_samples_per_second": 11.549,
-      "eval_steps_per_second": 5.774,
+      "eval_loss": 0.8013286590576172,
+      "eval_runtime": 85.9261,
+      "eval_samples_per_second": 11.638,
+      "eval_steps_per_second": 5.819,
       "step": 1000
     },
     {
       "epoch": 0.36,
-      "mmlu_eval_accuracy": 0.49380788778326273,
-      "mmlu_eval_accuracy_abstract_algebra": 0.36363636363636365,
-      "mmlu_eval_accuracy_anatomy": 0.6428571428571429,
-      "mmlu_eval_accuracy_astronomy": 0.4375,
+      "mmlu_eval_accuracy": 0.4977313575021329,
+      "mmlu_eval_accuracy_abstract_algebra": 0.45454545454545453,
+      "mmlu_eval_accuracy_anatomy": 0.7142857142857143,
+      "mmlu_eval_accuracy_astronomy": 0.5,
       "mmlu_eval_accuracy_business_ethics": 0.6363636363636364,
       "mmlu_eval_accuracy_clinical_knowledge": 0.5172413793103449,
       "mmlu_eval_accuracy_college_biology": 0.5,
@@ -912,25 +912,25 @@
       "mmlu_eval_accuracy_college_computer_science": 0.09090909090909091,
       "mmlu_eval_accuracy_college_mathematics": 0.2727272727272727,
       "mmlu_eval_accuracy_college_medicine": 0.36363636363636365,
-      "mmlu_eval_accuracy_college_physics": 0.45454545454545453,
+      "mmlu_eval_accuracy_college_physics": 0.36363636363636365,
       "mmlu_eval_accuracy_computer_security": 0.6363636363636364,
-      "mmlu_eval_accuracy_conceptual_physics": 0.46153846153846156,
-      "mmlu_eval_accuracy_econometrics": 0.08333333333333333,
+      "mmlu_eval_accuracy_conceptual_physics": 0.4230769230769231,
+      "mmlu_eval_accuracy_econometrics": 0.16666666666666666,
       "mmlu_eval_accuracy_electrical_engineering": 0.3125,
-      "mmlu_eval_accuracy_elementary_mathematics": 0.3902439024390244,
+      "mmlu_eval_accuracy_elementary_mathematics": 0.36585365853658536,
       "mmlu_eval_accuracy_formal_logic": 0.14285714285714285,
       "mmlu_eval_accuracy_global_facts": 0.5,
       "mmlu_eval_accuracy_high_school_biology": 0.4375,
       "mmlu_eval_accuracy_high_school_chemistry": 0.3181818181818182,
       "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
-      "mmlu_eval_accuracy_high_school_european_history": 0.5555555555555556,
+      "mmlu_eval_accuracy_high_school_european_history": 0.6111111111111112,
       "mmlu_eval_accuracy_high_school_geography": 0.9090909090909091,
       "mmlu_eval_accuracy_high_school_government_and_politics": 0.5238095238095238,
       "mmlu_eval_accuracy_high_school_macroeconomics": 0.46511627906976744,
-      "mmlu_eval_accuracy_high_school_mathematics": 0.3103448275862069,
+      "mmlu_eval_accuracy_high_school_mathematics": 0.2413793103448276,
       "mmlu_eval_accuracy_high_school_microeconomics": 0.34615384615384615,
       "mmlu_eval_accuracy_high_school_physics": 0.35294117647058826,
-      "mmlu_eval_accuracy_high_school_psychology": 0.85,
+      "mmlu_eval_accuracy_high_school_psychology": 0.8333333333333334,
       "mmlu_eval_accuracy_high_school_statistics": 0.391304347826087,
       "mmlu_eval_accuracy_high_school_us_history": 0.7272727272727273,
       "mmlu_eval_accuracy_high_school_world_history": 0.6923076923076923,
@@ -939,39 +939,39 @@
       "mmlu_eval_accuracy_international_law": 0.8461538461538461,
       "mmlu_eval_accuracy_jurisprudence": 0.2727272727272727,
       "mmlu_eval_accuracy_logical_fallacies": 0.6666666666666666,
-      "mmlu_eval_accuracy_machine_learning": 0.18181818181818182,
+      "mmlu_eval_accuracy_machine_learning": 0.36363636363636365,
       "mmlu_eval_accuracy_management": 0.5454545454545454,
-      "mmlu_eval_accuracy_marketing": 0.8,
+      "mmlu_eval_accuracy_marketing": 0.84,
       "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
       "mmlu_eval_accuracy_miscellaneous": 0.6744186046511628,
       "mmlu_eval_accuracy_moral_disputes": 0.5,
-      "mmlu_eval_accuracy_moral_scenarios": 0.23,
-      "mmlu_eval_accuracy_nutrition": 0.5757575757575758,
-      "mmlu_eval_accuracy_philosophy": 0.5,
+      "mmlu_eval_accuracy_moral_scenarios": 0.25,
+      "mmlu_eval_accuracy_nutrition": 0.6060606060606061,
+      "mmlu_eval_accuracy_philosophy": 0.47058823529411764,
       "mmlu_eval_accuracy_prehistory": 0.5142857142857142,
       "mmlu_eval_accuracy_professional_accounting": 0.3870967741935484,
-      "mmlu_eval_accuracy_professional_law": 0.3176470588235294,
-      "mmlu_eval_accuracy_professional_medicine": 0.5483870967741935,
-      "mmlu_eval_accuracy_professional_psychology": 0.5217391304347826,
+      "mmlu_eval_accuracy_professional_law": 0.3058823529411765,
+      "mmlu_eval_accuracy_professional_medicine": 0.4838709677419355,
+      "mmlu_eval_accuracy_professional_psychology": 0.5072463768115942,
       "mmlu_eval_accuracy_public_relations": 0.5833333333333334,
       "mmlu_eval_accuracy_security_studies": 0.4444444444444444,
       "mmlu_eval_accuracy_sociology": 0.6818181818181818,
       "mmlu_eval_accuracy_us_foreign_policy": 0.6363636363636364,
       "mmlu_eval_accuracy_virology": 0.5,
-      "mmlu_eval_accuracy_world_religions": 0.7894736842105263,
-      "mmlu_loss": 1.1652030370954434,
+      "mmlu_eval_accuracy_world_religions": 0.7368421052631579,
+      "mmlu_loss": 1.1683200510223604,
       "step": 1000
     },
     {
       "epoch": 0.36,
       "learning_rate": 0.0002,
-      "loss": 0.7968,
+      "loss": 0.7971,
       "step": 1010
     },
     {
       "epoch": 0.37,
       "learning_rate": 0.0002,
-      "loss": 0.8135,
+      "loss": 0.8128,
       "step": 1020
     },
     {
@@ -983,19 +983,19 @@
     {
       "epoch": 0.37,
       "learning_rate": 0.0002,
-      "loss": 0.8529,
+      "loss": 0.8536,
       "step": 1040
     },
     {
       "epoch": 0.38,
       "learning_rate": 0.0002,
-      "loss": 0.7968,
+      "loss": 0.7972,
       "step": 1050
     },
     {
       "epoch": 0.38,
       "learning_rate": 0.0002,
-      "loss": 0.8278,
+      "loss": 0.8284,
       "step": 1060
     },
     {
@@ -1013,532 +1013,150 @@
     {
       "epoch": 0.39,
       "learning_rate": 0.0002,
-      "loss": 0.7467,
+      "loss": 0.7461,
       "step": 1090
     },
     {
       "epoch": 0.39,
       "learning_rate": 0.0002,
-      "loss": 0.7878,
+      "loss": 0.7883,
       "step": 1100
     },
     {
       "epoch": 0.4,
       "learning_rate": 0.0002,
-      "loss": 0.8489,
+      "loss": 0.8483,
       "step": 1110
     },
     {
       "epoch": 0.4,
       "learning_rate": 0.0002,
-      "loss": 0.8087,
+      "loss": 0.8089,
       "step": 1120
     },
     {
       "epoch": 0.41,
       "learning_rate": 0.0002,
-      "loss": 0.8551,
+      "loss": 0.8567,
       "step": 1130
     },
     {
       "epoch": 0.41,
       "learning_rate": 0.0002,
-      "loss": 0.8632,
+      "loss": 0.8631,
       "step": 1140
     },
     {
       "epoch": 0.41,
       "learning_rate": 0.0002,
-      "loss": 0.7958,
+      "loss": 0.7947,
       "step": 1150
     },
     {
       "epoch": 0.42,
       "learning_rate": 0.0002,
-      "loss": 0.8076,
+      "loss": 0.8064,
       "step": 1160
     },
     {
       "epoch": 0.42,
       "learning_rate": 0.0002,
-      "loss": 0.7789,
+      "loss": 0.7775,
       "step": 1170
     },
     {
       "epoch": 0.42,
       "learning_rate": 0.0002,
-      "loss": 0.8105,
+      "loss": 0.81,
       "step": 1180
     },
     {
       "epoch": 0.43,
       "learning_rate": 0.0002,
-      "loss": 0.8721,
+      "loss": 0.8713,
       "step": 1190
     },
     {
       "epoch": 0.43,
       "learning_rate": 0.0002,
-      "loss": 0.8661,
+      "loss": 0.8668,
       "step": 1200
     },
     {
       "epoch": 0.43,
-      "eval_loss": 0.7999836206436157,
-      "eval_runtime": 86.1812,
-      "eval_samples_per_second": 11.603,
-      "eval_steps_per_second": 5.802,
+      "eval_loss": 0.80042564868927,
+      "eval_runtime": 95.8631,
+      "eval_samples_per_second": 10.432,
+      "eval_steps_per_second": 5.216,
       "step": 1200
     },
     {
       "epoch": 0.43,
-      "mmlu_eval_accuracy": 0.5025193430991057,
+      "mmlu_eval_accuracy": 0.4987799817825415,
       "mmlu_eval_accuracy_abstract_algebra": 0.2727272727272727,
       "mmlu_eval_accuracy_anatomy": 0.6428571428571429,
-      "mmlu_eval_accuracy_astronomy": 0.375,
-      "mmlu_eval_accuracy_business_ethics": 0.7272727272727273,
+      "mmlu_eval_accuracy_astronomy": 0.5,
+      "mmlu_eval_accuracy_business_ethics": 0.6363636363636364,
       "mmlu_eval_accuracy_clinical_knowledge": 0.5172413793103449,
-      "mmlu_eval_accuracy_college_biology": 0.4375,
+      "mmlu_eval_accuracy_college_biology": 0.375,
       "mmlu_eval_accuracy_college_chemistry": 0.25,
       "mmlu_eval_accuracy_college_computer_science": 0.2727272727272727,
       "mmlu_eval_accuracy_college_mathematics": 0.2727272727272727,
-      "mmlu_eval_accuracy_college_medicine": 0.4090909090909091,
+      "mmlu_eval_accuracy_college_medicine": 0.3181818181818182,
       "mmlu_eval_accuracy_college_physics": 0.36363636363636365,
-      "mmlu_eval_accuracy_computer_security": 0.5454545454545454,
-      "mmlu_eval_accuracy_conceptual_physics": 0.38461538461538464,
+      "mmlu_eval_accuracy_computer_security": 0.36363636363636365,
+      "mmlu_eval_accuracy_conceptual_physics": 0.46153846153846156,
       "mmlu_eval_accuracy_econometrics": 0.08333333333333333,
       "mmlu_eval_accuracy_electrical_engineering": 0.375,
-      "mmlu_eval_accuracy_elementary_mathematics": 0.3902439024390244,
+      "mmlu_eval_accuracy_elementary_mathematics": 0.36585365853658536,
       "mmlu_eval_accuracy_formal_logic": 0.14285714285714285,
-      "mmlu_eval_accuracy_global_facts": 0.4,
-      "mmlu_eval_accuracy_high_school_biology": 0.46875,
-      "mmlu_eval_accuracy_high_school_chemistry": 0.36363636363636365,
+      "mmlu_eval_accuracy_global_facts": 0.5,
+      "mmlu_eval_accuracy_high_school_biology": 0.4375,
+      "mmlu_eval_accuracy_high_school_chemistry": 0.45454545454545453,
       "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
       "mmlu_eval_accuracy_high_school_european_history": 0.6111111111111112,
-      "mmlu_eval_accuracy_high_school_geography": 0.8636363636363636,
-      "mmlu_eval_accuracy_high_school_government_and_politics": 0.47619047619047616,
-      "mmlu_eval_accuracy_high_school_macroeconomics": 0.4883720930232558,
-      "mmlu_eval_accuracy_high_school_mathematics": 0.27586206896551724,
+      "mmlu_eval_accuracy_high_school_geography": 0.9090909090909091,
+      "mmlu_eval_accuracy_high_school_government_and_politics": 0.5714285714285714,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.46511627906976744,
+      "mmlu_eval_accuracy_high_school_mathematics": 0.20689655172413793,
       "mmlu_eval_accuracy_high_school_microeconomics": 0.46153846153846156,
-      "mmlu_eval_accuracy_high_school_physics": 0.29411764705882354,
-      "mmlu_eval_accuracy_high_school_psychology": 0.8666666666666667,
-      "mmlu_eval_accuracy_high_school_statistics": 0.43478260869565216,
-      "mmlu_eval_accuracy_high_school_us_history": 0.7272727272727273,
-      "mmlu_eval_accuracy_high_school_world_history": 0.7307692307692307,
+      "mmlu_eval_accuracy_high_school_physics": 0.23529411764705882,
+      "mmlu_eval_accuracy_high_school_psychology": 0.85,
+      "mmlu_eval_accuracy_high_school_statistics": 0.4782608695652174,
+      "mmlu_eval_accuracy_high_school_us_history": 0.7727272727272727,
+      "mmlu_eval_accuracy_high_school_world_history": 0.6923076923076923,
       "mmlu_eval_accuracy_human_aging": 0.6956521739130435,
-      "mmlu_eval_accuracy_human_sexuality": 0.3333333333333333,
+      "mmlu_eval_accuracy_human_sexuality": 0.4166666666666667,
       "mmlu_eval_accuracy_international_law": 0.8461538461538461,
-      "mmlu_eval_accuracy_jurisprudence": 0.36363636363636365,
+      "mmlu_eval_accuracy_jurisprudence": 0.2727272727272727,
       "mmlu_eval_accuracy_logical_fallacies": 0.6666666666666666,
-      "mmlu_eval_accuracy_machine_learning": 0.18181818181818182,
-      "mmlu_eval_accuracy_management": 0.6363636363636364,
-      "mmlu_eval_accuracy_marketing": 0.88,
+      "mmlu_eval_accuracy_machine_learning": 0.2727272727272727,
+      "mmlu_eval_accuracy_management": 0.7272727272727273,
+      "mmlu_eval_accuracy_marketing": 0.92,
       "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
-      "mmlu_eval_accuracy_miscellaneous": 0.7093023255813954,
+      "mmlu_eval_accuracy_miscellaneous": 0.6511627906976745,
       "mmlu_eval_accuracy_moral_disputes": 0.4473684210526316,
       "mmlu_eval_accuracy_moral_scenarios": 0.24,
-      "mmlu_eval_accuracy_nutrition": 0.6060606060606061,
-      "mmlu_eval_accuracy_philosophy": 0.5,
-      "mmlu_eval_accuracy_prehistory": 0.5714285714285714,
-      "mmlu_eval_accuracy_professional_accounting": 0.3548387096774194,
-      "mmlu_eval_accuracy_professional_law": 0.3352941176470588,
-      "mmlu_eval_accuracy_professional_medicine": 0.5161290322580645,
-      "mmlu_eval_accuracy_professional_psychology": 0.5362318840579711,
-      "mmlu_eval_accuracy_public_relations": 0.5,
-      "mmlu_eval_accuracy_security_studies": 0.48148148148148145,
-      "mmlu_eval_accuracy_sociology": 0.6363636363636364,
-      "mmlu_eval_accuracy_us_foreign_policy": 0.8181818181818182,
-      "mmlu_eval_accuracy_virology": 0.5555555555555556,
-      "mmlu_eval_accuracy_world_religions": 0.8421052631578947,
-      "mmlu_loss": 1.1953769516462445,
-      "step": 1200
-    },
-    {
-      "epoch": 0.43,
-      "learning_rate": 0.0002,
-      "loss": 0.8461,
-      "step": 1210
-    },
-    {
-      "epoch": 0.44,
-      "learning_rate": 0.0002,
-      "loss": 0.7927,
-      "step": 1220
-    },
-    {
-      "epoch": 0.44,
-      "learning_rate": 0.0002,
-      "loss": 0.8313,
-      "step": 1230
-    },
-    {
-      "epoch": 0.44,
-      "learning_rate": 0.0002,
-      "loss": 0.7469,
-      "step": 1240
-    },
-    {
-      "epoch": 0.45,
-      "learning_rate": 0.0002,
-      "loss": 0.833,
-      "step": 1250
-    },
-    {
-      "epoch": 0.45,
-      "learning_rate": 0.0002,
-      "loss": 0.7905,
-      "step": 1260
-    },
-    {
-      "epoch": 0.46,
-      "learning_rate": 0.0002,
-      "loss": 0.8191,
-      "step": 1270
-    },
-    {
-      "epoch": 0.46,
-      "learning_rate": 0.0002,
-      "loss": 0.8205,
-      "step": 1280
-    },
-    {
-      "epoch": 0.46,
-      "learning_rate": 0.0002,
-      "loss": 0.7915,
-      "step": 1290
-    },
-    {
-      "epoch": 0.47,
-      "learning_rate": 0.0002,
-      "loss": 0.8808,
-      "step": 1300
-    },
-    {
-      "epoch": 0.47,
-      "learning_rate": 0.0002,
-      "loss": 0.827,
-      "step": 1310
-    },
-    {
-      "epoch": 0.47,
-      "learning_rate": 0.0002,
-      "loss": 0.8082,
-      "step": 1320
-    },
-    {
-      "epoch": 0.48,
-      "learning_rate": 0.0002,
-      "loss": 0.7505,
-      "step": 1330
-    },
-    {
-      "epoch": 0.48,
-      "learning_rate": 0.0002,
-      "loss": 0.7884,
-      "step": 1340
-    },
-    {
-      "epoch": 0.48,
-      "learning_rate": 0.0002,
-      "loss": 0.7948,
-      "step": 1350
-    },
-    {
-      "epoch": 0.49,
-      "learning_rate": 0.0002,
-      "loss": 0.7794,
-      "step": 1360
-    },
-    {
-      "epoch": 0.49,
-      "learning_rate": 0.0002,
-      "loss": 0.8277,
-      "step": 1370
-    },
-    {
-      "epoch": 0.5,
-      "learning_rate": 0.0002,
-      "loss": 0.8266,
-      "step": 1380
-    },
-    {
-      "epoch": 0.5,
-      "learning_rate": 0.0002,
-      "loss": 0.772,
-      "step": 1390
-    },
-    {
-      "epoch": 0.5,
-      "learning_rate": 0.0002,
-      "loss": 0.8114,
-      "step": 1400
-    },
-    {
-      "epoch": 0.5,
-      "eval_loss": 0.7933924198150635,
-      "eval_runtime": 86.2259,
-      "eval_samples_per_second": 11.597,
-      "eval_steps_per_second": 5.799,
-      "step": 1400
-    },
-    {
-      "epoch": 0.5,
-      "mmlu_eval_accuracy": 0.49326789615638883,
-      "mmlu_eval_accuracy_abstract_algebra": 0.36363636363636365,
-      "mmlu_eval_accuracy_anatomy": 0.7142857142857143,
-      "mmlu_eval_accuracy_astronomy": 0.4375,
-      "mmlu_eval_accuracy_business_ethics": 0.6363636363636364,
-      "mmlu_eval_accuracy_clinical_knowledge": 0.4827586206896552,
-      "mmlu_eval_accuracy_college_biology": 0.5,
-      "mmlu_eval_accuracy_college_chemistry": 0.25,
-      "mmlu_eval_accuracy_college_computer_science": 0.2727272727272727,
-      "mmlu_eval_accuracy_college_mathematics": 0.2727272727272727,
-      "mmlu_eval_accuracy_college_medicine": 0.36363636363636365,
-      "mmlu_eval_accuracy_college_physics": 0.45454545454545453,
-      "mmlu_eval_accuracy_computer_security": 0.45454545454545453,
-      "mmlu_eval_accuracy_conceptual_physics": 0.38461538461538464,
-      "mmlu_eval_accuracy_econometrics": 0.08333333333333333,
-      "mmlu_eval_accuracy_electrical_engineering": 0.3125,
-      "mmlu_eval_accuracy_elementary_mathematics": 0.34146341463414637,
-      "mmlu_eval_accuracy_formal_logic": 0.14285714285714285,
-      "mmlu_eval_accuracy_global_facts": 0.4,
-      "mmlu_eval_accuracy_high_school_biology": 0.4375,
-      "mmlu_eval_accuracy_high_school_chemistry": 0.2727272727272727,
-      "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
-      "mmlu_eval_accuracy_high_school_european_history": 0.5555555555555556,
-      "mmlu_eval_accuracy_high_school_geography": 0.9090909090909091,
-      "mmlu_eval_accuracy_high_school_government_and_politics": 0.5238095238095238,
-      "mmlu_eval_accuracy_high_school_macroeconomics": 0.4418604651162791,
-      "mmlu_eval_accuracy_high_school_mathematics": 0.27586206896551724,
-      "mmlu_eval_accuracy_high_school_microeconomics": 0.38461538461538464,
-      "mmlu_eval_accuracy_high_school_physics": 0.29411764705882354,
-      "mmlu_eval_accuracy_high_school_psychology": 0.85,
-      "mmlu_eval_accuracy_high_school_statistics": 0.43478260869565216,
-      "mmlu_eval_accuracy_high_school_us_history": 0.6363636363636364,
-      "mmlu_eval_accuracy_high_school_world_history": 0.7307692307692307,
-      "mmlu_eval_accuracy_human_aging": 0.6956521739130435,
-      "mmlu_eval_accuracy_human_sexuality": 0.3333333333333333,
-      "mmlu_eval_accuracy_international_law": 0.8461538461538461,
-      "mmlu_eval_accuracy_jurisprudence": 0.36363636363636365,
-      "mmlu_eval_accuracy_logical_fallacies": 0.6666666666666666,
-      "mmlu_eval_accuracy_machine_learning": 0.18181818181818182,
-      "mmlu_eval_accuracy_management": 0.6363636363636364,
-      "mmlu_eval_accuracy_marketing": 0.84,
-      "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
-      "mmlu_eval_accuracy_miscellaneous": 0.6976744186046512,
-      "mmlu_eval_accuracy_moral_disputes": 0.39473684210526316,
-      "mmlu_eval_accuracy_moral_scenarios": 0.23,
       "mmlu_eval_accuracy_nutrition": 0.6363636363636364,
-      "mmlu_eval_accuracy_philosophy": 0.5,
+      "mmlu_eval_accuracy_philosophy": 0.5294117647058824,
       "mmlu_eval_accuracy_prehistory": 0.4857142857142857,
       "mmlu_eval_accuracy_professional_accounting": 0.3548387096774194,
-      "mmlu_eval_accuracy_professional_law": 0.3352941176470588,
-      "mmlu_eval_accuracy_professional_medicine": 0.5806451612903226,
-      "mmlu_eval_accuracy_professional_psychology": 0.5217391304347826,
-      "mmlu_eval_accuracy_public_relations": 0.5833333333333334,
-      "mmlu_eval_accuracy_security_studies": 0.4444444444444444,
-      "mmlu_eval_accuracy_sociology": 0.6363636363636364,
-      "mmlu_eval_accuracy_us_foreign_policy": 0.7272727272727273,
-      "mmlu_eval_accuracy_virology": 0.5555555555555556,
-      "mmlu_eval_accuracy_world_religions": 0.7894736842105263,
-      "mmlu_loss": 1.1721096763256014,
-      "step": 1400
-    },
-    {
-      "epoch": 0.51,
-      "learning_rate": 0.0002,
-      "loss": 0.7291,
-      "step": 1410
-    },
-    {
-      "epoch": 0.51,
-      "learning_rate": 0.0002,
-      "loss": 0.8356,
-      "step": 1420
-    },
-    {
-      "epoch": 0.51,
-      "learning_rate": 0.0002,
-      "loss": 0.7977,
-      "step": 1430
-    },
-    {
-      "epoch": 0.52,
-      "learning_rate": 0.0002,
-      "loss": 0.8235,
-      "step": 1440
-    },
-    {
-      "epoch": 0.52,
-      "learning_rate": 0.0002,
-      "loss": 0.8132,
-      "step": 1450
-    },
-    {
-      "epoch": 0.52,
-      "learning_rate": 0.0002,
-      "loss": 0.8331,
-      "step": 1460
-    },
-    {
-      "epoch": 0.53,
-      "learning_rate": 0.0002,
-      "loss": 0.8069,
-      "step": 1470
-    },
-    {
-      "epoch": 0.53,
-      "learning_rate": 0.0002,
-      "loss": 0.8033,
-      "step": 1480
-    },
-    {
-      "epoch": 0.53,
-      "learning_rate": 0.0002,
-      "loss": 0.8513,
-      "step": 1490
-    },
-    {
-      "epoch": 0.54,
-      "learning_rate": 0.0002,
-      "loss": 0.8387,
-      "step": 1500
-    },
-    {
-      "epoch": 0.54,
-      "learning_rate": 0.0002,
-      "loss": 0.8607,
-      "step": 1510
-    },
-    {
-      "epoch": 0.55,
-      "learning_rate": 0.0002,
-      "loss": 0.7881,
-      "step": 1520
-    },
-    {
-      "epoch": 0.55,
-      "learning_rate": 0.0002,
-      "loss": 0.8003,
-      "step": 1530
-    },
-    {
-      "epoch": 0.55,
-      "learning_rate": 0.0002,
-      "loss": 0.8315,
-      "step": 1540
-    },
-    {
-      "epoch": 0.56,
-      "learning_rate": 0.0002,
-      "loss": 0.802,
-      "step": 1550
-    },
-    {
-      "epoch": 0.56,
-      "learning_rate": 0.0002,
-      "loss": 0.8384,
-      "step": 1560
-    },
-    {
-      "epoch": 0.56,
-      "learning_rate": 0.0002,
-      "loss": 0.8424,
-      "step": 1570
-    },
-    {
-      "epoch": 0.57,
-      "learning_rate": 0.0002,
-      "loss": 0.8516,
-      "step": 1580
-    },
-    {
-      "epoch": 0.57,
-      "learning_rate": 0.0002,
-      "loss": 0.7782,
-      "step": 1590
-    },
-    {
-      "epoch": 0.57,
-      "learning_rate": 0.0002,
-      "loss": 0.7565,
-      "step": 1600
-    },
-    {
-      "epoch": 0.57,
-      "eval_loss": 0.7887665629386902,
-      "eval_runtime": 86.463,
-      "eval_samples_per_second": 11.566,
-      "eval_steps_per_second": 5.783,
-      "step": 1600
-    },
-    {
-      "epoch": 0.57,
-      "mmlu_eval_accuracy": 0.5010287763423048,
-      "mmlu_eval_accuracy_abstract_algebra": 0.2727272727272727,
-      "mmlu_eval_accuracy_anatomy": 0.7142857142857143,
-      "mmlu_eval_accuracy_astronomy": 0.375,
-      "mmlu_eval_accuracy_business_ethics": 0.6363636363636364,
-      "mmlu_eval_accuracy_clinical_knowledge": 0.5517241379310345,
-      "mmlu_eval_accuracy_college_biology": 0.4375,
-      "mmlu_eval_accuracy_college_chemistry": 0.125,
-      "mmlu_eval_accuracy_college_computer_science": 0.36363636363636365,
-      "mmlu_eval_accuracy_college_mathematics": 0.2727272727272727,
-      "mmlu_eval_accuracy_college_medicine": 0.36363636363636365,
-      "mmlu_eval_accuracy_college_physics": 0.36363636363636365,
-      "mmlu_eval_accuracy_computer_security": 0.7272727272727273,
-      "mmlu_eval_accuracy_conceptual_physics": 0.38461538461538464,
-      "mmlu_eval_accuracy_econometrics": 0.08333333333333333,
-      "mmlu_eval_accuracy_electrical_engineering": 0.25,
-      "mmlu_eval_accuracy_elementary_mathematics": 0.3170731707317073,
-      "mmlu_eval_accuracy_formal_logic": 0.14285714285714285,
-      "mmlu_eval_accuracy_global_facts": 0.6,
-      "mmlu_eval_accuracy_high_school_biology": 0.46875,
-      "mmlu_eval_accuracy_high_school_chemistry": 0.3181818181818182,
-      "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
-      "mmlu_eval_accuracy_high_school_european_history": 0.5555555555555556,
-      "mmlu_eval_accuracy_high_school_geography": 0.9090909090909091,
-      "mmlu_eval_accuracy_high_school_government_and_politics": 0.5714285714285714,
-      "mmlu_eval_accuracy_high_school_macroeconomics": 0.5116279069767442,
-      "mmlu_eval_accuracy_high_school_mathematics": 0.2413793103448276,
-      "mmlu_eval_accuracy_high_school_microeconomics": 0.4230769230769231,
-      "mmlu_eval_accuracy_high_school_physics": 0.29411764705882354,
-      "mmlu_eval_accuracy_high_school_psychology": 0.8666666666666667,
-      "mmlu_eval_accuracy_high_school_statistics": 0.391304347826087,
-      "mmlu_eval_accuracy_high_school_us_history": 0.6818181818181818,
-      "mmlu_eval_accuracy_high_school_world_history": 0.7307692307692307,
-      "mmlu_eval_accuracy_human_aging": 0.6956521739130435,
-      "mmlu_eval_accuracy_human_sexuality": 0.3333333333333333,
-      "mmlu_eval_accuracy_international_law": 0.8461538461538461,
-      "mmlu_eval_accuracy_jurisprudence": 0.36363636363636365,
-      "mmlu_eval_accuracy_logical_fallacies": 0.6111111111111112,
-      "mmlu_eval_accuracy_machine_learning": 0.2727272727272727,
-      "mmlu_eval_accuracy_management": 0.6363636363636364,
-      "mmlu_eval_accuracy_marketing": 0.8,
-      "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
-      "mmlu_eval_accuracy_miscellaneous": 0.6511627906976745,
-      "mmlu_eval_accuracy_moral_disputes": 0.5,
-      "mmlu_eval_accuracy_moral_scenarios": 0.25,
-      "mmlu_eval_accuracy_nutrition": 0.5757575757575758,
-      "mmlu_eval_accuracy_philosophy": 0.5,
-      "mmlu_eval_accuracy_prehistory": 0.5714285714285714,
-      "mmlu_eval_accuracy_professional_accounting": 0.3548387096774194,
-      "mmlu_eval_accuracy_professional_law": 0.34705882352941175,
-      "mmlu_eval_accuracy_professional_medicine": 0.5483870967741935,
+      "mmlu_eval_accuracy_professional_law": 0.3411764705882353,
+      "mmlu_eval_accuracy_professional_medicine": 0.5161290322580645,
       "mmlu_eval_accuracy_professional_psychology": 0.5072463768115942,
-      "mmlu_eval_accuracy_public_relations": 0.6666666666666666,
+      "mmlu_eval_accuracy_public_relations": 0.5,
       "mmlu_eval_accuracy_security_studies": 0.48148148148148145,
-      "mmlu_eval_accuracy_sociology": 0.7272727272727273,
+      "mmlu_eval_accuracy_sociology": 0.6363636363636364,
       "mmlu_eval_accuracy_us_foreign_policy": 0.7272727272727273,
       "mmlu_eval_accuracy_virology": 0.4444444444444444,
-      "mmlu_eval_accuracy_world_religions": 0.7368421052631579,
-      "mmlu_loss": 1.1859235374485544,
-      "step": 1600
+      "mmlu_eval_accuracy_world_religions": 0.7894736842105263,
+      "mmlu_loss": 1.187385533209258,
+      "step": 1200
     }
   ],
   "max_steps": 10000,
   "num_train_epochs": 4,
-  "total_flos": 3.646329192767816e+17,
+  "total_flos": 2.7140414738733466e+17,
   "trial_name": null,
   "trial_params": null
 }

{checkpoint-1600 → checkpoint-1200}/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9874cb44496df17f18895492102c07eca04527d6d0ba92aeb9ab28d30e98705
+oid sha256:37f30cc8dd13ad132e6c71cbb22244a93e85fa4af8c1de9d359ad42128019415
 size 5819