Training in progress, step 1200

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fb7664cdb4bdb1de19610370b6df3da4bad0188d9c894f4b3e73c45514ae1e94
+oid sha256:b1f1e1ae02cb84082fe5a905ba174a9d939662d9cd523692c297ae5bc49eeafd
 size 319977229

checkpoint-1000/adapter_model/adapter_model/README.md

@@ -4,6 +4,17 @@ library_name: peft
 ## Training procedure
 
 
+The following `bitsandbytes` quantization config was used during training:
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+
 The following `bitsandbytes` quantization config was used during training:
 - load_in_8bit: False
 - load_in_4bit: True
@@ -16,5 +27,6 @@ The following `bitsandbytes` quantization config was used during training:
 - bnb_4bit_compute_dtype: bfloat16
 ### Framework versions
 
+- PEFT 0.4.0
 
 - PEFT 0.4.0

checkpoint-1000/adapter_model/adapter_model/adapter_config.json

@@ -14,13 +14,13 @@
   "r": 64,
   "revision": null,
   "target_modules": [
-    "k_proj",
-    "down_proj",
-    "o_proj",
-    "q_proj",
     "gate_proj",
     "up_proj",
-    "v_proj"
+    "o_proj",
+    "k_proj",
+    "v_proj",
+    "q_proj",
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

checkpoint-1000/adapter_model/adapter_model/adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a09d6408717cd7c0e780aa2b40e411422fb4512dd63969746100a440e8fd278b
+oid sha256:fb7664cdb4bdb1de19610370b6df3da4bad0188d9c894f4b3e73c45514ae1e94
 size 319977229

checkpoint-1200/README.md

@@ -4,6 +4,17 @@ library_name: peft
 ## Training procedure
 
 
+The following `bitsandbytes` quantization config was used during training:
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+
 The following `bitsandbytes` quantization config was used during training:
 - load_in_8bit: False
 - load_in_4bit: True
@@ -16,5 +27,6 @@ The following `bitsandbytes` quantization config was used during training:
 - bnb_4bit_compute_dtype: bfloat16
 ### Framework versions
 
+- PEFT 0.4.0
 
 - PEFT 0.4.0

checkpoint-1200/adapter_config.json

@@ -14,13 +14,13 @@
   "r": 64,
   "revision": null,
   "target_modules": [
-    "k_proj",
-    "down_proj",
-    "o_proj",
-    "q_proj",
     "gate_proj",
     "up_proj",
-    "v_proj"
+    "o_proj",
+    "k_proj",
+    "v_proj",
+    "q_proj",
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

checkpoint-1200/adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:53281bd7b359d3310f6ea45b1405573e0a5bf41ac9e9df912712d706c2adbe8f
+oid sha256:b1f1e1ae02cb84082fe5a905ba174a9d939662d9cd523692c297ae5bc49eeafd
 size 319977229

checkpoint-1200/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:132020a224fa642353e067cb493b6e07251279e2b23c3c5623f42419cfb29ea4
+oid sha256:7ce2466a7b129804640ea6e434a09e42f063df3cb804cfd7907d3dccfc746c04
 size 1279539973

checkpoint-1200/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b854c80065ddeab76c591a58f42c610c30efe83b1827be6da4e15487258b664
+oid sha256:22fc77f8085917eda3dd706d6d181c99b20e60e5f2c720f00f0a54e35df9ecd7
 size 14511

checkpoint-1200/trainer_state.json

@@ -1,5 +1,5 @@
 {
-  "best_metric": 0.7396283149719238,
+  "best_metric": 0.7392138838768005,
   "best_model_checkpoint": "experts/expert-26/checkpoint-1200",
   "epoch": 0.32132815637970275,
   "global_step": 1200,
@@ -10,25 +10,25 @@
     {
       "epoch": 0.0,
       "learning_rate": 0.0002,
-      "loss": 0.8329,
+      "loss": 0.8327,
       "step": 10
     },
     {
       "epoch": 0.01,
       "learning_rate": 0.0002,
-      "loss": 0.8048,
+      "loss": 0.805,
       "step": 20
     },
     {
       "epoch": 0.01,
       "learning_rate": 0.0002,
-      "loss": 0.8108,
+      "loss": 0.811,
       "step": 30
     },
     {
       "epoch": 0.01,
       "learning_rate": 0.0002,
-      "loss": 0.7864,
+      "loss": 0.7862,
       "step": 40
     },
     {
@@ -40,13 +40,13 @@
     {
       "epoch": 0.02,
       "learning_rate": 0.0002,
-      "loss": 0.7645,
+      "loss": 0.7647,
       "step": 60
     },
     {
       "epoch": 0.02,
       "learning_rate": 0.0002,
-      "loss": 0.757,
+      "loss": 0.7572,
       "step": 70
     },
     {
@@ -58,25 +58,25 @@
     {
       "epoch": 0.02,
       "learning_rate": 0.0002,
-      "loss": 0.7777,
+      "loss": 0.7775,
       "step": 90
     },
     {
       "epoch": 0.03,
       "learning_rate": 0.0002,
-      "loss": 0.7482,
+      "loss": 0.7483,
       "step": 100
     },
     {
       "epoch": 0.03,
       "learning_rate": 0.0002,
-      "loss": 0.7402,
+      "loss": 0.7404,
       "step": 110
     },
     {
       "epoch": 0.03,
       "learning_rate": 0.0002,
-      "loss": 0.7442,
+      "loss": 0.7444,
       "step": 120
     },
     {
@@ -94,83 +94,83 @@
     {
       "epoch": 0.04,
       "learning_rate": 0.0002,
-      "loss": 0.7632,
+      "loss": 0.763,
       "step": 150
     },
     {
       "epoch": 0.04,
       "learning_rate": 0.0002,
-      "loss": 0.7882,
+      "loss": 0.7881,
       "step": 160
     },
     {
       "epoch": 0.05,
       "learning_rate": 0.0002,
-      "loss": 0.7759,
+      "loss": 0.7757,
       "step": 170
     },
     {
       "epoch": 0.05,
       "learning_rate": 0.0002,
-      "loss": 0.8138,
+      "loss": 0.8139,
       "step": 180
     },
     {
       "epoch": 0.05,
       "learning_rate": 0.0002,
-      "loss": 0.7874,
+      "loss": 0.7873,
       "step": 190
     },
     {
       "epoch": 0.05,
       "learning_rate": 0.0002,
-      "loss": 0.7547,
+      "loss": 0.7542,
       "step": 200
     },
     {
       "epoch": 0.05,
-      "eval_loss": 0.7692526578903198,
-      "eval_runtime": 85.9008,
-      "eval_samples_per_second": 11.641,
-      "eval_steps_per_second": 5.821,
+      "eval_loss": 0.7692113518714905,
+      "eval_runtime": 63.4219,
+      "eval_samples_per_second": 15.767,
+      "eval_steps_per_second": 7.884,
       "step": 200
     },
     {
       "epoch": 0.05,
-      "mmlu_eval_accuracy": 0.4937716133877423,
+      "mmlu_eval_accuracy": 0.49597563739434053,
       "mmlu_eval_accuracy_abstract_algebra": 0.2727272727272727,
-      "mmlu_eval_accuracy_anatomy": 0.6428571428571429,
+      "mmlu_eval_accuracy_anatomy": 0.5714285714285714,
       "mmlu_eval_accuracy_astronomy": 0.4375,
-      "mmlu_eval_accuracy_business_ethics": 0.6363636363636364,
+      "mmlu_eval_accuracy_business_ethics": 0.5454545454545454,
       "mmlu_eval_accuracy_clinical_knowledge": 0.5862068965517241,
       "mmlu_eval_accuracy_college_biology": 0.5,
       "mmlu_eval_accuracy_college_chemistry": 0.25,
       "mmlu_eval_accuracy_college_computer_science": 0.36363636363636365,
       "mmlu_eval_accuracy_college_mathematics": 0.18181818181818182,
-      "mmlu_eval_accuracy_college_medicine": 0.4090909090909091,
+      "mmlu_eval_accuracy_college_medicine": 0.45454545454545453,
       "mmlu_eval_accuracy_college_physics": 0.36363636363636365,
-      "mmlu_eval_accuracy_computer_security": 0.36363636363636365,
-      "mmlu_eval_accuracy_conceptual_physics": 0.4230769230769231,
+      "mmlu_eval_accuracy_computer_security": 0.45454545454545453,
+      "mmlu_eval_accuracy_conceptual_physics": 0.38461538461538464,
       "mmlu_eval_accuracy_econometrics": 0.16666666666666666,
       "mmlu_eval_accuracy_electrical_engineering": 0.25,
       "mmlu_eval_accuracy_elementary_mathematics": 0.36585365853658536,
-      "mmlu_eval_accuracy_formal_logic": 0.14285714285714285,
+      "mmlu_eval_accuracy_formal_logic": 0.21428571428571427,
       "mmlu_eval_accuracy_global_facts": 0.6,
       "mmlu_eval_accuracy_high_school_biology": 0.375,
-      "mmlu_eval_accuracy_high_school_chemistry": 0.2727272727272727,
+      "mmlu_eval_accuracy_high_school_chemistry": 0.22727272727272727,
       "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
       "mmlu_eval_accuracy_high_school_european_history": 0.6111111111111112,
       "mmlu_eval_accuracy_high_school_geography": 0.9090909090909091,
       "mmlu_eval_accuracy_high_school_government_and_politics": 0.5238095238095238,
       "mmlu_eval_accuracy_high_school_macroeconomics": 0.4418604651162791,
       "mmlu_eval_accuracy_high_school_mathematics": 0.2413793103448276,
-      "mmlu_eval_accuracy_high_school_microeconomics": 0.4230769230769231,
+      "mmlu_eval_accuracy_high_school_microeconomics": 0.46153846153846156,
       "mmlu_eval_accuracy_high_school_physics": 0.17647058823529413,
-      "mmlu_eval_accuracy_high_school_psychology": 0.8666666666666667,
-      "mmlu_eval_accuracy_high_school_statistics": 0.43478260869565216,
-      "mmlu_eval_accuracy_high_school_us_history": 0.6363636363636364,
+      "mmlu_eval_accuracy_high_school_psychology": 0.85,
+      "mmlu_eval_accuracy_high_school_statistics": 0.391304347826087,
+      "mmlu_eval_accuracy_high_school_us_history": 0.6818181818181818,
       "mmlu_eval_accuracy_high_school_world_history": 0.7307692307692307,
-      "mmlu_eval_accuracy_human_aging": 0.6521739130434783,
+      "mmlu_eval_accuracy_human_aging": 0.6956521739130435,
       "mmlu_eval_accuracy_human_sexuality": 0.3333333333333333,
       "mmlu_eval_accuracy_international_law": 0.8461538461538461,
       "mmlu_eval_accuracy_jurisprudence": 0.36363636363636365,
@@ -180,100 +180,100 @@
       "mmlu_eval_accuracy_marketing": 0.92,
       "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
       "mmlu_eval_accuracy_miscellaneous": 0.6627906976744186,
-      "mmlu_eval_accuracy_moral_disputes": 0.42105263157894735,
-      "mmlu_eval_accuracy_moral_scenarios": 0.23,
-      "mmlu_eval_accuracy_nutrition": 0.6060606060606061,
+      "mmlu_eval_accuracy_moral_disputes": 0.39473684210526316,
+      "mmlu_eval_accuracy_moral_scenarios": 0.21,
+      "mmlu_eval_accuracy_nutrition": 0.6363636363636364,
       "mmlu_eval_accuracy_philosophy": 0.4411764705882353,
-      "mmlu_eval_accuracy_prehistory": 0.4857142857142857,
+      "mmlu_eval_accuracy_prehistory": 0.5428571428571428,
       "mmlu_eval_accuracy_professional_accounting": 0.3870967741935484,
       "mmlu_eval_accuracy_professional_law": 0.3235294117647059,
-      "mmlu_eval_accuracy_professional_medicine": 0.5161290322580645,
-      "mmlu_eval_accuracy_professional_psychology": 0.4927536231884058,
+      "mmlu_eval_accuracy_professional_medicine": 0.6129032258064516,
+      "mmlu_eval_accuracy_professional_psychology": 0.5072463768115942,
       "mmlu_eval_accuracy_public_relations": 0.6666666666666666,
       "mmlu_eval_accuracy_security_studies": 0.5185185185185185,
       "mmlu_eval_accuracy_sociology": 0.7272727272727273,
       "mmlu_eval_accuracy_us_foreign_policy": 0.7272727272727273,
-      "mmlu_eval_accuracy_virology": 0.5,
+      "mmlu_eval_accuracy_virology": 0.4444444444444444,
       "mmlu_eval_accuracy_world_religions": 0.6842105263157895,
-      "mmlu_loss": 1.2817495326382684,
+      "mmlu_loss": 1.286679525755093,
       "step": 200
     },
     {
       "epoch": 0.06,
       "learning_rate": 0.0002,
-      "loss": 0.8076,
+      "loss": 0.808,
       "step": 210
     },
     {
       "epoch": 0.06,
       "learning_rate": 0.0002,
-      "loss": 0.7546,
+      "loss": 0.7551,
       "step": 220
     },
     {
       "epoch": 0.06,
       "learning_rate": 0.0002,
-      "loss": 0.7894,
+      "loss": 0.7897,
       "step": 230
     },
     {
       "epoch": 0.06,
       "learning_rate": 0.0002,
-      "loss": 0.752,
+      "loss": 0.7519,
       "step": 240
     },
     {
       "epoch": 0.07,
       "learning_rate": 0.0002,
-      "loss": 0.7753,
+      "loss": 0.7755,
       "step": 250
     },
     {
       "epoch": 0.07,
       "learning_rate": 0.0002,
-      "loss": 0.7783,
+      "loss": 0.7785,
       "step": 260
     },
     {
       "epoch": 0.07,
       "learning_rate": 0.0002,
-      "loss": 0.7539,
+      "loss": 0.7537,
       "step": 270
     },
     {
       "epoch": 0.07,
       "learning_rate": 0.0002,
-      "loss": 0.7535,
+      "loss": 0.7534,
       "step": 280
     },
     {
       "epoch": 0.08,
       "learning_rate": 0.0002,
-      "loss": 0.7611,
+      "loss": 0.7615,
       "step": 290
     },
     {
       "epoch": 0.08,
       "learning_rate": 0.0002,
-      "loss": 0.7891,
+      "loss": 0.7902,
       "step": 300
     },
     {
       "epoch": 0.08,
       "learning_rate": 0.0002,
-      "loss": 0.7626,
+      "loss": 0.7624,
       "step": 310
     },
     {
       "epoch": 0.09,
       "learning_rate": 0.0002,
-      "loss": 0.7727,
+      "loss": 0.7725,
       "step": 320
     },
     {
       "epoch": 0.09,
       "learning_rate": 0.0002,
-      "loss": 0.7811,
+      "loss": 0.7815,
       "step": 330
     },
     {
@@ -297,13 +297,13 @@
     {
       "epoch": 0.1,
       "learning_rate": 0.0002,
-      "loss": 0.7269,
+      "loss": 0.7273,
       "step": 370
     },
     {
       "epoch": 0.1,
       "learning_rate": 0.0002,
-      "loss": 0.7491,
+      "loss": 0.7492,
       "step": 380
     },
     {
@@ -315,33 +315,33 @@
     {
       "epoch": 0.11,
       "learning_rate": 0.0002,
-      "loss": 0.7839,
+      "loss": 0.7833,
       "step": 400
     },
     {
       "epoch": 0.11,
-      "eval_loss": 0.7609446048736572,
-      "eval_runtime": 88.0049,
-      "eval_samples_per_second": 11.363,
-      "eval_steps_per_second": 5.682,
+      "eval_loss": 0.761160135269165,
+      "eval_runtime": 63.2385,
+      "eval_samples_per_second": 15.813,
+      "eval_steps_per_second": 7.907,
       "step": 400
     },
     {
       "epoch": 0.11,
-      "mmlu_eval_accuracy": 0.4932457176529807,
+      "mmlu_eval_accuracy": 0.5014046123130873,
       "mmlu_eval_accuracy_abstract_algebra": 0.2727272727272727,
       "mmlu_eval_accuracy_anatomy": 0.6428571428571429,
       "mmlu_eval_accuracy_astronomy": 0.4375,
       "mmlu_eval_accuracy_business_ethics": 0.5454545454545454,
       "mmlu_eval_accuracy_clinical_knowledge": 0.5517241379310345,
-      "mmlu_eval_accuracy_college_biology": 0.375,
+      "mmlu_eval_accuracy_college_biology": 0.4375,
       "mmlu_eval_accuracy_college_chemistry": 0.25,
       "mmlu_eval_accuracy_college_computer_science": 0.36363636363636365,
-      "mmlu_eval_accuracy_college_mathematics": 0.2727272727272727,
+      "mmlu_eval_accuracy_college_mathematics": 0.36363636363636365,
       "mmlu_eval_accuracy_college_medicine": 0.45454545454545453,
       "mmlu_eval_accuracy_college_physics": 0.45454545454545453,
       "mmlu_eval_accuracy_computer_security": 0.45454545454545453,
-      "mmlu_eval_accuracy_conceptual_physics": 0.4230769230769231,
+      "mmlu_eval_accuracy_conceptual_physics": 0.38461538461538464,
       "mmlu_eval_accuracy_econometrics": 0.16666666666666666,
       "mmlu_eval_accuracy_electrical_engineering": 0.3125,
       "mmlu_eval_accuracy_elementary_mathematics": 0.36585365853658536,
@@ -353,10 +353,10 @@
       "mmlu_eval_accuracy_high_school_european_history": 0.6111111111111112,
       "mmlu_eval_accuracy_high_school_geography": 0.9090909090909091,
       "mmlu_eval_accuracy_high_school_government_and_politics": 0.47619047619047616,
-      "mmlu_eval_accuracy_high_school_macroeconomics": 0.4418604651162791,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.46511627906976744,
       "mmlu_eval_accuracy_high_school_mathematics": 0.2413793103448276,
-      "mmlu_eval_accuracy_high_school_microeconomics": 0.38461538461538464,
-      "mmlu_eval_accuracy_high_school_physics": 0.17647058823529413,
+      "mmlu_eval_accuracy_high_school_microeconomics": 0.4230769230769231,
+      "mmlu_eval_accuracy_high_school_physics": 0.23529411764705882,
       "mmlu_eval_accuracy_high_school_psychology": 0.85,
       "mmlu_eval_accuracy_high_school_statistics": 0.43478260869565216,
       "mmlu_eval_accuracy_high_school_us_history": 0.5909090909090909,
@@ -370,29 +370,29 @@
       "mmlu_eval_accuracy_management": 0.6363636363636364,
       "mmlu_eval_accuracy_marketing": 0.88,
       "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
-      "mmlu_eval_accuracy_miscellaneous": 0.6511627906976745,
-      "mmlu_eval_accuracy_moral_disputes": 0.39473684210526316,
-      "mmlu_eval_accuracy_moral_scenarios": 0.24,
+      "mmlu_eval_accuracy_miscellaneous": 0.6627906976744186,
+      "mmlu_eval_accuracy_moral_disputes": 0.47368421052631576,
+      "mmlu_eval_accuracy_moral_scenarios": 0.23,
       "mmlu_eval_accuracy_nutrition": 0.6060606060606061,
       "mmlu_eval_accuracy_philosophy": 0.5,
-      "mmlu_eval_accuracy_prehistory": 0.4857142857142857,
-      "mmlu_eval_accuracy_professional_accounting": 0.3870967741935484,
-      "mmlu_eval_accuracy_professional_law": 0.3176470588235294,
-      "mmlu_eval_accuracy_professional_medicine": 0.4838709677419355,
-      "mmlu_eval_accuracy_professional_psychology": 0.5217391304347826,
+      "mmlu_eval_accuracy_prehistory": 0.5142857142857142,
+      "mmlu_eval_accuracy_professional_accounting": 0.41935483870967744,
+      "mmlu_eval_accuracy_professional_law": 0.3235294117647059,
+      "mmlu_eval_accuracy_professional_medicine": 0.5806451612903226,
+      "mmlu_eval_accuracy_professional_psychology": 0.5072463768115942,
       "mmlu_eval_accuracy_public_relations": 0.6666666666666666,
       "mmlu_eval_accuracy_security_studies": 0.5185185185185185,
       "mmlu_eval_accuracy_sociology": 0.6818181818181818,
       "mmlu_eval_accuracy_us_foreign_policy": 0.7272727272727273,
       "mmlu_eval_accuracy_virology": 0.5555555555555556,
       "mmlu_eval_accuracy_world_religions": 0.7368421052631579,
-      "mmlu_loss": 1.2718629106762516,
+      "mmlu_loss": 1.2976421791252828,
       "step": 400
     },
     {
       "epoch": 0.11,
       "learning_rate": 0.0002,
-      "loss": 0.7264,
+      "loss": 0.7266,
       "step": 410
     },
     {
@@ -404,85 +404,85 @@
     {
       "epoch": 0.12,
       "learning_rate": 0.0002,
-      "loss": 0.7225,
+      "loss": 0.7222,
       "step": 430
     },
     {
       "epoch": 0.12,
       "learning_rate": 0.0002,
-      "loss": 0.8026,
+      "loss": 0.8021,
       "step": 440
     },
     {
       "epoch": 0.12,
       "learning_rate": 0.0002,
-      "loss": 0.7473,
+      "loss": 0.747,
       "step": 450
     },
     {
       "epoch": 0.12,
       "learning_rate": 0.0002,
-      "loss": 0.7612,
+      "loss": 0.7616,
       "step": 460
     },
     {
       "epoch": 0.13,
       "learning_rate": 0.0002,
-      "loss": 0.8163,
+      "loss": 0.8169,
       "step": 470
     },
     {
       "epoch": 0.13,
       "learning_rate": 0.0002,
-      "loss": 0.7303,
+      "loss": 0.7304,
       "step": 480
     },
     {
       "epoch": 0.13,
       "learning_rate": 0.0002,
-      "loss": 0.7613,
+      "loss": 0.7616,
       "step": 490
     },
     {
       "epoch": 0.13,
       "learning_rate": 0.0002,
-      "loss": 0.7489,
+      "loss": 0.749,
       "step": 500
     },
     {
       "epoch": 0.14,
       "learning_rate": 0.0002,
-      "loss": 0.8101,
+      "loss": 0.8105,
       "step": 510
     },
     {
       "epoch": 0.14,
       "learning_rate": 0.0002,
-      "loss": 0.7452,
+      "loss": 0.7454,
       "step": 520
     },
     {
       "epoch": 0.14,
       "learning_rate": 0.0002,
-      "loss": 0.7951,
+      "loss": 0.7948,
       "step": 530
     },
     {
       "epoch": 0.14,
       "learning_rate": 0.0002,
-      "loss": 0.7857,
+      "loss": 0.7861,
       "step": 540
     },
     {
       "epoch": 0.15,
       "learning_rate": 0.0002,
-      "loss": 0.7716,
+      "loss": 0.7717,
       "step": 550
     },
     {
       "epoch": 0.15,
       "learning_rate": 0.0002,
-      "loss": 0.6853,
+      "loss": 0.6854,
       "step": 560
     },
     {
@@ -494,32 +494,32 @@
     {
       "epoch": 0.16,
       "learning_rate": 0.0002,
-      "loss": 0.7382,
+      "loss": 0.7377,
       "step": 580
     },
     {
       "epoch": 0.16,
       "learning_rate": 0.0002,
-      "loss": 0.76,
+      "loss": 0.7596,
       "step": 590
     },
     {
       "epoch": 0.16,
       "learning_rate": 0.0002,
-      "loss": 0.7022,
+      "loss": 0.7014,
       "step": 600
     },
     {
       "epoch": 0.16,
-      "eval_loss": 0.7534194588661194,
-      "eval_runtime": 85.945,
-      "eval_samples_per_second": 11.635,
-      "eval_steps_per_second": 5.818,
+      "eval_loss": 0.7531090974807739,
+      "eval_runtime": 63.4198,
+      "eval_samples_per_second": 15.768,
+      "eval_steps_per_second": 7.884,
       "step": 600
     },
     {
       "epoch": 0.16,
-      "mmlu_eval_accuracy": 0.4933727197010448,
+      "mmlu_eval_accuracy": 0.49973241736655605,
       "mmlu_eval_accuracy_abstract_algebra": 0.36363636363636365,
       "mmlu_eval_accuracy_anatomy": 0.5714285714285714,
       "mmlu_eval_accuracy_astronomy": 0.4375,
@@ -528,28 +528,28 @@
       "mmlu_eval_accuracy_college_biology": 0.375,
       "mmlu_eval_accuracy_college_chemistry": 0.25,
       "mmlu_eval_accuracy_college_computer_science": 0.36363636363636365,
-      "mmlu_eval_accuracy_college_mathematics": 0.18181818181818182,
-      "mmlu_eval_accuracy_college_medicine": 0.45454545454545453,
+      "mmlu_eval_accuracy_college_mathematics": 0.2727272727272727,
+      "mmlu_eval_accuracy_college_medicine": 0.5,
       "mmlu_eval_accuracy_college_physics": 0.36363636363636365,
-      "mmlu_eval_accuracy_computer_security": 0.5454545454545454,
+      "mmlu_eval_accuracy_computer_security": 0.45454545454545453,
       "mmlu_eval_accuracy_conceptual_physics": 0.38461538461538464,
       "mmlu_eval_accuracy_econometrics": 0.16666666666666666,
-      "mmlu_eval_accuracy_electrical_engineering": 0.375,
-      "mmlu_eval_accuracy_elementary_mathematics": 0.34146341463414637,
+      "mmlu_eval_accuracy_electrical_engineering": 0.3125,
+      "mmlu_eval_accuracy_elementary_mathematics": 0.36585365853658536,
       "mmlu_eval_accuracy_formal_logic": 0.14285714285714285,
       "mmlu_eval_accuracy_global_facts": 0.6,
       "mmlu_eval_accuracy_high_school_biology": 0.46875,
-      "mmlu_eval_accuracy_high_school_chemistry": 0.22727272727272727,
+      "mmlu_eval_accuracy_high_school_chemistry": 0.2727272727272727,
       "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
       "mmlu_eval_accuracy_high_school_european_history": 0.6666666666666666,
       "mmlu_eval_accuracy_high_school_geography": 0.9090909090909091,
-      "mmlu_eval_accuracy_high_school_government_and_politics": 0.5238095238095238,
+      "mmlu_eval_accuracy_high_school_government_and_politics": 0.47619047619047616,
       "mmlu_eval_accuracy_high_school_macroeconomics": 0.46511627906976744,
-      "mmlu_eval_accuracy_high_school_mathematics": 0.2413793103448276,
-      "mmlu_eval_accuracy_high_school_microeconomics": 0.34615384615384615,
+      "mmlu_eval_accuracy_high_school_mathematics": 0.27586206896551724,
+      "mmlu_eval_accuracy_high_school_microeconomics": 0.46153846153846156,
       "mmlu_eval_accuracy_high_school_physics": 0.23529411764705882,
       "mmlu_eval_accuracy_high_school_psychology": 0.85,
-      "mmlu_eval_accuracy_high_school_statistics": 0.391304347826087,
+      "mmlu_eval_accuracy_high_school_statistics": 0.43478260869565216,
       "mmlu_eval_accuracy_high_school_us_history": 0.6363636363636364,
       "mmlu_eval_accuracy_high_school_world_history": 0.6923076923076923,
       "mmlu_eval_accuracy_human_aging": 0.6956521739130435,
@@ -561,65 +561,65 @@
       "mmlu_eval_accuracy_management": 0.6363636363636364,
       "mmlu_eval_accuracy_marketing": 0.8,
       "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
-      "mmlu_eval_accuracy_miscellaneous": 0.6395348837209303,
-      "mmlu_eval_accuracy_moral_disputes": 0.39473684210526316,
-      "mmlu_eval_accuracy_moral_scenarios": 0.2,
-      "mmlu_eval_accuracy_nutrition": 0.6363636363636364,
-      "mmlu_eval_accuracy_philosophy": 0.47058823529411764,
+      "mmlu_eval_accuracy_miscellaneous": 0.6511627906976745,
+      "mmlu_eval_accuracy_moral_disputes": 0.4473684210526316,
+      "mmlu_eval_accuracy_moral_scenarios": 0.23,
+      "mmlu_eval_accuracy_nutrition": 0.6060606060606061,
+      "mmlu_eval_accuracy_philosophy": 0.5,
       "mmlu_eval_accuracy_prehistory": 0.45714285714285713,
-      "mmlu_eval_accuracy_professional_accounting": 0.3870967741935484,
-      "mmlu_eval_accuracy_professional_law": 0.31176470588235294,
-      "mmlu_eval_accuracy_professional_medicine": 0.5483870967741935,
-      "mmlu_eval_accuracy_professional_psychology": 0.5072463768115942,
+      "mmlu_eval_accuracy_professional_accounting": 0.41935483870967744,
+      "mmlu_eval_accuracy_professional_law": 0.3176470588235294,
+      "mmlu_eval_accuracy_professional_medicine": 0.4838709677419355,
+      "mmlu_eval_accuracy_professional_psychology": 0.5217391304347826,
       "mmlu_eval_accuracy_public_relations": 0.6666666666666666,
-      "mmlu_eval_accuracy_security_studies": 0.48148148148148145,
-      "mmlu_eval_accuracy_sociology": 0.7272727272727273,
+      "mmlu_eval_accuracy_security_studies": 0.5185185185185185,
+      "mmlu_eval_accuracy_sociology": 0.7727272727272727,
       "mmlu_eval_accuracy_us_foreign_policy": 0.8181818181818182,
       "mmlu_eval_accuracy_virology": 0.5,
       "mmlu_eval_accuracy_world_religions": 0.6842105263157895,
-      "mmlu_loss": 1.3060517932042752,
+      "mmlu_loss": 1.3179996946228392,
       "step": 600
     },
     {
       "epoch": 0.16,
       "learning_rate": 0.0002,
-      "loss": 0.7923,
+      "loss": 0.7915,
       "step": 610
     },
     {
       "epoch": 0.17,
       "learning_rate": 0.0002,
-      "loss": 0.7836,
+      "loss": 0.7833,
       "step": 620
     },
     {
       "epoch": 0.17,
       "learning_rate": 0.0002,
-      "loss": 0.777,
+      "loss": 0.7772,
       "step": 630
     },
     {
       "epoch": 0.17,
       "learning_rate": 0.0002,
-      "loss": 0.7528,
+      "loss": 0.7525,
       "step": 640
     },
     {
       "epoch": 0.17,
       "learning_rate": 0.0002,
-      "loss": 0.7358,
+      "loss": 0.7355,
       "step": 650
     },
     {
       "epoch": 0.18,
       "learning_rate": 0.0002,
-      "loss": 0.7448,
+      "loss": 0.7457,
       "step": 660
     },
     {
       "epoch": 0.18,
       "learning_rate": 0.0002,
-      "loss": 0.7845,
+      "loss": 0.7847,
       "step": 670
     },
     {
@@ -631,92 +631,92 @@
     {
       "epoch": 0.18,
       "learning_rate": 0.0002,
-      "loss": 0.7694,
+      "loss": 0.7697,
       "step": 690
     },
     {
       "epoch": 0.19,
       "learning_rate": 0.0002,
-      "loss": 0.7486,
+      "loss": 0.748,
       "step": 700
     },
     {
       "epoch": 0.19,
       "learning_rate": 0.0002,
-      "loss": 0.721,
+      "loss": 0.7205,
       "step": 710
     },
     {
       "epoch": 0.19,
       "learning_rate": 0.0002,
-      "loss": 0.7643,
+      "loss": 0.7637,
       "step": 720
     },
     {
       "epoch": 0.2,
       "learning_rate": 0.0002,
-      "loss": 0.7494,
+      "loss": 0.7501,
       "step": 730
     },
     {
       "epoch": 0.2,
       "learning_rate": 0.0002,
-      "loss": 0.7535,
+      "loss": 0.7531,
       "step": 740
     },
     {
       "epoch": 0.2,
       "learning_rate": 0.0002,
-      "loss": 0.7585,
+      "loss": 0.7591,
       "step": 750
     },
     {
       "epoch": 0.2,
       "learning_rate": 0.0002,
-      "loss": 0.7737,
+      "loss": 0.7747,
       "step": 760
     },
     {
       "epoch": 0.21,
       "learning_rate": 0.0002,
-      "loss": 0.694,
+      "loss": 0.6932,
       "step": 770
     },
     {
       "epoch": 0.21,
       "learning_rate": 0.0002,
-      "loss": 0.7594,
+      "loss": 0.7595,
       "step": 780
     },
     {
       "epoch": 0.21,
       "learning_rate": 0.0002,
-      "loss": 0.8029,
+      "loss": 0.8035,
       "step": 790
     },
     {
       "epoch": 0.21,
       "learning_rate": 0.0002,
-      "loss": 0.7367,
+      "loss": 0.7357,
       "step": 800
     },
     {
       "epoch": 0.21,
-      "eval_loss": 0.7466574907302856,
-      "eval_runtime": 85.8456,
-      "eval_samples_per_second": 11.649,
-      "eval_steps_per_second": 5.824,
+      "eval_loss": 0.7464295625686646,
+      "eval_runtime": 63.1499,
+      "eval_samples_per_second": 15.835,
+      "eval_steps_per_second": 7.918,
       "step": 800
     },
     {
       "epoch": 0.21,
-      "mmlu_eval_accuracy": 0.498798552422504,
+      "mmlu_eval_accuracy": 0.495286882959602,
       "mmlu_eval_accuracy_abstract_algebra": 0.36363636363636365,
       "mmlu_eval_accuracy_anatomy": 0.6428571428571429,
       "mmlu_eval_accuracy_astronomy": 0.4375,
       "mmlu_eval_accuracy_business_ethics": 0.45454545454545453,
       "mmlu_eval_accuracy_clinical_knowledge": 0.5517241379310345,
-      "mmlu_eval_accuracy_college_biology": 0.375,
+      "mmlu_eval_accuracy_college_biology": 0.3125,
       "mmlu_eval_accuracy_college_chemistry": 0.375,
       "mmlu_eval_accuracy_college_computer_science": 0.36363636363636365,
       "mmlu_eval_accuracy_college_mathematics": 0.2727272727272727,
@@ -726,55 +726,55 @@
       "mmlu_eval_accuracy_conceptual_physics": 0.38461538461538464,
       "mmlu_eval_accuracy_econometrics": 0.16666666666666666,
       "mmlu_eval_accuracy_electrical_engineering": 0.3125,
-      "mmlu_eval_accuracy_elementary_mathematics": 0.3902439024390244,
+      "mmlu_eval_accuracy_elementary_mathematics": 0.36585365853658536,
       "mmlu_eval_accuracy_formal_logic": 0.21428571428571427,
       "mmlu_eval_accuracy_global_facts": 0.6,
-      "mmlu_eval_accuracy_high_school_biology": 0.5,
+      "mmlu_eval_accuracy_high_school_biology": 0.4375,
       "mmlu_eval_accuracy_high_school_chemistry": 0.18181818181818182,
       "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
-      "mmlu_eval_accuracy_high_school_european_history": 0.5555555555555556,
+      "mmlu_eval_accuracy_high_school_european_history": 0.6111111111111112,
       "mmlu_eval_accuracy_high_school_geography": 0.9090909090909091,
-      "mmlu_eval_accuracy_high_school_government_and_politics": 0.5238095238095238,
+      "mmlu_eval_accuracy_high_school_government_and_politics": 0.47619047619047616,
       "mmlu_eval_accuracy_high_school_macroeconomics": 0.4186046511627907,
       "mmlu_eval_accuracy_high_school_mathematics": 0.1724137931034483,
       "mmlu_eval_accuracy_high_school_microeconomics": 0.38461538461538464,
-      "mmlu_eval_accuracy_high_school_physics": 0.23529411764705882,
+      "mmlu_eval_accuracy_high_school_physics": 0.17647058823529413,
       "mmlu_eval_accuracy_high_school_psychology": 0.85,
       "mmlu_eval_accuracy_high_school_statistics": 0.34782608695652173,
       "mmlu_eval_accuracy_high_school_us_history": 0.6818181818181818,
       "mmlu_eval_accuracy_high_school_world_history": 0.7307692307692307,
-      "mmlu_eval_accuracy_human_aging": 0.6521739130434783,
+      "mmlu_eval_accuracy_human_aging": 0.6956521739130435,
       "mmlu_eval_accuracy_human_sexuality": 0.3333333333333333,
       "mmlu_eval_accuracy_international_law": 0.8461538461538461,
       "mmlu_eval_accuracy_jurisprudence": 0.36363636363636365,
       "mmlu_eval_accuracy_logical_fallacies": 0.6666666666666666,
-      "mmlu_eval_accuracy_machine_learning": 0.36363636363636365,
+      "mmlu_eval_accuracy_machine_learning": 0.2727272727272727,
       "mmlu_eval_accuracy_management": 0.6363636363636364,
-      "mmlu_eval_accuracy_marketing": 0.8,
+      "mmlu_eval_accuracy_marketing": 0.76,
       "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
-      "mmlu_eval_accuracy_miscellaneous": 0.6744186046511628,
-      "mmlu_eval_accuracy_moral_disputes": 0.47368421052631576,
-      "mmlu_eval_accuracy_moral_scenarios": 0.24,
-      "mmlu_eval_accuracy_nutrition": 0.6666666666666666,
-      "mmlu_eval_accuracy_philosophy": 0.47058823529411764,
-      "mmlu_eval_accuracy_prehistory": 0.5142857142857142,
-      "mmlu_eval_accuracy_professional_accounting": 0.2903225806451613,
-      "mmlu_eval_accuracy_professional_law": 0.35294117647058826,
+      "mmlu_eval_accuracy_miscellaneous": 0.6627906976744186,
+      "mmlu_eval_accuracy_moral_disputes": 0.5,
+      "mmlu_eval_accuracy_moral_scenarios": 0.22,
+      "mmlu_eval_accuracy_nutrition": 0.6060606060606061,
+      "mmlu_eval_accuracy_philosophy": 0.5294117647058824,
+      "mmlu_eval_accuracy_prehistory": 0.5428571428571428,
+      "mmlu_eval_accuracy_professional_accounting": 0.3548387096774194,
+      "mmlu_eval_accuracy_professional_law": 0.3235294117647059,
       "mmlu_eval_accuracy_professional_medicine": 0.5161290322580645,
-      "mmlu_eval_accuracy_professional_psychology": 0.5217391304347826,
+      "mmlu_eval_accuracy_professional_psychology": 0.5072463768115942,
       "mmlu_eval_accuracy_public_relations": 0.6666666666666666,
       "mmlu_eval_accuracy_security_studies": 0.5185185185185185,
-      "mmlu_eval_accuracy_sociology": 0.6818181818181818,
+      "mmlu_eval_accuracy_sociology": 0.7272727272727273,
       "mmlu_eval_accuracy_us_foreign_policy": 0.7272727272727273,
       "mmlu_eval_accuracy_virology": 0.5,
       "mmlu_eval_accuracy_world_religions": 0.6842105263157895,
-      "mmlu_loss": 1.241851667939216,
+      "mmlu_loss": 1.2400291113878354,
       "step": 800
     },
     {
       "epoch": 0.22,
       "learning_rate": 0.0002,
-      "loss": 0.7249,
+      "loss": 0.7251,
       "step": 810
     },
     {
@@ -786,43 +786,43 @@
     {
       "epoch": 0.22,
       "learning_rate": 0.0002,
-      "loss": 0.7475,
+      "loss": 0.7474,
       "step": 830
     },
     {
       "epoch": 0.22,
       "learning_rate": 0.0002,
-      "loss": 0.7169,
+      "loss": 0.717,
       "step": 840
     },
     {
       "epoch": 0.23,
       "learning_rate": 0.0002,
-      "loss": 0.7307,
+      "loss": 0.7302,
       "step": 850
     },
     {
       "epoch": 0.23,
       "learning_rate": 0.0002,
-      "loss": 0.8059,
+      "loss": 0.8058,
       "step": 860
     },
     {
       "epoch": 0.23,
       "learning_rate": 0.0002,
-      "loss": 0.7409,
+      "loss": 0.7398,
       "step": 870
     },
     {
       "epoch": 0.24,
       "learning_rate": 0.0002,
-      "loss": 0.687,
+      "loss": 0.6869,
       "step": 880
     },
     {
       "epoch": 0.24,
       "learning_rate": 0.0002,
-      "loss": 0.7102,
+      "loss": 0.7099,
       "step": 890
     },
     {
@@ -834,37 +834,37 @@
     {
       "epoch": 0.24,
       "learning_rate": 0.0002,
-      "loss": 0.7471,
+      "loss": 0.7466,
       "step": 910
     },
     {
       "epoch": 0.25,
       "learning_rate": 0.0002,
-      "loss": 0.7669,
+      "loss": 0.7674,
       "step": 920
     },
     {
       "epoch": 0.25,
       "learning_rate": 0.0002,
-      "loss": 0.7415,
+      "loss": 0.7412,
       "step": 930
     },
     {
       "epoch": 0.25,
       "learning_rate": 0.0002,
-      "loss": 0.746,
+      "loss": 0.7454,
       "step": 940
     },
     {
       "epoch": 0.25,
       "learning_rate": 0.0002,
-      "loss": 0.7358,
+      "loss": 0.7348,
       "step": 950
     },
     {
       "epoch": 0.26,
       "learning_rate": 0.0002,
-      "loss": 0.7259,
+      "loss": 0.7256,
       "step": 960
     },
     {
@@ -876,96 +876,96 @@
     {
       "epoch": 0.26,
       "learning_rate": 0.0002,
-      "loss": 0.718,
+      "loss": 0.7173,
       "step": 980
     },
     {
       "epoch": 0.27,
       "learning_rate": 0.0002,
-      "loss": 0.7875,
+      "loss": 0.7879,
       "step": 990
     },
     {
       "epoch": 0.27,
       "learning_rate": 0.0002,
-      "loss": 0.743,
+      "loss": 0.7422,
       "step": 1000
     },
     {
       "epoch": 0.27,
-      "eval_loss": 0.7425828576087952,
-      "eval_runtime": 85.9225,
-      "eval_samples_per_second": 11.638,
-      "eval_steps_per_second": 5.819,
+      "eval_loss": 0.7426952123641968,
+      "eval_runtime": 63.1658,
+      "eval_samples_per_second": 15.831,
+      "eval_steps_per_second": 7.916,
       "step": 1000
     },
     {
       "epoch": 0.27,
-      "mmlu_eval_accuracy": 0.5012939874200372,
+      "mmlu_eval_accuracy": 0.5043401109660032,
       "mmlu_eval_accuracy_abstract_algebra": 0.45454545454545453,
       "mmlu_eval_accuracy_anatomy": 0.6428571428571429,
       "mmlu_eval_accuracy_astronomy": 0.4375,
-      "mmlu_eval_accuracy_business_ethics": 0.45454545454545453,
+      "mmlu_eval_accuracy_business_ethics": 0.5454545454545454,
       "mmlu_eval_accuracy_clinical_knowledge": 0.5517241379310345,
       "mmlu_eval_accuracy_college_biology": 0.375,
       "mmlu_eval_accuracy_college_chemistry": 0.25,
       "mmlu_eval_accuracy_college_computer_science": 0.36363636363636365,
       "mmlu_eval_accuracy_college_mathematics": 0.2727272727272727,
-      "mmlu_eval_accuracy_college_medicine": 0.4090909090909091,
+      "mmlu_eval_accuracy_college_medicine": 0.5454545454545454,
       "mmlu_eval_accuracy_college_physics": 0.45454545454545453,
-      "mmlu_eval_accuracy_computer_security": 0.5454545454545454,
-      "mmlu_eval_accuracy_conceptual_physics": 0.46153846153846156,
+      "mmlu_eval_accuracy_computer_security": 0.45454545454545453,
+      "mmlu_eval_accuracy_conceptual_physics": 0.4230769230769231,
       "mmlu_eval_accuracy_econometrics": 0.16666666666666666,
-      "mmlu_eval_accuracy_electrical_engineering": 0.3125,
+      "mmlu_eval_accuracy_electrical_engineering": 0.25,
       "mmlu_eval_accuracy_elementary_mathematics": 0.36585365853658536,
       "mmlu_eval_accuracy_formal_logic": 0.14285714285714285,
-      "mmlu_eval_accuracy_global_facts": 0.6,
+      "mmlu_eval_accuracy_global_facts": 0.5,
       "mmlu_eval_accuracy_high_school_biology": 0.4375,
       "mmlu_eval_accuracy_high_school_chemistry": 0.22727272727272727,
       "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
       "mmlu_eval_accuracy_high_school_european_history": 0.6111111111111112,
       "mmlu_eval_accuracy_high_school_geography": 0.9090909090909091,
-      "mmlu_eval_accuracy_high_school_government_and_politics": 0.5714285714285714,
-      "mmlu_eval_accuracy_high_school_macroeconomics": 0.46511627906976744,
+      "mmlu_eval_accuracy_high_school_government_and_politics": 0.5238095238095238,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.4418604651162791,
       "mmlu_eval_accuracy_high_school_mathematics": 0.2413793103448276,
-      "mmlu_eval_accuracy_high_school_microeconomics": 0.34615384615384615,
+      "mmlu_eval_accuracy_high_school_microeconomics": 0.4230769230769231,
       "mmlu_eval_accuracy_high_school_physics": 0.29411764705882354,
-      "mmlu_eval_accuracy_high_school_psychology": 0.8333333333333334,
-      "mmlu_eval_accuracy_high_school_statistics": 0.391304347826087,
-      "mmlu_eval_accuracy_high_school_us_history": 0.6363636363636364,
-      "mmlu_eval_accuracy_high_school_world_history": 0.6923076923076923,
+      "mmlu_eval_accuracy_high_school_psychology": 0.8666666666666667,
+      "mmlu_eval_accuracy_high_school_statistics": 0.34782608695652173,
+      "mmlu_eval_accuracy_high_school_us_history": 0.6818181818181818,
+      "mmlu_eval_accuracy_high_school_world_history": 0.7307692307692307,
       "mmlu_eval_accuracy_human_aging": 0.6521739130434783,
       "mmlu_eval_accuracy_human_sexuality": 0.3333333333333333,
       "mmlu_eval_accuracy_international_law": 0.8461538461538461,
-      "mmlu_eval_accuracy_jurisprudence": 0.2727272727272727,
+      "mmlu_eval_accuracy_jurisprudence": 0.36363636363636365,
       "mmlu_eval_accuracy_logical_fallacies": 0.6666666666666666,
       "mmlu_eval_accuracy_machine_learning": 0.36363636363636365,
       "mmlu_eval_accuracy_management": 0.6363636363636364,
       "mmlu_eval_accuracy_marketing": 0.8,
       "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
       "mmlu_eval_accuracy_miscellaneous": 0.6511627906976745,
-      "mmlu_eval_accuracy_moral_disputes": 0.5263157894736842,
-      "mmlu_eval_accuracy_moral_scenarios": 0.24,
-      "mmlu_eval_accuracy_nutrition": 0.6666666666666666,
-      "mmlu_eval_accuracy_philosophy": 0.5,
-      "mmlu_eval_accuracy_prehistory": 0.45714285714285713,
+      "mmlu_eval_accuracy_moral_disputes": 0.5789473684210527,
+      "mmlu_eval_accuracy_moral_scenarios": 0.25,
+      "mmlu_eval_accuracy_nutrition": 0.6060606060606061,
+      "mmlu_eval_accuracy_philosophy": 0.5294117647058824,
+      "mmlu_eval_accuracy_prehistory": 0.4857142857142857,
       "mmlu_eval_accuracy_professional_accounting": 0.3870967741935484,
-      "mmlu_eval_accuracy_professional_law": 0.34705882352941175,
-      "mmlu_eval_accuracy_professional_medicine": 0.45161290322580644,
-      "mmlu_eval_accuracy_professional_psychology": 0.5072463768115942,
+      "mmlu_eval_accuracy_professional_law": 0.32941176470588235,
+      "mmlu_eval_accuracy_professional_medicine": 0.5161290322580645,
+      "mmlu_eval_accuracy_professional_psychology": 0.5217391304347826,
       "mmlu_eval_accuracy_public_relations": 0.6666666666666666,
-      "mmlu_eval_accuracy_security_studies": 0.4444444444444444,
+      "mmlu_eval_accuracy_security_studies": 0.48148148148148145,
       "mmlu_eval_accuracy_sociology": 0.6818181818181818,
-      "mmlu_eval_accuracy_us_foreign_policy": 0.9090909090909091,
+      "mmlu_eval_accuracy_us_foreign_policy": 0.8181818181818182,
       "mmlu_eval_accuracy_virology": 0.5,
       "mmlu_eval_accuracy_world_religions": 0.6842105263157895,
-      "mmlu_loss": 1.2747252732515335,
+      "mmlu_loss": 1.2847360125098466,
       "step": 1000
     },
     {
       "epoch": 0.27,
       "learning_rate": 0.0002,
-      "loss": 0.7516,
+      "loss": 0.7518,
       "step": 1010
     },
     {
@@ -977,31 +977,31 @@
     {
       "epoch": 0.28,
       "learning_rate": 0.0002,
-      "loss": 0.7499,
+      "loss": 0.7497,
       "step": 1030
     },
     {
       "epoch": 0.28,
       "learning_rate": 0.0002,
-      "loss": 0.7352,
+      "loss": 0.735,
       "step": 1040
     },
     {
       "epoch": 0.28,
       "learning_rate": 0.0002,
-      "loss": 0.7281,
+      "loss": 0.7286,
       "step": 1050
     },
     {
       "epoch": 0.28,
       "learning_rate": 0.0002,
-      "loss": 0.7377,
+      "loss": 0.7376,
       "step": 1060
     },
     {
       "epoch": 0.29,
       "learning_rate": 0.0002,
-      "loss": 0.7857,
+      "loss": 0.7866,
       "step": 1070
     },
     {
@@ -1013,144 +1013,144 @@
     {
       "epoch": 0.29,
       "learning_rate": 0.0002,
-      "loss": 0.7211,
+      "loss": 0.7203,
       "step": 1090
     },
     {
       "epoch": 0.29,
       "learning_rate": 0.0002,
-      "loss": 0.7421,
+      "loss": 0.7419,
       "step": 1100
     },
     {
       "epoch": 0.3,
       "learning_rate": 0.0002,
-      "loss": 0.728,
+      "loss": 0.7282,
       "step": 1110
     },
     {
       "epoch": 0.3,
       "learning_rate": 0.0002,
-      "loss": 0.7564,
+      "loss": 0.7553,
       "step": 1120
     },
     {
       "epoch": 0.3,
       "learning_rate": 0.0002,
-      "loss": 0.7881,
+      "loss": 0.7885,
       "step": 1130
     },
     {
       "epoch": 0.31,
       "learning_rate": 0.0002,
-      "loss": 0.7232,
+      "loss": 0.7235,
       "step": 1140
     },
     {
       "epoch": 0.31,
       "learning_rate": 0.0002,
-      "loss": 0.7658,
+      "loss": 0.7657,
       "step": 1150
     },
     {
       "epoch": 0.31,
       "learning_rate": 0.0002,
-      "loss": 0.7311,
+      "loss": 0.7316,
       "step": 1160
     },
     {
       "epoch": 0.31,
       "learning_rate": 0.0002,
-      "loss": 0.7483,
+      "loss": 0.7473,
       "step": 1170
     },
     {
       "epoch": 0.32,
       "learning_rate": 0.0002,
-      "loss": 0.7307,
+      "loss": 0.7308,
       "step": 1180
     },
     {
       "epoch": 0.32,
       "learning_rate": 0.0002,
-      "loss": 0.7922,
+      "loss": 0.792,
       "step": 1190
     },
     {
       "epoch": 0.32,
       "learning_rate": 0.0002,
-      "loss": 0.7801,
+      "loss": 0.7802,
       "step": 1200
     },
     {
       "epoch": 0.32,
-      "eval_loss": 0.7396283149719238,
-      "eval_runtime": 85.8478,
-      "eval_samples_per_second": 11.649,
-      "eval_steps_per_second": 5.824,
+      "eval_loss": 0.7392138838768005,
+      "eval_runtime": 63.204,
+      "eval_samples_per_second": 15.822,
+      "eval_steps_per_second": 7.911,
       "step": 1200
     },
     {
       "epoch": 0.32,
-      "mmlu_eval_accuracy": 0.4963015119699251,
+      "mmlu_eval_accuracy": 0.49640006132716064,
       "mmlu_eval_accuracy_abstract_algebra": 0.2727272727272727,
       "mmlu_eval_accuracy_anatomy": 0.6428571428571429,
       "mmlu_eval_accuracy_astronomy": 0.4375,
-      "mmlu_eval_accuracy_business_ethics": 0.6363636363636364,
-      "mmlu_eval_accuracy_clinical_knowledge": 0.5172413793103449,
-      "mmlu_eval_accuracy_college_biology": 0.375,
+      "mmlu_eval_accuracy_business_ethics": 0.5454545454545454,
+      "mmlu_eval_accuracy_clinical_knowledge": 0.5862068965517241,
+      "mmlu_eval_accuracy_college_biology": 0.3125,
       "mmlu_eval_accuracy_college_chemistry": 0.25,
       "mmlu_eval_accuracy_college_computer_science": 0.45454545454545453,
       "mmlu_eval_accuracy_college_mathematics": 0.18181818181818182,
       "mmlu_eval_accuracy_college_medicine": 0.5,
       "mmlu_eval_accuracy_college_physics": 0.45454545454545453,
       "mmlu_eval_accuracy_computer_security": 0.45454545454545453,
-      "mmlu_eval_accuracy_conceptual_physics": 0.34615384615384615,
+      "mmlu_eval_accuracy_conceptual_physics": 0.38461538461538464,
       "mmlu_eval_accuracy_econometrics": 0.16666666666666666,
       "mmlu_eval_accuracy_electrical_engineering": 0.3125,
-      "mmlu_eval_accuracy_elementary_mathematics": 0.36585365853658536,
+      "mmlu_eval_accuracy_elementary_mathematics": 0.3902439024390244,
       "mmlu_eval_accuracy_formal_logic": 0.21428571428571427,
-      "mmlu_eval_accuracy_global_facts": 0.4,
+      "mmlu_eval_accuracy_global_facts": 0.5,
       "mmlu_eval_accuracy_high_school_biology": 0.28125,
       "mmlu_eval_accuracy_high_school_chemistry": 0.22727272727272727,
       "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
-      "mmlu_eval_accuracy_high_school_european_history": 0.6111111111111112,
-      "mmlu_eval_accuracy_high_school_geography": 0.8636363636363636,
+      "mmlu_eval_accuracy_high_school_european_history": 0.6666666666666666,
+      "mmlu_eval_accuracy_high_school_geography": 0.9090909090909091,
       "mmlu_eval_accuracy_high_school_government_and_politics": 0.5714285714285714,
-      "mmlu_eval_accuracy_high_school_macroeconomics": 0.4883720930232558,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.46511627906976744,
       "mmlu_eval_accuracy_high_school_mathematics": 0.2413793103448276,
       "mmlu_eval_accuracy_high_school_microeconomics": 0.38461538461538464,
       "mmlu_eval_accuracy_high_school_physics": 0.29411764705882354,
-      "mmlu_eval_accuracy_high_school_psychology": 0.8333333333333334,
+      "mmlu_eval_accuracy_high_school_psychology": 0.85,
       "mmlu_eval_accuracy_high_school_statistics": 0.391304347826087,
-      "mmlu_eval_accuracy_high_school_us_history": 0.6363636363636364,
+      "mmlu_eval_accuracy_high_school_us_history": 0.6818181818181818,
       "mmlu_eval_accuracy_high_school_world_history": 0.7307692307692307,
-      "mmlu_eval_accuracy_human_aging": 0.7391304347826086,
-      "mmlu_eval_accuracy_human_sexuality": 0.4166666666666667,
+      "mmlu_eval_accuracy_human_aging": 0.782608695652174,
+      "mmlu_eval_accuracy_human_sexuality": 0.3333333333333333,
       "mmlu_eval_accuracy_international_law": 0.8461538461538461,
       "mmlu_eval_accuracy_jurisprudence": 0.45454545454545453,
       "mmlu_eval_accuracy_logical_fallacies": 0.6666666666666666,
       "mmlu_eval_accuracy_machine_learning": 0.18181818181818182,
       "mmlu_eval_accuracy_management": 0.6363636363636364,
       "mmlu_eval_accuracy_marketing": 0.8,
-      "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
-      "mmlu_eval_accuracy_miscellaneous": 0.6744186046511628,
+      "mmlu_eval_accuracy_medical_genetics": 0.8181818181818182,
+      "mmlu_eval_accuracy_miscellaneous": 0.6511627906976745,
       "mmlu_eval_accuracy_moral_disputes": 0.5,
-      "mmlu_eval_accuracy_moral_scenarios": 0.22,
+      "mmlu_eval_accuracy_moral_scenarios": 0.27,
       "mmlu_eval_accuracy_nutrition": 0.5757575757575758,
-      "mmlu_eval_accuracy_philosophy": 0.4411764705882353,
-      "mmlu_eval_accuracy_prehistory": 0.4,
+      "mmlu_eval_accuracy_philosophy": 0.47058823529411764,
+      "mmlu_eval_accuracy_prehistory": 0.42857142857142855,
       "mmlu_eval_accuracy_professional_accounting": 0.45161290322580644,
-      "mmlu_eval_accuracy_professional_law": 0.3235294117647059,
-      "mmlu_eval_accuracy_professional_medicine": 0.5483870967741935,
+      "mmlu_eval_accuracy_professional_law": 0.3176470588235294,
+      "mmlu_eval_accuracy_professional_medicine": 0.45161290322580644,
       "mmlu_eval_accuracy_professional_psychology": 0.5072463768115942,
       "mmlu_eval_accuracy_public_relations": 0.6666666666666666,
-      "mmlu_eval_accuracy_security_studies": 0.4444444444444444,
-      "mmlu_eval_accuracy_sociology": 0.7727272727272727,
-      "mmlu_eval_accuracy_us_foreign_policy": 0.7272727272727273,
-      "mmlu_eval_accuracy_virology": 0.5555555555555556,
+      "mmlu_eval_accuracy_security_studies": 0.48148148148148145,
+      "mmlu_eval_accuracy_sociology": 0.6363636363636364,
+      "mmlu_eval_accuracy_us_foreign_policy": 0.8181818181818182,
+      "mmlu_eval_accuracy_virology": 0.5,
       "mmlu_eval_accuracy_world_religions": 0.7368421052631579,
-      "mmlu_loss": 1.3978832787384252,
+      "mmlu_loss": 1.384888773954257,
       "step": 1200
     }
   ],

checkpoint-1200/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1fdce65f27e1b27599ffadb916e306ae3c8008257aa650aa283cc44ffd79ab85
+oid sha256:fe4dd9a1fee85d1d6fa31030cdb57a7342525f029464f99168fb3a0bf75117ad
 size 5819