Training in progress, step 400

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6342fa32a534d427fda4a77c749de84133e553afd6fecd7274f137aa3e7d6ff7
+oid sha256:72d4d013e57515ff7938e3d88a61add2c0ad78f8e0893368e575e639de61bc0a
 size 319977229

checkpoint-200/adapter_model/adapter_model/README.md

@@ -4,6 +4,17 @@ library_name: peft
 ## Training procedure
 
 
+The following `bitsandbytes` quantization config was used during training:
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+
 The following `bitsandbytes` quantization config was used during training:
 - load_in_8bit: False
 - load_in_4bit: True
@@ -16,5 +27,6 @@ The following `bitsandbytes` quantization config was used during training:
 - bnb_4bit_compute_dtype: bfloat16
 ### Framework versions
 
+- PEFT 0.4.0
 
 - PEFT 0.4.0

checkpoint-200/adapter_model/adapter_model/adapter_config.json

@@ -15,11 +15,11 @@
   "revision": null,
   "target_modules": [
     "up_proj",
-    "down_proj",
-    "k_proj",
     "q_proj",
+    "k_proj",
     "gate_proj",
     "v_proj",
+    "down_proj",
     "o_proj"
   ],
   "task_type": "CAUSAL_LM"

checkpoint-200/adapter_model/adapter_model/adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6865aa6f8fa180a3e27dee19c155a2eebea142a7b162caa6cd4fb9ace1c13c56
+oid sha256:6342fa32a534d427fda4a77c749de84133e553afd6fecd7274f137aa3e7d6ff7
 size 319977229

checkpoint-400/README.md

@@ -4,6 +4,17 @@ library_name: peft
 ## Training procedure
 
 
+The following `bitsandbytes` quantization config was used during training:
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+
 The following `bitsandbytes` quantization config was used during training:
 - load_in_8bit: False
 - load_in_4bit: True
@@ -16,5 +27,6 @@ The following `bitsandbytes` quantization config was used during training:
 - bnb_4bit_compute_dtype: bfloat16
 ### Framework versions
 
+- PEFT 0.4.0
 
 - PEFT 0.4.0

checkpoint-400/adapter_config.json

@@ -15,11 +15,11 @@
   "revision": null,
   "target_modules": [
     "up_proj",
-    "down_proj",
-    "k_proj",
     "q_proj",
+    "k_proj",
     "gate_proj",
     "v_proj",
+    "down_proj",
     "o_proj"
   ],
   "task_type": "CAUSAL_LM"

checkpoint-400/adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8fdd9f99841e8aac861b4423eb3db6916c1b6d4b5f39ad2ee1b59b514bbc8a9d
+oid sha256:72d4d013e57515ff7938e3d88a61add2c0ad78f8e0893368e575e639de61bc0a
 size 319977229

checkpoint-400/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:77ec8f751e11e1c32ed50b57c0aec298504b70c0b81296f859d1f3177cb4c62f
+oid sha256:f085ba7380a7475ddac8cf01a682dcb062ba2f5c6643281688b50840fa71e2bc
 size 1279539973

checkpoint-400/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:50b57525a795eb258ada6d2b2b10c6edcd67155c82bdf9e044b06e7cc1de959d
+oid sha256:0c524af7e3397eebeb2aaf2f586bea973e178361ee5d4e590fae3ff922c28d0f
 size 14511

checkpoint-400/trainer_state.json

@@ -1,5 +1,5 @@
 {
-  "best_metric": 0.8780264258384705,
+  "best_metric": 0.878248929977417,
   "best_model_checkpoint": "experts/expert-11/checkpoint-400",
   "epoch": 0.15065913370998116,
   "global_step": 400,
@@ -16,109 +16,109 @@
     {
       "epoch": 0.01,
       "learning_rate": 0.0002,
-      "loss": 0.9795,
+      "loss": 0.9799,
       "step": 20
     },
     {
       "epoch": 0.01,
       "learning_rate": 0.0002,
-      "loss": 0.9392,
+      "loss": 0.939,
       "step": 30
     },
     {
       "epoch": 0.02,
       "learning_rate": 0.0002,
-      "loss": 0.9367,
+      "loss": 0.937,
       "step": 40
     },
     {
       "epoch": 0.02,
       "learning_rate": 0.0002,
-      "loss": 0.9282,
+      "loss": 0.9289,
       "step": 50
     },
     {
       "epoch": 0.02,
       "learning_rate": 0.0002,
-      "loss": 0.9043,
+      "loss": 0.9047,
       "step": 60
     },
     {
       "epoch": 0.03,
       "learning_rate": 0.0002,
-      "loss": 0.9321,
+      "loss": 0.9322,
       "step": 70
     },
     {
       "epoch": 0.03,
       "learning_rate": 0.0002,
-      "loss": 0.8929,
+      "loss": 0.8927,
       "step": 80
     },
     {
       "epoch": 0.03,
       "learning_rate": 0.0002,
-      "loss": 0.8651,
+      "loss": 0.8657,
       "step": 90
     },
     {
       "epoch": 0.04,
       "learning_rate": 0.0002,
-      "loss": 0.8973,
+      "loss": 0.8972,
       "step": 100
     },
     {
       "epoch": 0.04,
       "learning_rate": 0.0002,
-      "loss": 0.931,
+      "loss": 0.9312,
       "step": 110
     },
     {
       "epoch": 0.05,
       "learning_rate": 0.0002,
-      "loss": 0.8901,
+      "loss": 0.8905,
       "step": 120
     },
     {
       "epoch": 0.05,
       "learning_rate": 0.0002,
-      "loss": 0.8921,
+      "loss": 0.892,
       "step": 130
     },
     {
       "epoch": 0.05,
       "learning_rate": 0.0002,
-      "loss": 0.9044,
+      "loss": 0.9042,
       "step": 140
     },
     {
       "epoch": 0.06,
       "learning_rate": 0.0002,
-      "loss": 0.9332,
+      "loss": 0.9328,
       "step": 150
     },
     {
       "epoch": 0.06,
       "learning_rate": 0.0002,
-      "loss": 0.88,
+      "loss": 0.8804,
       "step": 160
     },
     {
       "epoch": 0.06,
       "learning_rate": 0.0002,
-      "loss": 0.8968,
+      "loss": 0.8963,
       "step": 170
     },
     {
       "epoch": 0.07,
       "learning_rate": 0.0002,
-      "loss": 0.9727,
+      "loss": 0.9726,
       "step": 180
     },
     {
       "epoch": 0.07,
       "learning_rate": 0.0002,
-      "loss": 0.8784,
+      "loss": 0.8783,
       "step": 190
     },
     {
@@ -129,15 +129,15 @@
     },
     {
       "epoch": 0.08,
-      "eval_loss": 0.8874365091323853,
-      "eval_runtime": 142.8182,
-      "eval_samples_per_second": 7.002,
-      "eval_steps_per_second": 3.501,
+      "eval_loss": 0.8874900937080383,
+      "eval_runtime": 103.8141,
+      "eval_samples_per_second": 9.633,
+      "eval_steps_per_second": 4.816,
       "step": 200
     },
     {
       "epoch": 0.08,
-      "mmlu_eval_accuracy": 0.49599278735507957,
+      "mmlu_eval_accuracy": 0.49658719618211383,
       "mmlu_eval_accuracy_abstract_algebra": 0.2727272727272727,
       "mmlu_eval_accuracy_anatomy": 0.6428571428571429,
       "mmlu_eval_accuracy_astronomy": 0.5625,
@@ -148,12 +148,12 @@
       "mmlu_eval_accuracy_college_computer_science": 0.2727272727272727,
       "mmlu_eval_accuracy_college_mathematics": 0.2727272727272727,
       "mmlu_eval_accuracy_college_medicine": 0.45454545454545453,
-      "mmlu_eval_accuracy_college_physics": 0.36363636363636365,
+      "mmlu_eval_accuracy_college_physics": 0.45454545454545453,
       "mmlu_eval_accuracy_computer_security": 0.45454545454545453,
-      "mmlu_eval_accuracy_conceptual_physics": 0.34615384615384615,
+      "mmlu_eval_accuracy_conceptual_physics": 0.38461538461538464,
       "mmlu_eval_accuracy_econometrics": 0.25,
       "mmlu_eval_accuracy_electrical_engineering": 0.3125,
-      "mmlu_eval_accuracy_elementary_mathematics": 0.3902439024390244,
+      "mmlu_eval_accuracy_elementary_mathematics": 0.36585365853658536,
       "mmlu_eval_accuracy_formal_logic": 0.07142857142857142,
       "mmlu_eval_accuracy_global_facts": 0.4,
       "mmlu_eval_accuracy_high_school_biology": 0.375,
@@ -162,10 +162,10 @@
       "mmlu_eval_accuracy_high_school_european_history": 0.6666666666666666,
       "mmlu_eval_accuracy_high_school_geography": 0.9090909090909091,
       "mmlu_eval_accuracy_high_school_government_and_politics": 0.6190476190476191,
-      "mmlu_eval_accuracy_high_school_macroeconomics": 0.4186046511627907,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.46511627906976744,
       "mmlu_eval_accuracy_high_school_mathematics": 0.2413793103448276,
       "mmlu_eval_accuracy_high_school_microeconomics": 0.5,
-      "mmlu_eval_accuracy_high_school_physics": 0.11764705882352941,
+      "mmlu_eval_accuracy_high_school_physics": 0.058823529411764705,
       "mmlu_eval_accuracy_high_school_psychology": 0.8666666666666667,
       "mmlu_eval_accuracy_high_school_statistics": 0.43478260869565216,
       "mmlu_eval_accuracy_high_school_us_history": 0.6363636363636364,
@@ -180,70 +180,70 @@
       "mmlu_eval_accuracy_marketing": 0.8,
       "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
       "mmlu_eval_accuracy_miscellaneous": 0.6627906976744186,
-      "mmlu_eval_accuracy_moral_disputes": 0.39473684210526316,
+      "mmlu_eval_accuracy_moral_disputes": 0.42105263157894735,
       "mmlu_eval_accuracy_moral_scenarios": 0.24,
       "mmlu_eval_accuracy_nutrition": 0.7272727272727273,
       "mmlu_eval_accuracy_philosophy": 0.5294117647058824,
       "mmlu_eval_accuracy_prehistory": 0.5142857142857142,
       "mmlu_eval_accuracy_professional_accounting": 0.3548387096774194,
-      "mmlu_eval_accuracy_professional_law": 0.32941176470588235,
+      "mmlu_eval_accuracy_professional_law": 0.3176470588235294,
       "mmlu_eval_accuracy_professional_medicine": 0.5483870967741935,
-      "mmlu_eval_accuracy_professional_psychology": 0.5072463768115942,
+      "mmlu_eval_accuracy_professional_psychology": 0.463768115942029,
       "mmlu_eval_accuracy_public_relations": 0.5833333333333334,
-      "mmlu_eval_accuracy_security_studies": 0.5185185185185185,
-      "mmlu_eval_accuracy_sociology": 0.7272727272727273,
+      "mmlu_eval_accuracy_security_studies": 0.48148148148148145,
+      "mmlu_eval_accuracy_sociology": 0.6818181818181818,
       "mmlu_eval_accuracy_us_foreign_policy": 0.7272727272727273,
       "mmlu_eval_accuracy_virology": 0.5,
-      "mmlu_eval_accuracy_world_religions": 0.6842105263157895,
-      "mmlu_loss": 1.6024464806906549,
+      "mmlu_eval_accuracy_world_religions": 0.7368421052631579,
+      "mmlu_loss": 1.606294181265968,
       "step": 200
     },
     {
       "epoch": 0.08,
       "learning_rate": 0.0002,
-      "loss": 0.8945,
+      "loss": 0.895,
       "step": 210
     },
     {
       "epoch": 0.08,
       "learning_rate": 0.0002,
-      "loss": 0.9126,
+      "loss": 0.9127,
       "step": 220
     },
     {
       "epoch": 0.09,
       "learning_rate": 0.0002,
-      "loss": 0.9186,
+      "loss": 0.9184,
       "step": 230
     },
     {
       "epoch": 0.09,
       "learning_rate": 0.0002,
-      "loss": 0.9349,
+      "loss": 0.9341,
       "step": 240
     },
     {
       "epoch": 0.09,
       "learning_rate": 0.0002,
-      "loss": 0.8852,
+      "loss": 0.885,
       "step": 250
     },
     {
       "epoch": 0.1,
       "learning_rate": 0.0002,
-      "loss": 0.9104,
+      "loss": 0.9103,
       "step": 260
     },
     {
       "epoch": 0.1,
       "learning_rate": 0.0002,
-      "loss": 0.943,
+      "loss": 0.9435,
       "step": 270
     },
     {
       "epoch": 0.11,
       "learning_rate": 0.0002,
-      "loss": 0.886,
+      "loss": 0.8857,
       "step": 280
     },
     {
@@ -255,112 +255,112 @@
     {
       "epoch": 0.11,
       "learning_rate": 0.0002,
-      "loss": 0.9065,
+      "loss": 0.9063,
       "step": 300
     },
     {
       "epoch": 0.12,
       "learning_rate": 0.0002,
-      "loss": 0.9395,
+      "loss": 0.939,
       "step": 310
     },
     {
       "epoch": 0.12,
       "learning_rate": 0.0002,
-      "loss": 0.9348,
+      "loss": 0.935,
       "step": 320
     },
     {
       "epoch": 0.12,
       "learning_rate": 0.0002,
-      "loss": 0.8581,
+      "loss": 0.8583,
       "step": 330
     },
     {
       "epoch": 0.13,
       "learning_rate": 0.0002,
-      "loss": 0.8796,
+      "loss": 0.8795,
       "step": 340
     },
     {
       "epoch": 0.13,
       "learning_rate": 0.0002,
-      "loss": 0.8214,
+      "loss": 0.8205,
       "step": 350
     },
     {
       "epoch": 0.14,
       "learning_rate": 0.0002,
-      "loss": 0.8293,
+      "loss": 0.8289,
       "step": 360
     },
     {
       "epoch": 0.14,
       "learning_rate": 0.0002,
-      "loss": 0.9048,
+      "loss": 0.9046,
       "step": 370
     },
     {
       "epoch": 0.14,
       "learning_rate": 0.0002,
-      "loss": 0.9161,
+      "loss": 0.9162,
       "step": 380
     },
     {
       "epoch": 0.15,
       "learning_rate": 0.0002,
-      "loss": 0.8699,
+      "loss": 0.8695,
       "step": 390
     },
     {
       "epoch": 0.15,
       "learning_rate": 0.0002,
-      "loss": 0.88,
+      "loss": 0.8804,
       "step": 400
     },
     {
       "epoch": 0.15,
-      "eval_loss": 0.8780264258384705,
-      "eval_runtime": 142.7037,
-      "eval_samples_per_second": 7.008,
-      "eval_steps_per_second": 3.504,
+      "eval_loss": 0.878248929977417,
+      "eval_runtime": 105.4959,
+      "eval_samples_per_second": 9.479,
+      "eval_steps_per_second": 4.74,
       "step": 400
     },
     {
       "epoch": 0.15,
-      "mmlu_eval_accuracy": 0.48513361890744183,
+      "mmlu_eval_accuracy": 0.4889546706999997,
       "mmlu_eval_accuracy_abstract_algebra": 0.2727272727272727,
       "mmlu_eval_accuracy_anatomy": 0.6428571428571429,
       "mmlu_eval_accuracy_astronomy": 0.4375,
       "mmlu_eval_accuracy_business_ethics": 0.5454545454545454,
       "mmlu_eval_accuracy_clinical_knowledge": 0.4482758620689655,
       "mmlu_eval_accuracy_college_biology": 0.4375,
-      "mmlu_eval_accuracy_college_chemistry": 0.125,
+      "mmlu_eval_accuracy_college_chemistry": 0.25,
       "mmlu_eval_accuracy_college_computer_science": 0.2727272727272727,
-      "mmlu_eval_accuracy_college_mathematics": 0.2727272727272727,
-      "mmlu_eval_accuracy_college_medicine": 0.4090909090909091,
+      "mmlu_eval_accuracy_college_mathematics": 0.36363636363636365,
+      "mmlu_eval_accuracy_college_medicine": 0.45454545454545453,
       "mmlu_eval_accuracy_college_physics": 0.45454545454545453,
       "mmlu_eval_accuracy_computer_security": 0.36363636363636365,
-      "mmlu_eval_accuracy_conceptual_physics": 0.38461538461538464,
+      "mmlu_eval_accuracy_conceptual_physics": 0.34615384615384615,
       "mmlu_eval_accuracy_econometrics": 0.25,
       "mmlu_eval_accuracy_electrical_engineering": 0.25,
       "mmlu_eval_accuracy_elementary_mathematics": 0.3170731707317073,
       "mmlu_eval_accuracy_formal_logic": 0.14285714285714285,
       "mmlu_eval_accuracy_global_facts": 0.4,
-      "mmlu_eval_accuracy_high_school_biology": 0.40625,
-      "mmlu_eval_accuracy_high_school_chemistry": 0.22727272727272727,
+      "mmlu_eval_accuracy_high_school_biology": 0.375,
+      "mmlu_eval_accuracy_high_school_chemistry": 0.36363636363636365,
       "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
-      "mmlu_eval_accuracy_high_school_european_history": 0.6111111111111112,
+      "mmlu_eval_accuracy_high_school_european_history": 0.5555555555555556,
       "mmlu_eval_accuracy_high_school_geography": 0.9090909090909091,
-      "mmlu_eval_accuracy_high_school_government_and_politics": 0.5714285714285714,
-      "mmlu_eval_accuracy_high_school_macroeconomics": 0.46511627906976744,
+      "mmlu_eval_accuracy_high_school_government_and_politics": 0.5238095238095238,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.4418604651162791,
       "mmlu_eval_accuracy_high_school_mathematics": 0.2413793103448276,
-      "mmlu_eval_accuracy_high_school_microeconomics": 0.46153846153846156,
+      "mmlu_eval_accuracy_high_school_microeconomics": 0.38461538461538464,
       "mmlu_eval_accuracy_high_school_physics": 0.17647058823529413,
-      "mmlu_eval_accuracy_high_school_psychology": 0.85,
+      "mmlu_eval_accuracy_high_school_psychology": 0.8666666666666667,
       "mmlu_eval_accuracy_high_school_statistics": 0.391304347826087,
       "mmlu_eval_accuracy_high_school_us_history": 0.6363636363636364,
-      "mmlu_eval_accuracy_high_school_world_history": 0.7307692307692307,
+      "mmlu_eval_accuracy_high_school_world_history": 0.7692307692307693,
       "mmlu_eval_accuracy_human_aging": 0.6521739130434783,
       "mmlu_eval_accuracy_human_sexuality": 0.3333333333333333,
       "mmlu_eval_accuracy_international_law": 0.8461538461538461,
@@ -372,21 +372,21 @@
       "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
       "mmlu_eval_accuracy_miscellaneous": 0.6511627906976745,
       "mmlu_eval_accuracy_moral_disputes": 0.39473684210526316,
-      "mmlu_eval_accuracy_moral_scenarios": 0.25,
+      "mmlu_eval_accuracy_moral_scenarios": 0.24,
       "mmlu_eval_accuracy_nutrition": 0.6666666666666666,
-      "mmlu_eval_accuracy_philosophy": 0.5588235294117647,
+      "mmlu_eval_accuracy_philosophy": 0.5294117647058824,
       "mmlu_eval_accuracy_prehistory": 0.45714285714285713,
-      "mmlu_eval_accuracy_professional_accounting": 0.3870967741935484,
-      "mmlu_eval_accuracy_professional_law": 0.3,
-      "mmlu_eval_accuracy_professional_medicine": 0.5483870967741935,
+      "mmlu_eval_accuracy_professional_accounting": 0.41935483870967744,
+      "mmlu_eval_accuracy_professional_law": 0.31176470588235294,
+      "mmlu_eval_accuracy_professional_medicine": 0.5161290322580645,
       "mmlu_eval_accuracy_professional_psychology": 0.4927536231884058,
       "mmlu_eval_accuracy_public_relations": 0.6666666666666666,
       "mmlu_eval_accuracy_security_studies": 0.5185185185185185,
-      "mmlu_eval_accuracy_sociology": 0.7272727272727273,
+      "mmlu_eval_accuracy_sociology": 0.6818181818181818,
       "mmlu_eval_accuracy_us_foreign_policy": 0.7272727272727273,
-      "mmlu_eval_accuracy_virology": 0.4444444444444444,
+      "mmlu_eval_accuracy_virology": 0.5555555555555556,
       "mmlu_eval_accuracy_world_religions": 0.7368421052631579,
-      "mmlu_loss": 1.496849655871603,
+      "mmlu_loss": 1.5168773670420634,
       "step": 400
     }
   ],

checkpoint-400/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f7da584018b8d6650a0d053d954b1aa99caf64436ee6adf2c05a7cc7135bee80
+oid sha256:6d25ed85aaaa1b1fcd8d0d7696607f4d0387a809d98218ea0ccfd02ccc95face
 size 5819