Training in progress, step 200

adapter_config.json

@@ -15,11 +15,11 @@
   "revision": null,
   "target_modules": [
     "up_proj",
-    "down_proj",
-    "k_proj",
     "q_proj",
+    "k_proj",
     "gate_proj",
     "v_proj",
+    "down_proj",
     "o_proj"
   ],
   "task_type": "CAUSAL_LM"

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c94e421ec723c624bfac2f6c00d7becee2b9ffd5e17dfa4bafe4889c4b17b755
+oid sha256:6342fa32a534d427fda4a77c749de84133e553afd6fecd7274f137aa3e7d6ff7
 size 319977229

checkpoint-200/README.md

@@ -4,6 +4,17 @@ library_name: peft
 ## Training procedure
 
 
+The following `bitsandbytes` quantization config was used during training:
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+
 The following `bitsandbytes` quantization config was used during training:
 - load_in_8bit: False
 - load_in_4bit: True
@@ -16,5 +27,6 @@ The following `bitsandbytes` quantization config was used during training:
 - bnb_4bit_compute_dtype: bfloat16
 ### Framework versions
 
+- PEFT 0.4.0
 
 - PEFT 0.4.0

checkpoint-200/adapter_config.json

@@ -15,11 +15,11 @@
   "revision": null,
   "target_modules": [
     "up_proj",
-    "down_proj",
-    "k_proj",
     "q_proj",
+    "k_proj",
     "gate_proj",
     "v_proj",
+    "down_proj",
     "o_proj"
   ],
   "task_type": "CAUSAL_LM"

checkpoint-200/adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6865aa6f8fa180a3e27dee19c155a2eebea142a7b162caa6cd4fb9ace1c13c56
+oid sha256:6342fa32a534d427fda4a77c749de84133e553afd6fecd7274f137aa3e7d6ff7
 size 319977229

checkpoint-200/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e199adcf368844165b55330595cb7d764a5456cd4083a894937ec1371ea721f2
+oid sha256:5b88232e1951f0342a572379c9010eeb6b5f4547e61164dd193d21ceacdef256
 size 1279539525

checkpoint-200/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8a873c071c01a34064bed4eebad2c70579320bf83b9ebf18f435df278558914
+oid sha256:63ee90d35cced36b7f030f659c1be1947a4b1a96d75a0a3b8e86bf2b7170dfc9
 size 14511

checkpoint-200/trainer_state.json

@@ -1,5 +1,5 @@
 {
-  "best_metric": 0.8874365091323853,
+  "best_metric": 0.8874900937080383,
   "best_model_checkpoint": "experts/expert-11/checkpoint-200",
   "epoch": 0.07532956685499058,
   "global_step": 200,
@@ -16,109 +16,109 @@
     {
       "epoch": 0.01,
       "learning_rate": 0.0002,
-      "loss": 0.9795,
+      "loss": 0.9799,
       "step": 20
     },
     {
       "epoch": 0.01,
       "learning_rate": 0.0002,
-      "loss": 0.9392,
+      "loss": 0.939,
       "step": 30
     },
     {
       "epoch": 0.02,
       "learning_rate": 0.0002,
-      "loss": 0.9367,
+      "loss": 0.937,
       "step": 40
     },
     {
       "epoch": 0.02,
       "learning_rate": 0.0002,
-      "loss": 0.9282,
+      "loss": 0.9289,
       "step": 50
     },
     {
       "epoch": 0.02,
       "learning_rate": 0.0002,
-      "loss": 0.9043,
+      "loss": 0.9047,
       "step": 60
     },
     {
       "epoch": 0.03,
       "learning_rate": 0.0002,
-      "loss": 0.9321,
+      "loss": 0.9322,
       "step": 70
     },
     {
       "epoch": 0.03,
       "learning_rate": 0.0002,
-      "loss": 0.8929,
+      "loss": 0.8927,
       "step": 80
     },
     {
       "epoch": 0.03,
       "learning_rate": 0.0002,
-      "loss": 0.8651,
+      "loss": 0.8657,
       "step": 90
     },
     {
       "epoch": 0.04,
       "learning_rate": 0.0002,
-      "loss": 0.8973,
+      "loss": 0.8972,
       "step": 100
     },
     {
       "epoch": 0.04,
       "learning_rate": 0.0002,
-      "loss": 0.931,
+      "loss": 0.9312,
       "step": 110
     },
     {
       "epoch": 0.05,
       "learning_rate": 0.0002,
-      "loss": 0.8901,
+      "loss": 0.8905,
       "step": 120
     },
     {
       "epoch": 0.05,
       "learning_rate": 0.0002,
-      "loss": 0.8921,
+      "loss": 0.892,
       "step": 130
     },
     {
       "epoch": 0.05,
       "learning_rate": 0.0002,
-      "loss": 0.9044,
+      "loss": 0.9042,
       "step": 140
     },
     {
       "epoch": 0.06,
       "learning_rate": 0.0002,
-      "loss": 0.9332,
+      "loss": 0.9328,
       "step": 150
     },
     {
       "epoch": 0.06,
       "learning_rate": 0.0002,
-      "loss": 0.88,
+      "loss": 0.8804,
       "step": 160
     },
     {
       "epoch": 0.06,
       "learning_rate": 0.0002,
-      "loss": 0.8968,
+      "loss": 0.8963,
       "step": 170
     },
     {
       "epoch": 0.07,
       "learning_rate": 0.0002,
-      "loss": 0.9727,
+      "loss": 0.9726,
       "step": 180
     },
     {
       "epoch": 0.07,
       "learning_rate": 0.0002,
-      "loss": 0.8784,
+      "loss": 0.8783,
       "step": 190
     },
     {
@@ -129,15 +129,15 @@
     },
     {
       "epoch": 0.08,
-      "eval_loss": 0.8874365091323853,
-      "eval_runtime": 142.8182,
-      "eval_samples_per_second": 7.002,
-      "eval_steps_per_second": 3.501,
+      "eval_loss": 0.8874900937080383,
+      "eval_runtime": 103.8141,
+      "eval_samples_per_second": 9.633,
+      "eval_steps_per_second": 4.816,
       "step": 200
     },
     {
       "epoch": 0.08,
-      "mmlu_eval_accuracy": 0.49599278735507957,
+      "mmlu_eval_accuracy": 0.49658719618211383,
       "mmlu_eval_accuracy_abstract_algebra": 0.2727272727272727,
       "mmlu_eval_accuracy_anatomy": 0.6428571428571429,
       "mmlu_eval_accuracy_astronomy": 0.5625,
@@ -148,12 +148,12 @@
       "mmlu_eval_accuracy_college_computer_science": 0.2727272727272727,
       "mmlu_eval_accuracy_college_mathematics": 0.2727272727272727,
       "mmlu_eval_accuracy_college_medicine": 0.45454545454545453,
-      "mmlu_eval_accuracy_college_physics": 0.36363636363636365,
+      "mmlu_eval_accuracy_college_physics": 0.45454545454545453,
       "mmlu_eval_accuracy_computer_security": 0.45454545454545453,
-      "mmlu_eval_accuracy_conceptual_physics": 0.34615384615384615,
+      "mmlu_eval_accuracy_conceptual_physics": 0.38461538461538464,
       "mmlu_eval_accuracy_econometrics": 0.25,
       "mmlu_eval_accuracy_electrical_engineering": 0.3125,
-      "mmlu_eval_accuracy_elementary_mathematics": 0.3902439024390244,
+      "mmlu_eval_accuracy_elementary_mathematics": 0.36585365853658536,
       "mmlu_eval_accuracy_formal_logic": 0.07142857142857142,
       "mmlu_eval_accuracy_global_facts": 0.4,
       "mmlu_eval_accuracy_high_school_biology": 0.375,
@@ -162,10 +162,10 @@
       "mmlu_eval_accuracy_high_school_european_history": 0.6666666666666666,
       "mmlu_eval_accuracy_high_school_geography": 0.9090909090909091,
       "mmlu_eval_accuracy_high_school_government_and_politics": 0.6190476190476191,
-      "mmlu_eval_accuracy_high_school_macroeconomics": 0.4186046511627907,
+      "mmlu_eval_accuracy_high_school_macroeconomics": 0.46511627906976744,
       "mmlu_eval_accuracy_high_school_mathematics": 0.2413793103448276,
       "mmlu_eval_accuracy_high_school_microeconomics": 0.5,
-      "mmlu_eval_accuracy_high_school_physics": 0.11764705882352941,
+      "mmlu_eval_accuracy_high_school_physics": 0.058823529411764705,
       "mmlu_eval_accuracy_high_school_psychology": 0.8666666666666667,
       "mmlu_eval_accuracy_high_school_statistics": 0.43478260869565216,
       "mmlu_eval_accuracy_high_school_us_history": 0.6363636363636364,
@@ -180,22 +180,22 @@
       "mmlu_eval_accuracy_marketing": 0.8,
       "mmlu_eval_accuracy_medical_genetics": 0.9090909090909091,
       "mmlu_eval_accuracy_miscellaneous": 0.6627906976744186,
-      "mmlu_eval_accuracy_moral_disputes": 0.39473684210526316,
+      "mmlu_eval_accuracy_moral_disputes": 0.42105263157894735,
       "mmlu_eval_accuracy_moral_scenarios": 0.24,
       "mmlu_eval_accuracy_nutrition": 0.7272727272727273,
       "mmlu_eval_accuracy_philosophy": 0.5294117647058824,
       "mmlu_eval_accuracy_prehistory": 0.5142857142857142,
       "mmlu_eval_accuracy_professional_accounting": 0.3548387096774194,
-      "mmlu_eval_accuracy_professional_law": 0.32941176470588235,
+      "mmlu_eval_accuracy_professional_law": 0.3176470588235294,
       "mmlu_eval_accuracy_professional_medicine": 0.5483870967741935,
-      "mmlu_eval_accuracy_professional_psychology": 0.5072463768115942,
+      "mmlu_eval_accuracy_professional_psychology": 0.463768115942029,
       "mmlu_eval_accuracy_public_relations": 0.5833333333333334,
-      "mmlu_eval_accuracy_security_studies": 0.5185185185185185,
-      "mmlu_eval_accuracy_sociology": 0.7272727272727273,
+      "mmlu_eval_accuracy_security_studies": 0.48148148148148145,
+      "mmlu_eval_accuracy_sociology": 0.6818181818181818,
       "mmlu_eval_accuracy_us_foreign_policy": 0.7272727272727273,
       "mmlu_eval_accuracy_virology": 0.5,
-      "mmlu_eval_accuracy_world_religions": 0.6842105263157895,
-      "mmlu_loss": 1.6024464806906549,
+      "mmlu_eval_accuracy_world_religions": 0.7368421052631579,
+      "mmlu_loss": 1.606294181265968,
       "step": 200
     }
   ],

checkpoint-200/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f7da584018b8d6650a0d053d954b1aa99caf64436ee6adf2c05a7cc7135bee80
+oid sha256:6d25ed85aaaa1b1fcd8d0d7696607f4d0387a809d98218ea0ccfd02ccc95face
 size 5819

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f7da584018b8d6650a0d053d954b1aa99caf64436ee6adf2c05a7cc7135bee80
+oid sha256:6d25ed85aaaa1b1fcd8d0d7696607f4d0387a809d98218ea0ccfd02ccc95face
 size 5819