Upload folder using huggingface_hub

README.md

@@ -199,4 +199,4 @@ Carbon emissions can be estimated using the [Machine Learning Impact calculator]
 [More Information Needed]
 ### Framework versions
 
-- PEFT 0.12.0
+- PEFT 0.13.2

adapter_config.json

@@ -10,20 +10,20 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 64,
-  "lora_dropout": 0.0001,
+  "lora_alpha": 128,
+  "lora_dropout": 0.05,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 32,
+  "r": 64,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "down_proj",
     "qkv_proj",
-    "gate_up_proj",
-    "o_proj"
+    "down_proj",
+    "o_proj",
+    "gate_up_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:34e8c063268c82d446f081987dac5fd9c69282ecfad89abd0570dc93517cdbc9
-size 201361312
+oid sha256:7d39d4b71c2d9c958752b7019b0481033ab8d7caa096419fe04a39f1e2c03e5f
+size 402688040

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0157da31bb4062434f031ff2dd7c51f693e094db4fe85815de38edaefd40b9fa
-size 402868986
+oid sha256:b2214c2d7be4e7002e6b458c215e56dc3cc1231d71e76dcf574cfefeb1df1f14
+size 805522170

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ed70ecedcd9a62bbb04bf9838304aced41ca983de90cea5987c3cff1d4f80fe3
+oid sha256:96a39edec8fd0ca2c66adccb7ddca2a246727221a5cedfcaa945c37683bd0907
 size 14244

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f94793ec3497737749203684f2a64875f06eeb7a4781950315fb5cb4ec740a8
+oid sha256:db20a34ad6b350b7c1ce1bf536f3e5516e15fa5f9d629c0ece20011d12bce789
 size 1064

trainer_state.json

@@ -1,408 +1,235 @@
 {
-  "best_metric": 0.4687739610671997,
-  "best_model_checkpoint": "//outputs/task7_microsoft/Phi-3.5-mini-instruct/checkpoint-250",
-  "epoch": 5.0,
+  "best_metric": 0.5998682379722595,
+  "best_model_checkpoint": "//outputs/task7_microsoft/Phi-3.5-mini-instruct/checkpoint-238",
+  "epoch": 7.0,
   "eval_steps": 500,
-  "global_step": 250,
+  "global_step": 238,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.1,
-      "grad_norm": 1.4559898376464844,
-      "learning_rate": 3.75e-06,
-      "loss": 1.8397,
-      "step": 5
-    },
-    {
-      "epoch": 0.2,
-      "grad_norm": 1.011980414390564,
-      "learning_rate": 7.5e-06,
-      "loss": 1.4061,
+      "epoch": 0.2962962962962963,
+      "grad_norm": 0.869780957698822,
+      "learning_rate": 7.692307692307694e-06,
+      "loss": 0.9461,
       "step": 10
     },
     {
-      "epoch": 0.3,
-      "grad_norm": 0.8619025945663452,
-      "learning_rate": 1.125e-05,
-      "loss": 1.037,
-      "step": 15
-    },
-    {
-      "epoch": 0.4,
-      "grad_norm": 1.4890649318695068,
-      "learning_rate": 1.5e-05,
-      "loss": 1.2559,
+      "epoch": 0.5925925925925926,
+      "grad_norm": 0.5883250832557678,
+      "learning_rate": 9.978490638616671e-06,
+      "loss": 0.6991,
       "step": 20
     },
     {
-      "epoch": 0.5,
-      "grad_norm": 0.3758047819137573,
-      "learning_rate": 1.8750000000000002e-05,
-      "loss": 1.0541,
-      "step": 25
-    },
-    {
-      "epoch": 0.6,
-      "grad_norm": 1.0797535181045532,
-      "learning_rate": 2.25e-05,
-      "loss": 1.2359,
+      "epoch": 0.8888888888888888,
+      "grad_norm": 0.922535240650177,
+      "learning_rate": 9.873583924954152e-06,
+      "loss": 0.7785,
       "step": 30
     },
     {
-      "epoch": 0.7,
-      "grad_norm": 0.32953447103500366,
-      "learning_rate": 2.625e-05,
-      "loss": 0.8877,
-      "step": 35
+      "epoch": 1.0,
+      "eval_loss": 0.6846582889556885,
+      "eval_runtime": 3.3866,
+      "eval_samples_per_second": 4.429,
+      "eval_steps_per_second": 0.591,
+      "step": 34
     },
     {
-      "epoch": 0.8,
-      "grad_norm": 0.319231241941452,
-      "learning_rate": 3e-05,
-      "loss": 1.0191,
+      "epoch": 1.1777777777777778,
+      "grad_norm": 0.3914264738559723,
+      "learning_rate": 9.68316749134364e-06,
+      "loss": 0.6765,
       "step": 40
     },
     {
-      "epoch": 0.9,
-      "grad_norm": 0.319320410490036,
-      "learning_rate": 2.9996796251818968e-05,
-      "loss": 0.8399,
-      "step": 45
-    },
-    {
-      "epoch": 1.0,
-      "grad_norm": 1.2043859958648682,
-      "learning_rate": 2.9987186375809513e-05,
-      "loss": 0.9834,
-      "step": 50
-    },
-    {
-      "epoch": 1.0,
-      "eval_loss": 0.6599301099777222,
-      "eval_runtime": 52.5278,
-      "eval_samples_per_second": 3.808,
-      "eval_steps_per_second": 0.476,
+      "epoch": 1.474074074074074,
+      "grad_norm": 0.5533085465431213,
+      "learning_rate": 9.410582299213574e-06,
+      "loss": 0.799,
       "step": 50
     },
     {
-      "epoch": 1.1,
-      "grad_norm": 0.39200976490974426,
-      "learning_rate": 2.997117447698802e-05,
-      "loss": 0.8063,
-      "step": 55
-    },
-    {
-      "epoch": 1.2,
-      "grad_norm": 0.42485809326171875,
-      "learning_rate": 2.994876739510005e-05,
-      "loss": 0.5906,
+      "epoch": 1.7703703703703704,
+      "grad_norm": 0.3141545355319977,
+      "learning_rate": 9.060611006213833e-06,
+      "loss": 0.5998,
       "step": 60
     },
     {
-      "epoch": 1.3,
-      "grad_norm": 0.5581662654876709,
-      "learning_rate": 2.9919974701698638e-05,
-      "loss": 0.7749,
-      "step": 65
+      "epoch": 2.0,
+      "eval_loss": 0.6442674994468689,
+      "eval_runtime": 3.3729,
+      "eval_samples_per_second": 4.447,
+      "eval_steps_per_second": 0.593,
+      "step": 68
     },
     {
-      "epoch": 1.4,
-      "grad_norm": 0.8188683390617371,
-      "learning_rate": 2.9884808696055675e-05,
-      "loss": 0.7623,
+      "epoch": 2.0592592592592593,
+      "grad_norm": 0.6881595849990845,
+      "learning_rate": 8.639394051847472e-06,
+      "loss": 0.6565,
       "step": 70
     },
     {
-      "epoch": 1.5,
-      "grad_norm": 0.4976309537887573,
-      "learning_rate": 2.984328439990804e-05,
-      "loss": 0.7587,
-      "step": 75
-    },
-    {
-      "epoch": 1.6,
-      "grad_norm": 0.515602171421051,
-      "learning_rate": 2.9795419551040836e-05,
-      "loss": 0.6395,
+      "epoch": 2.3555555555555556,
+      "grad_norm": 0.817984402179718,
+      "learning_rate": 8.154321920070415e-06,
+      "loss": 0.6779,
       "step": 80
     },
     {
-      "epoch": 1.7,
-      "grad_norm": 0.8577103018760681,
-      "learning_rate": 2.9741234595710393e-05,
-      "loss": 0.5315,
-      "step": 85
-    },
-    {
-      "epoch": 1.8,
-      "grad_norm": 0.6678707599639893,
-      "learning_rate": 2.968075267991032e-05,
-      "loss": 0.6739,
+      "epoch": 2.651851851851852,
+      "grad_norm": 0.585773229598999,
+      "learning_rate": 7.613905469171247e-06,
+      "loss": 0.5443,
       "step": 90
     },
     {
-      "epoch": 1.9,
-      "grad_norm": 0.3638306260108948,
-      "learning_rate": 2.9613999639484314e-05,
-      "loss": 0.6927,
-      "step": 95
-    },
-    {
-      "epoch": 2.0,
-      "grad_norm": 0.8823966383934021,
-      "learning_rate": 2.9541003989089956e-05,
-      "loss": 0.6094,
-      "step": 100
-    },
-    {
-      "epoch": 2.0,
-      "eval_loss": 0.5690982341766357,
-      "eval_runtime": 52.1876,
-      "eval_samples_per_second": 3.832,
-      "eval_steps_per_second": 0.479,
+      "epoch": 2.948148148148148,
+      "grad_norm": 0.5395255088806152,
+      "learning_rate": 7.02762660406497e-06,
+      "loss": 0.6243,
       "step": 100
     },
     {
-      "epoch": 2.1,
-      "grad_norm": 0.5922141671180725,
-      "learning_rate": 2.9461796910018204e-05,
-      "loss": 0.6031,
-      "step": 105
+      "epoch": 3.0,
+      "eval_loss": 0.6209592223167419,
+      "eval_runtime": 3.3724,
+      "eval_samples_per_second": 4.448,
+      "eval_steps_per_second": 0.593,
+      "step": 102
     },
     {
-      "epoch": 2.2,
-      "grad_norm": 0.5325513482093811,
-      "learning_rate": 2.9376412236873792e-05,
-      "loss": 0.493,
+      "epoch": 3.237037037037037,
+      "grad_norm": 0.7252342104911804,
+      "learning_rate": 6.405771911037698e-06,
+      "loss": 0.6189,
       "step": 110
     },
     {
-      "epoch": 2.3,
-      "grad_norm": 1.020575761795044,
-      "learning_rate": 2.928488644312222e-05,
-      "loss": 0.4483,
-      "step": 115
-    },
-    {
-      "epoch": 2.4,
-      "grad_norm": 0.9036449790000916,
-      "learning_rate": 2.9187258625509518e-05,
-      "loss": 0.5766,
+      "epoch": 3.533333333333333,
+      "grad_norm": 0.5255349278450012,
+      "learning_rate": 5.759252173912573e-06,
+      "loss": 0.5914,
       "step": 120
     },
     {
-      "epoch": 2.5,
-      "grad_norm": 1.0615090131759644,
-      "learning_rate": 2.9083570487361445e-05,
-      "loss": 0.4717,
-      "step": 125
-    },
-    {
-      "epoch": 2.6,
-      "grad_norm": 0.638048529624939,
-      "learning_rate": 2.8973866320769186e-05,
-      "loss": 0.3577,
+      "epoch": 3.8296296296296295,
+      "grad_norm": 0.5693609118461609,
+      "learning_rate": 5.099410938325351e-06,
+      "loss": 0.5872,
       "step": 130
     },
     {
-      "epoch": 2.7,
-      "grad_norm": 1.1508071422576904,
-      "learning_rate": 2.8858192987669303e-05,
-      "loss": 0.5615,
-      "step": 135
+      "epoch": 4.0,
+      "eval_loss": 0.6094754934310913,
+      "eval_runtime": 3.3715,
+      "eval_samples_per_second": 4.449,
+      "eval_steps_per_second": 0.593,
+      "step": 136
     },
     {
-      "epoch": 2.8,
-      "grad_norm": 0.6334187984466553,
-      "learning_rate": 2.873659989982586e-05,
-      "loss": 0.3704,
+      "epoch": 4.118518518518519,
+      "grad_norm": 0.38578182458877563,
+      "learning_rate": 4.43782548295514e-06,
+      "loss": 0.574,
       "step": 140
     },
     {
-      "epoch": 2.9,
-      "grad_norm": 0.53675377368927,
-      "learning_rate": 2.86091389977234e-05,
-      "loss": 0.3623,
-      "step": 145
-    },
-    {
-      "epoch": 3.0,
-      "grad_norm": 0.5917493104934692,
-      "learning_rate": 2.8475864728379682e-05,
-      "loss": 0.3345,
-      "step": 150
-    },
-    {
-      "epoch": 3.0,
-      "eval_loss": 0.5363968014717102,
-      "eval_runtime": 52.2028,
-      "eval_samples_per_second": 3.831,
-      "eval_steps_per_second": 0.479,
+      "epoch": 4.4148148148148145,
+      "grad_norm": 0.5012251138687134,
+      "learning_rate": 3.786103689779861e-06,
+      "loss": 0.5227,
       "step": 150
     },
     {
-      "epoch": 3.1,
-      "grad_norm": 1.654146671295166,
-      "learning_rate": 2.8336834022087776e-05,
-      "loss": 0.3779,
-      "step": 155
-    },
-    {
-      "epoch": 3.2,
-      "grad_norm": 0.9066053032875061,
-      "learning_rate": 2.8192106268097336e-05,
-      "loss": 0.2994,
+      "epoch": 4.711111111111111,
+      "grad_norm": 0.5396020412445068,
+      "learning_rate": 3.1556803773799616e-06,
+      "loss": 0.5366,
       "step": 160
     },
     {
-      "epoch": 3.3,
-      "grad_norm": 0.5281007289886475,
-      "learning_rate": 2.8041743289245503e-05,
-      "loss": 0.4545,
-      "step": 165
-    },
-    {
-      "epoch": 3.4,
-      "grad_norm": 0.8571799397468567,
-      "learning_rate": 2.788580931554828e-05,
-      "loss": 0.3399,
+      "epoch": 5.0,
+      "grad_norm": 0.5771762728691101,
+      "learning_rate": 2.5576166707349387e-06,
+      "loss": 0.6322,
       "step": 170
     },
     {
-      "epoch": 3.5,
-      "grad_norm": 0.43631649017333984,
-      "learning_rate": 2.7724370956763605e-05,
-      "loss": 0.2589,
-      "step": 175
+      "epoch": 5.0,
+      "eval_loss": 0.6032379269599915,
+      "eval_runtime": 3.372,
+      "eval_samples_per_second": 4.448,
+      "eval_steps_per_second": 0.593,
+      "step": 170
     },
     {
-      "epoch": 3.6,
-      "grad_norm": 0.7908278107643127,
-      "learning_rate": 2.7557497173937928e-05,
-      "loss": 0.3241,
+      "epoch": 5.296296296296296,
+      "grad_norm": 0.5676046013832092,
+      "learning_rate": 2.0024059276803742e-06,
+      "loss": 0.5883,
       "step": 180
     },
     {
-      "epoch": 3.7,
-      "grad_norm": 1.0415078401565552,
-      "learning_rate": 2.7385259249948338e-05,
-      "loss": 0.3205,
-      "step": 185
-    },
-    {
-      "epoch": 3.8,
-      "grad_norm": 0.5231990218162537,
-      "learning_rate": 2.7207730759052925e-05,
-      "loss": 0.1806,
+      "epoch": 5.592592592592593,
+      "grad_norm": 0.4530661702156067,
+      "learning_rate": 1.499789627152874e-06,
+      "loss": 0.5619,
       "step": 190
     },
     {
-      "epoch": 3.9,
-      "grad_norm": 0.48716872930526733,
-      "learning_rate": 2.7024987535462327e-05,
-      "loss": 0.172,
-      "step": 195
-    },
-    {
-      "epoch": 4.0,
-      "grad_norm": 0.6646760702133179,
-      "learning_rate": 2.6837107640945904e-05,
-      "loss": 0.2291,
+      "epoch": 5.888888888888889,
+      "grad_norm": 0.5783275365829468,
+      "learning_rate": 1.0585864495652899e-06,
+      "loss": 0.4661,
       "step": 200
     },
     {
-      "epoch": 4.0,
-      "eval_loss": 0.5222796201705933,
-      "eval_runtime": 52.1967,
-      "eval_samples_per_second": 3.832,
-      "eval_steps_per_second": 0.479,
-      "step": 200
-    },
-    {
-      "epoch": 4.1,
-      "grad_norm": 1.3394831418991089,
-      "learning_rate": 2.6644171331486363e-05,
-      "loss": 0.2097,
-      "step": 205
+      "epoch": 6.0,
+      "eval_loss": 0.6005836129188538,
+      "eval_runtime": 3.3722,
+      "eval_samples_per_second": 4.448,
+      "eval_steps_per_second": 0.593,
+      "step": 204
     },
     {
-      "epoch": 4.2,
-      "grad_norm": 0.6753952503204346,
-      "learning_rate": 2.6446261022997098e-05,
-      "loss": 0.2552,
+      "epoch": 6.177777777777778,
+      "grad_norm": 0.5576639175415039,
+      "learning_rate": 6.865375481914017e-07,
+      "loss": 0.5346,
       "step": 210
     },
     {
-      "epoch": 4.3,
-      "grad_norm": 0.5856276750564575,
-      "learning_rate": 2.6243461256116892e-05,
-      "loss": 0.1606,
-      "step": 215
-    },
-    {
-      "epoch": 4.4,
-      "grad_norm": 0.695767879486084,
-      "learning_rate": 2.6035858660096975e-05,
-      "loss": 0.2958,
+      "epoch": 6.474074074074074,
+      "grad_norm": 0.5694870948791504,
+      "learning_rate": 3.9017072635896716e-07,
+      "loss": 0.5697,
       "step": 220
     },
     {
-      "epoch": 4.5,
-      "grad_norm": 0.6565276980400085,
-      "learning_rate": 2.5823541915795932e-05,
-      "loss": 0.1491,
-      "step": 225
-    },
-    {
-      "epoch": 4.6,
-      "grad_norm": 0.497454971075058,
-      "learning_rate": 2.5606601717798212e-05,
-      "loss": 0.1945,
+      "epoch": 6.770370370370371,
+      "grad_norm": 0.35573288798332214,
+      "learning_rate": 1.7468590353731495e-07,
+      "loss": 0.5585,
       "step": 230
     },
     {
-      "epoch": 4.7,
-      "grad_norm": 0.7928630709648132,
-      "learning_rate": 2.5385130735672442e-05,
-      "loss": 0.1197,
-      "step": 235
-    },
-    {
-      "epoch": 4.8,
-      "grad_norm": 0.9403858780860901,
-      "learning_rate": 2.5159223574386117e-05,
-      "loss": 0.2448,
-      "step": 240
-    },
-    {
-      "epoch": 4.9,
-      "grad_norm": 0.41166239976882935,
-      "learning_rate": 2.49289767338935e-05,
-      "loss": 0.2321,
-      "step": 245
-    },
-    {
-      "epoch": 5.0,
-      "grad_norm": 0.4782765805721283,
-      "learning_rate": 2.469448856791411e-05,
-      "loss": 0.1126,
-      "step": 250
-    },
-    {
-      "epoch": 5.0,
-      "eval_loss": 0.4687739610671997,
-      "eval_runtime": 52.1992,
-      "eval_samples_per_second": 3.831,
-      "eval_steps_per_second": 0.479,
-      "step": 250
+      "epoch": 7.0,
+      "eval_loss": 0.5998682379722595,
+      "eval_runtime": 3.3736,
+      "eval_samples_per_second": 4.446,
+      "eval_steps_per_second": 0.593,
+      "step": 238
     }
   ],
-  "logging_steps": 5,
-  "max_steps": 800,
+  "logging_steps": 10,
+  "max_steps": 250,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 16,
+  "num_train_epochs": 8,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -416,8 +243,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 4.646855528216986e+16,
-  "train_batch_size": 2,
+  "total_flos": 2.291052678921216e+16,
+  "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:44a91374d47e061d44848107bfc25ebd1ed4e3cf32bfc6349d577cac835076d2
+oid sha256:057e998df0396c8c0743c2e8486036bb54b886294b8c5da9a7b7083bcb4e9d62
 size 5624