Upload folder using huggingface_hub

adapter_config.json

@@ -22,10 +22,10 @@
   "target_modules": [
     "qkv_proj",
     "gate_up_proj",
-    "o_proj",
-    "down_proj"
+    "down_proj",
+    "o_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,
-  "use_rslora": false
+  "use_rslora": true
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3380fafba548305bac72a211496228c87e172f5dfb41063a37070c04568b0c11
+oid sha256:714422880598a78601225359d0a2ea0b15762a21f326fd333b7c01e043ccf710
 size 445688440

all_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 2.0,
-    "total_flos": 7.49108138508288e+16,
-    "train_loss": 0.6441779155731201,
-    "train_runtime": 1839.4624,
-    "train_samples_per_second": 1.087,
-    "train_steps_per_second": 0.272
+    "epoch": 5.0,
+    "total_flos": 1.326817135982592e+17,
+    "train_loss": 0.1957190614938736,
+    "train_runtime": 1416.6552,
+    "train_samples_per_second": 0.565,
+    "train_steps_per_second": 0.035
 }

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 2.0,
-    "total_flos": 7.49108138508288e+16,
-    "train_loss": 0.6441779155731201,
-    "train_runtime": 1839.4624,
-    "train_samples_per_second": 1.087,
-    "train_steps_per_second": 0.272
+    "epoch": 5.0,
+    "total_flos": 1.326817135982592e+17,
+    "train_loss": 0.1957190614938736,
+    "train_runtime": 1416.6552,
+    "train_samples_per_second": 0.565,
+    "train_steps_per_second": 0.035
 }

trainer_state.json

@@ -1,202 +1,41 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.0,
+  "epoch": 5.0,
   "eval_steps": 500,
-  "global_step": 500,
+  "global_step": 50,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.08,
-      "grad_norm": 0.24795465171337128,
-      "learning_rate": 8.000000000000001e-06,
-      "loss": 0.9251,
+      "epoch": 2.0,
+      "grad_norm": 0.4947165846824646,
+      "learning_rate": 0.00021314021436425024,
+      "loss": 0.3985,
       "step": 20
     },
     {
-      "epoch": 0.16,
-      "grad_norm": 1.1689410209655762,
-      "learning_rate": 9.975414512725058e-06,
-      "loss": 1.1438,
+      "epoch": 4.0,
+      "grad_norm": 0.6319659352302551,
+      "learning_rate": 3.228009220820085e-05,
+      "loss": 0.082,
       "step": 40
     },
     {
-      "epoch": 0.24,
-      "grad_norm": 0.6293185949325562,
-      "learning_rate": 9.86663298624003e-06,
-      "loss": 0.9992,
-      "step": 60
-    },
-    {
-      "epoch": 0.32,
-      "grad_norm": 0.34476780891418457,
-      "learning_rate": 9.672822322997305e-06,
-      "loss": 0.8994,
-      "step": 80
-    },
-    {
-      "epoch": 0.4,
-      "grad_norm": 0.5161601901054382,
-      "learning_rate": 9.397368756032445e-06,
-      "loss": 0.7961,
-      "step": 100
-    },
-    {
-      "epoch": 0.48,
-      "grad_norm": 0.8914580941200256,
-      "learning_rate": 9.045084971874738e-06,
-      "loss": 0.8611,
-      "step": 120
-    },
-    {
-      "epoch": 0.56,
-      "grad_norm": 7.905534744262695,
-      "learning_rate": 8.622126023955446e-06,
-      "loss": 0.7744,
-      "step": 140
-    },
-    {
-      "epoch": 0.64,
-      "grad_norm": 0.37820664048194885,
-      "learning_rate": 8.135881792367686e-06,
-      "loss": 0.5589,
-      "step": 160
-    },
-    {
-      "epoch": 0.72,
-      "grad_norm": 0.931911289691925,
-      "learning_rate": 7.594847868906076e-06,
-      "loss": 0.6103,
-      "step": 180
-    },
-    {
-      "epoch": 0.8,
-      "grad_norm": 2.477545738220215,
-      "learning_rate": 7.008477123264849e-06,
-      "loss": 0.6874,
-      "step": 200
-    },
-    {
-      "epoch": 0.88,
-      "grad_norm": 11.537752151489258,
-      "learning_rate": 6.387014543809224e-06,
-      "loss": 0.5905,
-      "step": 220
-    },
-    {
-      "epoch": 0.96,
-      "grad_norm": 3.1809520721435547,
-      "learning_rate": 5.74131823855921e-06,
-      "loss": 0.6467,
-      "step": 240
-    },
-    {
-      "epoch": 1.04,
-      "grad_norm": 0.871376097202301,
-      "learning_rate": 5.082669723831793e-06,
-      "loss": 0.5445,
-      "step": 260
-    },
-    {
-      "epoch": 1.12,
-      "grad_norm": 4.610472679138184,
-      "learning_rate": 4.42257681515207e-06,
-      "loss": 0.5524,
-      "step": 280
-    },
-    {
-      "epoch": 1.2,
-      "grad_norm": 0.9732345342636108,
-      "learning_rate": 3.7725725642960047e-06,
-      "loss": 0.4875,
-      "step": 300
-    },
-    {
-      "epoch": 1.28,
-      "grad_norm": 1.0022259950637817,
-      "learning_rate": 3.1440137554088957e-06,
-      "loss": 0.5341,
-      "step": 320
-    },
-    {
-      "epoch": 1.3599999999999999,
-      "grad_norm": 0.8104701042175293,
-      "learning_rate": 2.5478824808474613e-06,
-      "loss": 0.4995,
-      "step": 340
-    },
-    {
-      "epoch": 1.44,
-      "grad_norm": 1.6915416717529297,
-      "learning_rate": 1.9945942635848745e-06,
-      "loss": 0.4579,
-      "step": 360
-    },
-    {
-      "epoch": 1.52,
-      "grad_norm": 0.820740282535553,
-      "learning_rate": 1.4938160786375571e-06,
-      "loss": 0.4705,
-      "step": 380
-    },
-    {
-      "epoch": 1.6,
-      "grad_norm": 1.1549257040023804,
-      "learning_rate": 1.0542974530180327e-06,
-      "loss": 0.5349,
-      "step": 400
-    },
-    {
-      "epoch": 1.6800000000000002,
-      "grad_norm": 2.226872444152832,
-      "learning_rate": 6.837175952121305e-07,
-      "loss": 0.5651,
-      "step": 420
-    },
-    {
-      "epoch": 1.76,
-      "grad_norm": 0.8398928046226501,
-      "learning_rate": 3.885512251130763e-07,
-      "loss": 0.5404,
-      "step": 440
-    },
-    {
-      "epoch": 1.8399999999999999,
-      "grad_norm": 1.174414038658142,
-      "learning_rate": 1.7395544861325718e-07,
-      "loss": 0.5067,
-      "step": 460
-    },
-    {
-      "epoch": 1.92,
-      "grad_norm": 0.28812170028686523,
-      "learning_rate": 4.367965336512403e-08,
-      "loss": 0.4795,
-      "step": 480
-    },
-    {
-      "epoch": 2.0,
-      "grad_norm": 0.9047369956970215,
-      "learning_rate": 0.0,
-      "loss": 0.4385,
-      "step": 500
-    },
-    {
-      "epoch": 2.0,
-      "step": 500,
-      "total_flos": 7.49108138508288e+16,
-      "train_loss": 0.6441779155731201,
-      "train_runtime": 1839.4624,
-      "train_samples_per_second": 1.087,
-      "train_steps_per_second": 0.272
+      "epoch": 5.0,
+      "step": 50,
+      "total_flos": 1.326817135982592e+17,
+      "train_loss": 0.1957190614938736,
+      "train_runtime": 1416.6552,
+      "train_samples_per_second": 0.565,
+      "train_steps_per_second": 0.035
     }
   ],
   "logging_steps": 20,
-  "max_steps": 500,
+  "max_steps": 50,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 2,
+  "num_train_epochs": 6,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -210,8 +49,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 7.49108138508288e+16,
-  "train_batch_size": 1,
+  "total_flos": 1.326817135982592e+17,
+  "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f6251580ffe6787e96068e8d94ea281067dbaa5f83ffe8fde0c66938195a435
+oid sha256:82da22da0251373d3062d31edc1f33a1d968f14dc7d511dff90a44fbc16d8b79
 size 5624