starlineventures/chatgpt

README.md

@@ -36,7 +36,7 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 0.0001
-- train_batch_size: 3
+- train_batch_size: 2
 - eval_batch_size: 16
 - seed: 3407
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08

adapter_config.json

@@ -23,13 +23,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "o_proj",
-    "up_proj",
-    "v_proj",
-    "k_proj",
     "gate_proj",
+    "down_proj",
+    "o_proj",
     "q_proj",
-    "down_proj"
+    "v_proj",
+    "up_proj",
+    "k_proj"
   ],
   "task_type": null,
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:073c24e6f5e00a1b88481cdfd1ea77dcab74b65cafb5fd6a7c44eac81a3910d5
+oid sha256:b39c284313876374a91b0b1225a89102450223a6bf61fe2f95ec5efcf735cc18
 size 94422368

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 3.0,
     "total_flos": 0.0,
-    "train_loss": 0.4677507205336702,
-    "train_runtime": 113.1172,
-    "train_samples_per_second": 5.384,
-    "train_steps_per_second": 1.803
+    "train_loss": 0.5464926295810275,
+    "train_runtime": 144.0922,
+    "train_samples_per_second": 4.226,
+    "train_steps_per_second": 2.124
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 3.0,
     "total_flos": 0.0,
-    "train_loss": 0.4677507205336702,
-    "train_runtime": 113.1172,
-    "train_samples_per_second": 5.384,
-    "train_steps_per_second": 1.803
+    "train_loss": 0.5464926295810275,
+    "train_runtime": 144.0922,
+    "train_samples_per_second": 4.226,
+    "train_steps_per_second": 2.124
 }

trainer_state.json

@@ -3,163 +3,233 @@
   "best_model_checkpoint": null,
   "epoch": 3.0,
   "eval_steps": 500,
-  "global_step": 204,
+  "global_step": 306,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.14705882352941177,
-      "grad_norm": 6.151556968688965,
-      "learning_rate": 9.509803921568627e-05,
-      "loss": 0.8078,
+      "epoch": 0.09803921568627451,
+      "grad_norm": 19.516904830932617,
+      "learning_rate": 9.673202614379085e-05,
+      "loss": 0.885,
       "step": 10
     },
     {
-      "epoch": 0.29411764705882354,
-      "grad_norm": 5.537124156951904,
-      "learning_rate": 9.019607843137255e-05,
-      "loss": 0.5717,
+      "epoch": 0.19607843137254902,
+      "grad_norm": 28.905141830444336,
+      "learning_rate": 9.34640522875817e-05,
+      "loss": 0.684,
       "step": 20
     },
     {
-      "epoch": 0.4411764705882353,
-      "grad_norm": 4.549717426300049,
-      "learning_rate": 8.529411764705883e-05,
-      "loss": 0.6521,
+      "epoch": 0.29411764705882354,
+      "grad_norm": 17.932239532470703,
+      "learning_rate": 9.019607843137255e-05,
+      "loss": 0.5972,
       "step": 30
     },
     {
-      "epoch": 0.5882352941176471,
-      "grad_norm": 2.5774660110473633,
-      "learning_rate": 8.039215686274511e-05,
-      "loss": 0.4809,
+      "epoch": 0.39215686274509803,
+      "grad_norm": 103.63544464111328,
+      "learning_rate": 8.692810457516341e-05,
+      "loss": 1.0098,
       "step": 40
     },
     {
-      "epoch": 0.7352941176470589,
-      "grad_norm": 2.7017040252685547,
-      "learning_rate": 7.549019607843137e-05,
-      "loss": 0.4491,
+      "epoch": 0.49019607843137253,
+      "grad_norm": 22.439851760864258,
+      "learning_rate": 8.366013071895425e-05,
+      "loss": 0.9728,
       "step": 50
     },
     {
-      "epoch": 0.8823529411764706,
-      "grad_norm": 1.613568902015686,
-      "learning_rate": 7.058823529411765e-05,
-      "loss": 0.5115,
+      "epoch": 0.5882352941176471,
+      "grad_norm": 20.24323081970215,
+      "learning_rate": 8.039215686274511e-05,
+      "loss": 0.5056,
       "step": 60
     },
     {
-      "epoch": 1.0294117647058822,
-      "grad_norm": 2.709432601928711,
-      "learning_rate": 6.568627450980392e-05,
-      "loss": 0.55,
+      "epoch": 0.6862745098039216,
+      "grad_norm": 21.69275665283203,
+      "learning_rate": 7.712418300653595e-05,
+      "loss": 0.5228,
       "step": 70
     },
     {
-      "epoch": 1.1764705882352942,
-      "grad_norm": 3.952850818634033,
-      "learning_rate": 6.078431372549019e-05,
-      "loss": 0.4256,
+      "epoch": 0.7843137254901961,
+      "grad_norm": 21.725149154663086,
+      "learning_rate": 7.385620915032681e-05,
+      "loss": 0.5238,
       "step": 80
     },
     {
-      "epoch": 1.3235294117647058,
-      "grad_norm": 2.875826358795166,
-      "learning_rate": 5.588235294117647e-05,
-      "loss": 0.4734,
+      "epoch": 0.8823529411764706,
+      "grad_norm": 22.388992309570312,
+      "learning_rate": 7.058823529411765e-05,
+      "loss": 0.5995,
       "step": 90
     },
     {
-      "epoch": 1.4705882352941178,
-      "grad_norm": 2.6799256801605225,
-      "learning_rate": 5.0980392156862745e-05,
-      "loss": 0.4256,
+      "epoch": 0.9803921568627451,
+      "grad_norm": 26.396589279174805,
+      "learning_rate": 6.73202614379085e-05,
+      "loss": 0.6626,
       "step": 100
     },
     {
-      "epoch": 1.6176470588235294,
-      "grad_norm": 1.358408808708191,
-      "learning_rate": 4.607843137254902e-05,
-      "loss": 0.4089,
+      "epoch": 1.0784313725490196,
+      "grad_norm": 30.482742309570312,
+      "learning_rate": 6.405228758169934e-05,
+      "loss": 0.4304,
       "step": 110
     },
     {
-      "epoch": 1.7647058823529411,
-      "grad_norm": 1.366873025894165,
-      "learning_rate": 4.11764705882353e-05,
-      "loss": 0.4921,
+      "epoch": 1.1764705882352942,
+      "grad_norm": 25.137704849243164,
+      "learning_rate": 6.078431372549019e-05,
+      "loss": 0.5027,
       "step": 120
     },
     {
-      "epoch": 1.9117647058823528,
-      "grad_norm": 3.361318588256836,
-      "learning_rate": 3.627450980392157e-05,
-      "loss": 0.4334,
+      "epoch": 1.2745098039215685,
+      "grad_norm": 26.78306007385254,
+      "learning_rate": 5.7516339869281044e-05,
+      "loss": 0.5641,
       "step": 130
     },
     {
-      "epoch": 2.0588235294117645,
-      "grad_norm": 1.263223648071289,
-      "learning_rate": 3.137254901960784e-05,
-      "loss": 0.3971,
+      "epoch": 1.3725490196078431,
+      "grad_norm": 41.55208206176758,
+      "learning_rate": 5.4248366013071894e-05,
+      "loss": 0.4953,
       "step": 140
     },
     {
-      "epoch": 2.2058823529411766,
-      "grad_norm": 1.3559168577194214,
-      "learning_rate": 2.647058823529412e-05,
-      "loss": 0.3945,
+      "epoch": 1.4705882352941178,
+      "grad_norm": 23.749731063842773,
+      "learning_rate": 5.0980392156862745e-05,
+      "loss": 0.4578,
       "step": 150
     },
     {
-      "epoch": 2.3529411764705883,
-      "grad_norm": 1.3529400825500488,
-      "learning_rate": 2.1568627450980395e-05,
-      "loss": 0.4233,
+      "epoch": 1.5686274509803921,
+      "grad_norm": 23.935508728027344,
+      "learning_rate": 4.77124183006536e-05,
+      "loss": 0.401,
       "step": 160
     },
     {
-      "epoch": 2.5,
-      "grad_norm": 1.8559266328811646,
-      "learning_rate": 1.6666666666666667e-05,
-      "loss": 0.3216,
+      "epoch": 1.6666666666666665,
+      "grad_norm": 22.33506202697754,
+      "learning_rate": 4.4444444444444447e-05,
+      "loss": 0.4554,
       "step": 170
     },
     {
-      "epoch": 2.6470588235294117,
-      "grad_norm": 1.4294967651367188,
-      "learning_rate": 1.1764705882352942e-05,
-      "loss": 0.4001,
+      "epoch": 1.7647058823529411,
+      "grad_norm": 23.189788818359375,
+      "learning_rate": 4.11764705882353e-05,
+      "loss": 0.6165,
       "step": 180
     },
     {
-      "epoch": 2.7941176470588234,
-      "grad_norm": 1.6639864444732666,
-      "learning_rate": 6.862745098039216e-06,
-      "loss": 0.3968,
+      "epoch": 1.8627450980392157,
+      "grad_norm": 21.400800704956055,
+      "learning_rate": 3.790849673202614e-05,
+      "loss": 0.3971,
       "step": 190
     },
+    {
+      "epoch": 1.9607843137254903,
+      "grad_norm": 20.903608322143555,
+      "learning_rate": 3.464052287581699e-05,
+      "loss": 0.5734,
+      "step": 200
+    },
+    {
+      "epoch": 2.0588235294117645,
+      "grad_norm": 23.240053176879883,
+      "learning_rate": 3.137254901960784e-05,
+      "loss": 0.4406,
+      "step": 210
+    },
+    {
+      "epoch": 2.156862745098039,
+      "grad_norm": 24.06924057006836,
+      "learning_rate": 2.8104575163398693e-05,
+      "loss": 0.4119,
+      "step": 220
+    },
+    {
+      "epoch": 2.2549019607843137,
+      "grad_norm": 24.296913146972656,
+      "learning_rate": 2.4836601307189544e-05,
+      "loss": 0.5279,
+      "step": 230
+    },
+    {
+      "epoch": 2.3529411764705883,
+      "grad_norm": 20.898887634277344,
+      "learning_rate": 2.1568627450980395e-05,
+      "loss": 0.5091,
+      "step": 240
+    },
+    {
+      "epoch": 2.450980392156863,
+      "grad_norm": 22.029281616210938,
+      "learning_rate": 1.8300653594771242e-05,
+      "loss": 0.4114,
+      "step": 250
+    },
+    {
+      "epoch": 2.549019607843137,
+      "grad_norm": 23.726993560791016,
+      "learning_rate": 1.5032679738562091e-05,
+      "loss": 0.4127,
+      "step": 260
+    },
+    {
+      "epoch": 2.6470588235294117,
+      "grad_norm": 20.965084075927734,
+      "learning_rate": 1.1764705882352942e-05,
+      "loss": 0.4927,
+      "step": 270
+    },
+    {
+      "epoch": 2.7450980392156863,
+      "grad_norm": 21.0610294342041,
+      "learning_rate": 8.496732026143791e-06,
+      "loss": 0.5405,
+      "step": 280
+    },
+    {
+      "epoch": 2.843137254901961,
+      "grad_norm": 20.865800857543945,
+      "learning_rate": 5.228758169934641e-06,
+      "loss": 0.4165,
+      "step": 290
+    },
     {
       "epoch": 2.9411764705882355,
-      "grad_norm": 1.9823282957077026,
+      "grad_norm": 21.134117126464844,
       "learning_rate": 1.96078431372549e-06,
-      "loss": 0.3785,
-      "step": 200
+      "loss": 0.4396,
+      "step": 300
     },
     {
       "epoch": 3.0,
-      "step": 204,
+      "step": 306,
       "total_flos": 0.0,
-      "train_loss": 0.4677507205336702,
-      "train_runtime": 113.1172,
-      "train_samples_per_second": 5.384,
-      "train_steps_per_second": 1.803
+      "train_loss": 0.5464926295810275,
+      "train_runtime": 144.0922,
+      "train_samples_per_second": 4.226,
+      "train_steps_per_second": 2.124
     }
   ],
   "logging_steps": 10,
-  "max_steps": 204,
+  "max_steps": 306,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 3,
   "save_steps": 500,
@@ -176,7 +246,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 3,
+  "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4f3ded48d9febf2f6f7ca41ec8be465c369d7768ebc7cf6b7b294f7132b117a7
+oid sha256:053be5ab108aa7512c29fea8130432a8346c0917d936d5ea4e067ce7993a94c3
 size 5432