Nivedita28/atomicdeepseek

README.md

@@ -37,9 +37,11 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 0.0001
-- train_batch_size: 5
+- train_batch_size: 1
 - eval_batch_size: 16
 - seed: 3407
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 2
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - num_epochs: 3

adapter_config.json

@@ -23,13 +23,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "k_proj",
+    "v_proj",
     "down_proj",
     "q_proj",
-    "o_proj",
+    "up_proj",
+    "k_proj",
     "gate_proj",
-    "v_proj",
-    "up_proj"
+    "o_proj"
   ],
   "task_type": null,
   "use_dora": false,

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 3.0,
-    "total_flos": 1.842821417926656e+16,
-    "train_loss": 0.37092622672207654,
-    "train_runtime": 190.7397,
-    "train_samples_per_second": 5.033,
-    "train_steps_per_second": 1.007
+    "total_flos": 0.0,
+    "train_loss": 0.04108585631474852,
+    "train_runtime": 388.3502,
+    "train_samples_per_second": 2.472,
+    "train_steps_per_second": 1.236
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e8dd6742316c796595513bbdedee22b2939a3cd37c4f2fe068b0465af5d9a0d0
+oid sha256:0823e7beeeeb987d2ad98788b70736eddd6aab9fda28190539a97f92eaa3eca5
 size 3554214752

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 3.0,
-    "total_flos": 1.842821417926656e+16,
-    "train_loss": 0.37092622672207654,
-    "train_runtime": 190.7397,
-    "train_samples_per_second": 5.033,
-    "train_steps_per_second": 1.007
+    "total_flos": 0.0,
+    "train_loss": 0.04108585631474852,
+    "train_runtime": 388.3502,
+    "train_samples_per_second": 2.472,
+    "train_steps_per_second": 1.236
 }

trainer_state.json

@@ -3,184 +3,387 @@
   "best_model_checkpoint": null,
   "epoch": 3.0,
   "eval_steps": 500,
-  "global_step": 192,
+  "global_step": 480,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.15625,
-      "grad_norm": 0.10847697407007217,
-      "learning_rate": 9.479166666666666e-05,
-      "loss": 5.5359,
+      "epoch": 0.0625,
+      "grad_norm": 0.20695899426937103,
+      "learning_rate": 9.791666666666667e-05,
+      "loss": 0.0491,
       "step": 10
     },
     {
-      "epoch": 0.3125,
-      "grad_norm": 0.0398186556994915,
-      "learning_rate": 8.958333333333335e-05,
-      "loss": 0.1263,
+      "epoch": 0.125,
+      "grad_norm": 0.18051496148109436,
+      "learning_rate": 9.583333333333334e-05,
+      "loss": 0.059,
       "step": 20
     },
     {
-      "epoch": 0.46875,
-      "grad_norm": 0.03957119584083557,
-      "learning_rate": 8.4375e-05,
-      "loss": 0.1074,
+      "epoch": 0.1875,
+      "grad_norm": 0.13114741444587708,
+      "learning_rate": 9.375e-05,
+      "loss": 0.0602,
       "step": 30
     },
     {
-      "epoch": 0.625,
-      "grad_norm": 0.027517227455973625,
-      "learning_rate": 7.916666666666666e-05,
-      "loss": 0.0965,
+      "epoch": 0.25,
+      "grad_norm": 0.18424402177333832,
+      "learning_rate": 9.166666666666667e-05,
+      "loss": 0.0497,
       "step": 40
     },
     {
-      "epoch": 0.78125,
-      "grad_norm": 0.02527707628905773,
-      "learning_rate": 7.395833333333335e-05,
-      "loss": 0.0884,
+      "epoch": 0.3125,
+      "grad_norm": 0.19860927760601044,
+      "learning_rate": 8.958333333333335e-05,
+      "loss": 0.0542,
       "step": 50
     },
+    {
+      "epoch": 0.375,
+      "grad_norm": 0.16605813801288605,
+      "learning_rate": 8.75e-05,
+      "loss": 0.0494,
+      "step": 60
+    },
+    {
+      "epoch": 0.4375,
+      "grad_norm": 0.16891923546791077,
+      "learning_rate": 8.541666666666666e-05,
+      "loss": 0.0544,
+      "step": 70
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 0.152054101228714,
+      "learning_rate": 8.333333333333334e-05,
+      "loss": 0.0543,
+      "step": 80
+    },
+    {
+      "epoch": 0.5625,
+      "grad_norm": 0.2942846715450287,
+      "learning_rate": 8.125000000000001e-05,
+      "loss": 0.0594,
+      "step": 90
+    },
+    {
+      "epoch": 0.625,
+      "grad_norm": 0.20024558901786804,
+      "learning_rate": 7.916666666666666e-05,
+      "loss": 0.0594,
+      "step": 100
+    },
+    {
+      "epoch": 0.6875,
+      "grad_norm": 0.2064783126115799,
+      "learning_rate": 7.708333333333334e-05,
+      "loss": 0.0602,
+      "step": 110
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 0.20470409095287323,
+      "learning_rate": 7.500000000000001e-05,
+      "loss": 0.0597,
+      "step": 120
+    },
+    {
+      "epoch": 0.75,
+      "eval_runtime": 5.9648,
+      "eval_samples_per_second": 13.412,
+      "eval_steps_per_second": 0.838,
+      "step": 120
+    },
+    {
+      "epoch": 0.8125,
+      "grad_norm": 0.1695672571659088,
+      "learning_rate": 7.291666666666667e-05,
+      "loss": 0.0659,
+      "step": 130
+    },
+    {
+      "epoch": 0.875,
+      "grad_norm": 0.1826922595500946,
+      "learning_rate": 7.083333333333334e-05,
+      "loss": 0.0726,
+      "step": 140
+    },
     {
       "epoch": 0.9375,
-      "grad_norm": 0.02493639849126339,
+      "grad_norm": 0.17495571076869965,
       "learning_rate": 6.875e-05,
-      "loss": 0.0884,
-      "step": 60
+      "loss": 0.0662,
+      "step": 150
     },
     {
       "epoch": 1.0,
-      "eval_runtime": 6.6509,
-      "eval_samples_per_second": 12.028,
-      "eval_steps_per_second": 0.752,
-      "step": 64
+      "grad_norm": 0.16560527682304382,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 0.0689,
+      "step": 160
     },
     {
-      "epoch": 1.09375,
-      "grad_norm": 0.02615288645029068,
-      "learning_rate": 6.354166666666666e-05,
-      "loss": 0.08,
-      "step": 70
+      "epoch": 1.0,
+      "eval_runtime": 5.4233,
+      "eval_samples_per_second": 14.751,
+      "eval_steps_per_second": 0.922,
+      "step": 160
+    },
+    {
+      "epoch": 1.0625,
+      "grad_norm": 0.15654611587524414,
+      "learning_rate": 6.458333333333334e-05,
+      "loss": 0.0461,
+      "step": 170
+    },
+    {
+      "epoch": 1.125,
+      "grad_norm": 0.19164249300956726,
+      "learning_rate": 6.25e-05,
+      "loss": 0.0396,
+      "step": 180
+    },
+    {
+      "epoch": 1.1875,
+      "grad_norm": 0.23551718890666962,
+      "learning_rate": 6.041666666666667e-05,
+      "loss": 0.0421,
+      "step": 190
     },
     {
       "epoch": 1.25,
-      "grad_norm": 0.027330700308084488,
+      "grad_norm": 0.21708479523658752,
       "learning_rate": 5.833333333333334e-05,
-      "loss": 0.0891,
-      "step": 80
+      "loss": 0.0336,
+      "step": 200
     },
     {
-      "epoch": 1.40625,
-      "grad_norm": 0.028688812628388405,
-      "learning_rate": 5.3125000000000004e-05,
-      "loss": 0.0947,
-      "step": 90
+      "epoch": 1.3125,
+      "grad_norm": 0.16753819584846497,
+      "learning_rate": 5.6250000000000005e-05,
+      "loss": 0.0457,
+      "step": 210
+    },
+    {
+      "epoch": 1.375,
+      "grad_norm": 0.19406142830848694,
+      "learning_rate": 5.4166666666666664e-05,
+      "loss": 0.0453,
+      "step": 220
+    },
+    {
+      "epoch": 1.4375,
+      "grad_norm": 0.16341769695281982,
+      "learning_rate": 5.208333333333334e-05,
+      "loss": 0.0363,
+      "step": 230
+    },
+    {
+      "epoch": 1.5,
+      "grad_norm": 0.20264258980751038,
+      "learning_rate": 5e-05,
+      "loss": 0.0342,
+      "step": 240
     },
     {
       "epoch": 1.5625,
-      "grad_norm": 0.030911659821867943,
+      "grad_norm": 0.170054629445076,
       "learning_rate": 4.791666666666667e-05,
-      "loss": 0.0803,
-      "step": 100
+      "loss": 0.0344,
+      "step": 250
     },
     {
-      "epoch": 1.71875,
-      "grad_norm": 0.028372356668114662,
-      "learning_rate": 4.270833333333333e-05,
-      "loss": 0.0791,
-      "step": 110
+      "epoch": 1.625,
+      "grad_norm": 0.2517170310020447,
+      "learning_rate": 4.5833333333333334e-05,
+      "loss": 0.0338,
+      "step": 260
+    },
+    {
+      "epoch": 1.6875,
+      "grad_norm": 0.2723373770713806,
+      "learning_rate": 4.375e-05,
+      "loss": 0.0349,
+      "step": 270
+    },
+    {
+      "epoch": 1.75,
+      "grad_norm": 0.2065780758857727,
+      "learning_rate": 4.166666666666667e-05,
+      "loss": 0.0431,
+      "step": 280
+    },
+    {
+      "epoch": 1.8125,
+      "grad_norm": 0.14495207369327545,
+      "learning_rate": 3.958333333333333e-05,
+      "loss": 0.0381,
+      "step": 290
     },
     {
       "epoch": 1.875,
-      "grad_norm": 0.022600064054131508,
+      "grad_norm": 0.16023993492126465,
       "learning_rate": 3.7500000000000003e-05,
-      "loss": 0.078,
-      "step": 120
+      "loss": 0.0341,
+      "step": 300
+    },
+    {
+      "epoch": 1.9375,
+      "grad_norm": 0.3680807054042816,
+      "learning_rate": 3.541666666666667e-05,
+      "loss": 0.0403,
+      "step": 310
     },
     {
       "epoch": 2.0,
-      "eval_runtime": 5.3296,
-      "eval_samples_per_second": 15.011,
-      "eval_steps_per_second": 0.938,
-      "step": 128
+      "grad_norm": 0.16842582821846008,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 0.0343,
+      "step": 320
     },
     {
-      "epoch": 2.03125,
-      "grad_norm": 0.02555151842534542,
-      "learning_rate": 3.229166666666667e-05,
-      "loss": 0.0811,
-      "step": 130
+      "epoch": 2.0,
+      "eval_runtime": 5.4235,
+      "eval_samples_per_second": 14.751,
+      "eval_steps_per_second": 0.922,
+      "step": 320
+    },
+    {
+      "epoch": 2.0625,
+      "grad_norm": 0.13369396328926086,
+      "learning_rate": 3.125e-05,
+      "loss": 0.0269,
+      "step": 330
+    },
+    {
+      "epoch": 2.125,
+      "grad_norm": 0.16148492693901062,
+      "learning_rate": 2.916666666666667e-05,
+      "loss": 0.0286,
+      "step": 340
     },
     {
       "epoch": 2.1875,
-      "grad_norm": 0.028641177341341972,
+      "grad_norm": 0.16799309849739075,
       "learning_rate": 2.7083333333333332e-05,
-      "loss": 0.0809,
-      "step": 140
+      "loss": 0.0276,
+      "step": 350
     },
     {
-      "epoch": 2.34375,
-      "grad_norm": 0.023602696135640144,
-      "learning_rate": 2.1875e-05,
-      "loss": 0.0738,
-      "step": 150
+      "epoch": 2.25,
+      "grad_norm": 0.15763401985168457,
+      "learning_rate": 2.5e-05,
+      "loss": 0.0233,
+      "step": 360
     },
     {
-      "epoch": 2.34375,
-      "eval_runtime": 5.3233,
-      "eval_samples_per_second": 15.028,
-      "eval_steps_per_second": 0.939,
-      "step": 150
+      "epoch": 2.3125,
+      "grad_norm": 0.2925902009010315,
+      "learning_rate": 2.2916666666666667e-05,
+      "loss": 0.0261,
+      "step": 370
+    },
+    {
+      "epoch": 2.375,
+      "grad_norm": 0.2722231149673462,
+      "learning_rate": 2.0833333333333336e-05,
+      "loss": 0.0268,
+      "step": 380
+    },
+    {
+      "epoch": 2.4375,
+      "grad_norm": 0.1751345992088318,
+      "learning_rate": 1.8750000000000002e-05,
+      "loss": 0.0248,
+      "step": 390
     },
     {
       "epoch": 2.5,
-      "grad_norm": 0.026037951931357384,
+      "grad_norm": 0.10352811962366104,
       "learning_rate": 1.6666666666666667e-05,
-      "loss": 0.0775,
-      "step": 160
+      "loss": 0.0234,
+      "step": 400
     },
     {
-      "epoch": 2.65625,
-      "grad_norm": 0.027666736394166946,
-      "learning_rate": 1.1458333333333333e-05,
-      "loss": 0.0829,
-      "step": 170
+      "epoch": 2.5625,
+      "grad_norm": 0.10212921351194382,
+      "learning_rate": 1.4583333333333335e-05,
+      "loss": 0.0278,
+      "step": 410
+    },
+    {
+      "epoch": 2.625,
+      "grad_norm": 0.19403204321861267,
+      "learning_rate": 1.25e-05,
+      "loss": 0.0254,
+      "step": 420
+    },
+    {
+      "epoch": 2.6875,
+      "grad_norm": 0.21788670122623444,
+      "learning_rate": 1.0416666666666668e-05,
+      "loss": 0.0227,
+      "step": 430
+    },
+    {
+      "epoch": 2.75,
+      "grad_norm": 0.1767636239528656,
+      "learning_rate": 8.333333333333334e-06,
+      "loss": 0.0253,
+      "step": 440
     },
     {
       "epoch": 2.8125,
-      "grad_norm": 0.025325452908873558,
+      "grad_norm": 0.18783213198184967,
       "learning_rate": 6.25e-06,
-      "loss": 0.0802,
-      "step": 180
+      "loss": 0.0284,
+      "step": 450
     },
     {
-      "epoch": 2.96875,
-      "grad_norm": 0.02601686492562294,
-      "learning_rate": 1.0416666666666667e-06,
-      "loss": 0.0858,
-      "step": 190
+      "epoch": 2.875,
+      "grad_norm": 0.1575060486793518,
+      "learning_rate": 4.166666666666667e-06,
+      "loss": 0.0257,
+      "step": 460
+    },
+    {
+      "epoch": 2.9375,
+      "grad_norm": 0.13178540766239166,
+      "learning_rate": 2.0833333333333334e-06,
+      "loss": 0.0271,
+      "step": 470
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 0.21420063078403473,
+      "learning_rate": 0.0,
+      "loss": 0.0234,
+      "step": 480
     },
     {
       "epoch": 3.0,
-      "eval_runtime": 5.3306,
-      "eval_samples_per_second": 15.008,
-      "eval_steps_per_second": 0.938,
-      "step": 192
+      "eval_runtime": 5.4227,
+      "eval_samples_per_second": 14.753,
+      "eval_steps_per_second": 0.922,
+      "step": 480
     },
     {
       "epoch": 3.0,
-      "step": 192,
-      "total_flos": 1.842821417926656e+16,
-      "train_loss": 0.37092622672207654,
-      "train_runtime": 190.7397,
-      "train_samples_per_second": 5.033,
-      "train_steps_per_second": 1.007
+      "step": 480,
+      "total_flos": 0.0,
+      "train_loss": 0.04108585631474852,
+      "train_runtime": 388.3502,
+      "train_samples_per_second": 2.472,
+      "train_steps_per_second": 1.236
     }
   ],
   "logging_steps": 10,
-  "max_steps": 192,
+  "max_steps": 480,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 3,
   "save_steps": 500,
@@ -196,8 +399,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.842821417926656e+16,
-  "train_batch_size": 5,
+  "total_flos": 0.0,
+  "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d54ce947218fd70f81e3e7c2d88f7e082fcc501ccbe553663dabd3839b514bde
+oid sha256:b40ff428d957547f7280b2289912b2261784bfe8a0bd220c4d53aa90cbd1575f
 size 5432