Training in progress, step 5200, checkpoint

last-checkpoint/adapter_config.json

@@ -33,13 +33,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "k_proj",
-    "o_proj",
+    "up_proj",
     "down_proj",
-    "v_proj",
     "gate_proj",
-    "up_proj",
-    "q_proj"
+    "q_proj",
+    "o_proj",
+    "v_proj",
+    "k_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:65b82decf38acdcd567075547ee240a671a4f19d269678a88a3a289761b3f0ef
+oid sha256:7b399e6c3c5830b10f6d3e1e89bd66e1998783b11fd12e3eeb4430bd526bc3a9
 size 119597408

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:84ee9654e961eb5d6fe07a13dd958dc3ad6ac419fc811c6661f3896869da4af4
-size 61232293
+oid sha256:4d382a155b75cafd529c54bda80b686d3d7c7a97e9d88c8aa8595145b4d7db6c
+size 62398051

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:098b29492211804ab324a36f37466821d948280bb74fce4ba895c03f13ecd878
+oid sha256:f4a9f217e852f439efa6bd32fde98d6867f11aa6ea13ddc021ba10af6a0b0934
 size 14645

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5ac1c46a2776d12775d23d0f587efc112188137ce2140da35bc15d301c9f620e
+oid sha256:afb655591367e732af512c0b489f6652e710a205ef4e8286da8729a948980ee0
 size 1383

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:048a46eeda80b2b8e45b52734073a9a9cb0b798967bc27cfaf7ff29d27c34922
+oid sha256:20b49bac01a6e00406b4ef3d368e5f8848c98e2ae220103f55426e9ca9511ec0
 size 1465

last-checkpoint/trainer_state.json

@@ -2,229 +2,383 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.6,
+  "epoch": 2.7733333333333334,
   "eval_steps": 500,
-  "global_step": 3000,
+  "global_step": 5200,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.05333333333333334,
-      "grad_norm": 0.23457413911819458,
+      "grad_norm": 0.2321965992450714,
       "learning_rate": 0.00019996188073020713,
       "loss": 0.7121,
       "step": 100
     },
     {
       "epoch": 0.10666666666666667,
-      "grad_norm": 0.20509809255599976,
+      "grad_norm": 0.2028595358133316,
       "learning_rate": 0.00019964771237806272,
-      "loss": 0.5663,
+      "loss": 0.5658,
       "step": 200
     },
     {
       "epoch": 0.16,
-      "grad_norm": 0.1555841565132141,
+      "grad_norm": 0.15056529641151428,
       "learning_rate": 0.0001990171979823181,
-      "loss": 0.572,
+      "loss": 0.5719,
       "step": 300
     },
     {
       "epoch": 0.21333333333333335,
-      "grad_norm": 0.18324309587478638,
+      "grad_norm": 0.18415790796279907,
       "learning_rate": 0.00019807233920191516,
-      "loss": 0.5441,
+      "loss": 0.5437,
       "step": 400
     },
     {
       "epoch": 0.26666666666666666,
-      "grad_norm": 0.20826920866966248,
+      "grad_norm": 0.21036416292190552,
       "learning_rate": 0.00019681613562740963,
-      "loss": 0.5466,
+      "loss": 0.5461,
       "step": 500
     },
     {
       "epoch": 0.32,
-      "grad_norm": 0.209468275308609,
+      "grad_norm": 0.2181209772825241,
       "learning_rate": 0.00019525257525833772,
-      "loss": 0.5744,
+      "loss": 0.5743,
       "step": 600
     },
     {
       "epoch": 0.37333333333333335,
-      "grad_norm": 0.2050088346004486,
+      "grad_norm": 0.197322279214859,
       "learning_rate": 0.00019338662184273617,
       "loss": 0.5383,
       "step": 700
     },
     {
       "epoch": 0.4266666666666667,
-      "grad_norm": 0.18242451548576355,
+      "grad_norm": 0.17933343350887299,
       "learning_rate": 0.0001912241991190079,
-      "loss": 0.5563,
+      "loss": 0.555,
       "step": 800
     },
     {
       "epoch": 0.48,
-      "grad_norm": 0.2052665799856186,
+      "grad_norm": 0.20802412927150726,
       "learning_rate": 0.0001887721720101592,
-      "loss": 0.5705,
+      "loss": 0.5698,
       "step": 900
     },
     {
       "epoch": 0.5333333333333333,
-      "grad_norm": 0.27574437856674194,
+      "grad_norm": 0.2803187966346741,
       "learning_rate": 0.00018603832483011145,
-      "loss": 0.5505,
+      "loss": 0.5502,
       "step": 1000
     },
     {
       "epoch": 0.5866666666666667,
-      "grad_norm": 0.30421534180641174,
+      "grad_norm": 0.3153877556324005,
       "learning_rate": 0.00018303133657127273,
-      "loss": 0.5265,
+      "loss": 0.5262,
       "step": 1100
     },
     {
       "epoch": 0.64,
-      "grad_norm": 0.2369503527879715,
+      "grad_norm": 0.23312756419181824,
       "learning_rate": 0.00017976075335182407,
-      "loss": 0.5407,
+      "loss": 0.5405,
       "step": 1200
     },
     {
       "epoch": 0.6933333333333334,
-      "grad_norm": 0.29343071579933167,
+      "grad_norm": 0.28737562894821167,
       "learning_rate": 0.0001762369581101898,
-      "loss": 0.5366,
+      "loss": 0.5363,
       "step": 1300
     },
     {
       "epoch": 0.7466666666666667,
-      "grad_norm": 0.1794026792049408,
+      "grad_norm": 0.180097296833992,
       "learning_rate": 0.00017247113764290082,
       "loss": 0.5745,
       "step": 1400
     },
     {
       "epoch": 0.8,
-      "grad_norm": 0.2688276469707489,
+      "grad_norm": 0.2748386859893799,
       "learning_rate": 0.00016847524709049475,
-      "loss": 0.4972,
+      "loss": 0.4973,
       "step": 1500
     },
     {
       "epoch": 0.8533333333333334,
-      "grad_norm": 0.17949211597442627,
+      "grad_norm": 0.17656417191028595,
       "learning_rate": 0.0001642619719841971,
-      "loss": 0.543,
+      "loss": 0.5428,
       "step": 1600
     },
     {
       "epoch": 0.9066666666666666,
-      "grad_norm": 0.24293017387390137,
+      "grad_norm": 0.23921695351600647,
       "learning_rate": 0.0001598446879738714,
-      "loss": 0.5566,
+      "loss": 0.5567,
       "step": 1700
     },
     {
       "epoch": 0.96,
-      "grad_norm": 0.17684029042720795,
+      "grad_norm": 0.17979004979133606,
       "learning_rate": 0.00015523741836508866,
-      "loss": 0.5058,
+      "loss": 0.5056,
       "step": 1800
     },
     {
       "epoch": 1.0133333333333334,
-      "grad_norm": 0.2043057680130005,
+      "grad_norm": 0.19811832904815674,
       "learning_rate": 0.00015045478960011983,
-      "loss": 0.5232,
+      "loss": 0.523,
       "step": 1900
     },
     {
       "epoch": 1.0666666666666667,
-      "grad_norm": 0.21241049468517303,
+      "grad_norm": 0.21939535439014435,
       "learning_rate": 0.00014551198482418425,
-      "loss": 0.4672,
+      "loss": 0.4665,
       "step": 2000
     },
     {
       "epoch": 1.12,
-      "grad_norm": 0.2556317150592804,
+      "grad_norm": 0.2566276788711548,
       "learning_rate": 0.00014042469568436382,
-      "loss": 0.4465,
+      "loss": 0.4455,
       "step": 2100
     },
     {
       "epoch": 1.1733333333333333,
-      "grad_norm": 0.26184341311454773,
+      "grad_norm": 0.2593347132205963,
       "learning_rate": 0.00013520907251420486,
-      "loss": 0.4223,
+      "loss": 0.4217,
       "step": 2200
     },
     {
       "epoch": 1.2266666666666666,
-      "grad_norm": 0.39901483058929443,
+      "grad_norm": 0.41315367817878723,
       "learning_rate": 0.00012988167306215355,
-      "loss": 0.4134,
+      "loss": 0.4131,
       "step": 2300
     },
     {
       "epoch": 1.28,
-      "grad_norm": 0.2483246624469757,
+      "grad_norm": 0.25102540850639343,
       "learning_rate": 0.00012445940992659383,
-      "loss": 0.4383,
+      "loss": 0.4376,
       "step": 2400
     },
     {
       "epoch": 1.3333333333333333,
-      "grad_norm": 0.30793285369873047,
+      "grad_norm": 0.30843883752822876,
       "learning_rate": 0.00011895949686436324,
-      "loss": 0.4512,
+      "loss": 0.4509,
       "step": 2500
     },
     {
       "epoch": 1.3866666666666667,
-      "grad_norm": 0.21273203194141388,
+      "grad_norm": 0.32662779092788696,
       "learning_rate": 0.00011339939414319763,
-      "loss": 0.4043,
+      "loss": 0.404,
       "step": 2600
     },
     {
       "epoch": 1.44,
-      "grad_norm": 0.21911175549030304,
+      "grad_norm": 0.24675920605659485,
       "learning_rate": 0.00010779675311159134,
-      "loss": 0.4142,
+      "loss": 0.3814,
       "step": 2700
     },
     {
       "epoch": 1.4933333333333334,
-      "grad_norm": 0.18140286207199097,
+      "grad_norm": 0.3210122883319855,
       "learning_rate": 0.00010216936016204415,
-      "loss": 0.4625,
+      "loss": 0.4201,
       "step": 2800
     },
     {
       "epoch": 1.5466666666666666,
-      "grad_norm": 0.26568442583084106,
+      "grad_norm": 0.30080801248550415,
       "learning_rate": 9.653508026559144e-05,
-      "loss": 0.4615,
+      "loss": 0.3838,
       "step": 2900
     },
     {
       "epoch": 1.6,
-      "grad_norm": 0.2561465799808502,
+      "grad_norm": 0.22295641899108887,
       "learning_rate": 9.091180025687534e-05,
-      "loss": 0.4113,
+      "loss": 0.3681,
       "step": 3000
+    },
+    {
+      "epoch": 1.6533333333333333,
+      "grad_norm": 0.34729060530662537,
+      "learning_rate": 8.531737204980706e-05,
+      "loss": 0.3631,
+      "step": 3100
+    },
+    {
+      "epoch": 1.7066666666666666,
+      "grad_norm": 0.36987531185150146,
+      "learning_rate": 7.976955596408955e-05,
+      "loss": 0.3779,
+      "step": 3200
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 0.510287880897522,
+      "learning_rate": 7.428596434251988e-05,
+      "loss": 0.3722,
+      "step": 3300
+    },
+    {
+      "epoch": 1.8133333333333335,
+      "grad_norm": 0.3648888170719147,
+      "learning_rate": 6.8884005638065e-05,
+      "loss": 0.3578,
+      "step": 3400
+    },
+    {
+      "epoch": 1.8666666666666667,
+      "grad_norm": 0.3440244793891907,
+      "learning_rate": 6.358082914821574e-05,
+      "loss": 0.3657,
+      "step": 3500
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 0.32496246695518494,
+      "learning_rate": 5.839327057206678e-05,
+      "loss": 0.3769,
+      "step": 3600
+    },
+    {
+      "epoch": 1.9733333333333334,
+      "grad_norm": 0.36488619446754456,
+      "learning_rate": 5.33377985629605e-05,
+      "loss": 0.3763,
+      "step": 3700
+    },
+    {
+      "epoch": 2.026666666666667,
+      "grad_norm": 0.3176770806312561,
+      "learning_rate": 4.843046244637005e-05,
+      "loss": 0.3526,
+      "step": 3800
+    },
+    {
+      "epoch": 2.08,
+      "grad_norm": 0.44687092304229736,
+      "learning_rate": 4.368684126899957e-05,
+      "loss": 0.4376,
+      "step": 3900
+    },
+    {
+      "epoch": 2.1333333333333333,
+      "grad_norm": 0.33409279584884644,
+      "learning_rate": 3.912199434085212e-05,
+      "loss": 0.39,
+      "step": 4000
+    },
+    {
+      "epoch": 2.1866666666666665,
+      "grad_norm": 0.34885501861572266,
+      "learning_rate": 3.475041342727693e-05,
+      "loss": 0.395,
+      "step": 4100
+    },
+    {
+      "epoch": 2.24,
+      "grad_norm": 0.4471144378185272,
+      "learning_rate": 3.0585976742768554e-05,
+      "loss": 0.3686,
+      "step": 4200
+    },
+    {
+      "epoch": 2.2933333333333334,
+      "grad_norm": 0.24638742208480835,
+      "learning_rate": 2.6641904892572556e-05,
+      "loss": 0.3695,
+      "step": 4300
+    },
+    {
+      "epoch": 2.3466666666666667,
+      "grad_norm": 0.36457329988479614,
+      "learning_rate": 2.2930718901966042e-05,
+      "loss": 0.3834,
+      "step": 4400
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 0.2564169764518738,
+      "learning_rate": 1.946420046645574e-05,
+      "loss": 0.3616,
+      "step": 4500
+    },
+    {
+      "epoch": 2.453333333333333,
+      "grad_norm": 0.31688305735588074,
+      "learning_rate": 1.625335454908574e-05,
+      "loss": 0.2725,
+      "step": 4600
+    },
+    {
+      "epoch": 2.506666666666667,
+      "grad_norm": 0.3402997553348541,
+      "learning_rate": 1.3308374443593851e-05,
+      "loss": 0.2798,
+      "step": 4700
+    },
+    {
+      "epoch": 2.56,
+      "grad_norm": 0.24339596927165985,
+      "learning_rate": 1.0638609414329625e-05,
+      "loss": 0.2731,
+      "step": 4800
+    },
+    {
+      "epoch": 2.6133333333333333,
+      "grad_norm": 0.3107380270957947,
+      "learning_rate": 8.252535015665497e-06,
+      "loss": 0.2547,
+      "step": 4900
+    },
+    {
+      "epoch": 2.6666666666666665,
+      "grad_norm": 0.4214955270290375,
+      "learning_rate": 6.157726185126322e-06,
+      "loss": 0.2694,
+      "step": 5000
+    },
+    {
+      "epoch": 2.7199999999999998,
+      "grad_norm": 0.3382337689399719,
+      "learning_rate": 4.360833195656866e-06,
+      "loss": 0.267,
+      "step": 5100
+    },
+    {
+      "epoch": 2.7733333333333334,
+      "grad_norm": 0.39369478821754456,
+      "learning_rate": 2.867560543370751e-06,
+      "loss": 0.2716,
+      "step": 5200
     }
   ],
   "logging_steps": 100,
   "max_steps": 5625,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 3,
-  "save_steps": 300,
+  "save_steps": 200,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
@@ -237,7 +391,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 3.0763934439174144e+17,
+  "total_flos": 5.3163646060732416e+17,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:45516608d837bffbed8e53d360cfd5448cb77674677db5e2316820ee907925c3
+oid sha256:4c0b253cece39709a89dda6cd57133689d6cbd728c9909c75212320dcfee45d7
 size 6097