Upload folder using huggingface_hub

adapter_config.json

@@ -14,7 +14,7 @@
   "loftq_config": {},
   "lora_alpha": 112,
   "lora_bias": false,
-  "lora_dropout": 0.1,
+  "lora_dropout": 0.05,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
@@ -23,10 +23,10 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "gate_up_proj",
     "down_proj",
-    "qkv_proj",
-    "o_proj"
+    "o_proj",
+    "gate_up_proj",
+    "qkv_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2ef79a75800f862f4e0fd92d66308bae954fcf321bde1c5dbf47d3acef07c03b
+oid sha256:79ccd2b0725d5caaf6fde82e92d52927391ad95545839ba41f7843cc6b907d55
 size 322996232

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a27c659290fdac5e7ec3deaaf773d7831bb182d85a453783d32fb3b42f3b2526
+oid sha256:019d1ca091101294bfbf2f45057dc925f538fd6c74c7db413e7331adcc9317d9
 size 646142667

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f01e6fa000027d3e605db84f7de61a67a161a02e812c74c22a886d0bd96eb5d4
+oid sha256:0a8c0aa69084bb9d94d9a6de1d8d9b79dc3923ff47dece5164ccf871ba1b2abc
 size 14645

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8785418585651bef269ecd0dbf2cde8c7100f2cba09bc9ef71d7b7bd8cb2c744
+oid sha256:ffb01ee3f21af64a917386cd80a40a480326ab2f88c05fd143f4f5d49daa613b
 size 1465

trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.8395061728395063,
+  "epoch": 2.9876543209876543,
   "eval_steps": 500,
-  "global_step": 59,
+  "global_step": 62,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -12,7 +12,7 @@
     {
       "entropy": 2.1045966744422913,
       "epoch": 0.04938271604938271,
-      "grad_norm": 17.07461929321289,
+      "grad_norm": 16.678590774536133,
       "learning_rate": 0.0,
       "loss": 2.4474,
       "mean_token_accuracy": 0.44178610295057297,
@@ -22,7 +22,7 @@
     {
       "entropy": 1.944445788860321,
       "epoch": 0.09876543209876543,
-      "grad_norm": 13.418954849243164,
+      "grad_norm": 12.955353736877441,
       "learning_rate": 3.3333333333333333e-06,
       "loss": 2.153,
       "mean_token_accuracy": 0.4885662570595741,
@@ -30,574 +30,604 @@
       "step": 2
     },
     {
-      "entropy": 2.1375502943992615,
+      "entropy": 2.133327841758728,
       "epoch": 0.14814814814814814,
-      "grad_norm": 13.263312339782715,
+      "grad_norm": 12.755569458007812,
       "learning_rate": 6.666666666666667e-06,
-      "loss": 2.3265,
-      "mean_token_accuracy": 0.4580386020243168,
+      "loss": 2.3206,
+      "mean_token_accuracy": 0.45916735380887985,
       "num_tokens": 6448.0,
       "step": 3
     },
     {
-      "entropy": 2.0441721975803375,
+      "entropy": 2.04616542160511,
       "epoch": 0.19753086419753085,
-      "grad_norm": 12.733418464660645,
+      "grad_norm": 12.627863883972168,
       "learning_rate": 1e-05,
-      "loss": 2.0155,
-      "mean_token_accuracy": 0.4843377247452736,
+      "loss": 2.0161,
+      "mean_token_accuracy": 0.4829055927693844,
       "num_tokens": 8532.0,
       "step": 4
     },
     {
-      "entropy": 2.224354177713394,
+      "entropy": 2.2228081226348877,
       "epoch": 0.24691358024691357,
-      "grad_norm": 6.445976734161377,
+      "grad_norm": 6.164249420166016,
       "learning_rate": 1.3333333333333333e-05,
-      "loss": 1.833,
-      "mean_token_accuracy": 0.4942014664411545,
+      "loss": 1.8315,
+      "mean_token_accuracy": 0.4869111068546772,
       "num_tokens": 11973.0,
       "step": 5
     },
     {
-      "entropy": 2.1060016453266144,
+      "entropy": 2.113390877842903,
       "epoch": 0.2962962962962963,
-      "grad_norm": 6.344198226928711,
+      "grad_norm": 6.344396591186523,
       "learning_rate": 1.6666666666666667e-05,
-      "loss": 1.9109,
-      "mean_token_accuracy": 0.5272197648882866,
+      "loss": 1.9057,
+      "mean_token_accuracy": 0.5254073962569237,
       "num_tokens": 14982.0,
       "step": 6
     },
     {
-      "entropy": 2.2028936445713043,
+      "entropy": 2.207372799515724,
       "epoch": 0.345679012345679,
-      "grad_norm": 7.122912406921387,
+      "grad_norm": 6.901336669921875,
       "learning_rate": 2e-05,
-      "loss": 1.8414,
-      "mean_token_accuracy": 0.5166565254330635,
+      "loss": 1.8379,
+      "mean_token_accuracy": 0.5119036361575127,
       "num_tokens": 17566.0,
       "step": 7
     },
     {
-      "entropy": 2.2383397221565247,
+      "entropy": 2.2413517236709595,
       "epoch": 0.3950617283950617,
-      "grad_norm": 6.923185348510742,
+      "grad_norm": 6.814880847930908,
       "learning_rate": 2.3333333333333336e-05,
-      "loss": 1.9645,
-      "mean_token_accuracy": 0.4945147931575775,
+      "loss": 1.9594,
+      "mean_token_accuracy": 0.49952177330851555,
       "num_tokens": 19825.0,
       "step": 8
     },
     {
-      "entropy": 2.2687272429466248,
+      "entropy": 2.2734450548887253,
       "epoch": 0.4444444444444444,
-      "grad_norm": 6.454328536987305,
+      "grad_norm": 6.185186386108398,
       "learning_rate": 2.6666666666666667e-05,
-      "loss": 1.9193,
-      "mean_token_accuracy": 0.520270973443985,
+      "loss": 1.9202,
+      "mean_token_accuracy": 0.5213926732540131,
       "num_tokens": 21855.0,
       "step": 9
     },
     {
-      "entropy": 2.137419432401657,
+      "entropy": 2.1350340843200684,
       "epoch": 0.49382716049382713,
-      "grad_norm": 6.179715156555176,
+      "grad_norm": 5.997181415557861,
       "learning_rate": 3e-05,
-      "loss": 1.7229,
-      "mean_token_accuracy": 0.5469484068453312,
+      "loss": 1.7231,
+      "mean_token_accuracy": 0.5484383963048458,
       "num_tokens": 24405.0,
       "step": 10
     },
     {
-      "entropy": 2.153471812605858,
+      "entropy": 2.1509426087141037,
       "epoch": 0.5432098765432098,
-      "grad_norm": 6.750667572021484,
+      "grad_norm": 6.595911502838135,
       "learning_rate": 3.3333333333333335e-05,
-      "loss": 1.7749,
-      "mean_token_accuracy": 0.5488381646573544,
+      "loss": 1.7791,
+      "mean_token_accuracy": 0.5505530126392841,
       "num_tokens": 26247.0,
       "step": 11
     },
     {
-      "entropy": 2.0210573077201843,
+      "entropy": 2.018960952758789,
       "epoch": 0.5925925925925926,
-      "grad_norm": 5.751638412475586,
+      "grad_norm": 5.737451076507568,
       "learning_rate": 3.6666666666666666e-05,
-      "loss": 1.7508,
-      "mean_token_accuracy": 0.5483265519142151,
+      "loss": 1.7496,
+      "mean_token_accuracy": 0.5482935383915901,
       "num_tokens": 28315.0,
       "step": 12
     },
     {
-      "entropy": 2.0588729083538055,
+      "entropy": 2.05392025411129,
       "epoch": 0.6419753086419753,
-      "grad_norm": 6.749085903167725,
+      "grad_norm": 6.572509765625,
       "learning_rate": 4e-05,
-      "loss": 1.78,
-      "mean_token_accuracy": 0.5516550652682781,
+      "loss": 1.7735,
+      "mean_token_accuracy": 0.5521397292613983,
       "num_tokens": 30028.0,
       "step": 13
     },
     {
-      "entropy": 1.8380699753761292,
+      "entropy": 1.830256313085556,
       "epoch": 0.691358024691358,
-      "grad_norm": 4.831390380859375,
+      "grad_norm": 4.667781352996826,
       "learning_rate": 4.3333333333333334e-05,
-      "loss": 1.639,
-      "mean_token_accuracy": 0.5657348707318306,
+      "loss": 1.6375,
+      "mean_token_accuracy": 0.5683313086628914,
       "num_tokens": 33151.0,
       "step": 14
     },
     {
-      "entropy": 2.002317950129509,
+      "entropy": 1.9952753484249115,
       "epoch": 0.7407407407407407,
-      "grad_norm": 6.0173773765563965,
+      "grad_norm": 5.7879743576049805,
       "learning_rate": 4.666666666666667e-05,
-      "loss": 1.6788,
-      "mean_token_accuracy": 0.5597697757184505,
+      "loss": 1.6752,
+      "mean_token_accuracy": 0.5599516853690147,
       "num_tokens": 35005.0,
       "step": 15
     },
     {
-      "entropy": 2.2061654031276703,
+      "entropy": 2.2150524109601974,
       "epoch": 0.7901234567901234,
-      "grad_norm": 6.6986517906188965,
+      "grad_norm": 6.519161224365234,
       "learning_rate": 5e-05,
-      "loss": 1.7774,
-      "mean_token_accuracy": 0.5585966594517231,
+      "loss": 1.7735,
+      "mean_token_accuracy": 0.5492940992116928,
       "num_tokens": 36505.0,
       "step": 16
     },
     {
-      "entropy": 2.003757819533348,
+      "entropy": 1.9987490773200989,
       "epoch": 0.8395061728395061,
-      "grad_norm": 6.035629749298096,
+      "grad_norm": 5.76025915145874,
       "learning_rate": 4.994647308096509e-05,
-      "loss": 1.657,
-      "mean_token_accuracy": 0.5558704920113087,
+      "loss": 1.6564,
+      "mean_token_accuracy": 0.5628282427787781,
       "num_tokens": 38467.0,
       "step": 17
     },
     {
-      "entropy": 1.9430343508720398,
+      "entropy": 1.949499949812889,
       "epoch": 0.8888888888888888,
-      "grad_norm": 5.070599555969238,
+      "grad_norm": 4.927590370178223,
       "learning_rate": 4.9786121534345265e-05,
-      "loss": 1.6259,
-      "mean_token_accuracy": 0.5617526397109032,
+      "loss": 1.6235,
+      "mean_token_accuracy": 0.5548277460038662,
       "num_tokens": 40889.0,
       "step": 18
     },
     {
-      "entropy": 2.037708967924118,
+      "entropy": 2.0500358641147614,
       "epoch": 0.9382716049382716,
-      "grad_norm": 5.155577182769775,
+      "grad_norm": 4.964672088623047,
       "learning_rate": 4.951963201008076e-05,
-      "loss": 1.5762,
-      "mean_token_accuracy": 0.5841848030686378,
+      "loss": 1.5668,
+      "mean_token_accuracy": 0.5864963307976723,
       "num_tokens": 43002.0,
       "step": 19
     },
     {
-      "entropy": 2.0400605648756027,
+      "entropy": 2.043752282857895,
       "epoch": 0.9876543209876543,
-      "grad_norm": 4.773712635040283,
+      "grad_norm": 4.934137344360352,
       "learning_rate": 4.914814565722671e-05,
-      "loss": 1.5847,
-      "mean_token_accuracy": 0.5751754343509674,
+      "loss": 1.5961,
+      "mean_token_accuracy": 0.5691769942641258,
       "num_tokens": 45100.0,
       "step": 20
     },
     {
-      "entropy": 1.6524975299835205,
+      "entropy": 1.6749918460845947,
       "epoch": 1.0,
-      "grad_norm": 10.611732482910156,
+      "grad_norm": 10.593884468078613,
       "learning_rate": 4.867325323737765e-05,
-      "loss": 1.3868,
-      "mean_token_accuracy": 0.6146115064620972,
+      "loss": 1.418,
+      "mean_token_accuracy": 0.6043022274971008,
       "num_tokens": 45620.0,
       "step": 21
     },
     {
-      "entropy": 1.9145531356334686,
+      "entropy": 1.888138085603714,
       "epoch": 1.0493827160493827,
-      "grad_norm": 4.789498805999756,
+      "grad_norm": 4.542983531951904,
       "learning_rate": 4.8096988312782174e-05,
-      "loss": 1.2313,
-      "mean_token_accuracy": 0.669404037296772,
+      "loss": 1.2251,
+      "mean_token_accuracy": 0.6697289943695068,
       "num_tokens": 47479.0,
       "step": 22
     },
     {
-      "entropy": 1.7801827937364578,
+      "entropy": 1.7415399253368378,
       "epoch": 1.0987654320987654,
-      "grad_norm": 4.739726543426514,
+      "grad_norm": 4.551285266876221,
       "learning_rate": 4.742181853831721e-05,
-      "loss": 1.26,
-      "mean_token_accuracy": 0.652519591152668,
+      "loss": 1.2521,
+      "mean_token_accuracy": 0.6552469357848167,
       "num_tokens": 49530.0,
       "step": 23
     },
     {
-      "entropy": 1.6698480248451233,
+      "entropy": 1.6383564174175262,
       "epoch": 1.1481481481481481,
-      "grad_norm": 4.931276798248291,
+      "grad_norm": 4.755136013031006,
       "learning_rate": 4.665063509461097e-05,
-      "loss": 1.2183,
-      "mean_token_accuracy": 0.6311887428164482,
+      "loss": 1.2116,
+      "mean_token_accuracy": 0.6445794999599457,
       "num_tokens": 51795.0,
       "step": 24
     },
     {
-      "entropy": 1.6189886331558228,
+      "entropy": 1.6066707968711853,
       "epoch": 1.1975308641975309,
-      "grad_norm": 4.8303542137146,
+      "grad_norm": 4.747236728668213,
       "learning_rate": 4.5786740307563636e-05,
-      "loss": 1.2323,
-      "mean_token_accuracy": 0.6584680005908012,
+      "loss": 1.2268,
+      "mean_token_accuracy": 0.6532000750303268,
       "num_tokens": 53775.0,
       "step": 25
     },
     {
-      "entropy": 1.7246621996164322,
+      "entropy": 1.7001492828130722,
       "epoch": 1.2469135802469136,
-      "grad_norm": 4.690735340118408,
+      "grad_norm": 4.45648193359375,
       "learning_rate": 4.4833833507280884e-05,
-      "loss": 1.2359,
-      "mean_token_accuracy": 0.6399939209222794,
+      "loss": 1.2339,
+      "mean_token_accuracy": 0.6452045813202858,
       "num_tokens": 56079.0,
       "step": 26
     },
     {
-      "entropy": 1.5530678927898407,
+      "entropy": 1.5360765606164932,
       "epoch": 1.2962962962962963,
-      "grad_norm": 4.586762428283691,
+      "grad_norm": 4.499372482299805,
       "learning_rate": 4.379599518697444e-05,
-      "loss": 1.2253,
-      "mean_token_accuracy": 0.6523294448852539,
+      "loss": 1.219,
+      "mean_token_accuracy": 0.6569549441337585,
       "num_tokens": 58349.0,
       "step": 27
     },
     {
-      "entropy": 1.551697015762329,
+      "entropy": 1.550552174448967,
       "epoch": 1.345679012345679,
-      "grad_norm": 4.633914947509766,
+      "grad_norm": 4.507914066314697,
       "learning_rate": 4.267766952966369e-05,
-      "loss": 1.1635,
-      "mean_token_accuracy": 0.6726551949977875,
+      "loss": 1.1618,
+      "mean_token_accuracy": 0.6673635169863701,
       "num_tokens": 60570.0,
       "step": 28
     },
     {
-      "entropy": 1.6292523592710495,
+      "entropy": 1.612103745341301,
       "epoch": 1.3950617283950617,
-      "grad_norm": 5.112277507781982,
+      "grad_norm": 5.213642597198486,
       "learning_rate": 4.148364537750172e-05,
-      "loss": 1.2572,
-      "mean_token_accuracy": 0.6387738659977913,
+      "loss": 1.2609,
+      "mean_token_accuracy": 0.6316312029957771,
       "num_tokens": 62650.0,
       "step": 29
     },
     {
-      "entropy": 1.4799976348876953,
+      "entropy": 1.4736015498638153,
       "epoch": 1.4444444444444444,
-      "grad_norm": 4.31527042388916,
+      "grad_norm": 4.291496753692627,
       "learning_rate": 4.021903572521802e-05,
-      "loss": 1.1865,
-      "mean_token_accuracy": 0.6671764701604843,
+      "loss": 1.1779,
+      "mean_token_accuracy": 0.6751257181167603,
       "num_tokens": 65419.0,
       "step": 30
     },
     {
-      "entropy": 1.549431949853897,
+      "entropy": 1.5430797338485718,
       "epoch": 1.4938271604938271,
-      "grad_norm": 4.670032024383545,
+      "grad_norm": 4.683901309967041,
       "learning_rate": 3.888925582549006e-05,
-      "loss": 1.0756,
-      "mean_token_accuracy": 0.6905995309352875,
+      "loss": 1.0674,
+      "mean_token_accuracy": 0.6916785389184952,
       "num_tokens": 67644.0,
       "step": 31
     },
     {
-      "entropy": 1.5958464741706848,
+      "entropy": 1.5857491940259933,
       "epoch": 1.5432098765432098,
-      "grad_norm": 5.251377582550049,
+      "grad_norm": 4.896679878234863,
       "learning_rate": 3.7500000000000003e-05,
-      "loss": 1.1905,
-      "mean_token_accuracy": 0.6532313749194145,
+      "loss": 1.1883,
+      "mean_token_accuracy": 0.6605922058224678,
       "num_tokens": 69764.0,
       "step": 32
     },
     {
-      "entropy": 1.6538591533899307,
+      "entropy": 1.6383006423711777,
       "epoch": 1.5925925925925926,
-      "grad_norm": 5.467432022094727,
+      "grad_norm": 5.426114559173584,
       "learning_rate": 3.6057217255475034e-05,
-      "loss": 1.1792,
-      "mean_token_accuracy": 0.6628076210618019,
+      "loss": 1.1589,
+      "mean_token_accuracy": 0.6661425679922104,
       "num_tokens": 71383.0,
       "step": 33
     },
     {
-      "entropy": 1.5740457475185394,
+      "entropy": 1.5539519935846329,
       "epoch": 1.6419753086419753,
-      "grad_norm": 5.274950981140137,
+      "grad_norm": 5.141643524169922,
       "learning_rate": 3.456708580912725e-05,
-      "loss": 1.1839,
-      "mean_token_accuracy": 0.6636194065213203,
+      "loss": 1.1704,
+      "mean_token_accuracy": 0.6650657132267952,
       "num_tokens": 73330.0,
       "step": 34
     },
     {
-      "entropy": 1.4796217530965805,
+      "entropy": 1.4722215682268143,
       "epoch": 1.691358024691358,
-      "grad_norm": 4.721081256866455,
+      "grad_norm": 4.59281587600708,
       "learning_rate": 3.303598663257904e-05,
-      "loss": 1.0261,
-      "mean_token_accuracy": 0.6876147240400314,
+      "loss": 1.0203,
+      "mean_token_accuracy": 0.6820086091756821,
       "num_tokens": 75429.0,
       "step": 35
     },
     {
-      "entropy": 1.4072065949440002,
+      "entropy": 1.4010685980319977,
       "epoch": 1.7407407407407407,
-      "grad_norm": 4.086071014404297,
+      "grad_norm": 4.383392333984375,
       "learning_rate": 3.147047612756302e-05,
-      "loss": 1.0271,
-      "mean_token_accuracy": 0.6869702711701393,
+      "loss": 1.0248,
+      "mean_token_accuracy": 0.6845208778977394,
       "num_tokens": 78459.0,
       "step": 36
     },
     {
-      "entropy": 1.5171602815389633,
+      "entropy": 1.5117781907320023,
       "epoch": 1.7901234567901234,
-      "grad_norm": 5.711819648742676,
+      "grad_norm": 5.551262378692627,
       "learning_rate": 2.9877258050403212e-05,
-      "loss": 1.1432,
-      "mean_token_accuracy": 0.6735518574714661,
+      "loss": 1.135,
+      "mean_token_accuracy": 0.6859684735536575,
       "num_tokens": 80102.0,
       "step": 37
     },
     {
-      "entropy": 1.4996187686920166,
+      "entropy": 1.4906423836946487,
       "epoch": 1.8395061728395061,
-      "grad_norm": 4.804397106170654,
+      "grad_norm": 4.837874412536621,
       "learning_rate": 2.8263154805501297e-05,
-      "loss": 1.1221,
-      "mean_token_accuracy": 0.6675888076424599,
+      "loss": 1.1213,
+      "mean_token_accuracy": 0.6715603843331337,
       "num_tokens": 82277.0,
       "step": 38
     },
     {
-      "entropy": 1.277234360575676,
+      "entropy": 1.2754715234041214,
       "epoch": 1.8888888888888888,
-      "grad_norm": 3.7578635215759277,
+      "grad_norm": 3.6830785274505615,
       "learning_rate": 2.663507823075358e-05,
-      "loss": 0.9613,
-      "mean_token_accuracy": 0.6974349915981293,
+      "loss": 0.9663,
+      "mean_token_accuracy": 0.7010771930217743,
       "num_tokens": 85933.0,
       "step": 39
     },
     {
-      "entropy": 1.4796784967184067,
+      "entropy": 1.4681971222162247,
       "epoch": 1.9382716049382716,
-      "grad_norm": 4.373854160308838,
+      "grad_norm": 4.174588680267334,
       "learning_rate": 2.5e-05,
-      "loss": 1.1684,
-      "mean_token_accuracy": 0.6698590070009232,
+      "loss": 1.154,
+      "mean_token_accuracy": 0.6745116859674454,
       "num_tokens": 88612.0,
       "step": 40
     },
     {
-      "entropy": 1.4071252942085266,
+      "entropy": 1.4022098779678345,
       "epoch": 1.9876543209876543,
-      "grad_norm": 5.221538543701172,
+      "grad_norm": 4.754171848297119,
       "learning_rate": 2.3364921769246423e-05,
-      "loss": 1.1439,
-      "mean_token_accuracy": 0.6814302429556847,
+      "loss": 1.1365,
+      "mean_token_accuracy": 0.6754558682441711,
       "num_tokens": 90823.0,
       "step": 41
     },
     {
-      "entropy": 1.5970491170883179,
+      "entropy": 1.588551640510559,
       "epoch": 2.0,
-      "grad_norm": 10.636951446533203,
+      "grad_norm": 10.379326820373535,
       "learning_rate": 2.173684519449872e-05,
-      "loss": 1.0735,
-      "mean_token_accuracy": 0.7276268303394318,
+      "loss": 1.0527,
+      "mean_token_accuracy": 0.7145833373069763,
       "num_tokens": 91240.0,
       "step": 42
     },
     {
-      "entropy": 1.39203742146492,
+      "entropy": 1.3871477395296097,
       "epoch": 2.049382716049383,
-      "grad_norm": 3.8262007236480713,
+      "grad_norm": 3.7378275394439697,
       "learning_rate": 2.0122741949596797e-05,
-      "loss": 0.7384,
-      "mean_token_accuracy": 0.7718699052929878,
+      "loss": 0.7188,
+      "mean_token_accuracy": 0.7726586982607841,
       "num_tokens": 93342.0,
       "step": 43
     },
     {
-      "entropy": 1.4726413637399673,
+      "entropy": 1.4595267176628113,
       "epoch": 2.0987654320987654,
-      "grad_norm": 4.435136795043945,
+      "grad_norm": 4.306966781616211,
       "learning_rate": 1.852952387243698e-05,
-      "loss": 0.7265,
-      "mean_token_accuracy": 0.7925485372543335,
+      "loss": 0.7157,
+      "mean_token_accuracy": 0.7996886223554611,
       "num_tokens": 95250.0,
       "step": 44
     },
     {
-      "entropy": 1.3003774285316467,
+      "entropy": 1.2950524985790253,
       "epoch": 2.148148148148148,
-      "grad_norm": 3.799497365951538,
+      "grad_norm": 3.7582972049713135,
       "learning_rate": 1.6964013367420966e-05,
-      "loss": 0.7015,
-      "mean_token_accuracy": 0.7795400172472,
+      "loss": 0.6962,
+      "mean_token_accuracy": 0.7863549143075943,
       "num_tokens": 97864.0,
       "step": 45
     },
     {
-      "entropy": 1.248021274805069,
+      "entropy": 1.2319310307502747,
       "epoch": 2.197530864197531,
-      "grad_norm": 3.650768756866455,
+      "grad_norm": 3.64840030670166,
       "learning_rate": 1.5432914190872757e-05,
-      "loss": 0.6776,
-      "mean_token_accuracy": 0.8115019127726555,
+      "loss": 0.6659,
+      "mean_token_accuracy": 0.8138639628887177,
       "num_tokens": 100323.0,
       "step": 46
     },
     {
-      "entropy": 1.2092759758234024,
+      "entropy": 1.19608373939991,
       "epoch": 2.246913580246914,
-      "grad_norm": 3.9729771614074707,
+      "grad_norm": 3.90803599357605,
       "learning_rate": 1.3942782744524973e-05,
-      "loss": 0.6091,
-      "mean_token_accuracy": 0.8202343359589577,
+      "loss": 0.6014,
+      "mean_token_accuracy": 0.8257261663675308,
       "num_tokens": 102615.0,
       "step": 47
     },
     {
-      "entropy": 1.2965480834245682,
+      "entropy": 1.2897031158208847,
       "epoch": 2.2962962962962963,
-      "grad_norm": 4.632357597351074,
+      "grad_norm": 4.506102561950684,
       "learning_rate": 1.2500000000000006e-05,
-      "loss": 0.7704,
-      "mean_token_accuracy": 0.7770233377814293,
+      "loss": 0.7683,
+      "mean_token_accuracy": 0.7799497023224831,
       "num_tokens": 104543.0,
       "step": 48
     },
     {
-      "entropy": 1.2238061875104904,
+      "entropy": 1.209401585161686,
       "epoch": 2.3456790123456788,
-      "grad_norm": 4.053952217102051,
+      "grad_norm": 3.957465887069702,
       "learning_rate": 1.1110744174509952e-05,
-      "loss": 0.6047,
-      "mean_token_accuracy": 0.8104883581399918,
+      "loss": 0.5976,
+      "mean_token_accuracy": 0.8096398636698723,
       "num_tokens": 106982.0,
       "step": 49
     },
     {
-      "entropy": 1.2117676883935928,
+      "entropy": 1.203671395778656,
       "epoch": 2.3950617283950617,
-      "grad_norm": 4.814194202423096,
+      "grad_norm": 4.684906005859375,
       "learning_rate": 9.780964274781984e-06,
-      "loss": 0.6945,
-      "mean_token_accuracy": 0.794559009373188,
+      "loss": 0.6821,
+      "mean_token_accuracy": 0.8014075681567192,
       "num_tokens": 108907.0,
       "step": 50
     },
     {
-      "entropy": 1.2279776483774185,
+      "entropy": 1.21255362033844,
       "epoch": 2.4444444444444446,
-      "grad_norm": 5.624629974365234,
+      "grad_norm": 5.399158954620361,
       "learning_rate": 8.51635462249828e-06,
-      "loss": 0.7707,
-      "mean_token_accuracy": 0.7722515612840652,
+      "loss": 0.7489,
+      "mean_token_accuracy": 0.7789968624711037,
       "num_tokens": 110649.0,
       "step": 51
     },
     {
-      "entropy": 1.130656473338604,
+      "entropy": 1.1142661720514297,
       "epoch": 2.493827160493827,
-      "grad_norm": 4.165122985839844,
+      "grad_norm": 4.029307842254639,
       "learning_rate": 7.3223304703363135e-06,
-      "loss": 0.6238,
-      "mean_token_accuracy": 0.8032507002353668,
+      "loss": 0.6049,
+      "mean_token_accuracy": 0.8112852945923805,
       "num_tokens": 113000.0,
       "step": 52
     },
     {
-      "entropy": 1.057595133781433,
+      "entropy": 1.0388604775071144,
       "epoch": 2.5432098765432096,
-      "grad_norm": 4.7867817878723145,
+      "grad_norm": 4.7035369873046875,
       "learning_rate": 6.204004813025568e-06,
-      "loss": 0.6951,
-      "mean_token_accuracy": 0.8001823052763939,
+      "loss": 0.6827,
+      "mean_token_accuracy": 0.8062720224261284,
       "num_tokens": 115252.0,
       "step": 53
     },
     {
-      "entropy": 1.0031295642256737,
+      "entropy": 0.9865575134754181,
       "epoch": 2.5925925925925926,
-      "grad_norm": 4.251824855804443,
+      "grad_norm": 4.302714824676514,
       "learning_rate": 5.166166492719124e-06,
-      "loss": 0.547,
-      "mean_token_accuracy": 0.8263749331235886,
+      "loss": 0.5451,
+      "mean_token_accuracy": 0.8259979337453842,
       "num_tokens": 117622.0,
       "step": 54
     },
     {
-      "entropy": 0.9957862570881844,
+      "entropy": 0.9599115997552872,
       "epoch": 2.6419753086419755,
-      "grad_norm": 4.6617279052734375,
+      "grad_norm": 4.7989349365234375,
       "learning_rate": 4.213259692436367e-06,
-      "loss": 0.6031,
-      "mean_token_accuracy": 0.8142141699790955,
+      "loss": 0.5905,
+      "mean_token_accuracy": 0.8102814555168152,
       "num_tokens": 120361.0,
       "step": 55
     },
     {
-      "entropy": 1.0976142808794975,
+      "entropy": 1.072813868522644,
       "epoch": 2.691358024691358,
-      "grad_norm": 4.915057182312012,
+      "grad_norm": 4.888575553894043,
       "learning_rate": 3.3493649053890326e-06,
-      "loss": 0.6949,
-      "mean_token_accuracy": 0.7822957187891006,
+      "loss": 0.689,
+      "mean_token_accuracy": 0.7948291003704071,
       "num_tokens": 122610.0,
       "step": 56
     },
     {
-      "entropy": 1.0492280274629593,
+      "entropy": 1.0314068496227264,
       "epoch": 2.7407407407407405,
-      "grad_norm": 5.099884033203125,
+      "grad_norm": 5.251216411590576,
       "learning_rate": 2.578181461682794e-06,
-      "loss": 0.6004,
-      "mean_token_accuracy": 0.8302762731909752,
+      "loss": 0.5865,
+      "mean_token_accuracy": 0.8308965489268303,
       "num_tokens": 124571.0,
       "step": 57
     },
     {
-      "entropy": 1.0981549099087715,
+      "entropy": 1.0771163925528526,
       "epoch": 2.7901234567901234,
-      "grad_norm": 5.698803901672363,
+      "grad_norm": 5.723106384277344,
       "learning_rate": 1.9030116872178316e-06,
-      "loss": 0.8066,
-      "mean_token_accuracy": 0.7803784757852554,
+      "loss": 0.7885,
+      "mean_token_accuracy": 0.7869210541248322,
       "num_tokens": 126741.0,
       "step": 58
     },
     {
-      "entropy": 1.0801162794232368,
+      "entropy": 1.0658350437879562,
       "epoch": 2.8395061728395063,
-      "grad_norm": 4.8652567863464355,
+      "grad_norm": 4.829237937927246,
       "learning_rate": 1.3267467626223606e-06,
-      "loss": 0.6778,
-      "mean_token_accuracy": 0.8027013018727303,
+      "loss": 0.6772,
+      "mean_token_accuracy": 0.8049523532390594,
       "num_tokens": 129201.0,
       "step": 59
+    },
+    {
+      "entropy": 1.0056668892502785,
+      "epoch": 2.888888888888889,
+      "grad_norm": 5.5694780349731445,
+      "learning_rate": 8.51854342773295e-07,
+      "loss": 0.6802,
+      "mean_token_accuracy": 0.8025626465678215,
+      "num_tokens": 131344.0,
+      "step": 60
+    },
+    {
+      "entropy": 1.0122049376368523,
+      "epoch": 2.9382716049382713,
+      "grad_norm": 4.719507217407227,
+      "learning_rate": 4.803679899192392e-07,
+      "loss": 0.6776,
+      "mean_token_accuracy": 0.79056666046381,
+      "num_tokens": 133881.0,
+      "step": 61
+    },
+    {
+      "entropy": 1.0161257833242416,
+      "epoch": 2.9876543209876543,
+      "grad_norm": 5.256667613983154,
+      "learning_rate": 2.1387846565474045e-07,
+      "loss": 0.6248,
+      "mean_token_accuracy": 0.812793679535389,
+      "num_tokens": 136287.0,
+      "step": 62
     }
   ],
   "logging_steps": 1,
@@ -617,7 +647,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 3467511535116288.0,
+  "total_flos": 3668970048184320.0,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6450a7549e5c3d80ba33c4a8d5dfdf16bf8472ec89ae7d85c7f2650870f43667
+oid sha256:857e29b87cbe8e931f9efcd116edb33ab62db020fffc73980185f4578c5fed6f
 size 6353