End of training

README.md

@@ -3,10 +3,10 @@ library_name: peft
 license: llama3.1
 base_model: meta-llama/Llama-3.1-8B-Instruct
 tags:
-- trl
-- kto
 - llama-factory
 - lora
+- trl
+- kto
 - generated_from_trainer
 model-index:
 - name: Llama-3.1-8B-Instruct_kto_sg_values
@@ -18,7 +18,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # Llama-3.1-8B-Instruct_kto_sg_values
 
-This model is a fine-tuned version of [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) on an unknown dataset.
+This model is a fine-tuned version of [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) on the kto_sg_values dataset.
 
 ## Model description
 

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
     "total_flos": 5.091091732590756e+17,
-    "train_loss": 0.23059077864843058,
-    "train_runtime": 10627.7324,
-    "train_samples_per_second": 5.307,
-    "train_steps_per_second": 0.332
+    "train_loss": 0.413429944193955,
+    "train_runtime": 10515.385,
+    "train_samples_per_second": 5.364,
+    "train_steps_per_second": 0.335
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
     "total_flos": 5.091091732590756e+17,
-    "train_loss": 0.23059077864843058,
-    "train_runtime": 10627.7324,
-    "train_samples_per_second": 5.307,
-    "train_steps_per_second": 0.332
+    "train_loss": 0.413429944193955,
+    "train_runtime": 10515.385,
+    "train_samples_per_second": 5.364,
+    "train_steps_per_second": 0.335
 }

trainer_state.json

@@ -10,297 +10,297 @@
   "log_history": [
     {
       "epoch": 0.05021276595744681,
-      "grad_norm": 0.7250668406486511,
-      "kl": 10.799270629882812,
-      "learning_rate": 2.5070821529745037e-07,
-      "logits/chosen": -15170619.076923076,
-      "logits/rejected": -16525174.033245845,
-      "logps/chosen": -2.1839165879693225,
-      "logps/rejected": -5.995521311789151,
-      "loss": 0.4996,
-      "rewards/chosen": 0.000273060312856248,
-      "rewards/margins": 0.000992770597812732,
-      "rewards/rejected": -0.000719710284956484,
+      "grad_norm": 0.732947051525116,
+      "kl": 12.475480079650879,
+      "learning_rate": 5.014164305949008e-08,
+      "logits/chosen": -15179654.095238095,
+      "logits/rejected": -16528289.93175853,
+      "logps/chosen": -2.1814988803514193,
+      "logps/rejected": -5.992375464785652,
+      "loss": 0.4997,
+      "rewards/chosen": 0.0005148474550072527,
+      "rewards/margins": 0.0009199919065958275,
+      "rewards/rejected": -0.00040514445158857477,
       "step": 177
     },
     {
       "epoch": 0.10042553191489362,
-      "grad_norm": 0.37310150265693665,
-      "kl": 0.8455712795257568,
-      "learning_rate": 4.9984237074401e-07,
-      "logits/chosen": -14958245.494949495,
-      "logits/rejected": -16451794.473637177,
-      "logps/chosen": -2.413396250920665,
-      "logps/rejected": -6.402490365281501,
-      "loss": 0.4952,
-      "rewards/chosen": -0.013412311823681147,
-      "rewards/margins": 0.012531101672975501,
-      "rewards/rejected": -0.02594341349665665,
+      "grad_norm": 0.26544979214668274,
+      "kl": 11.266656875610352,
+      "learning_rate": 9.996847414880202e-08,
+      "logits/chosen": -14902797.791245792,
+      "logits/rejected": -16395014.176943699,
+      "logps/chosen": -2.2833465678924663,
+      "logps/rejected": -6.153056317722297,
+      "loss": 0.4996,
+      "rewards/chosen": -0.00040731574048096886,
+      "rewards/margins": 0.000592714583635074,
+      "rewards/rejected": -0.0010000303241160429,
       "step": 354
     },
     {
       "epoch": 0.15063829787234043,
-      "grad_norm": 0.7330515384674072,
-      "kl": 0.0,
-      "learning_rate": 4.719419924337957e-07,
-      "logits/chosen": -15493149.883268483,
-      "logits/rejected": -17431733.1216566,
-      "logps/chosen": -10.362312287208171,
-      "logps/rejected": -17.032325145599653,
-      "loss": 0.3627,
-      "rewards/chosen": -0.8161516597762646,
-      "rewards/margins": 0.26472018556454646,
-      "rewards/rejected": -1.080871845340811,
+      "grad_norm": 0.6797279715538025,
+      "kl": 3.9419214725494385,
+      "learning_rate": 9.438839848675913e-08,
+      "logits/chosen": -14784543.875486381,
+      "logits/rejected": -16617535.61345988,
+      "logps/chosen": -2.2231409688867947,
+      "logps/rejected": -6.279140995739862,
+      "loss": 0.4988,
+      "rewards/chosen": -0.0022345268772733812,
+      "rewards/margins": 0.003318870893000191,
+      "rewards/rejected": -0.005553397770273572,
       "step": 531
     },
     {
       "epoch": 0.20085106382978724,
-      "grad_norm": 0.17463324964046478,
-      "kl": 0.0,
-      "learning_rate": 4.4404161412358134e-07,
-      "logits/chosen": -16503318.021505376,
-      "logits/rejected": -18315989.896218117,
-      "logps/chosen": -38.83077116935484,
-      "logps/rejected": -44.21139786719437,
-      "loss": 0.2129,
-      "rewards/chosen": -3.6382855377744177,
-      "rewards/margins": 0.1601535082348171,
-      "rewards/rejected": -3.7984390460092348,
+      "grad_norm": 0.5770508050918579,
+      "kl": 0.7956511974334717,
+      "learning_rate": 8.880832282471626e-08,
+      "logits/chosen": -14984102.078853047,
+      "logits/rejected": -16609355.65171504,
+      "logps/chosen": -2.5137440671202955,
+      "logps/rejected": -6.364399530013193,
+      "loss": 0.4975,
+      "rewards/chosen": -0.006582849341908664,
+      "rewards/margins": 0.007156510907123363,
+      "rewards/rejected": -0.013739360249032027,
       "step": 708
     },
     {
       "epoch": 0.251063829787234,
-      "grad_norm": 0.04723167046904564,
-      "kl": 0.0,
-      "learning_rate": 4.16141235813367e-07,
-      "logits/chosen": -16537055.085714286,
-      "logits/rejected": -18434136.33802817,
-      "logps/chosen": -50.28404366629464,
-      "logps/rejected": -56.05894104863556,
-      "loss": 0.1991,
-      "rewards/chosen": -4.803175571986607,
-      "rewards/margins": 0.18089855048258308,
-      "rewards/rejected": -4.98407412246919,
+      "grad_norm": 0.5370281934738159,
+      "kl": 0.05692530423402786,
+      "learning_rate": 8.32282471626734e-08,
+      "logits/chosen": -14969774.628571428,
+      "logits/rejected": -16552960.0,
+      "logps/chosen": -2.374119785853795,
+      "logps/rejected": -6.513669296049736,
+      "loss": 0.4945,
+      "rewards/chosen": -0.012183338403701782,
+      "rewards/margins": 0.01736304331833208,
+      "rewards/rejected": -0.029546381722033863,
       "step": 885
     },
     {
       "epoch": 0.30127659574468085,
-      "grad_norm": 0.03626934066414833,
-      "kl": 0.0,
-      "learning_rate": 3.882408575031526e-07,
-      "logits/chosen": -16553432.759124087,
-      "logits/rejected": -18578659.75481611,
-      "logps/chosen": -56.20960809192518,
-      "logps/rejected": -62.88292879816112,
-      "loss": 0.2012,
-      "rewards/chosen": -5.414033096202099,
-      "rewards/margins": 0.25663873091042344,
-      "rewards/rejected": -5.670671827112522,
+      "grad_norm": 0.7874128222465515,
+      "kl": 0.04236392676830292,
+      "learning_rate": 7.76481715006305e-08,
+      "logits/chosen": -14856485.372262774,
+      "logits/rejected": -16540230.837127846,
+      "logps/chosen": -2.2836047541486084,
+      "logps/rejected": -6.687177187089536,
+      "loss": 0.4906,
+      "rewards/chosen": -0.021432671233685346,
+      "rewards/margins": 0.029663742240055215,
+      "rewards/rejected": -0.05109641347374056,
       "step": 1062
     },
     {
       "epoch": 0.35148936170212763,
-      "grad_norm": 0.026356538757681847,
+      "grad_norm": 0.7597999572753906,
       "kl": 0.0,
-      "learning_rate": 3.6034047919293817e-07,
-      "logits/chosen": -16753125.946902655,
-      "logits/rejected": -18672338.82352941,
-      "logps/chosen": -61.49022573285398,
-      "logps/rejected": -67.57514443277311,
-      "loss": 0.1751,
-      "rewards/chosen": -5.908338158531526,
-      "rewards/margins": 0.23123231809662492,
-      "rewards/rejected": -6.139570476628151,
+      "learning_rate": 7.206809583858764e-08,
+      "logits/chosen": -15033836.743362831,
+      "logits/rejected": -16566120.551260505,
+      "logps/chosen": -2.7796039412506914,
+      "logps/rejected": -6.999364824054622,
+      "loss": 0.4848,
+      "rewards/chosen": -0.037276040136286645,
+      "rewards/margins": 0.04471597297755264,
+      "rewards/rejected": -0.08199201311383929,
       "step": 1239
     },
     {
       "epoch": 0.40170212765957447,
-      "grad_norm": 0.027685383334755898,
-      "kl": 0.0,
-      "learning_rate": 3.324401008827238e-07,
-      "logits/chosen": -16838802.810035843,
-      "logits/rejected": -18765256.612137202,
-      "logps/chosen": -64.57314068100358,
-      "logps/rejected": -71.2826311015831,
-      "loss": 0.1891,
-      "rewards/chosen": -6.220487861223118,
-      "rewards/margins": 0.2930856704116662,
-      "rewards/rejected": -6.513573531634784,
+      "grad_norm": 1.355948567390442,
+      "kl": 0.0319652259349823,
+      "learning_rate": 6.648802017654477e-08,
+      "logits/chosen": -15049823.426523298,
+      "logits/rejected": -16636860.453825857,
+      "logps/chosen": -3.031732155857975,
+      "logps/rejected": -7.528678471306069,
+      "loss": 0.4752,
+      "rewards/chosen": -0.06634654110050543,
+      "rewards/margins": 0.07183165772143553,
+      "rewards/rejected": -0.13817819882194096,
       "step": 1416
     },
     {
       "epoch": 0.45191489361702125,
-      "grad_norm": 0.01471030618995428,
+      "grad_norm": 0.9223116040229797,
       "kl": 0.0,
-      "learning_rate": 3.045397225725094e-07,
-      "logits/chosen": -17096763.36231884,
-      "logits/rejected": -18954482.52631579,
-      "logps/chosen": -67.2448341259058,
-      "logps/rejected": -74.1321683114035,
-      "loss": 0.1931,
-      "rewards/chosen": -6.504730666893116,
-      "rewards/margins": 0.28420505130863827,
-      "rewards/rejected": -6.7889357182017545,
+      "learning_rate": 6.090794451450188e-08,
+      "logits/chosen": -15345349.56521739,
+      "logits/rejected": -16821483.340350877,
+      "logps/chosen": -3.2707338194916216,
+      "logps/rejected": -8.287376644736842,
+      "loss": 0.4649,
+      "rewards/chosen": -0.10732046072033868,
+      "rewards/margins": 0.09713628355734771,
+      "rewards/rejected": -0.2044567442776864,
       "step": 1593
     },
     {
       "epoch": 0.502127659574468,
-      "grad_norm": 0.010299217887222767,
+      "grad_norm": 0.8558508157730103,
       "kl": 0.0,
-      "learning_rate": 2.7663934426229505e-07,
-      "logits/chosen": -17058365.519379847,
-      "logits/rejected": -18912892.683937825,
-      "logps/chosen": -69.01617762839147,
-      "logps/rejected": -75.52461814550949,
-      "loss": 0.1944,
-      "rewards/chosen": -6.667378270348837,
-      "rewards/margins": 0.3062005697850143,
-      "rewards/rejected": -6.973578840133851,
+      "learning_rate": 5.5327868852459016e-08,
+      "logits/chosen": -15458446.88372093,
+      "logits/rejected": -16827832.373056997,
+      "logps/chosen": -4.2163511764171515,
+      "logps/rejected": -8.720990696513384,
+      "loss": 0.4485,
+      "rewards/chosen": -0.18739568725112796,
+      "rewards/margins": 0.10582041852458987,
+      "rewards/rejected": -0.29321610577571783,
       "step": 1770
     },
     {
       "epoch": 0.5523404255319149,
-      "grad_norm": 0.00703179556876421,
+      "grad_norm": 1.2379802465438843,
       "kl": 0.0,
-      "learning_rate": 2.487389659520807e-07,
-      "logits/chosen": -16823254.34576271,
-      "logits/rejected": -19061004.560214095,
-      "logps/chosen": -69.84638506355932,
-      "logps/rejected": -77.77510314451382,
-      "loss": 0.1996,
-      "rewards/chosen": -6.773340671345339,
-      "rewards/margins": 0.3934298077191132,
-      "rewards/rejected": -7.166770479064452,
+      "learning_rate": 4.9747793190416137e-08,
+      "logits/chosen": -15368077.450847458,
+      "logits/rejected": -17055221.495093666,
+      "logps/chosen": -5.225153105137712,
+      "logps/rejected": -10.289466714986618,
+      "loss": 0.4349,
+      "rewards/chosen": -0.31121714963751324,
+      "rewards/margins": 0.10698996959893858,
+      "rewards/rejected": -0.4182071192364518,
       "step": 1947
     },
     {
       "epoch": 0.6025531914893617,
-      "grad_norm": 0.015442676842212677,
+      "grad_norm": 2.636925220489502,
       "kl": 0.0,
-      "learning_rate": 2.2083858764186634e-07,
-      "logits/chosen": -17488395.815384615,
-      "logits/rejected": -18972861.564013842,
-      "logps/chosen": -72.38798076923077,
-      "logps/rejected": -79.26926092128028,
-      "loss": 0.1886,
-      "rewards/chosen": -7.007599346454327,
-      "rewards/margins": 0.30762928406037826,
-      "rewards/rejected": -7.315228630514706,
+      "learning_rate": 4.4167717528373264e-08,
+      "logits/chosen": -15860050.707692308,
+      "logits/rejected": -17018940.23529412,
+      "logps/chosen": -6.8304584209735575,
+      "logps/rejected": -12.103902532980104,
+      "loss": 0.4076,
+      "rewards/chosen": -0.4518470470721905,
+      "rewards/margins": 0.14684447744662443,
+      "rewards/rejected": -0.5986915245188149,
       "step": 2124
     },
     {
       "epoch": 0.6527659574468085,
-      "grad_norm": 0.012100782245397568,
+      "grad_norm": 2.6684844493865967,
       "kl": 0.0,
-      "learning_rate": 1.9293820933165196e-07,
-      "logits/chosen": -17114164.0,
-      "logits/rejected": -19232628.524137933,
-      "logps/chosen": -72.89530944824219,
-      "logps/rejected": -81.06624461206897,
-      "loss": 0.1861,
-      "rewards/chosen": -7.082716941833496,
-      "rewards/margins": 0.38840038036477953,
-      "rewards/rejected": -7.471117322198276,
+      "learning_rate": 3.858764186633039e-08,
+      "logits/chosen": -15603414.0,
+      "logits/rejected": -17277662.455172412,
+      "logps/chosen": -8.103792190551758,
+      "logps/rejected": -14.19166049299569,
+      "loss": 0.3791,
+      "rewards/chosen": -0.6035651564598083,
+      "rewards/margins": 0.18009341708545024,
+      "rewards/rejected": -0.7836585735452586,
       "step": 2301
     },
     {
       "epoch": 0.7029787234042553,
-      "grad_norm": 0.009404444135725498,
+      "grad_norm": 2.259669542312622,
       "kl": 0.0,
-      "learning_rate": 1.6503783102143755e-07,
-      "logits/chosen": -17324397.714285713,
-      "logits/rejected": -19416927.721739132,
-      "logps/chosen": -74.51356907894737,
-      "logps/rejected": -81.89915760869565,
-      "loss": 0.1853,
-      "rewards/chosen": -7.221235461701128,
-      "rewards/margins": 0.35202370949452444,
-      "rewards/rejected": -7.5732591711956525,
+      "learning_rate": 3.300756620428751e-08,
+      "logits/chosen": -15778993.082706766,
+      "logits/rejected": -17491624.292173915,
+      "logps/chosen": -9.979437118186091,
+      "logps/rejected": -16.35701086956522,
+      "loss": 0.357,
+      "rewards/chosen": -0.7678221508972627,
+      "rewards/margins": 0.25122187933371554,
+      "rewards/rejected": -1.0190440302309782,
       "step": 2478
     },
     {
       "epoch": 0.7531914893617021,
-      "grad_norm": 0.00973726250231266,
+      "grad_norm": 3.3658533096313477,
       "kl": 0.0,
-      "learning_rate": 1.371374527112232e-07,
-      "logits/chosen": -16804548.204379562,
-      "logits/rejected": -19371664.364273205,
-      "logps/chosen": -75.09304430885037,
-      "logps/rejected": -82.9453535464098,
-      "loss": 0.1913,
-      "rewards/chosen": -7.289654585566834,
-      "rewards/margins": 0.3768818157575975,
-      "rewards/rejected": -7.666536401324431,
+      "learning_rate": 2.742749054224464e-08,
+      "logits/chosen": -15436312.291970802,
+      "logits/rejected": -17457363.614711035,
+      "logps/chosen": -11.540958599452555,
+      "logps/rejected": -18.07249138025394,
+      "loss": 0.3366,
+      "rewards/chosen": -0.9344464601391423,
+      "rewards/margins": 0.2448034252729635,
+      "rewards/rejected": -1.1792498854121058,
       "step": 2655
     },
     {
       "epoch": 0.8034042553191489,
-      "grad_norm": 0.01282684225589037,
+      "grad_norm": 2.318241834640503,
       "kl": 0.0,
-      "learning_rate": 1.0923707440100883e-07,
-      "logits/chosen": -17255675.78600823,
-      "logits/rejected": -19418751.89087809,
-      "logps/chosen": -75.07608346193416,
-      "logps/rejected": -83.94623827791986,
-      "loss": 0.1892,
-      "rewards/chosen": -7.311107996559928,
-      "rewards/margins": 0.4396163081075919,
-      "rewards/rejected": -7.75072430466752,
+      "learning_rate": 2.1847414880201765e-08,
+      "logits/chosen": -15859763.621399177,
+      "logits/rejected": -17528114.41432225,
+      "logps/chosen": -13.565882804462449,
+      "logps/rejected": -20.141737265558397,
+      "loss": 0.3189,
+      "rewards/chosen": -1.1600869261188271,
+      "rewards/margins": 0.21018625745827224,
+      "rewards/rejected": -1.3702731835770994,
       "step": 2832
     },
     {
       "epoch": 0.8536170212765958,
-      "grad_norm": 0.00687984237447381,
+      "grad_norm": 1.956950306892395,
       "kl": 0.0,
-      "learning_rate": 8.133669609079445e-08,
-      "logits/chosen": -16891363.235955056,
-      "logits/rejected": -19397185.50391645,
-      "logps/chosen": -76.50997776217228,
-      "logps/rejected": -84.4607743146214,
-      "loss": 0.1951,
-      "rewards/chosen": -7.409224949525983,
-      "rewards/margins": 0.40561744059586147,
-      "rewards/rejected": -7.814842390121845,
+      "learning_rate": 1.626733921815889e-08,
+      "logits/chosen": -15627038.68164794,
+      "logits/rejected": -17529227.69712794,
+      "logps/chosen": -15.227195253979401,
+      "logps/rejected": -21.2347761640557,
+      "loss": 0.3081,
+      "rewards/chosen": -1.2809460814972495,
+      "rewards/margins": 0.21129570738846626,
+      "rewards/rejected": -1.4922417888857158,
       "step": 3009
     },
     {
       "epoch": 0.9038297872340425,
-      "grad_norm": 0.0068199522793293,
+      "grad_norm": 2.3128018379211426,
       "kl": 0.0,
-      "learning_rate": 5.343631778058008e-08,
-      "logits/chosen": -17728405.0989011,
-      "logits/rejected": -19307877.459317584,
-      "logps/chosen": -77.02319425366301,
-      "logps/rejected": -84.82096183289589,
-      "loss": 0.1877,
-      "rewards/chosen": -7.465746086595696,
-      "rewards/margins": 0.4030767070394745,
-      "rewards/rejected": -7.868822793635171,
+      "learning_rate": 1.0687263556116015e-08,
+      "logits/chosen": -16287571.457875459,
+      "logits/rejected": -17474259.877515312,
+      "logps/chosen": -16.702005351419412,
+      "logps/rejected": -22.0252146216098,
+      "loss": 0.2956,
+      "rewards/chosen": -1.4336266150841346,
+      "rewards/margins": 0.1556206671159419,
+      "rewards/rejected": -1.5892472822000765,
       "step": 3186
     },
     {
       "epoch": 0.9540425531914893,
-      "grad_norm": 0.005797545425593853,
+      "grad_norm": 0.8989251255989075,
       "kl": 0.0,
-      "learning_rate": 2.55359394703657e-08,
-      "logits/chosen": -17274124.19047619,
-      "logits/rejected": -19408322.419243988,
-      "logps/chosen": -77.08465091765873,
-      "logps/rejected": -85.22552888745705,
-      "loss": 0.1786,
-      "rewards/chosen": -7.480488126240079,
-      "rewards/margins": 0.42159470371052254,
-      "rewards/rejected": -7.902082829950602,
+      "learning_rate": 5.1071878940731394e-09,
+      "logits/chosen": -15992458.158730159,
+      "logits/rejected": -17564913.04467354,
+      "logps/chosen": -16.995010618179563,
+      "logps/rejected": -22.893974173109967,
+      "loss": 0.2823,
+      "rewards/chosen": -1.4715233454628596,
+      "rewards/margins": 0.19740445351292446,
+      "rewards/rejected": -1.668927798975784,
       "step": 3363
     },
     {
       "epoch": 1.0,
       "step": 3525,
       "total_flos": 5.091091732590756e+17,
-      "train_loss": 0.23059077864843058,
-      "train_runtime": 10627.7324,
-      "train_samples_per_second": 5.307,
-      "train_steps_per_second": 0.332
+      "train_loss": 0.413429944193955,
+      "train_runtime": 10515.385,
+      "train_samples_per_second": 5.364,
+      "train_steps_per_second": 0.335
     }
   ],
   "logging_steps": 177,