Upload folder using huggingface_hub

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:30692ceb235c4429caf85385bb5458694b351ae55100bbfb3f50a74566d5f9f1
+oid sha256:ccb63f5f7e748555daeeb40663638cb4ee36e3274e2691792da29cde974199b6
 size 2384234968

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd62ddddee6c23a6650a91d1b2d97b6552130af70cc0c03b742eeb84fd96ec01
+oid sha256:bc572568881d9b759c8f59d75bf1b9cb28ba836994c8f509e1180f5fbe94eb68
 size 4768663315

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:250560ab3d528161ab3659b120def6e4a9ab4b457e3399603bbcfa40db3efc90
+oid sha256:2534e434cd5abbb8f7668d3eab0549db0ef95d6a797a3efa86b712e8e32266a7
 size 14645

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:60dc60940088350df2476daec612777256353a222718d12ad1d77c0a8edad709
+oid sha256:7eda8fb70e02ca89c72bc8ed5c2b4af18bf6ab7515831776b7b43399b9d94e12
 size 1465

trainer_state.json

@@ -2,453 +2,1353 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 3.0,
   "eval_steps": 500,
-  "global_step": 1493,
+  "global_step": 4479,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.033489618218352314,
-      "grad_norm": 90.51985931396484,
-      "learning_rate": 4.835900870730074e-06,
+      "grad_norm": 78.33113098144531,
+      "learning_rate": 2.1875e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -179.42369079589844,
-      "logps/rejected": -227.34011840820312,
-      "loss": 0.6272,
-      "rewards/accuracies": 0.4987500011920929,
-      "rewards/chosen": -0.41917884349823,
-      "rewards/margins": 0.4206826388835907,
-      "rewards/rejected": -0.8398614525794983,
+      "logps/chosen": -175.2264862060547,
+      "logps/rejected": -218.9656982421875,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.4137499928474426,
+      "rewards/chosen": 0.0005424434202723205,
+      "rewards/margins": 0.0029623538721352816,
+      "rewards/rejected": -0.0024199108593165874,
       "step": 50
     },
     {
       "epoch": 0.06697923643670463,
-      "grad_norm": 122.67408752441406,
-      "learning_rate": 4.668452779638312e-06,
+      "grad_norm": 106.17163848876953,
+      "learning_rate": 4.419642857142857e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -189.38446044921875,
-      "logps/rejected": -243.21636962890625,
-      "loss": 0.6008,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.9909499287605286,
-      "rewards/margins": 0.8664093613624573,
-      "rewards/rejected": -1.8573591709136963,
+      "logps/chosen": -179.5259246826172,
+      "logps/rejected": -224.7578887939453,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.42124998569488525,
+      "rewards/chosen": -0.005094751715660095,
+      "rewards/margins": 0.00641661649569869,
+      "rewards/rejected": -0.011511369608342648,
       "step": 100
     },
     {
       "epoch": 0.10046885465505694,
-      "grad_norm": 69.09490203857422,
-      "learning_rate": 4.501004688546551e-06,
+      "grad_norm": 86.04861450195312,
+      "learning_rate": 6.651785714285713e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -172.51051330566406,
-      "logps/rejected": -237.64398193359375,
-      "loss": 0.5445,
-      "rewards/accuracies": 0.5912500023841858,
-      "rewards/chosen": -0.773537814617157,
-      "rewards/margins": 1.0922305583953857,
-      "rewards/rejected": -1.8657684326171875,
+      "logps/chosen": -165.04095458984375,
+      "logps/rejected": -219.6518096923828,
+      "loss": 0.6756,
+      "rewards/accuracies": 0.5112500190734863,
+      "rewards/chosen": -0.026584235951304436,
+      "rewards/margins": 0.03996539115905762,
+      "rewards/rejected": -0.0665496289730072,
       "step": 150
     },
     {
       "epoch": 0.13395847287340926,
-      "grad_norm": 92.90730285644531,
-      "learning_rate": 4.333556597454789e-06,
+      "grad_norm": 82.77224731445312,
+      "learning_rate": 8.88392857142857e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -185.27850341796875,
-      "logps/rejected": -240.15431213378906,
-      "loss": 0.5958,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.8680741190910339,
-      "rewards/margins": 1.0129902362823486,
-      "rewards/rejected": -1.8810642957687378,
+      "logps/chosen": -177.75872802734375,
+      "logps/rejected": -223.51528930664062,
+      "loss": 0.6591,
+      "rewards/accuracies": 0.5099999904632568,
+      "rewards/chosen": -0.11609632521867752,
+      "rewards/margins": 0.10106377303600311,
+      "rewards/rejected": -0.21716010570526123,
       "step": 200
     },
     {
       "epoch": 0.16744809109176156,
-      "grad_norm": 41.30106735229492,
-      "learning_rate": 4.166108506363028e-06,
+      "grad_norm": 135.95346069335938,
+      "learning_rate": 1.1116071428571427e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -185.0808563232422,
-      "logps/rejected": -243.77664184570312,
-      "loss": 0.5232,
-      "rewards/accuracies": 0.5950000286102295,
-      "rewards/chosen": -0.6241927742958069,
-      "rewards/margins": 1.5013166666030884,
-      "rewards/rejected": -2.125509262084961,
+      "logps/chosen": -180.67254638671875,
+      "logps/rejected": -226.42140197753906,
+      "loss": 0.6295,
+      "rewards/accuracies": 0.5099999904632568,
+      "rewards/chosen": -0.18336135149002075,
+      "rewards/margins": 0.20662552118301392,
+      "rewards/rejected": -0.38998690247535706,
       "step": 250
     },
     {
       "epoch": 0.20093770931011387,
-      "grad_norm": 88.7403793334961,
-      "learning_rate": 3.998660415271266e-06,
+      "grad_norm": 89.77359771728516,
+      "learning_rate": 1.3348214285714285e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -184.0787353515625,
-      "logps/rejected": -243.6924285888672,
-      "loss": 0.5419,
-      "rewards/accuracies": 0.5950000286102295,
-      "rewards/chosen": -0.8100302219390869,
-      "rewards/margins": 1.521090030670166,
-      "rewards/rejected": -2.331120491027832,
+      "logps/chosen": -178.88687133789062,
+      "logps/rejected": -226.58355712890625,
+      "loss": 0.602,
+      "rewards/accuracies": 0.5637500286102295,
+      "rewards/chosen": -0.29084426164627075,
+      "rewards/margins": 0.32938891649246216,
+      "rewards/rejected": -0.6202332377433777,
       "step": 300
     },
     {
       "epoch": 0.23442732752846618,
-      "grad_norm": 70.95861053466797,
-      "learning_rate": 3.831212324179505e-06,
-      "logits/chosen": NaN,
-      "logits/rejected": -1.532848834991455,
-      "logps/chosen": -182.43124389648438,
-      "logps/rejected": -248.5233917236328,
-      "loss": 0.5533,
-      "rewards/accuracies": 0.5824999809265137,
-      "rewards/chosen": -1.069503903388977,
-      "rewards/margins": 1.6424156427383423,
-      "rewards/rejected": -2.7119195461273193,
+      "grad_norm": 89.93605041503906,
+      "learning_rate": 1.558035714285714e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": -1.608971118927002,
+      "logps/chosen": -176.1905059814453,
+      "logps/rejected": -231.0211944580078,
+      "loss": 0.5782,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.4454282522201538,
+      "rewards/margins": 0.5162708163261414,
+      "rewards/rejected": -0.9616988897323608,
       "step": 350
     },
     {
       "epoch": 0.2679169457468185,
-      "grad_norm": 71.3283462524414,
-      "learning_rate": 3.663764233087743e-06,
+      "grad_norm": 113.58289337158203,
+      "learning_rate": 1.7812499999999999e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -180.90724182128906,
-      "logps/rejected": -253.06655883789062,
-      "loss": 0.5131,
-      "rewards/accuracies": 0.6449999809265137,
-      "rewards/chosen": -0.9932506084442139,
-      "rewards/margins": 2.001958131790161,
-      "rewards/rejected": -2.995208740234375,
+      "logps/chosen": -176.52401733398438,
+      "logps/rejected": -236.76588439941406,
+      "loss": 0.5478,
+      "rewards/accuracies": 0.6150000095367432,
+      "rewards/chosen": -0.5549299120903015,
+      "rewards/margins": 0.8102107048034668,
+      "rewards/rejected": -1.3651405572891235,
       "step": 400
     },
     {
       "epoch": 0.3014065639651708,
-      "grad_norm": 52.41379928588867,
-      "learning_rate": 3.496316141995982e-06,
+      "grad_norm": 100.28213500976562,
+      "learning_rate": 1.999999696300462e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -187.8169708251953,
-      "logps/rejected": -250.7092742919922,
-      "loss": 0.574,
-      "rewards/accuracies": 0.6087499856948853,
-      "rewards/chosen": -0.9525413513183594,
-      "rewards/margins": 1.8562077283859253,
-      "rewards/rejected": -2.808749198913574,
+      "logps/chosen": -183.1260223388672,
+      "logps/rejected": -235.15631103515625,
+      "loss": 0.5635,
+      "rewards/accuracies": 0.5799999833106995,
+      "rewards/chosen": -0.48344433307647705,
+      "rewards/margins": 0.770007848739624,
+      "rewards/rejected": -1.253452181816101,
       "step": 450
     },
     {
       "epoch": 0.33489618218352313,
-      "grad_norm": 86.05554962158203,
-      "learning_rate": 3.32886805090422e-06,
+      "grad_norm": 90.32833099365234,
+      "learning_rate": 1.999210181452139e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -184.93011474609375,
-      "logps/rejected": -245.39427185058594,
-      "loss": 0.545,
-      "rewards/accuracies": 0.6200000047683716,
-      "rewards/chosen": -0.9822418093681335,
-      "rewards/margins": 1.7062770128250122,
-      "rewards/rejected": -2.68851900100708,
+      "logps/chosen": -180.36907958984375,
+      "logps/rejected": -232.14285278320312,
+      "loss": 0.5376,
+      "rewards/accuracies": 0.6087499856948853,
+      "rewards/chosen": -0.5261387825012207,
+      "rewards/margins": 0.8372372984886169,
+      "rewards/rejected": -1.3633761405944824,
       "step": 500
     },
     {
       "epoch": 0.3683858004018754,
-      "grad_norm": 24.4642391204834,
-      "learning_rate": 3.1614199598124583e-06,
+      "grad_norm": 72.57466125488281,
+      "learning_rate": 1.996903560165487e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -177.97482299804688,
-      "logps/rejected": -253.92169189453125,
-      "loss": 0.5249,
-      "rewards/accuracies": 0.6150000095367432,
-      "rewards/chosen": -0.7585690021514893,
-      "rewards/margins": 2.060235023498535,
-      "rewards/rejected": -2.8188037872314453,
+      "logps/chosen": -175.88233947753906,
+      "logps/rejected": -242.15728759765625,
+      "loss": 0.5083,
+      "rewards/accuracies": 0.6225000023841858,
+      "rewards/chosen": -0.5493210554122925,
+      "rewards/margins": 1.0930429697036743,
+      "rewards/rejected": -1.6423640251159668,
       "step": 550
     },
     {
       "epoch": 0.40187541862022774,
-      "grad_norm": 28.010169982910156,
-      "learning_rate": 2.993971868720697e-06,
+      "grad_norm": 47.55934143066406,
+      "learning_rate": 1.993083334596579e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -187.2054443359375,
-      "logps/rejected": -262.4362487792969,
-      "loss": 0.531,
-      "rewards/accuracies": 0.6212499737739563,
-      "rewards/chosen": -1.0288543701171875,
-      "rewards/margins": 2.0049030780792236,
-      "rewards/rejected": -3.033757209777832,
+      "logps/chosen": -184.1678924560547,
+      "logps/rejected": -251.43661499023438,
+      "loss": 0.5193,
+      "rewards/accuracies": 0.6225000023841858,
+      "rewards/chosen": -0.7250985503196716,
+      "rewards/margins": 1.2086968421936035,
+      "rewards/rejected": -1.9337953329086304,
       "step": 600
     },
     {
       "epoch": 0.43536503683858,
-      "grad_norm": 88.00039672851562,
-      "learning_rate": 2.8265237776289352e-06,
+      "grad_norm": 90.7481460571289,
+      "learning_rate": 1.987755305015383e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -199.0946044921875,
-      "logps/rejected": -257.4405212402344,
-      "loss": 0.506,
-      "rewards/accuracies": 0.6175000071525574,
-      "rewards/chosen": -0.9385867118835449,
-      "rewards/margins": 1.9474469423294067,
-      "rewards/rejected": -2.8860340118408203,
+      "logps/chosen": -196.693359375,
+      "logps/rejected": -247.3010711669922,
+      "loss": 0.516,
+      "rewards/accuracies": 0.6137499809265137,
+      "rewards/chosen": -0.6984607577323914,
+      "rewards/margins": 1.173628807067871,
+      "rewards/rejected": -1.8720895051956177,
       "step": 650
     },
     {
       "epoch": 0.46885465505693236,
-      "grad_norm": 76.95013427734375,
-      "learning_rate": 2.6590756865371737e-06,
+      "grad_norm": 86.08389282226562,
+      "learning_rate": 1.980927560999178e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -188.61651611328125,
-      "logps/rejected": -255.59820556640625,
-      "loss": 0.4841,
-      "rewards/accuracies": 0.6287500262260437,
-      "rewards/chosen": -0.9187755584716797,
-      "rewards/margins": 2.190307378768921,
-      "rewards/rejected": -3.1090831756591797,
+      "logps/chosen": -186.29693603515625,
+      "logps/rejected": -245.04824829101562,
+      "loss": 0.5057,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.6868166327476501,
+      "rewards/margins": 1.367271900177002,
+      "rewards/rejected": -2.0540883541107178,
       "step": 700
     },
     {
       "epoch": 0.5023442732752846,
-      "grad_norm": 37.079219818115234,
-      "learning_rate": 2.491627595445412e-06,
+      "grad_norm": 40.12553405761719,
+      "learning_rate": 1.9726104691501045e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -182.82937622070312,
-      "logps/rejected": -253.3831787109375,
-      "loss": 0.5061,
-      "rewards/accuracies": 0.6324999928474426,
-      "rewards/chosen": -0.8986356854438782,
-      "rewards/margins": 2.1806037425994873,
-      "rewards/rejected": -3.0792391300201416,
+      "logps/chosen": -179.41378784179688,
+      "logps/rejected": -240.62547302246094,
+      "loss": 0.5132,
+      "rewards/accuracies": 0.5975000262260437,
+      "rewards/chosen": -0.5570769309997559,
+      "rewards/margins": 1.2463946342468262,
+      "rewards/rejected": -1.803471326828003,
       "step": 750
     },
     {
       "epoch": 0.535833891493637,
-      "grad_norm": 31.89027976989746,
-      "learning_rate": 2.3241795043536505e-06,
+      "grad_norm": 36.09309005737305,
+      "learning_rate": 1.9628166573554945e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -172.42593383789062,
-      "logps/rejected": -251.63729858398438,
-      "loss": 0.4358,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.7772516012191772,
-      "rewards/margins": 2.509305953979492,
-      "rewards/rejected": -3.286557674407959,
+      "logps/chosen": -170.22169494628906,
+      "logps/rejected": -239.9406280517578,
+      "loss": 0.4553,
+      "rewards/accuracies": 0.6449999809265137,
+      "rewards/chosen": -0.5568282604217529,
+      "rewards/margins": 1.5600597858428955,
+      "rewards/rejected": -2.1168878078460693,
       "step": 800
     },
     {
       "epoch": 0.5693235097119893,
-      "grad_norm": 49.636253356933594,
-      "learning_rate": 2.156731413261889e-06,
+      "grad_norm": 88.8606185913086,
+      "learning_rate": 1.951560995614879e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -177.4981231689453,
-      "logps/rejected": -252.06234741210938,
-      "loss": 0.4752,
-      "rewards/accuracies": 0.6262500286102295,
-      "rewards/chosen": -0.8873915672302246,
-      "rewards/margins": 2.302338123321533,
-      "rewards/rejected": -3.1897289752960205,
+      "logps/chosen": -175.4136199951172,
+      "logps/rejected": -241.44386291503906,
+      "loss": 0.4912,
+      "rewards/accuracies": 0.6175000071525574,
+      "rewards/chosen": -0.6789398193359375,
+      "rewards/margins": 1.448940634727478,
+      "rewards/rejected": -2.127880573272705,
       "step": 850
     },
     {
       "epoch": 0.6028131279303416,
-      "grad_norm": 42.22234344482422,
-      "learning_rate": 1.9892833221701274e-06,
+      "grad_norm": 37.501346588134766,
+      "learning_rate": 1.9388605734627843e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -185.84585571289062,
-      "logps/rejected": -251.1869354248047,
-      "loss": 0.5284,
-      "rewards/accuracies": 0.6200000047683716,
-      "rewards/chosen": -0.9590955972671509,
-      "rewards/margins": 2.267340660095215,
-      "rewards/rejected": -3.2264363765716553,
+      "logps/chosen": -183.4543914794922,
+      "logps/rejected": -241.45433044433594,
+      "loss": 0.505,
+      "rewards/accuracies": 0.6212499737739563,
+      "rewards/chosen": -0.719947338104248,
+      "rewards/margins": 1.5332283973693848,
+      "rewards/rejected": -2.253175735473633,
       "step": 900
     },
     {
       "epoch": 0.6363027461486939,
-      "grad_norm": 60.56090545654297,
-      "learning_rate": 1.8218352310783657e-06,
+      "grad_norm": 58.78173065185547,
+      "learning_rate": 1.9247346740215936e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -182.73912048339844,
-      "logps/rejected": -245.673828125,
-      "loss": 0.4587,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.6209310293197632,
-      "rewards/margins": 2.47007155418396,
-      "rewards/rejected": -3.0910024642944336,
+      "logps/chosen": -182.4608612060547,
+      "logps/rejected": -236.8692169189453,
+      "loss": 0.4756,
+      "rewards/accuracies": 0.6274999976158142,
+      "rewards/chosen": -0.5931037068367004,
+      "rewards/margins": 1.6174336671829224,
+      "rewards/rejected": -2.2105374336242676,
       "step": 950
     },
     {
       "epoch": 0.6697923643670463,
-      "grad_norm": 47.82743453979492,
-      "learning_rate": 1.6543871399866043e-06,
+      "grad_norm": 53.627410888671875,
+      "learning_rate": 1.909204744723877e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -170.28599548339844,
-      "logps/rejected": -247.58837890625,
-      "loss": 0.4471,
-      "rewards/accuracies": 0.6424999833106995,
-      "rewards/chosen": -0.5806804895401001,
-      "rewards/margins": 2.4890031814575195,
-      "rewards/rejected": -3.06968355178833,
+      "logps/chosen": -169.64356994628906,
+      "logps/rejected": -238.07931518554688,
+      "loss": 0.4699,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.5164381265640259,
+      "rewards/margins": 1.6023368835449219,
+      "rewards/rejected": -2.1187753677368164,
       "step": 1000
     },
     {
       "epoch": 0.7032819825853985,
-      "grad_norm": 33.236053466796875,
-      "learning_rate": 1.4869390488948425e-06,
+      "grad_norm": 47.64691162109375,
+      "learning_rate": 1.8922943647486314e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -173.26913452148438,
-      "logps/rejected": -257.9516906738281,
-      "loss": 0.4204,
-      "rewards/accuracies": 0.6825000047683716,
-      "rewards/chosen": -0.4792047142982483,
-      "rewards/margins": 2.651007890701294,
-      "rewards/rejected": -3.1302123069763184,
+      "logps/chosen": -174.08212280273438,
+      "logps/rejected": -251.6885223388672,
+      "loss": 0.4309,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.560505211353302,
+      "rewards/margins": 1.9433872699737549,
+      "rewards/rejected": -2.503892421722412,
       "step": 1050
     },
     {
       "epoch": 0.7367716008037508,
-      "grad_norm": 39.73881530761719,
-      "learning_rate": 1.3194909578030812e-06,
+      "grad_norm": 58.94224166870117,
+      "learning_rate": 1.8740292092208816e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -161.0741424560547,
-      "logps/rejected": -243.2300262451172,
-      "loss": 0.4058,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.5020321607589722,
-      "rewards/margins": 2.7466940879821777,
-      "rewards/rejected": -3.2487261295318604,
+      "logps/chosen": -162.09487915039062,
+      "logps/rejected": -236.79824829101562,
+      "loss": 0.4293,
+      "rewards/accuracies": 0.6524999737739563,
+      "rewards/chosen": -0.6041057705879211,
+      "rewards/margins": 2.0014426708221436,
+      "rewards/rejected": -2.60554838180542,
       "step": 1100
     },
     {
       "epoch": 0.7702612190221031,
-      "grad_norm": 37.8878173828125,
-      "learning_rate": 1.1520428667113196e-06,
+      "grad_norm": 41.707763671875,
+      "learning_rate": 1.8544370102289943e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -177.0689697265625,
-      "logps/rejected": -248.60035705566406,
-      "loss": 0.4246,
-      "rewards/accuracies": 0.6575000286102295,
-      "rewards/chosen": -0.6515741944313049,
-      "rewards/margins": 2.5524582862854004,
-      "rewards/rejected": -3.2040326595306396,
+      "logps/chosen": -177.0761260986328,
+      "logps/rejected": -240.7725067138672,
+      "loss": 0.4419,
+      "rewards/accuracies": 0.6612499952316284,
+      "rewards/chosen": -0.6522895097732544,
+      "rewards/margins": 1.7689578533172607,
+      "rewards/rejected": -2.4212474822998047,
       "step": 1150
     },
     {
       "epoch": 0.8037508372404555,
-      "grad_norm": 54.17752456665039,
-      "learning_rate": 9.84594775619558e-07,
+      "grad_norm": 45.48369216918945,
+      "learning_rate": 1.83354751471889e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -184.8086395263672,
-      "logps/rejected": -270.40771484375,
-      "loss": 0.4499,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.5556185841560364,
-      "rewards/margins": 2.5394182205200195,
-      "rewards/rejected": -3.0950369834899902,
+      "logps/chosen": -184.2169952392578,
+      "logps/rejected": -264.9205322265625,
+      "loss": 0.4503,
+      "rewards/accuracies": 0.6549999713897705,
+      "rewards/chosen": -0.49645543098449707,
+      "rewards/margins": 2.04986572265625,
+      "rewards/rejected": -2.546321392059326,
       "step": 1200
     },
     {
       "epoch": 0.8372404554588078,
-      "grad_norm": 37.586639404296875,
-      "learning_rate": 8.171466845277964e-07,
+      "grad_norm": 51.16058349609375,
+      "learning_rate": 1.8113924393290904e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -182.44801330566406,
-      "logps/rejected": -255.1699676513672,
-      "loss": 0.4307,
-      "rewards/accuracies": 0.6537500023841858,
-      "rewards/chosen": -0.6889155507087708,
-      "rewards/margins": 2.603567123413086,
-      "rewards/rejected": -3.292482852935791,
+      "logps/chosen": -182.03074645996094,
+      "logps/rejected": -249.8163604736328,
+      "loss": 0.4319,
+      "rewards/accuracies": 0.6612499952316284,
+      "rewards/chosen": -0.6471911072731018,
+      "rewards/margins": 2.1099319458007812,
+      "rewards/rejected": -2.7571229934692383,
       "step": 1250
     },
     {
       "epoch": 0.87073007367716,
-      "grad_norm": 52.10836410522461,
-      "learning_rate": 6.496985934360349e-07,
+      "grad_norm": 64.02259063720703,
+      "learning_rate": 1.7880054222351658e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -179.65557861328125,
-      "logps/rejected": -245.46560668945312,
-      "loss": 0.4125,
-      "rewards/accuracies": 0.6637499928474426,
-      "rewards/chosen": -0.5343858003616333,
-      "rewards/margins": 2.6487746238708496,
-      "rewards/rejected": -3.1831603050231934,
+      "logps/chosen": -178.18972778320312,
+      "logps/rejected": -237.3641815185547,
+      "loss": 0.4155,
+      "rewards/accuracies": 0.6725000143051147,
+      "rewards/chosen": -0.38780125975608826,
+      "rewards/margins": 1.9852185249328613,
+      "rewards/rejected": -2.3730199337005615,
       "step": 1300
     },
     {
       "epoch": 0.9042196918955124,
-      "grad_norm": 30.635120391845703,
-      "learning_rate": 4.822505023442733e-07,
+      "grad_norm": 35.12641525268555,
+      "learning_rate": 1.763421972076705e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -175.81723022460938,
-      "logps/rejected": -254.23828125,
-      "loss": 0.4304,
-      "rewards/accuracies": 0.6524999737739563,
-      "rewards/chosen": -0.5225290060043335,
-      "rewards/margins": 2.57625675201416,
-      "rewards/rejected": -3.098785638809204,
+      "logps/chosen": -175.52285766601562,
+      "logps/rejected": -247.11244201660156,
+      "loss": 0.4359,
+      "rewards/accuracies": 0.6512500047683716,
+      "rewards/chosen": -0.493091344833374,
+      "rewards/margins": 1.8931076526641846,
+      "rewards/rejected": -2.3861987590789795,
       "step": 1350
     },
     {
       "epoch": 0.9377093101138647,
-      "grad_norm": 49.38210678100586,
-      "learning_rate": 3.1480241125251174e-07,
+      "grad_norm": 64.41110229492188,
+      "learning_rate": 1.7376794140443474e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -177.96661376953125,
-      "logps/rejected": -240.63694763183594,
-      "loss": 0.4411,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.4394516050815582,
-      "rewards/margins": 2.5781943798065186,
-      "rewards/rejected": -3.017645835876465,
+      "logps/chosen": -178.29629516601562,
+      "logps/rejected": -234.5249481201172,
+      "loss": 0.4512,
+      "rewards/accuracies": 0.6549999713897705,
+      "rewards/chosen": -0.4724200367927551,
+      "rewards/margins": 1.9340243339538574,
+      "rewards/rejected": -2.4064440727233887,
       "step": 1400
     },
     {
       "epoch": 0.971198928332217,
-      "grad_norm": 57.8912467956543,
-      "learning_rate": 1.4735432016075018e-07,
+      "grad_norm": 26.93653106689453,
+      "learning_rate": 1.7108168332087366e-06,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -176.60218811035156,
-      "logps/rejected": -250.90786743164062,
-      "loss": 0.4317,
-      "rewards/accuracies": 0.6474999785423279,
-      "rewards/chosen": -0.514062225818634,
-      "rewards/margins": 2.4711577892303467,
-      "rewards/rejected": -2.985220432281494,
+      "logps/chosen": -175.42259216308594,
+      "logps/rejected": -243.82032775878906,
+      "loss": 0.4343,
+      "rewards/accuracies": 0.6512500047683716,
+      "rewards/chosen": -0.3961036205291748,
+      "rewards/margins": 1.8803616762161255,
+      "rewards/rejected": -2.27646541595459,
       "step": 1450
+    },
+    {
+      "epoch": 1.0046885465505693,
+      "grad_norm": 74.74053955078125,
+      "learning_rate": 1.682875015177438e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -174.56732177734375,
+      "logps/rejected": -246.36451721191406,
+      "loss": 0.3957,
+      "rewards/accuracies": 0.6800000071525574,
+      "rewards/chosen": -0.34164169430732727,
+      "rewards/margins": 2.248396635055542,
+      "rewards/rejected": -2.590038537979126,
+      "step": 1500
+    },
+    {
+      "epoch": 1.0381781647689217,
+      "grad_norm": 58.65504455566406,
+      "learning_rate": 1.6538963841699207e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -176.5469207763672,
+      "logps/rejected": -258.92706298828125,
+      "loss": 0.2861,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.2739707827568054,
+      "rewards/margins": 3.0113985538482666,
+      "rewards/rejected": -3.2853691577911377,
+      "step": 1550
+    },
+    {
+      "epoch": 1.0716677829872738,
+      "grad_norm": 59.74324417114258,
+      "learning_rate": 1.6239249386046274e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -177.00692749023438,
+      "logps/rejected": -255.23556518554688,
+      "loss": 0.2914,
+      "rewards/accuracies": 0.7549999952316284,
+      "rewards/chosen": -0.4652925729751587,
+      "rewards/margins": 3.098710298538208,
+      "rewards/rejected": -3.564002752304077,
+      "step": 1600
+    },
+    {
+      "epoch": 1.1051574012056262,
+      "grad_norm": 37.80025863647461,
+      "learning_rate": 1.593006184295927e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -185.12716674804688,
+      "logps/rejected": -254.19509887695312,
+      "loss": 0.2798,
+      "rewards/accuracies": 0.7524999976158142,
+      "rewards/chosen": -0.28863173723220825,
+      "rewards/margins": 3.227825880050659,
+      "rewards/rejected": -3.516458034515381,
+      "step": 1650
+    },
+    {
+      "epoch": 1.1386470194239786,
+      "grad_norm": 40.97309875488281,
+      "learning_rate": 1.5611870653623825e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -182.1793975830078,
+      "logps/rejected": -245.0845184326172,
+      "loss": 0.2778,
+      "rewards/accuracies": 0.7450000047683716,
+      "rewards/chosen": -0.3949226438999176,
+      "rewards/margins": 3.3151471614837646,
+      "rewards/rejected": -3.7100696563720703,
+      "step": 1700
+    },
+    {
+      "epoch": 1.1721366376423308,
+      "grad_norm": 61.272247314453125,
+      "learning_rate": 1.5285158929512291e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -174.18487548828125,
+      "logps/rejected": -247.96957397460938,
+      "loss": 0.3048,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.4471362233161926,
+      "rewards/margins": 3.481740951538086,
+      "rewards/rejected": -3.928877830505371,
+      "step": 1750
+    },
+    {
+      "epoch": 1.2056262558606832,
+      "grad_norm": 20.384906768798828,
+      "learning_rate": 1.4950422718872916e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -177.91143798828125,
+      "logps/rejected": -264.8081970214844,
+      "loss": 0.2738,
+      "rewards/accuracies": 0.7574999928474426,
+      "rewards/chosen": -0.4734611213207245,
+      "rewards/margins": 3.4893076419830322,
+      "rewards/rejected": -3.962768793106079,
+      "step": 1800
+    },
+    {
+      "epoch": 1.2391158740790356,
+      "grad_norm": 46.84432601928711,
+      "learning_rate": 1.4608170253576945e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -171.02236938476562,
+      "logps/rejected": -259.7280578613281,
+      "loss": 0.2928,
+      "rewards/accuracies": 0.7262499928474426,
+      "rewards/chosen": -0.6498711109161377,
+      "rewards/margins": 3.556124210357666,
+      "rewards/rejected": -4.205995082855225,
+      "step": 1850
+    },
+    {
+      "epoch": 1.2726054922973877,
+      "grad_norm": 40.36602020263672,
+      "learning_rate": 1.4258921177467371e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -176.40257263183594,
+      "logps/rejected": -251.6402130126953,
+      "loss": 0.301,
+      "rewards/accuracies": 0.7325000166893005,
+      "rewards/chosen": -0.7374945878982544,
+      "rewards/margins": 3.618178606033325,
+      "rewards/rejected": -4.355673789978027,
+      "step": 1900
+    },
+    {
+      "epoch": 1.3060951105157401,
+      "grad_norm": 33.35322952270508,
+      "learning_rate": 1.3903205757380715e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -177.98854064941406,
+      "logps/rejected": -259.6983337402344,
+      "loss": 0.2985,
+      "rewards/accuracies": 0.7275000214576721,
+      "rewards/chosen": -0.7513535022735596,
+      "rewards/margins": 3.433237314224243,
+      "rewards/rejected": -4.184591293334961,
+      "step": 1950
+    },
+    {
+      "epoch": 1.3395847287340925,
+      "grad_norm": 31.858760833740234,
+      "learning_rate": 1.3541564078039942e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -176.97511291503906,
+      "logps/rejected": -267.1122131347656,
+      "loss": 0.307,
+      "rewards/accuracies": 0.7174999713897705,
+      "rewards/chosen": -0.6912581920623779,
+      "rewards/margins": 3.4836156368255615,
+      "rewards/rejected": -4.1748738288879395,
+      "step": 2000
+    },
+    {
+      "epoch": 1.3730743469524447,
+      "grad_norm": 40.272186279296875,
+      "learning_rate": 1.3174545222040757e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -181.2541046142578,
+      "logps/rejected": -267.8948974609375,
+      "loss": 0.2764,
+      "rewards/accuracies": 0.7612500190734863,
+      "rewards/chosen": -0.5613307356834412,
+      "rewards/margins": 3.6199841499328613,
+      "rewards/rejected": -4.181314468383789,
+      "step": 2050
+    },
+    {
+      "epoch": 1.406563965170797,
+      "grad_norm": 20.189088821411133,
+      "learning_rate": 1.2802706436176447e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -186.3399658203125,
+      "logps/rejected": -275.252685546875,
+      "loss": 0.2673,
+      "rewards/accuracies": 0.7512500286102295,
+      "rewards/chosen": -0.49821099638938904,
+      "rewards/margins": 3.6726813316345215,
+      "rewards/rejected": -4.170892238616943,
+      "step": 2100
+    },
+    {
+      "epoch": 1.4400535833891492,
+      "grad_norm": 28.09309196472168,
+      "learning_rate": 1.2426612285366904e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -180.54571533203125,
+      "logps/rejected": -272.14337158203125,
+      "loss": 0.2833,
+      "rewards/accuracies": 0.7649999856948853,
+      "rewards/chosen": -0.5274211168289185,
+      "rewards/margins": 3.785543203353882,
+      "rewards/rejected": -4.31296443939209,
+      "step": 2150
+    },
+    {
+      "epoch": 1.4735432016075016,
+      "grad_norm": 5.396151542663574,
+      "learning_rate": 1.2046833795476566e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -178.48960876464844,
+      "logps/rejected": -268.61944580078125,
+      "loss": 0.2594,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.3929290771484375,
+      "rewards/margins": 3.8942084312438965,
+      "rewards/rejected": -4.287137508392334,
+      "step": 2200
+    },
+    {
+      "epoch": 1.507032819825854,
+      "grad_norm": 26.636991500854492,
+      "learning_rate": 1.16639475863226e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -183.34547424316406,
+      "logps/rejected": -259.8311462402344,
+      "loss": 0.3026,
+      "rewards/accuracies": 0.7200000286102295,
+      "rewards/chosen": -0.5500699281692505,
+      "rewards/margins": 3.565783739089966,
+      "rewards/rejected": -4.115853786468506,
+      "step": 2250
+    },
+    {
+      "epoch": 1.5405224380442064,
+      "grad_norm": 14.03653335571289,
+      "learning_rate": 1.1278534996189831e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -182.8995361328125,
+      "logps/rejected": -273.84112548828125,
+      "loss": 0.2603,
+      "rewards/accuracies": 0.7487499713897705,
+      "rewards/chosen": -0.5162584185600281,
+      "rewards/margins": 4.0679030418396,
+      "rewards/rejected": -4.584161758422852,
+      "step": 2300
+    },
+    {
+      "epoch": 1.5740120562625586,
+      "grad_norm": 67.45540618896484,
+      "learning_rate": 1.0891181199181518e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -176.06849670410156,
+      "logps/rejected": -265.9678649902344,
+      "loss": 0.272,
+      "rewards/accuracies": 0.7475000023841858,
+      "rewards/chosen": -0.5778465867042542,
+      "rewards/margins": 3.9320404529571533,
+      "rewards/rejected": -4.509886264801025,
+      "step": 2350
+    },
+    {
+      "epoch": 1.607501674480911,
+      "grad_norm": 21.127580642700195,
+      "learning_rate": 1.0502474316746242e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -178.6305694580078,
+      "logps/rejected": -265.5202331542969,
+      "loss": 0.2839,
+      "rewards/accuracies": 0.7462499737739563,
+      "rewards/chosen": -0.5587973594665527,
+      "rewards/margins": 3.9246935844421387,
+      "rewards/rejected": -4.48349142074585,
+      "step": 2400
+    },
+    {
+      "epoch": 1.6409912926992631,
+      "grad_norm": 47.24773025512695,
+      "learning_rate": 1.0113004524729797e-06,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -196.45948791503906,
+      "logps/rejected": -272.1256408691406,
+      "loss": 0.2791,
+      "rewards/accuracies": 0.7587500214576721,
+      "rewards/chosen": -0.5817875862121582,
+      "rewards/margins": 3.766108989715576,
+      "rewards/rejected": -4.347896099090576,
+      "step": 2450
+    },
+    {
+      "epoch": 1.6744809109176155,
+      "grad_norm": 20.178668975830078,
+      "learning_rate": 9.723363157307888e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -183.7681427001953,
+      "logps/rejected": -268.5182800292969,
+      "loss": 0.2744,
+      "rewards/accuracies": 0.7475000023841858,
+      "rewards/chosen": -0.5075680017471313,
+      "rewards/margins": 3.9134867191314697,
+      "rewards/rejected": -4.421054840087891,
+      "step": 2500
+    },
+    {
+      "epoch": 1.707970529135968,
+      "grad_norm": 31.073015213012695,
+      "learning_rate": 9.334141809160118e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -178.35658264160156,
+      "logps/rejected": -265.6587829589844,
+      "loss": 0.2405,
+      "rewards/accuracies": 0.7712500095367432,
+      "rewards/chosen": -0.6600850820541382,
+      "rewards/margins": 4.134018421173096,
+      "rewards/rejected": -4.794103622436523,
+      "step": 2550
+    },
+    {
+      "epoch": 1.7414601473543203,
+      "grad_norm": 36.3228759765625,
+      "learning_rate": 8.945931437248468e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -178.47000122070312,
+      "logps/rejected": -270.1788635253906,
+      "loss": 0.2674,
+      "rewards/accuracies": 0.7524999976158142,
+      "rewards/chosen": -0.6616349816322327,
+      "rewards/margins": 4.066000461578369,
+      "rewards/rejected": -4.727634906768799,
+      "step": 2600
+    },
+    {
+      "epoch": 1.7749497655726725,
+      "grad_norm": 27.108051300048828,
+      "learning_rate": 8.559321463564014e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -175.67808532714844,
+      "logps/rejected": -261.2061767578125,
+      "loss": 0.2494,
+      "rewards/accuracies": 0.7549999952316284,
+      "rewards/chosen": -0.5604009032249451,
+      "rewards/margins": 4.31578254699707,
+      "rewards/rejected": -4.876183032989502,
+      "step": 2650
+    },
+    {
+      "epoch": 1.8084393837910246,
+      "grad_norm": 54.821876525878906,
+      "learning_rate": 8.174898880204195e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -178.19236755371094,
+      "logps/rejected": -269.1416015625,
+      "loss": 0.2817,
+      "rewards/accuracies": 0.7400000095367432,
+      "rewards/chosen": -0.5425779223442078,
+      "rewards/margins": 3.9950203895568848,
+      "rewards/rejected": -4.537598133087158,
+      "step": 2700
+    },
+    {
+      "epoch": 1.841929002009377,
+      "grad_norm": 36.13364791870117,
+      "learning_rate": 7.793247358139428e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -179.92677307128906,
+      "logps/rejected": -266.75799560546875,
+      "loss": 0.2885,
+      "rewards/accuracies": 0.7387499809265137,
+      "rewards/chosen": -0.5648588538169861,
+      "rewards/margins": 3.864666700363159,
+      "rewards/rejected": -4.429525852203369,
+      "step": 2750
+    },
+    {
+      "epoch": 1.8754186202277294,
+      "grad_norm": 24.641510009765625,
+      "learning_rate": 7.414946361022179e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -171.00909423828125,
+      "logps/rejected": -273.5279541015625,
+      "loss": 0.2695,
+      "rewards/accuracies": 0.7400000095367432,
+      "rewards/chosen": -0.4850202798843384,
+      "rewards/margins": 4.063894271850586,
+      "rewards/rejected": -4.548914432525635,
+      "step": 2800
+    },
+    {
+      "epoch": 1.9089082384460818,
+      "grad_norm": 25.44546127319336,
+      "learning_rate": 7.040570265384029e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -186.17147827148438,
+      "logps/rejected": -272.64111328125,
+      "loss": 0.2881,
+      "rewards/accuracies": 0.7512500286102295,
+      "rewards/chosen": -0.5362930297851562,
+      "rewards/margins": 4.026025295257568,
+      "rewards/rejected": -4.562318325042725,
+      "step": 2850
+    },
+    {
+      "epoch": 1.942397856664434,
+      "grad_norm": 62.34092330932617,
+      "learning_rate": 6.670687488556586e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -188.8939208984375,
+      "logps/rejected": -270.8504943847656,
+      "loss": 0.2685,
+      "rewards/accuracies": 0.7337499856948853,
+      "rewards/chosen": -0.3625078499317169,
+      "rewards/margins": 4.072076797485352,
+      "rewards/rejected": -4.434584617614746,
+      "step": 2900
+    },
+    {
+      "epoch": 1.9758874748827864,
+      "grad_norm": 16.188819885253906,
+      "learning_rate": 6.305859625640224e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -177.49630737304688,
+      "logps/rejected": -280.4139404296875,
+      "loss": 0.2755,
+      "rewards/accuracies": 0.7475000023841858,
+      "rewards/chosen": -0.6155076026916504,
+      "rewards/margins": 4.242664337158203,
+      "rewards/rejected": -4.8581719398498535,
+      "step": 2950
+    },
+    {
+      "epoch": 2.0093770931011385,
+      "grad_norm": 35.435707092285156,
+      "learning_rate": 5.946640596831101e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -166.32289123535156,
+      "logps/rejected": -263.216552734375,
+      "loss": 0.2391,
+      "rewards/accuracies": 0.7712500095367432,
+      "rewards/chosen": -0.6572730541229248,
+      "rewards/margins": 4.339555740356445,
+      "rewards/rejected": -4.996828556060791,
+      "step": 3000
+    },
+    {
+      "epoch": 2.042866711319491,
+      "grad_norm": 42.23343276977539,
+      "learning_rate": 5.59357580640101e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -179.9312744140625,
+      "logps/rejected": -277.5908508300781,
+      "loss": 0.213,
+      "rewards/accuracies": 0.7850000262260437,
+      "rewards/chosen": -0.35315731167793274,
+      "rewards/margins": 4.545411586761475,
+      "rewards/rejected": -4.898569107055664,
+      "step": 3050
+    },
+    {
+      "epoch": 2.0763563295378433,
+      "grad_norm": 2.853132486343384,
+      "learning_rate": 5.247201314606984e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -180.59486389160156,
+      "logps/rejected": -276.373291015625,
+      "loss": 0.2047,
+      "rewards/accuracies": 0.7950000166893005,
+      "rewards/chosen": -0.3648325800895691,
+      "rewards/margins": 4.745596885681152,
+      "rewards/rejected": -5.110429763793945,
+      "step": 3100
+    },
+    {
+      "epoch": 2.1098459477561957,
+      "grad_norm": 22.07088851928711,
+      "learning_rate": 4.90804302378802e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -177.32708740234375,
+      "logps/rejected": -260.5697021484375,
+      "loss": 0.2054,
+      "rewards/accuracies": 0.7925000190734863,
+      "rewards/chosen": -0.48022788763046265,
+      "rewards/margins": 4.517958641052246,
+      "rewards/rejected": -4.998186111450195,
+      "step": 3150
+    },
+    {
+      "epoch": 2.1433355659745477,
+      "grad_norm": 50.728519439697266,
+      "learning_rate": 4.57661587988459e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -177.0932159423828,
+      "logps/rejected": -270.6129150390625,
+      "loss": 0.236,
+      "rewards/accuracies": 0.7574999928474426,
+      "rewards/chosen": -0.4882276654243469,
+      "rewards/margins": 4.606672286987305,
+      "rewards/rejected": -5.094900131225586,
+      "step": 3200
+    },
+    {
+      "epoch": 2.1768251841929,
+      "grad_norm": 19.410276412963867,
+      "learning_rate": 4.253423090593318e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -185.2410125732422,
+      "logps/rejected": -282.7039794921875,
+      "loss": 0.2242,
+      "rewards/accuracies": 0.7612500190734863,
+      "rewards/chosen": -0.5257070064544678,
+      "rewards/margins": 4.692570209503174,
+      "rewards/rejected": -5.218277454376221,
+      "step": 3250
+    },
+    {
+      "epoch": 2.2103148024112524,
+      "grad_norm": 45.68756103515625,
+      "learning_rate": 3.938955361343912e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -175.8925018310547,
+      "logps/rejected": -284.1990966796875,
+      "loss": 0.2259,
+      "rewards/accuracies": 0.7699999809265137,
+      "rewards/chosen": -0.605311930179596,
+      "rewards/margins": 4.8395843505859375,
+      "rewards/rejected": -5.444896221160889,
+      "step": 3300
+    },
+    {
+      "epoch": 2.243804420629605,
+      "grad_norm": 51.53227996826172,
+      "learning_rate": 3.6336901502583364e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -177.85601806640625,
+      "logps/rejected": -275.8158874511719,
+      "loss": 0.2048,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.6794506907463074,
+      "rewards/margins": 4.734764575958252,
+      "rewards/rejected": -5.414215087890625,
+      "step": 3350
+    },
+    {
+      "epoch": 2.2772940388479572,
+      "grad_norm": 3.569408893585205,
+      "learning_rate": 3.3380909432234807e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -182.00836181640625,
+      "logps/rejected": -280.286376953125,
+      "loss": 0.1999,
+      "rewards/accuracies": 0.7950000166893005,
+      "rewards/chosen": -0.6098263263702393,
+      "rewards/margins": 4.961060047149658,
+      "rewards/rejected": -5.570886611938477,
+      "step": 3400
+    },
+    {
+      "epoch": 2.3107836570663096,
+      "grad_norm": 27.362163543701172,
+      "learning_rate": 3.0526065501779184e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -172.97593688964844,
+      "logps/rejected": -275.5477600097656,
+      "loss": 0.2184,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.6930285096168518,
+      "rewards/margins": 4.821885585784912,
+      "rewards/rejected": -5.514913558959961,
+      "step": 3450
+    },
+    {
+      "epoch": 2.3442732752846616,
+      "grad_norm": 28.243000030517578,
+      "learning_rate": 2.7776704236812454e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -182.44705200195312,
+      "logps/rejected": -277.888427734375,
+      "loss": 0.2128,
+      "rewards/accuracies": 0.7649999856948853,
+      "rewards/chosen": -0.6010170578956604,
+      "rewards/margins": 5.026294708251953,
+      "rewards/rejected": -5.6273112297058105,
+      "step": 3500
+    },
+    {
+      "epoch": 2.377762893503014,
+      "grad_norm": 14.03532886505127,
+      "learning_rate": 2.5137000008006437e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -182.77134704589844,
+      "logps/rejected": -279.57769775390625,
+      "loss": 0.21,
+      "rewards/accuracies": 0.7799999713897705,
+      "rewards/chosen": -0.7788973450660706,
+      "rewards/margins": 5.022655010223389,
+      "rewards/rejected": -5.801552772521973,
+      "step": 3550
+    },
+    {
+      "epoch": 2.4112525117213663,
+      "grad_norm": 35.019554138183594,
+      "learning_rate": 2.261096069313816e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -187.45738220214844,
+      "logps/rejected": -281.279541015625,
+      "loss": 0.1887,
+      "rewards/accuracies": 0.8075000047683716,
+      "rewards/chosen": -0.7265406847000122,
+      "rewards/margins": 5.097284317016602,
+      "rewards/rejected": -5.823824882507324,
+      "step": 3600
+    },
+    {
+      "epoch": 2.4447421299397187,
+      "grad_norm": 25.041046142578125,
+      "learning_rate": 2.020242159190646e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -176.86915588378906,
+      "logps/rejected": -277.746826171875,
+      "loss": 0.2311,
+      "rewards/accuracies": 0.7587500214576721,
+      "rewards/chosen": -0.786669135093689,
+      "rewards/margins": 4.789151191711426,
+      "rewards/rejected": -5.575820446014404,
+      "step": 3650
+    },
+    {
+      "epoch": 2.478231748158071,
+      "grad_norm": 20.99360466003418,
+      "learning_rate": 1.7915039602775062e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -182.3199462890625,
+      "logps/rejected": -273.0755920410156,
+      "loss": 0.2429,
+      "rewards/accuracies": 0.7737500071525574,
+      "rewards/chosen": -0.8147923946380615,
+      "rewards/margins": 4.847590446472168,
+      "rewards/rejected": -5.66238260269165,
+      "step": 3700
+    },
+    {
+      "epoch": 2.511721366376423,
+      "grad_norm": 18.44826889038086,
+      "learning_rate": 1.5752287670682861e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -170.71795654296875,
+      "logps/rejected": -276.1592102050781,
+      "loss": 0.2043,
+      "rewards/accuracies": 0.7862499952316284,
+      "rewards/chosen": -0.638399064540863,
+      "rewards/margins": 5.212125301361084,
+      "rewards/rejected": -5.850524425506592,
+      "step": 3750
+    },
+    {
+      "epoch": 2.5452109845947755,
+      "grad_norm": 40.779659271240234,
+      "learning_rate": 1.3717449514052314e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -180.7264404296875,
+      "logps/rejected": -284.6885986328125,
+      "loss": 0.2033,
+      "rewards/accuracies": 0.7962499856948853,
+      "rewards/chosen": -0.882935106754303,
+      "rewards/margins": 5.128498554229736,
+      "rewards/rejected": -6.0114336013793945,
+      "step": 3800
+    },
+    {
+      "epoch": 2.578700602813128,
+      "grad_norm": 44.556678771972656,
+      "learning_rate": 1.1813614639101088e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -183.99533081054688,
+      "logps/rejected": -275.25518798828125,
+      "loss": 0.2274,
+      "rewards/accuracies": 0.7774999737739563,
+      "rewards/chosen": -0.703125,
+      "rewards/margins": 5.014428615570068,
+      "rewards/rejected": -5.717553615570068,
+      "step": 3850
+    },
+    {
+      "epoch": 2.6121902210314802,
+      "grad_norm": 61.39085388183594,
+      "learning_rate": 1.0043673649027517e-07,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -178.3540802001953,
+      "logps/rejected": -282.1649475097656,
+      "loss": 0.2097,
+      "rewards/accuracies": 0.7662500143051147,
+      "rewards/chosen": -0.683403730392456,
+      "rewards/margins": 5.063638687133789,
+      "rewards/rejected": -5.747043609619141,
+      "step": 3900
+    },
+    {
+      "epoch": 2.6456798392498326,
+      "grad_norm": 58.0173454284668,
+      "learning_rate": 8.410313855191464e-08,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -178.94400024414062,
+      "logps/rejected": -286.5594177246094,
+      "loss": 0.2042,
+      "rewards/accuracies": 0.7862499952316284,
+      "rewards/chosen": -0.8088821172714233,
+      "rewards/margins": 5.067000865936279,
+      "rewards/rejected": -5.875882625579834,
+      "step": 3950
+    },
+    {
+      "epoch": 2.679169457468185,
+      "grad_norm": 16.31562042236328,
+      "learning_rate": 6.916015196954383e-08,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -185.46673583984375,
+      "logps/rejected": -288.2527770996094,
+      "loss": 0.217,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.7252050638198853,
+      "rewards/margins": 5.204960823059082,
+      "rewards/rejected": -5.930166244506836,
+      "step": 4000
+    },
+    {
+      "epoch": 2.7126590756865374,
+      "grad_norm": 20.799222946166992,
+      "learning_rate": 5.5630464763733787e-08,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -188.50820922851562,
+      "logps/rejected": -288.9837646484375,
+      "loss": 0.2258,
+      "rewards/accuracies": 0.7724999785423279,
+      "rewards/chosen": -0.7981621026992798,
+      "rewards/margins": 5.062735557556152,
+      "rewards/rejected": -5.860898017883301,
+      "step": 4050
+    },
+    {
+      "epoch": 2.7461486939048894,
+      "grad_norm": 18.682947158813477,
+      "learning_rate": 4.353461913466405e-08,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -178.44317626953125,
+      "logps/rejected": -266.35333251953125,
+      "loss": 0.2426,
+      "rewards/accuracies": 0.7524999976158142,
+      "rewards/chosen": -0.6803594827651978,
+      "rewards/margins": 4.8590497970581055,
+      "rewards/rejected": -5.539409160614014,
+      "step": 4100
+    },
+    {
+      "epoch": 2.7796383121232418,
+      "grad_norm": 54.06953048706055,
+      "learning_rate": 3.2890980272783255e-08,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -180.65658569335938,
+      "logps/rejected": -280.3162536621094,
+      "loss": 0.2086,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.812857449054718,
+      "rewards/margins": 5.271449565887451,
+      "rewards/rejected": -6.0843071937561035,
+      "step": 4150
+    },
+    {
+      "epoch": 2.813127930341594,
+      "grad_norm": 12.436116218566895,
+      "learning_rate": 2.371570847483839e-08,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -180.7625732421875,
+      "logps/rejected": -277.9272766113281,
+      "loss": 0.2046,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.6954517364501953,
+      "rewards/margins": 5.145771026611328,
+      "rewards/rejected": -5.841222763061523,
+      "step": 4200
+    },
+    {
+      "epoch": 2.8466175485599465,
+      "grad_norm": 66.9225845336914,
+      "learning_rate": 1.6022734607604393e-08,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -187.79019165039062,
+      "logps/rejected": -282.13323974609375,
+      "loss": 0.2096,
+      "rewards/accuracies": 0.7925000190734863,
+      "rewards/chosen": -0.8357629179954529,
+      "rewards/margins": 5.103863716125488,
+      "rewards/rejected": -5.939626693725586,
+      "step": 4250
+    },
+    {
+      "epoch": 2.8801071667782985,
+      "grad_norm": 15.983145713806152,
+      "learning_rate": 9.823738956571182e-09,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -191.03807067871094,
+      "logps/rejected": -292.3773193359375,
+      "loss": 0.206,
+      "rewards/accuracies": 0.7912499904632568,
+      "rewards/chosen": -0.6932557821273804,
+      "rewards/margins": 5.146268367767334,
+      "rewards/rejected": -5.839523792266846,
+      "step": 4300
+    },
+    {
+      "epoch": 2.913596784996651,
+      "grad_norm": 33.383487701416016,
+      "learning_rate": 5.128133491700715e-09,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -186.7404327392578,
+      "logps/rejected": -289.3056945800781,
+      "loss": 0.1936,
+      "rewards/accuracies": 0.7975000143051147,
+      "rewards/chosen": -0.7487243413925171,
+      "rewards/margins": 5.300227642059326,
+      "rewards/rejected": -6.048952579498291,
+      "step": 4350
+    },
+    {
+      "epoch": 2.9470864032150033,
+      "grad_norm": 3.542743682861328,
+      "learning_rate": 1.9430475771796684e-09,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -190.31752014160156,
+      "logps/rejected": -268.015380859375,
+      "loss": 0.2124,
+      "rewards/accuracies": 0.7862499952316284,
+      "rewards/chosen": -0.6255255937576294,
+      "rewards/margins": 4.9648332595825195,
+      "rewards/rejected": -5.590358257293701,
+      "step": 4400
+    },
+    {
+      "epoch": 2.9805760214333556,
+      "grad_norm": 19.205642700195312,
+      "learning_rate": 2.733171468656259e-10,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -177.03684997558594,
+      "logps/rejected": -277.01495361328125,
+      "loss": 0.2059,
+      "rewards/accuracies": 0.7837499976158142,
+      "rewards/chosen": -0.7730162739753723,
+      "rewards/margins": 5.190572738647461,
+      "rewards/rejected": -5.963588714599609,
+      "step": 4450
     }
   ],
   "logging_steps": 50,
-  "max_steps": 1493,
+  "max_steps": 4479,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 3,
   "save_steps": 200,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:83253cf573c71383f710bb1441ffc24338aa5407acd305912e4964de7e52bda0
+oid sha256:1b84867b7d993885f385b0892996ffe7c2611dc21555a94b8af64b6116d9bbd7
 size 6545