{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.23277467411545624,
  "eval_steps": 500,
  "global_step": 125,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.00186219739292365,
      "grad_norm": 40.418399810791016,
      "learning_rate": 9.259259259259259e-08,
      "logps/chosen": -143.09214782714844,
      "logps/rejected": -159.23831176757812,
      "loss": 0.6931,
      "losses/dpo": 0.6931471824645996,
      "losses/sft": 1.9754583835601807,
      "losses/total": 0.6931471824645996,
      "ref_logps/chosen": -143.09214782714844,
      "ref_logps/rejected": -159.23831176757812,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0037243947858473,
      "grad_norm": 40.48415756225586,
      "learning_rate": 1.8518518518518518e-07,
      "logps/chosen": -139.04046630859375,
      "logps/rejected": -167.1217041015625,
      "loss": 0.6931,
      "losses/dpo": 0.6931471824645996,
      "losses/sft": 2.354458808898926,
      "losses/total": 0.6931471824645996,
      "ref_logps/chosen": -139.04046630859375,
      "ref_logps/rejected": -167.1217041015625,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2
    },
    {
      "epoch": 0.00558659217877095,
      "grad_norm": 43.48381042480469,
      "learning_rate": 2.7777777777777776e-07,
      "logps/chosen": -145.57992553710938,
      "logps/rejected": -169.97146606445312,
      "loss": 0.6981,
      "losses/dpo": 0.6891708374023438,
      "losses/sft": 2.1739914417266846,
      "losses/total": 0.6891708374023438,
      "ref_logps/chosen": -145.53663635253906,
      "ref_logps/rejected": -170.02041625976562,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": -0.00432817917317152,
      "rewards/margins": -0.009223854169249535,
      "rewards/rejected": 0.004895674996078014,
      "step": 3
    },
    {
      "epoch": 0.0074487895716946,
      "grad_norm": 40.21121597290039,
      "learning_rate": 3.7037037037037036e-07,
      "logps/chosen": -145.57131958007812,
      "logps/rejected": -162.80123901367188,
      "loss": 0.6915,
      "losses/dpo": 0.6871472597122192,
      "losses/sft": 2.114041805267334,
      "losses/total": 0.6871472597122192,
      "ref_logps/chosen": -145.52674865722656,
      "ref_logps/rejected": -162.71473693847656,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -0.004457235801964998,
      "rewards/margins": 0.004192234016954899,
      "rewards/rejected": -0.00864946935325861,
      "step": 4
    },
    {
      "epoch": 0.00931098696461825,
      "grad_norm": 43.5928955078125,
      "learning_rate": 4.6296296296296297e-07,
      "logps/chosen": -149.61972045898438,
      "logps/rejected": -171.2332763671875,
      "loss": 0.6897,
      "losses/dpo": 0.6956955194473267,
      "losses/sft": 1.9878240823745728,
      "losses/total": 0.6956955194473267,
      "ref_logps/chosen": -149.57879638671875,
      "ref_logps/rejected": -171.1157684326172,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.004093348979949951,
      "rewards/margins": 0.007657159119844437,
      "rewards/rejected": -0.011750508099794388,
      "step": 5
    },
    {
      "epoch": 0.0111731843575419,
      "grad_norm": 43.37643051147461,
      "learning_rate": 5.555555555555555e-07,
      "logps/chosen": -144.1446075439453,
      "logps/rejected": -165.30984497070312,
      "loss": 0.6839,
      "losses/dpo": 0.7011494636535645,
      "losses/sft": 2.2325987815856934,
      "losses/total": 0.7011494636535645,
      "ref_logps/chosen": -144.15261840820312,
      "ref_logps/rejected": -165.12521362304688,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": 0.0007997989887371659,
      "rewards/margins": 0.019263559952378273,
      "rewards/rejected": -0.01846376061439514,
      "step": 6
    },
    {
      "epoch": 0.01303538175046555,
      "grad_norm": 41.759315490722656,
      "learning_rate": 6.481481481481481e-07,
      "logps/chosen": -140.94808959960938,
      "logps/rejected": -161.61846923828125,
      "loss": 0.6677,
      "losses/dpo": 0.6434986591339111,
      "losses/sft": 2.4416794776916504,
      "losses/total": 0.6434986591339111,
      "ref_logps/chosen": -140.9979248046875,
      "ref_logps/rejected": -161.13681030273438,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": 0.004983758553862572,
      "rewards/margins": 0.05314992740750313,
      "rewards/rejected": -0.048166170716285706,
      "step": 7
    },
    {
      "epoch": 0.0148975791433892,
      "grad_norm": 40.88422393798828,
      "learning_rate": 7.407407407407407e-07,
      "logps/chosen": -150.63833618164062,
      "logps/rejected": -174.64923095703125,
      "loss": 0.6588,
      "losses/dpo": 0.639857292175293,
      "losses/sft": 2.3609848022460938,
      "losses/total": 0.639857292175293,
      "ref_logps/chosen": -150.88568115234375,
      "ref_logps/rejected": -174.18106079101562,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.02473468706011772,
      "rewards/margins": 0.0715511366724968,
      "rewards/rejected": -0.046816445887088776,
      "step": 8
    },
    {
      "epoch": 0.01675977653631285,
      "grad_norm": 36.576690673828125,
      "learning_rate": 8.333333333333333e-07,
      "logps/chosen": -146.96463012695312,
      "logps/rejected": -174.3262939453125,
      "loss": 0.6259,
      "losses/dpo": 0.6055787801742554,
      "losses/sft": 2.1100568771362305,
      "losses/total": 0.6055787801742554,
      "ref_logps/chosen": -147.0458221435547,
      "ref_logps/rejected": -172.9729461669922,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.008118623867630959,
      "rewards/margins": 0.14345335960388184,
      "rewards/rejected": -0.13533474504947662,
      "step": 9
    },
    {
      "epoch": 0.0186219739292365,
      "grad_norm": 41.190025329589844,
      "learning_rate": 9.259259259259259e-07,
      "logps/chosen": -154.90782165527344,
      "logps/rejected": -160.24624633789062,
      "loss": 0.5904,
      "losses/dpo": 0.5785373449325562,
      "losses/sft": 2.5960636138916016,
      "losses/total": 0.5785373449325562,
      "ref_logps/chosen": -155.20986938476562,
      "ref_logps/rejected": -158.2913055419922,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 0.030205056071281433,
      "rewards/margins": 0.2256987988948822,
      "rewards/rejected": -0.19549374282360077,
      "step": 10
    },
    {
      "epoch": 0.020484171322160148,
      "grad_norm": 32.080810546875,
      "learning_rate": 1.0185185185185185e-06,
      "logps/chosen": -138.2403106689453,
      "logps/rejected": -162.02291870117188,
      "loss": 0.5437,
      "losses/dpo": 0.5474151968955994,
      "losses/sft": 1.9134595394134521,
      "losses/total": 0.5474151968955994,
      "ref_logps/chosen": -138.54551696777344,
      "ref_logps/rejected": -158.88693237304688,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.030520910397171974,
      "rewards/margins": 0.34412044286727905,
      "rewards/rejected": -0.31359952688217163,
      "step": 11
    },
    {
      "epoch": 0.0223463687150838,
      "grad_norm": 29.643903732299805,
      "learning_rate": 1.111111111111111e-06,
      "logps/chosen": -154.15118408203125,
      "logps/rejected": -180.0041961669922,
      "loss": 0.4393,
      "losses/dpo": 0.4857974946498871,
      "losses/sft": 2.032329797744751,
      "losses/total": 0.4857974946498871,
      "ref_logps/chosen": -154.87046813964844,
      "ref_logps/rejected": -174.29629516601562,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.0719279944896698,
      "rewards/margins": 0.6427179574966431,
      "rewards/rejected": -0.5707899332046509,
      "step": 12
    },
    {
      "epoch": 0.024208566108007448,
      "grad_norm": 27.43463134765625,
      "learning_rate": 1.2037037037037037e-06,
      "logps/chosen": -146.56866455078125,
      "logps/rejected": -173.71633911132812,
      "loss": 0.4487,
      "losses/dpo": 0.4681214988231659,
      "losses/sft": 2.5795841217041016,
      "losses/total": 0.4681214988231659,
      "ref_logps/chosen": -146.011962890625,
      "ref_logps/rejected": -166.83184814453125,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.055669985711574554,
      "rewards/margins": 0.6327812075614929,
      "rewards/rejected": -0.6884511709213257,
      "step": 13
    },
    {
      "epoch": 0.0260707635009311,
      "grad_norm": 24.864574432373047,
      "learning_rate": 1.2962962962962962e-06,
      "logps/chosen": -151.02015686035156,
      "logps/rejected": -181.14610290527344,
      "loss": 0.3711,
      "losses/dpo": 0.4647141695022583,
      "losses/sft": 2.417238473892212,
      "losses/total": 0.4647141695022583,
      "ref_logps/chosen": -151.75128173828125,
      "ref_logps/rejected": -172.87350463867188,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.07311299443244934,
      "rewards/margins": 0.9003715515136719,
      "rewards/rejected": -0.8272585272789001,
      "step": 14
    },
    {
      "epoch": 0.027932960893854747,
      "grad_norm": 22.773723602294922,
      "learning_rate": 1.3888888888888892e-06,
      "logps/chosen": -146.68460083007812,
      "logps/rejected": -180.87896728515625,
      "loss": 0.3584,
      "losses/dpo": 0.33828622102737427,
      "losses/sft": 2.191209077835083,
      "losses/total": 0.33828622102737427,
      "ref_logps/chosen": -145.39768981933594,
      "ref_logps/rejected": -170.00677490234375,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.12869027256965637,
      "rewards/margins": 0.9585269689559937,
      "rewards/rejected": -1.0872172117233276,
      "step": 15
    },
    {
      "epoch": 0.0297951582867784,
      "grad_norm": 17.319433212280273,
      "learning_rate": 1.4814814814814815e-06,
      "logps/chosen": -157.9118194580078,
      "logps/rejected": -180.584228515625,
      "loss": 0.253,
      "losses/dpo": 0.30005261301994324,
      "losses/sft": 2.218407154083252,
      "losses/total": 0.30005261301994324,
      "ref_logps/chosen": -155.03277587890625,
      "ref_logps/rejected": -161.18026733398438,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.2879039943218231,
      "rewards/margins": 1.652491807937622,
      "rewards/rejected": -1.940395712852478,
      "step": 16
    },
    {
      "epoch": 0.03165735567970205,
      "grad_norm": 14.82257080078125,
      "learning_rate": 1.5740740740740742e-06,
      "logps/chosen": -151.82461547851562,
      "logps/rejected": -182.52401733398438,
      "loss": 0.2197,
      "losses/dpo": 0.2500099837779999,
      "losses/sft": 2.3733954429626465,
      "losses/total": 0.2500099837779999,
      "ref_logps/chosen": -147.43585205078125,
      "ref_logps/rejected": -158.81658935546875,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": -0.438876748085022,
      "rewards/margins": 1.9318653345108032,
      "rewards/rejected": -2.3707423210144043,
      "step": 17
    },
    {
      "epoch": 0.0335195530726257,
      "grad_norm": 15.098052978515625,
      "learning_rate": 1.6666666666666667e-06,
      "logps/chosen": -146.62161254882812,
      "logps/rejected": -192.061767578125,
      "loss": 0.2086,
      "losses/dpo": 0.087049201130867,
      "losses/sft": 2.235438585281372,
      "losses/total": 0.087049201130867,
      "ref_logps/chosen": -142.35076904296875,
      "ref_logps/rejected": -164.73597717285156,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": -0.42708471417427063,
      "rewards/margins": 2.305494785308838,
      "rewards/rejected": -2.732579469680786,
      "step": 18
    },
    {
      "epoch": 0.035381750465549346,
      "grad_norm": 14.139907836914062,
      "learning_rate": 1.7592592592592594e-06,
      "logps/chosen": -151.61148071289062,
      "logps/rejected": -204.13427734375,
      "loss": 0.1831,
      "losses/dpo": 0.2179492712020874,
      "losses/sft": 2.547309637069702,
      "losses/total": 0.2179492712020874,
      "ref_logps/chosen": -145.25982666015625,
      "ref_logps/rejected": -172.7918701171875,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.6351636648178101,
      "rewards/margins": 2.4990787506103516,
      "rewards/rejected": -3.134242534637451,
      "step": 19
    },
    {
      "epoch": 0.037243947858473,
      "grad_norm": 15.214645385742188,
      "learning_rate": 1.8518518518518519e-06,
      "logps/chosen": -149.88876342773438,
      "logps/rejected": -218.4535675048828,
      "loss": 0.181,
      "losses/dpo": 0.18753334879875183,
      "losses/sft": 2.394906759262085,
      "losses/total": 0.18753334879875183,
      "ref_logps/chosen": -142.5288543701172,
      "ref_logps/rejected": -185.18878173828125,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7359906435012817,
      "rewards/margins": 2.590489387512207,
      "rewards/rejected": -3.3264803886413574,
      "step": 20
    },
    {
      "epoch": 0.03910614525139665,
      "grad_norm": 15.793328285217285,
      "learning_rate": 1.944444444444445e-06,
      "logps/chosen": -155.686279296875,
      "logps/rejected": -189.35899353027344,
      "loss": 0.1946,
      "losses/dpo": 0.2072586715221405,
      "losses/sft": 2.1763200759887695,
      "losses/total": 0.2072586715221405,
      "ref_logps/chosen": -148.760986328125,
      "ref_logps/rejected": -153.56321716308594,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.6925297379493713,
      "rewards/margins": 2.88704776763916,
      "rewards/rejected": -3.5795774459838867,
      "step": 21
    },
    {
      "epoch": 0.040968342644320296,
      "grad_norm": 15.932718276977539,
      "learning_rate": 2.037037037037037e-06,
      "logps/chosen": -147.47117614746094,
      "logps/rejected": -206.65597534179688,
      "loss": 0.1655,
      "losses/dpo": 0.10909190773963928,
      "losses/sft": 2.1695094108581543,
      "losses/total": 0.10909190773963928,
      "ref_logps/chosen": -137.73451232910156,
      "ref_logps/rejected": -160.2368927001953,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9736669063568115,
      "rewards/margins": 3.6682400703430176,
      "rewards/rejected": -4.641907691955566,
      "step": 22
    },
    {
      "epoch": 0.04283054003724395,
      "grad_norm": 11.131953239440918,
      "learning_rate": 2.1296296296296298e-06,
      "logps/chosen": -172.5790252685547,
      "logps/rejected": -234.46731567382812,
      "loss": 0.0974,
      "losses/dpo": 0.25966331362724304,
      "losses/sft": 2.784914255142212,
      "losses/total": 0.25966331362724304,
      "ref_logps/chosen": -154.8829345703125,
      "ref_logps/rejected": -172.43850708007812,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.7696071863174438,
      "rewards/margins": 4.433272361755371,
      "rewards/rejected": -6.202879905700684,
      "step": 23
    },
    {
      "epoch": 0.0446927374301676,
      "grad_norm": 9.715975761413574,
      "learning_rate": 2.222222222222222e-06,
      "logps/chosen": -161.45799255371094,
      "logps/rejected": -231.06072998046875,
      "loss": 0.0859,
      "losses/dpo": 0.04074842855334282,
      "losses/sft": 2.738722324371338,
      "losses/total": 0.04074842855334282,
      "ref_logps/chosen": -142.22097778320312,
      "ref_logps/rejected": -161.80909729003906,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.9237014055252075,
      "rewards/margins": 5.001461029052734,
      "rewards/rejected": -6.9251627922058105,
      "step": 24
    },
    {
      "epoch": 0.04655493482309125,
      "grad_norm": 14.314465522766113,
      "learning_rate": 2.314814814814815e-06,
      "logps/chosen": -170.95504760742188,
      "logps/rejected": -257.62115478515625,
      "loss": 0.124,
      "losses/dpo": 0.004803886171430349,
      "losses/sft": 2.606781244277954,
      "losses/total": 0.004803886171430349,
      "ref_logps/chosen": -146.23434448242188,
      "ref_logps/rejected": -173.50958251953125,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": -2.472069263458252,
      "rewards/margins": 5.939087867736816,
      "rewards/rejected": -8.411157608032227,
      "step": 25
    },
    {
      "epoch": 0.048417132216014895,
      "grad_norm": 16.382986068725586,
      "learning_rate": 2.4074074074074075e-06,
      "logps/chosen": -171.77096557617188,
      "logps/rejected": -249.99264526367188,
      "loss": 0.1927,
      "losses/dpo": 0.023840777575969696,
      "losses/sft": 2.574446201324463,
      "losses/total": 0.023840777575969696,
      "ref_logps/chosen": -146.96835327148438,
      "ref_logps/rejected": -168.13232421875,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.4802613258361816,
      "rewards/margins": 5.705770969390869,
      "rewards/rejected": -8.18603229522705,
      "step": 26
    },
    {
      "epoch": 0.05027932960893855,
      "grad_norm": 9.626073837280273,
      "learning_rate": 2.5e-06,
      "logps/chosen": -169.91448974609375,
      "logps/rejected": -251.7610626220703,
      "loss": 0.058,
      "losses/dpo": 0.0014569438062608242,
      "losses/sft": 2.5121726989746094,
      "losses/total": 0.0014569438062608242,
      "ref_logps/chosen": -141.93600463867188,
      "ref_logps/rejected": -160.8459014892578,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -2.7978479862213135,
      "rewards/margins": 6.293667793273926,
      "rewards/rejected": -9.091516494750977,
      "step": 27
    },
    {
      "epoch": 0.0521415270018622,
      "grad_norm": 9.746648788452148,
      "learning_rate": 2.5925925925925925e-06,
      "logps/chosen": -173.24583435058594,
      "logps/rejected": -267.49298095703125,
      "loss": 0.0456,
      "losses/dpo": 0.010660952888429165,
      "losses/sft": 2.7988812923431396,
      "losses/total": 0.010660952888429165,
      "ref_logps/chosen": -147.74620056152344,
      "ref_logps/rejected": -170.100830078125,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -2.5499653816223145,
      "rewards/margins": 7.189251899719238,
      "rewards/rejected": -9.739216804504395,
      "step": 28
    },
    {
      "epoch": 0.054003724394785846,
      "grad_norm": 19.729585647583008,
      "learning_rate": 2.6851851851851856e-06,
      "logps/chosen": -173.5810546875,
      "logps/rejected": -260.90985107421875,
      "loss": 0.1598,
      "losses/dpo": 0.11473169922828674,
      "losses/sft": 2.465557098388672,
      "losses/total": 0.11473169922828674,
      "ref_logps/chosen": -144.18157958984375,
      "ref_logps/rejected": -164.2210693359375,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": -2.939948081970215,
      "rewards/margins": 6.72892951965332,
      "rewards/rejected": -9.668878555297852,
      "step": 29
    },
    {
      "epoch": 0.055865921787709494,
      "grad_norm": 15.30510425567627,
      "learning_rate": 2.7777777777777783e-06,
      "logps/chosen": -170.92425537109375,
      "logps/rejected": -277.4226379394531,
      "loss": 0.1033,
      "losses/dpo": 0.13019727170467377,
      "losses/sft": 2.4969606399536133,
      "losses/total": 0.13019727170467377,
      "ref_logps/chosen": -142.30502319335938,
      "ref_logps/rejected": -169.90554809570312,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.8619208335876465,
      "rewards/margins": 7.889788627624512,
      "rewards/rejected": -10.751708984375,
      "step": 30
    },
    {
      "epoch": 0.05772811918063315,
      "grad_norm": 6.729362487792969,
      "learning_rate": 2.8703703703703706e-06,
      "logps/chosen": -177.1944122314453,
      "logps/rejected": -277.68206787109375,
      "loss": 0.0363,
      "losses/dpo": 0.03610475733876228,
      "losses/sft": 2.655313491821289,
      "losses/total": 0.03610475733876228,
      "ref_logps/chosen": -140.73678588867188,
      "ref_logps/rejected": -164.96824645996094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.6457643508911133,
      "rewards/margins": 7.625619888305664,
      "rewards/rejected": -11.271383285522461,
      "step": 31
    },
    {
      "epoch": 0.0595903165735568,
      "grad_norm": 15.916095733642578,
      "learning_rate": 2.962962962962963e-06,
      "logps/chosen": -168.07888793945312,
      "logps/rejected": -266.97467041015625,
      "loss": 0.1384,
      "losses/dpo": 0.003958229906857014,
      "losses/sft": 3.21946120262146,
      "losses/total": 0.003958229906857014,
      "ref_logps/chosen": -135.16700744628906,
      "ref_logps/rejected": -166.81027221679688,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.2911877632141113,
      "rewards/margins": 6.725253582000732,
      "rewards/rejected": -10.016440391540527,
      "step": 32
    },
    {
      "epoch": 0.061452513966480445,
      "grad_norm": 13.613255500793457,
      "learning_rate": 3.055555555555556e-06,
      "logps/chosen": -163.11419677734375,
      "logps/rejected": -267.9139099121094,
      "loss": 0.1,
      "losses/dpo": 0.0983101949095726,
      "losses/sft": 2.5492136478424072,
      "losses/total": 0.0983101949095726,
      "ref_logps/chosen": -140.14205932617188,
      "ref_logps/rejected": -160.53872680664062,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.297213554382324,
      "rewards/margins": 8.44030475616455,
      "rewards/rejected": -10.737519264221191,
      "step": 33
    },
    {
      "epoch": 0.0633147113594041,
      "grad_norm": 15.18605899810791,
      "learning_rate": 3.1481481481481483e-06,
      "logps/chosen": -167.43917846679688,
      "logps/rejected": -285.1209411621094,
      "loss": 0.1385,
      "losses/dpo": 0.010567452758550644,
      "losses/sft": 2.848440170288086,
      "losses/total": 0.010567452758550644,
      "ref_logps/chosen": -137.24496459960938,
      "ref_logps/rejected": -172.29844665527344,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.019423723220825,
      "rewards/margins": 8.262825965881348,
      "rewards/rejected": -11.282249450683594,
      "step": 34
    },
    {
      "epoch": 0.06517690875232775,
      "grad_norm": 14.179464340209961,
      "learning_rate": 3.240740740740741e-06,
      "logps/chosen": -163.87579345703125,
      "logps/rejected": -273.77777099609375,
      "loss": 0.1047,
      "losses/dpo": 0.010419070720672607,
      "losses/sft": 2.3946168422698975,
      "losses/total": 0.010419070720672607,
      "ref_logps/chosen": -144.74697875976562,
      "ref_logps/rejected": -167.2154541015625,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.9128817319869995,
      "rewards/margins": 8.743349075317383,
      "rewards/rejected": -10.656229972839355,
      "step": 35
    },
    {
      "epoch": 0.0670391061452514,
      "grad_norm": 7.81901741027832,
      "learning_rate": 3.3333333333333333e-06,
      "logps/chosen": -170.99649047851562,
      "logps/rejected": -260.6686096191406,
      "loss": 0.0408,
      "losses/dpo": 0.03330932930111885,
      "losses/sft": 2.9487311840057373,
      "losses/total": 0.03330932930111885,
      "ref_logps/chosen": -150.3196563720703,
      "ref_logps/rejected": -156.96363830566406,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -2.0676848888397217,
      "rewards/margins": 8.302813529968262,
      "rewards/rejected": -10.370498657226562,
      "step": 36
    },
    {
      "epoch": 0.06890130353817504,
      "grad_norm": 14.016197204589844,
      "learning_rate": 3.4259259259259265e-06,
      "logps/chosen": -161.38865661621094,
      "logps/rejected": -281.09271240234375,
      "loss": 0.1047,
      "losses/dpo": 0.5629028677940369,
      "losses/sft": 2.4373741149902344,
      "losses/total": 0.5629028677940369,
      "ref_logps/chosen": -143.14517211914062,
      "ref_logps/rejected": -172.78611755371094,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": -1.8243476152420044,
      "rewards/margins": 9.006311416625977,
      "rewards/rejected": -10.830659866333008,
      "step": 37
    },
    {
      "epoch": 0.07076350093109869,
      "grad_norm": 15.18514347076416,
      "learning_rate": 3.5185185185185187e-06,
      "logps/chosen": -160.60586547851562,
      "logps/rejected": -263.134765625,
      "loss": 0.1468,
      "losses/dpo": 0.018482662737369537,
      "losses/sft": 2.2324092388153076,
      "losses/total": 0.018482662737369537,
      "ref_logps/chosen": -141.36175537109375,
      "ref_logps/rejected": -164.61480712890625,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": -1.9244120121002197,
      "rewards/margins": 7.92758321762085,
      "rewards/rejected": -9.851995468139648,
      "step": 38
    },
    {
      "epoch": 0.07262569832402235,
      "grad_norm": 8.755450248718262,
      "learning_rate": 3.6111111111111115e-06,
      "logps/chosen": -170.1685791015625,
      "logps/rejected": -288.41119384765625,
      "loss": 0.0421,
      "losses/dpo": 9.010350913740695e-05,
      "losses/sft": 2.3953020572662354,
      "losses/total": 9.010350913740695e-05,
      "ref_logps/chosen": -152.65139770507812,
      "ref_logps/rejected": -172.203857421875,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.751717448234558,
      "rewards/margins": 9.86901569366455,
      "rewards/rejected": -11.620733261108398,
      "step": 39
    },
    {
      "epoch": 0.074487895716946,
      "grad_norm": 2.3320696353912354,
      "learning_rate": 3.7037037037037037e-06,
      "logps/chosen": -164.16981506347656,
      "logps/rejected": -263.49163818359375,
      "loss": 0.0103,
      "losses/dpo": 0.013380440883338451,
      "losses/sft": 2.408010482788086,
      "losses/total": 0.013380440883338451,
      "ref_logps/chosen": -153.0699462890625,
      "ref_logps/rejected": -166.4036407470703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1099885702133179,
      "rewards/margins": 8.598810195922852,
      "rewards/rejected": -9.708799362182617,
      "step": 40
    },
    {
      "epoch": 0.07635009310986965,
      "grad_norm": 13.349935531616211,
      "learning_rate": 3.796296296296297e-06,
      "logps/chosen": -162.9290771484375,
      "logps/rejected": -275.8412780761719,
      "loss": 0.0746,
      "losses/dpo": 0.0015751949977129698,
      "losses/sft": 2.3437771797180176,
      "losses/total": 0.0015751949977129698,
      "ref_logps/chosen": -151.49713134765625,
      "ref_logps/rejected": -168.54776000976562,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": -1.1431937217712402,
      "rewards/margins": 9.586158752441406,
      "rewards/rejected": -10.729352951049805,
      "step": 41
    },
    {
      "epoch": 0.0782122905027933,
      "grad_norm": 7.708398818969727,
      "learning_rate": 3.88888888888889e-06,
      "logps/chosen": -170.53414916992188,
      "logps/rejected": -267.343017578125,
      "loss": 0.0415,
      "losses/dpo": 0.16293640434741974,
      "losses/sft": 2.4445619583129883,
      "losses/total": 0.16293640434741974,
      "ref_logps/chosen": -158.01504516601562,
      "ref_logps/rejected": -162.6872100830078,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.2519094944000244,
      "rewards/margins": 9.213672637939453,
      "rewards/rejected": -10.465582847595215,
      "step": 42
    },
    {
      "epoch": 0.08007448789571694,
      "grad_norm": 8.482429504394531,
      "learning_rate": 3.9814814814814814e-06,
      "logps/chosen": -152.68186950683594,
      "logps/rejected": -256.3250427246094,
      "loss": 0.0489,
      "losses/dpo": 0.005869926419109106,
      "losses/sft": 2.2688705921173096,
      "losses/total": 0.005869926419109106,
      "ref_logps/chosen": -143.47671508789062,
      "ref_logps/rejected": -158.04229736328125,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.9205170273780823,
      "rewards/margins": 8.907758712768555,
      "rewards/rejected": -9.828275680541992,
      "step": 43
    },
    {
      "epoch": 0.08193668528864059,
      "grad_norm": 4.836800575256348,
      "learning_rate": 4.074074074074074e-06,
      "logps/chosen": -161.54238891601562,
      "logps/rejected": -249.66888427734375,
      "loss": 0.0293,
      "losses/dpo": 0.007885000668466091,
      "losses/sft": 2.5034093856811523,
      "losses/total": 0.007885000668466091,
      "ref_logps/chosen": -150.9593963623047,
      "ref_logps/rejected": -160.7205810546875,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.05829918384552,
      "rewards/margins": 7.8365325927734375,
      "rewards/rejected": -8.894831657409668,
      "step": 44
    },
    {
      "epoch": 0.08379888268156424,
      "grad_norm": 7.306879997253418,
      "learning_rate": 4.166666666666667e-06,
      "logps/chosen": -168.57354736328125,
      "logps/rejected": -273.81622314453125,
      "loss": 0.0409,
      "losses/dpo": 0.044236376881599426,
      "losses/sft": 2.285489559173584,
      "losses/total": 0.044236376881599426,
      "ref_logps/chosen": -153.68539428710938,
      "ref_logps/rejected": -173.37750244140625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4888172149658203,
      "rewards/margins": 8.555052757263184,
      "rewards/rejected": -10.043869972229004,
      "step": 45
    },
    {
      "epoch": 0.0856610800744879,
      "grad_norm": 8.716976165771484,
      "learning_rate": 4.2592592592592596e-06,
      "logps/chosen": -157.63235473632812,
      "logps/rejected": -256.28546142578125,
      "loss": 0.0498,
      "losses/dpo": 0.047989651560783386,
      "losses/sft": 2.594020366668701,
      "losses/total": 0.047989651560783386,
      "ref_logps/chosen": -149.37562561035156,
      "ref_logps/rejected": -163.25059509277344,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.8256728053092957,
      "rewards/margins": 8.477815628051758,
      "rewards/rejected": -9.303489685058594,
      "step": 46
    },
    {
      "epoch": 0.08752327746741155,
      "grad_norm": 14.776224136352539,
      "learning_rate": 4.351851851851852e-06,
      "logps/chosen": -159.35926818847656,
      "logps/rejected": -254.82000732421875,
      "loss": 0.0947,
      "losses/dpo": 0.0015063448809087276,
      "losses/sft": 2.186452627182007,
      "losses/total": 0.0015063448809087276,
      "ref_logps/chosen": -153.80572509765625,
      "ref_logps/rejected": -160.63690185546875,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": -0.555354654788971,
      "rewards/margins": 8.862955093383789,
      "rewards/rejected": -9.41831111907959,
      "step": 47
    },
    {
      "epoch": 0.0893854748603352,
      "grad_norm": 6.18978214263916,
      "learning_rate": 4.444444444444444e-06,
      "logps/chosen": -146.09950256347656,
      "logps/rejected": -269.2020568847656,
      "loss": 0.0442,
      "losses/dpo": 0.010637683793902397,
      "losses/sft": 2.3249499797821045,
      "losses/total": 0.010637683793902397,
      "ref_logps/chosen": -140.32867431640625,
      "ref_logps/rejected": -161.42984008789062,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.5770829319953918,
      "rewards/margins": 10.200138092041016,
      "rewards/rejected": -10.7772216796875,
      "step": 48
    },
    {
      "epoch": 0.09124767225325885,
      "grad_norm": 8.178156852722168,
      "learning_rate": 4.537037037037038e-06,
      "logps/chosen": -150.715087890625,
      "logps/rejected": -288.4056396484375,
      "loss": 0.0446,
      "losses/dpo": 0.2217281013727188,
      "losses/sft": 2.4317214488983154,
      "losses/total": 0.2217281013727188,
      "ref_logps/chosen": -143.90017700195312,
      "ref_logps/rejected": -177.4407958984375,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.6814901828765869,
      "rewards/margins": 10.414993286132812,
      "rewards/rejected": -11.09648323059082,
      "step": 49
    },
    {
      "epoch": 0.0931098696461825,
      "grad_norm": 11.958553314208984,
      "learning_rate": 4.62962962962963e-06,
      "logps/chosen": -156.93243408203125,
      "logps/rejected": -276.21051025390625,
      "loss": 0.0786,
      "losses/dpo": 0.020680470392107964,
      "losses/sft": 2.4808478355407715,
      "losses/total": 0.020680470392107964,
      "ref_logps/chosen": -146.65036010742188,
      "ref_logps/rejected": -166.49574279785156,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.0282070636749268,
      "rewards/margins": 9.943269729614258,
      "rewards/rejected": -10.971476554870605,
      "step": 50
    },
    {
      "epoch": 0.09497206703910614,
      "grad_norm": 2.3756279945373535,
      "learning_rate": 4.722222222222222e-06,
      "logps/chosen": -146.2957763671875,
      "logps/rejected": -290.5218505859375,
      "loss": 0.0115,
      "losses/dpo": 0.021228570491075516,
      "losses/sft": 2.0093941688537598,
      "losses/total": 0.021228570491075516,
      "ref_logps/chosen": -144.70184326171875,
      "ref_logps/rejected": -169.62815856933594,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15939302742481232,
      "rewards/margins": 11.929975509643555,
      "rewards/rejected": -12.08936882019043,
      "step": 51
    },
    {
      "epoch": 0.09683426443202979,
      "grad_norm": 4.650004863739014,
      "learning_rate": 4.814814814814815e-06,
      "logps/chosen": -151.42132568359375,
      "logps/rejected": -283.79669189453125,
      "loss": 0.0183,
      "losses/dpo": 0.005594337359070778,
      "losses/sft": 2.4590542316436768,
      "losses/total": 0.005594337359070778,
      "ref_logps/chosen": -148.097412109375,
      "ref_logps/rejected": -164.08462524414062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.33239230513572693,
      "rewards/margins": 11.638812065124512,
      "rewards/rejected": -11.97120475769043,
      "step": 52
    },
    {
      "epoch": 0.09869646182495345,
      "grad_norm": 7.309722423553467,
      "learning_rate": 4.907407407407408e-06,
      "logps/chosen": -163.71875,
      "logps/rejected": -280.90655517578125,
      "loss": 0.0367,
      "losses/dpo": 0.05281204357743263,
      "losses/sft": 2.374516725540161,
      "losses/total": 0.05281204357743263,
      "ref_logps/chosen": -151.98492431640625,
      "ref_logps/rejected": -164.55319213867188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.173383116722107,
      "rewards/margins": 10.461955070495605,
      "rewards/rejected": -11.635337829589844,
      "step": 53
    },
    {
      "epoch": 0.1005586592178771,
      "grad_norm": 6.878481388092041,
      "learning_rate": 5e-06,
      "logps/chosen": -158.5335693359375,
      "logps/rejected": -270.8926086425781,
      "loss": 0.0462,
      "losses/dpo": 0.008242600597441196,
      "losses/sft": 2.6928701400756836,
      "losses/total": 0.008242600597441196,
      "ref_logps/chosen": -154.32550048828125,
      "ref_logps/rejected": -162.00128173828125,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.4208071529865265,
      "rewards/margins": 10.468324661254883,
      "rewards/rejected": -10.889131546020508,
      "step": 54
    },
    {
      "epoch": 0.10242085661080075,
      "grad_norm": 5.541274070739746,
      "learning_rate": 4.989648033126294e-06,
      "logps/chosen": -150.91574096679688,
      "logps/rejected": -279.86248779296875,
      "loss": 0.0329,
      "losses/dpo": 0.0699252262711525,
      "losses/sft": 2.2539589405059814,
      "losses/total": 0.0699252262711525,
      "ref_logps/chosen": -147.29766845703125,
      "ref_logps/rejected": -170.07183837890625,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.36180704832077026,
      "rewards/margins": 10.61725902557373,
      "rewards/rejected": -10.97906494140625,
      "step": 55
    },
    {
      "epoch": 0.1042830540037244,
      "grad_norm": 6.928717613220215,
      "learning_rate": 4.9792960662525884e-06,
      "logps/chosen": -157.90420532226562,
      "logps/rejected": -274.0709228515625,
      "loss": 0.0401,
      "losses/dpo": 0.0023821014910936356,
      "losses/sft": 2.440980911254883,
      "losses/total": 0.0023821014910936356,
      "ref_logps/chosen": -145.0316925048828,
      "ref_logps/rejected": -164.71649169921875,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.2872496843338013,
      "rewards/margins": 9.648194313049316,
      "rewards/rejected": -10.935443878173828,
      "step": 56
    },
    {
      "epoch": 0.10614525139664804,
      "grad_norm": 12.181221961975098,
      "learning_rate": 4.968944099378882e-06,
      "logps/chosen": -164.98733520507812,
      "logps/rejected": -292.568115234375,
      "loss": 0.0608,
      "losses/dpo": 0.2203754484653473,
      "losses/sft": 2.4729645252227783,
      "losses/total": 0.2203754484653473,
      "ref_logps/chosen": -145.78073120117188,
      "ref_logps/rejected": -170.54788208007812,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.9206593036651611,
      "rewards/margins": 10.28136157989502,
      "rewards/rejected": -12.202020645141602,
      "step": 57
    },
    {
      "epoch": 0.10800744878957169,
      "grad_norm": 3.31730580329895,
      "learning_rate": 4.9585921325051765e-06,
      "logps/chosen": -144.76385498046875,
      "logps/rejected": -279.8059997558594,
      "loss": 0.0189,
      "losses/dpo": 0.023249687626957893,
      "losses/sft": 2.241105079650879,
      "losses/total": 0.023249687626957893,
      "ref_logps/chosen": -135.7710723876953,
      "ref_logps/rejected": -163.89041137695312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8992785215377808,
      "rewards/margins": 10.692280769348145,
      "rewards/rejected": -11.591558456420898,
      "step": 58
    },
    {
      "epoch": 0.10986964618249534,
      "grad_norm": 5.754706382751465,
      "learning_rate": 4.94824016563147e-06,
      "logps/chosen": -159.64266967773438,
      "logps/rejected": -272.6561584472656,
      "loss": 0.0327,
      "losses/dpo": 0.0525113008916378,
      "losses/sft": 2.5197577476501465,
      "losses/total": 0.0525113008916378,
      "ref_logps/chosen": -142.01556396484375,
      "ref_logps/rejected": -161.62533569335938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7627112865447998,
      "rewards/margins": 9.340368270874023,
      "rewards/rejected": -11.103079795837402,
      "step": 59
    },
    {
      "epoch": 0.11173184357541899,
      "grad_norm": 2.9009838104248047,
      "learning_rate": 4.9378881987577645e-06,
      "logps/chosen": -153.46998596191406,
      "logps/rejected": -300.505615234375,
      "loss": 0.0109,
      "losses/dpo": 0.05081873759627342,
      "losses/sft": 2.3817033767700195,
      "losses/total": 0.05081873759627342,
      "ref_logps/chosen": -137.58828735351562,
      "ref_logps/rejected": -164.65621948242188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5881686210632324,
      "rewards/margins": 11.996772766113281,
      "rewards/rejected": -13.584940910339355,
      "step": 60
    },
    {
      "epoch": 0.11359404096834265,
      "grad_norm": 10.7985258102417,
      "learning_rate": 4.927536231884059e-06,
      "logps/chosen": -161.22463989257812,
      "logps/rejected": -312.06597900390625,
      "loss": 0.0594,
      "losses/dpo": 0.02082718536257744,
      "losses/sft": 2.446563720703125,
      "losses/total": 0.02082718536257744,
      "ref_logps/chosen": -150.21656799316406,
      "ref_logps/rejected": -171.88088989257812,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.1008076667785645,
      "rewards/margins": 12.917702674865723,
      "rewards/rejected": -14.018510818481445,
      "step": 61
    },
    {
      "epoch": 0.1154562383612663,
      "grad_norm": 16.700149536132812,
      "learning_rate": 4.9171842650103525e-06,
      "logps/chosen": -157.88522338867188,
      "logps/rejected": -296.93817138671875,
      "loss": 0.0929,
      "losses/dpo": 0.0005755929159931839,
      "losses/sft": 2.514739751815796,
      "losses/total": 0.0005755929159931839,
      "ref_logps/chosen": -140.2955780029297,
      "ref_logps/rejected": -165.69281005859375,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.7589635848999023,
      "rewards/margins": 11.365575790405273,
      "rewards/rejected": -13.124539375305176,
      "step": 62
    },
    {
      "epoch": 0.11731843575418995,
      "grad_norm": 10.798019409179688,
      "learning_rate": 4.906832298136646e-06,
      "logps/chosen": -160.2606201171875,
      "logps/rejected": -316.3315734863281,
      "loss": 0.0485,
      "losses/dpo": 0.0018289771396666765,
      "losses/sft": 3.0345890522003174,
      "losses/total": 0.0018289771396666765,
      "ref_logps/chosen": -144.05250549316406,
      "ref_logps/rejected": -170.66879272460938,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.6208107471466064,
      "rewards/margins": 12.945466041564941,
      "rewards/rejected": -14.566276550292969,
      "step": 63
    },
    {
      "epoch": 0.1191806331471136,
      "grad_norm": 8.646073341369629,
      "learning_rate": 4.896480331262941e-06,
      "logps/chosen": -163.75540161132812,
      "logps/rejected": -315.20458984375,
      "loss": 0.0576,
      "losses/dpo": 0.014824606478214264,
      "losses/sft": 3.261763572692871,
      "losses/total": 0.014824606478214264,
      "ref_logps/chosen": -149.48907470703125,
      "ref_logps/rejected": -173.37435913085938,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.4266325235366821,
      "rewards/margins": 12.756391525268555,
      "rewards/rejected": -14.183023452758789,
      "step": 64
    },
    {
      "epoch": 0.12104283054003724,
      "grad_norm": 5.649742603302002,
      "learning_rate": 4.886128364389234e-06,
      "logps/chosen": -174.45712280273438,
      "logps/rejected": -324.653076171875,
      "loss": 0.018,
      "losses/dpo": 0.03694448992609978,
      "losses/sft": 2.400181293487549,
      "losses/total": 0.03694448992609978,
      "ref_logps/chosen": -155.43272399902344,
      "ref_logps/rejected": -175.6329345703125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.902438759803772,
      "rewards/margins": 12.999577522277832,
      "rewards/rejected": -14.902015686035156,
      "step": 65
    },
    {
      "epoch": 0.12290502793296089,
      "grad_norm": 4.2772345542907715,
      "learning_rate": 4.875776397515528e-06,
      "logps/chosen": -160.1391143798828,
      "logps/rejected": -298.70098876953125,
      "loss": 0.0198,
      "losses/dpo": 0.009300258941948414,
      "losses/sft": 2.3741018772125244,
      "losses/total": 0.009300258941948414,
      "ref_logps/chosen": -143.62374877929688,
      "ref_logps/rejected": -162.72705078125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6515378952026367,
      "rewards/margins": 11.945855140686035,
      "rewards/rejected": -13.597392082214355,
      "step": 66
    },
    {
      "epoch": 0.12476722532588454,
      "grad_norm": 12.112936019897461,
      "learning_rate": 4.865424430641822e-06,
      "logps/chosen": -171.04205322265625,
      "logps/rejected": -303.92144775390625,
      "loss": 0.0626,
      "losses/dpo": 0.36290252208709717,
      "losses/sft": 2.4964911937713623,
      "losses/total": 0.36290252208709717,
      "ref_logps/chosen": -152.29478454589844,
      "ref_logps/rejected": -161.23959350585938,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.8747248649597168,
      "rewards/margins": 12.393461227416992,
      "rewards/rejected": -14.268186569213867,
      "step": 67
    },
    {
      "epoch": 0.1266294227188082,
      "grad_norm": 5.256114959716797,
      "learning_rate": 4.855072463768117e-06,
      "logps/chosen": -174.6351776123047,
      "logps/rejected": -348.933837890625,
      "loss": 0.0251,
      "losses/dpo": 0.09207179397344589,
      "losses/sft": 2.219914436340332,
      "losses/total": 0.09207179397344589,
      "ref_logps/chosen": -151.7803955078125,
      "ref_logps/rejected": -185.33236694335938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.285478353500366,
      "rewards/margins": 14.074670791625977,
      "rewards/rejected": -16.36014747619629,
      "step": 68
    },
    {
      "epoch": 0.12849162011173185,
      "grad_norm": 5.340096950531006,
      "learning_rate": 4.84472049689441e-06,
      "logps/chosen": -161.56605529785156,
      "logps/rejected": -303.75201416015625,
      "loss": 0.027,
      "losses/dpo": 0.013381706550717354,
      "losses/sft": 2.1893093585968018,
      "losses/total": 0.013381706550717354,
      "ref_logps/chosen": -150.4385986328125,
      "ref_logps/rejected": -168.53863525390625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1127451658248901,
      "rewards/margins": 12.40859317779541,
      "rewards/rejected": -13.521337509155273,
      "step": 69
    },
    {
      "epoch": 0.1303538175046555,
      "grad_norm": 13.288958549499512,
      "learning_rate": 4.834368530020705e-06,
      "logps/chosen": -172.6832275390625,
      "logps/rejected": -334.8218078613281,
      "loss": 0.1264,
      "losses/dpo": 0.0077868797816336155,
      "losses/sft": 2.6989755630493164,
      "losses/total": 0.0077868797816336155,
      "ref_logps/chosen": -152.52114868164062,
      "ref_logps/rejected": -174.5692901611328,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.0162088871002197,
      "rewards/margins": 14.009044647216797,
      "rewards/rejected": -16.02525520324707,
      "step": 70
    },
    {
      "epoch": 0.13221601489757914,
      "grad_norm": 6.7856059074401855,
      "learning_rate": 4.824016563146998e-06,
      "logps/chosen": -166.0758056640625,
      "logps/rejected": -302.7711181640625,
      "loss": 0.0375,
      "losses/dpo": 0.008599307388067245,
      "losses/sft": 2.780581474304199,
      "losses/total": 0.008599307388067245,
      "ref_logps/chosen": -144.70916748046875,
      "ref_logps/rejected": -158.5432586669922,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -2.136662244796753,
      "rewards/margins": 12.286121368408203,
      "rewards/rejected": -14.422782897949219,
      "step": 71
    },
    {
      "epoch": 0.1340782122905028,
      "grad_norm": 7.3059282302856445,
      "learning_rate": 4.813664596273293e-06,
      "logps/chosen": -175.05950927734375,
      "logps/rejected": -346.2865905761719,
      "loss": 0.0272,
      "losses/dpo": 0.06259185075759888,
      "losses/sft": 2.080183982849121,
      "losses/total": 0.06259185075759888,
      "ref_logps/chosen": -153.53565979003906,
      "ref_logps/rejected": -175.8729248046875,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -2.152385950088501,
      "rewards/margins": 14.888980865478516,
      "rewards/rejected": -17.041364669799805,
      "step": 72
    },
    {
      "epoch": 0.13594040968342644,
      "grad_norm": 14.326214790344238,
      "learning_rate": 4.803312629399586e-06,
      "logps/chosen": -159.51559448242188,
      "logps/rejected": -303.34576416015625,
      "loss": 0.1862,
      "losses/dpo": 0.1816004067659378,
      "losses/sft": 2.4103786945343018,
      "losses/total": 0.1816004067659378,
      "ref_logps/chosen": -137.89926147460938,
      "ref_logps/rejected": -151.60308837890625,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.1616339683532715,
      "rewards/margins": 13.0126371383667,
      "rewards/rejected": -15.174270629882812,
      "step": 73
    },
    {
      "epoch": 0.1378026070763501,
      "grad_norm": 11.816011428833008,
      "learning_rate": 4.79296066252588e-06,
      "logps/chosen": -163.58856201171875,
      "logps/rejected": -307.9797058105469,
      "loss": 0.076,
      "losses/dpo": 0.04001971334218979,
      "losses/sft": 2.3666553497314453,
      "losses/total": 0.04001971334218979,
      "ref_logps/chosen": -140.401123046875,
      "ref_logps/rejected": -169.53005981445312,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.3187437057495117,
      "rewards/margins": 11.52622127532959,
      "rewards/rejected": -13.844964981079102,
      "step": 74
    },
    {
      "epoch": 0.13966480446927373,
      "grad_norm": 13.491276741027832,
      "learning_rate": 4.782608695652174e-06,
      "logps/chosen": -163.01483154296875,
      "logps/rejected": -305.1317443847656,
      "loss": 0.1175,
      "losses/dpo": 0.021199261769652367,
      "losses/sft": 2.5555906295776367,
      "losses/total": 0.021199261769652367,
      "ref_logps/chosen": -135.76666259765625,
      "ref_logps/rejected": -159.01651000976562,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.72481632232666,
      "rewards/margins": 11.88670825958252,
      "rewards/rejected": -14.61152458190918,
      "step": 75
    },
    {
      "epoch": 0.14152700186219738,
      "grad_norm": 13.310988426208496,
      "learning_rate": 4.772256728778468e-06,
      "logps/chosen": -166.88494873046875,
      "logps/rejected": -300.84918212890625,
      "loss": 0.0695,
      "losses/dpo": 0.23336642980575562,
      "losses/sft": 2.7684144973754883,
      "losses/total": 0.23336642980575562,
      "ref_logps/chosen": -145.76715087890625,
      "ref_logps/rejected": -162.08209228515625,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": -2.1117796897888184,
      "rewards/margins": 11.764933586120605,
      "rewards/rejected": -13.876713752746582,
      "step": 76
    },
    {
      "epoch": 0.14338919925512103,
      "grad_norm": 5.915053844451904,
      "learning_rate": 4.761904761904762e-06,
      "logps/chosen": -166.23074340820312,
      "logps/rejected": -321.7993469238281,
      "loss": 0.0245,
      "losses/dpo": 0.00781274028122425,
      "losses/sft": 2.6997854709625244,
      "losses/total": 0.00781274028122425,
      "ref_logps/chosen": -146.89723205566406,
      "ref_logps/rejected": -168.7777099609375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9333515167236328,
      "rewards/margins": 13.368810653686523,
      "rewards/rejected": -15.302162170410156,
      "step": 77
    },
    {
      "epoch": 0.1452513966480447,
      "grad_norm": 8.788707733154297,
      "learning_rate": 4.751552795031056e-06,
      "logps/chosen": -175.02999877929688,
      "logps/rejected": -321.0180969238281,
      "loss": 0.0271,
      "losses/dpo": 0.14333294332027435,
      "losses/sft": 2.538876533508301,
      "losses/total": 0.14333294332027435,
      "ref_logps/chosen": -157.10501098632812,
      "ref_logps/rejected": -181.61178588867188,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.7924991846084595,
      "rewards/margins": 12.148136138916016,
      "rewards/rejected": -13.940634727478027,
      "step": 78
    },
    {
      "epoch": 0.14711359404096835,
      "grad_norm": 3.836193323135376,
      "learning_rate": 4.74120082815735e-06,
      "logps/chosen": -161.6851806640625,
      "logps/rejected": -307.22308349609375,
      "loss": 0.018,
      "losses/dpo": 0.005441778339445591,
      "losses/sft": 2.4244680404663086,
      "losses/total": 0.005441778339445591,
      "ref_logps/chosen": -141.81394958496094,
      "ref_logps/rejected": -166.87547302246094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9871225357055664,
      "rewards/margins": 12.047636985778809,
      "rewards/rejected": -14.034758567810059,
      "step": 79
    },
    {
      "epoch": 0.148975791433892,
      "grad_norm": 1.0960124731063843,
      "learning_rate": 4.730848861283645e-06,
      "logps/chosen": -164.8990020751953,
      "logps/rejected": -306.0439453125,
      "loss": 0.0046,
      "losses/dpo": 0.00012883917952422053,
      "losses/sft": 2.337103843688965,
      "losses/total": 0.00012883917952422053,
      "ref_logps/chosen": -150.0802001953125,
      "ref_logps/rejected": -169.78543090820312,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.48188054561615,
      "rewards/margins": 12.143974304199219,
      "rewards/rejected": -13.6258544921875,
      "step": 80
    },
    {
      "epoch": 0.15083798882681565,
      "grad_norm": 10.833060264587402,
      "learning_rate": 4.7204968944099384e-06,
      "logps/chosen": -166.50360107421875,
      "logps/rejected": -291.5777893066406,
      "loss": 0.0699,
      "losses/dpo": 0.3678208291530609,
      "losses/sft": 2.360893726348877,
      "losses/total": 0.3678208291530609,
      "ref_logps/chosen": -139.50567626953125,
      "ref_logps/rejected": -159.87924194335938,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -2.699791193008423,
      "rewards/margins": 10.470064163208008,
      "rewards/rejected": -13.169855117797852,
      "step": 81
    },
    {
      "epoch": 0.1527001862197393,
      "grad_norm": 10.486247062683105,
      "learning_rate": 4.710144927536232e-06,
      "logps/chosen": -158.02520751953125,
      "logps/rejected": -298.325927734375,
      "loss": 0.0641,
      "losses/dpo": 0.04021751880645752,
      "losses/sft": 2.221280574798584,
      "losses/total": 0.04021751880645752,
      "ref_logps/chosen": -147.9010009765625,
      "ref_logps/rejected": -168.83761596679688,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.0124225616455078,
      "rewards/margins": 11.936410903930664,
      "rewards/rejected": -12.948833465576172,
      "step": 82
    },
    {
      "epoch": 0.15456238361266295,
      "grad_norm": 4.4983320236206055,
      "learning_rate": 4.6997929606625265e-06,
      "logps/chosen": -157.27609252929688,
      "logps/rejected": -284.30670166015625,
      "loss": 0.0134,
      "losses/dpo": 0.07455439865589142,
      "losses/sft": 2.680065631866455,
      "losses/total": 0.07455439865589142,
      "ref_logps/chosen": -143.0989990234375,
      "ref_logps/rejected": -166.63612365722656,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4177087545394897,
      "rewards/margins": 10.349348068237305,
      "rewards/rejected": -11.767056465148926,
      "step": 83
    },
    {
      "epoch": 0.1564245810055866,
      "grad_norm": 5.925443172454834,
      "learning_rate": 4.68944099378882e-06,
      "logps/chosen": -166.75436401367188,
      "logps/rejected": -307.4884338378906,
      "loss": 0.0294,
      "losses/dpo": 0.018268397077918053,
      "losses/sft": 2.3180205821990967,
      "losses/total": 0.018268397077918053,
      "ref_logps/chosen": -146.59835815429688,
      "ref_logps/rejected": -171.87164306640625,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -2.015601634979248,
      "rewards/margins": 11.546079635620117,
      "rewards/rejected": -13.561680793762207,
      "step": 84
    },
    {
      "epoch": 0.15828677839851024,
      "grad_norm": 10.823968887329102,
      "learning_rate": 4.679089026915114e-06,
      "logps/chosen": -164.76190185546875,
      "logps/rejected": -291.744873046875,
      "loss": 0.0787,
      "losses/dpo": 0.0033458764664828777,
      "losses/sft": 2.6443734169006348,
      "losses/total": 0.0033458764664828777,
      "ref_logps/chosen": -147.3948516845703,
      "ref_logps/rejected": -165.80938720703125,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.73670494556427,
      "rewards/margins": 10.856843948364258,
      "rewards/rejected": -12.593548774719238,
      "step": 85
    },
    {
      "epoch": 0.1601489757914339,
      "grad_norm": 10.07721996307373,
      "learning_rate": 4.668737060041408e-06,
      "logps/chosen": -170.19273376464844,
      "logps/rejected": -289.0329284667969,
      "loss": 0.0578,
      "losses/dpo": 0.22292722761631012,
      "losses/sft": 2.8669309616088867,
      "losses/total": 0.22292722761631012,
      "ref_logps/chosen": -155.04888916015625,
      "ref_logps/rejected": -169.506103515625,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.5143851041793823,
      "rewards/margins": 10.438299179077148,
      "rewards/rejected": -11.95268440246582,
      "step": 86
    },
    {
      "epoch": 0.16201117318435754,
      "grad_norm": 7.202996730804443,
      "learning_rate": 4.6583850931677025e-06,
      "logps/chosen": -152.85894775390625,
      "logps/rejected": -276.5400390625,
      "loss": 0.0464,
      "losses/dpo": 0.0008024298003874719,
      "losses/sft": 2.105069637298584,
      "losses/total": 0.0008024298003874719,
      "ref_logps/chosen": -135.02166748046875,
      "ref_logps/rejected": -162.55917358398438,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.783726453781128,
      "rewards/margins": 9.614358901977539,
      "rewards/rejected": -11.39808464050293,
      "step": 87
    },
    {
      "epoch": 0.16387337057728119,
      "grad_norm": 10.529645919799805,
      "learning_rate": 4.648033126293996e-06,
      "logps/chosen": -167.41766357421875,
      "logps/rejected": -312.9851989746094,
      "loss": 0.1059,
      "losses/dpo": 1.8586399164632894e-05,
      "losses/sft": 2.3417227268218994,
      "losses/total": 1.8586399164632894e-05,
      "ref_logps/chosen": -150.24447631835938,
      "ref_logps/rejected": -171.88534545898438,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.7173188924789429,
      "rewards/margins": 12.392667770385742,
      "rewards/rejected": -14.1099853515625,
      "step": 88
    },
    {
      "epoch": 0.16573556797020483,
      "grad_norm": 5.379014492034912,
      "learning_rate": 4.637681159420291e-06,
      "logps/chosen": -149.58767700195312,
      "logps/rejected": -283.6622009277344,
      "loss": 0.0265,
      "losses/dpo": 0.0076700253412127495,
      "losses/sft": 2.0566470623016357,
      "losses/total": 0.0076700253412127495,
      "ref_logps/chosen": -142.82745361328125,
      "ref_logps/rejected": -166.57040405273438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6760219931602478,
      "rewards/margins": 11.03315544128418,
      "rewards/rejected": -11.709177017211914,
      "step": 89
    },
    {
      "epoch": 0.16759776536312848,
      "grad_norm": 7.1271820068359375,
      "learning_rate": 4.627329192546584e-06,
      "logps/chosen": -166.07994079589844,
      "logps/rejected": -288.1270446777344,
      "loss": 0.0383,
      "losses/dpo": 0.006637689657509327,
      "losses/sft": 2.695986270904541,
      "losses/total": 0.006637689657509327,
      "ref_logps/chosen": -152.90316772460938,
      "ref_logps/rejected": -167.97433471679688,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.3176772594451904,
      "rewards/margins": 10.697591781616211,
      "rewards/rejected": -12.01526927947998,
      "step": 90
    },
    {
      "epoch": 0.16945996275605213,
      "grad_norm": 3.960089683532715,
      "learning_rate": 4.616977225672879e-06,
      "logps/chosen": -155.234619140625,
      "logps/rejected": -274.2052917480469,
      "loss": 0.0283,
      "losses/dpo": 0.029921425506472588,
      "losses/sft": 2.448662757873535,
      "losses/total": 0.029921425506472588,
      "ref_logps/chosen": -146.57998657226562,
      "ref_logps/rejected": -161.9693603515625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8654617071151733,
      "rewards/margins": 10.35813045501709,
      "rewards/rejected": -11.223592758178711,
      "step": 91
    },
    {
      "epoch": 0.1713221601489758,
      "grad_norm": 8.884647369384766,
      "learning_rate": 4.606625258799172e-06,
      "logps/chosen": -156.59239196777344,
      "logps/rejected": -269.66265869140625,
      "loss": 0.0549,
      "losses/dpo": 0.000325945409713313,
      "losses/sft": 2.27170991897583,
      "losses/total": 0.000325945409713313,
      "ref_logps/chosen": -143.44590759277344,
      "ref_logps/rejected": -160.02975463867188,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.3146480321884155,
      "rewards/margins": 9.648642539978027,
      "rewards/rejected": -10.963289260864258,
      "step": 92
    },
    {
      "epoch": 0.17318435754189945,
      "grad_norm": 8.483807563781738,
      "learning_rate": 4.596273291925466e-06,
      "logps/chosen": -157.52755737304688,
      "logps/rejected": -302.79736328125,
      "loss": 0.0525,
      "losses/dpo": 0.007826759479939938,
      "losses/sft": 2.404571533203125,
      "losses/total": 0.007826759479939938,
      "ref_logps/chosen": -151.92408752441406,
      "ref_logps/rejected": -174.8727569580078,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.5603480339050293,
      "rewards/margins": 12.232114791870117,
      "rewards/rejected": -12.792463302612305,
      "step": 93
    },
    {
      "epoch": 0.1750465549348231,
      "grad_norm": 2.803177833557129,
      "learning_rate": 4.58592132505176e-06,
      "logps/chosen": -154.108154296875,
      "logps/rejected": -316.1098937988281,
      "loss": 0.0115,
      "losses/dpo": 0.014974968507885933,
      "losses/sft": 2.5976779460906982,
      "losses/total": 0.014974968507885933,
      "ref_logps/chosen": -139.6196746826172,
      "ref_logps/rejected": -177.22061157226562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4488475322723389,
      "rewards/margins": 12.440080642700195,
      "rewards/rejected": -13.88892936706543,
      "step": 94
    },
    {
      "epoch": 0.17690875232774675,
      "grad_norm": 10.011286735534668,
      "learning_rate": 4.575569358178054e-06,
      "logps/chosen": -161.99615478515625,
      "logps/rejected": -290.70928955078125,
      "loss": 0.0521,
      "losses/dpo": 0.004248100332915783,
      "losses/sft": 2.6869282722473145,
      "losses/total": 0.004248100332915783,
      "ref_logps/chosen": -150.13644409179688,
      "ref_logps/rejected": -170.98199462890625,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.1859712600708008,
      "rewards/margins": 10.786758422851562,
      "rewards/rejected": -11.972728729248047,
      "step": 95
    },
    {
      "epoch": 0.1787709497206704,
      "grad_norm": 13.34592342376709,
      "learning_rate": 4.565217391304348e-06,
      "logps/chosen": -157.46038818359375,
      "logps/rejected": -293.8796691894531,
      "loss": 0.0613,
      "losses/dpo": 0.0029001892544329166,
      "losses/sft": 2.191887378692627,
      "losses/total": 0.0029001892544329166,
      "ref_logps/chosen": -147.4150390625,
      "ref_logps/rejected": -175.098876953125,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.004532814025879,
      "rewards/margins": 10.873546600341797,
      "rewards/rejected": -11.878079414367676,
      "step": 96
    },
    {
      "epoch": 0.18063314711359404,
      "grad_norm": 13.375811576843262,
      "learning_rate": 4.554865424430642e-06,
      "logps/chosen": -166.24191284179688,
      "logps/rejected": -305.58819580078125,
      "loss": 0.0981,
      "losses/dpo": 0.0007697511464357376,
      "losses/sft": 2.3007588386535645,
      "losses/total": 0.0007697511464357376,
      "ref_logps/chosen": -144.6219482421875,
      "ref_logps/rejected": -174.1811981201172,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": -2.1619954109191895,
      "rewards/margins": 10.978706359863281,
      "rewards/rejected": -13.140701293945312,
      "step": 97
    },
    {
      "epoch": 0.1824953445065177,
      "grad_norm": 11.116385459899902,
      "learning_rate": 4.544513457556936e-06,
      "logps/chosen": -151.30795288085938,
      "logps/rejected": -303.7850341796875,
      "loss": 0.0838,
      "losses/dpo": 0.006321331951767206,
      "losses/sft": 2.45919132232666,
      "losses/total": 0.006321331951767206,
      "ref_logps/chosen": -141.2546844482422,
      "ref_logps/rejected": -166.2907257080078,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.0053260326385498,
      "rewards/margins": 12.744101524353027,
      "rewards/rejected": -13.749427795410156,
      "step": 98
    },
    {
      "epoch": 0.18435754189944134,
      "grad_norm": 8.1275053024292,
      "learning_rate": 4.534161490683231e-06,
      "logps/chosen": -169.12844848632812,
      "logps/rejected": -291.58355712890625,
      "loss": 0.0379,
      "losses/dpo": 0.04353442043066025,
      "losses/sft": 2.9029669761657715,
      "losses/total": 0.04353442043066025,
      "ref_logps/chosen": -148.775390625,
      "ref_logps/rejected": -162.24786376953125,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -2.0353074073791504,
      "rewards/margins": 10.898263931274414,
      "rewards/rejected": -12.933570861816406,
      "step": 99
    },
    {
      "epoch": 0.186219739292365,
      "grad_norm": 3.8263306617736816,
      "learning_rate": 4.523809523809524e-06,
      "logps/chosen": -160.37161254882812,
      "logps/rejected": -291.48577880859375,
      "loss": 0.0159,
      "losses/dpo": 0.06080714985728264,
      "losses/sft": 2.056828498840332,
      "losses/total": 0.06080714985728264,
      "ref_logps/chosen": -143.4698486328125,
      "ref_logps/rejected": -165.9519500732422,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6901748180389404,
      "rewards/margins": 10.863210678100586,
      "rewards/rejected": -12.553384780883789,
      "step": 100
    },
    {
      "epoch": 0.18808193668528864,
      "grad_norm": 9.816635131835938,
      "learning_rate": 4.513457556935818e-06,
      "logps/chosen": -177.78240966796875,
      "logps/rejected": -330.1160888671875,
      "loss": 0.0489,
      "losses/dpo": 0.2933081388473511,
      "losses/sft": 2.7717084884643555,
      "losses/total": 0.2933081388473511,
      "ref_logps/chosen": -155.60540771484375,
      "ref_logps/rejected": -177.44467163085938,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -2.2176995277404785,
      "rewards/margins": 13.04944133758545,
      "rewards/rejected": -15.267141342163086,
      "step": 101
    },
    {
      "epoch": 0.18994413407821228,
      "grad_norm": 1.3710126876831055,
      "learning_rate": 4.503105590062112e-06,
      "logps/chosen": -166.66590881347656,
      "logps/rejected": -290.761962890625,
      "loss": 0.0054,
      "losses/dpo": 0.02758178301155567,
      "losses/sft": 3.162290096282959,
      "losses/total": 0.02758178301155567,
      "ref_logps/chosen": -147.08917236328125,
      "ref_logps/rejected": -151.07664489746094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9576739072799683,
      "rewards/margins": 12.010858535766602,
      "rewards/rejected": -13.96853256225586,
      "step": 102
    },
    {
      "epoch": 0.19180633147113593,
      "grad_norm": 7.93828010559082,
      "learning_rate": 4.492753623188406e-06,
      "logps/chosen": -146.8543701171875,
      "logps/rejected": -312.2845764160156,
      "loss": 0.0425,
      "losses/dpo": 0.2460930347442627,
      "losses/sft": 2.2462611198425293,
      "losses/total": 0.2460930347442627,
      "ref_logps/chosen": -135.14158630371094,
      "ref_logps/rejected": -168.27618408203125,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.1712779998779297,
      "rewards/margins": 13.229562759399414,
      "rewards/rejected": -14.400840759277344,
      "step": 103
    },
    {
      "epoch": 0.19366852886405958,
      "grad_norm": 13.03959846496582,
      "learning_rate": 4.4824016563146996e-06,
      "logps/chosen": -163.9506378173828,
      "logps/rejected": -338.9526062011719,
      "loss": 0.105,
      "losses/dpo": 1.4241080862120725e-05,
      "losses/sft": 2.1639623641967773,
      "losses/total": 1.4241080862120725e-05,
      "ref_logps/chosen": -144.27719116210938,
      "ref_logps/rejected": -172.1199951171875,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.9673457145690918,
      "rewards/margins": 14.71591567993164,
      "rewards/rejected": -16.683259963989258,
      "step": 104
    },
    {
      "epoch": 0.19553072625698323,
      "grad_norm": 12.940238952636719,
      "learning_rate": 4.472049689440994e-06,
      "logps/chosen": -167.60537719726562,
      "logps/rejected": -305.2105407714844,
      "loss": 0.1266,
      "losses/dpo": 0.016943011432886124,
      "losses/sft": 2.568720817565918,
      "losses/total": 0.016943011432886124,
      "ref_logps/chosen": -144.21282958984375,
      "ref_logps/rejected": -158.3541259765625,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": -2.339254140853882,
      "rewards/margins": 12.34638786315918,
      "rewards/rejected": -14.685641288757324,
      "step": 105
    },
    {
      "epoch": 0.1973929236499069,
      "grad_norm": 9.413414001464844,
      "learning_rate": 4.4616977225672884e-06,
      "logps/chosen": -171.18016052246094,
      "logps/rejected": -302.7274475097656,
      "loss": 0.0475,
      "losses/dpo": 0.0004924152162857354,
      "losses/sft": 2.5999226570129395,
      "losses/total": 0.0004924152162857354,
      "ref_logps/chosen": -144.5438690185547,
      "ref_logps/rejected": -162.2921905517578,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -2.6636300086975098,
      "rewards/margins": 11.37989616394043,
      "rewards/rejected": -14.043526649475098,
      "step": 106
    },
    {
      "epoch": 0.19925512104283055,
      "grad_norm": 1.439731240272522,
      "learning_rate": 4.451345755693582e-06,
      "logps/chosen": -169.63963317871094,
      "logps/rejected": -335.1871337890625,
      "loss": 0.0046,
      "losses/dpo": 0.01760055683553219,
      "losses/sft": 2.191661834716797,
      "losses/total": 0.01760055683553219,
      "ref_logps/chosen": -148.11293029785156,
      "ref_logps/rejected": -171.40560913085938,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1526713371276855,
      "rewards/margins": 14.225481986999512,
      "rewards/rejected": -16.37815284729004,
      "step": 107
    },
    {
      "epoch": 0.2011173184357542,
      "grad_norm": 12.107815742492676,
      "learning_rate": 4.4409937888198765e-06,
      "logps/chosen": -174.36929321289062,
      "logps/rejected": -322.1456604003906,
      "loss": 0.1067,
      "losses/dpo": 0.0005443975096568465,
      "losses/sft": 2.864640712738037,
      "losses/total": 0.0005443975096568465,
      "ref_logps/chosen": -149.93984985351562,
      "ref_logps/rejected": -166.4761962890625,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": -2.442944288253784,
      "rewards/margins": 13.124004364013672,
      "rewards/rejected": -15.566947937011719,
      "step": 108
    },
    {
      "epoch": 0.20297951582867785,
      "grad_norm": 7.122522354125977,
      "learning_rate": 4.43064182194617e-06,
      "logps/chosen": -155.75596618652344,
      "logps/rejected": -314.00762939453125,
      "loss": 0.0363,
      "losses/dpo": 0.1575506329536438,
      "losses/sft": 2.296907424926758,
      "losses/total": 0.1575506329536438,
      "ref_logps/chosen": -143.9917755126953,
      "ref_logps/rejected": -175.87164306640625,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.1764178276062012,
      "rewards/margins": 12.637179374694824,
      "rewards/rejected": -13.813596725463867,
      "step": 109
    },
    {
      "epoch": 0.2048417132216015,
      "grad_norm": 1.012307047843933,
      "learning_rate": 4.4202898550724645e-06,
      "logps/chosen": -164.65115356445312,
      "logps/rejected": -318.34326171875,
      "loss": 0.0034,
      "losses/dpo": 0.011834606528282166,
      "losses/sft": 2.363070249557495,
      "losses/total": 0.011834606528282166,
      "ref_logps/chosen": -149.04367065429688,
      "ref_logps/rejected": -166.6707000732422,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5607484579086304,
      "rewards/margins": 13.606508255004883,
      "rewards/rejected": -15.167256355285645,
      "step": 110
    },
    {
      "epoch": 0.20670391061452514,
      "grad_norm": 5.557791233062744,
      "learning_rate": 4.409937888198758e-06,
      "logps/chosen": -150.85084533691406,
      "logps/rejected": -309.1524658203125,
      "loss": 0.0232,
      "losses/dpo": 0.014252948574721813,
      "losses/sft": 2.4658865928649902,
      "losses/total": 0.014252948574721813,
      "ref_logps/chosen": -137.03976440429688,
      "ref_logps/rejected": -161.1869659423828,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.381108045578003,
      "rewards/margins": 13.415441513061523,
      "rewards/rejected": -14.796548843383789,
      "step": 111
    },
    {
      "epoch": 0.2085661080074488,
      "grad_norm": 9.207598686218262,
      "learning_rate": 4.399585921325052e-06,
      "logps/chosen": -167.4930419921875,
      "logps/rejected": -303.5692138671875,
      "loss": 0.0453,
      "losses/dpo": 0.0007799149607308209,
      "losses/sft": 2.518667459487915,
      "losses/total": 0.0007799149607308209,
      "ref_logps/chosen": -149.21981811523438,
      "ref_logps/rejected": -170.2943115234375,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.8273215293884277,
      "rewards/margins": 11.500171661376953,
      "rewards/rejected": -13.327491760253906,
      "step": 112
    },
    {
      "epoch": 0.21042830540037244,
      "grad_norm": 7.050368309020996,
      "learning_rate": 4.389233954451346e-06,
      "logps/chosen": -159.51071166992188,
      "logps/rejected": -307.4903869628906,
      "loss": 0.0301,
      "losses/dpo": 0.0039618429727852345,
      "losses/sft": 2.5831570625305176,
      "losses/total": 0.0039618429727852345,
      "ref_logps/chosen": -137.70912170410156,
      "ref_logps/rejected": -167.49789428710938,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -2.180156946182251,
      "rewards/margins": 11.819091796875,
      "rewards/rejected": -13.999248504638672,
      "step": 113
    },
    {
      "epoch": 0.2122905027932961,
      "grad_norm": 11.241766929626465,
      "learning_rate": 4.37888198757764e-06,
      "logps/chosen": -164.54208374023438,
      "logps/rejected": -306.73822021484375,
      "loss": 0.0491,
      "losses/dpo": 0.17672446370124817,
      "losses/sft": 2.2986695766448975,
      "losses/total": 0.17672446370124817,
      "ref_logps/chosen": -145.33531188964844,
      "ref_logps/rejected": -164.6031951904297,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.92067551612854,
      "rewards/margins": 12.292827606201172,
      "rewards/rejected": -14.213502883911133,
      "step": 114
    },
    {
      "epoch": 0.21415270018621974,
      "grad_norm": 4.248304843902588,
      "learning_rate": 4.368530020703934e-06,
      "logps/chosen": -160.98765563964844,
      "logps/rejected": -324.66845703125,
      "loss": 0.0157,
      "losses/dpo": 0.028906403109431267,
      "losses/sft": 2.51163387298584,
      "losses/total": 0.028906403109431267,
      "ref_logps/chosen": -142.23577880859375,
      "ref_logps/rejected": -178.46255493164062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.875186800956726,
      "rewards/margins": 12.745403289794922,
      "rewards/rejected": -14.620590209960938,
      "step": 115
    },
    {
      "epoch": 0.21601489757914338,
      "grad_norm": 11.712361335754395,
      "learning_rate": 4.358178053830228e-06,
      "logps/chosen": -176.92311096191406,
      "logps/rejected": -309.1515808105469,
      "loss": 0.0532,
      "losses/dpo": 0.0009635839378461242,
      "losses/sft": 2.519484758377075,
      "losses/total": 0.0009635839378461242,
      "ref_logps/chosen": -155.59686279296875,
      "ref_logps/rejected": -167.6163787841797,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -2.132624626159668,
      "rewards/margins": 12.020895004272461,
      "rewards/rejected": -14.153520584106445,
      "step": 116
    },
    {
      "epoch": 0.21787709497206703,
      "grad_norm": 12.869205474853516,
      "learning_rate": 4.347826086956522e-06,
      "logps/chosen": -164.499755859375,
      "logps/rejected": -322.2271728515625,
      "loss": 0.0481,
      "losses/dpo": 0.026883596554398537,
      "losses/sft": 2.4711999893188477,
      "losses/total": 0.026883596554398537,
      "ref_logps/chosen": -141.9335174560547,
      "ref_logps/rejected": -174.70437622070312,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -2.256622791290283,
      "rewards/margins": 12.495659828186035,
      "rewards/rejected": -14.75228214263916,
      "step": 117
    },
    {
      "epoch": 0.21973929236499068,
      "grad_norm": 7.2970499992370605,
      "learning_rate": 4.337474120082817e-06,
      "logps/chosen": -154.8804931640625,
      "logps/rejected": -307.47796630859375,
      "loss": 0.0217,
      "losses/dpo": 0.005061456002295017,
      "losses/sft": 2.7621004581451416,
      "losses/total": 0.005061456002295017,
      "ref_logps/chosen": -144.57373046875,
      "ref_logps/rejected": -164.53269958496094,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.0306763648986816,
      "rewards/margins": 13.263851165771484,
      "rewards/rejected": -14.294527053833008,
      "step": 118
    },
    {
      "epoch": 0.22160148975791433,
      "grad_norm": 12.324630737304688,
      "learning_rate": 4.32712215320911e-06,
      "logps/chosen": -169.20716857910156,
      "logps/rejected": -327.8011474609375,
      "loss": 0.0692,
      "losses/dpo": 0.05636470764875412,
      "losses/sft": 2.663618326187134,
      "losses/total": 0.05636470764875412,
      "ref_logps/chosen": -141.14559936523438,
      "ref_logps/rejected": -169.51641845703125,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.806156635284424,
      "rewards/margins": 13.022315979003906,
      "rewards/rejected": -15.828473091125488,
      "step": 119
    },
    {
      "epoch": 0.22346368715083798,
      "grad_norm": 7.082119464874268,
      "learning_rate": 4.316770186335404e-06,
      "logps/chosen": -162.06655883789062,
      "logps/rejected": -312.4548645019531,
      "loss": 0.0211,
      "losses/dpo": 0.14828312397003174,
      "losses/sft": 2.6526529788970947,
      "losses/total": 0.14828312397003174,
      "ref_logps/chosen": -147.12899780273438,
      "ref_logps/rejected": -163.53634643554688,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.4937548637390137,
      "rewards/margins": 13.39809799194336,
      "rewards/rejected": -14.891853332519531,
      "step": 120
    },
    {
      "epoch": 0.22532588454376165,
      "grad_norm": 7.707842826843262,
      "learning_rate": 4.306418219461698e-06,
      "logps/chosen": -169.38192749023438,
      "logps/rejected": -323.21514892578125,
      "loss": 0.0198,
      "losses/dpo": 0.1473950445652008,
      "losses/sft": 2.6331777572631836,
      "losses/total": 0.1473950445652008,
      "ref_logps/chosen": -149.7891845703125,
      "ref_logps/rejected": -174.02752685546875,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.9592747688293457,
      "rewards/margins": 12.959487915039062,
      "rewards/rejected": -14.918761253356934,
      "step": 121
    },
    {
      "epoch": 0.2271880819366853,
      "grad_norm": 6.156428337097168,
      "learning_rate": 4.296066252587992e-06,
      "logps/chosen": -177.64190673828125,
      "logps/rejected": -320.0240478515625,
      "loss": 0.0313,
      "losses/dpo": 0.0023224614560604095,
      "losses/sft": 2.282505512237549,
      "losses/total": 0.0023224614560604095,
      "ref_logps/chosen": -150.99745178222656,
      "ref_logps/rejected": -166.0125732421875,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -2.664444923400879,
      "rewards/margins": 12.736701011657715,
      "rewards/rejected": -15.40114688873291,
      "step": 122
    },
    {
      "epoch": 0.22905027932960895,
      "grad_norm": 12.972006797790527,
      "learning_rate": 4.2857142857142855e-06,
      "logps/chosen": -171.61099243164062,
      "logps/rejected": -314.13824462890625,
      "loss": 0.204,
      "losses/dpo": 0.11684274673461914,
      "losses/sft": 2.5414156913757324,
      "losses/total": 0.11684274673461914,
      "ref_logps/chosen": -152.2470703125,
      "ref_logps/rejected": -169.70111083984375,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": -1.936392903327942,
      "rewards/margins": 12.507322311401367,
      "rewards/rejected": -14.443716049194336,
      "step": 123
    },
    {
      "epoch": 0.2309124767225326,
      "grad_norm": 8.865942001342773,
      "learning_rate": 4.27536231884058e-06,
      "logps/chosen": -178.2666015625,
      "logps/rejected": -294.51971435546875,
      "loss": 0.0887,
      "losses/dpo": 0.04358779639005661,
      "losses/sft": 2.905651569366455,
      "losses/total": 0.04358779639005661,
      "ref_logps/chosen": -146.95541381835938,
      "ref_logps/rejected": -152.81776428222656,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -3.1311190128326416,
      "rewards/margins": 11.039077758789062,
      "rewards/rejected": -14.170197486877441,
      "step": 124
    },
    {
      "epoch": 0.23277467411545624,
      "grad_norm": 14.640778541564941,
      "learning_rate": 4.265010351966874e-06,
      "logps/chosen": -176.9116668701172,
      "logps/rejected": -332.41241455078125,
      "loss": 0.1211,
      "losses/dpo": 0.053478606045246124,
      "losses/sft": 2.7518341541290283,
      "losses/total": 0.053478606045246124,
      "ref_logps/chosen": -150.49269104003906,
      "ref_logps/rejected": -171.48587036132812,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -2.641897678375244,
      "rewards/margins": 13.450759887695312,
      "rewards/rejected": -16.0926570892334,
      "step": 125
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 537,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 25,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}