{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9997862861464104,
  "eval_steps": 500,
  "global_step": 3801,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.00026303243518716404,
      "grad_norm": 478.0,
      "learning_rate": 1.3123359580052492e-09,
      "logits/chosen": -0.6613709926605225,
      "logits/rejected": -0.6454611420631409,
      "logps/chosen": -981.9267578125,
      "logps/rejected": -909.8477783203125,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0005260648703743281,
      "grad_norm": 660.0,
      "learning_rate": 2.6246719160104985e-09,
      "logits/chosen": -0.6495126485824585,
      "logits/rejected": NaN,
      "logps/chosen": -1577.01806640625,
      "logps/rejected": -1305.827880859375,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2
    },
    {
      "epoch": 0.000789097305561492,
      "grad_norm": 504.0,
      "learning_rate": 3.937007874015748e-09,
      "logits/chosen": -0.6738831996917725,
      "logits/rejected": -0.6792718172073364,
      "logps/chosen": -1083.833740234375,
      "logps/rejected": -1063.00634765625,
      "loss": 0.6929,
      "rewards/accuracies": 0.0625,
      "rewards/chosen": 0.0004379272577352822,
      "rewards/margins": 0.0004379272577352822,
      "rewards/rejected": 0.0,
      "step": 3
    },
    {
      "epoch": 0.0010521297407486562,
      "grad_norm": 496.0,
      "learning_rate": 5.249343832020997e-09,
      "logits/chosen": -0.6315386295318604,
      "logits/rejected": -0.6478630900382996,
      "logps/chosen": -1111.81201171875,
      "logps/rejected": -1094.9241943359375,
      "loss": 0.6945,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.0028894427232444286,
      "rewards/margins": -0.0025873186532408,
      "rewards/rejected": -0.0003021239535883069,
      "step": 4
    },
    {
      "epoch": 0.00131516217593582,
      "grad_norm": 620.0,
      "learning_rate": 6.5616797900262466e-09,
      "logits/chosen": -0.6531481146812439,
      "logits/rejected": -0.6536089777946472,
      "logps/chosen": -1197.532470703125,
      "logps/rejected": -978.9539184570312,
      "loss": 0.6908,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0008089067414402962,
      "rewards/margins": 0.004808234982192516,
      "rewards/rejected": -0.0039993287064135075,
      "step": 5
    },
    {
      "epoch": 0.001578194611122984,
      "grad_norm": 640.0,
      "learning_rate": 7.874015748031496e-09,
      "logits/chosen": -0.6561139225959778,
      "logits/rejected": -0.6512961387634277,
      "logps/chosen": -1431.3724365234375,
      "logps/rejected": -968.4022216796875,
      "loss": 0.6894,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0071239471435546875,
      "rewards/margins": 0.007528210058808327,
      "rewards/rejected": -0.00040426268242299557,
      "step": 6
    },
    {
      "epoch": 0.0018412270463101482,
      "grad_norm": 616.0,
      "learning_rate": 9.186351706036744e-09,
      "logits/chosen": -0.6538051962852478,
      "logits/rejected": -0.6452628970146179,
      "logps/chosen": -1510.80810546875,
      "logps/rejected": -1266.880859375,
      "loss": 0.7088,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.018686486408114433,
      "rewards/margins": -0.030672647058963776,
      "rewards/rejected": 0.011986159719526768,
      "step": 7
    },
    {
      "epoch": 0.0021042594814973123,
      "grad_norm": 486.0,
      "learning_rate": 1.0498687664041994e-08,
      "logits/chosen": -0.64730304479599,
      "logits/rejected": -0.6551339626312256,
      "logps/chosen": -1049.4765625,
      "logps/rejected": -816.0979614257812,
      "loss": 0.694,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.006467532832175493,
      "rewards/margins": -0.001501083024777472,
      "rewards/rejected": -0.004966449923813343,
      "step": 8
    },
    {
      "epoch": 0.0023672919166844763,
      "grad_norm": 624.0,
      "learning_rate": 1.1811023622047243e-08,
      "logits/chosen": -0.6746657490730286,
      "logits/rejected": -0.6785321831703186,
      "logps/chosen": -1235.0616455078125,
      "logps/rejected": -1268.2205810546875,
      "loss": 0.6918,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0045032505877316,
      "rewards/margins": 0.004022980574518442,
      "rewards/rejected": -0.008526228368282318,
      "step": 9
    },
    {
      "epoch": 0.00263032435187164,
      "grad_norm": 468.0,
      "learning_rate": 1.3123359580052493e-08,
      "logits/chosen": -0.6271215081214905,
      "logits/rejected": -0.6320958137512207,
      "logps/chosen": -965.6876220703125,
      "logps/rejected": -892.5921630859375,
      "loss": 0.686,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -8.745177183300257e-05,
      "rewards/margins": 0.01535253506153822,
      "rewards/rejected": -0.015439988113939762,
      "step": 10
    },
    {
      "epoch": 0.002893356787058804,
      "grad_norm": 520.0,
      "learning_rate": 1.4435695538057741e-08,
      "logits/chosen": -0.6479083299636841,
      "logits/rejected": -0.6545549631118774,
      "logps/chosen": -1173.121826171875,
      "logps/rejected": -634.2754516601562,
      "loss": 0.6875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.011095333844423294,
      "rewards/margins": 0.011432838626205921,
      "rewards/rejected": -0.0003375053056515753,
      "step": 11
    },
    {
      "epoch": 0.003156389222245968,
      "grad_norm": 506.0,
      "learning_rate": 1.5748031496062992e-08,
      "logits/chosen": -0.6462807059288025,
      "logits/rejected": -0.6522711515426636,
      "logps/chosen": -1166.423583984375,
      "logps/rejected": -969.1693725585938,
      "loss": 0.6974,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.015190506353974342,
      "rewards/margins": -0.00775136798620224,
      "rewards/rejected": -0.00743913697078824,
      "step": 12
    },
    {
      "epoch": 0.0034194216574331324,
      "grad_norm": 548.0,
      "learning_rate": 1.706036745406824e-08,
      "logits/chosen": -0.6476821899414062,
      "logits/rejected": -0.651539146900177,
      "logps/chosen": -1084.5450439453125,
      "logps/rejected": -1102.34912109375,
      "loss": 0.6888,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0026543617714196444,
      "rewards/margins": 0.009313344955444336,
      "rewards/rejected": -0.006658983416855335,
      "step": 13
    },
    {
      "epoch": 0.0036824540926202964,
      "grad_norm": 568.0,
      "learning_rate": 1.837270341207349e-08,
      "logits/chosen": -0.6566973924636841,
      "logits/rejected": -0.6482314467430115,
      "logps/chosen": -1032.5177001953125,
      "logps/rejected": -1003.5614013671875,
      "loss": 0.6965,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.001129150390625,
      "rewards/margins": -0.005954647436738014,
      "rewards/rejected": 0.004825496580451727,
      "step": 14
    },
    {
      "epoch": 0.00394548652780746,
      "grad_norm": 584.0,
      "learning_rate": 1.968503937007874e-08,
      "logits/chosen": -0.6388291716575623,
      "logits/rejected": -0.6378220319747925,
      "logps/chosen": -1331.908447265625,
      "logps/rejected": -836.6995239257812,
      "loss": 0.7063,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.021807577461004257,
      "rewards/margins": -0.025357913225889206,
      "rewards/rejected": 0.00355033902451396,
      "step": 15
    },
    {
      "epoch": 0.004208518962994625,
      "grad_norm": 324.0,
      "learning_rate": 2.0997375328083988e-08,
      "logits/chosen": -0.6582619547843933,
      "logits/rejected": -0.6583358645439148,
      "logps/chosen": -468.46240234375,
      "logps/rejected": -572.666259765625,
      "loss": 0.6935,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0031709668692201376,
      "rewards/margins": -0.0004069809801876545,
      "rewards/rejected": -0.0027639856562018394,
      "step": 16
    },
    {
      "epoch": 0.004471551398181789,
      "grad_norm": 442.0,
      "learning_rate": 2.230971128608924e-08,
      "logits/chosen": -0.6362031698226929,
      "logits/rejected": -0.6343502998352051,
      "logps/chosen": -1079.895263671875,
      "logps/rejected": -774.8839111328125,
      "loss": 0.6981,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.005643177777528763,
      "rewards/margins": -0.009376143105328083,
      "rewards/rejected": 0.01501932181417942,
      "step": 17
    },
    {
      "epoch": 0.0047345838333689525,
      "grad_norm": 440.0,
      "learning_rate": 2.3622047244094487e-08,
      "logits/chosen": -0.6603103280067444,
      "logits/rejected": -0.6537511348724365,
      "logps/chosen": -1293.02001953125,
      "logps/rejected": -775.8257446289062,
      "loss": 0.6997,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.022523973137140274,
      "rewards/margins": -0.012728214263916016,
      "rewards/rejected": -0.009795760735869408,
      "step": 18
    },
    {
      "epoch": 0.0049976162685561164,
      "grad_norm": 478.0,
      "learning_rate": 2.4934383202099735e-08,
      "logits/chosen": -0.634665310382843,
      "logits/rejected": -0.6290925741195679,
      "logps/chosen": -1188.8642578125,
      "logps/rejected": -953.523193359375,
      "loss": 0.6885,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004985428415238857,
      "rewards/margins": 0.010046958923339844,
      "rewards/rejected": -0.005061530973762274,
      "step": 19
    },
    {
      "epoch": 0.00526064870374328,
      "grad_norm": 764.0,
      "learning_rate": 2.6246719160104986e-08,
      "logits/chosen": -0.6461503505706787,
      "logits/rejected": -0.64608234167099,
      "logps/chosen": -1323.8428955078125,
      "logps/rejected": -1102.487060546875,
      "loss": 0.7022,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0067094797268509865,
      "rewards/margins": -0.01769733428955078,
      "rewards/rejected": 0.010987851768732071,
      "step": 20
    },
    {
      "epoch": 0.005523681138930444,
      "grad_norm": 426.0,
      "learning_rate": 2.7559055118110234e-08,
      "logits/chosen": -0.6227860450744629,
      "logits/rejected": -0.6390319466590881,
      "logps/chosen": -789.1991577148438,
      "logps/rejected": -582.417236328125,
      "loss": 0.6893,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0020312308333814144,
      "rewards/margins": 0.008088779635727406,
      "rewards/rejected": -0.010120010934770107,
      "step": 21
    },
    {
      "epoch": 0.005786713574117608,
      "grad_norm": 620.0,
      "learning_rate": 2.8871391076115482e-08,
      "logits/chosen": -0.6394176483154297,
      "logits/rejected": -0.6542004942893982,
      "logps/chosen": -1395.189453125,
      "logps/rejected": -983.6468505859375,
      "loss": 0.6948,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.007534313946962357,
      "rewards/margins": -0.0026789666153490543,
      "rewards/rejected": -0.004855346865952015,
      "step": 22
    },
    {
      "epoch": 0.006049746009304772,
      "grad_norm": 600.0,
      "learning_rate": 3.0183727034120734e-08,
      "logits/chosen": -0.6426993608474731,
      "logits/rejected": -0.6616034507751465,
      "logps/chosen": -1344.54345703125,
      "logps/rejected": -886.1381225585938,
      "loss": 0.7096,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.04793548583984375,
      "rewards/margins": -0.031595516949892044,
      "rewards/rejected": -0.016339968889951706,
      "step": 23
    },
    {
      "epoch": 0.006312778444491936,
      "grad_norm": 432.0,
      "learning_rate": 3.1496062992125985e-08,
      "logits/chosen": -0.645313560962677,
      "logits/rejected": -0.6529985070228577,
      "logps/chosen": -1068.8375244140625,
      "logps/rejected": -931.7603149414062,
      "loss": 0.6861,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.009414147585630417,
      "rewards/margins": 0.014580106362700462,
      "rewards/rejected": -0.005165957845747471,
      "step": 24
    },
    {
      "epoch": 0.0065758108796791,
      "grad_norm": 580.0,
      "learning_rate": 3.280839895013123e-08,
      "logits/chosen": -0.6209381222724915,
      "logits/rejected": -0.6185789108276367,
      "logps/chosen": -1059.4046630859375,
      "logps/rejected": -849.1358032226562,
      "loss": 0.6942,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0033525472972542048,
      "rewards/margins": -0.0014457228826358914,
      "rewards/rejected": 0.004798269364982843,
      "step": 25
    },
    {
      "epoch": 0.006838843314866265,
      "grad_norm": 516.0,
      "learning_rate": 3.412073490813648e-08,
      "logits/chosen": -0.6207571029663086,
      "logits/rejected": -0.6236509084701538,
      "logps/chosen": -1143.9339599609375,
      "logps/rejected": -784.5245361328125,
      "loss": 0.6952,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.011875439435243607,
      "rewards/margins": -0.0026565538719296455,
      "rewards/rejected": -0.009218885563313961,
      "step": 26
    },
    {
      "epoch": 0.007101875750053429,
      "grad_norm": 424.0,
      "learning_rate": 3.543307086614173e-08,
      "logits/chosen": -0.6514330506324768,
      "logits/rejected": -0.6530667543411255,
      "logps/chosen": -773.1522216796875,
      "logps/rejected": -509.62176513671875,
      "loss": 0.6962,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01085586566478014,
      "rewards/margins": -0.005617333110421896,
      "rewards/rejected": -0.005238533020019531,
      "step": 27
    },
    {
      "epoch": 0.007364908185240593,
      "grad_norm": 564.0,
      "learning_rate": 3.674540682414698e-08,
      "logits/chosen": -0.625694990158081,
      "logits/rejected": -0.6209042072296143,
      "logps/chosen": -1010.1669921875,
      "logps/rejected": -1003.019775390625,
      "loss": 0.6878,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0008529662154614925,
      "rewards/margins": 0.011435603722929955,
      "rewards/rejected": -0.012288570404052734,
      "step": 28
    },
    {
      "epoch": 0.007627940620427757,
      "grad_norm": 628.0,
      "learning_rate": 3.805774278215223e-08,
      "logits/chosen": -0.6242140531539917,
      "logits/rejected": -0.625519871711731,
      "logps/chosen": -1141.395751953125,
      "logps/rejected": -908.4832763671875,
      "loss": 0.6858,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0075996387749910355,
      "rewards/margins": 0.015088795684278011,
      "rewards/rejected": -0.007489155977964401,
      "step": 29
    },
    {
      "epoch": 0.00789097305561492,
      "grad_norm": 524.0,
      "learning_rate": 3.937007874015748e-08,
      "logits/chosen": -0.6577723622322083,
      "logits/rejected": -0.6493313312530518,
      "logps/chosen": -1136.755859375,
      "logps/rejected": -835.238037109375,
      "loss": 0.6999,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.008542442694306374,
      "rewards/margins": -0.012964009307324886,
      "rewards/rejected": 0.004421568010002375,
      "step": 30
    },
    {
      "epoch": 0.008154005490802085,
      "grad_norm": 920.0,
      "learning_rate": 4.0682414698162724e-08,
      "logits/chosen": -0.6406484842300415,
      "logits/rejected": -0.6521078944206238,
      "logps/chosen": -1169.0692138671875,
      "logps/rejected": -1033.1300048828125,
      "loss": 0.683,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.010937213897705078,
      "rewards/margins": 0.020925331860780716,
      "rewards/rejected": -0.009988117963075638,
      "step": 31
    },
    {
      "epoch": 0.00841703792598925,
      "grad_norm": 568.0,
      "learning_rate": 4.1994750656167975e-08,
      "logits/chosen": -0.6679829359054565,
      "logits/rejected": -0.6634407639503479,
      "logps/chosen": -1217.44677734375,
      "logps/rejected": -821.7205810546875,
      "loss": 0.6862,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.020354462787508965,
      "rewards/margins": 0.015439844690263271,
      "rewards/rejected": 0.004914618097245693,
      "step": 32
    },
    {
      "epoch": 0.008680070361176413,
      "grad_norm": 462.0,
      "learning_rate": 4.330708661417323e-08,
      "logits/chosen": -0.6383431553840637,
      "logits/rejected": -0.6523004770278931,
      "logps/chosen": -977.1473999023438,
      "logps/rejected": -814.833984375,
      "loss": 0.6888,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0013907907996326685,
      "rewards/margins": 0.009458303451538086,
      "rewards/rejected": -0.008067512884736061,
      "step": 33
    },
    {
      "epoch": 0.008943102796363577,
      "grad_norm": 576.0,
      "learning_rate": 4.461942257217848e-08,
      "logits/chosen": -0.665785551071167,
      "logits/rejected": -0.6694854497909546,
      "logps/chosen": -1209.923583984375,
      "logps/rejected": -1071.947265625,
      "loss": 0.7048,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0017922879196703434,
      "rewards/margins": -0.022022390738129616,
      "rewards/rejected": 0.02023010328412056,
      "step": 34
    },
    {
      "epoch": 0.009206135231550741,
      "grad_norm": 544.0,
      "learning_rate": 4.593175853018372e-08,
      "logits/chosen": -0.6515253186225891,
      "logits/rejected": -0.6593830585479736,
      "logps/chosen": -1206.0894775390625,
      "logps/rejected": -793.4617309570312,
      "loss": 0.6887,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.008255863562226295,
      "rewards/margins": 0.009493446908891201,
      "rewards/rejected": -0.017749309539794922,
      "step": 35
    },
    {
      "epoch": 0.009469167666737905,
      "grad_norm": 540.0,
      "learning_rate": 4.7244094488188974e-08,
      "logits/chosen": -0.6429049372673035,
      "logits/rejected": -0.6440693140029907,
      "logps/chosen": -1196.6849365234375,
      "logps/rejected": -782.5664672851562,
      "loss": 0.6854,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.005351925268769264,
      "rewards/margins": 0.01674356311559677,
      "rewards/rejected": -0.022095490247011185,
      "step": 36
    },
    {
      "epoch": 0.009732200101925069,
      "grad_norm": 536.0,
      "learning_rate": 4.8556430446194225e-08,
      "logits/chosen": -0.6473021507263184,
      "logits/rejected": -0.6601806282997131,
      "logps/chosen": -1131.809814453125,
      "logps/rejected": -921.2569580078125,
      "loss": 0.682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.007221412844955921,
      "rewards/margins": 0.02343463897705078,
      "rewards/rejected": -0.030656052753329277,
      "step": 37
    },
    {
      "epoch": 0.009995232537112233,
      "grad_norm": 676.0,
      "learning_rate": 4.986876640419947e-08,
      "logits/chosen": -0.6587386727333069,
      "logits/rejected": -0.6617957949638367,
      "logps/chosen": -1572.87255859375,
      "logps/rejected": -1032.41259765625,
      "loss": 0.7027,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0022861487232148647,
      "rewards/margins": -0.01829981803894043,
      "rewards/rejected": 0.016013670712709427,
      "step": 38
    },
    {
      "epoch": 0.010258264972299397,
      "grad_norm": 636.0,
      "learning_rate": 5.118110236220472e-08,
      "logits/chosen": -0.6436494588851929,
      "logits/rejected": -0.652623176574707,
      "logps/chosen": -1101.4393310546875,
      "logps/rejected": -982.1671752929688,
      "loss": 0.7014,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03189497068524361,
      "rewards/margins": -0.015912534669041634,
      "rewards/rejected": -0.015982437878847122,
      "step": 39
    },
    {
      "epoch": 0.01052129740748656,
      "grad_norm": 572.0,
      "learning_rate": 5.249343832020997e-08,
      "logits/chosen": -0.6690911054611206,
      "logits/rejected": -0.6645552515983582,
      "logps/chosen": -1055.1851806640625,
      "logps/rejected": -815.9608154296875,
      "loss": 0.6776,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0016391761600971222,
      "rewards/margins": 0.032398127019405365,
      "rewards/rejected": -0.03403730317950249,
      "step": 40
    },
    {
      "epoch": 0.010784329842673725,
      "grad_norm": 560.0,
      "learning_rate": 5.380577427821522e-08,
      "logits/chosen": -0.6480713486671448,
      "logits/rejected": -0.6492923498153687,
      "logps/chosen": -1122.8189697265625,
      "logps/rejected": -578.2962036132812,
      "loss": 0.6904,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004717065021395683,
      "rewards/margins": 0.00614252220839262,
      "rewards/rejected": -0.0014254569541662931,
      "step": 41
    },
    {
      "epoch": 0.011047362277860889,
      "grad_norm": 564.0,
      "learning_rate": 5.511811023622047e-08,
      "logits/chosen": -0.6463696956634521,
      "logits/rejected": -0.6736325025558472,
      "logps/chosen": -1272.1763916015625,
      "logps/rejected": -981.54541015625,
      "loss": 0.6896,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.013799857348203659,
      "rewards/margins": 0.0075726984068751335,
      "rewards/rejected": 0.006227159406989813,
      "step": 42
    },
    {
      "epoch": 0.011310394713048053,
      "grad_norm": 552.0,
      "learning_rate": 5.643044619422572e-08,
      "logits/chosen": -0.6339900493621826,
      "logits/rejected": NaN,
      "logps/chosen": -1261.74951171875,
      "logps/rejected": -916.6941528320312,
      "loss": 0.6991,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0013702396536245942,
      "rewards/margins": -0.011715412139892578,
      "rewards/rejected": 0.010345172137022018,
      "step": 43
    },
    {
      "epoch": 0.011573427148235216,
      "grad_norm": 476.0,
      "learning_rate": 5.7742782152230965e-08,
      "logits/chosen": -0.6421727538108826,
      "logits/rejected": -0.6539686918258667,
      "logps/chosen": -965.207763671875,
      "logps/rejected": -838.1019897460938,
      "loss": 0.6989,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0031885148491710424,
      "rewards/margins": -0.01107177883386612,
      "rewards/rejected": 0.007883261889219284,
      "step": 44
    },
    {
      "epoch": 0.01183645958342238,
      "grad_norm": 496.0,
      "learning_rate": 5.9055118110236216e-08,
      "logits/chosen": -0.6374115347862244,
      "logits/rejected": -0.6306883096694946,
      "logps/chosen": -1024.2801513671875,
      "logps/rejected": -842.59228515625,
      "loss": 0.6953,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.00438423128798604,
      "rewards/margins": -0.0036731716245412827,
      "rewards/rejected": 0.00805740337818861,
      "step": 45
    },
    {
      "epoch": 0.012099492018609544,
      "grad_norm": 486.0,
      "learning_rate": 6.036745406824147e-08,
      "logits/chosen": -0.6535022258758545,
      "logits/rejected": -0.6586615443229675,
      "logps/chosen": -955.3035888671875,
      "logps/rejected": -714.7905883789062,
      "loss": 0.6917,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0009829518385231495,
      "rewards/margins": 0.0033273701556026936,
      "rewards/rejected": -0.004310321062803268,
      "step": 46
    },
    {
      "epoch": 0.012362524453796708,
      "grad_norm": 596.0,
      "learning_rate": 6.167979002624672e-08,
      "logits/chosen": -0.6287187337875366,
      "logits/rejected": -0.6289898753166199,
      "logps/chosen": -1154.611328125,
      "logps/rejected": -905.4376220703125,
      "loss": 0.7003,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.016869354993104935,
      "rewards/margins": -0.013419723138213158,
      "rewards/rejected": -0.003449631156399846,
      "step": 47
    },
    {
      "epoch": 0.012625556888983872,
      "grad_norm": 708.0,
      "learning_rate": 6.299212598425197e-08,
      "logits/chosen": -0.6427187919616699,
      "logits/rejected": -0.6474528312683105,
      "logps/chosen": -1228.32958984375,
      "logps/rejected": -986.4098510742188,
      "loss": 0.7053,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.005676841828972101,
      "rewards/margins": -0.02215537801384926,
      "rewards/rejected": 0.016478538513183594,
      "step": 48
    },
    {
      "epoch": 0.012888589324171036,
      "grad_norm": 516.0,
      "learning_rate": 6.430446194225722e-08,
      "logits/chosen": -0.6678164601325989,
      "logits/rejected": NaN,
      "logps/chosen": -1425.896728515625,
      "logps/rejected": -1257.0703125,
      "loss": 0.6917,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0022273065987974405,
      "rewards/margins": 0.003320979420095682,
      "rewards/rejected": -0.0010936741018667817,
      "step": 49
    },
    {
      "epoch": 0.0131516217593582,
      "grad_norm": 624.0,
      "learning_rate": 6.561679790026246e-08,
      "logits/chosen": -0.6566253304481506,
      "logits/rejected": -0.6389565467834473,
      "logps/chosen": -1115.861328125,
      "logps/rejected": -987.3572998046875,
      "loss": 0.6909,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.001755428034812212,
      "rewards/margins": 0.004685831256210804,
      "rewards/rejected": -0.006441259756684303,
      "step": 50
    },
    {
      "epoch": 0.013414654194545366,
      "grad_norm": 548.0,
      "learning_rate": 6.692913385826772e-08,
      "logits/chosen": -0.650773286819458,
      "logits/rejected": -0.6537303924560547,
      "logps/chosen": -1203.6383056640625,
      "logps/rejected": -1010.1041870117188,
      "loss": 0.6838,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.030438615009188652,
      "rewards/margins": 0.020061397925019264,
      "rewards/rejected": 0.010377216152846813,
      "step": 51
    },
    {
      "epoch": 0.01367768662973253,
      "grad_norm": 528.0,
      "learning_rate": 6.824146981627296e-08,
      "logits/chosen": -0.6454442143440247,
      "logits/rejected": -0.6545051336288452,
      "logps/chosen": -1138.6746826171875,
      "logps/rejected": -746.0814819335938,
      "loss": 0.6822,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0022541042417287827,
      "rewards/margins": 0.022426988929510117,
      "rewards/rejected": -0.020172882825136185,
      "step": 52
    },
    {
      "epoch": 0.013940719064919694,
      "grad_norm": 680.0,
      "learning_rate": 6.955380577427821e-08,
      "logits/chosen": -0.6754468679428101,
      "logits/rejected": -0.6692649126052856,
      "logps/chosen": -1030.5341796875,
      "logps/rejected": -1068.36279296875,
      "loss": 0.704,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.011090278625488281,
      "rewards/margins": -0.02095632627606392,
      "rewards/rejected": 0.009866046719253063,
      "step": 53
    },
    {
      "epoch": 0.014203751500106858,
      "grad_norm": 596.0,
      "learning_rate": 7.086614173228346e-08,
      "logits/chosen": -0.6554870009422302,
      "logits/rejected": -0.6556934714317322,
      "logps/chosen": -1174.1236572265625,
      "logps/rejected": -1065.7628173828125,
      "loss": 0.713,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.015700720250606537,
      "rewards/margins": -0.03826656565070152,
      "rewards/rejected": 0.022565841674804688,
      "step": 54
    },
    {
      "epoch": 0.014466783935294021,
      "grad_norm": 468.0,
      "learning_rate": 7.217847769028872e-08,
      "logits/chosen": -0.6528524160385132,
      "logits/rejected": -0.6453327536582947,
      "logps/chosen": -1241.2333984375,
      "logps/rejected": -1351.5711669921875,
      "loss": 0.6911,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.004076672717928886,
      "rewards/margins": 0.005133152939379215,
      "rewards/rejected": -0.009209822863340378,
      "step": 55
    },
    {
      "epoch": 0.014729816370481185,
      "grad_norm": 556.0,
      "learning_rate": 7.349081364829395e-08,
      "logits/chosen": -0.6388370990753174,
      "logits/rejected": -0.6383394002914429,
      "logps/chosen": -1200.1177978515625,
      "logps/rejected": -931.9652099609375,
      "loss": 0.7035,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.006584453862160444,
      "rewards/margins": -0.020203303545713425,
      "rewards/rejected": 0.013618851080536842,
      "step": 56
    },
    {
      "epoch": 0.01499284880566835,
      "grad_norm": 494.0,
      "learning_rate": 7.480314960629922e-08,
      "logits/chosen": -0.6486294269561768,
      "logits/rejected": -0.6614699363708496,
      "logps/chosen": -1119.9052734375,
      "logps/rejected": -651.143310546875,
      "loss": 0.6927,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.013533398509025574,
      "rewards/margins": 0.0016348820645362139,
      "rewards/rejected": 0.011898518539965153,
      "step": 57
    },
    {
      "epoch": 0.015255881240855513,
      "grad_norm": 656.0,
      "learning_rate": 7.611548556430446e-08,
      "logits/chosen": -0.6650611162185669,
      "logits/rejected": -0.6634715795516968,
      "logps/chosen": -1292.6005859375,
      "logps/rejected": -1205.146484375,
      "loss": 0.7103,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.02310619316995144,
      "rewards/margins": -0.03276882320642471,
      "rewards/rejected": 0.009662628173828125,
      "step": 58
    },
    {
      "epoch": 0.015518913676042677,
      "grad_norm": 684.0,
      "learning_rate": 7.742782152230971e-08,
      "logits/chosen": -0.6696011424064636,
      "logits/rejected": -0.6645325422286987,
      "logps/chosen": -1355.49072265625,
      "logps/rejected": -1272.9954833984375,
      "loss": 0.6934,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.000744151882827282,
      "rewards/margins": 0.0007336623966693878,
      "rewards/rejected": -0.0014778142794966698,
      "step": 59
    },
    {
      "epoch": 0.01578194611122984,
      "grad_norm": 520.0,
      "learning_rate": 7.874015748031496e-08,
      "logits/chosen": -0.6433677673339844,
      "logits/rejected": NaN,
      "logps/chosen": -1225.6917724609375,
      "logps/rejected": -883.4288940429688,
      "loss": 0.6949,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.012225532904267311,
      "rewards/margins": -0.002804471179842949,
      "rewards/rejected": -0.009421064518392086,
      "step": 60
    },
    {
      "epoch": 0.016044978546417003,
      "grad_norm": 404.0,
      "learning_rate": 8.005249343832021e-08,
      "logits/chosen": -0.6390970945358276,
      "logits/rejected": -0.6425827741622925,
      "logps/chosen": -807.92041015625,
      "logps/rejected": -823.8240966796875,
      "loss": 0.6868,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.002811240265145898,
      "rewards/margins": 0.013090752996504307,
      "rewards/rejected": -0.015901993960142136,
      "step": 61
    },
    {
      "epoch": 0.01630801098160417,
      "grad_norm": 556.0,
      "learning_rate": 8.136482939632545e-08,
      "logits/chosen": -0.6272599697113037,
      "logits/rejected": -0.629261314868927,
      "logps/chosen": -1317.435546875,
      "logps/rejected": -1185.9410400390625,
      "loss": 0.6961,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.003891182830557227,
      "rewards/margins": -0.005352020263671875,
      "rewards/rejected": 0.009243203327059746,
      "step": 62
    },
    {
      "epoch": 0.016571043416791335,
      "grad_norm": 466.0,
      "learning_rate": 8.267716535433071e-08,
      "logits/chosen": -0.6634899377822876,
      "logits/rejected": -0.6637872457504272,
      "logps/chosen": -747.8087158203125,
      "logps/rejected": -771.508056640625,
      "loss": 0.6921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004482840653508902,
      "rewards/margins": 0.0031440751627087593,
      "rewards/rejected": 0.0013387682847678661,
      "step": 63
    },
    {
      "epoch": 0.0168340758519785,
      "grad_norm": 556.0,
      "learning_rate": 8.398950131233595e-08,
      "logits/chosen": -0.6436450481414795,
      "logits/rejected": -0.6408189535140991,
      "logps/chosen": -1075.224853515625,
      "logps/rejected": -1025.9599609375,
      "loss": 0.6806,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.019418049603700638,
      "rewards/margins": 0.0264376662671566,
      "rewards/rejected": -0.0070196157321333885,
      "step": 64
    },
    {
      "epoch": 0.017097108287165663,
      "grad_norm": 536.0,
      "learning_rate": 8.530183727034122e-08,
      "logits/chosen": -0.6269121170043945,
      "logits/rejected": -0.6410995721817017,
      "logps/chosen": -1229.12109375,
      "logps/rejected": -1044.9771728515625,
      "loss": 0.6928,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.003551006782799959,
      "rewards/margins": 0.0009949689265340567,
      "rewards/rejected": 0.0025560371577739716,
      "step": 65
    },
    {
      "epoch": 0.017360140722352826,
      "grad_norm": 440.0,
      "learning_rate": 8.661417322834645e-08,
      "logits/chosen": -0.6630584597587585,
      "logits/rejected": -0.6530246138572693,
      "logps/chosen": -864.6857299804688,
      "logps/rejected": -614.0665283203125,
      "loss": 0.6855,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.005029201507568359,
      "rewards/margins": 0.016410063952207565,
      "rewards/rejected": -0.021439265459775925,
      "step": 66
    },
    {
      "epoch": 0.01762317315753999,
      "grad_norm": 600.0,
      "learning_rate": 8.79265091863517e-08,
      "logits/chosen": -0.6476324796676636,
      "logits/rejected": -0.6430042386054993,
      "logps/chosen": -1207.0330810546875,
      "logps/rejected": -1052.9666748046875,
      "loss": 0.706,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 1.1444091796875e-05,
      "rewards/margins": -0.02449226565659046,
      "rewards/rejected": 0.024503707885742188,
      "step": 67
    },
    {
      "epoch": 0.017886205592727154,
      "grad_norm": 470.0,
      "learning_rate": 8.923884514435696e-08,
      "logits/chosen": -0.6667293310165405,
      "logits/rejected": -0.6717281937599182,
      "logps/chosen": -874.5230102539062,
      "logps/rejected": -780.01611328125,
      "loss": 0.7049,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.009463788010179996,
      "rewards/margins": -0.02278413623571396,
      "rewards/rejected": 0.013320351019501686,
      "step": 68
    },
    {
      "epoch": 0.018149238027914318,
      "grad_norm": 460.0,
      "learning_rate": 9.055118110236221e-08,
      "logits/chosen": -0.6590813994407654,
      "logits/rejected": -0.6604346632957458,
      "logps/chosen": -991.8992919921875,
      "logps/rejected": -971.2369384765625,
      "loss": 0.6855,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0011934288777410984,
      "rewards/margins": 0.016167832538485527,
      "rewards/rejected": -0.017361260950565338,
      "step": 69
    },
    {
      "epoch": 0.018412270463101482,
      "grad_norm": 548.0,
      "learning_rate": 9.186351706036745e-08,
      "logits/chosen": -0.6486383080482483,
      "logits/rejected": -0.6490275263786316,
      "logps/chosen": -1020.6946411132812,
      "logps/rejected": -975.8842163085938,
      "loss": 0.6993,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009555434808135033,
      "rewards/margins": -0.011353873647749424,
      "rewards/rejected": 0.0017984382575377822,
      "step": 70
    },
    {
      "epoch": 0.018675302898288646,
      "grad_norm": 516.0,
      "learning_rate": 9.317585301837271e-08,
      "logits/chosen": -0.6590476036071777,
      "logits/rejected": -0.6603131294250488,
      "logps/chosen": -946.580322265625,
      "logps/rejected": -810.2112426757812,
      "loss": 0.6743,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.026568125933408737,
      "rewards/margins": 0.03902635723352432,
      "rewards/rejected": -0.012458228506147861,
      "step": 71
    },
    {
      "epoch": 0.01893833533347581,
      "grad_norm": 552.0,
      "learning_rate": 9.448818897637795e-08,
      "logits/chosen": -0.6483647227287292,
      "logits/rejected": -0.6447436809539795,
      "logps/chosen": -1237.6767578125,
      "logps/rejected": -899.4059448242188,
      "loss": 0.6784,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.013244818896055222,
      "rewards/margins": 0.030155183747410774,
      "rewards/rejected": -0.016910362988710403,
      "step": 72
    },
    {
      "epoch": 0.019201367768662974,
      "grad_norm": 500.0,
      "learning_rate": 9.58005249343832e-08,
      "logits/chosen": -0.6332172155380249,
      "logits/rejected": -0.6465437412261963,
      "logps/chosen": -1194.697265625,
      "logps/rejected": -930.92041015625,
      "loss": 0.6995,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0016160011291503906,
      "rewards/margins": -0.012108756229281425,
      "rewards/rejected": 0.010492755100131035,
      "step": 73
    },
    {
      "epoch": 0.019464400203850138,
      "grad_norm": 676.0,
      "learning_rate": 9.711286089238845e-08,
      "logits/chosen": -0.6632128357887268,
      "logits/rejected": -0.6675799489021301,
      "logps/chosen": -961.5637817382812,
      "logps/rejected": -870.0667114257812,
      "loss": 0.7072,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.008633995428681374,
      "rewards/margins": -0.02640991285443306,
      "rewards/rejected": 0.017775917425751686,
      "step": 74
    },
    {
      "epoch": 0.019727432639037302,
      "grad_norm": 556.0,
      "learning_rate": 9.84251968503937e-08,
      "logits/chosen": -0.6558970808982849,
      "logits/rejected": -0.6618886590003967,
      "logps/chosen": -1606.664306640625,
      "logps/rejected": -962.390380859375,
      "loss": 0.6957,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.013555527664721012,
      "rewards/margins": -0.004396628588438034,
      "rewards/rejected": -0.009158897213637829,
      "step": 75
    },
    {
      "epoch": 0.019990465074224466,
      "grad_norm": 576.0,
      "learning_rate": 9.973753280839894e-08,
      "logits/chosen": -0.6512588858604431,
      "logits/rejected": -0.6462149620056152,
      "logps/chosen": -1067.4432373046875,
      "logps/rejected": -749.668701171875,
      "loss": 0.693,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.015813350677490234,
      "rewards/margins": 0.0014084805734455585,
      "rewards/rejected": -0.017221832647919655,
      "step": 76
    },
    {
      "epoch": 0.02025349750941163,
      "grad_norm": 468.0,
      "learning_rate": 1.010498687664042e-07,
      "logits/chosen": -0.652782678604126,
      "logits/rejected": -0.654413104057312,
      "logps/chosen": -1070.6588134765625,
      "logps/rejected": -730.8727416992188,
      "loss": 0.7035,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.009283351711928844,
      "rewards/margins": -0.020475197583436966,
      "rewards/rejected": 0.029758550226688385,
      "step": 77
    },
    {
      "epoch": 0.020516529944598794,
      "grad_norm": 440.0,
      "learning_rate": 1.0236220472440944e-07,
      "logits/chosen": -0.6675430536270142,
      "logits/rejected": -0.6817749738693237,
      "logps/chosen": -854.1362915039062,
      "logps/rejected": -431.08270263671875,
      "loss": 0.6891,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0006029130890965462,
      "rewards/margins": 0.008838795125484467,
      "rewards/rejected": -0.008235884830355644,
      "step": 78
    },
    {
      "epoch": 0.020779562379785958,
      "grad_norm": 512.0,
      "learning_rate": 1.036745406824147e-07,
      "logits/chosen": -0.641825258731842,
      "logits/rejected": -0.6594170331954956,
      "logps/chosen": -994.7021484375,
      "logps/rejected": -520.4114990234375,
      "loss": 0.6896,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0034171096049249172,
      "rewards/margins": 0.00791168212890625,
      "rewards/rejected": -0.011328792199492455,
      "step": 79
    },
    {
      "epoch": 0.02104259481497312,
      "grad_norm": 604.0,
      "learning_rate": 1.0498687664041995e-07,
      "logits/chosen": -0.6439292430877686,
      "logits/rejected": -0.6462897658348083,
      "logps/chosen": -1077.8985595703125,
      "logps/rejected": -815.918212890625,
      "loss": 0.6982,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0018968585645779967,
      "rewards/margins": -0.009403801523149014,
      "rewards/rejected": 0.007506943307816982,
      "step": 80
    },
    {
      "epoch": 0.021305627250160285,
      "grad_norm": 572.0,
      "learning_rate": 1.062992125984252e-07,
      "logits/chosen": -0.6614235043525696,
      "logits/rejected": -0.6678154468536377,
      "logps/chosen": -1220.45263671875,
      "logps/rejected": -1016.094970703125,
      "loss": 0.683,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.017964934930205345,
      "rewards/margins": 0.021844863891601562,
      "rewards/rejected": -0.0038799280300736427,
      "step": 81
    },
    {
      "epoch": 0.02156865968534745,
      "grad_norm": 568.0,
      "learning_rate": 1.0761154855643043e-07,
      "logits/chosen": -0.6574364900588989,
      "logits/rejected": -0.6648485064506531,
      "logps/chosen": -1384.743896484375,
      "logps/rejected": -953.017578125,
      "loss": 0.7029,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.015233231708407402,
      "rewards/margins": -0.017672158777713776,
      "rewards/rejected": 0.0024389266036450863,
      "step": 82
    },
    {
      "epoch": 0.021831692120534613,
      "grad_norm": 560.0,
      "learning_rate": 1.089238845144357e-07,
      "logits/chosen": -0.6327518224716187,
      "logits/rejected": -0.6306644678115845,
      "logps/chosen": -1179.177490234375,
      "logps/rejected": -1162.572998046875,
      "loss": 0.7041,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0371156707406044,
      "rewards/margins": -0.020284557715058327,
      "rewards/rejected": -0.016831113025546074,
      "step": 83
    },
    {
      "epoch": 0.022094724555721777,
      "grad_norm": 468.0,
      "learning_rate": 1.1023622047244094e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6731138825416565,
      "logps/chosen": -768.2457885742188,
      "logps/rejected": -549.1614990234375,
      "loss": 0.6992,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.013354492373764515,
      "rewards/margins": -0.011455915868282318,
      "rewards/rejected": -0.0018985755741596222,
      "step": 84
    },
    {
      "epoch": 0.02235775699090894,
      "grad_norm": 596.0,
      "learning_rate": 1.1154855643044619e-07,
      "logits/chosen": -0.6745445728302002,
      "logits/rejected": -0.6721563339233398,
      "logps/chosen": -1450.4288330078125,
      "logps/rejected": -956.8093872070312,
      "loss": 0.6946,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.015703583136200905,
      "rewards/margins": -0.0018576616421341896,
      "rewards/rejected": -0.01384592056274414,
      "step": 85
    },
    {
      "epoch": 0.022620789426096105,
      "grad_norm": 604.0,
      "learning_rate": 1.1286089238845144e-07,
      "logits/chosen": -0.652224063873291,
      "logits/rejected": -0.654421865940094,
      "logps/chosen": -1159.897216796875,
      "logps/rejected": -857.185791015625,
      "loss": 0.7026,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.002898121252655983,
      "rewards/margins": -0.017615702003240585,
      "rewards/rejected": 0.014717579819262028,
      "step": 86
    },
    {
      "epoch": 0.02288382186128327,
      "grad_norm": 616.0,
      "learning_rate": 1.1417322834645669e-07,
      "logits/chosen": -0.6583680510520935,
      "logits/rejected": -0.65963214635849,
      "logps/chosen": -1528.080322265625,
      "logps/rejected": -1054.7440185546875,
      "loss": 0.7,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.012106322683393955,
      "rewards/margins": -0.01210718136280775,
      "rewards/rejected": 8.5815554484725e-07,
      "step": 87
    },
    {
      "epoch": 0.023146854296470433,
      "grad_norm": 532.0,
      "learning_rate": 1.1548556430446193e-07,
      "logits/chosen": -0.6387763619422913,
      "logits/rejected": -0.6282992362976074,
      "logps/chosen": -833.0134887695312,
      "logps/rejected": -811.2958984375,
      "loss": 0.6838,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01896057277917862,
      "rewards/margins": 0.019539976492524147,
      "rewards/rejected": -0.0005794053431600332,
      "step": 88
    },
    {
      "epoch": 0.023409886731657597,
      "grad_norm": 482.0,
      "learning_rate": 1.167979002624672e-07,
      "logits/chosen": -0.6625036001205444,
      "logits/rejected": -0.6705729961395264,
      "logps/chosen": -1143.0633544921875,
      "logps/rejected": -1152.207763671875,
      "loss": 0.6994,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0021528247743844986,
      "rewards/margins": -0.011566447094082832,
      "rewards/rejected": 0.013719271868467331,
      "step": 89
    },
    {
      "epoch": 0.02367291916684476,
      "grad_norm": 560.0,
      "learning_rate": 1.1811023622047243e-07,
      "logits/chosen": -0.6428525447845459,
      "logits/rejected": -0.6429169774055481,
      "logps/chosen": -1167.3192138671875,
      "logps/rejected": -894.5726928710938,
      "loss": 0.6914,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.014681626111268997,
      "rewards/margins": 0.004308986943215132,
      "rewards/rejected": 0.010372638702392578,
      "step": 90
    },
    {
      "epoch": 0.023935951602031925,
      "grad_norm": 470.0,
      "learning_rate": 1.1942257217847768e-07,
      "logits/chosen": -0.6648644804954529,
      "logits/rejected": -0.6818590760231018,
      "logps/chosen": -1082.2369384765625,
      "logps/rejected": -584.392333984375,
      "loss": 0.6813,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.022217940539121628,
      "rewards/margins": 0.024315834045410156,
      "rewards/rejected": -0.002097891876474023,
      "step": 91
    },
    {
      "epoch": 0.02419898403721909,
      "grad_norm": 620.0,
      "learning_rate": 1.2073490813648293e-07,
      "logits/chosen": -0.6473290324211121,
      "logits/rejected": -0.6534440517425537,
      "logps/chosen": -1387.5452880859375,
      "logps/rejected": -915.9308471679688,
      "loss": 0.698,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0022315974347293377,
      "rewards/margins": -0.008419988676905632,
      "rewards/rejected": 0.006188392639160156,
      "step": 92
    },
    {
      "epoch": 0.024462016472406253,
      "grad_norm": 604.0,
      "learning_rate": 1.2204724409448819e-07,
      "logits/chosen": -0.6490383744239807,
      "logits/rejected": -0.6508204936981201,
      "logps/chosen": -1669.7867431640625,
      "logps/rejected": -1401.859130859375,
      "loss": 0.7045,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.010508537292480469,
      "rewards/margins": -0.021072007715702057,
      "rewards/rejected": 0.010563468560576439,
      "step": 93
    },
    {
      "epoch": 0.024725048907593417,
      "grad_norm": 572.0,
      "learning_rate": 1.2335958005249344e-07,
      "logits/chosen": -0.6409127116203308,
      "logits/rejected": -0.6435267329216003,
      "logps/chosen": -1400.40380859375,
      "logps/rejected": -1041.4112548828125,
      "loss": 0.699,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0007992750033736229,
      "rewards/margins": -0.010796213522553444,
      "rewards/rejected": 0.009996939450502396,
      "step": 94
    },
    {
      "epoch": 0.02498808134278058,
      "grad_norm": 596.0,
      "learning_rate": 1.246719160104987e-07,
      "logits/chosen": -0.6618165373802185,
      "logits/rejected": -0.6547482013702393,
      "logps/chosen": -1442.2010498046875,
      "logps/rejected": -1443.297119140625,
      "loss": 0.6944,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.008986854925751686,
      "rewards/margins": -0.00018291501328349113,
      "rewards/rejected": 0.009169770404696465,
      "step": 95
    },
    {
      "epoch": 0.025251113777967744,
      "grad_norm": 660.0,
      "learning_rate": 1.2598425196850394e-07,
      "logits/chosen": -0.6429303288459778,
      "logits/rejected": -0.6414544582366943,
      "logps/chosen": -1411.7174072265625,
      "logps/rejected": -1044.54052734375,
      "loss": 0.7033,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.013068962842226028,
      "rewards/margins": -0.019350102171301842,
      "rewards/rejected": 0.006281138397753239,
      "step": 96
    },
    {
      "epoch": 0.02551414621315491,
      "grad_norm": 628.0,
      "learning_rate": 1.272965879265092e-07,
      "logits/chosen": -0.6580220460891724,
      "logits/rejected": -0.6619004607200623,
      "logps/chosen": -1541.75244140625,
      "logps/rejected": -1030.278564453125,
      "loss": 0.6893,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.014734363183379173,
      "rewards/margins": 0.008780481293797493,
      "rewards/rejected": -0.023514844477176666,
      "step": 97
    },
    {
      "epoch": 0.025777178648342072,
      "grad_norm": 528.0,
      "learning_rate": 1.2860892388451444e-07,
      "logits/chosen": -0.6590625643730164,
      "logits/rejected": -0.6597815752029419,
      "logps/chosen": -1485.60986328125,
      "logps/rejected": -1002.7106323242188,
      "loss": 0.7039,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.020105361938476562,
      "rewards/margins": -0.02083912119269371,
      "rewards/rejected": 0.0007337573915719986,
      "step": 98
    },
    {
      "epoch": 0.026040211083529236,
      "grad_norm": 832.0,
      "learning_rate": 1.2992125984251967e-07,
      "logits/chosen": -0.6383410096168518,
      "logits/rejected": -0.6598759889602661,
      "logps/chosen": -1321.9371337890625,
      "logps/rejected": -796.6106567382812,
      "loss": 0.6915,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010809613391757011,
      "rewards/margins": 0.004416752140969038,
      "rewards/rejected": 0.006392860785126686,
      "step": 99
    },
    {
      "epoch": 0.0263032435187164,
      "grad_norm": 504.0,
      "learning_rate": 1.3123359580052492e-07,
      "logits/chosen": -0.6737087368965149,
      "logits/rejected": -0.6666364669799805,
      "logps/chosen": -1172.2794189453125,
      "logps/rejected": -1041.3858642578125,
      "loss": 0.6932,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.001342391362413764,
      "rewards/margins": 0.000847339048050344,
      "rewards/rejected": 0.0004950524307787418,
      "step": 100
    },
    {
      "epoch": 0.026566275953903564,
      "grad_norm": 524.0,
      "learning_rate": 1.325459317585302e-07,
      "logits/chosen": -0.6498532295227051,
      "logits/rejected": -0.6444495916366577,
      "logps/chosen": -896.7752685546875,
      "logps/rejected": -914.8212890625,
      "loss": 0.6953,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.003770542563870549,
      "rewards/margins": -0.003038977272808552,
      "rewards/rejected": -0.0007315641269087791,
      "step": 101
    },
    {
      "epoch": 0.02682930838909073,
      "grad_norm": 616.0,
      "learning_rate": 1.3385826771653545e-07,
      "logits/chosen": -0.6652924418449402,
      "logits/rejected": -0.6655179858207703,
      "logps/chosen": -930.4898071289062,
      "logps/rejected": -983.84033203125,
      "loss": 0.6891,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.011583281680941582,
      "rewards/margins": 0.00953536108136177,
      "rewards/rejected": -0.021118640899658203,
      "step": 102
    },
    {
      "epoch": 0.027092340824277895,
      "grad_norm": 680.0,
      "learning_rate": 1.3517060367454067e-07,
      "logits/chosen": -0.6512477397918701,
      "logits/rejected": -0.6630122661590576,
      "logps/chosen": -1624.0438232421875,
      "logps/rejected": -1132.8804931640625,
      "loss": 0.7087,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.011258602142333984,
      "rewards/margins": -0.0290958434343338,
      "rewards/rejected": 0.01783723756670952,
      "step": 103
    },
    {
      "epoch": 0.02735537325946506,
      "grad_norm": 532.0,
      "learning_rate": 1.3648293963254592e-07,
      "logits/chosen": -0.6773630976676941,
      "logits/rejected": -0.6759374737739563,
      "logps/chosen": -1013.258056640625,
      "logps/rejected": -990.79931640625,
      "loss": 0.6916,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02291441150009632,
      "rewards/margins": 0.0032681464217603207,
      "rewards/rejected": 0.019646262750029564,
      "step": 104
    },
    {
      "epoch": 0.027618405694652223,
      "grad_norm": 468.0,
      "learning_rate": 1.3779527559055117e-07,
      "logits/chosen": -0.6446179747581482,
      "logits/rejected": -0.6440462470054626,
      "logps/chosen": -910.0596313476562,
      "logps/rejected": -794.4193115234375,
      "loss": 0.7074,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.017345810309052467,
      "rewards/margins": -0.02698802947998047,
      "rewards/rejected": 0.009642220102250576,
      "step": 105
    },
    {
      "epoch": 0.027881438129839387,
      "grad_norm": 560.0,
      "learning_rate": 1.3910761154855643e-07,
      "logits/chosen": -0.6417838335037231,
      "logits/rejected": -0.6492345333099365,
      "logps/chosen": -1083.3359375,
      "logps/rejected": -978.5983276367188,
      "loss": 0.6889,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0129995821043849,
      "rewards/margins": 0.009845927357673645,
      "rewards/rejected": -0.02284550853073597,
      "step": 106
    },
    {
      "epoch": 0.02814447056502655,
      "grad_norm": 540.0,
      "learning_rate": 1.4041994750656168e-07,
      "logits/chosen": -0.6487903594970703,
      "logits/rejected": NaN,
      "logps/chosen": -1121.177490234375,
      "logps/rejected": -836.255859375,
      "loss": 0.689,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005651378538459539,
      "rewards/margins": 0.009097002446651459,
      "rewards/rejected": -0.0034456257708370686,
      "step": 107
    },
    {
      "epoch": 0.028407503000213715,
      "grad_norm": 620.0,
      "learning_rate": 1.4173228346456693e-07,
      "logits/chosen": -0.6785897612571716,
      "logits/rejected": -0.6798429489135742,
      "logps/chosen": -1369.5643310546875,
      "logps/rejected": -1008.7618408203125,
      "loss": 0.6904,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.004759502597153187,
      "rewards/margins": 0.006230449769645929,
      "rewards/rejected": -0.010989952832460403,
      "step": 108
    },
    {
      "epoch": 0.02867053543540088,
      "grad_norm": 652.0,
      "learning_rate": 1.4304461942257218e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6084805727005005,
      "logps/chosen": -961.7047729492188,
      "logps/rejected": -963.2785034179688,
      "loss": 0.7092,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.024488354101777077,
      "rewards/margins": -0.031389713287353516,
      "rewards/rejected": 0.0069013601168990135,
      "step": 109
    },
    {
      "epoch": 0.028933567870588043,
      "grad_norm": 572.0,
      "learning_rate": 1.4435695538057743e-07,
      "logits/chosen": -0.6497198939323425,
      "logits/rejected": -0.6491546630859375,
      "logps/chosen": -1437.5703125,
      "logps/rejected": -1025.79296875,
      "loss": 0.6876,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.012715721502900124,
      "rewards/margins": 0.011871051974594593,
      "rewards/rejected": -0.02458677440881729,
      "step": 110
    },
    {
      "epoch": 0.029196600305775207,
      "grad_norm": 608.0,
      "learning_rate": 1.4566929133858266e-07,
      "logits/chosen": -0.6536111831665039,
      "logits/rejected": -0.6469756960868835,
      "logps/chosen": -1361.1495361328125,
      "logps/rejected": -1250.862060546875,
      "loss": 0.702,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.014045046642422676,
      "rewards/margins": -0.016698647290468216,
      "rewards/rejected": 0.002653599251061678,
      "step": 111
    },
    {
      "epoch": 0.02945963274096237,
      "grad_norm": 664.0,
      "learning_rate": 1.469816272965879e-07,
      "logits/chosen": -0.672586977481842,
      "logits/rejected": -0.6735091209411621,
      "logps/chosen": -1365.1573486328125,
      "logps/rejected": -1055.98193359375,
      "loss": 0.6843,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02622375637292862,
      "rewards/margins": 0.018584346398711205,
      "rewards/rejected": 0.007639408111572266,
      "step": 112
    },
    {
      "epoch": 0.029722665176149535,
      "grad_norm": 540.0,
      "learning_rate": 1.4829396325459319e-07,
      "logits/chosen": -0.6430389881134033,
      "logits/rejected": -0.6158244609832764,
      "logps/chosen": -1028.4473876953125,
      "logps/rejected": -540.0390625,
      "loss": 0.6945,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0030626305378973484,
      "rewards/margins": -0.0019155483460053802,
      "rewards/rejected": 0.00497817900031805,
      "step": 113
    },
    {
      "epoch": 0.0299856976113367,
      "grad_norm": 428.0,
      "learning_rate": 1.4960629921259844e-07,
      "logits/chosen": -0.6686395406723022,
      "logits/rejected": -0.6677947640419006,
      "logps/chosen": -868.5057373046875,
      "logps/rejected": -863.9393310546875,
      "loss": 0.6851,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.002024078043177724,
      "rewards/margins": 0.017639826983213425,
      "rewards/rejected": -0.015615750104188919,
      "step": 114
    },
    {
      "epoch": 0.030248730046523863,
      "grad_norm": 512.0,
      "learning_rate": 1.5091863517060366e-07,
      "logits/chosen": -0.6472195386886597,
      "logits/rejected": -0.6526005864143372,
      "logps/chosen": -1054.8924560546875,
      "logps/rejected": -904.5728149414062,
      "loss": 0.6926,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0021983161568641663,
      "rewards/margins": 0.0018188473768532276,
      "rewards/rejected": -0.004017162136733532,
      "step": 115
    },
    {
      "epoch": 0.030511762481711027,
      "grad_norm": 438.0,
      "learning_rate": 1.522309711286089e-07,
      "logits/chosen": -0.64715576171875,
      "logits/rejected": -0.6523733139038086,
      "logps/chosen": -962.71142578125,
      "logps/rejected": -749.036376953125,
      "loss": 0.6935,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00016956403851509094,
      "rewards/margins": 0.00042962958104908466,
      "rewards/rejected": -0.0005991924554109573,
      "step": 116
    },
    {
      "epoch": 0.03077479491689819,
      "grad_norm": 596.0,
      "learning_rate": 1.5354330708661416e-07,
      "logits/chosen": -0.6582873463630676,
      "logits/rejected": -0.6561271548271179,
      "logps/chosen": -1589.960693359375,
      "logps/rejected": -831.090087890625,
      "loss": 0.6986,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.00888977199792862,
      "rewards/margins": -0.010261631570756435,
      "rewards/rejected": 0.01915140263736248,
      "step": 117
    },
    {
      "epoch": 0.031037827352085354,
      "grad_norm": 500.0,
      "learning_rate": 1.5485564304461942e-07,
      "logits/chosen": -0.6244059205055237,
      "logits/rejected": -0.6262105107307434,
      "logps/chosen": -946.11572265625,
      "logps/rejected": -718.927490234375,
      "loss": 0.6937,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.02040405198931694,
      "rewards/margins": -0.00035762879997491837,
      "rewards/rejected": 0.020761679857969284,
      "step": 118
    },
    {
      "epoch": 0.03130085978727252,
      "grad_norm": 572.0,
      "learning_rate": 1.5616797900262467e-07,
      "logits/chosen": -0.6423102021217346,
      "logits/rejected": -0.6468573808670044,
      "logps/chosen": -1261.1246337890625,
      "logps/rejected": -866.6472778320312,
      "loss": 0.6826,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0024516116827726364,
      "rewards/margins": 0.0215929988771677,
      "rewards/rejected": -0.024044610559940338,
      "step": 119
    },
    {
      "epoch": 0.03156389222245968,
      "grad_norm": 540.0,
      "learning_rate": 1.5748031496062992e-07,
      "logits/chosen": -0.6713907718658447,
      "logits/rejected": -0.6750301122665405,
      "logps/chosen": -1097.6748046875,
      "logps/rejected": -1128.4312744140625,
      "loss": 0.7014,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.005674266256392002,
      "rewards/margins": -0.01600952073931694,
      "rewards/rejected": 0.010335253551602364,
      "step": 120
    },
    {
      "epoch": 0.031826924657646846,
      "grad_norm": 568.0,
      "learning_rate": 1.5879265091863517e-07,
      "logits/chosen": -0.6585685014724731,
      "logits/rejected": -0.6606056094169617,
      "logps/chosen": -1133.682373046875,
      "logps/rejected": -950.2247924804688,
      "loss": 0.6983,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.020866870880126953,
      "rewards/margins": -0.00957422237843275,
      "rewards/rejected": -0.011292650364339352,
      "step": 121
    },
    {
      "epoch": 0.03208995709283401,
      "grad_norm": 506.0,
      "learning_rate": 1.6010498687664042e-07,
      "logits/chosen": -0.6591353416442871,
      "logits/rejected": -0.6645353436470032,
      "logps/chosen": -1084.0191650390625,
      "logps/rejected": -1150.1143798828125,
      "loss": 0.69,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004020309075713158,
      "rewards/margins": 0.007074547000229359,
      "rewards/rejected": -0.003054236527532339,
      "step": 122
    },
    {
      "epoch": 0.032352989528021174,
      "grad_norm": 588.0,
      "learning_rate": 1.6141732283464565e-07,
      "logits/chosen": -0.650454580783844,
      "logits/rejected": -0.6408450603485107,
      "logps/chosen": -1083.82568359375,
      "logps/rejected": -808.3011474609375,
      "loss": 0.691,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005265616811811924,
      "rewards/margins": 0.005886172875761986,
      "rewards/rejected": -0.011151790618896484,
      "step": 123
    },
    {
      "epoch": 0.03261602196320834,
      "grad_norm": 520.0,
      "learning_rate": 1.627296587926509e-07,
      "logits/chosen": -0.6372298002243042,
      "logits/rejected": -0.6574360728263855,
      "logps/chosen": -1177.017333984375,
      "logps/rejected": -737.2098999023438,
      "loss": 0.6844,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.029219819232821465,
      "rewards/margins": 0.018529225140810013,
      "rewards/rejected": 0.010690595023334026,
      "step": 124
    },
    {
      "epoch": 0.0328790543983955,
      "grad_norm": 512.0,
      "learning_rate": 1.6404199475065617e-07,
      "logits/chosen": -0.6459711194038391,
      "logits/rejected": -0.6455292701721191,
      "logps/chosen": -1235.0416259765625,
      "logps/rejected": -1169.53857421875,
      "loss": 0.6839,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0031824111938476562,
      "rewards/margins": 0.019691945984959602,
      "rewards/rejected": -0.016509532928466797,
      "step": 125
    },
    {
      "epoch": 0.03314208683358267,
      "grad_norm": 584.0,
      "learning_rate": 1.6535433070866143e-07,
      "logits/chosen": -0.6716091632843018,
      "logits/rejected": -0.665132462978363,
      "logps/chosen": -1390.890625,
      "logps/rejected": -912.44873046875,
      "loss": 0.6847,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.015256785787642002,
      "rewards/margins": 0.018321702256798744,
      "rewards/rejected": -0.003064919263124466,
      "step": 126
    },
    {
      "epoch": 0.03340511926876983,
      "grad_norm": 568.0,
      "learning_rate": 1.6666666666666665e-07,
      "logits/chosen": -0.6361861228942871,
      "logits/rejected": NaN,
      "logps/chosen": -1290.5814208984375,
      "logps/rejected": -784.6868286132812,
      "loss": 0.6904,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0040683746337890625,
      "rewards/margins": 0.006489276885986328,
      "rewards/rejected": -0.010557650588452816,
      "step": 127
    },
    {
      "epoch": 0.033668151703957,
      "grad_norm": 490.0,
      "learning_rate": 1.679790026246719e-07,
      "logits/chosen": -0.6313784718513489,
      "logits/rejected": -0.6361284852027893,
      "logps/chosen": -1027.375732421875,
      "logps/rejected": -777.8204345703125,
      "loss": 0.6918,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.008790016174316406,
      "rewards/margins": 0.0033023846335709095,
      "rewards/rejected": 0.005487632937729359,
      "step": 128
    },
    {
      "epoch": 0.03393118413914416,
      "grad_norm": 524.0,
      "learning_rate": 1.6929133858267715e-07,
      "logits/chosen": -0.6193291544914246,
      "logits/rejected": -0.6393106579780579,
      "logps/chosen": -1174.30419921875,
      "logps/rejected": -1052.031494140625,
      "loss": 0.6972,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012887382879853249,
      "rewards/margins": -0.006917095277458429,
      "rewards/rejected": 0.01980447769165039,
      "step": 129
    },
    {
      "epoch": 0.034194216574331325,
      "grad_norm": 556.0,
      "learning_rate": 1.7060367454068243e-07,
      "logits/chosen": -0.622719407081604,
      "logits/rejected": -0.6256909370422363,
      "logps/chosen": -1094.28955078125,
      "logps/rejected": -597.7046508789062,
      "loss": 0.7004,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.019174860790371895,
      "rewards/margins": -0.013591576367616653,
      "rewards/rejected": -0.005583286285400391,
      "step": 130
    },
    {
      "epoch": 0.034457249009518486,
      "grad_norm": 584.0,
      "learning_rate": 1.7191601049868766e-07,
      "logits/chosen": -0.6481513977050781,
      "logits/rejected": -0.6685305237770081,
      "logps/chosen": -1089.91552734375,
      "logps/rejected": -1109.7947998046875,
      "loss": 0.6871,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006201267242431641,
      "rewards/margins": 0.012852096930146217,
      "rewards/rejected": -0.006650829222053289,
      "step": 131
    },
    {
      "epoch": 0.03472028144470565,
      "grad_norm": 564.0,
      "learning_rate": 1.732283464566929e-07,
      "logits/chosen": -0.6496647596359253,
      "logits/rejected": -0.646241307258606,
      "logps/chosen": -1114.0977783203125,
      "logps/rejected": -807.0661010742188,
      "loss": 0.6988,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0014527318999171257,
      "rewards/margins": -0.010260437615215778,
      "rewards/rejected": 0.011713171377778053,
      "step": 132
    },
    {
      "epoch": 0.03498331387989281,
      "grad_norm": 486.0,
      "learning_rate": 1.7454068241469816e-07,
      "logits/chosen": -0.6430667638778687,
      "logits/rejected": -0.656929075717926,
      "logps/chosen": -1112.7408447265625,
      "logps/rejected": -1044.7509765625,
      "loss": 0.6962,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.012630178593099117,
      "rewards/margins": -0.005127906799316406,
      "rewards/rejected": -0.007502268068492413,
      "step": 133
    },
    {
      "epoch": 0.03524634631507998,
      "grad_norm": 532.0,
      "learning_rate": 1.758530183727034e-07,
      "logits/chosen": -0.6819089651107788,
      "logits/rejected": -0.681180477142334,
      "logps/chosen": -1187.88916015625,
      "logps/rejected": -1134.199951171875,
      "loss": 0.7036,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.007580186240375042,
      "rewards/margins": -0.02049856446683407,
      "rewards/rejected": 0.012918377295136452,
      "step": 134
    },
    {
      "epoch": 0.03550937875026714,
      "grad_norm": 476.0,
      "learning_rate": 1.7716535433070863e-07,
      "logits/chosen": -0.670299768447876,
      "logits/rejected": -0.6790396571159363,
      "logps/chosen": -1099.957275390625,
      "logps/rejected": -828.4228515625,
      "loss": 0.6994,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.011410904116928577,
      "rewards/margins": -0.011872483417391777,
      "rewards/rejected": 0.0004615788348019123,
      "step": 135
    },
    {
      "epoch": 0.03577241118545431,
      "grad_norm": 520.0,
      "learning_rate": 1.784776902887139e-07,
      "logits/chosen": -0.6410176157951355,
      "logits/rejected": -0.6373354196548462,
      "logps/chosen": -1191.1402587890625,
      "logps/rejected": -849.6145629882812,
      "loss": 0.6923,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0076240538619458675,
      "rewards/margins": 0.002615642733871937,
      "rewards/rejected": -0.010239697061479092,
      "step": 136
    },
    {
      "epoch": 0.03603544362064147,
      "grad_norm": 640.0,
      "learning_rate": 1.7979002624671916e-07,
      "logits/chosen": -0.6377418041229248,
      "logits/rejected": NaN,
      "logps/chosen": -1375.43115234375,
      "logps/rejected": -586.9110107421875,
      "loss": 0.6894,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01369934156537056,
      "rewards/margins": 0.008589362725615501,
      "rewards/rejected": 0.005109977908432484,
      "step": 137
    },
    {
      "epoch": 0.036298476055828637,
      "grad_norm": 588.0,
      "learning_rate": 1.8110236220472441e-07,
      "logits/chosen": -0.6702321767807007,
      "logits/rejected": -0.6761676073074341,
      "logps/chosen": -1240.321533203125,
      "logps/rejected": -975.1643676757812,
      "loss": 0.6991,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.00663495110347867,
      "rewards/margins": -0.011009836569428444,
      "rewards/rejected": 0.0043748850002884865,
      "step": 138
    },
    {
      "epoch": 0.0365615084910158,
      "grad_norm": 452.0,
      "learning_rate": 1.8241469816272964e-07,
      "logits/chosen": -0.6323966383934021,
      "logits/rejected": -0.6392496228218079,
      "logps/chosen": -889.60546875,
      "logps/rejected": -680.923095703125,
      "loss": 0.696,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.00718841515481472,
      "rewards/margins": -0.005342195741832256,
      "rewards/rejected": -0.0018462184816598892,
      "step": 139
    },
    {
      "epoch": 0.036824540926202964,
      "grad_norm": 608.0,
      "learning_rate": 1.837270341207349e-07,
      "logits/chosen": -0.6656695604324341,
      "logits/rejected": -0.6715975999832153,
      "logps/chosen": -986.3863525390625,
      "logps/rejected": -785.9706420898438,
      "loss": 0.6891,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.005256843753159046,
      "rewards/margins": 0.009022044949233532,
      "rewards/rejected": -0.014278887771070004,
      "step": 140
    },
    {
      "epoch": 0.037087573361390125,
      "grad_norm": 490.0,
      "learning_rate": 1.8503937007874014e-07,
      "logits/chosen": -0.6407849788665771,
      "logits/rejected": -0.6558778882026672,
      "logps/chosen": -960.9951171875,
      "logps/rejected": -597.4873046875,
      "loss": 0.699,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0047273640520870686,
      "rewards/margins": -0.01122426986694336,
      "rewards/rejected": 0.006496906280517578,
      "step": 141
    },
    {
      "epoch": 0.03735060579657729,
      "grad_norm": 676.0,
      "learning_rate": 1.8635170603674542e-07,
      "logits/chosen": -0.6403671503067017,
      "logits/rejected": -0.6556288003921509,
      "logps/chosen": -1313.753173828125,
      "logps/rejected": -1134.660888671875,
      "loss": 0.6973,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.004966639913618565,
      "rewards/margins": -0.007860660552978516,
      "rewards/rejected": 0.002894020639359951,
      "step": 142
    },
    {
      "epoch": 0.03761363823176445,
      "grad_norm": 492.0,
      "learning_rate": 1.8766404199475064e-07,
      "logits/chosen": -0.6636741161346436,
      "logits/rejected": -0.6641364693641663,
      "logps/chosen": -1002.41259765625,
      "logps/rejected": -895.4652099609375,
      "loss": 0.6888,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.001305675134062767,
      "rewards/margins": 0.009362127631902695,
      "rewards/rejected": -0.008056451566517353,
      "step": 143
    },
    {
      "epoch": 0.03787667066695162,
      "grad_norm": 548.0,
      "learning_rate": 1.889763779527559e-07,
      "logits/chosen": -0.6478816270828247,
      "logits/rejected": -0.6456509828567505,
      "logps/chosen": -1170.7188720703125,
      "logps/rejected": -918.7308349609375,
      "loss": 0.699,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.015614700503647327,
      "rewards/margins": -0.0106201171875,
      "rewards/rejected": 0.026234818622469902,
      "step": 144
    },
    {
      "epoch": 0.03813970310213878,
      "grad_norm": 468.0,
      "learning_rate": 1.9028871391076115e-07,
      "logits/chosen": -0.6595015525817871,
      "logits/rejected": -0.6792489886283875,
      "logps/chosen": -780.314697265625,
      "logps/rejected": -808.2157592773438,
      "loss": 0.6841,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.017458438873291016,
      "rewards/margins": 0.018729783594608307,
      "rewards/rejected": -0.0012713427422568202,
      "step": 145
    },
    {
      "epoch": 0.03840273553732595,
      "grad_norm": 676.0,
      "learning_rate": 1.916010498687664e-07,
      "logits/chosen": -0.6136724948883057,
      "logits/rejected": -0.6228630542755127,
      "logps/chosen": -1521.4188232421875,
      "logps/rejected": -1177.7987060546875,
      "loss": 0.6828,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006443404126912355,
      "rewards/margins": 0.02133655548095703,
      "rewards/rejected": -0.014893149957060814,
      "step": 146
    },
    {
      "epoch": 0.03866576797251311,
      "grad_norm": 560.0,
      "learning_rate": 1.9291338582677162e-07,
      "logits/chosen": -0.6503449082374573,
      "logits/rejected": -0.6461342573165894,
      "logps/chosen": -886.2039794921875,
      "logps/rejected": -1022.9850463867188,
      "loss": 0.696,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0010157818906009197,
      "rewards/margins": -0.005259108729660511,
      "rewards/rejected": 0.006274891085922718,
      "step": 147
    },
    {
      "epoch": 0.038928800407700276,
      "grad_norm": 600.0,
      "learning_rate": 1.942257217847769e-07,
      "logits/chosen": -0.6739921569824219,
      "logits/rejected": -0.6692219972610474,
      "logps/chosen": -1558.8614501953125,
      "logps/rejected": -1081.86572265625,
      "loss": 0.6825,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012275028973817825,
      "rewards/margins": 0.02227339893579483,
      "rewards/rejected": -0.009998369961977005,
      "step": 148
    },
    {
      "epoch": 0.039191832842887436,
      "grad_norm": 628.0,
      "learning_rate": 1.9553805774278215e-07,
      "logits/chosen": -0.6331160664558411,
      "logits/rejected": -0.6471138000488281,
      "logps/chosen": -1351.064453125,
      "logps/rejected": -1019.9130249023438,
      "loss": 0.6958,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.00038623809814453125,
      "rewards/margins": -0.0035767527297139168,
      "rewards/rejected": 0.003962993621826172,
      "step": 149
    },
    {
      "epoch": 0.039454865278074604,
      "grad_norm": 564.0,
      "learning_rate": 1.968503937007874e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6621302366256714,
      "logps/chosen": -943.87890625,
      "logps/rejected": -875.265625,
      "loss": 0.6941,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.010566902346909046,
      "rewards/margins": -0.0014950751792639494,
      "rewards/rejected": 0.012061978690326214,
      "step": 150
    },
    {
      "epoch": 0.039717897713261764,
      "grad_norm": 604.0,
      "learning_rate": 1.9816272965879263e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6439089179039001,
      "logps/chosen": -1123.5224609375,
      "logps/rejected": -826.7777709960938,
      "loss": 0.6879,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.007737922482192516,
      "rewards/margins": 0.012290382757782936,
      "rewards/rejected": -0.004552459344267845,
      "step": 151
    },
    {
      "epoch": 0.03998093014844893,
      "grad_norm": 506.0,
      "learning_rate": 1.9947506561679788e-07,
      "logits/chosen": -0.6276299953460693,
      "logits/rejected": -0.6387923359870911,
      "logps/chosen": -997.2647705078125,
      "logps/rejected": -1001.1608276367188,
      "loss": 0.6823,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.018098928034305573,
      "rewards/margins": 0.022268343716859818,
      "rewards/rejected": -0.00416941661387682,
      "step": 152
    },
    {
      "epoch": 0.04024396258363609,
      "grad_norm": 568.0,
      "learning_rate": 2.0078740157480316e-07,
      "logits/chosen": -0.6739535331726074,
      "logits/rejected": -0.6773117184638977,
      "logps/chosen": -1093.9720458984375,
      "logps/rejected": -1030.314208984375,
      "loss": 0.6917,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01855916902422905,
      "rewards/margins": 0.004058551508933306,
      "rewards/rejected": 0.014500617049634457,
      "step": 153
    },
    {
      "epoch": 0.04050699501882326,
      "grad_norm": 620.0,
      "learning_rate": 2.020997375328084e-07,
      "logits/chosen": -0.6769409775733948,
      "logits/rejected": -0.6771318912506104,
      "logps/chosen": -1662.4361572265625,
      "logps/rejected": -1090.8690185546875,
      "loss": 0.707,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.015667632222175598,
      "rewards/margins": -0.02653498575091362,
      "rewards/rejected": 0.01086735725402832,
      "step": 154
    },
    {
      "epoch": 0.04077002745401042,
      "grad_norm": 536.0,
      "learning_rate": 2.0341207349081363e-07,
      "logits/chosen": -0.6509108543395996,
      "logits/rejected": -0.6485505700111389,
      "logps/chosen": -1139.8125,
      "logps/rejected": -821.041259765625,
      "loss": 0.6939,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0028749462217092514,
      "rewards/margins": -0.0005982404109090567,
      "rewards/rejected": 0.0034731868654489517,
      "step": 155
    },
    {
      "epoch": 0.04103305988919759,
      "grad_norm": 580.0,
      "learning_rate": 2.0472440944881889e-07,
      "logits/chosen": -0.6566342115402222,
      "logits/rejected": -0.6538740992546082,
      "logps/chosen": -1047.9442138671875,
      "logps/rejected": -492.25421142578125,
      "loss": 0.6953,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.012102034874260426,
      "rewards/margins": -0.003575420007109642,
      "rewards/rejected": 0.015677452087402344,
      "step": 156
    },
    {
      "epoch": 0.04129609232438475,
      "grad_norm": 636.0,
      "learning_rate": 2.0603674540682414e-07,
      "logits/chosen": -0.6586996912956238,
      "logits/rejected": -0.6547606587409973,
      "logps/chosen": -1381.224853515625,
      "logps/rejected": -1130.41845703125,
      "loss": 0.688,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.010731222108006477,
      "rewards/margins": 0.01091470755636692,
      "rewards/rejected": -0.0001834868744481355,
      "step": 157
    },
    {
      "epoch": 0.041559124759571915,
      "grad_norm": 516.0,
      "learning_rate": 2.073490813648294e-07,
      "logits/chosen": -0.6231178641319275,
      "logits/rejected": -0.6351503133773804,
      "logps/chosen": -995.0567016601562,
      "logps/rejected": -799.1051635742188,
      "loss": 0.6943,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.004308701027184725,
      "rewards/margins": -0.0018595214933156967,
      "rewards/rejected": -0.002449179068207741,
      "step": 158
    },
    {
      "epoch": 0.041822157194759076,
      "grad_norm": 616.0,
      "learning_rate": 2.0866141732283464e-07,
      "logits/chosen": -0.6346429586410522,
      "logits/rejected": -0.6357970237731934,
      "logps/chosen": -1212.0888671875,
      "logps/rejected": -996.753662109375,
      "loss": 0.6896,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0008132923394441605,
      "rewards/margins": 0.00888214074075222,
      "rewards/rejected": -0.00806884653866291,
      "step": 159
    },
    {
      "epoch": 0.04208518962994624,
      "grad_norm": 640.0,
      "learning_rate": 2.099737532808399e-07,
      "logits/chosen": -0.642386257648468,
      "logits/rejected": -0.6494432687759399,
      "logps/chosen": -1289.384521484375,
      "logps/rejected": -690.872314453125,
      "loss": 0.6885,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0011980053968727589,
      "rewards/margins": 0.010298348031938076,
      "rewards/rejected": -0.00910034216940403,
      "step": 160
    },
    {
      "epoch": 0.042348222065133403,
      "grad_norm": 536.0,
      "learning_rate": 2.1128608923884514e-07,
      "logits/chosen": -0.6469371318817139,
      "logits/rejected": -0.6517741084098816,
      "logps/chosen": -1102.7420654296875,
      "logps/rejected": -747.0505981445312,
      "loss": 0.699,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.01643075980246067,
      "rewards/margins": -0.011131476610898972,
      "rewards/rejected": -0.00529928132891655,
      "step": 161
    },
    {
      "epoch": 0.04261125450032057,
      "grad_norm": 540.0,
      "learning_rate": 2.125984251968504e-07,
      "logits/chosen": -0.6387971639633179,
      "logits/rejected": -0.6428045630455017,
      "logps/chosen": -1030.740966796875,
      "logps/rejected": -719.9903564453125,
      "loss": 0.6929,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.005468941293656826,
      "rewards/margins": 0.001611995860002935,
      "rewards/rejected": 0.0038569453172385693,
      "step": 162
    },
    {
      "epoch": 0.04287428693550773,
      "grad_norm": 540.0,
      "learning_rate": 2.1391076115485562e-07,
      "logits/chosen": -0.6682126522064209,
      "logits/rejected": -0.6647217273712158,
      "logps/chosen": -1188.6016845703125,
      "logps/rejected": -1108.383056640625,
      "loss": 0.6933,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.003918646834790707,
      "rewards/margins": 0.0002891540643759072,
      "rewards/rejected": 0.0036294935271143913,
      "step": 163
    },
    {
      "epoch": 0.0431373193706949,
      "grad_norm": 544.0,
      "learning_rate": 2.1522309711286087e-07,
      "logits/chosen": -0.6258804798126221,
      "logits/rejected": -0.6338955760002136,
      "logps/chosen": -1007.4208984375,
      "logps/rejected": -787.7755737304688,
      "loss": 0.6811,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0021362295374274254,
      "rewards/margins": 0.025175049901008606,
      "rewards/rejected": -0.023038815706968307,
      "step": 164
    },
    {
      "epoch": 0.043400351805882066,
      "grad_norm": 616.0,
      "learning_rate": 2.1653543307086615e-07,
      "logits/chosen": -0.657672643661499,
      "logits/rejected": NaN,
      "logps/chosen": -1247.53076171875,
      "logps/rejected": -882.1461181640625,
      "loss": 0.6943,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.008572387509047985,
      "rewards/margins": -0.0016937265172600746,
      "rewards/rejected": -0.006878662388771772,
      "step": 165
    },
    {
      "epoch": 0.04366338424106923,
      "grad_norm": 430.0,
      "learning_rate": 2.178477690288714e-07,
      "logits/chosen": -0.6739400625228882,
      "logits/rejected": -0.6699235439300537,
      "logps/chosen": -682.491455078125,
      "logps/rejected": -490.64801025390625,
      "loss": 0.6986,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.003162479493767023,
      "rewards/margins": -0.010198308154940605,
      "rewards/rejected": 0.00703582726418972,
      "step": 166
    },
    {
      "epoch": 0.043926416676256394,
      "grad_norm": 848.0,
      "learning_rate": 2.1916010498687662e-07,
      "logits/chosen": -0.6421108245849609,
      "logits/rejected": -0.6569357514381409,
      "logps/chosen": -1251.7508544921875,
      "logps/rejected": -999.281005859375,
      "loss": 0.697,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.008310414850711823,
      "rewards/margins": -0.006734943483024836,
      "rewards/rejected": -0.0015754695050418377,
      "step": 167
    },
    {
      "epoch": 0.044189449111443554,
      "grad_norm": 372.0,
      "learning_rate": 2.2047244094488187e-07,
      "logits/chosen": -0.6080148816108704,
      "logits/rejected": -0.6135621666908264,
      "logps/chosen": -588.7601318359375,
      "logps/rejected": -600.26025390625,
      "loss": 0.6937,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004939842503517866,
      "rewards/margins": -0.0008674621349200606,
      "rewards/rejected": 0.005807304289191961,
      "step": 168
    },
    {
      "epoch": 0.04445248154663072,
      "grad_norm": 492.0,
      "learning_rate": 2.2178477690288713e-07,
      "logits/chosen": -0.6486470699310303,
      "logits/rejected": -0.6405966877937317,
      "logps/chosen": -1275.6898193359375,
      "logps/rejected": -1077.193359375,
      "loss": 0.6948,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.00547943077981472,
      "rewards/margins": -0.0028480528853833675,
      "rewards/rejected": -0.00263137836009264,
      "step": 169
    },
    {
      "epoch": 0.04471551398181788,
      "grad_norm": 552.0,
      "learning_rate": 2.2309711286089238e-07,
      "logits/chosen": -0.6720563769340515,
      "logits/rejected": -0.671728253364563,
      "logps/chosen": -1320.3448486328125,
      "logps/rejected": -1159.0299072265625,
      "loss": 0.6888,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.00343570695258677,
      "rewards/margins": 0.009407330304384232,
      "rewards/rejected": -0.012843037024140358,
      "step": 170
    },
    {
      "epoch": 0.04497854641700505,
      "grad_norm": 444.0,
      "learning_rate": 2.2440944881889763e-07,
      "logits/chosen": -0.6325276494026184,
      "logits/rejected": -0.6365100741386414,
      "logps/chosen": -803.6257934570312,
      "logps/rejected": -702.935791015625,
      "loss": 0.6873,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.005678844638168812,
      "rewards/margins": 0.012663746252655983,
      "rewards/rejected": -0.006984901614487171,
      "step": 171
    },
    {
      "epoch": 0.04524157885219221,
      "grad_norm": 688.0,
      "learning_rate": 2.2572178477690288e-07,
      "logits/chosen": -0.6477915048599243,
      "logits/rejected": -0.6616644859313965,
      "logps/chosen": -1464.08837890625,
      "logps/rejected": -813.4791870117188,
      "loss": 0.6844,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006414985284209251,
      "rewards/margins": 0.01883401721715927,
      "rewards/rejected": -0.012419032864272594,
      "step": 172
    },
    {
      "epoch": 0.04550461128737938,
      "grad_norm": 656.0,
      "learning_rate": 2.2703412073490813e-07,
      "logits/chosen": -0.6405099034309387,
      "logits/rejected": NaN,
      "logps/chosen": -1524.1402587890625,
      "logps/rejected": -695.8150024414062,
      "loss": 0.7099,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.013130380772054195,
      "rewards/margins": -0.0322691947221756,
      "rewards/rejected": 0.019138814881443977,
      "step": 173
    },
    {
      "epoch": 0.04576764372256654,
      "grad_norm": 536.0,
      "learning_rate": 2.2834645669291338e-07,
      "logits/chosen": -0.6480164527893066,
      "logits/rejected": -0.653152585029602,
      "logps/chosen": -1066.4207763671875,
      "logps/rejected": -1221.49853515625,
      "loss": 0.7021,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.028661014512181282,
      "rewards/margins": -0.01670985296368599,
      "rewards/rejected": -0.011951161548495293,
      "step": 174
    },
    {
      "epoch": 0.046030676157753705,
      "grad_norm": 520.0,
      "learning_rate": 2.296587926509186e-07,
      "logits/chosen": -0.6478853821754456,
      "logits/rejected": -0.6489351391792297,
      "logps/chosen": -1149.61865234375,
      "logps/rejected": -764.1865844726562,
      "loss": 0.6933,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.024858666583895683,
      "rewards/margins": 0.0005493159405887127,
      "rewards/rejected": 0.024309350177645683,
      "step": 175
    },
    {
      "epoch": 0.046293708592940866,
      "grad_norm": 426.0,
      "learning_rate": 2.3097112860892386e-07,
      "logits/chosen": -0.6682780385017395,
      "logits/rejected": -0.6729027628898621,
      "logps/chosen": -802.4763793945312,
      "logps/rejected": -437.84173583984375,
      "loss": 0.6906,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.002623557345941663,
      "rewards/margins": 0.005454064346849918,
      "rewards/rejected": -0.0028305056039243937,
      "step": 176
    },
    {
      "epoch": 0.04655674102812803,
      "grad_norm": 612.0,
      "learning_rate": 2.3228346456692914e-07,
      "logits/chosen": -0.6707938313484192,
      "logits/rejected": NaN,
      "logps/chosen": -1062.6573486328125,
      "logps/rejected": -626.5631103515625,
      "loss": 0.6872,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0077560413628816605,
      "rewards/margins": 0.012795830145478249,
      "rewards/rejected": -0.005039787385612726,
      "step": 177
    },
    {
      "epoch": 0.046819773463315194,
      "grad_norm": 524.0,
      "learning_rate": 2.335958005249344e-07,
      "logits/chosen": -0.6476251482963562,
      "logits/rejected": -0.6590272188186646,
      "logps/chosen": -1249.658203125,
      "logps/rejected": -952.749755859375,
      "loss": 0.6868,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.024695489555597305,
      "rewards/margins": 0.013487052172422409,
      "rewards/rejected": 0.01120843831449747,
      "step": 178
    },
    {
      "epoch": 0.04708280589850236,
      "grad_norm": 454.0,
      "learning_rate": 2.349081364829396e-07,
      "logits/chosen": -0.6528888940811157,
      "logits/rejected": -0.6545329689979553,
      "logps/chosen": -810.317626953125,
      "logps/rejected": -709.342529296875,
      "loss": 0.7031,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.010278129018843174,
      "rewards/margins": -0.018921375274658203,
      "rewards/rejected": 0.029199503362178802,
      "step": 179
    },
    {
      "epoch": 0.04734583833368952,
      "grad_norm": 520.0,
      "learning_rate": 2.3622047244094486e-07,
      "logits/chosen": -0.6580276489257812,
      "logits/rejected": -0.660503089427948,
      "logps/chosen": -993.73486328125,
      "logps/rejected": -832.9864501953125,
      "loss": 0.7017,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.012546444311738014,
      "rewards/margins": -0.015984345227479935,
      "rewards/rejected": 0.0034379009157419205,
      "step": 180
    },
    {
      "epoch": 0.04760887076887669,
      "grad_norm": 478.0,
      "learning_rate": 2.3753280839895011e-07,
      "logits/chosen": -0.6389084458351135,
      "logits/rejected": -0.6435239911079407,
      "logps/chosen": -1247.4146728515625,
      "logps/rejected": -1039.228271484375,
      "loss": 0.6899,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02553420141339302,
      "rewards/margins": 0.007093477062880993,
      "rewards/rejected": 0.018440723419189453,
      "step": 181
    },
    {
      "epoch": 0.04787190320406385,
      "grad_norm": 716.0,
      "learning_rate": 2.3884514435695537e-07,
      "logits/chosen": -0.6200729012489319,
      "logits/rejected": NaN,
      "logps/chosen": -1533.27685546875,
      "logps/rejected": -1213.04248046875,
      "loss": 0.7007,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.019236277788877487,
      "rewards/margins": -0.013663576915860176,
      "rewards/rejected": -0.005572701338678598,
      "step": 182
    },
    {
      "epoch": 0.04813493563925102,
      "grad_norm": 624.0,
      "learning_rate": 2.401574803149606e-07,
      "logits/chosen": -0.6636444926261902,
      "logits/rejected": -0.6659044623374939,
      "logps/chosen": -1323.50634765625,
      "logps/rejected": -1145.371826171875,
      "loss": 0.6896,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.011841201223433018,
      "rewards/margins": 0.00920934695750475,
      "rewards/rejected": -0.021050548180937767,
      "step": 183
    },
    {
      "epoch": 0.04839796807443818,
      "grad_norm": 486.0,
      "learning_rate": 2.4146981627296587e-07,
      "logits/chosen": -0.6558758616447449,
      "logits/rejected": -0.6642827987670898,
      "logps/chosen": -1079.6669921875,
      "logps/rejected": -1080.0760498046875,
      "loss": 0.6981,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006880361121147871,
      "rewards/margins": -0.009491939097642899,
      "rewards/rejected": 0.016372300684452057,
      "step": 184
    },
    {
      "epoch": 0.048661000509625345,
      "grad_norm": 580.0,
      "learning_rate": 2.4278215223097115e-07,
      "logits/chosen": -0.6489015817642212,
      "logits/rejected": -0.6499336957931519,
      "logps/chosen": -1494.6865234375,
      "logps/rejected": -1414.94970703125,
      "loss": 0.7008,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.001120567787438631,
      "rewards/margins": -0.013475995510816574,
      "rewards/rejected": 0.012355426326394081,
      "step": 185
    },
    {
      "epoch": 0.048924032944812505,
      "grad_norm": 664.0,
      "learning_rate": 2.4409448818897637e-07,
      "logits/chosen": -0.6605331897735596,
      "logits/rejected": -0.6639459729194641,
      "logps/chosen": -1561.5499267578125,
      "logps/rejected": -1287.1905517578125,
      "loss": 0.7218,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.027819251641631126,
      "rewards/margins": -0.055567167699337006,
      "rewards/rejected": 0.02774791605770588,
      "step": 186
    },
    {
      "epoch": 0.04918706537999967,
      "grad_norm": 510.0,
      "learning_rate": 2.454068241469816e-07,
      "logits/chosen": -0.6814215183258057,
      "logits/rejected": -0.686382532119751,
      "logps/chosen": -1175.1947021484375,
      "logps/rejected": -1096.156005859375,
      "loss": 0.6901,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0008806223049759865,
      "rewards/margins": 0.007646369747817516,
      "rewards/rejected": -0.006765746511518955,
      "step": 187
    },
    {
      "epoch": 0.04945009781518683,
      "grad_norm": 544.0,
      "learning_rate": 2.467191601049869e-07,
      "logits/chosen": -0.6429194211959839,
      "logits/rejected": -0.6560116410255432,
      "logps/chosen": -1130.4404296875,
      "logps/rejected": -886.9219360351562,
      "loss": 0.6892,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.005955316126346588,
      "rewards/margins": 0.008696556091308594,
      "rewards/rejected": -0.014651871286332607,
      "step": 188
    },
    {
      "epoch": 0.049713130250374,
      "grad_norm": 596.0,
      "learning_rate": 2.480314960629921e-07,
      "logits/chosen": -0.6632444262504578,
      "logits/rejected": -0.6708757281303406,
      "logps/chosen": -1242.6373291015625,
      "logps/rejected": -988.210205078125,
      "loss": 0.6996,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.010944461449980736,
      "rewards/margins": -0.01218943577259779,
      "rewards/rejected": 0.001244974322617054,
      "step": 189
    },
    {
      "epoch": 0.04997616268556116,
      "grad_norm": 420.0,
      "learning_rate": 2.493438320209974e-07,
      "logits/chosen": -0.6431402564048767,
      "logits/rejected": -0.6592947244644165,
      "logps/chosen": -897.5260620117188,
      "logps/rejected": -723.5248413085938,
      "loss": 0.6875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00712509173899889,
      "rewards/margins": 0.011575507931411266,
      "rewards/rejected": -0.004450416192412376,
      "step": 190
    },
    {
      "epoch": 0.05023919512074833,
      "grad_norm": 500.0,
      "learning_rate": 2.5065616797900265e-07,
      "logits/chosen": -0.6397775411605835,
      "logits/rejected": -0.6445972323417664,
      "logps/chosen": -1058.3369140625,
      "logps/rejected": -861.7650756835938,
      "loss": 0.6868,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.020495031028985977,
      "rewards/margins": 0.013363649137318134,
      "rewards/rejected": 0.007131386082619429,
      "step": 191
    },
    {
      "epoch": 0.05050222755593549,
      "grad_norm": 612.0,
      "learning_rate": 2.519685039370079e-07,
      "logits/chosen": -0.6534507274627686,
      "logits/rejected": -0.6537026762962341,
      "logps/chosen": -1639.73974609375,
      "logps/rejected": -931.2974853515625,
      "loss": 0.6966,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.001328229671344161,
      "rewards/margins": -0.005866288207471371,
      "rewards/rejected": 0.004538059700280428,
      "step": 192
    },
    {
      "epoch": 0.050765259991122656,
      "grad_norm": 596.0,
      "learning_rate": 2.532808398950131e-07,
      "logits/chosen": -0.6411073803901672,
      "logits/rejected": -0.6294708847999573,
      "logps/chosen": -1098.517333984375,
      "logps/rejected": -902.0569458007812,
      "loss": 0.7041,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.013903905637562275,
      "rewards/margins": -0.020866677165031433,
      "rewards/rejected": 0.006962775718420744,
      "step": 193
    },
    {
      "epoch": 0.05102829242630982,
      "grad_norm": 502.0,
      "learning_rate": 2.545931758530184e-07,
      "logits/chosen": -0.6629586219787598,
      "logits/rejected": -0.6667455434799194,
      "logps/chosen": -929.1187744140625,
      "logps/rejected": -803.96337890625,
      "loss": 0.6949,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.023139096796512604,
      "rewards/margins": -0.0024677275214344263,
      "rewards/rejected": -0.020671367645263672,
      "step": 194
    },
    {
      "epoch": 0.051291324861496984,
      "grad_norm": 548.0,
      "learning_rate": 2.559055118110236e-07,
      "logits/chosen": -0.6544424295425415,
      "logits/rejected": -0.660040020942688,
      "logps/chosen": -1043.0499267578125,
      "logps/rejected": -886.1975708007812,
      "loss": 0.6883,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.002444744575768709,
      "rewards/margins": 0.010622883215546608,
      "rewards/rejected": -0.01306762732565403,
      "step": 195
    },
    {
      "epoch": 0.051554357296684145,
      "grad_norm": 572.0,
      "learning_rate": 2.572178477690289e-07,
      "logits/chosen": -0.6630423069000244,
      "logits/rejected": -0.6706511974334717,
      "logps/chosen": -1169.66015625,
      "logps/rejected": -962.062744140625,
      "loss": 0.6957,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.011507797986268997,
      "rewards/margins": -0.00401687528938055,
      "rewards/rejected": 0.015524674206972122,
      "step": 196
    },
    {
      "epoch": 0.05181738973187131,
      "grad_norm": 576.0,
      "learning_rate": 2.585301837270341e-07,
      "logits/chosen": -0.6509425044059753,
      "logits/rejected": -0.6654580235481262,
      "logps/chosen": -1336.3155517578125,
      "logps/rejected": -1071.27685546875,
      "loss": 0.6896,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.014598083682358265,
      "rewards/margins": 0.0075977337546646595,
      "rewards/rejected": 0.007000351324677467,
      "step": 197
    },
    {
      "epoch": 0.05208042216705847,
      "grad_norm": 496.0,
      "learning_rate": 2.5984251968503933e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6487581133842468,
      "logps/chosen": -824.3193359375,
      "logps/rejected": -1046.800048828125,
      "loss": 0.6844,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.004351425915956497,
      "rewards/margins": 0.018295764923095703,
      "rewards/rejected": -0.013944339007139206,
      "step": 198
    },
    {
      "epoch": 0.05234345460224564,
      "grad_norm": 540.0,
      "learning_rate": 2.611548556430446e-07,
      "logits/chosen": -0.6457973122596741,
      "logits/rejected": -0.6428192257881165,
      "logps/chosen": -1166.8922119140625,
      "logps/rejected": -769.8594970703125,
      "loss": 0.6835,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.007059955503791571,
      "rewards/margins": 0.0205046646296978,
      "rewards/rejected": -0.013444710522890091,
      "step": 199
    },
    {
      "epoch": 0.0526064870374328,
      "grad_norm": 488.0,
      "learning_rate": 2.6246719160104984e-07,
      "logits/chosen": -0.6343008279800415,
      "logits/rejected": -0.643351674079895,
      "logps/chosen": -711.837646484375,
      "logps/rejected": -684.895751953125,
      "loss": 0.7116,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.006773758679628372,
      "rewards/margins": -0.035717107355594635,
      "rewards/rejected": 0.028943348675966263,
      "step": 200
    },
    {
      "epoch": 0.05286951947261997,
      "grad_norm": 620.0,
      "learning_rate": 2.6377952755905506e-07,
      "logits/chosen": -0.6603890657424927,
      "logits/rejected": -0.6507285833358765,
      "logps/chosen": -1276.2965087890625,
      "logps/rejected": -1128.033203125,
      "loss": 0.6935,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.013014602474868298,
      "rewards/margins": -0.0004647737368941307,
      "rewards/rejected": 0.013479376211762428,
      "step": 201
    },
    {
      "epoch": 0.05313255190780713,
      "grad_norm": 532.0,
      "learning_rate": 2.650918635170604e-07,
      "logits/chosen": -0.661770761013031,
      "logits/rejected": -0.6687109470367432,
      "logps/chosen": -1096.3887939453125,
      "logps/rejected": -1005.1268920898438,
      "loss": 0.6887,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.007310677319765091,
      "rewards/margins": 0.009572505950927734,
      "rewards/rejected": -0.0022618304938077927,
      "step": 202
    },
    {
      "epoch": 0.053395584342994296,
      "grad_norm": 446.0,
      "learning_rate": 2.664041994750656e-07,
      "logits/chosen": -0.6327085494995117,
      "logits/rejected": -0.6399305462837219,
      "logps/chosen": -865.3645629882812,
      "logps/rejected": -641.5222778320312,
      "loss": 0.6952,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.008043194189667702,
      "rewards/margins": -0.0035264971666038036,
      "rewards/rejected": -0.004516697954386473,
      "step": 203
    },
    {
      "epoch": 0.05365861677818146,
      "grad_norm": 684.0,
      "learning_rate": 2.677165354330709e-07,
      "logits/chosen": -0.6260887980461121,
      "logits/rejected": -0.6450051069259644,
      "logps/chosen": -1170.5355224609375,
      "logps/rejected": -1013.0426025390625,
      "loss": 0.6888,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.010009098798036575,
      "rewards/margins": 0.009754370898008347,
      "rewards/rejected": -0.019763469696044922,
      "step": 204
    },
    {
      "epoch": 0.05392164921336862,
      "grad_norm": 564.0,
      "learning_rate": 2.690288713910761e-07,
      "logits/chosen": -0.6525919437408447,
      "logits/rejected": -0.6500747203826904,
      "logps/chosen": -1279.8641357421875,
      "logps/rejected": -1190.08349609375,
      "loss": 0.6987,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.01746187172830105,
      "rewards/margins": -0.010366249829530716,
      "rewards/rejected": -0.007095622830092907,
      "step": 205
    },
    {
      "epoch": 0.05418468164855579,
      "grad_norm": 478.0,
      "learning_rate": 2.7034120734908134e-07,
      "logits/chosen": -0.6645711660385132,
      "logits/rejected": -0.6709896922111511,
      "logps/chosen": -920.47119140625,
      "logps/rejected": -713.474609375,
      "loss": 0.6871,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.001297378446906805,
      "rewards/margins": 0.013420678675174713,
      "rewards/rejected": -0.012123298831284046,
      "step": 206
    },
    {
      "epoch": 0.05444771408374295,
      "grad_norm": 448.0,
      "learning_rate": 2.716535433070866e-07,
      "logits/chosen": -0.6281125545501709,
      "logits/rejected": -0.6318945288658142,
      "logps/chosen": -894.6412353515625,
      "logps/rejected": -867.2469482421875,
      "loss": 0.6898,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.001167392823845148,
      "rewards/margins": 0.006877041421830654,
      "rewards/rejected": -0.005709648597985506,
      "step": 207
    },
    {
      "epoch": 0.05471074651893012,
      "grad_norm": 580.0,
      "learning_rate": 2.7296587926509185e-07,
      "logits/chosen": -0.6359333395957947,
      "logits/rejected": -0.6376537084579468,
      "logps/chosen": -1439.845947265625,
      "logps/rejected": -1240.228515625,
      "loss": 0.6874,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.016063310205936432,
      "rewards/margins": 0.012308408506214619,
      "rewards/rejected": 0.003754901699721813,
      "step": 208
    },
    {
      "epoch": 0.05497377895411728,
      "grad_norm": 492.0,
      "learning_rate": 2.7427821522309707e-07,
      "logits/chosen": -0.6489044427871704,
      "logits/rejected": -0.6361191868782043,
      "logps/chosen": -1027.7608642578125,
      "logps/rejected": -832.9791259765625,
      "loss": 0.6851,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004324627574533224,
      "rewards/margins": 0.01706666871905327,
      "rewards/rejected": -0.012742042541503906,
      "step": 209
    },
    {
      "epoch": 0.05523681138930445,
      "grad_norm": 500.0,
      "learning_rate": 2.7559055118110235e-07,
      "logits/chosen": -0.6445798873901367,
      "logits/rejected": -0.655692458152771,
      "logps/chosen": -1237.78955078125,
      "logps/rejected": -1039.4429931640625,
      "loss": 0.6938,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.001141166314482689,
      "rewards/margins": -0.0005393023602664471,
      "rewards/rejected": -0.0006018655840307474,
      "step": 210
    },
    {
      "epoch": 0.05549984382449161,
      "grad_norm": 484.0,
      "learning_rate": 2.769028871391076e-07,
      "logits/chosen": -0.6446409821510315,
      "logits/rejected": -0.648796796798706,
      "logps/chosen": -1088.5699462890625,
      "logps/rejected": -806.5174560546875,
      "loss": 0.705,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0034367565531283617,
      "rewards/margins": -0.02264413982629776,
      "rewards/rejected": 0.019207384437322617,
      "step": 211
    },
    {
      "epoch": 0.055762876259678774,
      "grad_norm": 572.0,
      "learning_rate": 2.7821522309711285e-07,
      "logits/chosen": -0.6703362464904785,
      "logits/rejected": -0.6645433306694031,
      "logps/chosen": -1161.7869873046875,
      "logps/rejected": -1191.700439453125,
      "loss": 0.6868,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.024042608216404915,
      "rewards/margins": 0.013963223434984684,
      "rewards/rejected": 0.01007938478142023,
      "step": 212
    },
    {
      "epoch": 0.056025908694865935,
      "grad_norm": 580.0,
      "learning_rate": 2.795275590551181e-07,
      "logits/chosen": -0.6570540070533752,
      "logits/rejected": -0.6600320935249329,
      "logps/chosen": -1254.09912109375,
      "logps/rejected": -1017.1392822265625,
      "loss": 0.6904,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.014642143622040749,
      "rewards/margins": 0.006773186381906271,
      "rewards/rejected": -0.021415332332253456,
      "step": 213
    },
    {
      "epoch": 0.0562889411300531,
      "grad_norm": 644.0,
      "learning_rate": 2.8083989501312335e-07,
      "logits/chosen": -0.6394319534301758,
      "logits/rejected": -0.6311585307121277,
      "logps/chosen": -1262.3450927734375,
      "logps/rejected": -1078.8646240234375,
      "loss": 0.7117,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.01698589138686657,
      "rewards/margins": -0.035050392150878906,
      "rewards/rejected": 0.018064498901367188,
      "step": 214
    },
    {
      "epoch": 0.05655197356524026,
      "grad_norm": 520.0,
      "learning_rate": 2.8215223097112863e-07,
      "logits/chosen": -0.6522241830825806,
      "logits/rejected": -0.6469199657440186,
      "logps/chosen": -890.803466796875,
      "logps/rejected": -906.4737548828125,
      "loss": 0.6989,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0074935914017260075,
      "rewards/margins": -0.010819530114531517,
      "rewards/rejected": 0.0033259389456361532,
      "step": 215
    },
    {
      "epoch": 0.05681500600042743,
      "grad_norm": 584.0,
      "learning_rate": 2.8346456692913386e-07,
      "logits/chosen": -0.6567167043685913,
      "logits/rejected": -0.665936291217804,
      "logps/chosen": -1420.06396484375,
      "logps/rejected": -1064.367431640625,
      "loss": 0.701,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.012634849175810814,
      "rewards/margins": -0.014315224252641201,
      "rewards/rejected": 0.02695007249712944,
      "step": 216
    },
    {
      "epoch": 0.05707803843561459,
      "grad_norm": 656.0,
      "learning_rate": 2.847769028871391e-07,
      "logits/chosen": -0.6079094409942627,
      "logits/rejected": -0.6294989585876465,
      "logps/chosen": -1606.7603759765625,
      "logps/rejected": -1345.4986572265625,
      "loss": 0.6987,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.013304710388183594,
      "rewards/margins": -0.009956072084605694,
      "rewards/rejected": -0.0033486373722553253,
      "step": 217
    },
    {
      "epoch": 0.05734107087080176,
      "grad_norm": 524.0,
      "learning_rate": 2.8608923884514436e-07,
      "logits/chosen": -0.6730504035949707,
      "logits/rejected": -0.6751164197921753,
      "logps/chosen": -931.9920654296875,
      "logps/rejected": -902.3679809570312,
      "loss": 0.707,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.023534012958407402,
      "rewards/margins": -0.02651824988424778,
      "rewards/rejected": 0.0029842383228242397,
      "step": 218
    },
    {
      "epoch": 0.05760410330598892,
      "grad_norm": 444.0,
      "learning_rate": 2.874015748031496e-07,
      "logits/chosen": -0.6676087379455566,
      "logits/rejected": -0.6636544466018677,
      "logps/chosen": -993.7527465820312,
      "logps/rejected": -740.0318603515625,
      "loss": 0.6844,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.027707383036613464,
      "rewards/margins": 0.018487263470888138,
      "rewards/rejected": 0.009220123291015625,
      "step": 219
    },
    {
      "epoch": 0.057867135741176086,
      "grad_norm": 628.0,
      "learning_rate": 2.8871391076115486e-07,
      "logits/chosen": -0.6577138304710388,
      "logits/rejected": -0.6511857509613037,
      "logps/chosen": -1542.944091796875,
      "logps/rejected": -1193.1319580078125,
      "loss": 0.6996,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0042772297747433186,
      "rewards/margins": -0.011966419406235218,
      "rewards/rejected": 0.016243647783994675,
      "step": 220
    },
    {
      "epoch": 0.058130168176363246,
      "grad_norm": 600.0,
      "learning_rate": 2.900262467191601e-07,
      "logits/chosen": -0.6522630453109741,
      "logits/rejected": NaN,
      "logps/chosen": -1375.6717529296875,
      "logps/rejected": -1066.1729736328125,
      "loss": 0.7162,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.019515419378876686,
      "rewards/margins": -0.04494886472821236,
      "rewards/rejected": 0.02543344721198082,
      "step": 221
    },
    {
      "epoch": 0.058393200611550414,
      "grad_norm": 608.0,
      "learning_rate": 2.913385826771653e-07,
      "logits/chosen": -0.6714162826538086,
      "logits/rejected": -0.6669720411300659,
      "logps/chosen": -838.969482421875,
      "logps/rejected": -600.285400390625,
      "loss": 0.684,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.014755535870790482,
      "rewards/margins": 0.01926608383655548,
      "rewards/rejected": -0.0045105465687811375,
      "step": 222
    },
    {
      "epoch": 0.058656233046737574,
      "grad_norm": 510.0,
      "learning_rate": 2.926509186351706e-07,
      "logits/chosen": -0.6207312941551208,
      "logits/rejected": -0.6353626251220703,
      "logps/chosen": -1291.47265625,
      "logps/rejected": -1164.3203125,
      "loss": 0.6931,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.015393543057143688,
      "rewards/margins": 0.0012891776859760284,
      "rewards/rejected": 0.014104367233812809,
      "step": 223
    },
    {
      "epoch": 0.05891926548192474,
      "grad_norm": 556.0,
      "learning_rate": 2.939632545931758e-07,
      "logits/chosen": -0.6373709440231323,
      "logits/rejected": -0.6328381896018982,
      "logps/chosen": -1143.54541015625,
      "logps/rejected": -1102.6002197265625,
      "loss": 0.6912,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00842742808163166,
      "rewards/margins": 0.004566859919577837,
      "rewards/rejected": 0.0038605695590376854,
      "step": 224
    },
    {
      "epoch": 0.0591822979171119,
      "grad_norm": 560.0,
      "learning_rate": 2.9527559055118104e-07,
      "logits/chosen": -0.6703706383705139,
      "logits/rejected": -0.6691528558731079,
      "logps/chosen": -1107.851318359375,
      "logps/rejected": -1155.216552734375,
      "loss": 0.6954,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.006274033337831497,
      "rewards/margins": -0.0036531458608806133,
      "rewards/rejected": 0.009927177801728249,
      "step": 225
    },
    {
      "epoch": 0.05944533035229907,
      "grad_norm": 624.0,
      "learning_rate": 2.9658792650918637e-07,
      "logits/chosen": -0.6391873955726624,
      "logits/rejected": -0.631041407585144,
      "logps/chosen": -1424.0523681640625,
      "logps/rejected": -1020.201904296875,
      "loss": 0.7004,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.003956986125558615,
      "rewards/margins": -0.01353378500789404,
      "rewards/rejected": 0.009576797485351562,
      "step": 226
    },
    {
      "epoch": 0.05970836278748623,
      "grad_norm": 620.0,
      "learning_rate": 2.979002624671916e-07,
      "logits/chosen": -0.6491602659225464,
      "logits/rejected": -0.6524882912635803,
      "logps/chosen": -1024.7144775390625,
      "logps/rejected": -992.422607421875,
      "loss": 0.7012,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.009047319181263447,
      "rewards/margins": -0.01537446677684784,
      "rewards/rejected": 0.02442178875207901,
      "step": 227
    },
    {
      "epoch": 0.0599713952226734,
      "grad_norm": 552.0,
      "learning_rate": 2.9921259842519687e-07,
      "logits/chosen": -0.652782142162323,
      "logits/rejected": -0.6508370637893677,
      "logps/chosen": -1178.6668701171875,
      "logps/rejected": -963.5601806640625,
      "loss": 0.6931,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00585174560546875,
      "rewards/margins": 0.0006870270008221269,
      "rewards/rejected": 0.00516471965238452,
      "step": 228
    },
    {
      "epoch": 0.06023442765786056,
      "grad_norm": 564.0,
      "learning_rate": 3.005249343832021e-07,
      "logits/chosen": -0.6550735235214233,
      "logits/rejected": -0.6512598395347595,
      "logps/chosen": -1120.5562744140625,
      "logps/rejected": -856.09228515625,
      "loss": 0.693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01975269243121147,
      "rewards/margins": 0.0011728284880518913,
      "rewards/rejected": 0.018579863011837006,
      "step": 229
    },
    {
      "epoch": 0.060497460093047725,
      "grad_norm": 548.0,
      "learning_rate": 3.018372703412073e-07,
      "logits/chosen": -0.6479599475860596,
      "logits/rejected": -0.6524536609649658,
      "logps/chosen": -1294.2763671875,
      "logps/rejected": -1098.02880859375,
      "loss": 0.6908,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0020080560352653265,
      "rewards/margins": 0.005371761508285999,
      "rewards/rejected": -0.007379817310720682,
      "step": 230
    },
    {
      "epoch": 0.060760492528234886,
      "grad_norm": 528.0,
      "learning_rate": 3.031496062992126e-07,
      "logits/chosen": -0.6615718603134155,
      "logits/rejected": -0.6657954454421997,
      "logps/chosen": -1437.4237060546875,
      "logps/rejected": -1274.1492919921875,
      "loss": 0.6877,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.012323187664151192,
      "rewards/margins": 0.012197209522128105,
      "rewards/rejected": -0.024520397186279297,
      "step": 231
    },
    {
      "epoch": 0.06102352496342205,
      "grad_norm": 490.0,
      "learning_rate": 3.044619422572178e-07,
      "logits/chosen": -0.6470730900764465,
      "logits/rejected": NaN,
      "logps/chosen": -1320.71240234375,
      "logps/rejected": -758.5291748046875,
      "loss": 0.6933,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.013622760772705078,
      "rewards/margins": 0.0008701332844793797,
      "rewards/rejected": -0.01449289359152317,
      "step": 232
    },
    {
      "epoch": 0.061286557398609214,
      "grad_norm": 492.0,
      "learning_rate": 3.0577427821522305e-07,
      "logits/chosen": -0.6666160225868225,
      "logits/rejected": -0.6720753908157349,
      "logps/chosen": -735.7352294921875,
      "logps/rejected": -781.3399047851562,
      "loss": 0.6946,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0076745981350541115,
      "rewards/margins": -0.0021949773654341698,
      "rewards/rejected": -0.005479621235281229,
      "step": 233
    },
    {
      "epoch": 0.06154958983379638,
      "grad_norm": 640.0,
      "learning_rate": 3.0708661417322833e-07,
      "logits/chosen": -0.6626201272010803,
      "logits/rejected": -0.6691379547119141,
      "logps/chosen": -1063.3154296875,
      "logps/rejected": -1289.497314453125,
      "loss": 0.6898,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.01085195504128933,
      "rewards/margins": 0.007399273570626974,
      "rewards/rejected": -0.01825122907757759,
      "step": 234
    },
    {
      "epoch": 0.06181262226898354,
      "grad_norm": 510.0,
      "learning_rate": 3.0839895013123355e-07,
      "logits/chosen": -0.6659311056137085,
      "logits/rejected": NaN,
      "logps/chosen": -1129.597900390625,
      "logps/rejected": -842.25634765625,
      "loss": 0.6865,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.006999111734330654,
      "rewards/margins": 0.014112666249275208,
      "rewards/rejected": -0.02111177332699299,
      "step": 235
    },
    {
      "epoch": 0.06207565470417071,
      "grad_norm": 552.0,
      "learning_rate": 3.0971128608923883e-07,
      "logits/chosen": -0.6754056215286255,
      "logits/rejected": -0.679322361946106,
      "logps/chosen": -1273.062255859375,
      "logps/rejected": -1240.431640625,
      "loss": 0.6913,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01623544655740261,
      "rewards/margins": 0.0042869579046964645,
      "rewards/rejected": -0.020522404462099075,
      "step": 236
    },
    {
      "epoch": 0.06233868713935787,
      "grad_norm": 516.0,
      "learning_rate": 3.110236220472441e-07,
      "logits/chosen": -0.6596537828445435,
      "logits/rejected": -0.6560080051422119,
      "logps/chosen": -1224.908203125,
      "logps/rejected": -968.6658325195312,
      "loss": 0.6909,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012663649395108223,
      "rewards/margins": 0.0053750984370708466,
      "rewards/rejected": 0.007288550958037376,
      "step": 237
    },
    {
      "epoch": 0.06260171957454504,
      "grad_norm": 584.0,
      "learning_rate": 3.1233595800524933e-07,
      "logits/chosen": -0.6579158306121826,
      "logits/rejected": -0.6561657190322876,
      "logps/chosen": -1041.2425537109375,
      "logps/rejected": -1025.013427734375,
      "loss": 0.6991,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.005525493994355202,
      "rewards/margins": -0.010969829745590687,
      "rewards/rejected": 0.016495324671268463,
      "step": 238
    },
    {
      "epoch": 0.0628647520097322,
      "grad_norm": 496.0,
      "learning_rate": 3.136482939632546e-07,
      "logits/chosen": -0.6609362959861755,
      "logits/rejected": -0.6639521718025208,
      "logps/chosen": -1059.1962890625,
      "logps/rejected": -742.7137451171875,
      "loss": 0.6996,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01297235582023859,
      "rewards/margins": -0.011010886169970036,
      "rewards/rejected": -0.0019614705815911293,
      "step": 239
    },
    {
      "epoch": 0.06312778444491936,
      "grad_norm": 510.0,
      "learning_rate": 3.1496062992125984e-07,
      "logits/chosen": -0.6785593032836914,
      "logits/rejected": -0.6541599631309509,
      "logps/chosen": -1284.397216796875,
      "logps/rejected": -867.9552612304688,
      "loss": 0.6797,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.006996249780058861,
      "rewards/margins": 0.028154563158750534,
      "rewards/rejected": -0.035150814801454544,
      "step": 240
    },
    {
      "epoch": 0.06339081688010653,
      "grad_norm": 520.0,
      "learning_rate": 3.1627296587926506e-07,
      "logits/chosen": -0.645885705947876,
      "logits/rejected": -0.6419854164123535,
      "logps/chosen": -983.212646484375,
      "logps/rejected": -752.5016479492188,
      "loss": 0.6946,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0042947763577103615,
      "rewards/margins": -0.002197647001594305,
      "rewards/rejected": -0.0020971298217773438,
      "step": 241
    },
    {
      "epoch": 0.06365384931529369,
      "grad_norm": 548.0,
      "learning_rate": 3.1758530183727034e-07,
      "logits/chosen": -0.6618828773498535,
      "logits/rejected": NaN,
      "logps/chosen": -1044.2054443359375,
      "logps/rejected": -753.8661499023438,
      "loss": 0.6924,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0036457059904932976,
      "rewards/margins": 0.002443697303533554,
      "rewards/rejected": -0.006089404225349426,
      "step": 242
    },
    {
      "epoch": 0.06391688175048085,
      "grad_norm": 410.0,
      "learning_rate": 3.1889763779527556e-07,
      "logits/chosen": -0.6533990502357483,
      "logits/rejected": -0.6601124405860901,
      "logps/chosen": -829.646240234375,
      "logps/rejected": -845.0447998046875,
      "loss": 0.6882,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.001080273650586605,
      "rewards/margins": 0.010344361886382103,
      "rewards/rejected": -0.009264087304472923,
      "step": 243
    },
    {
      "epoch": 0.06417991418566801,
      "grad_norm": 628.0,
      "learning_rate": 3.2020997375328084e-07,
      "logits/chosen": -0.6528266072273254,
      "logits/rejected": -0.6577106714248657,
      "logps/chosen": -1451.298583984375,
      "logps/rejected": -860.9277954101562,
      "loss": 0.683,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01164855808019638,
      "rewards/margins": 0.021075531840324402,
      "rewards/rejected": -0.00942697562277317,
      "step": 244
    },
    {
      "epoch": 0.06444294662085519,
      "grad_norm": 486.0,
      "learning_rate": 3.2152230971128607e-07,
      "logits/chosen": -0.6530910730361938,
      "logits/rejected": -0.6482900381088257,
      "logps/chosen": -1128.6920166015625,
      "logps/rejected": -1035.798583984375,
      "loss": 0.6969,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.02426758036017418,
      "rewards/margins": -0.006976031698286533,
      "rewards/rejected": -0.017291545867919922,
      "step": 245
    },
    {
      "epoch": 0.06470597905604235,
      "grad_norm": 548.0,
      "learning_rate": 3.228346456692913e-07,
      "logits/chosen": -0.6404434442520142,
      "logits/rejected": -0.6531354784965515,
      "logps/chosen": -1043.7762451171875,
      "logps/rejected": -715.377197265625,
      "loss": 0.6982,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.008378506638109684,
      "rewards/margins": -0.009554006159305573,
      "rewards/rejected": 0.0011754999868571758,
      "step": 246
    },
    {
      "epoch": 0.06496901149122951,
      "grad_norm": 524.0,
      "learning_rate": 3.2414698162729657e-07,
      "logits/chosen": -0.6558781862258911,
      "logits/rejected": -0.6615175604820251,
      "logps/chosen": -1052.2518310546875,
      "logps/rejected": -852.4507446289062,
      "loss": 0.7096,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.011655425652861595,
      "rewards/margins": -0.03171500936150551,
      "rewards/rejected": 0.020059585571289062,
      "step": 247
    },
    {
      "epoch": 0.06523204392641668,
      "grad_norm": 604.0,
      "learning_rate": 3.254593175853018e-07,
      "logits/chosen": -0.6315042972564697,
      "logits/rejected": -0.6293298006057739,
      "logps/chosen": -1448.3974609375,
      "logps/rejected": -861.8954467773438,
      "loss": 0.6916,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0020496370270848274,
      "rewards/margins": 0.0037741665728390217,
      "rewards/rejected": -0.0017245309427380562,
      "step": 248
    },
    {
      "epoch": 0.06549507636160384,
      "grad_norm": 580.0,
      "learning_rate": 3.2677165354330707e-07,
      "logits/chosen": -0.6569128036499023,
      "logits/rejected": -0.6426726579666138,
      "logps/chosen": -1092.09765625,
      "logps/rejected": -661.063720703125,
      "loss": 0.6977,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01204080693423748,
      "rewards/margins": -0.00797433964908123,
      "rewards/rejected": -0.00406646728515625,
      "step": 249
    },
    {
      "epoch": 0.065758108796791,
      "grad_norm": 648.0,
      "learning_rate": 3.2808398950131235e-07,
      "logits/chosen": -0.6278527975082397,
      "logits/rejected": NaN,
      "logps/chosen": -1674.215576171875,
      "logps/rejected": -1336.81982421875,
      "loss": 0.6974,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.040663909167051315,
      "rewards/margins": -0.007493495941162109,
      "rewards/rejected": -0.033170416951179504,
      "step": 250
    },
    {
      "epoch": 0.06602114123197816,
      "grad_norm": 624.0,
      "learning_rate": 3.2939632545931757e-07,
      "logits/chosen": -0.6667503118515015,
      "logits/rejected": -0.6659904718399048,
      "logps/chosen": -1703.78662109375,
      "logps/rejected": -1419.568115234375,
      "loss": 0.6868,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.007192706689238548,
      "rewards/margins": 0.013651085086166859,
      "rewards/rejected": -0.02084379270672798,
      "step": 251
    },
    {
      "epoch": 0.06628417366716534,
      "grad_norm": 600.0,
      "learning_rate": 3.3070866141732285e-07,
      "logits/chosen": -0.6412410736083984,
      "logits/rejected": NaN,
      "logps/chosen": -1229.572265625,
      "logps/rejected": -651.200927734375,
      "loss": 0.6863,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0020234095863997936,
      "rewards/margins": 0.015001010149717331,
      "rewards/rejected": -0.01297760009765625,
      "step": 252
    },
    {
      "epoch": 0.0665472061023525,
      "grad_norm": 576.0,
      "learning_rate": 3.320209973753281e-07,
      "logits/chosen": -0.6711891293525696,
      "logits/rejected": -0.6826565265655518,
      "logps/chosen": -1667.0850830078125,
      "logps/rejected": -1143.8936767578125,
      "loss": 0.6953,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.017447471618652344,
      "rewards/margins": -0.0034441472962498665,
      "rewards/rejected": -0.014003324322402477,
      "step": 253
    },
    {
      "epoch": 0.06681023853753966,
      "grad_norm": 416.0,
      "learning_rate": 3.333333333333333e-07,
      "logits/chosen": -0.644903302192688,
      "logits/rejected": -0.6564706563949585,
      "logps/chosen": -799.4824829101562,
      "logps/rejected": -547.0015869140625,
      "loss": 0.6954,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006193828769028187,
      "rewards/margins": -0.0037369728088378906,
      "rewards/rejected": 0.009930801577866077,
      "step": 254
    },
    {
      "epoch": 0.06707327097272682,
      "grad_norm": 438.0,
      "learning_rate": 3.346456692913386e-07,
      "logits/chosen": -0.6638983488082886,
      "logits/rejected": -0.6577736139297485,
      "logps/chosen": -1154.508544921875,
      "logps/rejected": -726.4736938476562,
      "loss": 0.689,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.005758953280746937,
      "rewards/margins": 0.008807563222944736,
      "rewards/rejected": -0.014566516503691673,
      "step": 255
    },
    {
      "epoch": 0.067336303407914,
      "grad_norm": 556.0,
      "learning_rate": 3.359580052493438e-07,
      "logits/chosen": -0.6391483545303345,
      "logits/rejected": -0.6351225972175598,
      "logps/chosen": -1337.34814453125,
      "logps/rejected": -1218.4527587890625,
      "loss": 0.704,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.03294835239648819,
      "rewards/margins": -0.02093386836349964,
      "rewards/rejected": -0.012014484032988548,
      "step": 256
    },
    {
      "epoch": 0.06759933584310115,
      "grad_norm": 580.0,
      "learning_rate": 3.3727034120734903e-07,
      "logits/chosen": -0.6529752612113953,
      "logits/rejected": -0.646112859249115,
      "logps/chosen": -1109.556396484375,
      "logps/rejected": -867.5125732421875,
      "loss": 0.6939,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008285046555101871,
      "rewards/margins": 0.00045356713235378265,
      "rewards/rejected": -0.008738611824810505,
      "step": 257
    },
    {
      "epoch": 0.06786236827828832,
      "grad_norm": 568.0,
      "learning_rate": 3.385826771653543e-07,
      "logits/chosen": -0.6767556071281433,
      "logits/rejected": -0.6795201301574707,
      "logps/chosen": -1327.17578125,
      "logps/rejected": -1014.0072021484375,
      "loss": 0.6865,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0010107513517141342,
      "rewards/margins": 0.013786411844193935,
      "rewards/rejected": -0.01479716319590807,
      "step": 258
    },
    {
      "epoch": 0.06812540071347548,
      "grad_norm": 474.0,
      "learning_rate": 3.3989501312335953e-07,
      "logits/chosen": -0.659292459487915,
      "logits/rejected": -0.6629323363304138,
      "logps/chosen": -871.2627563476562,
      "logps/rejected": -595.5875854492188,
      "loss": 0.6944,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.021514413878321648,
      "rewards/margins": -0.0021037100814282894,
      "rewards/rejected": -0.01941070519387722,
      "step": 259
    },
    {
      "epoch": 0.06838843314866265,
      "grad_norm": 752.0,
      "learning_rate": 3.4120734908136486e-07,
      "logits/chosen": -0.6758798360824585,
      "logits/rejected": NaN,
      "logps/chosen": -1422.399169921875,
      "logps/rejected": -1225.0625,
      "loss": 0.7045,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.004854393657296896,
      "rewards/margins": -0.021451760083436966,
      "rewards/rejected": 0.016597365960478783,
      "step": 260
    },
    {
      "epoch": 0.06865146558384981,
      "grad_norm": 604.0,
      "learning_rate": 3.425196850393701e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6770823001861572,
      "logps/chosen": -1138.8768310546875,
      "logps/rejected": -1109.4735107421875,
      "loss": 0.6983,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.008134364150464535,
      "rewards/margins": -0.008702229708433151,
      "rewards/rejected": 0.01683659665286541,
      "step": 261
    },
    {
      "epoch": 0.06891449801903697,
      "grad_norm": 498.0,
      "learning_rate": 3.438320209973753e-07,
      "logits/chosen": -0.6510261297225952,
      "logits/rejected": -0.6527219414710999,
      "logps/chosen": -1127.134765625,
      "logps/rejected": -1041.7783203125,
      "loss": 0.6893,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0032745355274528265,
      "rewards/margins": 0.008713483810424805,
      "rewards/rejected": -0.01198802050203085,
      "step": 262
    },
    {
      "epoch": 0.06917753045422413,
      "grad_norm": 464.0,
      "learning_rate": 3.451443569553806e-07,
      "logits/chosen": -0.6522761583328247,
      "logits/rejected": -0.6600165963172913,
      "logps/chosen": -879.314453125,
      "logps/rejected": -948.681640625,
      "loss": 0.6928,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0007865899242460728,
      "rewards/margins": 0.0010147090069949627,
      "rewards/rejected": -0.00022811908274888992,
      "step": 263
    },
    {
      "epoch": 0.0694405628894113,
      "grad_norm": 496.0,
      "learning_rate": 3.464566929133858e-07,
      "logits/chosen": -0.6525784134864807,
      "logits/rejected": -0.6586518287658691,
      "logps/chosen": -1160.165283203125,
      "logps/rejected": -615.3844604492188,
      "loss": 0.6844,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009471512399613857,
      "rewards/margins": 0.018221475183963776,
      "rewards/rejected": -0.008749961853027344,
      "step": 264
    },
    {
      "epoch": 0.06970359532459847,
      "grad_norm": 696.0,
      "learning_rate": 3.4776902887139104e-07,
      "logits/chosen": -0.6704614162445068,
      "logits/rejected": -0.664938747882843,
      "logps/chosen": -1000.4151000976562,
      "logps/rejected": -1049.18408203125,
      "loss": 0.7069,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0024328234139829874,
      "rewards/margins": -0.02615041844546795,
      "rewards/rejected": 0.023717595264315605,
      "step": 265
    },
    {
      "epoch": 0.06996662775978563,
      "grad_norm": 532.0,
      "learning_rate": 3.490813648293963e-07,
      "logits/chosen": -0.6293448209762573,
      "logits/rejected": -0.643260657787323,
      "logps/chosen": -1504.987548828125,
      "logps/rejected": -1369.097412109375,
      "loss": 0.7005,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.001880359835922718,
      "rewards/margins": -0.013903332874178886,
      "rewards/rejected": 0.01578368991613388,
      "step": 266
    },
    {
      "epoch": 0.07022966019497279,
      "grad_norm": 346.0,
      "learning_rate": 3.5039370078740154e-07,
      "logits/chosen": -0.6376094818115234,
      "logits/rejected": -0.6307231783866882,
      "logps/chosen": -939.0899658203125,
      "logps/rejected": -1027.8807373046875,
      "loss": 0.6807,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.008753776550292969,
      "rewards/margins": 0.025488758459687233,
      "rewards/rejected": -0.016734980046749115,
      "step": 267
    },
    {
      "epoch": 0.07049269263015996,
      "grad_norm": 492.0,
      "learning_rate": 3.517060367454068e-07,
      "logits/chosen": -0.6313166618347168,
      "logits/rejected": -0.6396786570549011,
      "logps/chosen": -823.7127685546875,
      "logps/rejected": -684.97216796875,
      "loss": 0.6843,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.001386165153235197,
      "rewards/margins": 0.018451310694217682,
      "rewards/rejected": -0.019837476313114166,
      "step": 268
    },
    {
      "epoch": 0.07075572506534712,
      "grad_norm": 680.0,
      "learning_rate": 3.5301837270341204e-07,
      "logits/chosen": -0.6303259134292603,
      "logits/rejected": -0.6234923601150513,
      "logps/chosen": -1117.355224609375,
      "logps/rejected": -956.0250244140625,
      "loss": 0.6989,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02456073835492134,
      "rewards/margins": -0.010167882777750492,
      "rewards/rejected": 0.03472862392663956,
      "step": 269
    },
    {
      "epoch": 0.07101875750053428,
      "grad_norm": 496.0,
      "learning_rate": 3.5433070866141727e-07,
      "logits/chosen": -0.6447986960411072,
      "logits/rejected": -0.6424817442893982,
      "logps/chosen": -998.4840698242188,
      "logps/rejected": -737.5504150390625,
      "loss": 0.6895,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005258751101791859,
      "rewards/margins": 0.007679367437958717,
      "rewards/rejected": -0.0024206158705055714,
      "step": 270
    },
    {
      "epoch": 0.07128178993572144,
      "grad_norm": 652.0,
      "learning_rate": 3.5564304461942255e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6634913682937622,
      "logps/chosen": -1054.9622802734375,
      "logps/rejected": -1164.832275390625,
      "loss": 0.6977,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.021865272894501686,
      "rewards/margins": -0.008540153503417969,
      "rewards/rejected": -0.013325119391083717,
      "step": 271
    },
    {
      "epoch": 0.07154482237090862,
      "grad_norm": 528.0,
      "learning_rate": 3.569553805774278e-07,
      "logits/chosen": -0.6487042903900146,
      "logits/rejected": -0.6430377960205078,
      "logps/chosen": -1115.3856201171875,
      "logps/rejected": -1104.4427490234375,
      "loss": 0.704,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.006620311643928289,
      "rewards/margins": -0.020443344488739967,
      "rewards/rejected": 0.013823031447827816,
      "step": 272
    },
    {
      "epoch": 0.07180785480609578,
      "grad_norm": 500.0,
      "learning_rate": 3.5826771653543305e-07,
      "logits/chosen": -0.6656239032745361,
      "logits/rejected": -0.6741122007369995,
      "logps/chosen": -1028.3631591796875,
      "logps/rejected": -814.8997802734375,
      "loss": 0.6869,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.024081135168671608,
      "rewards/margins": 0.013095282949507236,
      "rewards/rejected": 0.010985852219164371,
      "step": 273
    },
    {
      "epoch": 0.07207088724128294,
      "grad_norm": 516.0,
      "learning_rate": 3.5958005249343833e-07,
      "logits/chosen": -0.680946409702301,
      "logits/rejected": -0.6814238429069519,
      "logps/chosen": -928.3062744140625,
      "logps/rejected": -752.7227783203125,
      "loss": 0.6949,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.001876449678093195,
      "rewards/margins": -0.0028281209524720907,
      "rewards/rejected": 0.0009516717400401831,
      "step": 274
    },
    {
      "epoch": 0.0723339196764701,
      "grad_norm": 524.0,
      "learning_rate": 3.6089238845144355e-07,
      "logits/chosen": -0.6215833425521851,
      "logits/rejected": -0.6319592595100403,
      "logps/chosen": -1199.7894287109375,
      "logps/rejected": -1123.418701171875,
      "loss": 0.6974,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008648490533232689,
      "rewards/margins": -0.007791042327880859,
      "rewards/rejected": 0.016439534723758698,
      "step": 275
    },
    {
      "epoch": 0.07259695211165727,
      "grad_norm": 452.0,
      "learning_rate": 3.6220472440944883e-07,
      "logits/chosen": -0.6505846977233887,
      "logits/rejected": -0.6514597535133362,
      "logps/chosen": -809.4046630859375,
      "logps/rejected": -832.36083984375,
      "loss": 0.6824,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.008590411394834518,
      "rewards/margins": 0.022570420056581497,
      "rewards/rejected": -0.013980008661746979,
      "step": 276
    },
    {
      "epoch": 0.07285998454684443,
      "grad_norm": 596.0,
      "learning_rate": 3.6351706036745405e-07,
      "logits/chosen": -0.648588240146637,
      "logits/rejected": -0.652485728263855,
      "logps/chosen": -1113.803466796875,
      "logps/rejected": -932.2403564453125,
      "loss": 0.7082,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0023553851060569286,
      "rewards/margins": -0.02849569357931614,
      "rewards/rejected": 0.026140308007597923,
      "step": 277
    },
    {
      "epoch": 0.0731230169820316,
      "grad_norm": 480.0,
      "learning_rate": 3.648293963254593e-07,
      "logits/chosen": -0.648916482925415,
      "logits/rejected": -0.6551617383956909,
      "logps/chosen": -1252.36474609375,
      "logps/rejected": -1066.474853515625,
      "loss": 0.688,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.014356088824570179,
      "rewards/margins": 0.01140599325299263,
      "rewards/rejected": -0.025762081146240234,
      "step": 278
    },
    {
      "epoch": 0.07338604941721875,
      "grad_norm": 498.0,
      "learning_rate": 3.6614173228346456e-07,
      "logits/chosen": -0.6281301379203796,
      "logits/rejected": -0.6223104596138,
      "logps/chosen": -1060.449462890625,
      "logps/rejected": -822.7742919921875,
      "loss": 0.6881,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.02236786112189293,
      "rewards/margins": 0.011000441387295723,
      "rewards/rejected": 0.011367416940629482,
      "step": 279
    },
    {
      "epoch": 0.07364908185240593,
      "grad_norm": 504.0,
      "learning_rate": 3.674540682414698e-07,
      "logits/chosen": -0.6149517893791199,
      "logits/rejected": -0.6394557356834412,
      "logps/chosen": -1295.5799560546875,
      "logps/rejected": -1210.942138671875,
      "loss": 0.6952,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.021969985216856003,
      "rewards/margins": -0.0034797671250998974,
      "rewards/rejected": 0.025449752807617188,
      "step": 280
    },
    {
      "epoch": 0.07391211428759309,
      "grad_norm": 496.0,
      "learning_rate": 3.68766404199475e-07,
      "logits/chosen": -0.669288694858551,
      "logits/rejected": -0.6624835133552551,
      "logps/chosen": -914.8475952148438,
      "logps/rejected": -611.3168334960938,
      "loss": 0.6937,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.006021023262292147,
      "rewards/margins": -0.0005590431392192841,
      "rewards/rejected": -0.005461979191750288,
      "step": 281
    },
    {
      "epoch": 0.07417514672278025,
      "grad_norm": 656.0,
      "learning_rate": 3.700787401574803e-07,
      "logits/chosen": -0.6875889301300049,
      "logits/rejected": -0.6871373653411865,
      "logps/chosen": -1146.330078125,
      "logps/rejected": -1111.092041015625,
      "loss": 0.6802,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01683368720114231,
      "rewards/margins": 0.026833057403564453,
      "rewards/rejected": -0.009999371133744717,
      "step": 282
    },
    {
      "epoch": 0.07443817915796741,
      "grad_norm": 504.0,
      "learning_rate": 3.7139107611548556e-07,
      "logits/chosen": -0.6377272605895996,
      "logits/rejected": -0.6574225425720215,
      "logps/chosen": -1145.170166015625,
      "logps/rejected": -779.3514404296875,
      "loss": 0.6942,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0003867150517180562,
      "rewards/margins": -0.0017632485833019018,
      "rewards/rejected": 0.0013765335315838456,
      "step": 283
    },
    {
      "epoch": 0.07470121159315458,
      "grad_norm": 524.0,
      "learning_rate": 3.7270341207349084e-07,
      "logits/chosen": -0.6564843058586121,
      "logits/rejected": NaN,
      "logps/chosen": -1085.8931884765625,
      "logps/rejected": -585.3638305664062,
      "loss": 0.6924,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009985638782382011,
      "rewards/margins": 0.0022500022314488888,
      "rewards/rejected": -0.012235641479492188,
      "step": 284
    },
    {
      "epoch": 0.07496424402834174,
      "grad_norm": 620.0,
      "learning_rate": 3.7401574803149606e-07,
      "logits/chosen": -0.6380633115768433,
      "logits/rejected": -0.6366631388664246,
      "logps/chosen": -1164.2335205078125,
      "logps/rejected": -835.4154663085938,
      "loss": 0.6893,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005470752716064453,
      "rewards/margins": 0.008221054449677467,
      "rewards/rejected": -0.0027503015007823706,
      "step": 285
    },
    {
      "epoch": 0.0752272764635289,
      "grad_norm": 520.0,
      "learning_rate": 3.753280839895013e-07,
      "logits/chosen": -0.6355050206184387,
      "logits/rejected": -0.6419182419776917,
      "logps/chosen": -909.714599609375,
      "logps/rejected": -861.7562255859375,
      "loss": 0.6939,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010997294448316097,
      "rewards/margins": -0.0008930200710892677,
      "rewards/rejected": 0.011890316382050514,
      "step": 286
    },
    {
      "epoch": 0.07549030889871608,
      "grad_norm": 592.0,
      "learning_rate": 3.7664041994750657e-07,
      "logits/chosen": -0.6612749695777893,
      "logits/rejected": -0.6672670841217041,
      "logps/chosen": -1470.8828125,
      "logps/rejected": -981.255615234375,
      "loss": 0.6984,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.019954873248934746,
      "rewards/margins": -0.01004114281386137,
      "rewards/rejected": -0.009913730435073376,
      "step": 287
    },
    {
      "epoch": 0.07575334133390324,
      "grad_norm": 528.0,
      "learning_rate": 3.779527559055118e-07,
      "logits/chosen": -0.6621054410934448,
      "logits/rejected": -0.6694252490997314,
      "logps/chosen": -951.9699096679688,
      "logps/rejected": -764.3212890625,
      "loss": 0.6928,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0014675145503133535,
      "rewards/margins": 0.0016566766425967216,
      "rewards/rejected": -0.0031241897959262133,
      "step": 288
    },
    {
      "epoch": 0.0760163737690904,
      "grad_norm": 580.0,
      "learning_rate": 3.79265091863517e-07,
      "logits/chosen": -0.6492139101028442,
      "logits/rejected": -0.6513727307319641,
      "logps/chosen": -1156.82275390625,
      "logps/rejected": -958.750244140625,
      "loss": 0.7032,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.007457638159394264,
      "rewards/margins": -0.017909908667206764,
      "rewards/rejected": 0.0104522705078125,
      "step": 289
    },
    {
      "epoch": 0.07627940620427756,
      "grad_norm": 446.0,
      "learning_rate": 3.805774278215223e-07,
      "logits/chosen": -0.6591653227806091,
      "logits/rejected": -0.6685588359832764,
      "logps/chosen": -830.6842651367188,
      "logps/rejected": -604.954833984375,
      "loss": 0.6939,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0005403998075053096,
      "rewards/margins": -0.0008945935405790806,
      "rewards/rejected": 0.001434993464499712,
      "step": 290
    },
    {
      "epoch": 0.07654243863946474,
      "grad_norm": 564.0,
      "learning_rate": 3.818897637795275e-07,
      "logits/chosen": -0.6540361642837524,
      "logits/rejected": -0.6697030067443848,
      "logps/chosen": -945.4405517578125,
      "logps/rejected": -700.9091796875,
      "loss": 0.7024,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.009421158581972122,
      "rewards/margins": -0.017499731853604317,
      "rewards/rejected": 0.026920892298221588,
      "step": 291
    },
    {
      "epoch": 0.0768054710746519,
      "grad_norm": 440.0,
      "learning_rate": 3.832020997375328e-07,
      "logits/chosen": -0.6488121747970581,
      "logits/rejected": -0.6554242968559265,
      "logps/chosen": -907.3612670898438,
      "logps/rejected": -694.8921508789062,
      "loss": 0.6866,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.014776710420846939,
      "rewards/margins": 0.013621235266327858,
      "rewards/rejected": 0.0011554728262126446,
      "step": 292
    },
    {
      "epoch": 0.07706850350983906,
      "grad_norm": 516.0,
      "learning_rate": 3.84514435695538e-07,
      "logits/chosen": -0.6797102093696594,
      "logits/rejected": -0.6755685806274414,
      "logps/chosen": -905.8797607421875,
      "logps/rejected": -947.4730224609375,
      "loss": 0.7012,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.009938430041074753,
      "rewards/margins": -0.01573972776532173,
      "rewards/rejected": 0.005801296792924404,
      "step": 293
    },
    {
      "epoch": 0.07733153594502622,
      "grad_norm": 580.0,
      "learning_rate": 3.8582677165354325e-07,
      "logits/chosen": -0.6458734273910522,
      "logits/rejected": -0.6548214554786682,
      "logps/chosen": -1192.5618896484375,
      "logps/rejected": -858.5386962890625,
      "loss": 0.7032,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.003719232976436615,
      "rewards/margins": -0.018166830763220787,
      "rewards/rejected": 0.021886061877012253,
      "step": 294
    },
    {
      "epoch": 0.07759456838021339,
      "grad_norm": 474.0,
      "learning_rate": 3.871391076115486e-07,
      "logits/chosen": -0.6221662163734436,
      "logits/rejected": -0.6342501640319824,
      "logps/chosen": -943.25439453125,
      "logps/rejected": -555.9083251953125,
      "loss": 0.7024,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.007568264380097389,
      "rewards/margins": -0.01805896870791912,
      "rewards/rejected": 0.010490703396499157,
      "step": 295
    },
    {
      "epoch": 0.07785760081540055,
      "grad_norm": 476.0,
      "learning_rate": 3.884514435695538e-07,
      "logits/chosen": -0.6628819704055786,
      "logits/rejected": -0.6671411991119385,
      "logps/chosen": -1142.74755859375,
      "logps/rejected": -884.37939453125,
      "loss": 0.6959,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0031286231242120266,
      "rewards/margins": -0.004720305558294058,
      "rewards/rejected": 0.0015916824340820312,
      "step": 296
    },
    {
      "epoch": 0.07812063325058771,
      "grad_norm": 552.0,
      "learning_rate": 3.8976377952755903e-07,
      "logits/chosen": -0.6677764654159546,
      "logits/rejected": -0.6639803647994995,
      "logps/chosen": -1117.0487060546875,
      "logps/rejected": -858.6883544921875,
      "loss": 0.7025,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.0009479522705078125,
      "rewards/margins": -0.01827859878540039,
      "rewards/rejected": 0.019226552918553352,
      "step": 297
    },
    {
      "epoch": 0.07838366568577487,
      "grad_norm": 494.0,
      "learning_rate": 3.910761154855643e-07,
      "logits/chosen": -0.6394590139389038,
      "logits/rejected": -0.6501412391662598,
      "logps/chosen": -1390.4141845703125,
      "logps/rejected": -1113.39306640625,
      "loss": 0.6874,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0148995416238904,
      "rewards/margins": 0.012362862937152386,
      "rewards/rejected": 0.002536678919568658,
      "step": 298
    },
    {
      "epoch": 0.07864669812096205,
      "grad_norm": 540.0,
      "learning_rate": 3.9238845144356953e-07,
      "logits/chosen": -0.6696509122848511,
      "logits/rejected": -0.6796689033508301,
      "logps/chosen": -1077.276123046875,
      "logps/rejected": -862.341552734375,
      "loss": 0.7023,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.012256432324647903,
      "rewards/margins": -0.015931177884340286,
      "rewards/rejected": 0.0036747450940310955,
      "step": 299
    },
    {
      "epoch": 0.07890973055614921,
      "grad_norm": 422.0,
      "learning_rate": 3.937007874015748e-07,
      "logits/chosen": -0.645210862159729,
      "logits/rejected": -0.6478235125541687,
      "logps/chosen": -755.4135131835938,
      "logps/rejected": -641.76171875,
      "loss": 0.7003,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00035667442716658115,
      "rewards/margins": -0.013493156060576439,
      "rewards/rejected": 0.013849831186234951,
      "step": 300
    },
    {
      "epoch": 0.07917276299133637,
      "grad_norm": 470.0,
      "learning_rate": 3.9501312335958003e-07,
      "logits/chosen": -0.6601244211196899,
      "logits/rejected": -0.6564587950706482,
      "logps/chosen": -711.2291259765625,
      "logps/rejected": -851.2891845703125,
      "loss": 0.6972,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.010273934341967106,
      "rewards/margins": -0.006413744296878576,
      "rewards/rejected": -0.0038601872511208057,
      "step": 301
    },
    {
      "epoch": 0.07943579542652353,
      "grad_norm": 524.0,
      "learning_rate": 3.9632545931758526e-07,
      "logits/chosen": -0.645393967628479,
      "logits/rejected": -0.6422467827796936,
      "logps/chosen": -1489.0777587890625,
      "logps/rejected": -1144.843017578125,
      "loss": 0.7024,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.009588527493178844,
      "rewards/margins": -0.017386915162205696,
      "rewards/rejected": 0.007798385806381702,
      "step": 302
    },
    {
      "epoch": 0.0796988278617107,
      "grad_norm": 568.0,
      "learning_rate": 3.9763779527559053e-07,
      "logits/chosen": -0.6655370593070984,
      "logits/rejected": -0.6692866683006287,
      "logps/chosen": -1215.259765625,
      "logps/rejected": -1140.0477294921875,
      "loss": 0.6996,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.01259632222354412,
      "rewards/margins": -0.0111173614859581,
      "rewards/rejected": -0.001478957012295723,
      "step": 303
    },
    {
      "epoch": 0.07996186029689786,
      "grad_norm": 560.0,
      "learning_rate": 3.9895013123359576e-07,
      "logits/chosen": -0.6688210368156433,
      "logits/rejected": -0.671970784664154,
      "logps/chosen": -998.2176513671875,
      "logps/rejected": -862.2885131835938,
      "loss": 0.6961,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.018679236993193626,
      "rewards/margins": -0.0054856291972100735,
      "rewards/rejected": 0.024164868518710136,
      "step": 304
    },
    {
      "epoch": 0.08022489273208502,
      "grad_norm": 764.0,
      "learning_rate": 4.00262467191601e-07,
      "logits/chosen": -0.6371362805366516,
      "logits/rejected": -0.637810230255127,
      "logps/chosen": -1223.288330078125,
      "logps/rejected": -1067.036376953125,
      "loss": 0.691,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01522054709494114,
      "rewards/margins": 0.004624462686479092,
      "rewards/rejected": 0.010596084408462048,
      "step": 305
    },
    {
      "epoch": 0.08048792516727218,
      "grad_norm": 556.0,
      "learning_rate": 4.015748031496063e-07,
      "logits/chosen": -0.6688053607940674,
      "logits/rejected": -0.6693486571311951,
      "logps/chosen": -1081.871826171875,
      "logps/rejected": -975.42529296875,
      "loss": 0.6852,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.015407180413603783,
      "rewards/margins": 0.01652665063738823,
      "rewards/rejected": -0.0011194697581231594,
      "step": 306
    },
    {
      "epoch": 0.08075095760245936,
      "grad_norm": 604.0,
      "learning_rate": 4.0288713910761154e-07,
      "logits/chosen": -0.6601340174674988,
      "logits/rejected": -0.6580657958984375,
      "logps/chosen": -1555.8927001953125,
      "logps/rejected": -1142.19482421875,
      "loss": 0.6938,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005479430314153433,
      "rewards/margins": -0.0007202147971838713,
      "rewards/rejected": 0.006199646275490522,
      "step": 307
    },
    {
      "epoch": 0.08101399003764652,
      "grad_norm": 564.0,
      "learning_rate": 4.041994750656168e-07,
      "logits/chosen": -0.6677992343902588,
      "logits/rejected": -0.6459469795227051,
      "logps/chosen": -928.517333984375,
      "logps/rejected": -828.2236938476562,
      "loss": 0.6866,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0069786072708666325,
      "rewards/margins": 0.014068982563912868,
      "rewards/rejected": -0.021047594025731087,
      "step": 308
    },
    {
      "epoch": 0.08127702247283368,
      "grad_norm": 460.0,
      "learning_rate": 4.0551181102362204e-07,
      "logits/chosen": -0.6328941583633423,
      "logits/rejected": -0.6418446898460388,
      "logps/chosen": -826.9599609375,
      "logps/rejected": -718.1134033203125,
      "loss": 0.6951,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0033727651461958885,
      "rewards/margins": -0.0026304256170988083,
      "rewards/rejected": 0.006003189831972122,
      "step": 309
    },
    {
      "epoch": 0.08154005490802084,
      "grad_norm": 468.0,
      "learning_rate": 4.0682414698162727e-07,
      "logits/chosen": -0.6517041921615601,
      "logits/rejected": -0.6444616317749023,
      "logps/chosen": -930.96728515625,
      "logps/rejected": -897.7252197265625,
      "loss": 0.6983,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0027921688742935658,
      "rewards/margins": -0.009891653433442116,
      "rewards/rejected": 0.007099485024809837,
      "step": 310
    },
    {
      "epoch": 0.08180308734320801,
      "grad_norm": 576.0,
      "learning_rate": 4.0813648293963255e-07,
      "logits/chosen": -0.6348099708557129,
      "logits/rejected": -0.6343958377838135,
      "logps/chosen": -1065.724853515625,
      "logps/rejected": -973.275146484375,
      "loss": 0.7015,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.019366074353456497,
      "rewards/margins": -0.015261173248291016,
      "rewards/rejected": -0.004104900173842907,
      "step": 311
    },
    {
      "epoch": 0.08206611977839517,
      "grad_norm": 516.0,
      "learning_rate": 4.0944881889763777e-07,
      "logits/chosen": -0.6479460597038269,
      "logits/rejected": -0.6576790809631348,
      "logps/chosen": -1179.1973876953125,
      "logps/rejected": -728.3736572265625,
      "loss": 0.6915,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00240936316549778,
      "rewards/margins": 0.003691578283905983,
      "rewards/rejected": -0.0012822146527469158,
      "step": 312
    },
    {
      "epoch": 0.08232915221358234,
      "grad_norm": 604.0,
      "learning_rate": 4.10761154855643e-07,
      "logits/chosen": -0.6693635582923889,
      "logits/rejected": -0.6698324680328369,
      "logps/chosen": -1311.9993896484375,
      "logps/rejected": -916.3141479492188,
      "loss": 0.6882,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008720780722796917,
      "rewards/margins": 0.010567665100097656,
      "rewards/rejected": -0.0018468850757926702,
      "step": 313
    },
    {
      "epoch": 0.0825921846487695,
      "grad_norm": 620.0,
      "learning_rate": 4.1207349081364827e-07,
      "logits/chosen": -0.6143415570259094,
      "logits/rejected": -0.6325668096542358,
      "logps/chosen": -1214.889892578125,
      "logps/rejected": -1133.6531982421875,
      "loss": 0.7072,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01774015463888645,
      "rewards/margins": -0.027235889807343483,
      "rewards/rejected": 0.009495736099779606,
      "step": 314
    },
    {
      "epoch": 0.08285521708395667,
      "grad_norm": 552.0,
      "learning_rate": 4.133858267716535e-07,
      "logits/chosen": -0.6553505659103394,
      "logits/rejected": -0.6654460430145264,
      "logps/chosen": -1020.3807983398438,
      "logps/rejected": -1138.481689453125,
      "loss": 0.6964,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01307897549122572,
      "rewards/margins": -0.005926513113081455,
      "rewards/rejected": -0.007152461912482977,
      "step": 315
    },
    {
      "epoch": 0.08311824951914383,
      "grad_norm": 544.0,
      "learning_rate": 4.146981627296588e-07,
      "logits/chosen": -0.6451642513275146,
      "logits/rejected": -0.6428130865097046,
      "logps/chosen": -1249.8876953125,
      "logps/rejected": -976.5205688476562,
      "loss": 0.6861,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02142639085650444,
      "rewards/margins": 0.015574648045003414,
      "rewards/rejected": 0.005851745139807463,
      "step": 316
    },
    {
      "epoch": 0.08338128195433099,
      "grad_norm": 696.0,
      "learning_rate": 4.16010498687664e-07,
      "logits/chosen": -0.6217223405838013,
      "logits/rejected": -0.6201914548873901,
      "logps/chosen": -1367.0252685546875,
      "logps/rejected": -895.6494750976562,
      "loss": 0.6995,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008623315021395683,
      "rewards/margins": -0.010660458356142044,
      "rewards/rejected": 0.019283771514892578,
      "step": 317
    },
    {
      "epoch": 0.08364431438951815,
      "grad_norm": 462.0,
      "learning_rate": 4.173228346456693e-07,
      "logits/chosen": -0.6721180081367493,
      "logits/rejected": -0.6693371534347534,
      "logps/chosen": -832.2236328125,
      "logps/rejected": -721.9007568359375,
      "loss": 0.6838,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005961179733276367,
      "rewards/margins": 0.01991109922528267,
      "rewards/rejected": -0.013949919492006302,
      "step": 318
    },
    {
      "epoch": 0.08390734682470533,
      "grad_norm": 486.0,
      "learning_rate": 4.1863517060367456e-07,
      "logits/chosen": -0.6457740664482117,
      "logits/rejected": NaN,
      "logps/chosen": -1042.0567626953125,
      "logps/rejected": -554.03173828125,
      "loss": 0.6906,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01461486890912056,
      "rewards/margins": 0.005528449080884457,
      "rewards/rejected": 0.009086418896913528,
      "step": 319
    },
    {
      "epoch": 0.08417037925989249,
      "grad_norm": 616.0,
      "learning_rate": 4.199475065616798e-07,
      "logits/chosen": -0.6453023552894592,
      "logits/rejected": -0.6407389044761658,
      "logps/chosen": -1381.42822265625,
      "logps/rejected": -846.5745849609375,
      "loss": 0.6995,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.010937407612800598,
      "rewards/margins": -0.011781216599047184,
      "rewards/rejected": 0.0008438111981377006,
      "step": 320
    },
    {
      "epoch": 0.08443341169507965,
      "grad_norm": 468.0,
      "learning_rate": 4.21259842519685e-07,
      "logits/chosen": -0.6562632322311401,
      "logits/rejected": -0.6630125045776367,
      "logps/chosen": -1037.980224609375,
      "logps/rejected": -829.2412109375,
      "loss": 0.6956,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006737804505974054,
      "rewards/margins": -0.004456806927919388,
      "rewards/rejected": 0.011194610968232155,
      "step": 321
    },
    {
      "epoch": 0.08469644413026681,
      "grad_norm": 498.0,
      "learning_rate": 4.225721784776903e-07,
      "logits/chosen": -0.6467600464820862,
      "logits/rejected": -0.6449259519577026,
      "logps/chosen": -1318.5992431640625,
      "logps/rejected": -1421.7777099609375,
      "loss": 0.6814,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.028503846377134323,
      "rewards/margins": 0.02495741844177246,
      "rewards/rejected": 0.003546429332345724,
      "step": 322
    },
    {
      "epoch": 0.08495947656545398,
      "grad_norm": 660.0,
      "learning_rate": 4.238845144356955e-07,
      "logits/chosen": -0.624576210975647,
      "logits/rejected": -0.6306819319725037,
      "logps/chosen": -1376.3800048828125,
      "logps/rejected": -922.5819091796875,
      "loss": 0.7059,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.017032338306307793,
      "rewards/margins": -0.024225901812314987,
      "rewards/rejected": 0.007193566299974918,
      "step": 323
    },
    {
      "epoch": 0.08522250900064114,
      "grad_norm": 486.0,
      "learning_rate": 4.251968503937008e-07,
      "logits/chosen": -0.6485639214515686,
      "logits/rejected": -0.6478336453437805,
      "logps/chosen": -1088.67919921875,
      "logps/rejected": -871.0502319335938,
      "loss": 0.6941,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02263345569372177,
      "rewards/margins": -0.0016667358577251434,
      "rewards/rejected": -0.020966721698641777,
      "step": 324
    },
    {
      "epoch": 0.0854855414358283,
      "grad_norm": 588.0,
      "learning_rate": 4.26509186351706e-07,
      "logits/chosen": -0.6636873483657837,
      "logits/rejected": -0.6739016771316528,
      "logps/chosen": -1513.06298828125,
      "logps/rejected": -1039.01416015625,
      "loss": 0.6892,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0034511564299464226,
      "rewards/margins": 0.008462239056825638,
      "rewards/rejected": -0.01191339548677206,
      "step": 325
    },
    {
      "epoch": 0.08574857387101546,
      "grad_norm": 532.0,
      "learning_rate": 4.2782152230971124e-07,
      "logits/chosen": -0.6463096141815186,
      "logits/rejected": -0.6410671472549438,
      "logps/chosen": -1183.61083984375,
      "logps/rejected": -1027.8082275390625,
      "loss": 0.6903,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007108116056770086,
      "rewards/margins": 0.006554793566465378,
      "rewards/rejected": 0.0005533224903047085,
      "step": 326
    },
    {
      "epoch": 0.08601160630620264,
      "grad_norm": 588.0,
      "learning_rate": 4.291338582677165e-07,
      "logits/chosen": -0.6778362989425659,
      "logits/rejected": -0.6765240430831909,
      "logps/chosen": -1252.3221435546875,
      "logps/rejected": -898.236572265625,
      "loss": 0.6891,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0175431277602911,
      "rewards/margins": 0.00989541970193386,
      "rewards/rejected": 0.007647705264389515,
      "step": 327
    },
    {
      "epoch": 0.0862746387413898,
      "grad_norm": 556.0,
      "learning_rate": 4.3044619422572174e-07,
      "logits/chosen": -0.6367539763450623,
      "logits/rejected": -0.6558355689048767,
      "logps/chosen": -1478.9158935546875,
      "logps/rejected": -942.15087890625,
      "loss": 0.6911,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.009474755264818668,
      "rewards/margins": 0.004687451291829348,
      "rewards/rejected": -0.014162207022309303,
      "step": 328
    },
    {
      "epoch": 0.08653767117657696,
      "grad_norm": 510.0,
      "learning_rate": 4.3175853018372696e-07,
      "logits/chosen": -0.6456934809684753,
      "logits/rejected": -0.656043291091919,
      "logps/chosen": -1061.24169921875,
      "logps/rejected": -903.4969482421875,
      "loss": 0.6974,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0016147608403116465,
      "rewards/margins": -0.007200860884040594,
      "rewards/rejected": 0.008815623819828033,
      "step": 329
    },
    {
      "epoch": 0.08680070361176413,
      "grad_norm": 772.0,
      "learning_rate": 4.330708661417323e-07,
      "logits/chosen": -0.6521505117416382,
      "logits/rejected": -0.6456344723701477,
      "logps/chosen": -1116.2132568359375,
      "logps/rejected": -755.3931274414062,
      "loss": 0.7035,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.008582115173339844,
      "rewards/margins": -0.02001018449664116,
      "rewards/rejected": 0.01142807025462389,
      "step": 330
    },
    {
      "epoch": 0.08706373604695129,
      "grad_norm": 416.0,
      "learning_rate": 4.343832020997375e-07,
      "logits/chosen": -0.6598215103149414,
      "logits/rejected": -0.665395975112915,
      "logps/chosen": -673.4774169921875,
      "logps/rejected": -859.8544311523438,
      "loss": 0.6921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.001678180880844593,
      "rewards/margins": 0.0024773594923317432,
      "rewards/rejected": -0.000799178727902472,
      "step": 331
    },
    {
      "epoch": 0.08732676848213845,
      "grad_norm": 616.0,
      "learning_rate": 4.356955380577428e-07,
      "logits/chosen": -0.6614571213722229,
      "logits/rejected": -0.6661038398742676,
      "logps/chosen": -1462.594482421875,
      "logps/rejected": -1319.1505126953125,
      "loss": 0.6896,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02012805826961994,
      "rewards/margins": 0.008162498474121094,
      "rewards/rejected": -0.028290558606386185,
      "step": 332
    },
    {
      "epoch": 0.08758980091732561,
      "grad_norm": 520.0,
      "learning_rate": 4.37007874015748e-07,
      "logits/chosen": -0.6466075778007507,
      "logits/rejected": -0.6534801125526428,
      "logps/chosen": -1426.7747802734375,
      "logps/rejected": -1390.5435791015625,
      "loss": 0.7038,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.01843853108584881,
      "rewards/margins": -0.020663641393184662,
      "rewards/rejected": 0.002225113334134221,
      "step": 333
    },
    {
      "epoch": 0.08785283335251279,
      "grad_norm": 560.0,
      "learning_rate": 4.3832020997375325e-07,
      "logits/chosen": -0.6374636888504028,
      "logits/rejected": -0.6368136405944824,
      "logps/chosen": -1185.226806640625,
      "logps/rejected": -1111.69677734375,
      "loss": 0.6949,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.01053609885275364,
      "rewards/margins": -0.0025975217577069998,
      "rewards/rejected": -0.007938575930893421,
      "step": 334
    },
    {
      "epoch": 0.08811586578769995,
      "grad_norm": 438.0,
      "learning_rate": 4.396325459317585e-07,
      "logits/chosen": -0.6613527536392212,
      "logits/rejected": -0.6644643545150757,
      "logps/chosen": -1251.9486083984375,
      "logps/rejected": -1051.9251708984375,
      "loss": 0.6855,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03288383409380913,
      "rewards/margins": 0.016278456896543503,
      "rewards/rejected": 0.016605377197265625,
      "step": 335
    },
    {
      "epoch": 0.08837889822288711,
      "grad_norm": 580.0,
      "learning_rate": 4.4094488188976375e-07,
      "logits/chosen": -0.6231614351272583,
      "logits/rejected": -0.6333091259002686,
      "logps/chosen": -1294.750732421875,
      "logps/rejected": -1177.544921875,
      "loss": 0.7081,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.006430863868445158,
      "rewards/margins": -0.02897944673895836,
      "rewards/rejected": 0.022548580542206764,
      "step": 336
    },
    {
      "epoch": 0.08864193065807427,
      "grad_norm": 502.0,
      "learning_rate": 4.4225721784776897e-07,
      "logits/chosen": -0.6669552326202393,
      "logits/rejected": -0.6849139332771301,
      "logps/chosen": -1199.0579833984375,
      "logps/rejected": -819.72265625,
      "loss": 0.6921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 4.6729459427297115e-05,
      "rewards/margins": 0.0025122645311057568,
      "rewards/rejected": -0.002465534256771207,
      "step": 337
    },
    {
      "epoch": 0.08890496309326144,
      "grad_norm": 580.0,
      "learning_rate": 4.4356955380577425e-07,
      "logits/chosen": -0.6275092363357544,
      "logits/rejected": NaN,
      "logps/chosen": -1288.9111328125,
      "logps/rejected": -1015.1377563476562,
      "loss": 0.6883,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.005341242998838425,
      "rewards/margins": 0.01050658244639635,
      "rewards/rejected": -0.0158478245139122,
      "step": 338
    },
    {
      "epoch": 0.0891679955284486,
      "grad_norm": 616.0,
      "learning_rate": 4.448818897637795e-07,
      "logits/chosen": -0.6069040298461914,
      "logits/rejected": -0.6135823130607605,
      "logps/chosen": -941.4371337890625,
      "logps/rejected": -1097.9150390625,
      "loss": 0.6964,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.014163970947265625,
      "rewards/margins": -0.004886055830866098,
      "rewards/rejected": -0.009277915582060814,
      "step": 339
    },
    {
      "epoch": 0.08943102796363576,
      "grad_norm": 540.0,
      "learning_rate": 4.4619422572178475e-07,
      "logits/chosen": -0.6526578664779663,
      "logits/rejected": -0.6510878801345825,
      "logps/chosen": -1234.7666015625,
      "logps/rejected": -819.76318359375,
      "loss": 0.6941,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.024567414075136185,
      "rewards/margins": -0.0005670543760061264,
      "rewards/rejected": -0.02400035783648491,
      "step": 340
    },
    {
      "epoch": 0.08969406039882293,
      "grad_norm": 652.0,
      "learning_rate": 4.4750656167979003e-07,
      "logits/chosen": -0.655733585357666,
      "logits/rejected": -0.6701127886772156,
      "logps/chosen": -1274.8785400390625,
      "logps/rejected": -859.04736328125,
      "loss": 0.6823,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.020443059504032135,
      "rewards/margins": 0.022854233160614967,
      "rewards/rejected": -0.04329729452729225,
      "step": 341
    },
    {
      "epoch": 0.0899570928340101,
      "grad_norm": 584.0,
      "learning_rate": 4.4881889763779526e-07,
      "logits/chosen": -0.6339419484138489,
      "logits/rejected": -0.6193982362747192,
      "logps/chosen": -1158.531005859375,
      "logps/rejected": -1121.8988037109375,
      "loss": 0.6873,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.005501747131347656,
      "rewards/margins": 0.013061238452792168,
      "rewards/rejected": -0.018562985584139824,
      "step": 342
    },
    {
      "epoch": 0.09022012526919726,
      "grad_norm": 540.0,
      "learning_rate": 4.5013123359580053e-07,
      "logits/chosen": -0.6591323614120483,
      "logits/rejected": -0.6599884629249573,
      "logps/chosen": -1220.9061279296875,
      "logps/rejected": -917.7898559570312,
      "loss": 0.6957,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.001958847977221012,
      "rewards/margins": -0.004246139898896217,
      "rewards/rejected": 0.0022872923873364925,
      "step": 343
    },
    {
      "epoch": 0.09048315770438442,
      "grad_norm": 588.0,
      "learning_rate": 4.5144356955380576e-07,
      "logits/chosen": -0.6604488492012024,
      "logits/rejected": -0.6771016716957092,
      "logps/chosen": -1136.9378662109375,
      "logps/rejected": -820.0826416015625,
      "loss": 0.6834,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.025425434112548828,
      "rewards/margins": 0.020054150372743607,
      "rewards/rejected": 0.0053712851367890835,
      "step": 344
    },
    {
      "epoch": 0.09074619013957158,
      "grad_norm": 510.0,
      "learning_rate": 4.52755905511811e-07,
      "logits/chosen": -0.6455289125442505,
      "logits/rejected": -0.6505630612373352,
      "logps/chosen": -1186.1549072265625,
      "logps/rejected": -674.1807250976562,
      "loss": 0.6965,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.00524520967155695,
      "rewards/margins": -0.005996083840727806,
      "rewards/rejected": 0.0007508760318160057,
      "step": 345
    },
    {
      "epoch": 0.09100922257475876,
      "grad_norm": 536.0,
      "learning_rate": 4.5406824146981626e-07,
      "logits/chosen": -0.6417142152786255,
      "logits/rejected": -0.644270122051239,
      "logps/chosen": -982.8403930664062,
      "logps/rejected": -881.130126953125,
      "loss": 0.6924,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.013568781316280365,
      "rewards/margins": 0.002014159457758069,
      "rewards/rejected": 0.011554623953998089,
      "step": 346
    },
    {
      "epoch": 0.09127225500994592,
      "grad_norm": 612.0,
      "learning_rate": 4.553805774278215e-07,
      "logits/chosen": -0.6561776399612427,
      "logits/rejected": -0.6646116971969604,
      "logps/chosen": -1302.4442138671875,
      "logps/rejected": -1142.475341796875,
      "loss": 0.6928,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0038027758710086346,
      "rewards/margins": 0.0013325689360499382,
      "rewards/rejected": 0.0024702069349586964,
      "step": 347
    },
    {
      "epoch": 0.09153528744513308,
      "grad_norm": 544.0,
      "learning_rate": 4.5669291338582676e-07,
      "logits/chosen": -0.6691831350326538,
      "logits/rejected": -0.6748553514480591,
      "logps/chosen": -1261.6497802734375,
      "logps/rejected": -821.3449096679688,
      "loss": 0.6959,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.003939437679946423,
      "rewards/margins": -0.004597330000251532,
      "rewards/rejected": 0.0006578926695510745,
      "step": 348
    },
    {
      "epoch": 0.09179831988032024,
      "grad_norm": 664.0,
      "learning_rate": 4.58005249343832e-07,
      "logits/chosen": -0.6582449674606323,
      "logits/rejected": -0.6623384952545166,
      "logps/chosen": -1647.9708251953125,
      "logps/rejected": -1201.5267333984375,
      "loss": 0.6955,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.01722707971930504,
      "rewards/margins": -0.0029200566932559013,
      "rewards/rejected": -0.014307022094726562,
      "step": 349
    },
    {
      "epoch": 0.09206135231550741,
      "grad_norm": 544.0,
      "learning_rate": 4.593175853018372e-07,
      "logits/chosen": -0.6657758951187134,
      "logits/rejected": -0.6636642217636108,
      "logps/chosen": -1247.4637451171875,
      "logps/rejected": -1062.354248046875,
      "loss": 0.7018,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.008515644818544388,
      "rewards/margins": -0.016467858105897903,
      "rewards/rejected": 0.00795221421867609,
      "step": 350
    },
    {
      "epoch": 0.09232438475069457,
      "grad_norm": 592.0,
      "learning_rate": 4.606299212598425e-07,
      "logits/chosen": -0.6555737853050232,
      "logits/rejected": -0.6680029630661011,
      "logps/chosen": -1337.2735595703125,
      "logps/rejected": -1101.5445556640625,
      "loss": 0.6788,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.025840329006314278,
      "rewards/margins": 0.030787279829382896,
      "rewards/rejected": -0.004946947563439608,
      "step": 351
    },
    {
      "epoch": 0.09258741718588173,
      "grad_norm": 736.0,
      "learning_rate": 4.619422572178477e-07,
      "logits/chosen": -0.6813198328018188,
      "logits/rejected": -0.6869717240333557,
      "logps/chosen": -1779.8074951171875,
      "logps/rejected": -1628.3963623046875,
      "loss": 0.6754,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.019542312249541283,
      "rewards/margins": 0.03758535534143448,
      "rewards/rejected": -0.018043041229248047,
      "step": 352
    },
    {
      "epoch": 0.09285044962106889,
      "grad_norm": 528.0,
      "learning_rate": 4.63254593175853e-07,
      "logits/chosen": -0.6470504403114319,
      "logits/rejected": -0.6575362682342529,
      "logps/chosen": -947.9443359375,
      "logps/rejected": -700.9642944335938,
      "loss": 0.6854,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008911516517400742,
      "rewards/margins": 0.01609363593161106,
      "rewards/rejected": -0.007182121276855469,
      "step": 353
    },
    {
      "epoch": 0.09311348205625607,
      "grad_norm": 564.0,
      "learning_rate": 4.6456692913385827e-07,
      "logits/chosen": -0.6679884791374207,
      "logits/rejected": -0.6727485656738281,
      "logps/chosen": -1478.8516845703125,
      "logps/rejected": -1344.7056884765625,
      "loss": 0.7044,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004106998443603516,
      "rewards/margins": -0.021081257611513138,
      "rewards/rejected": 0.025188256055116653,
      "step": 354
    },
    {
      "epoch": 0.09337651449144323,
      "grad_norm": 580.0,
      "learning_rate": 4.658792650918635e-07,
      "logits/chosen": -0.664135754108429,
      "logits/rejected": -0.6773480176925659,
      "logps/chosen": -1168.7042236328125,
      "logps/rejected": -717.1077880859375,
      "loss": 0.686,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.015994643792510033,
      "rewards/margins": 0.014564752578735352,
      "rewards/rejected": 0.0014298907481133938,
      "step": 355
    },
    {
      "epoch": 0.09363954692663039,
      "grad_norm": 628.0,
      "learning_rate": 4.671916010498688e-07,
      "logits/chosen": -0.670951247215271,
      "logits/rejected": -0.6681044101715088,
      "logps/chosen": -1032.6256103515625,
      "logps/rejected": -1066.7144775390625,
      "loss": 0.7058,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.016932964324951172,
      "rewards/margins": -0.024973394349217415,
      "rewards/rejected": 0.008040429092943668,
      "step": 356
    },
    {
      "epoch": 0.09390257936181755,
      "grad_norm": 456.0,
      "learning_rate": 4.68503937007874e-07,
      "logits/chosen": -0.6277486681938171,
      "logits/rejected": -0.6259077787399292,
      "logps/chosen": -937.046142578125,
      "logps/rejected": -703.3759155273438,
      "loss": 0.6891,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006030750460922718,
      "rewards/margins": 0.008304690942168236,
      "rewards/rejected": -0.0022739411797374487,
      "step": 357
    },
    {
      "epoch": 0.09416561179700472,
      "grad_norm": 628.0,
      "learning_rate": 4.698162729658792e-07,
      "logits/chosen": -0.6527631878852844,
      "logits/rejected": -0.6479618549346924,
      "logps/chosen": -1450.4285888671875,
      "logps/rejected": -1279.2713623046875,
      "loss": 0.6883,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0010967242997139692,
      "rewards/margins": 0.010249137878417969,
      "rewards/rejected": -0.009152411483228207,
      "step": 358
    },
    {
      "epoch": 0.09442864423219188,
      "grad_norm": 490.0,
      "learning_rate": 4.711286089238845e-07,
      "logits/chosen": -0.6617475748062134,
      "logits/rejected": -0.6694413423538208,
      "logps/chosen": -1027.1656494140625,
      "logps/rejected": -808.7401733398438,
      "loss": 0.695,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.008673667907714844,
      "rewards/margins": -0.00277214078232646,
      "rewards/rejected": -0.0059015266597270966,
      "step": 359
    },
    {
      "epoch": 0.09469167666737904,
      "grad_norm": 462.0,
      "learning_rate": 4.7244094488188973e-07,
      "logits/chosen": -0.6627187132835388,
      "logits/rejected": -0.6719405651092529,
      "logps/chosen": -1030.349609375,
      "logps/rejected": -792.9718017578125,
      "loss": 0.6776,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.005264568142592907,
      "rewards/margins": 0.031690314412117004,
      "rewards/rejected": -0.03695487976074219,
      "step": 360
    },
    {
      "epoch": 0.0949547091025662,
      "grad_norm": 544.0,
      "learning_rate": 4.7375328083989495e-07,
      "logits/chosen": -0.6845085024833679,
      "logits/rejected": -0.6698288917541504,
      "logps/chosen": -961.2928466796875,
      "logps/rejected": -642.7725219726562,
      "loss": 0.7084,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.025252344086766243,
      "rewards/margins": -0.028422638773918152,
      "rewards/rejected": 0.0031703002750873566,
      "step": 361
    },
    {
      "epoch": 0.09521774153775338,
      "grad_norm": 520.0,
      "learning_rate": 4.7506561679790023e-07,
      "logits/chosen": -0.6602206230163574,
      "logits/rejected": -0.6641651391983032,
      "logps/chosen": -1048.6236572265625,
      "logps/rejected": -1062.366943359375,
      "loss": 0.6983,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006479551084339619,
      "rewards/margins": -0.009500313550233841,
      "rewards/rejected": 0.015979861840605736,
      "step": 362
    },
    {
      "epoch": 0.09548077397294054,
      "grad_norm": 600.0,
      "learning_rate": 4.7637795275590545e-07,
      "logits/chosen": -0.6193004846572876,
      "logits/rejected": -0.6262727975845337,
      "logps/chosen": -1242.4127197265625,
      "logps/rejected": -1034.482177734375,
      "loss": 0.6941,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01174106728285551,
      "rewards/margins": -0.0008250228129327297,
      "rewards/rejected": -0.010916042141616344,
      "step": 363
    },
    {
      "epoch": 0.0957438064081277,
      "grad_norm": 436.0,
      "learning_rate": 4.776902887139107e-07,
      "logits/chosen": -0.6615593433380127,
      "logits/rejected": -0.661186695098877,
      "logps/chosen": -1105.869140625,
      "logps/rejected": -822.6203002929688,
      "loss": 0.6852,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.005316353868693113,
      "rewards/margins": 0.017119694501161575,
      "rewards/rejected": -0.011803342029452324,
      "step": 364
    },
    {
      "epoch": 0.09600683884331486,
      "grad_norm": 420.0,
      "learning_rate": 4.79002624671916e-07,
      "logits/chosen": -0.6654348969459534,
      "logits/rejected": -0.6627000570297241,
      "logps/chosen": -879.7646484375,
      "logps/rejected": -747.5318603515625,
      "loss": 0.6919,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.018766213208436966,
      "rewards/margins": 0.002869749441742897,
      "rewards/rejected": 0.01589646376669407,
      "step": 365
    },
    {
      "epoch": 0.09626987127850203,
      "grad_norm": 596.0,
      "learning_rate": 4.803149606299212e-07,
      "logits/chosen": -0.6517578959465027,
      "logits/rejected": -0.6643199324607849,
      "logps/chosen": -1294.841064453125,
      "logps/rejected": -1120.679931640625,
      "loss": 0.6923,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.001168681774288416,
      "rewards/margins": 0.0025620474480092525,
      "rewards/rejected": -0.0013933655573055148,
      "step": 366
    },
    {
      "epoch": 0.0965329037136892,
      "grad_norm": 604.0,
      "learning_rate": 4.816272965879265e-07,
      "logits/chosen": -0.6596790552139282,
      "logits/rejected": -0.6662644743919373,
      "logps/chosen": -904.9344482421875,
      "logps/rejected": -999.66748046875,
      "loss": 0.6903,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008775759488344193,
      "rewards/margins": 0.006797790061682463,
      "rewards/rejected": -0.015573550015687943,
      "step": 367
    },
    {
      "epoch": 0.09679593614887635,
      "grad_norm": 572.0,
      "learning_rate": 4.829396325459317e-07,
      "logits/chosen": -0.6444803476333618,
      "logits/rejected": -0.6541239023208618,
      "logps/chosen": -1005.17138671875,
      "logps/rejected": -705.955322265625,
      "loss": 0.6989,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.007965183816850185,
      "rewards/margins": -0.011142779141664505,
      "rewards/rejected": 0.0031775953248143196,
      "step": 368
    },
    {
      "epoch": 0.09705896858406353,
      "grad_norm": 494.0,
      "learning_rate": 4.84251968503937e-07,
      "logits/chosen": -0.63264000415802,
      "logits/rejected": -0.6420748233795166,
      "logps/chosen": -931.1685180664062,
      "logps/rejected": -769.2515258789062,
      "loss": 0.6905,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.019947433844208717,
      "rewards/margins": 0.005692958831787109,
      "rewards/rejected": 0.014254475012421608,
      "step": 369
    },
    {
      "epoch": 0.09732200101925069,
      "grad_norm": 556.0,
      "learning_rate": 4.855643044619423e-07,
      "logits/chosen": -0.6799954771995544,
      "logits/rejected": -0.6771104335784912,
      "logps/chosen": -1690.837890625,
      "logps/rejected": -1343.6231689453125,
      "loss": 0.6804,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.019279100000858307,
      "rewards/margins": 0.02647409588098526,
      "rewards/rejected": -0.007194995414465666,
      "step": 370
    },
    {
      "epoch": 0.09758503345443785,
      "grad_norm": 440.0,
      "learning_rate": 4.868766404199475e-07,
      "logits/chosen": -0.6714980602264404,
      "logits/rejected": -0.6779274940490723,
      "logps/chosen": -987.0328979492188,
      "logps/rejected": -852.0817260742188,
      "loss": 0.7013,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.016957569867372513,
      "rewards/margins": -0.015298845246434212,
      "rewards/rejected": -0.0016587254358455539,
      "step": 371
    },
    {
      "epoch": 0.09784806588962501,
      "grad_norm": 604.0,
      "learning_rate": 4.881889763779527e-07,
      "logits/chosen": -0.6568538546562195,
      "logits/rejected": -0.659113347530365,
      "logps/chosen": -1635.8292236328125,
      "logps/rejected": -1217.1900634765625,
      "loss": 0.6943,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0023496635258197784,
      "rewards/margins": -0.0013109201099723577,
      "rewards/rejected": -0.0010387427173554897,
      "step": 372
    },
    {
      "epoch": 0.09811109832481218,
      "grad_norm": 458.0,
      "learning_rate": 4.89501312335958e-07,
      "logits/chosen": -0.6740940809249878,
      "logits/rejected": -0.6654078364372253,
      "logps/chosen": -953.443359375,
      "logps/rejected": -765.1026000976562,
      "loss": 0.6942,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00015382753917947412,
      "rewards/margins": -0.0013262736611068249,
      "rewards/rejected": 0.001172446645796299,
      "step": 373
    },
    {
      "epoch": 0.09837413075999935,
      "grad_norm": 544.0,
      "learning_rate": 4.908136482939632e-07,
      "logits/chosen": -0.6432079672813416,
      "logits/rejected": -0.6473474502563477,
      "logps/chosen": -1195.49755859375,
      "logps/rejected": -804.1455078125,
      "loss": 0.6867,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.00845251139253378,
      "rewards/margins": 0.013295507058501244,
      "rewards/rejected": -0.021748017519712448,
      "step": 374
    },
    {
      "epoch": 0.0986371631951865,
      "grad_norm": 490.0,
      "learning_rate": 4.921259842519685e-07,
      "logits/chosen": -0.6702641248703003,
      "logits/rejected": -0.6723368167877197,
      "logps/chosen": -845.9915161132812,
      "logps/rejected": -803.5270385742188,
      "loss": 0.6924,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0048376088961958885,
      "rewards/margins": 0.0025856976862996817,
      "rewards/rejected": 0.0022519114427268505,
      "step": 375
    },
    {
      "epoch": 0.09890019563037367,
      "grad_norm": 482.0,
      "learning_rate": 4.934383202099737e-07,
      "logits/chosen": -0.6513277888298035,
      "logits/rejected": -0.6466772556304932,
      "logps/chosen": -830.4884033203125,
      "logps/rejected": -685.7301025390625,
      "loss": 0.693,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.016599560156464577,
      "rewards/margins": 0.0018201838247478008,
      "rewards/rejected": 0.014779376797378063,
      "step": 376
    },
    {
      "epoch": 0.09916322806556084,
      "grad_norm": 500.0,
      "learning_rate": 4.94750656167979e-07,
      "logits/chosen": -0.6731439828872681,
      "logits/rejected": -0.657768189907074,
      "logps/chosen": -1013.1187744140625,
      "logps/rejected": -816.9703979492188,
      "loss": 0.6967,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.007097337394952774,
      "rewards/margins": -0.006486797239631414,
      "rewards/rejected": -0.0006105424836277962,
      "step": 377
    },
    {
      "epoch": 0.099426260500748,
      "grad_norm": 508.0,
      "learning_rate": 4.960629921259842e-07,
      "logits/chosen": -0.6607010364532471,
      "logits/rejected": -0.6733332872390747,
      "logps/chosen": -1073.70654296875,
      "logps/rejected": -1067.963623046875,
      "loss": 0.6958,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.005278110504150391,
      "rewards/margins": -0.004849147982895374,
      "rewards/rejected": -0.00042896275408566,
      "step": 378
    },
    {
      "epoch": 0.09968929293593516,
      "grad_norm": 478.0,
      "learning_rate": 4.973753280839894e-07,
      "logits/chosen": -0.6343443393707275,
      "logits/rejected": -0.6461348533630371,
      "logps/chosen": -1133.5338134765625,
      "logps/rejected": -783.0164184570312,
      "loss": 0.6992,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0034609793219715357,
      "rewards/margins": -0.011598683893680573,
      "rewards/rejected": 0.008137702941894531,
      "step": 379
    },
    {
      "epoch": 0.09995232537112232,
      "grad_norm": 604.0,
      "learning_rate": 4.986876640419948e-07,
      "logits/chosen": -0.6583479642868042,
      "logits/rejected": -0.644252598285675,
      "logps/chosen": -1076.775390625,
      "logps/rejected": -1109.1422119140625,
      "loss": 0.6945,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003186130430549383,
      "rewards/margins": -0.0009279260411858559,
      "rewards/rejected": 0.004114056006073952,
      "step": 380
    },
    {
      "epoch": 0.1002153578063095,
      "grad_norm": 560.0,
      "learning_rate": 5e-07,
      "logits/chosen": -0.648545503616333,
      "logits/rejected": NaN,
      "logps/chosen": -1098.2987060546875,
      "logps/rejected": -826.2868041992188,
      "loss": 0.698,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.006715678609907627,
      "rewards/margins": -0.009214402176439762,
      "rewards/rejected": 0.002498722169548273,
      "step": 381
    },
    {
      "epoch": 0.10047839024149666,
      "grad_norm": 512.0,
      "learning_rate": 4.998538011695906e-07,
      "logits/chosen": -0.6757872104644775,
      "logits/rejected": -0.6775391101837158,
      "logps/chosen": -1098.189453125,
      "logps/rejected": -850.1490478515625,
      "loss": 0.6925,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0021767623256891966,
      "rewards/margins": 0.0018335343338549137,
      "rewards/rejected": -0.004010295961052179,
      "step": 382
    },
    {
      "epoch": 0.10074142267668382,
      "grad_norm": 528.0,
      "learning_rate": 4.997076023391812e-07,
      "logits/chosen": -0.6460634469985962,
      "logits/rejected": -0.6551867723464966,
      "logps/chosen": -962.4361572265625,
      "logps/rejected": -693.2891845703125,
      "loss": 0.6933,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004437350668013096,
      "rewards/margins": 0.0003967280499637127,
      "rewards/rejected": 0.004040622618049383,
      "step": 383
    },
    {
      "epoch": 0.10100445511187098,
      "grad_norm": 464.0,
      "learning_rate": 4.995614035087719e-07,
      "logits/chosen": -0.6641680598258972,
      "logits/rejected": -0.6430263519287109,
      "logps/chosen": -922.1253662109375,
      "logps/rejected": -758.2979125976562,
      "loss": 0.6918,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.009790231473743916,
      "rewards/margins": 0.003285027574747801,
      "rewards/rejected": 0.006505203898996115,
      "step": 384
    },
    {
      "epoch": 0.10126748754705815,
      "grad_norm": 496.0,
      "learning_rate": 4.994152046783626e-07,
      "logits/chosen": -0.6495983004570007,
      "logits/rejected": -0.6486555337905884,
      "logps/chosen": -1130.6583251953125,
      "logps/rejected": -775.8767700195312,
      "loss": 0.6973,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.020624544471502304,
      "rewards/margins": -0.0069273002445697784,
      "rewards/rejected": -0.013697242364287376,
      "step": 385
    },
    {
      "epoch": 0.10153051998224531,
      "grad_norm": 532.0,
      "learning_rate": 4.992690058479532e-07,
      "logits/chosen": -0.6377093195915222,
      "logits/rejected": -0.6405943632125854,
      "logps/chosen": -1190.715087890625,
      "logps/rejected": -837.0263061523438,
      "loss": 0.7015,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.027240943163633347,
      "rewards/margins": -0.01623692363500595,
      "rewards/rejected": -0.011004018597304821,
      "step": 386
    },
    {
      "epoch": 0.10179355241743247,
      "grad_norm": 788.0,
      "learning_rate": 4.991228070175438e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6607757210731506,
      "logps/chosen": -1351.0816650390625,
      "logps/rejected": -757.5221557617188,
      "loss": 0.6943,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.017868900671601295,
      "rewards/margins": -0.001141357235610485,
      "rewards/rejected": -0.016727544367313385,
      "step": 387
    },
    {
      "epoch": 0.10205658485261963,
      "grad_norm": 510.0,
      "learning_rate": 4.989766081871345e-07,
      "logits/chosen": -0.6160472631454468,
      "logits/rejected": -0.6205061078071594,
      "logps/chosen": -980.1285400390625,
      "logps/rejected": -730.5191650390625,
      "loss": 0.6893,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010351181030273438,
      "rewards/margins": 0.008693408221006393,
      "rewards/rejected": 0.001657772110775113,
      "step": 388
    },
    {
      "epoch": 0.10231961728780681,
      "grad_norm": 516.0,
      "learning_rate": 4.988304093567251e-07,
      "logits/chosen": -0.6675488948822021,
      "logits/rejected": -0.6590033173561096,
      "logps/chosen": -1447.4691162109375,
      "logps/rejected": -1177.7138671875,
      "loss": 0.6868,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.015342521481215954,
      "rewards/margins": 0.014174842275679111,
      "rewards/rejected": 0.0011676792055368423,
      "step": 389
    },
    {
      "epoch": 0.10258264972299397,
      "grad_norm": 520.0,
      "learning_rate": 4.986842105263158e-07,
      "logits/chosen": -0.6326026320457458,
      "logits/rejected": NaN,
      "logps/chosen": -1061.00390625,
      "logps/rejected": -997.64111328125,
      "loss": 0.6907,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.017512226477265358,
      "rewards/margins": 0.00538587523624301,
      "rewards/rejected": -0.022898100316524506,
      "step": 390
    },
    {
      "epoch": 0.10284568215818113,
      "grad_norm": 424.0,
      "learning_rate": 4.985380116959064e-07,
      "logits/chosen": -0.6564021110534668,
      "logits/rejected": -0.6635516285896301,
      "logps/chosen": -681.7681274414062,
      "logps/rejected": -829.8527221679688,
      "loss": 0.6988,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.017841624096035957,
      "rewards/margins": -0.010982227511703968,
      "rewards/rejected": -0.006859397981315851,
      "step": 391
    },
    {
      "epoch": 0.10310871459336829,
      "grad_norm": 604.0,
      "learning_rate": 4.98391812865497e-07,
      "logits/chosen": -0.6488210558891296,
      "logits/rejected": NaN,
      "logps/chosen": -1184.73486328125,
      "logps/rejected": -939.4691162109375,
      "loss": 0.6891,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.00293579138815403,
      "rewards/margins": 0.008504104800522327,
      "rewards/rejected": -0.011439896188676357,
      "step": 392
    },
    {
      "epoch": 0.10337174702855546,
      "grad_norm": 684.0,
      "learning_rate": 4.982456140350877e-07,
      "logits/chosen": -0.647111177444458,
      "logits/rejected": -0.6450287103652954,
      "logps/chosen": -1184.73095703125,
      "logps/rejected": -946.3756713867188,
      "loss": 0.6982,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.009802342392504215,
      "rewards/margins": -0.008929537609219551,
      "rewards/rejected": 0.01873188093304634,
      "step": 393
    },
    {
      "epoch": 0.10363477946374262,
      "grad_norm": 584.0,
      "learning_rate": 4.980994152046784e-07,
      "logits/chosen": -0.64289391040802,
      "logits/rejected": -0.6353098750114441,
      "logps/chosen": -1007.2327270507812,
      "logps/rejected": -927.1550903320312,
      "loss": 0.6803,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.012617874890565872,
      "rewards/margins": 0.026142023503780365,
      "rewards/rejected": -0.013524151407182217,
      "step": 394
    },
    {
      "epoch": 0.10389781189892978,
      "grad_norm": 556.0,
      "learning_rate": 4.97953216374269e-07,
      "logits/chosen": -0.6283326745033264,
      "logits/rejected": -0.6349169611930847,
      "logps/chosen": -1146.84814453125,
      "logps/rejected": -988.6134033203125,
      "loss": 0.6968,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.018285132944583893,
      "rewards/margins": -0.006732653826475143,
      "rewards/rejected": -0.0115524772554636,
      "step": 395
    },
    {
      "epoch": 0.10416084433411694,
      "grad_norm": 434.0,
      "learning_rate": 4.978070175438596e-07,
      "logits/chosen": -0.6631174087524414,
      "logits/rejected": -0.665276288986206,
      "logps/chosen": -874.229736328125,
      "logps/rejected": -808.2489013671875,
      "loss": 0.6919,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00966720562428236,
      "rewards/margins": 0.002786444965749979,
      "rewards/rejected": 0.006880760192871094,
      "step": 396
    },
    {
      "epoch": 0.10442387676930412,
      "grad_norm": 588.0,
      "learning_rate": 4.976608187134503e-07,
      "logits/chosen": -0.6509300470352173,
      "logits/rejected": -0.6535791754722595,
      "logps/chosen": -1173.158935546875,
      "logps/rejected": -754.3465576171875,
      "loss": 0.6769,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.023432349786162376,
      "rewards/margins": 0.03338055685162544,
      "rewards/rejected": -0.009948206134140491,
      "step": 397
    },
    {
      "epoch": 0.10468690920449128,
      "grad_norm": 456.0,
      "learning_rate": 4.975146198830409e-07,
      "logits/chosen": -0.6562961935997009,
      "logits/rejected": -0.6768209338188171,
      "logps/chosen": -1064.37890625,
      "logps/rejected": -871.4591674804688,
      "loss": 0.698,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.017628813162446022,
      "rewards/margins": -0.00858292542397976,
      "rewards/rejected": -0.009045887738466263,
      "step": 398
    },
    {
      "epoch": 0.10494994163967844,
      "grad_norm": 430.0,
      "learning_rate": 4.973684210526316e-07,
      "logits/chosen": -0.6530295610427856,
      "logits/rejected": -0.6614413261413574,
      "logps/chosen": -734.1513061523438,
      "logps/rejected": -692.0745849609375,
      "loss": 0.6888,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.010198784992098808,
      "rewards/margins": 0.009164620190858841,
      "rewards/rejected": 0.001034163637086749,
      "step": 399
    },
    {
      "epoch": 0.1052129740748656,
      "grad_norm": 498.0,
      "learning_rate": 4.972222222222222e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6461234092712402,
      "logps/chosen": -933.185791015625,
      "logps/rejected": -964.748779296875,
      "loss": 0.6955,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.005076312925666571,
      "rewards/margins": -0.0040420531295239925,
      "rewards/rejected": 0.009118366986513138,
      "step": 400
    },
    {
      "epoch": 0.10547600651005277,
      "grad_norm": 478.0,
      "learning_rate": 4.970760233918128e-07,
      "logits/chosen": -0.6772772073745728,
      "logits/rejected": -0.6863045692443848,
      "logps/chosen": -939.7379150390625,
      "logps/rejected": -921.865234375,
      "loss": 0.7072,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.024318885058164597,
      "rewards/margins": -0.026868246495723724,
      "rewards/rejected": 0.0025493628345429897,
      "step": 401
    },
    {
      "epoch": 0.10573903894523994,
      "grad_norm": 552.0,
      "learning_rate": 4.969298245614035e-07,
      "logits/chosen": -0.6580210328102112,
      "logits/rejected": -0.6672474145889282,
      "logps/chosen": -1342.829345703125,
      "logps/rejected": -988.1507568359375,
      "loss": 0.6866,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02575702778995037,
      "rewards/margins": 0.013594292104244232,
      "rewards/rejected": 0.01216273382306099,
      "step": 402
    },
    {
      "epoch": 0.1060020713804271,
      "grad_norm": 712.0,
      "learning_rate": 4.967836257309941e-07,
      "logits/chosen": -0.6646366715431213,
      "logits/rejected": -0.6568801999092102,
      "logps/chosen": -1533.6546630859375,
      "logps/rejected": -1233.322021484375,
      "loss": 0.7034,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0044548045843839645,
      "rewards/margins": -0.019608305767178535,
      "rewards/rejected": 0.0240631103515625,
      "step": 403
    },
    {
      "epoch": 0.10626510381561426,
      "grad_norm": 416.0,
      "learning_rate": 4.966374269005848e-07,
      "logits/chosen": -0.6700597405433655,
      "logits/rejected": -0.6715225577354431,
      "logps/chosen": -661.3177490234375,
      "logps/rejected": -749.3204345703125,
      "loss": 0.6921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0048188213258981705,
      "rewards/margins": 0.0026041045784950256,
      "rewards/rejected": -0.007422924507409334,
      "step": 404
    },
    {
      "epoch": 0.10652813625080143,
      "grad_norm": 560.0,
      "learning_rate": 4.964912280701754e-07,
      "logits/chosen": -0.6695969700813293,
      "logits/rejected": -0.6702657341957092,
      "logps/chosen": -1320.3748779296875,
      "logps/rejected": -971.001953125,
      "loss": 0.6918,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.020376110449433327,
      "rewards/margins": 0.0030992510728538036,
      "rewards/rejected": -0.02347536012530327,
      "step": 405
    },
    {
      "epoch": 0.10679116868598859,
      "grad_norm": 434.0,
      "learning_rate": 4.963450292397661e-07,
      "logits/chosen": -0.6265073418617249,
      "logits/rejected": -0.6266322135925293,
      "logps/chosen": -974.2664184570312,
      "logps/rejected": -641.9275512695312,
      "loss": 0.6956,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.011164188385009766,
      "rewards/margins": -0.004169464111328125,
      "rewards/rejected": -0.006994724273681641,
      "step": 406
    },
    {
      "epoch": 0.10705420112117575,
      "grad_norm": 480.0,
      "learning_rate": 4.961988304093567e-07,
      "logits/chosen": -0.6465390920639038,
      "logits/rejected": -0.6569474935531616,
      "logps/chosen": -1079.066162109375,
      "logps/rejected": -1113.3319091796875,
      "loss": 0.6972,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.017443275079131126,
      "rewards/margins": -0.006731128320097923,
      "rewards/rejected": 0.02417440339922905,
      "step": 407
    },
    {
      "epoch": 0.10731723355636293,
      "grad_norm": 498.0,
      "learning_rate": 4.960526315789474e-07,
      "logits/chosen": -0.6789365410804749,
      "logits/rejected": -0.6747094392776489,
      "logps/chosen": -1065.556640625,
      "logps/rejected": -1103.8992919921875,
      "loss": 0.6954,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0016559602227061987,
      "rewards/margins": -0.003854083362966776,
      "rewards/rejected": 0.0055100442841649055,
      "step": 408
    },
    {
      "epoch": 0.10758026599155009,
      "grad_norm": 390.0,
      "learning_rate": 4.95906432748538e-07,
      "logits/chosen": -0.6513721942901611,
      "logits/rejected": -0.6466161012649536,
      "logps/chosen": -617.5242309570312,
      "logps/rejected": -659.999755859375,
      "loss": 0.6822,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.007896710187196732,
      "rewards/margins": 0.022522974759340286,
      "rewards/rejected": -0.01462626364082098,
      "step": 409
    },
    {
      "epoch": 0.10784329842673725,
      "grad_norm": 596.0,
      "learning_rate": 4.957602339181287e-07,
      "logits/chosen": -0.6477194428443909,
      "logits/rejected": -0.6494531631469727,
      "logps/chosen": -1385.3138427734375,
      "logps/rejected": -1246.540771484375,
      "loss": 0.6914,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.021145058795809746,
      "rewards/margins": 0.004136275965720415,
      "rewards/rejected": 0.017008783295750618,
      "step": 410
    },
    {
      "epoch": 0.10810633086192441,
      "grad_norm": 548.0,
      "learning_rate": 4.956140350877192e-07,
      "logits/chosen": -0.6374384164810181,
      "logits/rejected": -0.6389647722244263,
      "logps/chosen": -1219.482666015625,
      "logps/rejected": -917.78271484375,
      "loss": 0.7117,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.020290182903409004,
      "rewards/margins": -0.03589935228228569,
      "rewards/rejected": 0.015609169378876686,
      "step": 411
    },
    {
      "epoch": 0.10836936329711158,
      "grad_norm": 474.0,
      "learning_rate": 4.954678362573099e-07,
      "logits/chosen": -0.6620480418205261,
      "logits/rejected": -0.6594375371932983,
      "logps/chosen": -1061.3751220703125,
      "logps/rejected": -720.4833374023438,
      "loss": 0.6885,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008317947387695312,
      "rewards/margins": 0.01034068875014782,
      "rewards/rejected": -0.0020227434579283,
      "step": 412
    },
    {
      "epoch": 0.10863239573229874,
      "grad_norm": 568.0,
      "learning_rate": 4.953216374269006e-07,
      "logits/chosen": -0.6475723385810852,
      "logits/rejected": -0.6579403877258301,
      "logps/chosen": -1117.982666015625,
      "logps/rejected": -892.6618041992188,
      "loss": 0.6844,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.013082598336040974,
      "rewards/margins": 0.018273258581757545,
      "rewards/rejected": -0.005190657917410135,
      "step": 413
    },
    {
      "epoch": 0.1088954281674859,
      "grad_norm": 472.0,
      "learning_rate": 4.951754385964912e-07,
      "logits/chosen": -0.6802256107330322,
      "logits/rejected": -0.6693291068077087,
      "logps/chosen": -1137.5804443359375,
      "logps/rejected": -1010.0615844726562,
      "loss": 0.6918,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0018917082343250513,
      "rewards/margins": 0.0031585688702762127,
      "rewards/rejected": -0.0012668618001043797,
      "step": 414
    },
    {
      "epoch": 0.10915846060267306,
      "grad_norm": 544.0,
      "learning_rate": 4.950292397660819e-07,
      "logits/chosen": -0.6561927795410156,
      "logits/rejected": NaN,
      "logps/chosen": -1385.2281494140625,
      "logps/rejected": -746.2745361328125,
      "loss": 0.6992,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0030251494608819485,
      "rewards/margins": -0.010986708104610443,
      "rewards/rejected": 0.014011859893798828,
      "step": 415
    },
    {
      "epoch": 0.10942149303786024,
      "grad_norm": 478.0,
      "learning_rate": 4.948830409356725e-07,
      "logits/chosen": -0.6582752466201782,
      "logits/rejected": -0.655479907989502,
      "logps/chosen": -950.408447265625,
      "logps/rejected": -819.005126953125,
      "loss": 0.6958,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.010255146771669388,
      "rewards/margins": -0.004325579851865768,
      "rewards/rejected": 0.014580724760890007,
      "step": 416
    },
    {
      "epoch": 0.1096845254730474,
      "grad_norm": 716.0,
      "learning_rate": 4.947368421052631e-07,
      "logits/chosen": -0.623702883720398,
      "logits/rejected": -0.6284883618354797,
      "logps/chosen": -1270.1884765625,
      "logps/rejected": -871.7545776367188,
      "loss": 0.6939,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.012632180005311966,
      "rewards/margins": -6.0082413256168365e-05,
      "rewards/rejected": -0.012572098523378372,
      "step": 417
    },
    {
      "epoch": 0.10994755790823456,
      "grad_norm": 426.0,
      "learning_rate": 4.945906432748538e-07,
      "logits/chosen": -0.6373027563095093,
      "logits/rejected": -0.6380037665367126,
      "logps/chosen": -811.5731201171875,
      "logps/rejected": -513.7674560546875,
      "loss": 0.6976,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.011730480939149857,
      "rewards/margins": -0.00833754613995552,
      "rewards/rejected": -0.003392934799194336,
      "step": 418
    },
    {
      "epoch": 0.11021059034342172,
      "grad_norm": 502.0,
      "learning_rate": 4.944444444444445e-07,
      "logits/chosen": -0.6712849736213684,
      "logits/rejected": -0.6762508153915405,
      "logps/chosen": -1021.4556884765625,
      "logps/rejected": -780.7979125976562,
      "loss": 0.6899,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.011895084753632545,
      "rewards/margins": 0.007075786124914885,
      "rewards/rejected": 0.004819298163056374,
      "step": 419
    },
    {
      "epoch": 0.1104736227786089,
      "grad_norm": 556.0,
      "learning_rate": 4.94298245614035e-07,
      "logits/chosen": -0.6525319814682007,
      "logits/rejected": -0.6712098717689514,
      "logps/chosen": -1122.7117919921875,
      "logps/rejected": -912.10595703125,
      "loss": 0.6943,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00543823279440403,
      "rewards/margins": -0.0011725444346666336,
      "rewards/rejected": -0.0042656902223825455,
      "step": 420
    },
    {
      "epoch": 0.11073665521379605,
      "grad_norm": 676.0,
      "learning_rate": 4.941520467836257e-07,
      "logits/chosen": -0.6379592418670654,
      "logits/rejected": -0.6547951698303223,
      "logps/chosen": -1256.466552734375,
      "logps/rejected": -1230.877685546875,
      "loss": 0.6918,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0028044702485203743,
      "rewards/margins": 0.0041407570242881775,
      "rewards/rejected": -0.0013362881727516651,
      "step": 421
    },
    {
      "epoch": 0.11099968764898321,
      "grad_norm": 458.0,
      "learning_rate": 4.940058479532163e-07,
      "logits/chosen": -0.641041100025177,
      "logits/rejected": -0.6582581996917725,
      "logps/chosen": -876.3052368164062,
      "logps/rejected": -469.0206298828125,
      "loss": 0.6968,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.003149890573695302,
      "rewards/margins": -0.006825351621955633,
      "rewards/rejected": 0.003675461281090975,
      "step": 422
    },
    {
      "epoch": 0.11126272008417037,
      "grad_norm": 576.0,
      "learning_rate": 4.93859649122807e-07,
      "logits/chosen": -0.6622021198272705,
      "logits/rejected": -0.655548095703125,
      "logps/chosen": -1174.82861328125,
      "logps/rejected": -924.9749755859375,
      "loss": 0.6983,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.01314392127096653,
      "rewards/margins": -0.009348297491669655,
      "rewards/rejected": -0.0037956233136355877,
      "step": 423
    },
    {
      "epoch": 0.11152575251935755,
      "grad_norm": 394.0,
      "learning_rate": 4.937134502923976e-07,
      "logits/chosen": -0.6334827542304993,
      "logits/rejected": -0.6417543292045593,
      "logps/chosen": -710.669921875,
      "logps/rejected": -652.84912109375,
      "loss": 0.6943,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.0048239706084132195,
      "rewards/margins": -0.0018152245320379734,
      "rewards/rejected": 0.0066391946747899055,
      "step": 424
    },
    {
      "epoch": 0.11178878495454471,
      "grad_norm": 552.0,
      "learning_rate": 4.935672514619883e-07,
      "logits/chosen": -0.6570732593536377,
      "logits/rejected": -0.6614696979522705,
      "logps/chosen": -1196.081298828125,
      "logps/rejected": -1221.447265625,
      "loss": 0.6904,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.008006571792066097,
      "rewards/margins": 0.006163312122225761,
      "rewards/rejected": -0.014169884845614433,
      "step": 425
    },
    {
      "epoch": 0.11205181738973187,
      "grad_norm": 580.0,
      "learning_rate": 4.934210526315789e-07,
      "logits/chosen": -0.6502673625946045,
      "logits/rejected": -0.6564635038375854,
      "logps/chosen": -1481.185546875,
      "logps/rejected": -994.21337890625,
      "loss": 0.6899,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.003949357196688652,
      "rewards/margins": 0.007239199243485928,
      "rewards/rejected": -0.003289843210950494,
      "step": 426
    },
    {
      "epoch": 0.11231484982491903,
      "grad_norm": 510.0,
      "learning_rate": 4.932748538011696e-07,
      "logits/chosen": -0.649726927280426,
      "logits/rejected": -0.6471753716468811,
      "logps/chosen": -1343.54150390625,
      "logps/rejected": -972.6571044921875,
      "loss": 0.6793,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.02503242716193199,
      "rewards/margins": 0.028229428455233574,
      "rewards/rejected": -0.003197002224624157,
      "step": 427
    },
    {
      "epoch": 0.1125778822601062,
      "grad_norm": 482.0,
      "learning_rate": 4.931286549707603e-07,
      "logits/chosen": -0.6408566832542419,
      "logits/rejected": -0.6436323523521423,
      "logps/chosen": -967.6197509765625,
      "logps/rejected": -758.3827514648438,
      "loss": 0.697,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.011100675910711288,
      "rewards/margins": -0.007018375676125288,
      "rewards/rejected": 0.018119048327207565,
      "step": 428
    },
    {
      "epoch": 0.11284091469529337,
      "grad_norm": 632.0,
      "learning_rate": 4.929824561403508e-07,
      "logits/chosen": -0.6474649906158447,
      "logits/rejected": -0.6546456217765808,
      "logps/chosen": -1189.7103271484375,
      "logps/rejected": -850.369384765625,
      "loss": 0.6885,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.019398879259824753,
      "rewards/margins": 0.009976767934858799,
      "rewards/rejected": 0.009422111324965954,
      "step": 429
    },
    {
      "epoch": 0.11310394713048053,
      "grad_norm": 584.0,
      "learning_rate": 4.928362573099415e-07,
      "logits/chosen": -0.632500410079956,
      "logits/rejected": -0.6357671618461609,
      "logps/chosen": -870.778076171875,
      "logps/rejected": -818.5460205078125,
      "loss": 0.6942,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0036073687952011824,
      "rewards/margins": -0.0017198568675667048,
      "rewards/rejected": -0.001887512276880443,
      "step": 430
    },
    {
      "epoch": 0.11336697956566769,
      "grad_norm": 664.0,
      "learning_rate": 4.926900584795321e-07,
      "logits/chosen": -0.637924075126648,
      "logits/rejected": -0.6375539302825928,
      "logps/chosen": -997.6199951171875,
      "logps/rejected": -760.9223022460938,
      "loss": 0.69,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.012534906156361103,
      "rewards/margins": 0.007072876673191786,
      "rewards/rejected": -0.0196077823638916,
      "step": 431
    },
    {
      "epoch": 0.11363001200085486,
      "grad_norm": 524.0,
      "learning_rate": 4.925438596491228e-07,
      "logits/chosen": -0.6618976593017578,
      "logits/rejected": -0.654763400554657,
      "logps/chosen": -978.6613159179688,
      "logps/rejected": -700.9021606445312,
      "loss": 0.6903,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006518364883959293,
      "rewards/margins": 0.0063149454072117805,
      "rewards/rejected": 0.00020341743947938085,
      "step": 432
    },
    {
      "epoch": 0.11389304443604202,
      "grad_norm": 740.0,
      "learning_rate": 4.923976608187134e-07,
      "logits/chosen": -0.6782190203666687,
      "logits/rejected": -0.6877949833869934,
      "logps/chosen": -1606.3846435546875,
      "logps/rejected": -1074.387939453125,
      "loss": 0.6903,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004541302099823952,
      "rewards/margins": 0.006964874919503927,
      "rewards/rejected": -0.0024235728196799755,
      "step": 433
    },
    {
      "epoch": 0.11415607687122918,
      "grad_norm": 548.0,
      "learning_rate": 4.922514619883041e-07,
      "logits/chosen": -0.6566163301467896,
      "logits/rejected": -0.6624320149421692,
      "logps/chosen": -990.8555297851562,
      "logps/rejected": -759.8629760742188,
      "loss": 0.6805,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.031916145235300064,
      "rewards/margins": 0.025928687304258347,
      "rewards/rejected": 0.005987453740090132,
      "step": 434
    },
    {
      "epoch": 0.11441910930641634,
      "grad_norm": 668.0,
      "learning_rate": 4.921052631578947e-07,
      "logits/chosen": -0.6891883015632629,
      "logits/rejected": -0.7005688548088074,
      "logps/chosen": -1884.705322265625,
      "logps/rejected": -1377.092041015625,
      "loss": 0.692,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.022716524079442024,
      "rewards/margins": 0.0031872752588242292,
      "rewards/rejected": 0.019529247656464577,
      "step": 435
    },
    {
      "epoch": 0.11468214174160352,
      "grad_norm": 580.0,
      "learning_rate": 4.919590643274853e-07,
      "logits/chosen": -0.6688225865364075,
      "logits/rejected": -0.6687368750572205,
      "logps/chosen": -1260.5853271484375,
      "logps/rejected": -1000.1238403320312,
      "loss": 0.6929,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0052101146429777145,
      "rewards/margins": 0.0015459060668945312,
      "rewards/rejected": -0.006756019778549671,
      "step": 436
    },
    {
      "epoch": 0.11494517417679068,
      "grad_norm": 492.0,
      "learning_rate": 4.91812865497076e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6407371759414673,
      "logps/chosen": -1039.739501953125,
      "logps/rejected": -1013.825439453125,
      "loss": 0.6957,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.023665238171815872,
      "rewards/margins": -0.004816723521798849,
      "rewards/rejected": -0.018848516047000885,
      "step": 437
    },
    {
      "epoch": 0.11520820661197784,
      "grad_norm": 498.0,
      "learning_rate": 4.916666666666666e-07,
      "logits/chosen": -0.6669853925704956,
      "logits/rejected": -0.6711788177490234,
      "logps/chosen": -1172.8128662109375,
      "logps/rejected": -927.5643920898438,
      "loss": 0.694,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0014010434970259666,
      "rewards/margins": -0.0011658663861453533,
      "rewards/rejected": 0.00256690988317132,
      "step": 438
    },
    {
      "epoch": 0.115471239047165,
      "grad_norm": 490.0,
      "learning_rate": 4.915204678362573e-07,
      "logits/chosen": -0.6595950722694397,
      "logits/rejected": -0.661357045173645,
      "logps/chosen": -1005.705322265625,
      "logps/rejected": -829.6026000976562,
      "loss": 0.6988,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.007011605892330408,
      "rewards/margins": -0.010509680956602097,
      "rewards/rejected": 0.0034980783239006996,
      "step": 439
    },
    {
      "epoch": 0.11573427148235217,
      "grad_norm": 560.0,
      "learning_rate": 4.913742690058479e-07,
      "logits/chosen": -0.6613905429840088,
      "logits/rejected": -0.6704486608505249,
      "logps/chosen": -1099.102294921875,
      "logps/rejected": -973.2681274414062,
      "loss": 0.6966,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.018236063420772552,
      "rewards/margins": -0.005321788601577282,
      "rewards/rejected": 0.023557856678962708,
      "step": 440
    },
    {
      "epoch": 0.11599730391753933,
      "grad_norm": 408.0,
      "learning_rate": 4.912280701754385e-07,
      "logits/chosen": -0.6514667868614197,
      "logits/rejected": -0.649681568145752,
      "logps/chosen": -1037.820068359375,
      "logps/rejected": -1033.3524169921875,
      "loss": 0.6906,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0034064287319779396,
      "rewards/margins": 0.005371284671127796,
      "rewards/rejected": -0.001964855007827282,
      "step": 441
    },
    {
      "epoch": 0.11626033635272649,
      "grad_norm": 576.0,
      "learning_rate": 4.910818713450292e-07,
      "logits/chosen": -0.6405940651893616,
      "logits/rejected": -0.6481306552886963,
      "logps/chosen": -1146.455810546875,
      "logps/rejected": -934.0137939453125,
      "loss": 0.6953,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00996322650462389,
      "rewards/margins": -0.0038928035646677017,
      "rewards/rejected": 0.013856029137969017,
      "step": 442
    },
    {
      "epoch": 0.11652336878791365,
      "grad_norm": 552.0,
      "learning_rate": 4.909356725146199e-07,
      "logits/chosen": -0.6487240195274353,
      "logits/rejected": -0.654698371887207,
      "logps/chosen": -1421.2747802734375,
      "logps/rejected": -1238.619873046875,
      "loss": 0.6984,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.00788202416151762,
      "rewards/margins": -0.01002960093319416,
      "rewards/rejected": 0.017911624163389206,
      "step": 443
    },
    {
      "epoch": 0.11678640122310083,
      "grad_norm": 812.0,
      "learning_rate": 4.907894736842105e-07,
      "logits/chosen": -0.6558486819267273,
      "logits/rejected": -0.662229061126709,
      "logps/chosen": -1340.640625,
      "logps/rejected": -1651.473388671875,
      "loss": 0.6878,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.005771923344582319,
      "rewards/margins": 0.012363624759018421,
      "rewards/rejected": -0.018135547637939453,
      "step": 444
    },
    {
      "epoch": 0.11704943365828799,
      "grad_norm": 688.0,
      "learning_rate": 4.906432748538011e-07,
      "logits/chosen": -0.6547192931175232,
      "logits/rejected": -0.6794427633285522,
      "logps/chosen": -1673.4765625,
      "logps/rejected": -1158.641357421875,
      "loss": 0.6964,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00939865130931139,
      "rewards/margins": -0.005473900120705366,
      "rewards/rejected": -0.0039247507229447365,
      "step": 445
    },
    {
      "epoch": 0.11731246609347515,
      "grad_norm": 560.0,
      "learning_rate": 4.904970760233918e-07,
      "logits/chosen": -0.6492964625358582,
      "logits/rejected": -0.6608262062072754,
      "logps/chosen": -1394.31787109375,
      "logps/rejected": -909.8809814453125,
      "loss": 0.7029,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.012937736697494984,
      "rewards/margins": -0.018783092498779297,
      "rewards/rejected": 0.005845355801284313,
      "step": 446
    },
    {
      "epoch": 0.11757549852866231,
      "grad_norm": 624.0,
      "learning_rate": 4.903508771929825e-07,
      "logits/chosen": -0.6534320712089539,
      "logits/rejected": NaN,
      "logps/chosen": -1561.83203125,
      "logps/rejected": -935.690673828125,
      "loss": 0.6792,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.016530990600585938,
      "rewards/margins": 0.0290590301156044,
      "rewards/rejected": -0.012528038583695889,
      "step": 447
    },
    {
      "epoch": 0.11783853096384948,
      "grad_norm": 656.0,
      "learning_rate": 4.902046783625731e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6764371991157532,
      "logps/chosen": -1288.59716796875,
      "logps/rejected": -957.0806274414062,
      "loss": 0.7072,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0213959701359272,
      "rewards/margins": -0.026488494127988815,
      "rewards/rejected": 0.005092525854706764,
      "step": 448
    },
    {
      "epoch": 0.11810156339903664,
      "grad_norm": 504.0,
      "learning_rate": 4.900584795321637e-07,
      "logits/chosen": -0.6690979599952698,
      "logits/rejected": -0.6672987937927246,
      "logps/chosen": -1172.7657470703125,
      "logps/rejected": -900.905517578125,
      "loss": 0.6926,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.018794726580381393,
      "rewards/margins": 0.0019073492148891091,
      "rewards/rejected": 0.016887377947568893,
      "step": 449
    },
    {
      "epoch": 0.1183645958342238,
      "grad_norm": 540.0,
      "learning_rate": 4.899122807017543e-07,
      "logits/chosen": -0.6343806385993958,
      "logits/rejected": -0.6302553415298462,
      "logps/chosen": -1315.7581787109375,
      "logps/rejected": -1057.3045654296875,
      "loss": 0.7081,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.02464752271771431,
      "rewards/margins": -0.028812218457460403,
      "rewards/rejected": 0.004164694808423519,
      "step": 450
    },
    {
      "epoch": 0.11862762826941098,
      "grad_norm": 462.0,
      "learning_rate": 4.89766081871345e-07,
      "logits/chosen": -0.6548861265182495,
      "logits/rejected": -0.6533276438713074,
      "logps/chosen": -1118.9281005859375,
      "logps/rejected": -766.8286743164062,
      "loss": 0.7025,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004336929880082607,
      "rewards/margins": -0.018058249726891518,
      "rewards/rejected": 0.0223951805382967,
      "step": 451
    },
    {
      "epoch": 0.11889066070459814,
      "grad_norm": 468.0,
      "learning_rate": 4.896198830409357e-07,
      "logits/chosen": -0.648120105266571,
      "logits/rejected": -0.6353003978729248,
      "logps/chosen": -1425.86328125,
      "logps/rejected": -1181.094482421875,
      "loss": 0.6919,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.004661655053496361,
      "rewards/margins": 0.003265952691435814,
      "rewards/rejected": -0.007927609607577324,
      "step": 452
    },
    {
      "epoch": 0.1191536931397853,
      "grad_norm": 488.0,
      "learning_rate": 4.894736842105263e-07,
      "logits/chosen": -0.6345666646957397,
      "logits/rejected": -0.6400364637374878,
      "logps/chosen": -893.5048828125,
      "logps/rejected": -811.1396484375,
      "loss": 0.6871,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0212263111025095,
      "rewards/margins": 0.012777996249496937,
      "rewards/rejected": 0.008448313921689987,
      "step": 453
    },
    {
      "epoch": 0.11941672557497246,
      "grad_norm": 436.0,
      "learning_rate": 4.893274853801169e-07,
      "logits/chosen": -0.6492687463760376,
      "logits/rejected": -0.6511315703392029,
      "logps/chosen": -862.75732421875,
      "logps/rejected": -758.02490234375,
      "loss": 0.6869,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0031505580991506577,
      "rewards/margins": 0.013158511370420456,
      "rewards/rejected": -0.016309071332216263,
      "step": 454
    },
    {
      "epoch": 0.11967975801015963,
      "grad_norm": 548.0,
      "learning_rate": 4.891812865497075e-07,
      "logits/chosen": -0.6440595388412476,
      "logits/rejected": -0.6525145769119263,
      "logps/chosen": -1088.24267578125,
      "logps/rejected": -809.7687377929688,
      "loss": 0.711,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0327395424246788,
      "rewards/margins": -0.0345248207449913,
      "rewards/rejected": 0.0017852778546512127,
      "step": 455
    },
    {
      "epoch": 0.1199427904453468,
      "grad_norm": 502.0,
      "learning_rate": 4.890350877192983e-07,
      "logits/chosen": -0.6230167150497437,
      "logits/rejected": -0.6108574271202087,
      "logps/chosen": -1184.6884765625,
      "logps/rejected": -976.4376220703125,
      "loss": 0.7055,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.016309738159179688,
      "rewards/margins": -0.023158933967351913,
      "rewards/rejected": 0.006849193014204502,
      "step": 456
    },
    {
      "epoch": 0.12020582288053396,
      "grad_norm": 584.0,
      "learning_rate": 4.888888888888889e-07,
      "logits/chosen": -0.6597912311553955,
      "logits/rejected": -0.6560087203979492,
      "logps/chosen": -1397.6527099609375,
      "logps/rejected": -925.844970703125,
      "loss": 0.6933,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00021266844123601913,
      "rewards/margins": 0.00038604671135544777,
      "rewards/rejected": -0.0005987172480672598,
      "step": 457
    },
    {
      "epoch": 0.12046885531572112,
      "grad_norm": 414.0,
      "learning_rate": 4.887426900584795e-07,
      "logits/chosen": -0.6261149048805237,
      "logits/rejected": -0.6267373561859131,
      "logps/chosen": -932.5118408203125,
      "logps/rejected": -925.6280517578125,
      "loss": 0.7035,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.014931775629520416,
      "rewards/margins": -0.02005796507000923,
      "rewards/rejected": 0.00512619037181139,
      "step": 458
    },
    {
      "epoch": 0.12073188775090829,
      "grad_norm": 548.0,
      "learning_rate": 4.885964912280701e-07,
      "logits/chosen": -0.6553400158882141,
      "logits/rejected": -0.6701475381851196,
      "logps/chosen": -1223.78271484375,
      "logps/rejected": -963.9017944335938,
      "loss": 0.682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.015366362407803535,
      "rewards/margins": 0.023986291140317917,
      "rewards/rejected": -0.008619928732514381,
      "step": 459
    },
    {
      "epoch": 0.12099492018609545,
      "grad_norm": 676.0,
      "learning_rate": 4.884502923976608e-07,
      "logits/chosen": -0.6598070859909058,
      "logits/rejected": -0.6688258647918701,
      "logps/chosen": -1158.373779296875,
      "logps/rejected": -900.0425415039062,
      "loss": 0.6857,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012372016906738281,
      "rewards/margins": 0.016020964831113815,
      "rewards/rejected": -0.003648948622867465,
      "step": 460
    },
    {
      "epoch": 0.12125795262128261,
      "grad_norm": 504.0,
      "learning_rate": 4.883040935672515e-07,
      "logits/chosen": -0.6382073163986206,
      "logits/rejected": NaN,
      "logps/chosen": -1226.057373046875,
      "logps/rejected": -809.2221069335938,
      "loss": 0.6923,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0023064608685672283,
      "rewards/margins": 0.0022115702740848064,
      "rewards/rejected": -0.004518031608313322,
      "step": 461
    },
    {
      "epoch": 0.12152098505646977,
      "grad_norm": 540.0,
      "learning_rate": 4.881578947368421e-07,
      "logits/chosen": -0.6457525491714478,
      "logits/rejected": -0.6389673352241516,
      "logps/chosen": -1085.1395263671875,
      "logps/rejected": -826.2454833984375,
      "loss": 0.6968,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.001690769218839705,
      "rewards/margins": -0.006767703220248222,
      "rewards/rejected": 0.005076933652162552,
      "step": 462
    },
    {
      "epoch": 0.12178401749165695,
      "grad_norm": 402.0,
      "learning_rate": 4.880116959064327e-07,
      "logits/chosen": -0.6486125588417053,
      "logits/rejected": NaN,
      "logps/chosen": -564.6585083007812,
      "logps/rejected": -469.13134765625,
      "loss": 0.6889,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.014987422153353691,
      "rewards/margins": 0.008719492703676224,
      "rewards/rejected": 0.006267929915338755,
      "step": 463
    },
    {
      "epoch": 0.1220470499268441,
      "grad_norm": 520.0,
      "learning_rate": 4.878654970760233e-07,
      "logits/chosen": -0.6273882985115051,
      "logits/rejected": -0.6317552328109741,
      "logps/chosen": -1000.71142578125,
      "logps/rejected": -855.29638671875,
      "loss": 0.683,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00958719290792942,
      "rewards/margins": 0.020761966705322266,
      "rewards/rejected": -0.01117477472871542,
      "step": 464
    },
    {
      "epoch": 0.12231008236203127,
      "grad_norm": 664.0,
      "learning_rate": 4.877192982456141e-07,
      "logits/chosen": -0.6608648300170898,
      "logits/rejected": -0.6669347286224365,
      "logps/chosen": -1484.809326171875,
      "logps/rejected": -1021.1392822265625,
      "loss": 0.7021,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01711559295654297,
      "rewards/margins": -0.016025735065340996,
      "rewards/rejected": -0.0010898587061092257,
      "step": 465
    },
    {
      "epoch": 0.12257311479721843,
      "grad_norm": 624.0,
      "learning_rate": 4.875730994152047e-07,
      "logits/chosen": -0.6585571765899658,
      "logits/rejected": -0.6497384309768677,
      "logps/chosen": -1390.97265625,
      "logps/rejected": -1295.92236328125,
      "loss": 0.7037,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.007139395922422409,
      "rewards/margins": -0.02025279961526394,
      "rewards/rejected": 0.01311340369284153,
      "step": 466
    },
    {
      "epoch": 0.1228361472324056,
      "grad_norm": 506.0,
      "learning_rate": 4.874269005847953e-07,
      "logits/chosen": -0.6589363217353821,
      "logits/rejected": -0.6441799402236938,
      "logps/chosen": -1024.96337890625,
      "logps/rejected": -609.979248046875,
      "loss": 0.6878,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0018026353791356087,
      "rewards/margins": 0.011275529861450195,
      "rewards/rejected": -0.013078165240585804,
      "step": 467
    },
    {
      "epoch": 0.12309917966759276,
      "grad_norm": 474.0,
      "learning_rate": 4.872807017543859e-07,
      "logits/chosen": -0.6254754066467285,
      "logits/rejected": -0.6355024576187134,
      "logps/chosen": -925.7980346679688,
      "logps/rejected": -591.854736328125,
      "loss": 0.683,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.013832664117217064,
      "rewards/margins": 0.0207353588193655,
      "rewards/rejected": -0.006902695167809725,
      "step": 468
    },
    {
      "epoch": 0.12336221210277992,
      "grad_norm": 520.0,
      "learning_rate": 4.871345029239765e-07,
      "logits/chosen": -0.6541805863380432,
      "logits/rejected": -0.6539812088012695,
      "logps/chosen": -1150.2108154296875,
      "logps/rejected": -1115.0096435546875,
      "loss": 0.6953,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004450702108442783,
      "rewards/margins": -0.003258991986513138,
      "rewards/rejected": 0.007709694094955921,
      "step": 469
    },
    {
      "epoch": 0.12362524453796708,
      "grad_norm": 500.0,
      "learning_rate": 4.869883040935673e-07,
      "logits/chosen": -0.6510951519012451,
      "logits/rejected": -0.6668254733085632,
      "logps/chosen": -837.047607421875,
      "logps/rejected": -684.4010620117188,
      "loss": 0.6902,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01953563652932644,
      "rewards/margins": 0.006593703292310238,
      "rewards/rejected": 0.012941932305693626,
      "step": 470
    },
    {
      "epoch": 0.12388827697315426,
      "grad_norm": 528.0,
      "learning_rate": 4.868421052631579e-07,
      "logits/chosen": -0.6415792107582092,
      "logits/rejected": -0.6529781818389893,
      "logps/chosen": -1397.5892333984375,
      "logps/rejected": -1001.2159423828125,
      "loss": 0.703,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.026717757806181908,
      "rewards/margins": -0.01910429075360298,
      "rewards/rejected": -0.007613467052578926,
      "step": 471
    },
    {
      "epoch": 0.12415130940834142,
      "grad_norm": 588.0,
      "learning_rate": 4.866959064327485e-07,
      "logits/chosen": -0.6420029401779175,
      "logits/rejected": -0.6517320871353149,
      "logps/chosen": -1073.255126953125,
      "logps/rejected": -561.1179809570312,
      "loss": 0.6835,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006833935156464577,
      "rewards/margins": 0.020499708130955696,
      "rewards/rejected": -0.013665772043168545,
      "step": 472
    },
    {
      "epoch": 0.12441434184352858,
      "grad_norm": 628.0,
      "learning_rate": 4.865497076023391e-07,
      "logits/chosen": -0.6930283308029175,
      "logits/rejected": -0.6855157017707825,
      "logps/chosen": -1148.962158203125,
      "logps/rejected": -892.2872314453125,
      "loss": 0.7022,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.015285111963748932,
      "rewards/margins": -0.017420005053281784,
      "rewards/rejected": 0.0021348954178392887,
      "step": 473
    },
    {
      "epoch": 0.12467737427871574,
      "grad_norm": 1376.0,
      "learning_rate": 4.864035087719299e-07,
      "logits/chosen": -0.6437627673149109,
      "logits/rejected": -0.6486901044845581,
      "logps/chosen": -1166.4041748046875,
      "logps/rejected": -1072.503173828125,
      "loss": 0.6855,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0025899894535541534,
      "rewards/margins": 0.016553593799471855,
      "rewards/rejected": -0.013963603414595127,
      "step": 474
    },
    {
      "epoch": 0.12494040671390291,
      "grad_norm": 1152.0,
      "learning_rate": 4.862573099415205e-07,
      "logits/chosen": -0.6163784265518188,
      "logits/rejected": -0.6124188303947449,
      "logps/chosen": -1114.771728515625,
      "logps/rejected": -739.03271484375,
      "loss": 0.6839,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007243156433105469,
      "rewards/margins": 0.019089076668024063,
      "rewards/rejected": -0.011845925822854042,
      "step": 475
    },
    {
      "epoch": 0.12520343914909007,
      "grad_norm": 390.0,
      "learning_rate": 4.861111111111111e-07,
      "logits/chosen": -0.6309524774551392,
      "logits/rejected": -0.6319814324378967,
      "logps/chosen": -788.746826171875,
      "logps/rejected": -485.1277770996094,
      "loss": 0.7003,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.002976846881210804,
      "rewards/margins": -0.013920927420258522,
      "rewards/rejected": 0.0168977752327919,
      "step": 476
    },
    {
      "epoch": 0.12546647158427723,
      "grad_norm": 640.0,
      "learning_rate": 4.859649122807017e-07,
      "logits/chosen": -0.6574693918228149,
      "logits/rejected": NaN,
      "logps/chosen": -1525.87158203125,
      "logps/rejected": -829.5984497070312,
      "loss": 0.683,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0038453591987490654,
      "rewards/margins": 0.021307993680238724,
      "rewards/rejected": -0.017462637275457382,
      "step": 477
    },
    {
      "epoch": 0.1257295040194644,
      "grad_norm": 516.0,
      "learning_rate": 4.858187134502923e-07,
      "logits/chosen": -0.6385027766227722,
      "logits/rejected": -0.650617778301239,
      "logps/chosen": -828.6520385742188,
      "logps/rejected": -668.8001708984375,
      "loss": 0.6935,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.014794254675507545,
      "rewards/margins": -0.0002989765489473939,
      "rewards/rejected": -0.014495277777314186,
      "step": 478
    },
    {
      "epoch": 0.12599253645465155,
      "grad_norm": 532.0,
      "learning_rate": 4.856725146198831e-07,
      "logits/chosen": -0.6760532855987549,
      "logits/rejected": -0.6697038412094116,
      "logps/chosen": -903.2620239257812,
      "logps/rejected": -854.5142822265625,
      "loss": 0.7059,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.020189572125673294,
      "rewards/margins": -0.024651192128658295,
      "rewards/rejected": 0.004461622331291437,
      "step": 479
    },
    {
      "epoch": 0.12625556888983872,
      "grad_norm": 652.0,
      "learning_rate": 4.855263157894737e-07,
      "logits/chosen": -0.6264652013778687,
      "logits/rejected": -0.6247514486312866,
      "logps/chosen": -1565.491455078125,
      "logps/rejected": -1161.4844970703125,
      "loss": 0.6909,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.002096748910844326,
      "rewards/margins": 0.005349920596927404,
      "rewards/rejected": -0.0032531730830669403,
      "step": 480
    },
    {
      "epoch": 0.1265186013250259,
      "grad_norm": 382.0,
      "learning_rate": 4.853801169590643e-07,
      "logits/chosen": -0.6584673523902893,
      "logits/rejected": -0.6665544509887695,
      "logps/chosen": -1031.97412109375,
      "logps/rejected": -816.8914794921875,
      "loss": 0.6943,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004962253849953413,
      "rewards/margins": -0.0021880147978663445,
      "rewards/rejected": 0.00715026818215847,
      "step": 481
    },
    {
      "epoch": 0.12678163376021306,
      "grad_norm": 478.0,
      "learning_rate": 4.852339181286549e-07,
      "logits/chosen": -0.637371838092804,
      "logits/rejected": -0.6418337225914001,
      "logps/chosen": -842.7682495117188,
      "logps/rejected": -692.6647338867188,
      "loss": 0.6888,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.001556301023811102,
      "rewards/margins": 0.009653760120272636,
      "rewards/rejected": -0.008097458630800247,
      "step": 482
    },
    {
      "epoch": 0.12704466619540022,
      "grad_norm": 7680.0,
      "learning_rate": 4.850877192982456e-07,
      "logits/chosen": -0.6717047691345215,
      "logits/rejected": -0.6595069766044617,
      "logps/chosen": -1460.17822265625,
      "logps/rejected": -1216.4056396484375,
      "loss": 0.7011,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.00040459586307406425,
      "rewards/margins": -0.014338876120746136,
      "rewards/rejected": 0.013934277929365635,
      "step": 483
    },
    {
      "epoch": 0.12730769863058738,
      "grad_norm": 552.0,
      "learning_rate": 4.849415204678363e-07,
      "logits/chosen": -0.6358560919761658,
      "logits/rejected": -0.6400071978569031,
      "logps/chosen": -1408.602294921875,
      "logps/rejected": -967.60107421875,
      "loss": 0.7036,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.006542206276208162,
      "rewards/margins": -0.020281411707401276,
      "rewards/rejected": 0.013739204965531826,
      "step": 484
    },
    {
      "epoch": 0.12757073106577455,
      "grad_norm": 444.0,
      "learning_rate": 4.847953216374269e-07,
      "logits/chosen": -0.6620662212371826,
      "logits/rejected": -0.6684134602546692,
      "logps/chosen": -629.3146362304688,
      "logps/rejected": -747.7568359375,
      "loss": 0.6911,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.014173317700624466,
      "rewards/margins": 0.004753685090690851,
      "rewards/rejected": -0.01892700046300888,
      "step": 485
    },
    {
      "epoch": 0.1278337635009617,
      "grad_norm": 560.0,
      "learning_rate": 4.846491228070175e-07,
      "logits/chosen": -0.6379152536392212,
      "logits/rejected": -0.6404035687446594,
      "logps/chosen": -1216.5155029296875,
      "logps/rejected": -1187.354248046875,
      "loss": 0.6892,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.009013176895678043,
      "rewards/margins": 0.00878763198852539,
      "rewards/rejected": -0.01780080795288086,
      "step": 486
    },
    {
      "epoch": 0.12809679593614887,
      "grad_norm": 548.0,
      "learning_rate": 4.845029239766082e-07,
      "logits/chosen": -0.6475176811218262,
      "logits/rejected": -0.6437628865242004,
      "logps/chosen": -1185.4185791015625,
      "logps/rejected": -1012.613525390625,
      "loss": 0.6944,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.021749306470155716,
      "rewards/margins": -0.0017985336016863585,
      "rewards/rejected": -0.01995077170431614,
      "step": 487
    },
    {
      "epoch": 0.12835982837133603,
      "grad_norm": 416.0,
      "learning_rate": 4.843567251461988e-07,
      "logits/chosen": -0.6631720066070557,
      "logits/rejected": -0.6624504923820496,
      "logps/chosen": -704.355224609375,
      "logps/rejected": -510.6564025878906,
      "loss": 0.6868,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01696052774786949,
      "rewards/margins": 0.01418628916144371,
      "rewards/rejected": 0.0027742383535951376,
      "step": 488
    },
    {
      "epoch": 0.12862286080652321,
      "grad_norm": 652.0,
      "learning_rate": 4.842105263157895e-07,
      "logits/chosen": -0.6479825973510742,
      "logits/rejected": -0.6427233815193176,
      "logps/chosen": -1297.000244140625,
      "logps/rejected": -874.7701416015625,
      "loss": 0.6989,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0012496942654252052,
      "rewards/margins": -0.010835360735654831,
      "rewards/rejected": 0.012085056863725185,
      "step": 489
    },
    {
      "epoch": 0.12888589324171038,
      "grad_norm": 474.0,
      "learning_rate": 4.840643274853801e-07,
      "logits/chosen": -0.6299773454666138,
      "logits/rejected": -0.6340715885162354,
      "logps/chosen": -802.4874267578125,
      "logps/rejected": -651.0548095703125,
      "loss": 0.7002,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0064650545828044415,
      "rewards/margins": -0.013617992401123047,
      "rewards/rejected": 0.007152939680963755,
      "step": 490
    },
    {
      "epoch": 0.12914892567689754,
      "grad_norm": 400.0,
      "learning_rate": 4.839181286549707e-07,
      "logits/chosen": -0.6571359634399414,
      "logits/rejected": -0.6718398332595825,
      "logps/chosen": -888.20068359375,
      "logps/rejected": -720.225341796875,
      "loss": 0.693,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0006063454784452915,
      "rewards/margins": 0.0005041602998971939,
      "rewards/rejected": -0.0011105055455118418,
      "step": 491
    },
    {
      "epoch": 0.1294119581120847,
      "grad_norm": 498.0,
      "learning_rate": 4.837719298245614e-07,
      "logits/chosen": -0.6640750765800476,
      "logits/rejected": -0.6708216071128845,
      "logps/chosen": -972.9691772460938,
      "logps/rejected": -1018.4635620117188,
      "loss": 0.7067,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.014795970171689987,
      "rewards/margins": -0.026091478765010834,
      "rewards/rejected": 0.011295508593320847,
      "step": 492
    },
    {
      "epoch": 0.12967499054727186,
      "grad_norm": 532.0,
      "learning_rate": 4.836257309941521e-07,
      "logits/chosen": -0.6393430233001709,
      "logits/rejected": -0.6345587968826294,
      "logps/chosen": -1411.5556640625,
      "logps/rejected": -871.4142456054688,
      "loss": 0.6981,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.019999120384454727,
      "rewards/margins": -0.009113597683608532,
      "rewards/rejected": -0.01088552363216877,
      "step": 493
    },
    {
      "epoch": 0.12993802298245902,
      "grad_norm": 470.0,
      "learning_rate": 4.834795321637427e-07,
      "logits/chosen": -0.6625242829322815,
      "logits/rejected": -0.668809711933136,
      "logps/chosen": -840.5152587890625,
      "logps/rejected": -824.7583618164062,
      "loss": 0.6939,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0032976155634969473,
      "rewards/margins": -0.000873565673828125,
      "rewards/rejected": 0.004171181004494429,
      "step": 494
    },
    {
      "epoch": 0.13020105541764618,
      "grad_norm": 464.0,
      "learning_rate": 4.833333333333333e-07,
      "logits/chosen": -0.6351169347763062,
      "logits/rejected": -0.6494114995002747,
      "logps/chosen": -1010.057861328125,
      "logps/rejected": -878.3966064453125,
      "loss": 0.6973,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0008570682257413864,
      "rewards/margins": -0.007884408347308636,
      "rewards/rejected": 0.007027340587228537,
      "step": 495
    },
    {
      "epoch": 0.13046408785283337,
      "grad_norm": 712.0,
      "learning_rate": 4.83187134502924e-07,
      "logits/chosen": -0.6359748244285583,
      "logits/rejected": -0.6490459442138672,
      "logps/chosen": -1408.23095703125,
      "logps/rejected": -1057.9925537109375,
      "loss": 0.7111,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.023152543231844902,
      "rewards/margins": -0.034037020057439804,
      "rewards/rejected": 0.010884474962949753,
      "step": 496
    },
    {
      "epoch": 0.13072712028802053,
      "grad_norm": 486.0,
      "learning_rate": 4.830409356725145e-07,
      "logits/chosen": -0.6493430137634277,
      "logits/rejected": -0.652683675289154,
      "logps/chosen": -1008.0975341796875,
      "logps/rejected": -1020.5653076171875,
      "loss": 0.7015,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0035530095919966698,
      "rewards/margins": -0.016226578503847122,
      "rewards/rejected": 0.012673568911850452,
      "step": 497
    },
    {
      "epoch": 0.1309901527232077,
      "grad_norm": 446.0,
      "learning_rate": 4.828947368421053e-07,
      "logits/chosen": -0.634191632270813,
      "logits/rejected": -0.6426963806152344,
      "logps/chosen": -1190.978515625,
      "logps/rejected": -1008.3136596679688,
      "loss": 0.6906,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0077645303681492805,
      "rewards/margins": 0.005211353302001953,
      "rewards/rejected": 0.0025531775318086147,
      "step": 498
    },
    {
      "epoch": 0.13125318515839485,
      "grad_norm": 486.0,
      "learning_rate": 4.827485380116959e-07,
      "logits/chosen": -0.651569128036499,
      "logits/rejected": -0.6489272117614746,
      "logps/chosen": -969.9178466796875,
      "logps/rejected": -792.765380859375,
      "loss": 0.6947,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.005021191202104092,
      "rewards/margins": -0.002221678616479039,
      "rewards/rejected": 0.0072428700514137745,
      "step": 499
    },
    {
      "epoch": 0.131516217593582,
      "grad_norm": 680.0,
      "learning_rate": 4.826023391812865e-07,
      "logits/chosen": -0.6196260452270508,
      "logits/rejected": -0.6308959126472473,
      "logps/chosen": -1093.8433837890625,
      "logps/rejected": -914.7958984375,
      "loss": 0.6963,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.016752814874053,
      "rewards/margins": -0.004709959961473942,
      "rewards/rejected": -0.012042857706546783,
      "step": 500
    },
    {
      "epoch": 0.13177925002876917,
      "grad_norm": 424.0,
      "learning_rate": 4.824561403508772e-07,
      "logits/chosen": -0.6682766079902649,
      "logits/rejected": -0.6699283123016357,
      "logps/chosen": -744.862060546875,
      "logps/rejected": -609.9517211914062,
      "loss": 0.6812,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.015089226886630058,
      "rewards/margins": 0.024673795327544212,
      "rewards/rejected": -0.009584570303559303,
      "step": 501
    },
    {
      "epoch": 0.13204228246395633,
      "grad_norm": 510.0,
      "learning_rate": 4.823099415204678e-07,
      "logits/chosen": -0.6513470411300659,
      "logits/rejected": -0.6581114530563354,
      "logps/chosen": -866.8416748046875,
      "logps/rejected": -771.6387939453125,
      "loss": 0.6901,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.022140026092529297,
      "rewards/margins": 0.006749915890395641,
      "rewards/rejected": 0.01539011113345623,
      "step": 502
    },
    {
      "epoch": 0.1323053148991435,
      "grad_norm": 512.0,
      "learning_rate": 4.821637426900585e-07,
      "logits/chosen": -0.6337032318115234,
      "logits/rejected": -0.6349467039108276,
      "logps/chosen": -895.94580078125,
      "logps/rejected": -854.8506469726562,
      "loss": 0.6907,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004441071301698685,
      "rewards/margins": 0.005282212048768997,
      "rewards/rejected": -0.0008411409799009562,
      "step": 503
    },
    {
      "epoch": 0.13256834733433068,
      "grad_norm": 636.0,
      "learning_rate": 4.820175438596491e-07,
      "logits/chosen": -0.6617169976234436,
      "logits/rejected": NaN,
      "logps/chosen": -1355.8397216796875,
      "logps/rejected": -835.518798828125,
      "loss": 0.6916,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.021565057337284088,
      "rewards/margins": 0.0035863874945789576,
      "rewards/rejected": 0.017978670075535774,
      "step": 504
    },
    {
      "epoch": 0.13283137976951784,
      "grad_norm": 474.0,
      "learning_rate": 4.818713450292398e-07,
      "logits/chosen": -0.6412705779075623,
      "logits/rejected": -0.6402278542518616,
      "logps/chosen": -936.7109375,
      "logps/rejected": -777.92822265625,
      "loss": 0.692,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.004138469696044922,
      "rewards/margins": 0.0030287746340036392,
      "rewards/rejected": -0.007167243864387274,
      "step": 505
    },
    {
      "epoch": 0.133094412204705,
      "grad_norm": 644.0,
      "learning_rate": 4.817251461988303e-07,
      "logits/chosen": -0.632429301738739,
      "logits/rejected": -0.6309314370155334,
      "logps/chosen": -1257.4735107421875,
      "logps/rejected": -754.9366455078125,
      "loss": 0.6878,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.002307224553078413,
      "rewards/margins": 0.011636019684374332,
      "rewards/rejected": -0.009328793734312057,
      "step": 506
    },
    {
      "epoch": 0.13335744463989216,
      "grad_norm": 496.0,
      "learning_rate": 4.81578947368421e-07,
      "logits/chosen": -0.6436784267425537,
      "logits/rejected": -0.6470980048179626,
      "logps/chosen": -756.3818359375,
      "logps/rejected": -747.8235473632812,
      "loss": 0.6964,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0005214689299464226,
      "rewards/margins": -0.0059011452831327915,
      "rewards/rejected": 0.005379676818847656,
      "step": 507
    },
    {
      "epoch": 0.13362047707507932,
      "grad_norm": 496.0,
      "learning_rate": 4.814327485380117e-07,
      "logits/chosen": -0.6451561450958252,
      "logits/rejected": -0.6443666219711304,
      "logps/chosen": -910.962158203125,
      "logps/rejected": -874.3042602539062,
      "loss": 0.69,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0016632081242278218,
      "rewards/margins": 0.0067986492067575455,
      "rewards/rejected": -0.008461857214570045,
      "step": 508
    },
    {
      "epoch": 0.13388350951026648,
      "grad_norm": 564.0,
      "learning_rate": 4.812865497076023e-07,
      "logits/chosen": -0.674325704574585,
      "logits/rejected": -0.6712091565132141,
      "logps/chosen": -1258.7723388671875,
      "logps/rejected": -987.6766967773438,
      "loss": 0.6895,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00744581175968051,
      "rewards/margins": 0.007972432300448418,
      "rewards/rejected": -0.01541824359446764,
      "step": 509
    },
    {
      "epoch": 0.13414654194545364,
      "grad_norm": 640.0,
      "learning_rate": 4.811403508771929e-07,
      "logits/chosen": -0.6535459756851196,
      "logits/rejected": -0.6625362634658813,
      "logps/chosen": -1039.622802734375,
      "logps/rejected": -837.0830078125,
      "loss": 0.6916,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02295513264834881,
      "rewards/margins": 0.003825997933745384,
      "rewards/rejected": -0.026781130582094193,
      "step": 510
    },
    {
      "epoch": 0.1344095743806408,
      "grad_norm": 508.0,
      "learning_rate": 4.809941520467836e-07,
      "logits/chosen": -0.6579539179801941,
      "logits/rejected": -0.6491963267326355,
      "logps/chosen": -1174.4166259765625,
      "logps/rejected": -672.60595703125,
      "loss": 0.6865,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0057579027488827705,
      "rewards/margins": 0.01404876820743084,
      "rewards/rejected": -0.019806673750281334,
      "step": 511
    },
    {
      "epoch": 0.134672606815828,
      "grad_norm": 544.0,
      "learning_rate": 4.808479532163743e-07,
      "logits/chosen": -0.6552273035049438,
      "logits/rejected": -0.6429293155670166,
      "logps/chosen": -1278.2847900390625,
      "logps/rejected": -809.2093505859375,
      "loss": 0.6952,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.016009902581572533,
      "rewards/margins": -0.00396156357601285,
      "rewards/rejected": -0.012048340402543545,
      "step": 512
    },
    {
      "epoch": 0.13493563925101515,
      "grad_norm": 516.0,
      "learning_rate": 4.807017543859649e-07,
      "logits/chosen": -0.6431457996368408,
      "logits/rejected": -0.6404819488525391,
      "logps/chosen": -1084.869140625,
      "logps/rejected": -799.14990234375,
      "loss": 0.688,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.02252369001507759,
      "rewards/margins": 0.010861396789550781,
      "rewards/rejected": 0.011662292294204235,
      "step": 513
    },
    {
      "epoch": 0.1351986716862023,
      "grad_norm": 540.0,
      "learning_rate": 4.805555555555555e-07,
      "logits/chosen": -0.6702117919921875,
      "logits/rejected": NaN,
      "logps/chosen": -998.6484985351562,
      "logps/rejected": -767.8541259765625,
      "loss": 0.6898,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00481233699247241,
      "rewards/margins": 0.007064628414809704,
      "rewards/rejected": -0.002252293284982443,
      "step": 514
    },
    {
      "epoch": 0.13546170412138947,
      "grad_norm": 556.0,
      "learning_rate": 4.804093567251461e-07,
      "logits/chosen": -0.6512441635131836,
      "logits/rejected": -0.650234580039978,
      "logps/chosen": -1117.7078857421875,
      "logps/rejected": -878.1242065429688,
      "loss": 0.6816,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.03379325941205025,
      "rewards/margins": 0.02419443055987358,
      "rewards/rejected": 0.009598826989531517,
      "step": 515
    },
    {
      "epoch": 0.13572473655657663,
      "grad_norm": 404.0,
      "learning_rate": 4.802631578947368e-07,
      "logits/chosen": -0.6471911668777466,
      "logits/rejected": -0.6344019770622253,
      "logps/chosen": -621.2020874023438,
      "logps/rejected": -646.8888549804688,
      "loss": 0.6974,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.012826251797378063,
      "rewards/margins": -0.008065415546298027,
      "rewards/rejected": -0.004760838113725185,
      "step": 516
    },
    {
      "epoch": 0.1359877689917638,
      "grad_norm": 588.0,
      "learning_rate": 4.801169590643275e-07,
      "logits/chosen": -0.6078386306762695,
      "logits/rejected": -0.6192202568054199,
      "logps/chosen": -1068.2911376953125,
      "logps/rejected": -935.0166015625,
      "loss": 0.6935,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.017098044976592064,
      "rewards/margins": 4.739593714475632e-05,
      "rewards/rejected": 0.017050649970769882,
      "step": 517
    },
    {
      "epoch": 0.13625080142695095,
      "grad_norm": 628.0,
      "learning_rate": 4.799707602339181e-07,
      "logits/chosen": -0.6669657826423645,
      "logits/rejected": -0.6579009890556335,
      "logps/chosen": -1029.417236328125,
      "logps/rejected": -1010.452880859375,
      "loss": 0.6841,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.00032882695086300373,
      "rewards/margins": 0.019343184307217598,
      "rewards/rejected": -0.019014358520507812,
      "step": 518
    },
    {
      "epoch": 0.1365138338621381,
      "grad_norm": 2592.0,
      "learning_rate": 4.798245614035087e-07,
      "logits/chosen": -0.655178964138031,
      "logits/rejected": -0.6612006425857544,
      "logps/chosen": -958.66845703125,
      "logps/rejected": -800.6452026367188,
      "loss": 0.6854,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0022922507487237453,
      "rewards/margins": 0.016683101654052734,
      "rewards/rejected": -0.018975352868437767,
      "step": 519
    },
    {
      "epoch": 0.1367768662973253,
      "grad_norm": 524.0,
      "learning_rate": 4.796783625730994e-07,
      "logits/chosen": -0.6422751545906067,
      "logits/rejected": NaN,
      "logps/chosen": -1049.5809326171875,
      "logps/rejected": -816.2612915039062,
      "loss": 0.7023,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0166489128023386,
      "rewards/margins": -0.017706917598843575,
      "rewards/rejected": 0.001058005727827549,
      "step": 520
    },
    {
      "epoch": 0.13703989873251246,
      "grad_norm": 490.0,
      "learning_rate": 4.7953216374269e-07,
      "logits/chosen": -0.6500913500785828,
      "logits/rejected": -0.6544511318206787,
      "logps/chosen": -927.1670532226562,
      "logps/rejected": -782.174560546875,
      "loss": 0.697,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01775316894054413,
      "rewards/margins": -0.006945657078176737,
      "rewards/rejected": 0.024698829278349876,
      "step": 521
    },
    {
      "epoch": 0.13730293116769962,
      "grad_norm": 488.0,
      "learning_rate": 4.793859649122807e-07,
      "logits/chosen": -0.6461738348007202,
      "logits/rejected": -0.6428927183151245,
      "logps/chosen": -1122.0419921875,
      "logps/rejected": -848.4646606445312,
      "loss": 0.7014,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008051300421357155,
      "rewards/margins": -0.015896223485469818,
      "rewards/rejected": 0.007844923064112663,
      "step": 522
    },
    {
      "epoch": 0.13756596360288678,
      "grad_norm": 596.0,
      "learning_rate": 4.792397660818713e-07,
      "logits/chosen": -0.684341549873352,
      "logits/rejected": -0.685996413230896,
      "logps/chosen": -877.0186157226562,
      "logps/rejected": -915.4061279296875,
      "loss": 0.6953,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0018650058191269636,
      "rewards/margins": -0.0037704003043472767,
      "rewards/rejected": 0.0019053937867283821,
      "step": 523
    },
    {
      "epoch": 0.13782899603807394,
      "grad_norm": 564.0,
      "learning_rate": 4.79093567251462e-07,
      "logits/chosen": -0.6292560696601868,
      "logits/rejected": -0.6331432461738586,
      "logps/chosen": -984.267333984375,
      "logps/rejected": -849.928955078125,
      "loss": 0.6972,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0048833382315933704,
      "rewards/margins": -0.006951000075787306,
      "rewards/rejected": 0.0020676618441939354,
      "step": 524
    },
    {
      "epoch": 0.1380920284732611,
      "grad_norm": 458.0,
      "learning_rate": 4.789473684210526e-07,
      "logits/chosen": -0.6631412506103516,
      "logits/rejected": -0.6766126155853271,
      "logps/chosen": -808.2734375,
      "logps/rejected": -695.8335571289062,
      "loss": 0.7031,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01882624812424183,
      "rewards/margins": -0.019152499735355377,
      "rewards/rejected": 0.00032625114545226097,
      "step": 525
    },
    {
      "epoch": 0.13835506090844826,
      "grad_norm": 486.0,
      "learning_rate": 4.788011695906433e-07,
      "logits/chosen": -0.6542309522628784,
      "logits/rejected": -0.6591788530349731,
      "logps/chosen": -954.7708129882812,
      "logps/rejected": -875.6358642578125,
      "loss": 0.6766,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.017879582941532135,
      "rewards/margins": 0.0348973274230957,
      "rewards/rejected": -0.017017746344208717,
      "step": 526
    },
    {
      "epoch": 0.13861809334363542,
      "grad_norm": 508.0,
      "learning_rate": 4.786549707602339e-07,
      "logits/chosen": -0.6309075355529785,
      "logits/rejected": -0.6374235153198242,
      "logps/chosen": -1306.224609375,
      "logps/rejected": -840.7925415039062,
      "loss": 0.6873,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.020236587151885033,
      "rewards/margins": 0.0138098718598485,
      "rewards/rejected": 0.006426715757697821,
      "step": 527
    },
    {
      "epoch": 0.1388811257788226,
      "grad_norm": 676.0,
      "learning_rate": 4.785087719298245e-07,
      "logits/chosen": -0.6423572897911072,
      "logits/rejected": -0.644288182258606,
      "logps/chosen": -1342.372314453125,
      "logps/rejected": -924.3419189453125,
      "loss": 0.7083,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0011693965643644333,
      "rewards/margins": -0.0287915226072073,
      "rewards/rejected": 0.027622127905488014,
      "step": 528
    },
    {
      "epoch": 0.13914415821400977,
      "grad_norm": 620.0,
      "learning_rate": 4.783625730994152e-07,
      "logits/chosen": -0.6555459499359131,
      "logits/rejected": -0.6632274985313416,
      "logps/chosen": -1384.801513671875,
      "logps/rejected": -1464.75732421875,
      "loss": 0.7051,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.013442231342196465,
      "rewards/margins": -0.02294149249792099,
      "rewards/rejected": 0.036383725702762604,
      "step": 529
    },
    {
      "epoch": 0.13940719064919693,
      "grad_norm": 536.0,
      "learning_rate": 4.782163742690058e-07,
      "logits/chosen": -0.6626526117324829,
      "logits/rejected": NaN,
      "logps/chosen": -1333.065185546875,
      "logps/rejected": -867.9591064453125,
      "loss": 0.6919,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.018688105046749115,
      "rewards/margins": 0.0032207488548010588,
      "rewards/rejected": 0.015467358753085136,
      "step": 530
    },
    {
      "epoch": 0.1396702230843841,
      "grad_norm": 632.0,
      "learning_rate": 4.780701754385965e-07,
      "logits/chosen": -0.6286462545394897,
      "logits/rejected": NaN,
      "logps/chosen": -1260.0224609375,
      "logps/rejected": -1154.17578125,
      "loss": 0.6912,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.015214918181300163,
      "rewards/margins": 0.004807282239198685,
      "rewards/rejected": 0.010407639667391777,
      "step": 531
    },
    {
      "epoch": 0.13993325551957125,
      "grad_norm": 516.0,
      "learning_rate": 4.779239766081871e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6515749096870422,
      "logps/chosen": -715.6073608398438,
      "logps/rejected": -1058.9375,
      "loss": 0.6936,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003263091901317239,
      "rewards/margins": 0.00045094452798366547,
      "rewards/rejected": 0.002812147606164217,
      "step": 532
    },
    {
      "epoch": 0.14019628795475841,
      "grad_norm": 760.0,
      "learning_rate": 4.777777777777778e-07,
      "logits/chosen": -0.649348795413971,
      "logits/rejected": NaN,
      "logps/chosen": -1461.40478515625,
      "logps/rejected": -1107.4271240234375,
      "loss": 0.7058,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.008192826062440872,
      "rewards/margins": -0.024122335016727448,
      "rewards/rejected": 0.015929508954286575,
      "step": 533
    },
    {
      "epoch": 0.14045932038994557,
      "grad_norm": 604.0,
      "learning_rate": 4.776315789473684e-07,
      "logits/chosen": -0.6498348712921143,
      "logits/rejected": -0.6673755049705505,
      "logps/chosen": -1059.34130859375,
      "logps/rejected": -1049.4617919921875,
      "loss": 0.6959,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.002810955047607422,
      "rewards/margins": -0.004633045289665461,
      "rewards/rejected": 0.001822090707719326,
      "step": 534
    },
    {
      "epoch": 0.14072235282513276,
      "grad_norm": 496.0,
      "learning_rate": 4.77485380116959e-07,
      "logits/chosen": -0.6363334655761719,
      "logits/rejected": -0.6400913596153259,
      "logps/chosen": -950.833251953125,
      "logps/rejected": -723.5020751953125,
      "loss": 0.6882,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.019244767725467682,
      "rewards/margins": 0.010345982387661934,
      "rewards/rejected": 0.008898783475160599,
      "step": 535
    },
    {
      "epoch": 0.14098538526031992,
      "grad_norm": 556.0,
      "learning_rate": 4.773391812865497e-07,
      "logits/chosen": -0.6657078266143799,
      "logits/rejected": -0.6732887029647827,
      "logps/chosen": -1077.7119140625,
      "logps/rejected": -832.7321166992188,
      "loss": 0.6994,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0006950856186449528,
      "rewards/margins": -0.011647176928818226,
      "rewards/rejected": 0.010952090844511986,
      "step": 536
    },
    {
      "epoch": 0.14124841769550708,
      "grad_norm": 560.0,
      "learning_rate": 4.771929824561403e-07,
      "logits/chosen": -0.6514765024185181,
      "logits/rejected": -0.6639765501022339,
      "logps/chosen": -1229.75830078125,
      "logps/rejected": -815.6502075195312,
      "loss": 0.6904,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.012634370476007462,
      "rewards/margins": 0.006032132543623447,
      "rewards/rejected": 0.006602239795029163,
      "step": 537
    },
    {
      "epoch": 0.14151145013069424,
      "grad_norm": 438.0,
      "learning_rate": 4.77046783625731e-07,
      "logits/chosen": -0.6368830800056458,
      "logits/rejected": -0.6306840181350708,
      "logps/chosen": -732.5057983398438,
      "logps/rejected": -567.5560302734375,
      "loss": 0.6859,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.019089316949248314,
      "rewards/margins": 0.015027761459350586,
      "rewards/rejected": 0.004061556421220303,
      "step": 538
    },
    {
      "epoch": 0.1417744825658814,
      "grad_norm": 532.0,
      "learning_rate": 4.769005847953216e-07,
      "logits/chosen": -0.6470096111297607,
      "logits/rejected": -0.6509332060813904,
      "logps/chosen": -1108.208740234375,
      "logps/rejected": -967.0982666015625,
      "loss": 0.682,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00044822791824117303,
      "rewards/margins": 0.02402782440185547,
      "rewards/rejected": -0.02357959747314453,
      "step": 539
    },
    {
      "epoch": 0.14203751500106856,
      "grad_norm": 502.0,
      "learning_rate": 4.7675438596491225e-07,
      "logits/chosen": -0.6155234575271606,
      "logits/rejected": -0.6162593960762024,
      "logps/chosen": -997.8795166015625,
      "logps/rejected": -751.7327880859375,
      "loss": 0.6976,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.007163905538618565,
      "rewards/margins": -0.007981490343809128,
      "rewards/rejected": 0.0008175848051905632,
      "step": 540
    },
    {
      "epoch": 0.14230054743625573,
      "grad_norm": 560.0,
      "learning_rate": 4.766081871345029e-07,
      "logits/chosen": -0.6745294332504272,
      "logits/rejected": -0.6755213141441345,
      "logps/chosen": -961.755126953125,
      "logps/rejected": -897.1589965820312,
      "loss": 0.6976,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0007805358618497849,
      "rewards/margins": -0.008357001468539238,
      "rewards/rejected": 0.007576465606689453,
      "step": 541
    },
    {
      "epoch": 0.14256357987144289,
      "grad_norm": 486.0,
      "learning_rate": 4.7646198830409354e-07,
      "logits/chosen": -0.6293255090713501,
      "logits/rejected": -0.6258423328399658,
      "logps/chosen": -902.443603515625,
      "logps/rejected": -779.45654296875,
      "loss": 0.6984,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.004289102274924517,
      "rewards/margins": -0.010097838006913662,
      "rewards/rejected": 0.005808735266327858,
      "step": 542
    },
    {
      "epoch": 0.14282661230663007,
      "grad_norm": 588.0,
      "learning_rate": 4.7631578947368416e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6698393225669861,
      "logps/chosen": -1033.1466064453125,
      "logps/rejected": -1204.5506591796875,
      "loss": 0.6912,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0038934708572924137,
      "rewards/margins": 0.004987715743482113,
      "rewards/rejected": -0.0010942455846816301,
      "step": 543
    },
    {
      "epoch": 0.14308964474181723,
      "grad_norm": 584.0,
      "learning_rate": 4.7616959064327483e-07,
      "logits/chosen": -0.6545750498771667,
      "logits/rejected": NaN,
      "logps/chosen": -1179.218505859375,
      "logps/rejected": -945.1787109375,
      "loss": 0.6858,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.031015396118164062,
      "rewards/margins": 0.015330218710005283,
      "rewards/rejected": -0.04634561762213707,
      "step": 544
    },
    {
      "epoch": 0.1433526771770044,
      "grad_norm": 452.0,
      "learning_rate": 4.760233918128655e-07,
      "logits/chosen": -0.6556026339530945,
      "logits/rejected": -0.6543760299682617,
      "logps/chosen": -984.29541015625,
      "logps/rejected": -754.7111206054688,
      "loss": 0.6865,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.006456135306507349,
      "rewards/margins": 0.013967849314212799,
      "rewards/rejected": -0.02042398601770401,
      "step": 545
    },
    {
      "epoch": 0.14361570961219156,
      "grad_norm": 592.0,
      "learning_rate": 4.758771929824561e-07,
      "logits/chosen": -0.657542884349823,
      "logits/rejected": -0.667889416217804,
      "logps/chosen": -1408.24951171875,
      "logps/rejected": -1111.666259765625,
      "loss": 0.6988,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0028821928426623344,
      "rewards/margins": -0.010772132314741611,
      "rewards/rejected": 0.007889938540756702,
      "step": 546
    },
    {
      "epoch": 0.14387874204737872,
      "grad_norm": 528.0,
      "learning_rate": 4.7573099415204675e-07,
      "logits/chosen": -0.6796157956123352,
      "logits/rejected": -0.6849287748336792,
      "logps/chosen": -1119.3182373046875,
      "logps/rejected": -752.9148559570312,
      "loss": 0.6937,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.016573525965213776,
      "rewards/margins": -0.0003426545299589634,
      "rewards/rejected": 0.01691618002951145,
      "step": 547
    },
    {
      "epoch": 0.14414177448256588,
      "grad_norm": 436.0,
      "learning_rate": 4.755847953216374e-07,
      "logits/chosen": -0.6693822741508484,
      "logits/rejected": -0.6594333648681641,
      "logps/chosen": -795.62939453125,
      "logps/rejected": -948.8435668945312,
      "loss": 0.7024,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.015607594512403011,
      "rewards/margins": -0.01784949190914631,
      "rewards/rejected": 0.002241897163912654,
      "step": 548
    },
    {
      "epoch": 0.14440480691775304,
      "grad_norm": 604.0,
      "learning_rate": 4.7543859649122804e-07,
      "logits/chosen": -0.6597892045974731,
      "logits/rejected": -0.6558645963668823,
      "logps/chosen": -1196.97021484375,
      "logps/rejected": -986.2417602539062,
      "loss": 0.6871,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0157668124884367,
      "rewards/margins": 0.01330108568072319,
      "rewards/rejected": 0.0024657249450683594,
      "step": 549
    },
    {
      "epoch": 0.1446678393529402,
      "grad_norm": 580.0,
      "learning_rate": 4.752923976608187e-07,
      "logits/chosen": -0.6637647151947021,
      "logits/rejected": NaN,
      "logps/chosen": -1431.882568359375,
      "logps/rejected": -897.135498046875,
      "loss": 0.6949,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.004445934668183327,
      "rewards/margins": -0.0030455589294433594,
      "rewards/rejected": -0.0014003748074173927,
      "step": 550
    },
    {
      "epoch": 0.14493087178812739,
      "grad_norm": 464.0,
      "learning_rate": 4.7514619883040933e-07,
      "logits/chosen": -0.6397905349731445,
      "logits/rejected": -0.6490638852119446,
      "logps/chosen": -943.5575561523438,
      "logps/rejected": -879.4398193359375,
      "loss": 0.6913,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.014638328924775124,
      "rewards/margins": 0.0045423489063978195,
      "rewards/rejected": 0.010095978155732155,
      "step": 551
    },
    {
      "epoch": 0.14519390422331455,
      "grad_norm": 460.0,
      "learning_rate": 4.7499999999999995e-07,
      "logits/chosen": -0.6447945237159729,
      "logits/rejected": NaN,
      "logps/chosen": -1189.177490234375,
      "logps/rejected": -1068.3358154296875,
      "loss": 0.6974,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0075168609619140625,
      "rewards/margins": -0.007968425750732422,
      "rewards/rejected": 0.0004515652544796467,
      "step": 552
    },
    {
      "epoch": 0.1454569366585017,
      "grad_norm": 528.0,
      "learning_rate": 4.748538011695906e-07,
      "logits/chosen": -0.6389585733413696,
      "logits/rejected": -0.6405737996101379,
      "logps/chosen": -1217.77490234375,
      "logps/rejected": -885.9113159179688,
      "loss": 0.6873,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.006237124092876911,
      "rewards/margins": 0.012143516913056374,
      "rewards/rejected": -0.0059063914231956005,
      "step": 553
    },
    {
      "epoch": 0.14571996909368887,
      "grad_norm": 548.0,
      "learning_rate": 4.747076023391813e-07,
      "logits/chosen": -0.636116087436676,
      "logits/rejected": -0.6316486597061157,
      "logps/chosen": -1170.5423583984375,
      "logps/rejected": -928.7910766601562,
      "loss": 0.697,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.004344940185546875,
      "rewards/margins": -0.006950569339096546,
      "rewards/rejected": 0.002605628687888384,
      "step": 554
    },
    {
      "epoch": 0.14598300152887603,
      "grad_norm": 516.0,
      "learning_rate": 4.7456140350877186e-07,
      "logits/chosen": -0.6373350024223328,
      "logits/rejected": -0.6378212571144104,
      "logps/chosen": -1024.843505859375,
      "logps/rejected": -660.0047607421875,
      "loss": 0.6734,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.017117025330662727,
      "rewards/margins": 0.041603900492191315,
      "rewards/rejected": -0.024486875161528587,
      "step": 555
    },
    {
      "epoch": 0.1462460339640632,
      "grad_norm": 552.0,
      "learning_rate": 4.7441520467836254e-07,
      "logits/chosen": -0.6584852933883667,
      "logits/rejected": -0.6666289567947388,
      "logps/chosen": -1097.46875,
      "logps/rejected": -967.7063598632812,
      "loss": 0.6911,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01124506164342165,
      "rewards/margins": 0.004377365577965975,
      "rewards/rejected": -0.015622426755726337,
      "step": 556
    },
    {
      "epoch": 0.14650906639925035,
      "grad_norm": 442.0,
      "learning_rate": 4.742690058479532e-07,
      "logits/chosen": -0.6647803783416748,
      "logits/rejected": -0.6673501133918762,
      "logps/chosen": -1037.9697265625,
      "logps/rejected": -688.4361572265625,
      "loss": 0.6955,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.009186267852783203,
      "rewards/margins": -0.004523181356489658,
      "rewards/rejected": -0.00466308556497097,
      "step": 557
    },
    {
      "epoch": 0.1467720988344375,
      "grad_norm": 540.0,
      "learning_rate": 4.7412280701754383e-07,
      "logits/chosen": -0.6360700130462646,
      "logits/rejected": -0.6503647565841675,
      "logps/chosen": -1286.628173828125,
      "logps/rejected": -676.6072387695312,
      "loss": 0.6913,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009073732420802116,
      "rewards/margins": 0.0041166311129927635,
      "rewards/rejected": 0.004957105498760939,
      "step": 558
    },
    {
      "epoch": 0.1470351312696247,
      "grad_norm": 628.0,
      "learning_rate": 4.7397660818713445e-07,
      "logits/chosen": -0.6680763959884644,
      "logits/rejected": -0.6663932204246521,
      "logps/chosen": -1205.354736328125,
      "logps/rejected": -998.494873046875,
      "loss": 0.7018,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.009743976406753063,
      "rewards/margins": -0.015917014330625534,
      "rewards/rejected": 0.006173037923872471,
      "step": 559
    },
    {
      "epoch": 0.14729816370481186,
      "grad_norm": 568.0,
      "learning_rate": 4.738304093567251e-07,
      "logits/chosen": -0.664301872253418,
      "logits/rejected": NaN,
      "logps/chosen": -1189.426025390625,
      "logps/rejected": -902.5806884765625,
      "loss": 0.6975,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006959343329071999,
      "rewards/margins": -0.007527255918830633,
      "rewards/rejected": 0.014486599713563919,
      "step": 560
    },
    {
      "epoch": 0.14756119613999902,
      "grad_norm": 600.0,
      "learning_rate": 4.7368421052631574e-07,
      "logits/chosen": -0.6621152758598328,
      "logits/rejected": NaN,
      "logps/chosen": -1172.7376708984375,
      "logps/rejected": -1044.28759765625,
      "loss": 0.6837,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.023128606379032135,
      "rewards/margins": 0.0203157439827919,
      "rewards/rejected": 0.0028128623962402344,
      "step": 561
    },
    {
      "epoch": 0.14782422857518618,
      "grad_norm": 644.0,
      "learning_rate": 4.735380116959064e-07,
      "logits/chosen": -0.6508535146713257,
      "logits/rejected": -0.6361100673675537,
      "logps/chosen": -1328.62890625,
      "logps/rejected": -1285.684814453125,
      "loss": 0.702,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010887861251831055,
      "rewards/margins": -0.016469242051243782,
      "rewards/rejected": 0.027357101440429688,
      "step": 562
    },
    {
      "epoch": 0.14808726101037334,
      "grad_norm": 572.0,
      "learning_rate": 4.733918128654971e-07,
      "logits/chosen": -0.6670821905136108,
      "logits/rejected": -0.6673285961151123,
      "logps/chosen": -895.0045166015625,
      "logps/rejected": -1032.4765625,
      "loss": 0.7042,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01159811019897461,
      "rewards/margins": -0.021222783252596855,
      "rewards/rejected": 0.009624672122299671,
      "step": 563
    },
    {
      "epoch": 0.1483502934455605,
      "grad_norm": 564.0,
      "learning_rate": 4.7324561403508766e-07,
      "logits/chosen": -0.6390162110328674,
      "logits/rejected": -0.635673463344574,
      "logps/chosen": -1011.8121948242188,
      "logps/rejected": -969.6964111328125,
      "loss": 0.6953,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0057098399847745895,
      "rewards/margins": -0.0034353258088231087,
      "rewards/rejected": 0.009145164862275124,
      "step": 564
    },
    {
      "epoch": 0.14861332588074766,
      "grad_norm": 422.0,
      "learning_rate": 4.7309941520467833e-07,
      "logits/chosen": -0.652207612991333,
      "logits/rejected": -0.6727631092071533,
      "logps/chosen": -933.9305419921875,
      "logps/rejected": -680.3701171875,
      "loss": 0.6834,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02395191229879856,
      "rewards/margins": 0.02019472047686577,
      "rewards/rejected": 0.0037571906577795744,
      "step": 565
    },
    {
      "epoch": 0.14887635831593482,
      "grad_norm": 596.0,
      "learning_rate": 4.72953216374269e-07,
      "logits/chosen": -0.6496362090110779,
      "logits/rejected": -0.651808500289917,
      "logps/chosen": -987.5576782226562,
      "logps/rejected": -840.6793823242188,
      "loss": 0.6934,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.007026957347989082,
      "rewards/margins": 0.0002282140776515007,
      "rewards/rejected": 0.006798745132982731,
      "step": 566
    },
    {
      "epoch": 0.149139390751122,
      "grad_norm": 552.0,
      "learning_rate": 4.728070175438597e-07,
      "logits/chosen": -0.6632911562919617,
      "logits/rejected": -0.6766233444213867,
      "logps/chosen": -1534.4166259765625,
      "logps/rejected": -1074.1905517578125,
      "loss": 0.6987,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0036736486945301294,
      "rewards/margins": -0.01013708021491766,
      "rewards/rejected": 0.006463432684540749,
      "step": 567
    },
    {
      "epoch": 0.14940242318630917,
      "grad_norm": 444.0,
      "learning_rate": 4.7266081871345024e-07,
      "logits/chosen": -0.6814846396446228,
      "logits/rejected": NaN,
      "logps/chosen": -1400.7298583984375,
      "logps/rejected": -1070.0599365234375,
      "loss": 0.6838,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01795320399105549,
      "rewards/margins": 0.01939544826745987,
      "rewards/rejected": -0.0014422410167753696,
      "step": 568
    },
    {
      "epoch": 0.14966545562149633,
      "grad_norm": 520.0,
      "learning_rate": 4.725146198830409e-07,
      "logits/chosen": -0.6589382886886597,
      "logits/rejected": -0.6568370461463928,
      "logps/chosen": -1337.086181640625,
      "logps/rejected": -1100.120849609375,
      "loss": 0.6852,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012731838040053844,
      "rewards/margins": 0.016766168177127838,
      "rewards/rejected": -0.004034329205751419,
      "step": 569
    },
    {
      "epoch": 0.1499284880566835,
      "grad_norm": 492.0,
      "learning_rate": 4.723684210526316e-07,
      "logits/chosen": -0.6468349099159241,
      "logits/rejected": NaN,
      "logps/chosen": -997.56396484375,
      "logps/rejected": -719.9695434570312,
      "loss": 0.6956,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.011832332238554955,
      "rewards/margins": -0.00399866234511137,
      "rewards/rejected": 0.01583099365234375,
      "step": 570
    },
    {
      "epoch": 0.15019152049187065,
      "grad_norm": 474.0,
      "learning_rate": 4.722222222222222e-07,
      "logits/chosen": -0.6511151790618896,
      "logits/rejected": -0.6562613248825073,
      "logps/chosen": -1226.0401611328125,
      "logps/rejected": -962.8599853515625,
      "loss": 0.6902,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.004833889193832874,
      "rewards/margins": 0.006450795102864504,
      "rewards/rejected": -0.01128468383103609,
      "step": 571
    },
    {
      "epoch": 0.1504545529270578,
      "grad_norm": 528.0,
      "learning_rate": 4.7207602339181283e-07,
      "logits/chosen": -0.6181662678718567,
      "logits/rejected": -0.6204691529273987,
      "logps/chosen": -1171.846435546875,
      "logps/rejected": -924.55517578125,
      "loss": 0.6976,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.020044326782226562,
      "rewards/margins": -0.008498812094330788,
      "rewards/rejected": -0.01154551561921835,
      "step": 572
    },
    {
      "epoch": 0.15071758536224497,
      "grad_norm": 454.0,
      "learning_rate": 4.719298245614035e-07,
      "logits/chosen": -0.6323500871658325,
      "logits/rejected": -0.6556678414344788,
      "logps/chosen": -994.1142578125,
      "logps/rejected": -808.7297973632812,
      "loss": 0.6947,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.000615311786532402,
      "rewards/margins": -0.002795505104586482,
      "rewards/rejected": 0.003410816891118884,
      "step": 573
    },
    {
      "epoch": 0.15098061779743216,
      "grad_norm": 516.0,
      "learning_rate": 4.717836257309941e-07,
      "logits/chosen": -0.65614253282547,
      "logits/rejected": -0.6562457084655762,
      "logps/chosen": -1140.3682861328125,
      "logps/rejected": -1036.6968994140625,
      "loss": 0.7063,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.005929184146225452,
      "rewards/margins": -0.025693418458104134,
      "rewards/rejected": 0.019764233380556107,
      "step": 574
    },
    {
      "epoch": 0.15124365023261932,
      "grad_norm": 580.0,
      "learning_rate": 4.716374269005848e-07,
      "logits/chosen": -0.657097339630127,
      "logits/rejected": -0.655438244342804,
      "logps/chosen": -1206.1781005859375,
      "logps/rejected": -848.6461181640625,
      "loss": 0.691,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.015420245006680489,
      "rewards/margins": 0.004963206127285957,
      "rewards/rejected": 0.010457038879394531,
      "step": 575
    },
    {
      "epoch": 0.15150668266780648,
      "grad_norm": 478.0,
      "learning_rate": 4.7149122807017547e-07,
      "logits/chosen": -0.6526088118553162,
      "logits/rejected": -0.6472732424736023,
      "logps/chosen": -1021.0940551757812,
      "logps/rejected": -778.2271728515625,
      "loss": 0.698,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.007116653490811586,
      "rewards/margins": -0.008984232321381569,
      "rewards/rejected": 0.0018675797618925571,
      "step": 576
    },
    {
      "epoch": 0.15176971510299364,
      "grad_norm": 508.0,
      "learning_rate": 4.7134502923976603e-07,
      "logits/chosen": -0.6555579900741577,
      "logits/rejected": -0.6575942635536194,
      "logps/chosen": -1251.1600341796875,
      "logps/rejected": -1204.4788818359375,
      "loss": 0.6967,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00695571955293417,
      "rewards/margins": -0.006651591509580612,
      "rewards/rejected": 0.013607311993837357,
      "step": 577
    },
    {
      "epoch": 0.1520327475381808,
      "grad_norm": 568.0,
      "learning_rate": 4.711988304093567e-07,
      "logits/chosen": -0.6406919360160828,
      "logits/rejected": -0.64703369140625,
      "logps/chosen": -1035.18310546875,
      "logps/rejected": -990.2181396484375,
      "loss": 0.6936,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.009360980242490768,
      "rewards/margins": -0.00022211088798940182,
      "rewards/rejected": -0.009138870052993298,
      "step": 578
    },
    {
      "epoch": 0.15229577997336796,
      "grad_norm": 536.0,
      "learning_rate": 4.710526315789474e-07,
      "logits/chosen": -0.6684260368347168,
      "logits/rejected": -0.6620067358016968,
      "logps/chosen": -982.3534545898438,
      "logps/rejected": -732.427001953125,
      "loss": 0.692,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.004316044505685568,
      "rewards/margins": 0.003099536057561636,
      "rewards/rejected": -0.0074155814945697784,
      "step": 579
    },
    {
      "epoch": 0.15255881240855512,
      "grad_norm": 466.0,
      "learning_rate": 4.70906432748538e-07,
      "logits/chosen": -0.6727544069290161,
      "logits/rejected": -0.6767103672027588,
      "logps/chosen": -1069.6021728515625,
      "logps/rejected": -885.20361328125,
      "loss": 0.6937,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00821695476770401,
      "rewards/margins": -0.0004134178161621094,
      "rewards/rejected": -0.007803535554558039,
      "step": 580
    },
    {
      "epoch": 0.15282184484374228,
      "grad_norm": 580.0,
      "learning_rate": 4.707602339181286e-07,
      "logits/chosen": -0.6747865080833435,
      "logits/rejected": -0.6875367164611816,
      "logps/chosen": -1455.50244140625,
      "logps/rejected": -1243.8857421875,
      "loss": 0.6948,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01625671610236168,
      "rewards/margins": -0.002950097434222698,
      "rewards/rejected": -0.013306617736816406,
      "step": 581
    },
    {
      "epoch": 0.15308487727892947,
      "grad_norm": 568.0,
      "learning_rate": 4.706140350877193e-07,
      "logits/chosen": -0.6402923464775085,
      "logits/rejected": -0.6507517099380493,
      "logps/chosen": -1059.0849609375,
      "logps/rejected": -1159.94775390625,
      "loss": 0.6824,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0024478912819176912,
      "rewards/margins": 0.021981239318847656,
      "rewards/rejected": -0.019533349201083183,
      "step": 582
    },
    {
      "epoch": 0.15334790971411663,
      "grad_norm": 644.0,
      "learning_rate": 4.704678362573099e-07,
      "logits/chosen": -0.6439639329910278,
      "logits/rejected": -0.6461595892906189,
      "logps/chosen": -1305.0106201171875,
      "logps/rejected": -1252.6900634765625,
      "loss": 0.6798,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.018677901476621628,
      "rewards/margins": 0.027949048206210136,
      "rewards/rejected": -0.009271145798265934,
      "step": 583
    },
    {
      "epoch": 0.1536109421493038,
      "grad_norm": 612.0,
      "learning_rate": 4.703216374269006e-07,
      "logits/chosen": -0.6471026539802551,
      "logits/rejected": -0.6542971730232239,
      "logps/chosen": -1540.3778076171875,
      "logps/rejected": -1118.171875,
      "loss": 0.7049,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.02843198925256729,
      "rewards/margins": -0.022427942603826523,
      "rewards/rejected": -0.00600404804572463,
      "step": 584
    },
    {
      "epoch": 0.15387397458449095,
      "grad_norm": 418.0,
      "learning_rate": 4.701754385964912e-07,
      "logits/chosen": -0.6527692079544067,
      "logits/rejected": -0.6523643732070923,
      "logps/chosen": -670.271728515625,
      "logps/rejected": -569.227294921875,
      "loss": 0.6876,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009289837442338467,
      "rewards/margins": 0.01164550706744194,
      "rewards/rejected": -0.02093534544110298,
      "step": 585
    },
    {
      "epoch": 0.1541370070196781,
      "grad_norm": 418.0,
      "learning_rate": 4.700292397660818e-07,
      "logits/chosen": -0.6513493061065674,
      "logits/rejected": -0.6553128957748413,
      "logps/chosen": -845.6691284179688,
      "logps/rejected": -739.1240844726562,
      "loss": 0.6938,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0009282107930630445,
      "rewards/margins": -0.00066719064489007,
      "rewards/rejected": 0.0015954016707837582,
      "step": 586
    },
    {
      "epoch": 0.15440003945486527,
      "grad_norm": 548.0,
      "learning_rate": 4.698830409356725e-07,
      "logits/chosen": -0.6587492823600769,
      "logits/rejected": NaN,
      "logps/chosen": -1011.8173828125,
      "logps/rejected": -921.4862670898438,
      "loss": 0.6886,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0013225566362962127,
      "rewards/margins": 0.009807491675019264,
      "rewards/rejected": -0.011130047030746937,
      "step": 587
    },
    {
      "epoch": 0.15466307189005243,
      "grad_norm": 466.0,
      "learning_rate": 4.6973684210526317e-07,
      "logits/chosen": -0.6641072630882263,
      "logits/rejected": -0.6657620072364807,
      "logps/chosen": -961.7951049804688,
      "logps/rejected": -858.6510620117188,
      "loss": 0.6897,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0016757971607148647,
      "rewards/margins": 0.007203388959169388,
      "rewards/rejected": -0.00552759226411581,
      "step": 588
    },
    {
      "epoch": 0.1549261043252396,
      "grad_norm": 576.0,
      "learning_rate": 4.6959064327485374e-07,
      "logits/chosen": -0.6491438746452332,
      "logits/rejected": -0.6547327041625977,
      "logps/chosen": -1439.5765380859375,
      "logps/rejected": -1068.7139892578125,
      "loss": 0.6936,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.010054206475615501,
      "rewards/margins": 0.0010286332108080387,
      "rewards/rejected": -0.011082840152084827,
      "step": 589
    },
    {
      "epoch": 0.15518913676042678,
      "grad_norm": 604.0,
      "learning_rate": 4.694444444444444e-07,
      "logits/chosen": -0.6324167251586914,
      "logits/rejected": -0.6361513733863831,
      "logps/chosen": -1290.504638671875,
      "logps/rejected": -1021.6541748046875,
      "loss": 0.6913,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.005943965632468462,
      "rewards/margins": 0.005778503604233265,
      "rewards/rejected": -0.011722469702363014,
      "step": 590
    },
    {
      "epoch": 0.15545216919561394,
      "grad_norm": 494.0,
      "learning_rate": 4.692982456140351e-07,
      "logits/chosen": -0.6544032692909241,
      "logits/rejected": -0.6492961645126343,
      "logps/chosen": -902.549072265625,
      "logps/rejected": -843.0955200195312,
      "loss": 0.6931,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.023426149040460587,
      "rewards/margins": 0.0006309514865279198,
      "rewards/rejected": 0.02279520034790039,
      "step": 591
    },
    {
      "epoch": 0.1557152016308011,
      "grad_norm": 406.0,
      "learning_rate": 4.691520467836257e-07,
      "logits/chosen": -0.6450624465942383,
      "logits/rejected": -0.6464312076568604,
      "logps/chosen": -563.7039184570312,
      "logps/rejected": -620.029296875,
      "loss": 0.6972,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.002726936247199774,
      "rewards/margins": -0.007549429312348366,
      "rewards/rejected": 0.010276365093886852,
      "step": 592
    },
    {
      "epoch": 0.15597823406598826,
      "grad_norm": 572.0,
      "learning_rate": 4.690058479532164e-07,
      "logits/chosen": -0.6592616438865662,
      "logits/rejected": -0.671554446220398,
      "logps/chosen": -1253.2796630859375,
      "logps/rejected": -1082.9796142578125,
      "loss": 0.692,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01432356983423233,
      "rewards/margins": 0.003393937135115266,
      "rewards/rejected": 0.010929632931947708,
      "step": 593
    },
    {
      "epoch": 0.15624126650117542,
      "grad_norm": 608.0,
      "learning_rate": 4.68859649122807e-07,
      "logits/chosen": -0.6540321111679077,
      "logits/rejected": -0.6515912413597107,
      "logps/chosen": -1497.9337158203125,
      "logps/rejected": -984.5587768554688,
      "loss": 0.716,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.03819427266716957,
      "rewards/margins": -0.04340524598956108,
      "rewards/rejected": 0.005210971925407648,
      "step": 594
    },
    {
      "epoch": 0.15650429893636258,
      "grad_norm": 450.0,
      "learning_rate": 4.687134502923976e-07,
      "logits/chosen": -0.6414198279380798,
      "logits/rejected": -0.6439805030822754,
      "logps/chosen": -862.289306640625,
      "logps/rejected": -814.0860595703125,
      "loss": 0.683,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01594839245080948,
      "rewards/margins": 0.02102356031537056,
      "rewards/rejected": -0.005075169261544943,
      "step": 595
    },
    {
      "epoch": 0.15676733137154975,
      "grad_norm": 696.0,
      "learning_rate": 4.685672514619883e-07,
      "logits/chosen": -0.6494045257568359,
      "logits/rejected": -0.6632829308509827,
      "logps/chosen": -1444.204833984375,
      "logps/rejected": -637.3975830078125,
      "loss": 0.6982,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.003885936923325062,
      "rewards/margins": -0.00895996019244194,
      "rewards/rejected": 0.012845898047089577,
      "step": 596
    },
    {
      "epoch": 0.1570303638067369,
      "grad_norm": 404.0,
      "learning_rate": 4.6842105263157896e-07,
      "logits/chosen": -0.640990674495697,
      "logits/rejected": -0.6457639932632446,
      "logps/chosen": -737.6881713867188,
      "logps/rejected": -757.93701171875,
      "loss": 0.7031,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.001244163140654564,
      "rewards/margins": -0.01954956166446209,
      "rewards/rejected": 0.018305396661162376,
      "step": 597
    },
    {
      "epoch": 0.1572933962419241,
      "grad_norm": 800.0,
      "learning_rate": 4.6827485380116953e-07,
      "logits/chosen": -0.6411548852920532,
      "logits/rejected": NaN,
      "logps/chosen": -1419.5577392578125,
      "logps/rejected": -995.4225463867188,
      "loss": 0.7045,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.012004803866147995,
      "rewards/margins": -0.021561861038208008,
      "rewards/rejected": 0.009557055309414864,
      "step": 598
    },
    {
      "epoch": 0.15755642867711125,
      "grad_norm": 480.0,
      "learning_rate": 4.681286549707602e-07,
      "logits/chosen": -0.6084703207015991,
      "logits/rejected": -0.6063169836997986,
      "logps/chosen": -942.1055297851562,
      "logps/rejected": -836.2926025390625,
      "loss": 0.696,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.00835266150534153,
      "rewards/margins": -0.005267953500151634,
      "rewards/rejected": -0.0030847089365124702,
      "step": 599
    },
    {
      "epoch": 0.15781946111229841,
      "grad_norm": 448.0,
      "learning_rate": 4.6798245614035087e-07,
      "logits/chosen": -0.6491280198097229,
      "logits/rejected": -0.6531782150268555,
      "logps/chosen": -879.969482421875,
      "logps/rejected": -589.9409790039062,
      "loss": 0.7023,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.022472098469734192,
      "rewards/margins": -0.017426539212465286,
      "rewards/rejected": -0.005045556928962469,
      "step": 600
    },
    {
      "epoch": 0.15808249354748558,
      "grad_norm": 492.0,
      "learning_rate": 4.678362573099415e-07,
      "logits/chosen": -0.6128121614456177,
      "logits/rejected": -0.6179262399673462,
      "logps/chosen": -932.4656982421875,
      "logps/rejected": -595.1422729492188,
      "loss": 0.6923,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.002834511687979102,
      "rewards/margins": 0.0025705343578010798,
      "rewards/rejected": 0.00026397709734737873,
      "step": 601
    },
    {
      "epoch": 0.15834552598267274,
      "grad_norm": 404.0,
      "learning_rate": 4.676900584795321e-07,
      "logits/chosen": -0.6552446484565735,
      "logits/rejected": -0.6523796319961548,
      "logps/chosen": -1078.81494140625,
      "logps/rejected": -916.8479614257812,
      "loss": 0.6894,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.004864407703280449,
      "rewards/margins": 0.008066559210419655,
      "rewards/rejected": -0.012930966913700104,
      "step": 602
    },
    {
      "epoch": 0.1586085584178599,
      "grad_norm": 532.0,
      "learning_rate": 4.675438596491228e-07,
      "logits/chosen": -0.6900719404220581,
      "logits/rejected": -0.6926895976066589,
      "logps/chosen": -1747.53515625,
      "logps/rejected": -1248.38720703125,
      "loss": 0.6994,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.022893141955137253,
      "rewards/margins": -0.011319349519908428,
      "rewards/rejected": 0.034212492406368256,
      "step": 603
    },
    {
      "epoch": 0.15887159085304706,
      "grad_norm": 478.0,
      "learning_rate": 4.673976608187134e-07,
      "logits/chosen": -0.6581219434738159,
      "logits/rejected": -0.6553754210472107,
      "logps/chosen": -1064.0035400390625,
      "logps/rejected": -996.272705078125,
      "loss": 0.7004,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.006839943118393421,
      "rewards/margins": -0.013355635106563568,
      "rewards/rejected": 0.020195579156279564,
      "step": 604
    },
    {
      "epoch": 0.15913462328823422,
      "grad_norm": 568.0,
      "learning_rate": 4.672514619883041e-07,
      "logits/chosen": -0.6439696550369263,
      "logits/rejected": -0.6433199644088745,
      "logps/chosen": -1211.4022216796875,
      "logps/rejected": -945.382080078125,
      "loss": 0.7105,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.016016483306884766,
      "rewards/margins": -0.033408164978027344,
      "rewards/rejected": 0.017391681671142578,
      "step": 605
    },
    {
      "epoch": 0.1593976557234214,
      "grad_norm": 540.0,
      "learning_rate": 4.6710526315789475e-07,
      "logits/chosen": -0.6391346454620361,
      "logits/rejected": NaN,
      "logps/chosen": -1208.23095703125,
      "logps/rejected": -992.7986450195312,
      "loss": 0.697,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0031981943175196648,
      "rewards/margins": -0.006733560934662819,
      "rewards/rejected": 0.009931755252182484,
      "step": 606
    },
    {
      "epoch": 0.15966068815860857,
      "grad_norm": 492.0,
      "learning_rate": 4.6695906432748537e-07,
      "logits/chosen": -0.6544333696365356,
      "logits/rejected": -0.6612458825111389,
      "logps/chosen": -1351.373046875,
      "logps/rejected": -972.63720703125,
      "loss": 0.6989,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.012973785400390625,
      "rewards/margins": -0.01114797592163086,
      "rewards/rejected": -0.0018258094787597656,
      "step": 607
    },
    {
      "epoch": 0.15992372059379573,
      "grad_norm": 568.0,
      "learning_rate": 4.66812865497076e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6558018922805786,
      "logps/chosen": -988.6917724609375,
      "logps/rejected": -1108.125244140625,
      "loss": 0.688,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.007395648863166571,
      "rewards/margins": 0.01099080964922905,
      "rewards/rejected": -0.0035951617173850536,
      "step": 608
    },
    {
      "epoch": 0.1601867530289829,
      "grad_norm": 512.0,
      "learning_rate": 4.6666666666666666e-07,
      "logits/chosen": -0.6430438160896301,
      "logits/rejected": -0.6400179266929626,
      "logps/chosen": -1016.9786987304688,
      "logps/rejected": -949.7998657226562,
      "loss": 0.6829,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.03193988651037216,
      "rewards/margins": 0.02156829833984375,
      "rewards/rejected": 0.010371590033173561,
      "step": 609
    },
    {
      "epoch": 0.16044978546417005,
      "grad_norm": 532.0,
      "learning_rate": 4.6652046783625734e-07,
      "logits/chosen": -0.6418235301971436,
      "logits/rejected": -0.6619388461112976,
      "logps/chosen": -930.57568359375,
      "logps/rejected": -803.1010131835938,
      "loss": 0.6943,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.015452288091182709,
      "rewards/margins": -0.0018200874328613281,
      "rewards/rejected": 0.017272377386689186,
      "step": 610
    },
    {
      "epoch": 0.1607128178993572,
      "grad_norm": 426.0,
      "learning_rate": 4.663742690058479e-07,
      "logits/chosen": -0.65666663646698,
      "logits/rejected": -0.6576167345046997,
      "logps/chosen": -758.9026489257812,
      "logps/rejected": -692.592041015625,
      "loss": 0.6765,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.015479277819395065,
      "rewards/margins": 0.03446197509765625,
      "rewards/rejected": -0.018982697278261185,
      "step": 611
    },
    {
      "epoch": 0.16097585033454437,
      "grad_norm": 472.0,
      "learning_rate": 4.662280701754386e-07,
      "logits/chosen": -0.6591235995292664,
      "logits/rejected": -0.6547662615776062,
      "logps/chosen": -1015.907470703125,
      "logps/rejected": -819.3258056640625,
      "loss": 0.6855,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008265495300292969,
      "rewards/margins": 0.01663227006793022,
      "rewards/rejected": -0.008366775698959827,
      "step": 612
    },
    {
      "epoch": 0.16123888276973156,
      "grad_norm": 640.0,
      "learning_rate": 4.6608187134502925e-07,
      "logits/chosen": -0.6616694331169128,
      "logits/rejected": -0.6642777323722839,
      "logps/chosen": -947.7955322265625,
      "logps/rejected": -1035.9886474609375,
      "loss": 0.6842,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.012627124786376953,
      "rewards/margins": 0.020005226135253906,
      "rewards/rejected": -0.03263235092163086,
      "step": 613
    },
    {
      "epoch": 0.16150191520491872,
      "grad_norm": 556.0,
      "learning_rate": 4.6593567251461987e-07,
      "logits/chosen": -0.6470766067504883,
      "logits/rejected": -0.6531392931938171,
      "logps/chosen": -1003.5410766601562,
      "logps/rejected": -889.65576171875,
      "loss": 0.6988,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0003473288379609585,
      "rewards/margins": -0.01050653401762247,
      "rewards/rejected": 0.0101592056453228,
      "step": 614
    },
    {
      "epoch": 0.16176494764010588,
      "grad_norm": 448.0,
      "learning_rate": 4.657894736842105e-07,
      "logits/chosen": -0.650424063205719,
      "logits/rejected": -0.6462191939353943,
      "logps/chosen": -756.3436889648438,
      "logps/rejected": -709.0968017578125,
      "loss": 0.6893,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.016525747254490852,
      "rewards/margins": 0.007933616638183594,
      "rewards/rejected": 0.00859212875366211,
      "step": 615
    },
    {
      "epoch": 0.16202798007529304,
      "grad_norm": 464.0,
      "learning_rate": 4.6564327485380116e-07,
      "logits/chosen": -0.649438738822937,
      "logits/rejected": -0.6562921404838562,
      "logps/chosen": -917.2308349609375,
      "logps/rejected": -699.2810668945312,
      "loss": 0.689,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.002286480739712715,
      "rewards/margins": 0.008445214480161667,
      "rewards/rejected": -0.006158733740448952,
      "step": 616
    },
    {
      "epoch": 0.1622910125104802,
      "grad_norm": 604.0,
      "learning_rate": 4.654970760233918e-07,
      "logits/chosen": -0.6636006236076355,
      "logits/rejected": -0.675687313079834,
      "logps/chosen": -956.2766723632812,
      "logps/rejected": -1043.9761962890625,
      "loss": 0.6933,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.00188369769603014,
      "rewards/margins": 0.00042104627937078476,
      "rewards/rejected": 0.0014626504853367805,
      "step": 617
    },
    {
      "epoch": 0.16255404494566736,
      "grad_norm": 592.0,
      "learning_rate": 4.6535087719298245e-07,
      "logits/chosen": -0.6683772206306458,
      "logits/rejected": -0.6651363372802734,
      "logps/chosen": -1325.2598876953125,
      "logps/rejected": -1057.5224609375,
      "loss": 0.6893,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.002490806393325329,
      "rewards/margins": 0.009081268683075905,
      "rewards/rejected": -0.011572074145078659,
      "step": 618
    },
    {
      "epoch": 0.16281707738085452,
      "grad_norm": 528.0,
      "learning_rate": 4.652046783625731e-07,
      "logits/chosen": -0.6601821184158325,
      "logits/rejected": -0.6509106159210205,
      "logps/chosen": -1300.499755859375,
      "logps/rejected": -1126.980712890625,
      "loss": 0.6909,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.008894538506865501,
      "rewards/margins": 0.0053956047631800175,
      "rewards/rejected": -0.014290141873061657,
      "step": 619
    },
    {
      "epoch": 0.16308010981604168,
      "grad_norm": 724.0,
      "learning_rate": 4.650584795321637e-07,
      "logits/chosen": -0.6339240074157715,
      "logits/rejected": -0.6429381370544434,
      "logps/chosen": -1163.7109375,
      "logps/rejected": -721.8901977539062,
      "loss": 0.6898,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.006753731518983841,
      "rewards/margins": 0.008273027837276459,
      "rewards/rejected": -0.0015192989958450198,
      "step": 620
    },
    {
      "epoch": 0.16334314225122887,
      "grad_norm": 374.0,
      "learning_rate": 4.6491228070175437e-07,
      "logits/chosen": -0.647847056388855,
      "logits/rejected": -0.6540321111679077,
      "logps/chosen": -917.5108642578125,
      "logps/rejected": -688.3504638671875,
      "loss": 0.7026,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.001913643442094326,
      "rewards/margins": -0.018618440255522728,
      "rewards/rejected": 0.02053208462893963,
      "step": 621
    },
    {
      "epoch": 0.16360617468641603,
      "grad_norm": 592.0,
      "learning_rate": 4.6476608187134504e-07,
      "logits/chosen": -0.6355500221252441,
      "logits/rejected": -0.637879490852356,
      "logps/chosen": -1069.0250244140625,
      "logps/rejected": -860.7989501953125,
      "loss": 0.6993,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011551856994628906,
      "rewards/margins": -0.011195802129805088,
      "rewards/rejected": -0.00035605416633188725,
      "step": 622
    },
    {
      "epoch": 0.1638692071216032,
      "grad_norm": 600.0,
      "learning_rate": 4.6461988304093566e-07,
      "logits/chosen": -0.6572977304458618,
      "logits/rejected": -0.6615984439849854,
      "logps/chosen": -1385.99658203125,
      "logps/rejected": -1117.8319091796875,
      "loss": 0.6837,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.010710431262850761,
      "rewards/margins": 0.019510937854647636,
      "rewards/rejected": -0.008800506591796875,
      "step": 623
    },
    {
      "epoch": 0.16413223955679035,
      "grad_norm": 528.0,
      "learning_rate": 4.644736842105263e-07,
      "logits/chosen": -0.6525132060050964,
      "logits/rejected": -0.6760110855102539,
      "logps/chosen": -1239.646728515625,
      "logps/rejected": -950.6162109375,
      "loss": 0.7056,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.009344863705337048,
      "rewards/margins": -0.024012945592403412,
      "rewards/rejected": 0.014668083749711514,
      "step": 624
    },
    {
      "epoch": 0.1643952719919775,
      "grad_norm": 580.0,
      "learning_rate": 4.6432748538011695e-07,
      "logits/chosen": -0.6649733781814575,
      "logits/rejected": -0.66163170337677,
      "logps/chosen": -1085.30859375,
      "logps/rejected": -870.9446411132812,
      "loss": 0.6958,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.00727920513600111,
      "rewards/margins": -0.004736137576401234,
      "rewards/rejected": 0.012015344575047493,
      "step": 625
    },
    {
      "epoch": 0.16465830442716467,
      "grad_norm": 478.0,
      "learning_rate": 4.6418128654970757e-07,
      "logits/chosen": -0.6386498808860779,
      "logits/rejected": -0.6532381176948547,
      "logps/chosen": -971.453125,
      "logps/rejected": -765.7872314453125,
      "loss": 0.6927,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00551605224609375,
      "rewards/margins": 0.0014951694756746292,
      "rewards/rejected": -0.007011222653090954,
      "step": 626
    },
    {
      "epoch": 0.16492133686235183,
      "grad_norm": 624.0,
      "learning_rate": 4.6403508771929825e-07,
      "logits/chosen": -0.6673092842102051,
      "logits/rejected": NaN,
      "logps/chosen": -1460.5869140625,
      "logps/rejected": -996.58154296875,
      "loss": 0.6843,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.027661897242069244,
      "rewards/margins": 0.01905984990298748,
      "rewards/rejected": 0.008602047339081764,
      "step": 627
    },
    {
      "epoch": 0.165184369297539,
      "grad_norm": 464.0,
      "learning_rate": 4.6388888888888886e-07,
      "logits/chosen": -0.6479542851448059,
      "logits/rejected": -0.6454523205757141,
      "logps/chosen": -1065.26220703125,
      "logps/rejected": -1043.3228759765625,
      "loss": 0.6928,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0012118343729525805,
      "rewards/margins": 0.0012098317965865135,
      "rewards/rejected": -0.0024216650053858757,
      "step": 628
    },
    {
      "epoch": 0.16544740173272618,
      "grad_norm": 406.0,
      "learning_rate": 4.637426900584795e-07,
      "logits/chosen": -0.6795663833618164,
      "logits/rejected": -0.6850870847702026,
      "logps/chosen": -1399.7880859375,
      "logps/rejected": -1053.44384765625,
      "loss": 0.6908,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0011433600448071957,
      "rewards/margins": 0.0049679758958518505,
      "rewards/rejected": -0.006111336871981621,
      "step": 629
    },
    {
      "epoch": 0.16571043416791334,
      "grad_norm": 556.0,
      "learning_rate": 4.6359649122807016e-07,
      "logits/chosen": -0.670711100101471,
      "logits/rejected": NaN,
      "logps/chosen": -1393.0081787109375,
      "logps/rejected": -744.76806640625,
      "loss": 0.6954,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.005586384795606136,
      "rewards/margins": -0.003727317787706852,
      "rewards/rejected": -0.0018590688705444336,
      "step": 630
    },
    {
      "epoch": 0.1659734666031005,
      "grad_norm": 620.0,
      "learning_rate": 4.6345029239766083e-07,
      "logits/chosen": -0.6572563052177429,
      "logits/rejected": -0.6539417505264282,
      "logps/chosen": -1185.9931640625,
      "logps/rejected": -729.248046875,
      "loss": 0.7036,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.011548042297363281,
      "rewards/margins": -0.020135309547185898,
      "rewards/rejected": 0.008587265387177467,
      "step": 631
    },
    {
      "epoch": 0.16623649903828766,
      "grad_norm": 648.0,
      "learning_rate": 4.633040935672514e-07,
      "logits/chosen": -0.6524266004562378,
      "logits/rejected": -0.6525443196296692,
      "logps/chosen": -1112.8756103515625,
      "logps/rejected": -1302.09423828125,
      "loss": 0.6992,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.003885603277012706,
      "rewards/margins": -0.010755394585430622,
      "rewards/rejected": 0.014640998095273972,
      "step": 632
    },
    {
      "epoch": 0.16649953147347482,
      "grad_norm": 528.0,
      "learning_rate": 4.6315789473684207e-07,
      "logits/chosen": -0.6764792799949646,
      "logits/rejected": -0.6754053831100464,
      "logps/chosen": -1221.600341796875,
      "logps/rejected": -1120.4947509765625,
      "loss": 0.7018,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.006460856180638075,
      "rewards/margins": -0.016872026026248932,
      "rewards/rejected": 0.010411168448626995,
      "step": 633
    },
    {
      "epoch": 0.16676256390866198,
      "grad_norm": 460.0,
      "learning_rate": 4.6301169590643274e-07,
      "logits/chosen": -0.6311095356941223,
      "logits/rejected": -0.625081479549408,
      "logps/chosen": -770.1863403320312,
      "logps/rejected": -770.638427734375,
      "loss": 0.6863,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.00833053607493639,
      "rewards/margins": 0.01445460319519043,
      "rewards/rejected": -0.006124068517237902,
      "step": 634
    },
    {
      "epoch": 0.16702559634384914,
      "grad_norm": 544.0,
      "learning_rate": 4.6286549707602336e-07,
      "logits/chosen": -0.6549522280693054,
      "logits/rejected": -0.6497362852096558,
      "logps/chosen": -1037.9290771484375,
      "logps/rejected": -978.8318481445312,
      "loss": 0.68,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.011898328550159931,
      "rewards/margins": 0.027684878557920456,
      "rewards/rejected": -0.01578655280172825,
      "step": 635
    },
    {
      "epoch": 0.1672886287790363,
      "grad_norm": 592.0,
      "learning_rate": 4.62719298245614e-07,
      "logits/chosen": -0.6759097576141357,
      "logits/rejected": -0.6831099390983582,
      "logps/chosen": -1242.489990234375,
      "logps/rejected": -874.7149047851562,
      "loss": 0.675,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.026665497571229935,
      "rewards/margins": 0.03775444254279137,
      "rewards/rejected": -0.011088944971561432,
      "step": 636
    },
    {
      "epoch": 0.1675516612142235,
      "grad_norm": 528.0,
      "learning_rate": 4.6257309941520466e-07,
      "logits/chosen": -0.6362646222114563,
      "logits/rejected": NaN,
      "logps/chosen": -1461.423583984375,
      "logps/rejected": -926.4964599609375,
      "loss": 0.6951,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.003828524611890316,
      "rewards/margins": -0.0034300812985748053,
      "rewards/rejected": -0.00039844587445259094,
      "step": 637
    },
    {
      "epoch": 0.16781469364941065,
      "grad_norm": 532.0,
      "learning_rate": 4.624269005847953e-07,
      "logits/chosen": -0.6443532109260559,
      "logits/rejected": -0.64585942029953,
      "logps/chosen": -872.7605590820312,
      "logps/rejected": -904.7603149414062,
      "loss": 0.703,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.001926659606397152,
      "rewards/margins": -0.019133903086185455,
      "rewards/rejected": 0.021060561761260033,
      "step": 638
    },
    {
      "epoch": 0.1680777260845978,
      "grad_norm": 484.0,
      "learning_rate": 4.6228070175438595e-07,
      "logits/chosen": -0.6899157762527466,
      "logits/rejected": -0.6830337643623352,
      "logps/chosen": -856.8184814453125,
      "logps/rejected": -737.677978515625,
      "loss": 0.6817,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.018322275951504707,
      "rewards/margins": 0.02420673333108425,
      "rewards/rejected": -0.0058844564482569695,
      "step": 639
    },
    {
      "epoch": 0.16834075851978497,
      "grad_norm": 472.0,
      "learning_rate": 4.621345029239766e-07,
      "logits/chosen": -0.6545671224594116,
      "logits/rejected": -0.6478638052940369,
      "logps/chosen": -931.981201171875,
      "logps/rejected": -753.8602294921875,
      "loss": 0.7009,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.00989074632525444,
      "rewards/margins": -0.014875601977109909,
      "rewards/rejected": 0.004984855651855469,
      "step": 640
    },
    {
      "epoch": 0.16860379095497213,
      "grad_norm": 608.0,
      "learning_rate": 4.619883040935672e-07,
      "logits/chosen": -0.6420859694480896,
      "logits/rejected": -0.636662483215332,
      "logps/chosen": -1469.325439453125,
      "logps/rejected": -1224.0106201171875,
      "loss": 0.6908,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.005093289539217949,
      "rewards/margins": 0.005812313407659531,
      "rewards/rejected": -0.01090560108423233,
      "step": 641
    },
    {
      "epoch": 0.1688668233901593,
      "grad_norm": 548.0,
      "learning_rate": 4.6184210526315786e-07,
      "logits/chosen": -0.6588228344917297,
      "logits/rejected": -0.6599839329719543,
      "logps/chosen": -1308.170654296875,
      "logps/rejected": -745.5248413085938,
      "loss": 0.6995,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.02308216318488121,
      "rewards/margins": -0.011808491311967373,
      "rewards/rejected": -0.011273670010268688,
      "step": 642
    },
    {
      "epoch": 0.16912985582534645,
      "grad_norm": 680.0,
      "learning_rate": 4.6169590643274853e-07,
      "logits/chosen": -0.660077691078186,
      "logits/rejected": -0.677581787109375,
      "logps/chosen": -1181.0291748046875,
      "logps/rejected": -879.0274658203125,
      "loss": 0.7006,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0016775126568973064,
      "rewards/margins": -0.014317704364657402,
      "rewards/rejected": 0.012640189379453659,
      "step": 643
    },
    {
      "epoch": 0.16939288826053361,
      "grad_norm": 576.0,
      "learning_rate": 4.6154970760233915e-07,
      "logits/chosen": -0.6602239012718201,
      "logits/rejected": -0.6677110195159912,
      "logps/chosen": -1070.5419921875,
      "logps/rejected": -901.53125,
      "loss": 0.7017,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0033366200514137745,
      "rewards/margins": -0.016645049676299095,
      "rewards/rejected": 0.01998167112469673,
      "step": 644
    },
    {
      "epoch": 0.1696559206957208,
      "grad_norm": 524.0,
      "learning_rate": 4.614035087719298e-07,
      "logits/chosen": -0.6600430607795715,
      "logits/rejected": -0.6559502482414246,
      "logps/chosen": -857.5313720703125,
      "logps/rejected": -639.4733276367188,
      "loss": 0.6972,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01218872144818306,
      "rewards/margins": -0.007568406872451305,
      "rewards/rejected": -0.004620314110070467,
      "step": 645
    },
    {
      "epoch": 0.16991895313090796,
      "grad_norm": 484.0,
      "learning_rate": 4.6125730994152045e-07,
      "logits/chosen": -0.6512450575828552,
      "logits/rejected": -0.661704421043396,
      "logps/chosen": -1402.625,
      "logps/rejected": -1094.3994140625,
      "loss": 0.6872,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011508655734360218,
      "rewards/margins": 0.012406541034579277,
      "rewards/rejected": -0.02391519583761692,
      "step": 646
    },
    {
      "epoch": 0.17018198556609512,
      "grad_norm": 446.0,
      "learning_rate": 4.611111111111111e-07,
      "logits/chosen": -0.6739441752433777,
      "logits/rejected": -0.676597535610199,
      "logps/chosen": -865.2835693359375,
      "logps/rejected": -684.4969482421875,
      "loss": 0.6764,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.03604907914996147,
      "rewards/margins": 0.034582044929265976,
      "rewards/rejected": 0.0014670370146632195,
      "step": 647
    },
    {
      "epoch": 0.17044501800128228,
      "grad_norm": 544.0,
      "learning_rate": 4.6096491228070174e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6479510068893433,
      "logps/chosen": -1083.4622802734375,
      "logps/rejected": -1356.57080078125,
      "loss": 0.6983,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.009426307864487171,
      "rewards/margins": -0.009334182366728783,
      "rewards/rejected": -9.212549775838852e-05,
      "step": 648
    },
    {
      "epoch": 0.17070805043646944,
      "grad_norm": 620.0,
      "learning_rate": 4.6081871345029236e-07,
      "logits/chosen": -0.6821374297142029,
      "logits/rejected": -0.6783127784729004,
      "logps/chosen": -933.8197631835938,
      "logps/rejected": -631.5008544921875,
      "loss": 0.7076,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.009608983062207699,
      "rewards/margins": -0.027953529730439186,
      "rewards/rejected": 0.018344547599554062,
      "step": 649
    },
    {
      "epoch": 0.1709710828716566,
      "grad_norm": 564.0,
      "learning_rate": 4.6067251461988303e-07,
      "logits/chosen": -0.6801784038543701,
      "logits/rejected": -0.6879419684410095,
      "logps/chosen": -1061.033447265625,
      "logps/rejected": -954.2124633789062,
      "loss": 0.6901,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.017193222418427467,
      "rewards/margins": 0.006732321344316006,
      "rewards/rejected": 0.010460902005434036,
      "step": 650
    },
    {
      "epoch": 0.17123411530684376,
      "grad_norm": 644.0,
      "learning_rate": 4.6052631578947365e-07,
      "logits/chosen": -0.6406633853912354,
      "logits/rejected": -0.6443125605583191,
      "logps/chosen": -1742.260009765625,
      "logps/rejected": -1219.6861572265625,
      "loss": 0.695,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004571151919662952,
      "rewards/margins": -0.0026114454958587885,
      "rewards/rejected": 0.007182598114013672,
      "step": 651
    },
    {
      "epoch": 0.17149714774203093,
      "grad_norm": 496.0,
      "learning_rate": 4.603801169590643e-07,
      "logits/chosen": -0.6638538837432861,
      "logits/rejected": -0.668156087398529,
      "logps/chosen": -961.4178466796875,
      "logps/rejected": -756.8173217773438,
      "loss": 0.6917,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012229538522660732,
      "rewards/margins": 0.0031833648681640625,
      "rewards/rejected": 0.009046172723174095,
      "step": 652
    },
    {
      "epoch": 0.1717601801772181,
      "grad_norm": 478.0,
      "learning_rate": 4.60233918128655e-07,
      "logits/chosen": -0.6351841688156128,
      "logits/rejected": -0.6368878483772278,
      "logps/chosen": -1149.31640625,
      "logps/rejected": -684.7774658203125,
      "loss": 0.6888,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.010816669091582298,
      "rewards/margins": 0.00934519898146391,
      "rewards/rejected": 0.0014714719727635384,
      "step": 653
    },
    {
      "epoch": 0.17202321261240527,
      "grad_norm": 474.0,
      "learning_rate": 4.6008771929824556e-07,
      "logits/chosen": -0.6623499989509583,
      "logits/rejected": -0.6653239727020264,
      "logps/chosen": -1324.885498046875,
      "logps/rejected": -832.5585327148438,
      "loss": 0.7075,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.03422670066356659,
      "rewards/margins": -0.027720261365175247,
      "rewards/rejected": -0.006506443023681641,
      "step": 654
    },
    {
      "epoch": 0.17228624504759243,
      "grad_norm": 588.0,
      "learning_rate": 4.5994152046783624e-07,
      "logits/chosen": -0.6622382402420044,
      "logits/rejected": NaN,
      "logps/chosen": -1168.2470703125,
      "logps/rejected": -1191.270751953125,
      "loss": 0.7033,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.03303203731775284,
      "rewards/margins": -0.01947164535522461,
      "rewards/rejected": -0.01356039009988308,
      "step": 655
    },
    {
      "epoch": 0.1725492774827796,
      "grad_norm": 446.0,
      "learning_rate": 4.597953216374269e-07,
      "logits/chosen": -0.6483920812606812,
      "logits/rejected": -0.6548329591751099,
      "logps/chosen": -987.1036376953125,
      "logps/rejected": -684.447021484375,
      "loss": 0.6871,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.011769820004701614,
      "rewards/margins": 0.01282267551869154,
      "rewards/rejected": -0.0010528565617278218,
      "step": 656
    },
    {
      "epoch": 0.17281230991796676,
      "grad_norm": 540.0,
      "learning_rate": 4.5964912280701753e-07,
      "logits/chosen": -0.6480388045310974,
      "logits/rejected": -0.6492023468017578,
      "logps/chosen": -1103.59716796875,
      "logps/rejected": -918.759765625,
      "loss": 0.6987,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0017939568497240543,
      "rewards/margins": -0.010383701883256435,
      "rewards/rejected": 0.008589744567871094,
      "step": 657
    },
    {
      "epoch": 0.17307534235315392,
      "grad_norm": 576.0,
      "learning_rate": 4.5950292397660815e-07,
      "logits/chosen": -0.6406853199005127,
      "logits/rejected": -0.6450140476226807,
      "logps/chosen": -1168.095703125,
      "logps/rejected": -877.1400146484375,
      "loss": 0.6835,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0485471747815609,
      "rewards/margins": 0.02007603459060192,
      "rewards/rejected": 0.028471138328313828,
      "step": 658
    },
    {
      "epoch": 0.17333837478834108,
      "grad_norm": 588.0,
      "learning_rate": 4.593567251461988e-07,
      "logits/chosen": -0.6772143244743347,
      "logits/rejected": -0.6723844408988953,
      "logps/chosen": -1081.2314453125,
      "logps/rejected": -1075.686767578125,
      "loss": 0.6956,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.021831845864653587,
      "rewards/margins": -0.0033207423985004425,
      "rewards/rejected": 0.02515258640050888,
      "step": 659
    },
    {
      "epoch": 0.17360140722352826,
      "grad_norm": 720.0,
      "learning_rate": 4.5921052631578944e-07,
      "logits/chosen": -0.6420338153839111,
      "logits/rejected": -0.6662768125534058,
      "logps/chosen": -1394.9893798828125,
      "logps/rejected": -1094.008544921875,
      "loss": 0.6938,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006850433070212603,
      "rewards/margins": 0.00019087904365733266,
      "rewards/rejected": 0.006659555248916149,
      "step": 660
    },
    {
      "epoch": 0.17386443965871543,
      "grad_norm": 402.0,
      "learning_rate": 4.590643274853801e-07,
      "logits/chosen": -0.6475464105606079,
      "logits/rejected": -0.6532906293869019,
      "logps/chosen": -790.0838012695312,
      "logps/rejected": -548.1459350585938,
      "loss": 0.6826,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02077193185687065,
      "rewards/margins": 0.022324133664369583,
      "rewards/rejected": -0.0015522001776844263,
      "step": 661
    },
    {
      "epoch": 0.17412747209390259,
      "grad_norm": 536.0,
      "learning_rate": 4.5891812865497074e-07,
      "logits/chosen": -0.6796097159385681,
      "logits/rejected": -0.669807493686676,
      "logps/chosen": -1025.6595458984375,
      "logps/rejected": -717.2975463867188,
      "loss": 0.6946,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.00011587096378207207,
      "rewards/margins": -0.001328753773123026,
      "rewards/rejected": 0.0014446255518123507,
      "step": 662
    },
    {
      "epoch": 0.17439050452908975,
      "grad_norm": 572.0,
      "learning_rate": 4.5877192982456136e-07,
      "logits/chosen": -0.6413443684577942,
      "logits/rejected": -0.6527271270751953,
      "logps/chosen": -1323.782470703125,
      "logps/rejected": -1010.6643676757812,
      "loss": 0.701,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01614398881793022,
      "rewards/margins": -0.01499161683022976,
      "rewards/rejected": -0.001152372220531106,
      "step": 663
    },
    {
      "epoch": 0.1746535369642769,
      "grad_norm": 468.0,
      "learning_rate": 4.5862573099415203e-07,
      "logits/chosen": -0.6352505683898926,
      "logits/rejected": -0.6472588181495667,
      "logps/chosen": -955.2815551757812,
      "logps/rejected": -560.502685546875,
      "loss": 0.6866,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0037085525691509247,
      "rewards/margins": 0.013453674502670765,
      "rewards/rejected": -0.017162229865789413,
      "step": 664
    },
    {
      "epoch": 0.17491656939946407,
      "grad_norm": 588.0,
      "learning_rate": 4.584795321637427e-07,
      "logits/chosen": -0.6509827375411987,
      "logits/rejected": -0.6648935079574585,
      "logps/chosen": -1680.55810546875,
      "logps/rejected": -1486.48828125,
      "loss": 0.7066,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01960649900138378,
      "rewards/margins": -0.025792311877012253,
      "rewards/rejected": 0.0061858175322413445,
      "step": 665
    },
    {
      "epoch": 0.17517960183465123,
      "grad_norm": 544.0,
      "learning_rate": 4.5833333333333327e-07,
      "logits/chosen": -0.6379156112670898,
      "logits/rejected": -0.6393079161643982,
      "logps/chosen": -1074.4376220703125,
      "logps/rejected": -680.0652465820312,
      "loss": 0.6938,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0011207587085664272,
      "rewards/margins": -0.0002634050324559212,
      "rewards/rejected": 0.0013841614127159119,
      "step": 666
    },
    {
      "epoch": 0.1754426342698384,
      "grad_norm": 644.0,
      "learning_rate": 4.5818713450292394e-07,
      "logits/chosen": -0.6872091293334961,
      "logits/rejected": -0.6873466372489929,
      "logps/chosen": -1500.3408203125,
      "logps/rejected": -1323.6348876953125,
      "loss": 0.697,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01432876754552126,
      "rewards/margins": -0.006065368186682463,
      "rewards/rejected": -0.00826339703053236,
      "step": 667
    },
    {
      "epoch": 0.17570566670502558,
      "grad_norm": 612.0,
      "learning_rate": 4.580409356725146e-07,
      "logits/chosen": -0.6500923037528992,
      "logits/rejected": NaN,
      "logps/chosen": -1327.0386962890625,
      "logps/rejected": -885.4765014648438,
      "loss": 0.6845,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.002838707994669676,
      "rewards/margins": 0.018357276916503906,
      "rewards/rejected": -0.015518571250140667,
      "step": 668
    },
    {
      "epoch": 0.17596869914021274,
      "grad_norm": 506.0,
      "learning_rate": 4.5789473684210523e-07,
      "logits/chosen": -0.6594017744064331,
      "logits/rejected": -0.6425626277923584,
      "logps/chosen": -801.919189453125,
      "logps/rejected": -855.4094848632812,
      "loss": 0.6981,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00592279527336359,
      "rewards/margins": -0.009219836443662643,
      "rewards/rejected": 0.003297042567282915,
      "step": 669
    },
    {
      "epoch": 0.1762317315753999,
      "grad_norm": 496.0,
      "learning_rate": 4.577485380116959e-07,
      "logits/chosen": -0.645927906036377,
      "logits/rejected": -0.6492242813110352,
      "logps/chosen": -1228.087158203125,
      "logps/rejected": -687.6332397460938,
      "loss": 0.6876,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01735715940594673,
      "rewards/margins": 0.01144104078412056,
      "rewards/rejected": 0.0059161195531487465,
      "step": 670
    },
    {
      "epoch": 0.17649476401058706,
      "grad_norm": 592.0,
      "learning_rate": 4.5760233918128653e-07,
      "logits/chosen": -0.6506455540657043,
      "logits/rejected": -0.6568708419799805,
      "logps/chosen": -1275.1495361328125,
      "logps/rejected": -769.5081787109375,
      "loss": 0.7099,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.026969242841005325,
      "rewards/margins": -0.03216390684247017,
      "rewards/rejected": 0.005194662604480982,
      "step": 671
    },
    {
      "epoch": 0.17675779644577422,
      "grad_norm": 492.0,
      "learning_rate": 4.5745614035087715e-07,
      "logits/chosen": -0.6535452008247375,
      "logits/rejected": -0.6556299924850464,
      "logps/chosen": -1398.078125,
      "logps/rejected": -996.4866943359375,
      "loss": 0.6999,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0007474906742572784,
      "rewards/margins": -0.013053607195615768,
      "rewards/rejected": 0.01230611838400364,
      "step": 672
    },
    {
      "epoch": 0.17702082888096138,
      "grad_norm": 544.0,
      "learning_rate": 4.573099415204678e-07,
      "logits/chosen": -0.652185320854187,
      "logits/rejected": -0.6541866064071655,
      "logps/chosen": -935.67919921875,
      "logps/rejected": -1028.971435546875,
      "loss": 0.7031,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.028034929186105728,
      "rewards/margins": -0.019338179379701614,
      "rewards/rejected": -0.008696747943758965,
      "step": 673
    },
    {
      "epoch": 0.17728386131614854,
      "grad_norm": 494.0,
      "learning_rate": 4.571637426900585e-07,
      "logits/chosen": -0.645858883857727,
      "logits/rejected": -0.6461138129234314,
      "logps/chosen": -957.1597900390625,
      "logps/rejected": -1021.0820922851562,
      "loss": 0.6864,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.015526103787124157,
      "rewards/margins": 0.013872243463993073,
      "rewards/rejected": -0.029398346319794655,
      "step": 674
    },
    {
      "epoch": 0.1775468937513357,
      "grad_norm": 572.0,
      "learning_rate": 4.5701754385964906e-07,
      "logits/chosen": -0.6534539461135864,
      "logits/rejected": -0.648851215839386,
      "logps/chosen": -1285.216552734375,
      "logps/rejected": -762.2664794921875,
      "loss": 0.6867,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.004258443135768175,
      "rewards/margins": 0.013961123302578926,
      "rewards/rejected": -0.018219565972685814,
      "step": 675
    },
    {
      "epoch": 0.1778099261865229,
      "grad_norm": 552.0,
      "learning_rate": 4.5687134502923973e-07,
      "logits/chosen": -0.6651393175125122,
      "logits/rejected": NaN,
      "logps/chosen": -1210.8154296875,
      "logps/rejected": -626.6005249023438,
      "loss": 0.7039,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.023878097534179688,
      "rewards/margins": -0.020720576867461205,
      "rewards/rejected": -0.003157520666718483,
      "step": 676
    },
    {
      "epoch": 0.17807295862171005,
      "grad_norm": 498.0,
      "learning_rate": 4.567251461988304e-07,
      "logits/chosen": -0.6645187139511108,
      "logits/rejected": -0.6662808060646057,
      "logps/chosen": -1021.8192749023438,
      "logps/rejected": -762.6009521484375,
      "loss": 0.6873,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.017307233065366745,
      "rewards/margins": 0.012361478991806507,
      "rewards/rejected": 0.004945756401866674,
      "step": 677
    },
    {
      "epoch": 0.1783359910568972,
      "grad_norm": 498.0,
      "learning_rate": 4.56578947368421e-07,
      "logits/chosen": -0.6450657844543457,
      "logits/rejected": -0.640282928943634,
      "logps/chosen": -1544.8211669921875,
      "logps/rejected": -1204.3367919921875,
      "loss": 0.6798,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00025167432613670826,
      "rewards/margins": 0.027689792215824127,
      "rewards/rejected": -0.027941465377807617,
      "step": 678
    },
    {
      "epoch": 0.17859902349208437,
      "grad_norm": 624.0,
      "learning_rate": 4.5643274853801164e-07,
      "logits/chosen": -0.648372232913971,
      "logits/rejected": -0.6453104615211487,
      "logps/chosen": -1042.7969970703125,
      "logps/rejected": -695.660400390625,
      "loss": 0.697,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.015947582200169563,
      "rewards/margins": -0.007413243874907494,
      "rewards/rejected": -0.008534337393939495,
      "step": 679
    },
    {
      "epoch": 0.17886205592727153,
      "grad_norm": 672.0,
      "learning_rate": 4.562865497076023e-07,
      "logits/chosen": -0.6526268124580383,
      "logits/rejected": -0.6492290496826172,
      "logps/chosen": -1007.3619384765625,
      "logps/rejected": -1007.4280395507812,
      "loss": 0.6922,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.009574556723237038,
      "rewards/margins": 0.002411365043371916,
      "rewards/rejected": -0.01198592223227024,
      "step": 680
    },
    {
      "epoch": 0.1791250883624587,
      "grad_norm": 560.0,
      "learning_rate": 4.5614035087719294e-07,
      "logits/chosen": -0.6509470343589783,
      "logits/rejected": -0.6420130729675293,
      "logps/chosen": -1204.3905029296875,
      "logps/rejected": -881.9530029296875,
      "loss": 0.6911,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.012878607958555222,
      "rewards/margins": 0.004485797602683306,
      "rewards/rejected": 0.008392810821533203,
      "step": 681
    },
    {
      "epoch": 0.17938812079764585,
      "grad_norm": 528.0,
      "learning_rate": 4.559941520467836e-07,
      "logits/chosen": -0.646864652633667,
      "logits/rejected": -0.6308355331420898,
      "logps/chosen": -1140.21533203125,
      "logps/rejected": -964.585693359375,
      "loss": 0.6884,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.013562393374741077,
      "rewards/margins": 0.010549355298280716,
      "rewards/rejected": 0.0030130380764603615,
      "step": 682
    },
    {
      "epoch": 0.179651153232833,
      "grad_norm": 576.0,
      "learning_rate": 4.558479532163743e-07,
      "logits/chosen": -0.6731962561607361,
      "logits/rejected": -0.6755200624465942,
      "logps/chosen": -1673.778076171875,
      "logps/rejected": -1103.115478515625,
      "loss": 0.6832,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00815734826028347,
      "rewards/margins": 0.02083425596356392,
      "rewards/rejected": -0.012676907703280449,
      "step": 683
    },
    {
      "epoch": 0.1799141856680202,
      "grad_norm": 648.0,
      "learning_rate": 4.5570175438596485e-07,
      "logits/chosen": -0.6373763084411621,
      "logits/rejected": -0.6506407260894775,
      "logps/chosen": -1430.5474853515625,
      "logps/rejected": -1026.4542236328125,
      "loss": 0.6996,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.005757140927016735,
      "rewards/margins": -0.011945437639951706,
      "rewards/rejected": 0.006188295781612396,
      "step": 684
    },
    {
      "epoch": 0.18017721810320736,
      "grad_norm": 486.0,
      "learning_rate": 4.555555555555555e-07,
      "logits/chosen": -0.6577614545822144,
      "logits/rejected": -0.6573840975761414,
      "logps/chosen": -1353.5196533203125,
      "logps/rejected": -1047.7652587890625,
      "loss": 0.6843,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00899286288768053,
      "rewards/margins": 0.01851492002606392,
      "rewards/rejected": -0.009522057138383389,
      "step": 685
    },
    {
      "epoch": 0.18044025053839452,
      "grad_norm": 612.0,
      "learning_rate": 4.554093567251462e-07,
      "logits/chosen": -0.647865355014801,
      "logits/rejected": -0.6470797657966614,
      "logps/chosen": -992.3675537109375,
      "logps/rejected": -923.7362060546875,
      "loss": 0.6876,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.012101506814360619,
      "rewards/margins": 0.011899088509380817,
      "rewards/rejected": 0.00020241830497980118,
      "step": 686
    },
    {
      "epoch": 0.18070328297358168,
      "grad_norm": 450.0,
      "learning_rate": 4.5526315789473687e-07,
      "logits/chosen": -0.6360988020896912,
      "logits/rejected": -0.6486783027648926,
      "logps/chosen": -991.2210693359375,
      "logps/rejected": -639.8688354492188,
      "loss": 0.6918,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0025600437074899673,
      "rewards/margins": 0.0031267632730305195,
      "rewards/rejected": -0.0005667211953550577,
      "step": 687
    },
    {
      "epoch": 0.18096631540876884,
      "grad_norm": 536.0,
      "learning_rate": 4.5511695906432744e-07,
      "logits/chosen": -0.6371111273765564,
      "logits/rejected": -0.6315031051635742,
      "logps/chosen": -1247.3681640625,
      "logps/rejected": -1086.0225830078125,
      "loss": 0.6931,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.015214920043945312,
      "rewards/margins": 0.0011767377145588398,
      "rewards/rejected": -0.016391659155488014,
      "step": 688
    },
    {
      "epoch": 0.181229347843956,
      "grad_norm": 400.0,
      "learning_rate": 4.549707602339181e-07,
      "logits/chosen": -0.6669952273368835,
      "logits/rejected": -0.6723837852478027,
      "logps/chosen": -693.1636962890625,
      "logps/rejected": -490.64208984375,
      "loss": 0.6801,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03195839002728462,
      "rewards/margins": 0.027569198980927467,
      "rewards/rejected": 0.004389191046357155,
      "step": 689
    },
    {
      "epoch": 0.18149238027914316,
      "grad_norm": 532.0,
      "learning_rate": 4.548245614035088e-07,
      "logits/chosen": -0.6426583528518677,
      "logits/rejected": -0.6392993927001953,
      "logps/chosen": -1147.2236328125,
      "logps/rejected": -1019.8736572265625,
      "loss": 0.6804,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.003381536342203617,
      "rewards/margins": 0.026433560997247696,
      "rewards/rejected": -0.023052025586366653,
      "step": 690
    },
    {
      "epoch": 0.18175541271433032,
      "grad_norm": 512.0,
      "learning_rate": 4.546783625730994e-07,
      "logits/chosen": -0.6624666452407837,
      "logits/rejected": -0.6684360504150391,
      "logps/chosen": -923.652587890625,
      "logps/rejected": -776.1763916015625,
      "loss": 0.6965,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.000889874529093504,
      "rewards/margins": -0.006122398190200329,
      "rewards/rejected": 0.0070122722536325455,
      "step": 691
    },
    {
      "epoch": 0.1820184451495175,
      "grad_norm": 540.0,
      "learning_rate": 4.5453216374269e-07,
      "logits/chosen": -0.6513867378234863,
      "logits/rejected": -0.6412699222564697,
      "logps/chosen": -1186.4639892578125,
      "logps/rejected": -984.4757080078125,
      "loss": 0.6897,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004696083255112171,
      "rewards/margins": 0.007339287083595991,
      "rewards/rejected": -0.00264320382848382,
      "step": 692
    },
    {
      "epoch": 0.18228147758470467,
      "grad_norm": 428.0,
      "learning_rate": 4.543859649122807e-07,
      "logits/chosen": -0.6692111492156982,
      "logits/rejected": -0.6650774478912354,
      "logps/chosen": -956.0091552734375,
      "logps/rejected": -721.0761108398438,
      "loss": 0.6827,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.02759256400167942,
      "rewards/margins": 0.021480560302734375,
      "rewards/rejected": 0.0061120036989450455,
      "step": 693
    },
    {
      "epoch": 0.18254451001989183,
      "grad_norm": 524.0,
      "learning_rate": 4.542397660818713e-07,
      "logits/chosen": -0.6446396708488464,
      "logits/rejected": -0.645882785320282,
      "logps/chosen": -1544.9681396484375,
      "logps/rejected": -1224.28564453125,
      "loss": 0.6885,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004899310413748026,
      "rewards/margins": 0.010236836038529873,
      "rewards/rejected": -0.005337524693459272,
      "step": 694
    },
    {
      "epoch": 0.182807542455079,
      "grad_norm": 560.0,
      "learning_rate": 4.54093567251462e-07,
      "logits/chosen": -0.638068675994873,
      "logits/rejected": -0.6495839357376099,
      "logps/chosen": -1519.6075439453125,
      "logps/rejected": -1088.3643798828125,
      "loss": 0.6918,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.005098534282296896,
      "rewards/margins": 0.0037232404574751854,
      "rewards/rejected": -0.00882177334278822,
      "step": 695
    },
    {
      "epoch": 0.18307057489026615,
      "grad_norm": 500.0,
      "learning_rate": 4.5394736842105266e-07,
      "logits/chosen": -0.6327572464942932,
      "logits/rejected": -0.6333155632019043,
      "logps/chosen": -905.5890502929688,
      "logps/rejected": -805.4906616210938,
      "loss": 0.6971,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -3.2807060051709414e-05,
      "rewards/margins": -0.007597350515425205,
      "rewards/rejected": 0.007564546074718237,
      "step": 696
    },
    {
      "epoch": 0.1833336073254533,
      "grad_norm": 528.0,
      "learning_rate": 4.5380116959064323e-07,
      "logits/chosen": -0.6695257425308228,
      "logits/rejected": -0.6723732948303223,
      "logps/chosen": -979.3114624023438,
      "logps/rejected": -798.7437744140625,
      "loss": 0.701,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00863561686128378,
      "rewards/margins": -0.014925003983080387,
      "rewards/rejected": 0.006289387121796608,
      "step": 697
    },
    {
      "epoch": 0.18359663976064047,
      "grad_norm": 494.0,
      "learning_rate": 4.536549707602339e-07,
      "logits/chosen": -0.6156856417655945,
      "logits/rejected": -0.6167691349983215,
      "logps/chosen": -917.764892578125,
      "logps/rejected": -660.7918090820312,
      "loss": 0.685,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.003991604782640934,
      "rewards/margins": 0.016696453094482422,
      "rewards/rejected": -0.012704849243164062,
      "step": 698
    },
    {
      "epoch": 0.18385967219582766,
      "grad_norm": 536.0,
      "learning_rate": 4.5350877192982457e-07,
      "logits/chosen": -0.6303366422653198,
      "logits/rejected": -0.6334330439567566,
      "logps/chosen": -962.2940673828125,
      "logps/rejected": -679.2992553710938,
      "loss": 0.6895,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004458046518266201,
      "rewards/margins": 0.007739925757050514,
      "rewards/rejected": -0.003281879238784313,
      "step": 699
    },
    {
      "epoch": 0.18412270463101482,
      "grad_norm": 636.0,
      "learning_rate": 4.533625730994152e-07,
      "logits/chosen": -0.6585320830345154,
      "logits/rejected": -0.6703499555587769,
      "logps/chosen": -1380.5262451171875,
      "logps/rejected": -1076.5177001953125,
      "loss": 0.6886,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.009824943728744984,
      "rewards/margins": 0.010901881381869316,
      "rewards/rejected": -0.0010769357904791832,
      "step": 700
    },
    {
      "epoch": 0.18438573706620198,
      "grad_norm": 600.0,
      "learning_rate": 4.532163742690058e-07,
      "logits/chosen": -0.6297889947891235,
      "logits/rejected": -0.6326335668563843,
      "logps/chosen": -1395.7266845703125,
      "logps/rejected": -991.288818359375,
      "loss": 0.6923,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0275789275765419,
      "rewards/margins": 0.0032555575016885996,
      "rewards/rejected": 0.024323368445038795,
      "step": 701
    },
    {
      "epoch": 0.18464876950138914,
      "grad_norm": 556.0,
      "learning_rate": 4.530701754385965e-07,
      "logits/chosen": -0.6607016324996948,
      "logits/rejected": -0.6531972289085388,
      "logps/chosen": -1336.6329345703125,
      "logps/rejected": -1220.2574462890625,
      "loss": 0.699,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.00026817317120730877,
      "rewards/margins": -0.011176491156220436,
      "rewards/rejected": 0.010908316820859909,
      "step": 702
    },
    {
      "epoch": 0.1849118019365763,
      "grad_norm": 660.0,
      "learning_rate": 4.529239766081871e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6646823883056641,
      "logps/chosen": -1177.8646240234375,
      "logps/rejected": -828.2650756835938,
      "loss": 0.6983,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.002808857709169388,
      "rewards/margins": -0.009525062516331673,
      "rewards/rejected": 0.012333917431533337,
      "step": 703
    },
    {
      "epoch": 0.18517483437176346,
      "grad_norm": 688.0,
      "learning_rate": 4.527777777777778e-07,
      "logits/chosen": -0.629706084728241,
      "logits/rejected": -0.6336047053337097,
      "logps/chosen": -1464.25439453125,
      "logps/rejected": -1034.655029296875,
      "loss": 0.6922,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.013882448896765709,
      "rewards/margins": 0.002639484591782093,
      "rewards/rejected": -0.016521930694580078,
      "step": 704
    },
    {
      "epoch": 0.18543786680695062,
      "grad_norm": 448.0,
      "learning_rate": 4.526315789473684e-07,
      "logits/chosen": -0.6503704786300659,
      "logits/rejected": -0.6587463021278381,
      "logps/chosen": -791.1139526367188,
      "logps/rejected": -617.254638671875,
      "loss": 0.6893,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.008926916867494583,
      "rewards/margins": 0.008050777018070221,
      "rewards/rejected": 0.000876140664331615,
      "step": 705
    },
    {
      "epoch": 0.18570089924213778,
      "grad_norm": 338.0,
      "learning_rate": 4.52485380116959e-07,
      "logits/chosen": -0.6361703872680664,
      "logits/rejected": -0.6347763538360596,
      "logps/chosen": -760.4567260742188,
      "logps/rejected": -576.458740234375,
      "loss": 0.6918,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -6.25615066383034e-05,
      "rewards/margins": 0.002807426266372204,
      "rewards/rejected": -0.0028699873946607113,
      "step": 706
    },
    {
      "epoch": 0.18596393167732497,
      "grad_norm": 584.0,
      "learning_rate": 4.523391812865497e-07,
      "logits/chosen": -0.6604491472244263,
      "logits/rejected": -0.6597015261650085,
      "logps/chosen": -1255.995361328125,
      "logps/rejected": -868.73681640625,
      "loss": 0.6867,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0007294652750715613,
      "rewards/margins": 0.014817904680967331,
      "rewards/rejected": -0.014088439755141735,
      "step": 707
    },
    {
      "epoch": 0.18622696411251213,
      "grad_norm": 520.0,
      "learning_rate": 4.5219298245614036e-07,
      "logits/chosen": -0.6329449415206909,
      "logits/rejected": -0.6324520111083984,
      "logps/chosen": -1221.93701171875,
      "logps/rejected": -815.06689453125,
      "loss": 0.6859,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007984447292983532,
      "rewards/margins": 0.01604938507080078,
      "rewards/rejected": -0.00806493777781725,
      "step": 708
    },
    {
      "epoch": 0.1864899965476993,
      "grad_norm": 468.0,
      "learning_rate": 4.5204678362573093e-07,
      "logits/chosen": -0.6345410346984863,
      "logits/rejected": -0.6536726355552673,
      "logps/chosen": -808.6947631835938,
      "logps/rejected": -849.1183471679688,
      "loss": 0.6951,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.014065217226743698,
      "rewards/margins": -0.0024709231220185757,
      "rewards/rejected": 0.016536138951778412,
      "step": 709
    },
    {
      "epoch": 0.18675302898288645,
      "grad_norm": 508.0,
      "learning_rate": 4.519005847953216e-07,
      "logits/chosen": -0.6536576747894287,
      "logits/rejected": -0.6497605443000793,
      "logps/chosen": -1158.8427734375,
      "logps/rejected": -911.9163208007812,
      "loss": 0.6913,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007209586910903454,
      "rewards/margins": 0.004279899410903454,
      "rewards/rejected": 0.0029296870343387127,
      "step": 710
    },
    {
      "epoch": 0.18701606141807361,
      "grad_norm": 580.0,
      "learning_rate": 4.517543859649123e-07,
      "logits/chosen": -0.6358242630958557,
      "logits/rejected": -0.6448075771331787,
      "logps/chosen": -1490.203857421875,
      "logps/rejected": -866.2582397460938,
      "loss": 0.693,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0041792867705225945,
      "rewards/margins": 0.0011592868249863386,
      "rewards/rejected": 0.003020000644028187,
      "step": 711
    },
    {
      "epoch": 0.18727909385326078,
      "grad_norm": 516.0,
      "learning_rate": 4.516081871345029e-07,
      "logits/chosen": -0.6621646285057068,
      "logits/rejected": -0.6682507991790771,
      "logps/chosen": -1186.7838134765625,
      "logps/rejected": -927.5475463867188,
      "loss": 0.6987,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.011769103817641735,
      "rewards/margins": -0.010672377422451973,
      "rewards/rejected": -0.0010967261623591185,
      "step": 712
    },
    {
      "epoch": 0.18754212628844794,
      "grad_norm": 536.0,
      "learning_rate": 4.5146198830409357e-07,
      "logits/chosen": -0.6544944047927856,
      "logits/rejected": NaN,
      "logps/chosen": -1066.1802978515625,
      "logps/rejected": -921.6251220703125,
      "loss": 0.7077,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0015595441218465567,
      "rewards/margins": -0.027076054364442825,
      "rewards/rejected": 0.028635596856474876,
      "step": 713
    },
    {
      "epoch": 0.1878051587236351,
      "grad_norm": 564.0,
      "learning_rate": 4.513157894736842e-07,
      "logits/chosen": -0.6656642556190491,
      "logits/rejected": NaN,
      "logps/chosen": -1266.2113037109375,
      "logps/rejected": -912.82470703125,
      "loss": 0.685,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.015463684685528278,
      "rewards/margins": 0.017262745648622513,
      "rewards/rejected": -0.0017990589840337634,
      "step": 714
    },
    {
      "epoch": 0.18806819115882228,
      "grad_norm": 548.0,
      "learning_rate": 4.511695906432748e-07,
      "logits/chosen": -0.6567980647087097,
      "logits/rejected": -0.6748414039611816,
      "logps/chosen": -974.7615966796875,
      "logps/rejected": -832.1028442382812,
      "loss": 0.6899,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.013705873861908913,
      "rewards/margins": 0.007369612343609333,
      "rewards/rejected": 0.00633625965565443,
      "step": 715
    },
    {
      "epoch": 0.18833122359400944,
      "grad_norm": 540.0,
      "learning_rate": 4.510233918128655e-07,
      "logits/chosen": -0.6632890105247498,
      "logits/rejected": -0.6593891978263855,
      "logps/chosen": -1358.3751220703125,
      "logps/rejected": -960.101318359375,
      "loss": 0.7018,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.007807445712387562,
      "rewards/margins": -0.016212748363614082,
      "rewards/rejected": 0.008405303582549095,
      "step": 716
    },
    {
      "epoch": 0.1885942560291966,
      "grad_norm": 632.0,
      "learning_rate": 4.5087719298245615e-07,
      "logits/chosen": -0.651888370513916,
      "logits/rejected": -0.6606912016868591,
      "logps/chosen": -1213.4718017578125,
      "logps/rejected": -993.6044311523438,
      "loss": 0.6932,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.001909446669742465,
      "rewards/margins": 0.0006975163705646992,
      "rewards/rejected": 0.0012119296006858349,
      "step": 717
    },
    {
      "epoch": 0.18885728846438377,
      "grad_norm": 458.0,
      "learning_rate": 4.507309941520467e-07,
      "logits/chosen": -0.6415412425994873,
      "logits/rejected": -0.6430887579917908,
      "logps/chosen": -1067.627685546875,
      "logps/rejected": -884.4302368164062,
      "loss": 0.6996,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.035043954849243164,
      "rewards/margins": -0.012580870650708675,
      "rewards/rejected": -0.022463083267211914,
      "step": 718
    },
    {
      "epoch": 0.18912032089957093,
      "grad_norm": 600.0,
      "learning_rate": 4.505847953216374e-07,
      "logits/chosen": -0.6531956195831299,
      "logits/rejected": -0.6567687392234802,
      "logps/chosen": -1170.83740234375,
      "logps/rejected": -1174.44482421875,
      "loss": 0.7022,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008985616266727448,
      "rewards/margins": -0.0166334118694067,
      "rewards/rejected": 0.025619031861424446,
      "step": 719
    },
    {
      "epoch": 0.1893833533347581,
      "grad_norm": 620.0,
      "learning_rate": 4.5043859649122807e-07,
      "logits/chosen": -0.6682187914848328,
      "logits/rejected": -0.6611223816871643,
      "logps/chosen": -1412.259033203125,
      "logps/rejected": -1370.46630859375,
      "loss": 0.6816,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.002512836130335927,
      "rewards/margins": 0.024980830028653145,
      "rewards/rejected": -0.022467993199825287,
      "step": 720
    },
    {
      "epoch": 0.18964638576994525,
      "grad_norm": 604.0,
      "learning_rate": 4.502923976608187e-07,
      "logits/chosen": -0.6636388301849365,
      "logits/rejected": -0.6737022399902344,
      "logps/chosen": -1281.5604248046875,
      "logps/rejected": -1121.956787109375,
      "loss": 0.6934,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0012163156643509865,
      "rewards/margins": 0.0007938388735055923,
      "rewards/rejected": -0.002010154537856579,
      "step": 721
    },
    {
      "epoch": 0.1899094182051324,
      "grad_norm": 592.0,
      "learning_rate": 4.501461988304093e-07,
      "logits/chosen": -0.6437097787857056,
      "logits/rejected": -0.6440637707710266,
      "logps/chosen": -1195.6722412109375,
      "logps/rejected": -1155.2144775390625,
      "loss": 0.6829,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.010977460071444511,
      "rewards/margins": 0.022224141284823418,
      "rewards/rejected": -0.011246681213378906,
      "step": 722
    },
    {
      "epoch": 0.1901724506403196,
      "grad_norm": 680.0,
      "learning_rate": 4.5e-07,
      "logits/chosen": -0.6577469110488892,
      "logits/rejected": -0.6582343578338623,
      "logps/chosen": -1428.575439453125,
      "logps/rejected": -1048.0836181640625,
      "loss": 0.6935,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0048691751435399055,
      "rewards/margins": 0.00032844487577676773,
      "rewards/rejected": -0.005197620484977961,
      "step": 723
    },
    {
      "epoch": 0.19043548307550676,
      "grad_norm": 486.0,
      "learning_rate": 4.498538011695906e-07,
      "logits/chosen": -0.6419447660446167,
      "logits/rejected": -0.6344998478889465,
      "logps/chosen": -1232.05029296875,
      "logps/rejected": -1098.9765625,
      "loss": 0.6882,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.006622028537094593,
      "rewards/margins": 0.010558033362030983,
      "rewards/rejected": -0.00393600482493639,
      "step": 724
    },
    {
      "epoch": 0.19069851551069392,
      "grad_norm": 1144.0,
      "learning_rate": 4.4970760233918127e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6619886159896851,
      "logps/chosen": -1231.3988037109375,
      "logps/rejected": -892.8746337890625,
      "loss": 0.7003,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.016280557960271835,
      "rewards/margins": -0.01360244769603014,
      "rewards/rejected": -0.002678109332919121,
      "step": 725
    },
    {
      "epoch": 0.19096154794588108,
      "grad_norm": 568.0,
      "learning_rate": 4.4956140350877195e-07,
      "logits/chosen": -0.6627321243286133,
      "logits/rejected": -0.6621801853179932,
      "logps/chosen": -1134.9246826171875,
      "logps/rejected": -891.5624389648438,
      "loss": 0.6914,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.005432798061519861,
      "rewards/margins": 0.00541000347584486,
      "rewards/rejected": -0.01084280014038086,
      "step": 726
    },
    {
      "epoch": 0.19122458038106824,
      "grad_norm": 612.0,
      "learning_rate": 4.494152046783625e-07,
      "logits/chosen": -0.650719940662384,
      "logits/rejected": -0.6550860404968262,
      "logps/chosen": -1268.548828125,
      "logps/rejected": -927.727783203125,
      "loss": 0.7029,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.011428260244429111,
      "rewards/margins": -0.0180188175290823,
      "rewards/rejected": 0.0065905568189918995,
      "step": 727
    },
    {
      "epoch": 0.1914876128162554,
      "grad_norm": 612.0,
      "learning_rate": 4.492690058479532e-07,
      "logits/chosen": -0.6567432880401611,
      "logits/rejected": -0.6570137143135071,
      "logps/chosen": -1608.3319091796875,
      "logps/rejected": -1406.853515625,
      "loss": 0.693,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0010457991156727076,
      "rewards/margins": 0.0008108147885650396,
      "rewards/rejected": -0.0018566134385764599,
      "step": 728
    },
    {
      "epoch": 0.19175064525144256,
      "grad_norm": 430.0,
      "learning_rate": 4.4912280701754386e-07,
      "logits/chosen": -0.6462405323982239,
      "logits/rejected": -0.639239490032196,
      "logps/chosen": -861.8684692382812,
      "logps/rejected": -764.4345703125,
      "loss": 0.6957,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007321452256292105,
      "rewards/margins": -0.004557703621685505,
      "rewards/rejected": 0.011879157274961472,
      "step": 729
    },
    {
      "epoch": 0.19201367768662972,
      "grad_norm": 552.0,
      "learning_rate": 4.4897660818713453e-07,
      "logits/chosen": -0.6366018056869507,
      "logits/rejected": -0.6372234225273132,
      "logps/chosen": -1258.7919921875,
      "logps/rejected": -1031.4854736328125,
      "loss": 0.6882,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0016983032692223787,
      "rewards/margins": 0.010709955357015133,
      "rewards/rejected": -0.009011650457978249,
      "step": 730
    },
    {
      "epoch": 0.1922767101218169,
      "grad_norm": 680.0,
      "learning_rate": 4.488304093567251e-07,
      "logits/chosen": -0.6532309651374817,
      "logits/rejected": -0.6388663649559021,
      "logps/chosen": -1248.135009765625,
      "logps/rejected": -904.9610595703125,
      "loss": 0.6984,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.004203987307846546,
      "rewards/margins": -0.008976934477686882,
      "rewards/rejected": 0.004772949032485485,
      "step": 731
    },
    {
      "epoch": 0.19253974255700407,
      "grad_norm": 524.0,
      "learning_rate": 4.4868421052631577e-07,
      "logits/chosen": -0.6640313863754272,
      "logits/rejected": -0.6633997559547424,
      "logps/chosen": -1196.6578369140625,
      "logps/rejected": -969.0513916015625,
      "loss": 0.6822,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.014466570690274239,
      "rewards/margins": 0.022891424596309662,
      "rewards/rejected": -0.008424854837357998,
      "step": 732
    },
    {
      "epoch": 0.19280277499219123,
      "grad_norm": 434.0,
      "learning_rate": 4.4853801169590644e-07,
      "logits/chosen": -0.6533514261245728,
      "logits/rejected": -0.6648159027099609,
      "logps/chosen": -753.6069946289062,
      "logps/rejected": -834.5172729492188,
      "loss": 0.6858,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.010428144596517086,
      "rewards/margins": 0.015240194275975227,
      "rewards/rejected": -0.004812050145119429,
      "step": 733
    },
    {
      "epoch": 0.1930658074273784,
      "grad_norm": 524.0,
      "learning_rate": 4.4839181286549706e-07,
      "logits/chosen": -0.6663745045661926,
      "logits/rejected": NaN,
      "logps/chosen": -1184.41650390625,
      "logps/rejected": -948.6272583007812,
      "loss": 0.6989,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.01624603196978569,
      "rewards/margins": -0.010484503582119942,
      "rewards/rejected": -0.005761528387665749,
      "step": 734
    },
    {
      "epoch": 0.19332883986256555,
      "grad_norm": 478.0,
      "learning_rate": 4.482456140350877e-07,
      "logits/chosen": -0.6295549273490906,
      "logits/rejected": -0.6309410929679871,
      "logps/chosen": -947.2687377929688,
      "logps/rejected": -1119.0609130859375,
      "loss": 0.6994,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0020413401070982218,
      "rewards/margins": -0.011728191748261452,
      "rewards/rejected": 0.013769530691206455,
      "step": 735
    },
    {
      "epoch": 0.1935918722977527,
      "grad_norm": 466.0,
      "learning_rate": 4.4809941520467836e-07,
      "logits/chosen": -0.6579045653343201,
      "logits/rejected": -0.6678513288497925,
      "logps/chosen": -1042.34326171875,
      "logps/rejected": -794.25634765625,
      "loss": 0.6903,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.004667567554861307,
      "rewards/margins": 0.006070231087505817,
      "rewards/rejected": -0.0014026642311364412,
      "step": 736
    },
    {
      "epoch": 0.19385490473293987,
      "grad_norm": 540.0,
      "learning_rate": 4.47953216374269e-07,
      "logits/chosen": -0.6484693288803101,
      "logits/rejected": -0.6382700204849243,
      "logps/chosen": -1132.8304443359375,
      "logps/rejected": -794.4981689453125,
      "loss": 0.6854,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.015555668622255325,
      "rewards/margins": 0.01605663262307644,
      "rewards/rejected": -0.0005009651649743319,
      "step": 737
    },
    {
      "epoch": 0.19411793716812706,
      "grad_norm": 588.0,
      "learning_rate": 4.4780701754385965e-07,
      "logits/chosen": -0.6554692387580872,
      "logits/rejected": -0.6605231761932373,
      "logps/chosen": -1005.5405883789062,
      "logps/rejected": -1057.169189453125,
      "loss": 0.6902,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.010391090996563435,
      "rewards/margins": 0.006622648332268,
      "rewards/rejected": 0.0037684431299567223,
      "step": 738
    },
    {
      "epoch": 0.19438096960331422,
      "grad_norm": 502.0,
      "learning_rate": 4.4766081871345027e-07,
      "logits/chosen": -0.6488897800445557,
      "logits/rejected": -0.6644623875617981,
      "logps/chosen": -987.8038940429688,
      "logps/rejected": -719.1437377929688,
      "loss": 0.6865,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.023150920867919922,
      "rewards/margins": 0.013920022174715996,
      "rewards/rejected": 0.009230898693203926,
      "step": 739
    },
    {
      "epoch": 0.19464400203850138,
      "grad_norm": 580.0,
      "learning_rate": 4.475146198830409e-07,
      "logits/chosen": -0.6441448330879211,
      "logits/rejected": -0.6440749168395996,
      "logps/chosen": -1182.17529296875,
      "logps/rejected": -863.3211669921875,
      "loss": 0.7012,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.014169501140713692,
      "rewards/margins": -0.014938638545572758,
      "rewards/rejected": 0.0007691378705203533,
      "step": 740
    },
    {
      "epoch": 0.19490703447368854,
      "grad_norm": 378.0,
      "learning_rate": 4.4736842105263156e-07,
      "logits/chosen": -0.6633780002593994,
      "logits/rejected": -0.6549820899963379,
      "logps/chosen": -800.2330932617188,
      "logps/rejected": -808.9683227539062,
      "loss": 0.6893,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02985982783138752,
      "rewards/margins": 0.008232259191572666,
      "rewards/rejected": 0.02162756957113743,
      "step": 741
    },
    {
      "epoch": 0.1951700669088757,
      "grad_norm": 540.0,
      "learning_rate": 4.4722222222222223e-07,
      "logits/chosen": -0.6439536213874817,
      "logits/rejected": -0.6548182964324951,
      "logps/chosen": -1182.625732421875,
      "logps/rejected": -880.6935424804688,
      "loss": 0.6955,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.009184932336211205,
      "rewards/margins": -0.003823566250503063,
      "rewards/rejected": -0.005361366085708141,
      "step": 742
    },
    {
      "epoch": 0.19543309934406286,
      "grad_norm": 468.0,
      "learning_rate": 4.4707602339181285e-07,
      "logits/chosen": -0.626203715801239,
      "logits/rejected": -0.6265105605125427,
      "logps/chosen": -840.2626342773438,
      "logps/rejected": -726.5039672851562,
      "loss": 0.6951,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.013984489254653454,
      "rewards/margins": -0.0028801916632801294,
      "rewards/rejected": -0.011104298755526543,
      "step": 743
    },
    {
      "epoch": 0.19569613177925002,
      "grad_norm": 752.0,
      "learning_rate": 4.469298245614035e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6619726419448853,
      "logps/chosen": -1041.708740234375,
      "logps/rejected": -1202.5506591796875,
      "loss": 0.6849,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00023250491358339787,
      "rewards/margins": 0.01776285097002983,
      "rewards/rejected": -0.017995359376072884,
      "step": 744
    },
    {
      "epoch": 0.19595916421443718,
      "grad_norm": 580.0,
      "learning_rate": 4.4678362573099415e-07,
      "logits/chosen": -0.6512508988380432,
      "logits/rejected": -0.6562142372131348,
      "logps/chosen": -1407.2183837890625,
      "logps/rejected": -837.1385498046875,
      "loss": 0.6819,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.024277497082948685,
      "rewards/margins": 0.02444915845990181,
      "rewards/rejected": -0.00017166044563055038,
      "step": 745
    },
    {
      "epoch": 0.19622219664962437,
      "grad_norm": 510.0,
      "learning_rate": 4.4663742690058477e-07,
      "logits/chosen": -0.6418107748031616,
      "logits/rejected": -0.655471682548523,
      "logps/chosen": -1149.5992431640625,
      "logps/rejected": -1006.5118408203125,
      "loss": 0.6986,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.011410237289965153,
      "rewards/margins": -0.009966659359633923,
      "rewards/rejected": 0.021376894786953926,
      "step": 746
    },
    {
      "epoch": 0.19648522908481153,
      "grad_norm": 648.0,
      "learning_rate": 4.4649122807017544e-07,
      "logits/chosen": -0.652443528175354,
      "logits/rejected": -0.6511761546134949,
      "logps/chosen": -1016.4933471679688,
      "logps/rejected": -936.0607299804688,
      "loss": 0.6972,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01308898814022541,
      "rewards/margins": -0.006847668439149857,
      "rewards/rejected": -0.006241322495043278,
      "step": 747
    },
    {
      "epoch": 0.1967482615199987,
      "grad_norm": 532.0,
      "learning_rate": 4.4634502923976606e-07,
      "logits/chosen": -0.6608214974403381,
      "logits/rejected": -0.6722173690795898,
      "logps/chosen": -976.24658203125,
      "logps/rejected": -941.0308227539062,
      "loss": 0.6963,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0190398208796978,
      "rewards/margins": -0.005814647302031517,
      "rewards/rejected": -0.013225174508988857,
      "step": 748
    },
    {
      "epoch": 0.19701129395518585,
      "grad_norm": 462.0,
      "learning_rate": 4.461988304093567e-07,
      "logits/chosen": -0.6607558727264404,
      "logits/rejected": -0.6618559956550598,
      "logps/chosen": -974.9990234375,
      "logps/rejected": -816.43505859375,
      "loss": 0.6993,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.004752159584313631,
      "rewards/margins": -0.011489678174257278,
      "rewards/rejected": 0.006737518589943647,
      "step": 749
    },
    {
      "epoch": 0.197274326390373,
      "grad_norm": 374.0,
      "learning_rate": 4.4605263157894735e-07,
      "logits/chosen": -0.6401820182800293,
      "logits/rejected": -0.6480700373649597,
      "logps/chosen": -793.6942749023438,
      "logps/rejected": -504.8950500488281,
      "loss": 0.683,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.026088906452059746,
      "rewards/margins": 0.020896198228001595,
      "rewards/rejected": 0.005192707758396864,
      "step": 750
    },
    {
      "epoch": 0.19753735882556017,
      "grad_norm": 484.0,
      "learning_rate": 4.45906432748538e-07,
      "logits/chosen": -0.6518751978874207,
      "logits/rejected": -0.6601479053497314,
      "logps/chosen": -1399.3961181640625,
      "logps/rejected": -1073.482666015625,
      "loss": 0.6938,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008722878061234951,
      "rewards/margins": -0.0003355985973030329,
      "rewards/rejected": -0.008387279696762562,
      "step": 751
    },
    {
      "epoch": 0.19780039126074733,
      "grad_norm": 516.0,
      "learning_rate": 4.457602339181286e-07,
      "logits/chosen": -0.6242809295654297,
      "logits/rejected": -0.6290614604949951,
      "logps/chosen": -1330.4664306640625,
      "logps/rejected": -1128.166748046875,
      "loss": 0.6833,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.03634987026453018,
      "rewards/margins": 0.021875856444239616,
      "rewards/rejected": 0.014474010095000267,
      "step": 752
    },
    {
      "epoch": 0.1980634236959345,
      "grad_norm": 564.0,
      "learning_rate": 4.4561403508771927e-07,
      "logits/chosen": -0.662363588809967,
      "logits/rejected": -0.6743966937065125,
      "logps/chosen": -1163.9974365234375,
      "logps/rejected": -1000.3201904296875,
      "loss": 0.6957,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006549644749611616,
      "rewards/margins": -0.004045868292450905,
      "rewards/rejected": 0.010595513507723808,
      "step": 753
    },
    {
      "epoch": 0.19832645613112168,
      "grad_norm": 392.0,
      "learning_rate": 4.4546783625730994e-07,
      "logits/chosen": -0.6636279821395874,
      "logits/rejected": -0.6735292673110962,
      "logps/chosen": -1356.413818359375,
      "logps/rejected": -1185.8214111328125,
      "loss": 0.6959,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0030334480106830597,
      "rewards/margins": -0.004495619796216488,
      "rewards/rejected": 0.007529067806899548,
      "step": 754
    },
    {
      "epoch": 0.19858948856630884,
      "grad_norm": 612.0,
      "learning_rate": 4.4532163742690056e-07,
      "logits/chosen": -0.6783542633056641,
      "logits/rejected": -0.6562831997871399,
      "logps/chosen": -972.4913330078125,
      "logps/rejected": -784.241943359375,
      "loss": 0.689,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.012099266052246094,
      "rewards/margins": 0.008960437029600143,
      "rewards/rejected": 0.0031388283241540194,
      "step": 755
    },
    {
      "epoch": 0.198852521001496,
      "grad_norm": 596.0,
      "learning_rate": 4.451754385964912e-07,
      "logits/chosen": -0.6751278638839722,
      "logits/rejected": -0.6803988218307495,
      "logps/chosen": -1084.0009765625,
      "logps/rejected": -970.068359375,
      "loss": 0.6947,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.002829932840541005,
      "rewards/margins": -0.0027263634838163853,
      "rewards/rejected": -0.00010356889106333256,
      "step": 756
    },
    {
      "epoch": 0.19911555343668316,
      "grad_norm": 620.0,
      "learning_rate": 4.4502923976608185e-07,
      "logits/chosen": -0.6520630121231079,
      "logits/rejected": -0.6481403708457947,
      "logps/chosen": -1707.6650390625,
      "logps/rejected": -1283.7020263671875,
      "loss": 0.6948,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0012609483674168587,
      "rewards/margins": -0.002241801004856825,
      "rewards/rejected": 0.003502750303596258,
      "step": 757
    },
    {
      "epoch": 0.19937858587187032,
      "grad_norm": 592.0,
      "learning_rate": 4.4488304093567247e-07,
      "logits/chosen": -0.6630868911743164,
      "logits/rejected": -0.6611250638961792,
      "logps/chosen": -1506.060302734375,
      "logps/rejected": -1275.40966796875,
      "loss": 0.6965,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009127045050263405,
      "rewards/margins": -0.006395722273737192,
      "rewards/rejected": -0.0027313230093568563,
      "step": 758
    },
    {
      "epoch": 0.19964161830705748,
      "grad_norm": 580.0,
      "learning_rate": 4.4473684210526314e-07,
      "logits/chosen": -0.6506256461143494,
      "logits/rejected": -0.662623405456543,
      "logps/chosen": -1325.2635498046875,
      "logps/rejected": -1301.3382568359375,
      "loss": 0.6973,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.004987239371985197,
      "rewards/margins": -0.0076697347685694695,
      "rewards/rejected": 0.0026824946980923414,
      "step": 759
    },
    {
      "epoch": 0.19990465074224464,
      "grad_norm": 544.0,
      "learning_rate": 4.445906432748538e-07,
      "logits/chosen": -0.654667317867279,
      "logits/rejected": -0.6577562689781189,
      "logps/chosen": -1289.4005126953125,
      "logps/rejected": -1067.6058349609375,
      "loss": 0.6834,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.007168580312281847,
      "rewards/margins": 0.02044553868472576,
      "rewards/rejected": -0.013276958838105202,
      "step": 760
    },
    {
      "epoch": 0.2001676831774318,
      "grad_norm": 434.0,
      "learning_rate": 4.444444444444444e-07,
      "logits/chosen": -0.6562015414237976,
      "logits/rejected": -0.6726572513580322,
      "logps/chosen": -882.508544921875,
      "logps/rejected": -815.301513671875,
      "loss": 0.6993,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0002567295450717211,
      "rewards/margins": -0.011734629049897194,
      "rewards/rejected": 0.011477899737656116,
      "step": 761
    },
    {
      "epoch": 0.200430715612619,
      "grad_norm": 524.0,
      "learning_rate": 4.4429824561403506e-07,
      "logits/chosen": -0.6497802734375,
      "logits/rejected": -0.6495064496994019,
      "logps/chosen": -963.5411987304688,
      "logps/rejected": -855.60302734375,
      "loss": 0.686,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007481575012207031,
      "rewards/margins": 0.014880087226629257,
      "rewards/rejected": -0.007398509886115789,
      "step": 762
    },
    {
      "epoch": 0.20069374804780615,
      "grad_norm": 420.0,
      "learning_rate": 4.4415204678362573e-07,
      "logits/chosen": -0.6612011194229126,
      "logits/rejected": -0.6657717227935791,
      "logps/chosen": -1000.9288940429688,
      "logps/rejected": -828.897216796875,
      "loss": 0.7022,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.002003001980483532,
      "rewards/margins": -0.01768532022833824,
      "rewards/rejected": 0.0196883212774992,
      "step": 763
    },
    {
      "epoch": 0.2009567804829933,
      "grad_norm": 536.0,
      "learning_rate": 4.4400584795321635e-07,
      "logits/chosen": -0.6429092288017273,
      "logits/rejected": -0.6403162479400635,
      "logps/chosen": -1122.017333984375,
      "logps/rejected": -586.678955078125,
      "loss": 0.6982,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.009196187369525433,
      "rewards/margins": -0.008974837139248848,
      "rewards/rejected": 0.018171025440096855,
      "step": 764
    },
    {
      "epoch": 0.20121981291818047,
      "grad_norm": 516.0,
      "learning_rate": 4.4385964912280697e-07,
      "logits/chosen": -0.6607496738433838,
      "logits/rejected": -0.6609890460968018,
      "logps/chosen": -1128.817138671875,
      "logps/rejected": -848.3609619140625,
      "loss": 0.6993,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.014628028497099876,
      "rewards/margins": -0.010871696285903454,
      "rewards/rejected": -0.003756331978365779,
      "step": 765
    },
    {
      "epoch": 0.20148284535336763,
      "grad_norm": 462.0,
      "learning_rate": 4.4371345029239764e-07,
      "logits/chosen": -0.6511843800544739,
      "logits/rejected": -0.6483480930328369,
      "logps/chosen": -1081.75244140625,
      "logps/rejected": -805.920654296875,
      "loss": 0.6911,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.014067363925278187,
      "rewards/margins": 0.005776499398052692,
      "rewards/rejected": 0.00829086359590292,
      "step": 766
    },
    {
      "epoch": 0.2017458777885548,
      "grad_norm": 454.0,
      "learning_rate": 4.4356725146198826e-07,
      "logits/chosen": -0.655484676361084,
      "logits/rejected": -0.6580122709274292,
      "logps/chosen": -1070.681396484375,
      "logps/rejected": -833.6152954101562,
      "loss": 0.6867,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.021213959902524948,
      "rewards/margins": 0.013197183609008789,
      "rewards/rejected": 0.008016777224838734,
      "step": 767
    },
    {
      "epoch": 0.20200891022374196,
      "grad_norm": 414.0,
      "learning_rate": 4.4342105263157893e-07,
      "logits/chosen": -0.643466591835022,
      "logits/rejected": -0.6472280025482178,
      "logps/chosen": -859.0689697265625,
      "logps/rejected": -543.5916137695312,
      "loss": 0.6972,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.021432017907500267,
      "rewards/margins": -0.007208443246781826,
      "rewards/rejected": -0.01422357652336359,
      "step": 768
    },
    {
      "epoch": 0.20227194265892912,
      "grad_norm": 352.0,
      "learning_rate": 4.4327485380116955e-07,
      "logits/chosen": -0.6342786550521851,
      "logits/rejected": -0.6500223278999329,
      "logps/chosen": -565.81787109375,
      "logps/rejected": -696.4588623046875,
      "loss": 0.6904,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0099334716796875,
      "rewards/margins": 0.0059356689453125,
      "rewards/rejected": 0.003997802734375,
      "step": 769
    },
    {
      "epoch": 0.2025349750941163,
      "grad_norm": 1112.0,
      "learning_rate": 4.4312865497076023e-07,
      "logits/chosen": -0.6243571043014526,
      "logits/rejected": -0.6671913266181946,
      "logps/chosen": -1536.7562255859375,
      "logps/rejected": -1195.444580078125,
      "loss": 0.6908,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.007890988141298294,
      "rewards/margins": 0.005607223138213158,
      "rewards/rejected": 0.0022837647702544928,
      "step": 770
    },
    {
      "epoch": 0.20279800752930346,
      "grad_norm": 656.0,
      "learning_rate": 4.4298245614035085e-07,
      "logits/chosen": -0.6747786402702332,
      "logits/rejected": -0.6741942763328552,
      "logps/chosen": -1435.6949462890625,
      "logps/rejected": -1307.8994140625,
      "loss": 0.6926,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008592176251113415,
      "rewards/margins": 0.0014653695980086923,
      "rewards/rejected": -0.010057544335722923,
      "step": 771
    },
    {
      "epoch": 0.20306103996449062,
      "grad_norm": 656.0,
      "learning_rate": 4.428362573099415e-07,
      "logits/chosen": -0.640631377696991,
      "logits/rejected": -0.6262953281402588,
      "logps/chosen": -1344.938720703125,
      "logps/rejected": -1047.923828125,
      "loss": 0.6827,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0172819122672081,
      "rewards/margins": 0.02213554084300995,
      "rewards/rejected": -0.004853629972785711,
      "step": 772
    },
    {
      "epoch": 0.20332407239967779,
      "grad_norm": 652.0,
      "learning_rate": 4.426900584795322e-07,
      "logits/chosen": -0.6791410446166992,
      "logits/rejected": -0.681085467338562,
      "logps/chosen": -1542.120849609375,
      "logps/rejected": -1502.9747314453125,
      "loss": 0.6935,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.021631622686982155,
      "rewards/margins": -6.046285852789879e-05,
      "rewards/rejected": -0.02157115936279297,
      "step": 773
    },
    {
      "epoch": 0.20358710483486495,
      "grad_norm": 528.0,
      "learning_rate": 4.4254385964912276e-07,
      "logits/chosen": -0.653032660484314,
      "logits/rejected": -0.6601405739784241,
      "logps/chosen": -1001.2109375,
      "logps/rejected": -845.225830078125,
      "loss": 0.7029,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.018195200711488724,
      "rewards/margins": -0.01795940473675728,
      "rewards/rejected": -0.00023579690605401993,
      "step": 774
    },
    {
      "epoch": 0.2038501372700521,
      "grad_norm": 564.0,
      "learning_rate": 4.4239766081871343e-07,
      "logits/chosen": -0.6239919066429138,
      "logits/rejected": -0.6468684673309326,
      "logps/chosen": -1119.9423828125,
      "logps/rejected": -849.74951171875,
      "loss": 0.7035,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0050171855837106705,
      "rewards/margins": -0.020021725445985794,
      "rewards/rejected": 0.025038909167051315,
      "step": 775
    },
    {
      "epoch": 0.20411316970523927,
      "grad_norm": 560.0,
      "learning_rate": 4.422514619883041e-07,
      "logits/chosen": -0.6459144353866577,
      "logits/rejected": -0.6383178234100342,
      "logps/chosen": -1237.9542236328125,
      "logps/rejected": -1018.4423828125,
      "loss": 0.6701,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.009999371133744717,
      "rewards/margins": 0.047510623931884766,
      "rewards/rejected": -0.03751125559210777,
      "step": 776
    },
    {
      "epoch": 0.20437620214042646,
      "grad_norm": 548.0,
      "learning_rate": 4.421052631578947e-07,
      "logits/chosen": -0.6556012034416199,
      "logits/rejected": NaN,
      "logps/chosen": -1101.35546875,
      "logps/rejected": -815.5090942382812,
      "loss": 0.6985,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0020343780051916838,
      "rewards/margins": -0.009798241779208183,
      "rewards/rejected": 0.007763862609863281,
      "step": 777
    },
    {
      "epoch": 0.20463923457561362,
      "grad_norm": 484.0,
      "learning_rate": 4.4195906432748535e-07,
      "logits/chosen": -0.667720377445221,
      "logits/rejected": -0.6619106531143188,
      "logps/chosen": -845.734375,
      "logps/rejected": -619.7615356445312,
      "loss": 0.6941,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.011107826605439186,
      "rewards/margins": -0.001706599839963019,
      "rewards/rejected": 0.01281442679464817,
      "step": 778
    },
    {
      "epoch": 0.20490226701080078,
      "grad_norm": 532.0,
      "learning_rate": 4.41812865497076e-07,
      "logits/chosen": -0.6645890474319458,
      "logits/rejected": -0.6508137583732605,
      "logps/chosen": -1320.583984375,
      "logps/rejected": -1416.760498046875,
      "loss": 0.6951,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.010963821783661842,
      "rewards/margins": -0.002670479705557227,
      "rewards/rejected": 0.01363429892808199,
      "step": 779
    },
    {
      "epoch": 0.20516529944598794,
      "grad_norm": 512.0,
      "learning_rate": 4.4166666666666664e-07,
      "logits/chosen": -0.6669124364852905,
      "logits/rejected": -0.6750966906547546,
      "logps/chosen": -1118.456298828125,
      "logps/rejected": -1032.2325439453125,
      "loss": 0.7043,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.00792384147644043,
      "rewards/margins": -0.021852826699614525,
      "rewards/rejected": 0.013928985223174095,
      "step": 780
    },
    {
      "epoch": 0.2054283318811751,
      "grad_norm": 438.0,
      "learning_rate": 4.415204678362573e-07,
      "logits/chosen": -0.6737874746322632,
      "logits/rejected": -0.6749814748764038,
      "logps/chosen": -992.33447265625,
      "logps/rejected": -731.0540161132812,
      "loss": 0.7019,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005043649580329657,
      "rewards/margins": -0.016936445608735085,
      "rewards/rejected": 0.011892796494066715,
      "step": 781
    },
    {
      "epoch": 0.20569136431636226,
      "grad_norm": 580.0,
      "learning_rate": 4.4137426900584793e-07,
      "logits/chosen": -0.6496281623840332,
      "logits/rejected": -0.6538664698600769,
      "logps/chosen": -1218.224853515625,
      "logps/rejected": -859.3951416015625,
      "loss": 0.6853,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01923828199505806,
      "rewards/margins": 0.017232799902558327,
      "rewards/rejected": 0.002005481394007802,
      "step": 782
    },
    {
      "epoch": 0.20595439675154942,
      "grad_norm": 784.0,
      "learning_rate": 4.4122807017543855e-07,
      "logits/chosen": -0.6585565209388733,
      "logits/rejected": -0.6549550294876099,
      "logps/chosen": -1339.4005126953125,
      "logps/rejected": -932.4124755859375,
      "loss": 0.6934,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008797741495072842,
      "rewards/margins": 0.0007012840360403061,
      "rewards/rejected": -0.009499025531113148,
      "step": 783
    },
    {
      "epoch": 0.20621742918673658,
      "grad_norm": 492.0,
      "learning_rate": 4.410818713450292e-07,
      "logits/chosen": -0.6271328926086426,
      "logits/rejected": NaN,
      "logps/chosen": -1041.88671875,
      "logps/rejected": -722.9102783203125,
      "loss": 0.6883,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.010249901562929153,
      "rewards/margins": 0.010814474895596504,
      "rewards/rejected": -0.0005645751953125,
      "step": 784
    },
    {
      "epoch": 0.20648046162192377,
      "grad_norm": 616.0,
      "learning_rate": 4.409356725146199e-07,
      "logits/chosen": -0.6531401872634888,
      "logits/rejected": -0.6524772047996521,
      "logps/chosen": -1319.1322021484375,
      "logps/rejected": -1219.7496337890625,
      "loss": 0.6999,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.02205047756433487,
      "rewards/margins": -0.011806774884462357,
      "rewards/rejected": -0.010243702679872513,
      "step": 785
    },
    {
      "epoch": 0.20674349405711093,
      "grad_norm": 564.0,
      "learning_rate": 4.4078947368421046e-07,
      "logits/chosen": -0.644480288028717,
      "logits/rejected": -0.6560184955596924,
      "logps/chosen": -1357.14599609375,
      "logps/rejected": -1019.3382568359375,
      "loss": 0.7081,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.00419883755967021,
      "rewards/margins": -0.02848644368350506,
      "rewards/rejected": 0.032685279846191406,
      "step": 786
    },
    {
      "epoch": 0.2070065264922981,
      "grad_norm": 430.0,
      "learning_rate": 4.4064327485380114e-07,
      "logits/chosen": -0.6442382335662842,
      "logits/rejected": -0.6412817239761353,
      "logps/chosen": -847.58740234375,
      "logps/rejected": -477.5646667480469,
      "loss": 0.6978,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.004938507452607155,
      "rewards/margins": -0.00885624997317791,
      "rewards/rejected": 0.003917742054909468,
      "step": 787
    },
    {
      "epoch": 0.20726955892748525,
      "grad_norm": 540.0,
      "learning_rate": 4.404970760233918e-07,
      "logits/chosen": -0.6335230469703674,
      "logits/rejected": -0.6479284763336182,
      "logps/chosen": -1316.3868408203125,
      "logps/rejected": -716.8987426757812,
      "loss": 0.6969,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0004093172028660774,
      "rewards/margins": -0.007162189576774836,
      "rewards/rejected": 0.006752872373908758,
      "step": 788
    },
    {
      "epoch": 0.2075325913626724,
      "grad_norm": 600.0,
      "learning_rate": 4.4035087719298243e-07,
      "logits/chosen": -0.6707294583320618,
      "logits/rejected": -0.6761391162872314,
      "logps/chosen": -1483.3924560546875,
      "logps/rejected": -1024.822998046875,
      "loss": 0.6972,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0013369559310376644,
      "rewards/margins": -0.007195951417088509,
      "rewards/rejected": 0.008532905951142311,
      "step": 789
    },
    {
      "epoch": 0.20779562379785957,
      "grad_norm": 680.0,
      "learning_rate": 4.402046783625731e-07,
      "logits/chosen": -0.6288598775863647,
      "logits/rejected": -0.6389007568359375,
      "logps/chosen": -1177.2498779296875,
      "logps/rejected": -1263.7783203125,
      "loss": 0.7057,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.005622481927275658,
      "rewards/margins": -0.024268530309200287,
      "rewards/rejected": 0.01864604838192463,
      "step": 790
    },
    {
      "epoch": 0.20805865623304673,
      "grad_norm": 572.0,
      "learning_rate": 4.400584795321637e-07,
      "logits/chosen": -0.635841429233551,
      "logits/rejected": -0.6397991180419922,
      "logps/chosen": -1144.23291015625,
      "logps/rejected": -979.5231323242188,
      "loss": 0.7001,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.009451961144804955,
      "rewards/margins": -0.012460805475711823,
      "rewards/rejected": 0.0030088431667536497,
      "step": 791
    },
    {
      "epoch": 0.2083216886682339,
      "grad_norm": 616.0,
      "learning_rate": 4.3991228070175434e-07,
      "logits/chosen": -0.6779416799545288,
      "logits/rejected": -0.6812231540679932,
      "logps/chosen": -1267.140380859375,
      "logps/rejected": -935.7254638671875,
      "loss": 0.6921,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.008132457733154297,
      "rewards/margins": 0.002804708667099476,
      "rewards/rejected": -0.010937165468931198,
      "step": 792
    },
    {
      "epoch": 0.20858472110342108,
      "grad_norm": 568.0,
      "learning_rate": 4.39766081871345e-07,
      "logits/chosen": -0.622276246547699,
      "logits/rejected": -0.622241199016571,
      "logps/chosen": -1145.3980712890625,
      "logps/rejected": -966.7175903320312,
      "loss": 0.6949,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.012794208712875843,
      "rewards/margins": -0.0027873986400663853,
      "rewards/rejected": -0.01000680960714817,
      "step": 793
    },
    {
      "epoch": 0.20884775353860824,
      "grad_norm": 584.0,
      "learning_rate": 4.396198830409357e-07,
      "logits/chosen": -0.6600505709648132,
      "logits/rejected": -0.6570634841918945,
      "logps/chosen": -948.3492431640625,
      "logps/rejected": -896.6544799804688,
      "loss": 0.7079,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.019662857055664062,
      "rewards/margins": -0.027063990011811256,
      "rewards/rejected": 0.007401132490485907,
      "step": 794
    },
    {
      "epoch": 0.2091107859737954,
      "grad_norm": 616.0,
      "learning_rate": 4.3947368421052625e-07,
      "logits/chosen": -0.6815725564956665,
      "logits/rejected": -0.6699313521385193,
      "logps/chosen": -1416.39599609375,
      "logps/rejected": -795.698486328125,
      "loss": 0.6832,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01911640167236328,
      "rewards/margins": 0.0213604923337698,
      "rewards/rejected": -0.0022440897300839424,
      "step": 795
    },
    {
      "epoch": 0.20937381840898256,
      "grad_norm": 588.0,
      "learning_rate": 4.3932748538011693e-07,
      "logits/chosen": -0.6640876531600952,
      "logits/rejected": -0.6580400466918945,
      "logps/chosen": -1184.756103515625,
      "logps/rejected": -949.47265625,
      "loss": 0.6925,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.021609114482998848,
      "rewards/margins": 0.00255403365008533,
      "rewards/rejected": 0.019055083394050598,
      "step": 796
    },
    {
      "epoch": 0.20963685084416972,
      "grad_norm": 556.0,
      "learning_rate": 4.391812865497076e-07,
      "logits/chosen": -0.6508691310882568,
      "logits/rejected": -0.6495875120162964,
      "logps/chosen": -1087.895263671875,
      "logps/rejected": -1065.6585693359375,
      "loss": 0.6968,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0037169461138546467,
      "rewards/margins": -0.006332587916404009,
      "rewards/rejected": 0.010049534030258656,
      "step": 797
    },
    {
      "epoch": 0.20989988327935688,
      "grad_norm": 430.0,
      "learning_rate": 4.390350877192982e-07,
      "logits/chosen": -0.661754846572876,
      "logits/rejected": -0.6519618034362793,
      "logps/chosen": -861.8496704101562,
      "logps/rejected": -787.765625,
      "loss": 0.6951,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.015801431611180305,
      "rewards/margins": -0.0031321533024311066,
      "rewards/rejected": -0.012669277377426624,
      "step": 798
    },
    {
      "epoch": 0.21016291571454404,
      "grad_norm": 604.0,
      "learning_rate": 4.3888888888888884e-07,
      "logits/chosen": -0.6534398794174194,
      "logits/rejected": -0.656624436378479,
      "logps/chosen": -1505.72900390625,
      "logps/rejected": -1361.83447265625,
      "loss": 0.6951,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.004929160233587027,
      "rewards/margins": -0.003658437170088291,
      "rewards/rejected": -0.0012707235291600227,
      "step": 799
    },
    {
      "epoch": 0.2104259481497312,
      "grad_norm": 532.0,
      "learning_rate": 4.387426900584795e-07,
      "logits/chosen": -0.6393918991088867,
      "logits/rejected": -0.6448347568511963,
      "logps/chosen": -1007.0714721679688,
      "logps/rejected": -870.5504150390625,
      "loss": 0.6971,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.010501575656235218,
      "rewards/margins": -0.007706356234848499,
      "rewards/rejected": -0.0027952194213867188,
      "step": 800
    },
    {
      "epoch": 0.2106889805849184,
      "grad_norm": 524.0,
      "learning_rate": 4.3859649122807013e-07,
      "logits/chosen": -0.6587250828742981,
      "logits/rejected": -0.6646015048027039,
      "logps/chosen": -1500.003662109375,
      "logps/rejected": -1280.10205078125,
      "loss": 0.6872,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.001419639214873314,
      "rewards/margins": 0.012457944452762604,
      "rewards/rejected": -0.013877583667635918,
      "step": 801
    },
    {
      "epoch": 0.21095201302010555,
      "grad_norm": 588.0,
      "learning_rate": 4.384502923976608e-07,
      "logits/chosen": -0.6516584157943726,
      "logits/rejected": -0.6484222412109375,
      "logps/chosen": -1375.5943603515625,
      "logps/rejected": -969.0824584960938,
      "loss": 0.6972,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.002269458957016468,
      "rewards/margins": -0.0063328733667731285,
      "rewards/rejected": 0.004063415341079235,
      "step": 802
    },
    {
      "epoch": 0.2112150454552927,
      "grad_norm": 420.0,
      "learning_rate": 4.383040935672515e-07,
      "logits/chosen": -0.6523887515068054,
      "logits/rejected": -0.6535546779632568,
      "logps/chosen": -865.0643920898438,
      "logps/rejected": -863.5477905273438,
      "loss": 0.6875,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.025189019739627838,
      "rewards/margins": 0.011604071594774723,
      "rewards/rejected": -0.03679309040307999,
      "step": 803
    },
    {
      "epoch": 0.21147807789047987,
      "grad_norm": 612.0,
      "learning_rate": 4.3815789473684205e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6595463752746582,
      "logps/chosen": -1214.68603515625,
      "logps/rejected": -1338.209228515625,
      "loss": 0.6854,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.007104970049113035,
      "rewards/margins": 0.017328547313809395,
      "rewards/rejected": -0.024433517828583717,
      "step": 804
    },
    {
      "epoch": 0.21174111032566703,
      "grad_norm": 548.0,
      "learning_rate": 4.380116959064327e-07,
      "logits/chosen": -0.6252679824829102,
      "logits/rejected": -0.6356551647186279,
      "logps/chosen": -1176.0623779296875,
      "logps/rejected": -1067.6199951171875,
      "loss": 0.6777,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.013387154787778854,
      "rewards/margins": 0.03199901804327965,
      "rewards/rejected": -0.018611861392855644,
      "step": 805
    },
    {
      "epoch": 0.2120041427608542,
      "grad_norm": 592.0,
      "learning_rate": 4.378654970760234e-07,
      "logits/chosen": -0.6704615354537964,
      "logits/rejected": -0.6882315874099731,
      "logps/chosen": -1342.844482421875,
      "logps/rejected": -920.3446655273438,
      "loss": 0.7053,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.012595177628099918,
      "rewards/margins": -0.023678112775087357,
      "rewards/rejected": 0.011082934215664864,
      "step": 806
    },
    {
      "epoch": 0.21226717519604135,
      "grad_norm": 596.0,
      "learning_rate": 4.37719298245614e-07,
      "logits/chosen": -0.6354945302009583,
      "logits/rejected": -0.6310251355171204,
      "logps/chosen": -1242.80908203125,
      "logps/rejected": -1042.005126953125,
      "loss": 0.6892,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.001097202766686678,
      "rewards/margins": 0.008711719885468483,
      "rewards/rejected": -0.00761451805010438,
      "step": 807
    },
    {
      "epoch": 0.2125302076312285,
      "grad_norm": 564.0,
      "learning_rate": 4.3757309941520463e-07,
      "logits/chosen": -0.6573998928070068,
      "logits/rejected": -0.6717562675476074,
      "logps/chosen": -1312.3853759765625,
      "logps/rejected": -694.7807006835938,
      "loss": 0.6965,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003173160832375288,
      "rewards/margins": -0.005718897562474012,
      "rewards/rejected": 0.008892059326171875,
      "step": 808
    },
    {
      "epoch": 0.2127932400664157,
      "grad_norm": 592.0,
      "learning_rate": 4.374269005847953e-07,
      "logits/chosen": -0.6605919599533081,
      "logits/rejected": -0.6615513563156128,
      "logps/chosen": -1317.720703125,
      "logps/rejected": -1137.0740966796875,
      "loss": 0.6886,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01684741862118244,
      "rewards/margins": 0.009630776010453701,
      "rewards/rejected": 0.00721664447337389,
      "step": 809
    },
    {
      "epoch": 0.21305627250160286,
      "grad_norm": 532.0,
      "learning_rate": 4.37280701754386e-07,
      "logits/chosen": -0.6460017561912537,
      "logits/rejected": -0.647057831287384,
      "logps/chosen": -971.1797485351562,
      "logps/rejected": -923.414306640625,
      "loss": 0.6814,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.017192363739013672,
      "rewards/margins": 0.02498035505414009,
      "rewards/rejected": -0.007787990849465132,
      "step": 810
    },
    {
      "epoch": 0.21331930493679002,
      "grad_norm": 504.0,
      "learning_rate": 4.371345029239766e-07,
      "logits/chosen": -0.6526768207550049,
      "logits/rejected": -0.6597069501876831,
      "logps/chosen": -1060.8126220703125,
      "logps/rejected": -990.5165405273438,
      "loss": 0.6955,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.019948100671172142,
      "rewards/margins": -0.003991127014160156,
      "rewards/rejected": -0.015956975519657135,
      "step": 811
    },
    {
      "epoch": 0.21358233737197718,
      "grad_norm": 688.0,
      "learning_rate": 4.369883040935672e-07,
      "logits/chosen": -0.6598244309425354,
      "logits/rejected": -0.654676616191864,
      "logps/chosen": -1656.1378173828125,
      "logps/rejected": -1056.1754150390625,
      "loss": 0.6741,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.019435692578554153,
      "rewards/margins": 0.039788246154785156,
      "rewards/rejected": -0.020352553576231003,
      "step": 812
    },
    {
      "epoch": 0.21384536980716434,
      "grad_norm": 612.0,
      "learning_rate": 4.368421052631579e-07,
      "logits/chosen": -0.6304666996002197,
      "logits/rejected": -0.6402153372764587,
      "logps/chosen": -1372.925537109375,
      "logps/rejected": -971.3444213867188,
      "loss": 0.7109,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.017415620386600494,
      "rewards/margins": -0.033936597406864166,
      "rewards/rejected": 0.016520977020263672,
      "step": 813
    },
    {
      "epoch": 0.2141084022423515,
      "grad_norm": 560.0,
      "learning_rate": 4.366959064327485e-07,
      "logits/chosen": -0.6482072472572327,
      "logits/rejected": -0.661095142364502,
      "logps/chosen": -1213.2178955078125,
      "logps/rejected": -1097.7030029296875,
      "loss": 0.6991,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.017000962048768997,
      "rewards/margins": -0.011106728576123714,
      "rewards/rejected": 0.028107691556215286,
      "step": 814
    },
    {
      "epoch": 0.21437143467753866,
      "grad_norm": 460.0,
      "learning_rate": 4.365497076023392e-07,
      "logits/chosen": -0.667288601398468,
      "logits/rejected": -0.6715567708015442,
      "logps/chosen": -1041.27001953125,
      "logps/rejected": -905.7879638671875,
      "loss": 0.6845,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004284095484763384,
      "rewards/margins": 0.017969941720366478,
      "rewards/rejected": -0.013685845769941807,
      "step": 815
    },
    {
      "epoch": 0.21463446711272585,
      "grad_norm": 836.0,
      "learning_rate": 4.3640350877192985e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6435962319374084,
      "logps/chosen": -776.484130859375,
      "logps/rejected": -854.091552734375,
      "loss": 0.6844,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.032824039459228516,
      "rewards/margins": 0.018055344000458717,
      "rewards/rejected": 0.014768695458769798,
      "step": 816
    },
    {
      "epoch": 0.214897499547913,
      "grad_norm": 480.0,
      "learning_rate": 4.362573099415204e-07,
      "logits/chosen": -0.6440244317054749,
      "logits/rejected": -0.6496699452400208,
      "logps/chosen": -1414.7381591796875,
      "logps/rejected": -1272.3828125,
      "loss": 0.6961,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.003811454400420189,
      "rewards/margins": -0.004800605587661266,
      "rewards/rejected": 0.0009891502559185028,
      "step": 817
    },
    {
      "epoch": 0.21516053198310017,
      "grad_norm": 484.0,
      "learning_rate": 4.361111111111111e-07,
      "logits/chosen": -0.6491208672523499,
      "logits/rejected": -0.6450731754302979,
      "logps/chosen": -1235.13134765625,
      "logps/rejected": -978.9249267578125,
      "loss": 0.6901,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.018970202654600143,
      "rewards/margins": 0.00642766896635294,
      "rewards/rejected": -0.025397872552275658,
      "step": 818
    },
    {
      "epoch": 0.21542356441828733,
      "grad_norm": 492.0,
      "learning_rate": 4.3596491228070177e-07,
      "logits/chosen": -0.6724022030830383,
      "logits/rejected": -0.6660445928573608,
      "logps/chosen": -869.1982421875,
      "logps/rejected": -623.9981689453125,
      "loss": 0.6934,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00846633780747652,
      "rewards/margins": 8.707121014595032e-05,
      "rewards/rejected": -0.00855341088026762,
      "step": 819
    },
    {
      "epoch": 0.2156865968534745,
      "grad_norm": 556.0,
      "learning_rate": 4.358187134502924e-07,
      "logits/chosen": -0.6555745005607605,
      "logits/rejected": -0.6604610681533813,
      "logps/chosen": -894.3395385742188,
      "logps/rejected": -828.9657592773438,
      "loss": 0.6968,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.006514072883874178,
      "rewards/margins": -0.006579923443496227,
      "rewards/rejected": 0.013093994930386543,
      "step": 820
    },
    {
      "epoch": 0.21594962928866165,
      "grad_norm": 568.0,
      "learning_rate": 4.35672514619883e-07,
      "logits/chosen": -0.6635108590126038,
      "logits/rejected": -0.666929304599762,
      "logps/chosen": -1339.51708984375,
      "logps/rejected": -846.65185546875,
      "loss": 0.682,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008404494263231754,
      "rewards/margins": 0.023697232827544212,
      "rewards/rejected": -0.015292741358280182,
      "step": 821
    },
    {
      "epoch": 0.21621266172384881,
      "grad_norm": 560.0,
      "learning_rate": 4.355263157894737e-07,
      "logits/chosen": -0.6399483680725098,
      "logits/rejected": -0.6489025354385376,
      "logps/chosen": -1112.149658203125,
      "logps/rejected": -726.7469482421875,
      "loss": 0.6794,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.021131323650479317,
      "rewards/margins": 0.0282090175896883,
      "rewards/rejected": -0.007077693939208984,
      "step": 822
    },
    {
      "epoch": 0.21647569415903598,
      "grad_norm": 672.0,
      "learning_rate": 4.353801169590643e-07,
      "logits/chosen": -0.6573113799095154,
      "logits/rejected": -0.6652599573135376,
      "logps/chosen": -1316.136474609375,
      "logps/rejected": -1120.9915771484375,
      "loss": 0.7007,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.008861731737852097,
      "rewards/margins": -0.014249132946133614,
      "rewards/rejected": 0.005387401673942804,
      "step": 823
    },
    {
      "epoch": 0.21673872659422316,
      "grad_norm": 452.0,
      "learning_rate": 4.3523391812865497e-07,
      "logits/chosen": -0.6158350706100464,
      "logits/rejected": -0.6180128455162048,
      "logps/chosen": -1051.4276123046875,
      "logps/rejected": -955.781494140625,
      "loss": 0.6995,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0034147254191339016,
      "rewards/margins": -0.011894701980054379,
      "rewards/rejected": 0.008479977026581764,
      "step": 824
    },
    {
      "epoch": 0.21700175902941032,
      "grad_norm": 510.0,
      "learning_rate": 4.350877192982456e-07,
      "logits/chosen": -0.639191746711731,
      "logits/rejected": -0.6472294330596924,
      "logps/chosen": -1090.1175537109375,
      "logps/rejected": -1096.095458984375,
      "loss": 0.6875,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.024121476337313652,
      "rewards/margins": 0.01180953998118639,
      "rewards/rejected": 0.012311935424804688,
      "step": 825
    },
    {
      "epoch": 0.21726479146459748,
      "grad_norm": 490.0,
      "learning_rate": 4.349415204678362e-07,
      "logits/chosen": -0.6592907905578613,
      "logits/rejected": -0.6728937029838562,
      "logps/chosen": -1174.04541015625,
      "logps/rejected": -984.4191284179688,
      "loss": 0.6955,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0015500076115131378,
      "rewards/margins": -0.0037281981203705072,
      "rewards/rejected": 0.0021781926043331623,
      "step": 826
    },
    {
      "epoch": 0.21752782389978464,
      "grad_norm": 568.0,
      "learning_rate": 4.347953216374269e-07,
      "logits/chosen": -0.6489138603210449,
      "logits/rejected": -0.6410279273986816,
      "logps/chosen": -1332.3524169921875,
      "logps/rejected": -1205.3927001953125,
      "loss": 0.7022,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.021468734368681908,
      "rewards/margins": -0.016532326117157936,
      "rewards/rejected": -0.004936409182846546,
      "step": 827
    },
    {
      "epoch": 0.2177908563349718,
      "grad_norm": 772.0,
      "learning_rate": 4.3464912280701756e-07,
      "logits/chosen": -0.6676679849624634,
      "logits/rejected": -0.6774855256080627,
      "logps/chosen": -1192.6904296875,
      "logps/rejected": -1081.0416259765625,
      "loss": 0.698,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0046442970633506775,
      "rewards/margins": -0.008277608081698418,
      "rewards/rejected": 0.012921905145049095,
      "step": 828
    },
    {
      "epoch": 0.21805388877015897,
      "grad_norm": 536.0,
      "learning_rate": 4.345029239766081e-07,
      "logits/chosen": -0.6646314859390259,
      "logits/rejected": -0.6758843660354614,
      "logps/chosen": -1186.0433349609375,
      "logps/rejected": -823.5877075195312,
      "loss": 0.6926,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004502580966800451,
      "rewards/margins": 0.002196693792939186,
      "rewards/rejected": 0.0023058895021677017,
      "step": 829
    },
    {
      "epoch": 0.21831692120534613,
      "grad_norm": 520.0,
      "learning_rate": 4.343567251461988e-07,
      "logits/chosen": -0.6583808660507202,
      "logits/rejected": -0.6648484468460083,
      "logps/chosen": -1036.5050048828125,
      "logps/rejected": -1100.016357421875,
      "loss": 0.6933,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0005438798107206821,
      "rewards/margins": 0.00015287427231669426,
      "rewards/rejected": 0.00039100623689591885,
      "step": 830
    },
    {
      "epoch": 0.2185799536405333,
      "grad_norm": 440.0,
      "learning_rate": 4.3421052631578947e-07,
      "logits/chosen": -0.6431485414505005,
      "logits/rejected": -0.6479921936988831,
      "logps/chosen": -990.8639526367188,
      "logps/rejected": -884.9867553710938,
      "loss": 0.6866,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03700628504157066,
      "rewards/margins": 0.01362905465066433,
      "rewards/rejected": 0.023377228528261185,
      "step": 831
    },
    {
      "epoch": 0.21884298607572047,
      "grad_norm": 572.0,
      "learning_rate": 4.340643274853801e-07,
      "logits/chosen": -0.651874840259552,
      "logits/rejected": -0.6631288528442383,
      "logps/chosen": -1166.5494384765625,
      "logps/rejected": -952.0700073242188,
      "loss": 0.6868,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.016393661499023438,
      "rewards/margins": 0.01311416644603014,
      "rewards/rejected": 0.0032794950529932976,
      "step": 832
    },
    {
      "epoch": 0.21910601851090764,
      "grad_norm": 600.0,
      "learning_rate": 4.3391812865497076e-07,
      "logits/chosen": -0.6335805058479309,
      "logits/rejected": -0.63664710521698,
      "logps/chosen": -1116.4951171875,
      "logps/rejected": -917.7718505859375,
      "loss": 0.6998,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0016927719116210938,
      "rewards/margins": -0.012289906851947308,
      "rewards/rejected": 0.013982677832245827,
      "step": 833
    },
    {
      "epoch": 0.2193690509460948,
      "grad_norm": 474.0,
      "learning_rate": 4.337719298245614e-07,
      "logits/chosen": -0.6464242339134216,
      "logits/rejected": -0.6609668731689453,
      "logps/chosen": -947.1666259765625,
      "logps/rejected": -582.15869140625,
      "loss": 0.6992,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02464446797966957,
      "rewards/margins": -0.011487198062241077,
      "rewards/rejected": -0.013157272711396217,
      "step": 834
    },
    {
      "epoch": 0.21963208338128196,
      "grad_norm": 508.0,
      "learning_rate": 4.33625730994152e-07,
      "logits/chosen": -0.6345809698104858,
      "logits/rejected": -0.6358705759048462,
      "logps/chosen": -1150.98486328125,
      "logps/rejected": -796.6553955078125,
      "loss": 0.6891,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.012774467468261719,
      "rewards/margins": 0.008544063195586205,
      "rewards/rejected": -0.021318530663847923,
      "step": 835
    },
    {
      "epoch": 0.21989511581646912,
      "grad_norm": 580.0,
      "learning_rate": 4.334795321637427e-07,
      "logits/chosen": -0.6494818925857544,
      "logits/rejected": -0.6598688364028931,
      "logps/chosen": -1273.494140625,
      "logps/rejected": -670.211669921875,
      "loss": 0.6859,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.009761713445186615,
      "rewards/margins": 0.015354728326201439,
      "rewards/rejected": -0.005593014881014824,
      "step": 836
    },
    {
      "epoch": 0.22015814825165628,
      "grad_norm": 564.0,
      "learning_rate": 4.3333333333333335e-07,
      "logits/chosen": -0.6567484140396118,
      "logits/rejected": -0.6412634253501892,
      "logps/chosen": -1335.728759765625,
      "logps/rejected": -1254.5638427734375,
      "loss": 0.6922,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.017114736139774323,
      "rewards/margins": 0.003052998334169388,
      "rewards/rejected": 0.014061737805604935,
      "step": 837
    },
    {
      "epoch": 0.22042118068684344,
      "grad_norm": 604.0,
      "learning_rate": 4.331871345029239e-07,
      "logits/chosen": -0.6430237293243408,
      "logits/rejected": -0.63265460729599,
      "logps/chosen": -1254.5537109375,
      "logps/rejected": -893.4974365234375,
      "loss": 0.6911,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.013673638924956322,
      "rewards/margins": 0.0045126439072191715,
      "rewards/rejected": -0.018186284229159355,
      "step": 838
    },
    {
      "epoch": 0.2206842131220306,
      "grad_norm": 612.0,
      "learning_rate": 4.330409356725146e-07,
      "logits/chosen": -0.6317944526672363,
      "logits/rejected": -0.6372591853141785,
      "logps/chosen": -1269.3919677734375,
      "logps/rejected": -990.6099243164062,
      "loss": 0.6821,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.002483939751982689,
      "rewards/margins": 0.023173000663518906,
      "rewards/rejected": -0.02068905718624592,
      "step": 839
    },
    {
      "epoch": 0.2209472455572178,
      "grad_norm": 510.0,
      "learning_rate": 4.3289473684210526e-07,
      "logits/chosen": -0.6344320178031921,
      "logits/rejected": -0.6459878087043762,
      "logps/chosen": -1139.860595703125,
      "logps/rejected": -647.7550048828125,
      "loss": 0.6959,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.01595029979944229,
      "rewards/margins": -0.005015087779611349,
      "rewards/rejected": -0.010935211554169655,
      "step": 840
    },
    {
      "epoch": 0.22121027799240495,
      "grad_norm": 448.0,
      "learning_rate": 4.327485380116959e-07,
      "logits/chosen": -0.6406762599945068,
      "logits/rejected": -0.6433092355728149,
      "logps/chosen": -856.4383544921875,
      "logps/rejected": -708.157958984375,
      "loss": 0.6912,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00624074786901474,
      "rewards/margins": 0.004692078568041325,
      "rewards/rejected": 0.001548671629279852,
      "step": 841
    },
    {
      "epoch": 0.2214733104275921,
      "grad_norm": 478.0,
      "learning_rate": 4.326023391812865e-07,
      "logits/chosen": -0.6576591730117798,
      "logits/rejected": -0.6698178052902222,
      "logps/chosen": -1011.1871337890625,
      "logps/rejected": -721.8387451171875,
      "loss": 0.6793,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.016004038974642754,
      "rewards/margins": 0.02981419861316681,
      "rewards/rejected": -0.01381015870720148,
      "step": 842
    },
    {
      "epoch": 0.22173634286277927,
      "grad_norm": 454.0,
      "learning_rate": 4.324561403508772e-07,
      "logits/chosen": -0.6455374956130981,
      "logits/rejected": -0.6560652256011963,
      "logps/chosen": -972.1410522460938,
      "logps/rejected": -811.5577392578125,
      "loss": 0.6936,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.010145331732928753,
      "rewards/margins": -0.00018439278937876225,
      "rewards/rejected": 0.010329724289476871,
      "step": 843
    },
    {
      "epoch": 0.22199937529796643,
      "grad_norm": 632.0,
      "learning_rate": 4.323099415204678e-07,
      "logits/chosen": -0.6580116152763367,
      "logits/rejected": -0.6607521772384644,
      "logps/chosen": -1508.905029296875,
      "logps/rejected": -966.08740234375,
      "loss": 0.7076,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.008885001763701439,
      "rewards/margins": -0.02775859832763672,
      "rewards/rejected": 0.03664360195398331,
      "step": 844
    },
    {
      "epoch": 0.2222624077331536,
      "grad_norm": 424.0,
      "learning_rate": 4.3216374269005847e-07,
      "logits/chosen": -0.6534767746925354,
      "logits/rejected": -0.667229175567627,
      "logps/chosen": -739.85498046875,
      "logps/rejected": -625.1873779296875,
      "loss": 0.6934,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.010534428991377354,
      "rewards/margins": -3.409339115023613e-05,
      "rewards/rejected": -0.010500336065888405,
      "step": 845
    },
    {
      "epoch": 0.22252544016834075,
      "grad_norm": 596.0,
      "learning_rate": 4.3201754385964914e-07,
      "logits/chosen": -0.6283923387527466,
      "logits/rejected": -0.6176068782806396,
      "logps/chosen": -1472.6639404296875,
      "logps/rejected": -800.1642456054688,
      "loss": 0.6969,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.010673236101865768,
      "rewards/margins": -0.00620799046009779,
      "rewards/rejected": -0.004465246107429266,
      "step": 846
    },
    {
      "epoch": 0.2227884726035279,
      "grad_norm": 540.0,
      "learning_rate": 4.318713450292397e-07,
      "logits/chosen": -0.662883460521698,
      "logits/rejected": -0.6620143055915833,
      "logps/chosen": -1195.2061767578125,
      "logps/rejected": -721.4503784179688,
      "loss": 0.6904,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012880709022283554,
      "rewards/margins": 0.006820966489613056,
      "rewards/rejected": 0.006059742067009211,
      "step": 847
    },
    {
      "epoch": 0.2230515050387151,
      "grad_norm": 608.0,
      "learning_rate": 4.317251461988304e-07,
      "logits/chosen": -0.6520161032676697,
      "logits/rejected": -0.6564663648605347,
      "logps/chosen": -1244.665283203125,
      "logps/rejected": -958.7894287109375,
      "loss": 0.6843,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01604309119284153,
      "rewards/margins": 0.01879758946597576,
      "rewards/rejected": -0.0027544971089810133,
      "step": 848
    },
    {
      "epoch": 0.22331453747390226,
      "grad_norm": 434.0,
      "learning_rate": 4.3157894736842105e-07,
      "logits/chosen": -0.6578177809715271,
      "logits/rejected": -0.6478297710418701,
      "logps/chosen": -1225.79541015625,
      "logps/rejected": -1239.2060546875,
      "loss": 0.6909,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0002681249752640724,
      "rewards/margins": 0.004876280203461647,
      "rewards/rejected": -0.0051444051787257195,
      "step": 849
    },
    {
      "epoch": 0.22357756990908942,
      "grad_norm": 478.0,
      "learning_rate": 4.314327485380117e-07,
      "logits/chosen": -0.6504977345466614,
      "logits/rejected": -0.6721453666687012,
      "logps/chosen": -1000.8265991210938,
      "logps/rejected": -842.8904418945312,
      "loss": 0.6962,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.012708951719105244,
      "rewards/margins": -0.0053910259157419205,
      "rewards/rejected": -0.007317925337702036,
      "step": 850
    },
    {
      "epoch": 0.22384060234427658,
      "grad_norm": 532.0,
      "learning_rate": 4.312865497076023e-07,
      "logits/chosen": -0.6521130204200745,
      "logits/rejected": -0.6558492183685303,
      "logps/chosen": -980.9907836914062,
      "logps/rejected": -762.17724609375,
      "loss": 0.6941,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.013884162530303001,
      "rewards/margins": -0.0013333316892385483,
      "rewards/rejected": 0.01521749421954155,
      "step": 851
    },
    {
      "epoch": 0.22410363477946374,
      "grad_norm": 576.0,
      "learning_rate": 4.3114035087719297e-07,
      "logits/chosen": -0.6386855244636536,
      "logits/rejected": NaN,
      "logps/chosen": -839.327880859375,
      "logps/rejected": -501.22943115234375,
      "loss": 0.7013,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.011155320331454277,
      "rewards/margins": -0.015580891631543636,
      "rewards/rejected": 0.004425574094057083,
      "step": 852
    },
    {
      "epoch": 0.2243666672146509,
      "grad_norm": 540.0,
      "learning_rate": 4.3099415204678364e-07,
      "logits/chosen": -0.667325496673584,
      "logits/rejected": -0.660779595375061,
      "logps/chosen": -1293.87109375,
      "logps/rejected": -893.7847900390625,
      "loss": 0.6891,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02774372324347496,
      "rewards/margins": 0.00879058800637722,
      "rewards/rejected": 0.01895313337445259,
      "step": 853
    },
    {
      "epoch": 0.22462969964983806,
      "grad_norm": 432.0,
      "learning_rate": 4.3084795321637426e-07,
      "logits/chosen": -0.639907956123352,
      "logits/rejected": -0.6407199501991272,
      "logps/chosen": -671.8045043945312,
      "logps/rejected": -598.6275634765625,
      "loss": 0.6921,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.005404090974479914,
      "rewards/margins": 0.003039885312318802,
      "rewards/rejected": -0.008443975821137428,
      "step": 854
    },
    {
      "epoch": 0.22489273208502522,
      "grad_norm": 532.0,
      "learning_rate": 4.307017543859649e-07,
      "logits/chosen": -0.6517372131347656,
      "logits/rejected": NaN,
      "logps/chosen": -1108.2899169921875,
      "logps/rejected": -738.0327758789062,
      "loss": 0.7035,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.016665268689393997,
      "rewards/margins": -0.020205404609441757,
      "rewards/rejected": 0.003540134523063898,
      "step": 855
    },
    {
      "epoch": 0.2251557645202124,
      "grad_norm": 540.0,
      "learning_rate": 4.3055555555555555e-07,
      "logits/chosen": -0.6430526971817017,
      "logits/rejected": -0.6380457878112793,
      "logps/chosen": -1152.6346435546875,
      "logps/rejected": -911.484130859375,
      "loss": 0.6864,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.014775753021240234,
      "rewards/margins": 0.014666080474853516,
      "rewards/rejected": 0.0001096727792173624,
      "step": 856
    },
    {
      "epoch": 0.22541879695539957,
      "grad_norm": 540.0,
      "learning_rate": 4.3040935672514617e-07,
      "logits/chosen": -0.6535139679908752,
      "logits/rejected": -0.6664895415306091,
      "logps/chosen": -1079.5916748046875,
      "logps/rejected": -773.4505615234375,
      "loss": 0.7041,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.025535108521580696,
      "rewards/margins": -0.020915033295750618,
      "rewards/rejected": -0.004620076157152653,
      "step": 857
    },
    {
      "epoch": 0.22568182939058673,
      "grad_norm": 430.0,
      "learning_rate": 4.3026315789473684e-07,
      "logits/chosen": -0.6388133764266968,
      "logits/rejected": -0.6465859413146973,
      "logps/chosen": -731.8033447265625,
      "logps/rejected": -735.6280517578125,
      "loss": 0.6938,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.004616976249963045,
      "rewards/margins": -0.0010282035218551755,
      "rewards/rejected": -0.0035887733101844788,
      "step": 858
    },
    {
      "epoch": 0.2259448618257739,
      "grad_norm": 584.0,
      "learning_rate": 4.3011695906432746e-07,
      "logits/chosen": -0.6762039661407471,
      "logits/rejected": -0.6723034977912903,
      "logps/chosen": -991.0286254882812,
      "logps/rejected": -771.4771728515625,
      "loss": 0.6933,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0018990515964105725,
      "rewards/margins": 0.0003376970998942852,
      "rewards/rejected": -0.002236747881397605,
      "step": 859
    },
    {
      "epoch": 0.22620789426096105,
      "grad_norm": 552.0,
      "learning_rate": 4.299707602339181e-07,
      "logits/chosen": -0.6445938944816589,
      "logits/rejected": -0.6227478981018066,
      "logps/chosen": -1182.7359619140625,
      "logps/rejected": -1095.48095703125,
      "loss": 0.686,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0011880872771143913,
      "rewards/margins": 0.015610980801284313,
      "rewards/rejected": -0.014422893524169922,
      "step": 860
    },
    {
      "epoch": 0.2264709266961482,
      "grad_norm": 470.0,
      "learning_rate": 4.2982456140350876e-07,
      "logits/chosen": -0.6582797169685364,
      "logits/rejected": -0.6576478481292725,
      "logps/chosen": -1273.6416015625,
      "logps/rejected": -998.322021484375,
      "loss": 0.7094,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.02810077741742134,
      "rewards/margins": -0.03157348558306694,
      "rewards/rejected": 0.003472709096968174,
      "step": 861
    },
    {
      "epoch": 0.22673395913133537,
      "grad_norm": 608.0,
      "learning_rate": 4.2967836257309943e-07,
      "logits/chosen": -0.6638142466545105,
      "logits/rejected": -0.6633443236351013,
      "logps/chosen": -1458.6077880859375,
      "logps/rejected": -1306.641845703125,
      "loss": 0.7013,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01209716685116291,
      "rewards/margins": -0.01564807817339897,
      "rewards/rejected": 0.0035509110894054174,
      "step": 862
    },
    {
      "epoch": 0.22699699156652256,
      "grad_norm": 504.0,
      "learning_rate": 4.2953216374269005e-07,
      "logits/chosen": -0.662648618221283,
      "logits/rejected": -0.6688656806945801,
      "logps/chosen": -1151.4718017578125,
      "logps/rejected": -934.0843505859375,
      "loss": 0.6842,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0013099671341478825,
      "rewards/margins": 0.01846475526690483,
      "rewards/rejected": -0.017154788598418236,
      "step": 863
    },
    {
      "epoch": 0.22726002400170972,
      "grad_norm": 544.0,
      "learning_rate": 4.2938596491228067e-07,
      "logits/chosen": -0.6667197942733765,
      "logits/rejected": -0.66978520154953,
      "logps/chosen": -1190.945068359375,
      "logps/rejected": -1257.4901123046875,
      "loss": 0.7048,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.021747158840298653,
      "rewards/margins": -0.022187281399965286,
      "rewards/rejected": 0.04393444210290909,
      "step": 864
    },
    {
      "epoch": 0.22752305643689688,
      "grad_norm": 708.0,
      "learning_rate": 4.2923976608187134e-07,
      "logits/chosen": -0.664799153804779,
      "logits/rejected": -0.6634653806686401,
      "logps/chosen": -1547.351806640625,
      "logps/rejected": -1359.3751220703125,
      "loss": 0.6835,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0076026925817132,
      "rewards/margins": 0.021210718899965286,
      "rewards/rejected": -0.013608025386929512,
      "step": 865
    },
    {
      "epoch": 0.22778608887208404,
      "grad_norm": 660.0,
      "learning_rate": 4.2909356725146196e-07,
      "logits/chosen": -0.6463688611984253,
      "logits/rejected": -0.6361218690872192,
      "logps/chosen": -1500.9267578125,
      "logps/rejected": -1375.2362060546875,
      "loss": 0.6831,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.013908100314438343,
      "rewards/margins": 0.021990297362208366,
      "rewards/rejected": -0.008082199841737747,
      "step": 866
    },
    {
      "epoch": 0.2280491213072712,
      "grad_norm": 688.0,
      "learning_rate": 4.2894736842105263e-07,
      "logits/chosen": -0.6766576170921326,
      "logits/rejected": -0.6635469198226929,
      "logps/chosen": -869.7906494140625,
      "logps/rejected": -720.0200805664062,
      "loss": 0.6909,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02160034142434597,
      "rewards/margins": 0.005223417654633522,
      "rewards/rejected": 0.016376923769712448,
      "step": 867
    },
    {
      "epoch": 0.22831215374245836,
      "grad_norm": 624.0,
      "learning_rate": 4.2880116959064325e-07,
      "logits/chosen": -0.6601732969284058,
      "logits/rejected": -0.6710987687110901,
      "logps/chosen": -1639.6187744140625,
      "logps/rejected": -1316.950927734375,
      "loss": 0.6922,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.0002429969608783722,
      "rewards/margins": 0.0027555469423532486,
      "rewards/rejected": -0.0025125492829829454,
      "step": 868
    },
    {
      "epoch": 0.22857518617764552,
      "grad_norm": 556.0,
      "learning_rate": 4.286549707602339e-07,
      "logits/chosen": -0.6712678670883179,
      "logits/rejected": -0.662216305732727,
      "logps/chosen": -953.675537109375,
      "logps/rejected": -950.01953125,
      "loss": 0.7002,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.020055104047060013,
      "rewards/margins": -0.012712049297988415,
      "rewards/rejected": -0.007343054283410311,
      "step": 869
    },
    {
      "epoch": 0.22883821861283268,
      "grad_norm": 468.0,
      "learning_rate": 4.2850877192982455e-07,
      "logits/chosen": -0.6149875521659851,
      "logits/rejected": -0.6250731945037842,
      "logps/chosen": -876.5697631835938,
      "logps/rejected": -823.7353515625,
      "loss": 0.695,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -4.6108849346637726e-05,
      "rewards/margins": -0.0031315800733864307,
      "rewards/rejected": 0.003085470525547862,
      "step": 870
    },
    {
      "epoch": 0.22910125104801987,
      "grad_norm": 462.0,
      "learning_rate": 4.283625730994152e-07,
      "logits/chosen": -0.6835023164749146,
      "logits/rejected": -0.6922206282615662,
      "logps/chosen": -1339.838134765625,
      "logps/rejected": -1100.82421875,
      "loss": 0.6925,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.005047225393354893,
      "rewards/margins": 0.001970673445612192,
      "rewards/rejected": -0.007017899304628372,
      "step": 871
    },
    {
      "epoch": 0.22936428348320703,
      "grad_norm": 620.0,
      "learning_rate": 4.282163742690058e-07,
      "logits/chosen": -0.635147213935852,
      "logits/rejected": NaN,
      "logps/chosen": -1394.3463134765625,
      "logps/rejected": -1088.1103515625,
      "loss": 0.6871,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.016169453039765358,
      "rewards/margins": 0.012930582277476788,
      "rewards/rejected": 0.0032388679683208466,
      "step": 872
    },
    {
      "epoch": 0.2296273159183942,
      "grad_norm": 492.0,
      "learning_rate": 4.2807017543859646e-07,
      "logits/chosen": -0.6574622392654419,
      "logits/rejected": -0.656360924243927,
      "logps/chosen": -1160.81982421875,
      "logps/rejected": -779.7403564453125,
      "loss": 0.6934,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0076669687405228615,
      "rewards/margins": -1.5163328498601913e-05,
      "rewards/rejected": 0.007682132534682751,
      "step": 873
    },
    {
      "epoch": 0.22989034835358135,
      "grad_norm": 540.0,
      "learning_rate": 4.2792397660818713e-07,
      "logits/chosen": -0.6481013894081116,
      "logits/rejected": NaN,
      "logps/chosen": -1142.72998046875,
      "logps/rejected": -1125.4716796875,
      "loss": 0.6951,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006975936703383923,
      "rewards/margins": -0.0035740849561989307,
      "rewards/rejected": 0.010550020262598991,
      "step": 874
    },
    {
      "epoch": 0.2301533807887685,
      "grad_norm": 454.0,
      "learning_rate": 4.2777777777777775e-07,
      "logits/chosen": -0.6659290790557861,
      "logits/rejected": -0.6688430309295654,
      "logps/chosen": -1238.548828125,
      "logps/rejected": -1081.14697265625,
      "loss": 0.6925,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.004638958722352982,
      "rewards/margins": 0.001966094598174095,
      "rewards/rejected": 0.0026728627271950245,
      "step": 875
    },
    {
      "epoch": 0.23041641322395567,
      "grad_norm": 748.0,
      "learning_rate": 4.2763157894736837e-07,
      "logits/chosen": -0.646831750869751,
      "logits/rejected": NaN,
      "logps/chosen": -1497.468994140625,
      "logps/rejected": -917.7943115234375,
      "loss": 0.6785,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.018980028107762337,
      "rewards/margins": 0.031047439202666283,
      "rewards/rejected": -0.01206741388887167,
      "step": 876
    },
    {
      "epoch": 0.23067944565914283,
      "grad_norm": 624.0,
      "learning_rate": 4.2748538011695905e-07,
      "logits/chosen": -0.6727848649024963,
      "logits/rejected": NaN,
      "logps/chosen": -1582.393798828125,
      "logps/rejected": -1130.9178466796875,
      "loss": 0.7023,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0020991319324821234,
      "rewards/margins": -0.01650114543735981,
      "rewards/rejected": 0.01860027387738228,
      "step": 877
    },
    {
      "epoch": 0.23094247809433,
      "grad_norm": 506.0,
      "learning_rate": 4.2733918128654967e-07,
      "logits/chosen": -0.6576783657073975,
      "logits/rejected": -0.6583809852600098,
      "logps/chosen": -1010.55419921875,
      "logps/rejected": -989.2208251953125,
      "loss": 0.7011,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.014603041112422943,
      "rewards/margins": -0.01447906345129013,
      "rewards/rejected": -0.00012397835962474346,
      "step": 878
    },
    {
      "epoch": 0.23120551052951718,
      "grad_norm": 552.0,
      "learning_rate": 4.2719298245614034e-07,
      "logits/chosen": -0.6486835479736328,
      "logits/rejected": -0.6548222303390503,
      "logps/chosen": -1397.095703125,
      "logps/rejected": -1145.6754150390625,
      "loss": 0.689,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02333507500588894,
      "rewards/margins": 0.009081650525331497,
      "rewards/rejected": 0.014253425411880016,
      "step": 879
    },
    {
      "epoch": 0.23146854296470434,
      "grad_norm": 668.0,
      "learning_rate": 4.27046783625731e-07,
      "logits/chosen": -0.6430803537368774,
      "logits/rejected": -0.639268159866333,
      "logps/chosen": -1352.3389892578125,
      "logps/rejected": -1075.939453125,
      "loss": 0.691,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00018196133896708488,
      "rewards/margins": 0.004814528860151768,
      "rewards/rejected": -0.004996491130441427,
      "step": 880
    },
    {
      "epoch": 0.2317315753998915,
      "grad_norm": 474.0,
      "learning_rate": 4.269005847953216e-07,
      "logits/chosen": -0.6355385184288025,
      "logits/rejected": -0.6339321136474609,
      "logps/chosen": -1258.9385986328125,
      "logps/rejected": -1187.7784423828125,
      "loss": 0.6922,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.027421284466981888,
      "rewards/margins": 0.0024227621033787727,
      "rewards/rejected": 0.02499852329492569,
      "step": 881
    },
    {
      "epoch": 0.23199460783507866,
      "grad_norm": 560.0,
      "learning_rate": 4.2675438596491225e-07,
      "logits/chosen": -0.6527127623558044,
      "logits/rejected": -0.6623032093048096,
      "logps/chosen": -1314.462890625,
      "logps/rejected": -1213.800537109375,
      "loss": 0.6917,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.011784171685576439,
      "rewards/margins": 0.003445433219894767,
      "rewards/rejected": 0.008338737301528454,
      "step": 882
    },
    {
      "epoch": 0.23225764027026582,
      "grad_norm": 512.0,
      "learning_rate": 4.266081871345029e-07,
      "logits/chosen": -0.6838600635528564,
      "logits/rejected": -0.6944271326065063,
      "logps/chosen": -1225.80224609375,
      "logps/rejected": -1031.6514892578125,
      "loss": 0.6951,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.01706400141119957,
      "rewards/margins": -0.002279376145452261,
      "rewards/rejected": -0.014784622937440872,
      "step": 883
    },
    {
      "epoch": 0.23252067270545299,
      "grad_norm": 648.0,
      "learning_rate": 4.2646198830409354e-07,
      "logits/chosen": -0.6693555116653442,
      "logits/rejected": -0.6646584868431091,
      "logps/chosen": -1381.623291015625,
      "logps/rejected": -904.5198974609375,
      "loss": 0.6853,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009057236835360527,
      "rewards/margins": 0.016524124890565872,
      "rewards/rejected": -0.007466889452189207,
      "step": 884
    },
    {
      "epoch": 0.23278370514064015,
      "grad_norm": 580.0,
      "learning_rate": 4.2631578947368416e-07,
      "logits/chosen": -0.6598042845726013,
      "logits/rejected": -0.6704519391059875,
      "logps/chosen": -1040.103515625,
      "logps/rejected": -947.5610961914062,
      "loss": 0.6976,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.007202816661447287,
      "rewards/margins": -0.00847549457103014,
      "rewards/rejected": 0.0012726783752441406,
      "step": 885
    },
    {
      "epoch": 0.2330467375758273,
      "grad_norm": 596.0,
      "learning_rate": 4.2616959064327484e-07,
      "logits/chosen": -0.6540853381156921,
      "logits/rejected": -0.650750458240509,
      "logps/chosen": -1544.2713623046875,
      "logps/rejected": -1200.158447265625,
      "loss": 0.6998,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.025407982990145683,
      "rewards/margins": -0.012563228607177734,
      "rewards/rejected": -0.012844755314290524,
      "step": 886
    },
    {
      "epoch": 0.2333097700110145,
      "grad_norm": 596.0,
      "learning_rate": 4.2602339181286546e-07,
      "logits/chosen": -0.6289834976196289,
      "logits/rejected": -0.6347373723983765,
      "logps/chosen": -1499.5792236328125,
      "logps/rejected": -1215.696533203125,
      "loss": 0.7087,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.021971799433231354,
      "rewards/margins": -0.029514122754335403,
      "rewards/rejected": 0.007542324252426624,
      "step": 887
    },
    {
      "epoch": 0.23357280244620165,
      "grad_norm": 472.0,
      "learning_rate": 4.2587719298245613e-07,
      "logits/chosen": -0.6565306186676025,
      "logits/rejected": -0.6657038927078247,
      "logps/chosen": -1331.9046630859375,
      "logps/rejected": -1096.242431640625,
      "loss": 0.6937,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.003745271125808358,
      "rewards/margins": 0.00012912601232528687,
      "rewards/rejected": -0.0038743980694562197,
      "step": 888
    },
    {
      "epoch": 0.23383583488138882,
      "grad_norm": 484.0,
      "learning_rate": 4.2573099415204675e-07,
      "logits/chosen": -0.654082179069519,
      "logits/rejected": -0.6524882316589355,
      "logps/chosen": -922.2520751953125,
      "logps/rejected": -718.129638671875,
      "loss": 0.7006,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.017943812534213066,
      "rewards/margins": -0.014629794284701347,
      "rewards/rejected": -0.003314018016681075,
      "step": 889
    },
    {
      "epoch": 0.23409886731657598,
      "grad_norm": 676.0,
      "learning_rate": 4.255847953216374e-07,
      "logits/chosen": -0.6540122628211975,
      "logits/rejected": -0.6478042006492615,
      "logps/chosen": -1453.74365234375,
      "logps/rejected": -1380.8197021484375,
      "loss": 0.6921,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.003359222551807761,
      "rewards/margins": 0.003178692189976573,
      "rewards/rejected": 0.00018053071107715368,
      "step": 890
    },
    {
      "epoch": 0.23436189975176314,
      "grad_norm": 500.0,
      "learning_rate": 4.2543859649122804e-07,
      "logits/chosen": -0.6470010280609131,
      "logits/rejected": -0.6519954204559326,
      "logps/chosen": -1049.544677734375,
      "logps/rejected": -658.0455932617188,
      "loss": 0.6975,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.006639384664595127,
      "rewards/margins": -0.00733375595882535,
      "rewards/rejected": 0.0006943720509298146,
      "step": 891
    },
    {
      "epoch": 0.2346249321869503,
      "grad_norm": 488.0,
      "learning_rate": 4.252923976608187e-07,
      "logits/chosen": -0.6382570266723633,
      "logits/rejected": -0.6391891837120056,
      "logps/chosen": -1119.5345458984375,
      "logps/rejected": -826.5848388671875,
      "loss": 0.6872,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0076551432721316814,
      "rewards/margins": 0.012663744390010834,
      "rewards/rejected": -0.005008603446185589,
      "step": 892
    },
    {
      "epoch": 0.23488796462213746,
      "grad_norm": 498.0,
      "learning_rate": 4.251461988304094e-07,
      "logits/chosen": -0.6261445879936218,
      "logits/rejected": -0.631615161895752,
      "logps/chosen": -922.0223999023438,
      "logps/rejected": -752.93798828125,
      "loss": 0.6892,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.002913952339440584,
      "rewards/margins": 0.00881900917738676,
      "rewards/rejected": -0.005905057303607464,
      "step": 893
    },
    {
      "epoch": 0.23515099705732462,
      "grad_norm": 592.0,
      "learning_rate": 4.2499999999999995e-07,
      "logits/chosen": -0.6493598222732544,
      "logits/rejected": -0.6565423011779785,
      "logps/chosen": -1426.53466796875,
      "logps/rejected": -1058.2913818359375,
      "loss": 0.6999,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.00048484839498996735,
      "rewards/margins": -0.012539099901914597,
      "rewards/rejected": 0.012054253369569778,
      "step": 894
    },
    {
      "epoch": 0.2354140294925118,
      "grad_norm": 524.0,
      "learning_rate": 4.2485380116959063e-07,
      "logits/chosen": -0.6413501501083374,
      "logits/rejected": -0.6431332230567932,
      "logps/chosen": -1127.4658203125,
      "logps/rejected": -1139.112060546875,
      "loss": 0.6979,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.027405595406889915,
      "rewards/margins": -0.009227896109223366,
      "rewards/rejected": -0.018177703022956848,
      "step": 895
    },
    {
      "epoch": 0.23567706192769897,
      "grad_norm": 592.0,
      "learning_rate": 4.247076023391813e-07,
      "logits/chosen": -0.6376802921295166,
      "logits/rejected": -0.6481179594993591,
      "logps/chosen": -1455.93359375,
      "logps/rejected": -967.8805541992188,
      "loss": 0.6944,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0018619541078805923,
      "rewards/margins": -0.0009324084967374802,
      "rewards/rejected": 0.002794360974803567,
      "step": 896
    },
    {
      "epoch": 0.23594009436288613,
      "grad_norm": 418.0,
      "learning_rate": 4.245614035087719e-07,
      "logits/chosen": -0.6587619185447693,
      "logits/rejected": -0.6658751368522644,
      "logps/chosen": -759.63916015625,
      "logps/rejected": -556.5771484375,
      "loss": 0.6879,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0028210640884935856,
      "rewards/margins": 0.011132908053696156,
      "rewards/rejected": -0.008311844430863857,
      "step": 897
    },
    {
      "epoch": 0.2362031267980733,
      "grad_norm": 672.0,
      "learning_rate": 4.2441520467836254e-07,
      "logits/chosen": -0.6709924936294556,
      "logits/rejected": -0.6621079444885254,
      "logps/chosen": -1528.0504150390625,
      "logps/rejected": -1027.7425537109375,
      "loss": 0.68,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.00840912014245987,
      "rewards/margins": 0.027515795081853867,
      "rewards/rejected": -0.019106673076748848,
      "step": 898
    },
    {
      "epoch": 0.23646615923326045,
      "grad_norm": 544.0,
      "learning_rate": 4.242690058479532e-07,
      "logits/chosen": -0.6478431224822998,
      "logits/rejected": -0.6515421271324158,
      "logps/chosen": -947.7338256835938,
      "logps/rejected": -1034.696533203125,
      "loss": 0.695,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01071920432150364,
      "rewards/margins": -0.0034603122621774673,
      "rewards/rejected": -0.007258892059326172,
      "step": 899
    },
    {
      "epoch": 0.2367291916684476,
      "grad_norm": 532.0,
      "learning_rate": 4.2412280701754383e-07,
      "logits/chosen": -0.6486972570419312,
      "logits/rejected": -0.6548302173614502,
      "logps/chosen": -1032.2056884765625,
      "logps/rejected": -1009.3548583984375,
      "loss": 0.7034,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.00094594806432724,
      "rewards/margins": -0.019376564770936966,
      "rewards/rejected": 0.020322512835264206,
      "step": 900
    },
    {
      "epoch": 0.23699222410363477,
      "grad_norm": 572.0,
      "learning_rate": 4.239766081871345e-07,
      "logits/chosen": -0.6554297208786011,
      "logits/rejected": -0.6626624464988708,
      "logps/chosen": -1196.591064453125,
      "logps/rejected": -852.71435546875,
      "loss": 0.6862,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.017056085169315338,
      "rewards/margins": 0.014958571642637253,
      "rewards/rejected": 0.0020975121296942234,
      "step": 901
    },
    {
      "epoch": 0.23725525653882196,
      "grad_norm": 528.0,
      "learning_rate": 4.238304093567251e-07,
      "logits/chosen": -0.6441904306411743,
      "logits/rejected": -0.6509107947349548,
      "logps/chosen": -1227.4884033203125,
      "logps/rejected": -1006.7327880859375,
      "loss": 0.6806,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.018128108233213425,
      "rewards/margins": 0.02617359533905983,
      "rewards/rejected": -0.008045483380556107,
      "step": 902
    },
    {
      "epoch": 0.23751828897400912,
      "grad_norm": 498.0,
      "learning_rate": 4.2368421052631575e-07,
      "logits/chosen": -0.6471742391586304,
      "logits/rejected": -0.6523598432540894,
      "logps/chosen": -1134.4337158203125,
      "logps/rejected": -944.07080078125,
      "loss": 0.6888,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01704397238790989,
      "rewards/margins": 0.009661389514803886,
      "rewards/rejected": 0.007382584735751152,
      "step": 903
    },
    {
      "epoch": 0.23778132140919628,
      "grad_norm": 532.0,
      "learning_rate": 4.235380116959064e-07,
      "logits/chosen": -0.6521209478378296,
      "logits/rejected": -0.6468965411186218,
      "logps/chosen": -1010.8550415039062,
      "logps/rejected": -633.4574584960938,
      "loss": 0.6805,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.014462423510849476,
      "rewards/margins": 0.025991681963205338,
      "rewards/rejected": -0.011529255658388138,
      "step": 904
    },
    {
      "epoch": 0.23804435384438344,
      "grad_norm": 552.0,
      "learning_rate": 4.233918128654971e-07,
      "logits/chosen": -0.6457709074020386,
      "logits/rejected": -0.6566892266273499,
      "logps/chosen": -1042.475341796875,
      "logps/rejected": -723.513671875,
      "loss": 0.6858,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.00287551898509264,
      "rewards/margins": 0.014894009567797184,
      "rewards/rejected": -0.017769526690244675,
      "step": 905
    },
    {
      "epoch": 0.2383073862795706,
      "grad_norm": 628.0,
      "learning_rate": 4.2324561403508766e-07,
      "logits/chosen": -0.6626263856887817,
      "logits/rejected": -0.6586578488349915,
      "logps/chosen": -1045.025634765625,
      "logps/rejected": -992.6514892578125,
      "loss": 0.6946,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.007508181966841221,
      "rewards/margins": -0.0018329117447137833,
      "rewards/rejected": -0.005675267893821001,
      "step": 906
    },
    {
      "epoch": 0.23857041871475776,
      "grad_norm": 524.0,
      "learning_rate": 4.2309941520467833e-07,
      "logits/chosen": -0.6580490469932556,
      "logits/rejected": -0.6593243479728699,
      "logps/chosen": -1058.534912109375,
      "logps/rejected": -1046.4051513671875,
      "loss": 0.6807,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.021741963922977448,
      "rewards/margins": 0.026550674811005592,
      "rewards/rejected": -0.004808712285012007,
      "step": 907
    },
    {
      "epoch": 0.23883345114994492,
      "grad_norm": 564.0,
      "learning_rate": 4.22953216374269e-07,
      "logits/chosen": -0.6293663382530212,
      "logits/rejected": -0.6408403515815735,
      "logps/chosen": -1139.6544189453125,
      "logps/rejected": -687.2424926757812,
      "loss": 0.6902,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.003840065561234951,
      "rewards/margins": 0.006707476917654276,
      "rewards/rejected": -0.0028674122877418995,
      "step": 908
    },
    {
      "epoch": 0.23909648358513208,
      "grad_norm": 588.0,
      "learning_rate": 4.228070175438596e-07,
      "logits/chosen": -0.6104028820991516,
      "logits/rejected": -0.6132823824882507,
      "logps/chosen": -1629.984375,
      "logps/rejected": -1452.47119140625,
      "loss": 0.6886,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.017409514635801315,
      "rewards/margins": 0.009449196048080921,
      "rewards/rejected": 0.007960320450365543,
      "step": 909
    },
    {
      "epoch": 0.23935951602031927,
      "grad_norm": 524.0,
      "learning_rate": 4.226608187134503e-07,
      "logits/chosen": -0.6668891310691833,
      "logits/rejected": -0.6703274250030518,
      "logps/chosen": -1148.832275390625,
      "logps/rejected": -779.6400146484375,
      "loss": 0.6898,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0009624478407204151,
      "rewards/margins": 0.007446479983627796,
      "rewards/rejected": -0.008408928290009499,
      "step": 910
    },
    {
      "epoch": 0.23962254845550643,
      "grad_norm": 564.0,
      "learning_rate": 4.225146198830409e-07,
      "logits/chosen": -0.6619659066200256,
      "logits/rejected": -0.6683390140533447,
      "logps/chosen": -1514.9395751953125,
      "logps/rejected": -1213.4749755859375,
      "loss": 0.7015,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.021883107721805573,
      "rewards/margins": -0.015762709081172943,
      "rewards/rejected": -0.006120396312326193,
      "step": 911
    },
    {
      "epoch": 0.2398855808906936,
      "grad_norm": 492.0,
      "learning_rate": 4.2236842105263154e-07,
      "logits/chosen": -0.6570926904678345,
      "logits/rejected": -0.6619292497634888,
      "logps/chosen": -747.0951538085938,
      "logps/rejected": -921.114990234375,
      "loss": 0.6895,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0025812152307480574,
      "rewards/margins": 0.007744503207504749,
      "rewards/rejected": -0.005163287743926048,
      "step": 912
    },
    {
      "epoch": 0.24014861332588075,
      "grad_norm": 612.0,
      "learning_rate": 4.222222222222222e-07,
      "logits/chosen": -0.6602206826210022,
      "logits/rejected": -0.6631293892860413,
      "logps/chosen": -1400.0423583984375,
      "logps/rejected": -1108.2841796875,
      "loss": 0.689,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00417375611141324,
      "rewards/margins": 0.009189318865537643,
      "rewards/rejected": -0.005015564151108265,
      "step": 913
    },
    {
      "epoch": 0.2404116457610679,
      "grad_norm": 470.0,
      "learning_rate": 4.220760233918129e-07,
      "logits/chosen": -0.6397500038146973,
      "logits/rejected": -0.6364994049072266,
      "logps/chosen": -831.3400268554688,
      "logps/rejected": -708.343994140625,
      "loss": 0.6972,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.013440608978271484,
      "rewards/margins": -0.007079507224261761,
      "rewards/rejected": -0.006361103616654873,
      "step": 914
    },
    {
      "epoch": 0.24067467819625507,
      "grad_norm": 528.0,
      "learning_rate": 4.2192982456140345e-07,
      "logits/chosen": -0.6338838934898376,
      "logits/rejected": -0.6407709121704102,
      "logps/chosen": -1121.96044921875,
      "logps/rejected": -966.068359375,
      "loss": 0.6932,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01351394783705473,
      "rewards/margins": 0.0006361952982842922,
      "rewards/rejected": -0.014150142669677734,
      "step": 915
    },
    {
      "epoch": 0.24093771063144223,
      "grad_norm": 458.0,
      "learning_rate": 4.217836257309941e-07,
      "logits/chosen": -0.6421201825141907,
      "logits/rejected": -0.6473686099052429,
      "logps/chosen": -1057.8087158203125,
      "logps/rejected": -633.873291015625,
      "loss": 0.6951,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005066298879683018,
      "rewards/margins": -0.0032515525817871094,
      "rewards/rejected": -0.001814747229218483,
      "step": 916
    },
    {
      "epoch": 0.2412007430666294,
      "grad_norm": 488.0,
      "learning_rate": 4.216374269005848e-07,
      "logits/chosen": -0.6728191375732422,
      "logits/rejected": -0.6737523674964905,
      "logps/chosen": -1193.1923828125,
      "logps/rejected": -1050.3173828125,
      "loss": 0.6863,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.011368656530976295,
      "rewards/margins": 0.014731883071362972,
      "rewards/rejected": -0.003363228403031826,
      "step": 917
    },
    {
      "epoch": 0.24146377550181658,
      "grad_norm": 422.0,
      "learning_rate": 4.214912280701754e-07,
      "logits/chosen": -0.6181214451789856,
      "logits/rejected": -0.6188530921936035,
      "logps/chosen": -835.8681640625,
      "logps/rejected": -850.8530883789062,
      "loss": 0.6929,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.00398182962089777,
      "rewards/margins": 0.000859689200296998,
      "rewards/rejected": 0.003122139722108841,
      "step": 918
    },
    {
      "epoch": 0.24172680793700374,
      "grad_norm": 564.0,
      "learning_rate": 4.2134502923976603e-07,
      "logits/chosen": -0.6256563663482666,
      "logits/rejected": -0.6312570571899414,
      "logps/chosen": -947.5220947265625,
      "logps/rejected": -988.0272827148438,
      "loss": 0.6938,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.013106966391205788,
      "rewards/margins": -0.0005857956130057573,
      "rewards/rejected": -0.012521171942353249,
      "step": 919
    },
    {
      "epoch": 0.2419898403721909,
      "grad_norm": 604.0,
      "learning_rate": 4.211988304093567e-07,
      "logits/chosen": -0.6546767354011536,
      "logits/rejected": -0.6532192230224609,
      "logps/chosen": -1046.8782958984375,
      "logps/rejected": -1314.2176513671875,
      "loss": 0.6965,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.003243350423872471,
      "rewards/margins": -0.005753516685217619,
      "rewards/rejected": 0.0025101657956838608,
      "step": 920
    },
    {
      "epoch": 0.24225287280737806,
      "grad_norm": 636.0,
      "learning_rate": 4.2105263157894733e-07,
      "logits/chosen": -0.6299022436141968,
      "logits/rejected": -0.6480986475944519,
      "logps/chosen": -1586.593017578125,
      "logps/rejected": -1016.1822509765625,
      "loss": 0.6982,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0161069855093956,
      "rewards/margins": -0.00928802415728569,
      "rewards/rejected": -0.006818961817771196,
      "step": 921
    },
    {
      "epoch": 0.24251590524256522,
      "grad_norm": 510.0,
      "learning_rate": 4.20906432748538e-07,
      "logits/chosen": -0.6389152407646179,
      "logits/rejected": -0.6472741365432739,
      "logps/chosen": -1174.499267578125,
      "logps/rejected": -906.8936767578125,
      "loss": 0.7072,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.5258730854839087e-05,
      "rewards/margins": -0.02647418901324272,
      "rewards/rejected": 0.02645893022418022,
      "step": 922
    },
    {
      "epoch": 0.24277893767775238,
      "grad_norm": 418.0,
      "learning_rate": 4.2076023391812867e-07,
      "logits/chosen": -0.6681522727012634,
      "logits/rejected": -0.6661462783813477,
      "logps/chosen": -1200.568359375,
      "logps/rejected": -1101.7628173828125,
      "loss": 0.6979,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.004012107849121094,
      "rewards/margins": -0.008824682794511318,
      "rewards/rejected": 0.0048125749453902245,
      "step": 923
    },
    {
      "epoch": 0.24304197011293954,
      "grad_norm": 564.0,
      "learning_rate": 4.2061403508771924e-07,
      "logits/chosen": -0.664409875869751,
      "logits/rejected": -0.6760498285293579,
      "logps/chosen": -1554.7550048828125,
      "logps/rejected": -1272.03271484375,
      "loss": 0.6844,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.017483044415712357,
      "rewards/margins": 0.019138049334287643,
      "rewards/rejected": -0.0016550077125430107,
      "step": 924
    },
    {
      "epoch": 0.2433050025481267,
      "grad_norm": 536.0,
      "learning_rate": 4.204678362573099e-07,
      "logits/chosen": -0.6578445434570312,
      "logits/rejected": -0.6586265563964844,
      "logps/chosen": -927.6338500976562,
      "logps/rejected": -1016.1383056640625,
      "loss": 0.6884,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.013580989092588425,
      "rewards/margins": 0.010125827044248581,
      "rewards/rejected": 0.003455161117017269,
      "step": 925
    },
    {
      "epoch": 0.2435680349833139,
      "grad_norm": 528.0,
      "learning_rate": 4.203216374269006e-07,
      "logits/chosen": -0.656714141368866,
      "logits/rejected": -0.6451117992401123,
      "logps/chosen": -1037.1591796875,
      "logps/rejected": -873.2415771484375,
      "loss": 0.6981,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.00511093158274889,
      "rewards/margins": -0.009246635250747204,
      "rewards/rejected": 0.014357566833496094,
      "step": 926
    },
    {
      "epoch": 0.24383106741850105,
      "grad_norm": 688.0,
      "learning_rate": 4.201754385964912e-07,
      "logits/chosen": -0.6542782783508301,
      "logits/rejected": -0.6638826131820679,
      "logps/chosen": -1347.8094482421875,
      "logps/rejected": -1218.0079345703125,
      "loss": 0.702,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0021791455801576376,
      "rewards/margins": -0.017060374841094017,
      "rewards/rejected": 0.01488122995942831,
      "step": 927
    },
    {
      "epoch": 0.2440940998536882,
      "grad_norm": 644.0,
      "learning_rate": 4.200292397660818e-07,
      "logits/chosen": -0.649683952331543,
      "logits/rejected": -0.65986567735672,
      "logps/chosen": -1125.9305419921875,
      "logps/rejected": -1116.3148193359375,
      "loss": 0.6861,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00983133353292942,
      "rewards/margins": 0.015117742121219635,
      "rewards/rejected": -0.005286408122628927,
      "step": 928
    },
    {
      "epoch": 0.24435713228887537,
      "grad_norm": 438.0,
      "learning_rate": 4.198830409356725e-07,
      "logits/chosen": -0.638198733329773,
      "logits/rejected": -0.6359167098999023,
      "logps/chosen": -861.5576782226562,
      "logps/rejected": -563.738525390625,
      "loss": 0.6825,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.022594641894102097,
      "rewards/margins": 0.02188901975750923,
      "rewards/rejected": 0.0007056239992380142,
      "step": 929
    },
    {
      "epoch": 0.24462016472406253,
      "grad_norm": 504.0,
      "learning_rate": 4.197368421052631e-07,
      "logits/chosen": -0.6318085789680481,
      "logits/rejected": -0.6318012475967407,
      "logps/chosen": -1311.75732421875,
      "logps/rejected": -939.59814453125,
      "loss": 0.679,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.03400106728076935,
      "rewards/margins": 0.029186725616455078,
      "rewards/rejected": 0.0048143393360078335,
      "step": 930
    },
    {
      "epoch": 0.2448831971592497,
      "grad_norm": 548.0,
      "learning_rate": 4.195906432748538e-07,
      "logits/chosen": -0.6668189764022827,
      "logits/rejected": -0.6594454050064087,
      "logps/chosen": -1336.30224609375,
      "logps/rejected": -1071.995361328125,
      "loss": 0.6726,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.030321311205625534,
      "rewards/margins": 0.04346472769975662,
      "rewards/rejected": -0.013143414631485939,
      "step": 931
    },
    {
      "epoch": 0.24514622959443685,
      "grad_norm": 536.0,
      "learning_rate": 4.194444444444444e-07,
      "logits/chosen": -0.6218857765197754,
      "logits/rejected": -0.6330381035804749,
      "logps/chosen": -1197.1927490234375,
      "logps/rejected": -1101.2611083984375,
      "loss": 0.6954,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01946573331952095,
      "rewards/margins": -0.0037076007574796677,
      "rewards/rejected": -0.015758132562041283,
      "step": 932
    },
    {
      "epoch": 0.24540926202962401,
      "grad_norm": 624.0,
      "learning_rate": 4.192982456140351e-07,
      "logits/chosen": -0.6566147208213806,
      "logits/rejected": -0.6662856936454773,
      "logps/chosen": -1149.4246826171875,
      "logps/rejected": -873.6394653320312,
      "loss": 0.7058,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.002914619166404009,
      "rewards/margins": -0.02470398135483265,
      "rewards/rejected": 0.02761860005557537,
      "step": 933
    },
    {
      "epoch": 0.2456722944648112,
      "grad_norm": 544.0,
      "learning_rate": 4.191520467836257e-07,
      "logits/chosen": -0.6534942388534546,
      "logits/rejected": -0.6632649898529053,
      "logps/chosen": -1231.4635009765625,
      "logps/rejected": -868.723388671875,
      "loss": 0.7008,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01444778498262167,
      "rewards/margins": -0.01459503173828125,
      "rewards/rejected": 0.00014724675565958023,
      "step": 934
    },
    {
      "epoch": 0.24593532689999836,
      "grad_norm": 660.0,
      "learning_rate": 4.190058479532164e-07,
      "logits/chosen": -0.6623749136924744,
      "logits/rejected": NaN,
      "logps/chosen": -1710.11572265625,
      "logps/rejected": -926.841064453125,
      "loss": 0.6843,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.013453865423798561,
      "rewards/margins": 0.018909549340605736,
      "rewards/rejected": -0.005455684382468462,
      "step": 935
    },
    {
      "epoch": 0.24619835933518552,
      "grad_norm": 572.0,
      "learning_rate": 4.1885964912280705e-07,
      "logits/chosen": -0.6671855449676514,
      "logits/rejected": -0.6702539920806885,
      "logps/chosen": -1379.165771484375,
      "logps/rejected": -1133.8062744140625,
      "loss": 0.7053,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.008199644275009632,
      "rewards/margins": -0.02367115020751953,
      "rewards/rejected": 0.015471506863832474,
      "step": 936
    },
    {
      "epoch": 0.24646139177037268,
      "grad_norm": 680.0,
      "learning_rate": 4.187134502923976e-07,
      "logits/chosen": -0.6368398666381836,
      "logits/rejected": -0.6383780837059021,
      "logps/chosen": -1379.7298583984375,
      "logps/rejected": -1010.9422607421875,
      "loss": 0.6965,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0009054187685251236,
      "rewards/margins": -0.005238533020019531,
      "rewards/rejected": 0.0061439513228833675,
      "step": 937
    },
    {
      "epoch": 0.24672442420555984,
      "grad_norm": 596.0,
      "learning_rate": 4.185672514619883e-07,
      "logits/chosen": -0.6531285643577576,
      "logits/rejected": -0.6675831079483032,
      "logps/chosen": -993.51953125,
      "logps/rejected": -872.1209106445312,
      "loss": 0.6979,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.01012487430125475,
      "rewards/margins": -0.00869445689022541,
      "rewards/rejected": 0.018819332122802734,
      "step": 938
    },
    {
      "epoch": 0.246987456640747,
      "grad_norm": 500.0,
      "learning_rate": 4.1842105263157896e-07,
      "logits/chosen": -0.6548840403556824,
      "logits/rejected": -0.6586740016937256,
      "logps/chosen": -1030.630126953125,
      "logps/rejected": -658.2689819335938,
      "loss": 0.7101,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0267746914178133,
      "rewards/margins": -0.03306436538696289,
      "rewards/rejected": 0.006289672572165728,
      "step": 939
    },
    {
      "epoch": 0.24725048907593417,
      "grad_norm": 508.0,
      "learning_rate": 4.182748538011696e-07,
      "logits/chosen": -0.6490989923477173,
      "logits/rejected": -0.6547738313674927,
      "logps/chosen": -1057.1375732421875,
      "logps/rejected": -885.9509887695312,
      "loss": 0.6852,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.006193924229592085,
      "rewards/margins": 0.016273880377411842,
      "rewards/rejected": -0.01007995754480362,
      "step": 940
    },
    {
      "epoch": 0.24751352151112135,
      "grad_norm": 394.0,
      "learning_rate": 4.181286549707602e-07,
      "logits/chosen": -0.6534513235092163,
      "logits/rejected": -0.6515899300575256,
      "logps/chosen": -528.9981079101562,
      "logps/rejected": -589.5171508789062,
      "loss": 0.6874,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01296076737344265,
      "rewards/margins": 0.011711454018950462,
      "rewards/rejected": 0.001249313703738153,
      "step": 941
    },
    {
      "epoch": 0.24777655394630851,
      "grad_norm": 552.0,
      "learning_rate": 4.179824561403509e-07,
      "logits/chosen": -0.6523987054824829,
      "logits/rejected": -0.6572139859199524,
      "logps/chosen": -1478.381103515625,
      "logps/rejected": -1107.2392578125,
      "loss": 0.7019,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0015441890573129058,
      "rewards/margins": -0.016761304810643196,
      "rewards/rejected": 0.015217114239931107,
      "step": 942
    },
    {
      "epoch": 0.24803958638149567,
      "grad_norm": 444.0,
      "learning_rate": 4.178362573099415e-07,
      "logits/chosen": -0.6492806077003479,
      "logits/rejected": -0.6577972173690796,
      "logps/chosen": -983.4747924804688,
      "logps/rejected": -835.034912109375,
      "loss": 0.688,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0248844176530838,
      "rewards/margins": 0.011253930628299713,
      "rewards/rejected": 0.013630485162138939,
      "step": 943
    },
    {
      "epoch": 0.24830261881668284,
      "grad_norm": 744.0,
      "learning_rate": 4.1769005847953217e-07,
      "logits/chosen": -0.6535940170288086,
      "logits/rejected": -0.6710982918739319,
      "logps/chosen": -1092.96240234375,
      "logps/rejected": -940.3051147460938,
      "loss": 0.7013,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.011361218057572842,
      "rewards/margins": -0.014706609770655632,
      "rewards/rejected": 0.0033453935757279396,
      "step": 944
    },
    {
      "epoch": 0.24856565125187,
      "grad_norm": 628.0,
      "learning_rate": 4.175438596491228e-07,
      "logits/chosen": -0.6783431768417358,
      "logits/rejected": -0.6687673330307007,
      "logps/chosen": -1535.752197265625,
      "logps/rejected": -1211.2891845703125,
      "loss": 0.7009,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00014352984726428986,
      "rewards/margins": -0.01480498444288969,
      "rewards/rejected": 0.014948509633541107,
      "step": 945
    },
    {
      "epoch": 0.24882868368705716,
      "grad_norm": 470.0,
      "learning_rate": 4.173976608187134e-07,
      "logits/chosen": -0.6390160918235779,
      "logits/rejected": -0.6407660841941833,
      "logps/chosen": -812.281982421875,
      "logps/rejected": -888.0386352539062,
      "loss": 0.6905,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.003564214799553156,
      "rewards/margins": 0.005888890940696001,
      "rewards/rejected": -0.009453105740249157,
      "step": 946
    },
    {
      "epoch": 0.24909171612224432,
      "grad_norm": 552.0,
      "learning_rate": 4.172514619883041e-07,
      "logits/chosen": -0.6453909873962402,
      "logits/rejected": -0.6623314619064331,
      "logps/chosen": -1355.2547607421875,
      "logps/rejected": -1081.2435302734375,
      "loss": 0.6835,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.012732649222016335,
      "rewards/margins": 0.02012333832681179,
      "rewards/rejected": -0.007390690501779318,
      "step": 947
    },
    {
      "epoch": 0.24935474855743148,
      "grad_norm": 704.0,
      "learning_rate": 4.1710526315789475e-07,
      "logits/chosen": -0.6571630835533142,
      "logits/rejected": -0.6697350740432739,
      "logps/chosen": -1300.3885498046875,
      "logps/rejected": -1138.1015625,
      "loss": 0.6986,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.003518296405673027,
      "rewards/margins": -0.010310412384569645,
      "rewards/rejected": 0.013828707858920097,
      "step": 948
    },
    {
      "epoch": 0.24961778099261867,
      "grad_norm": 604.0,
      "learning_rate": 4.169590643274853e-07,
      "logits/chosen": -0.6633532047271729,
      "logits/rejected": -0.6613842844963074,
      "logps/chosen": -1204.6719970703125,
      "logps/rejected": -1011.9022216796875,
      "loss": 0.7013,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.004476356785744429,
      "rewards/margins": -0.015715407207608223,
      "rewards/rejected": 0.011239051818847656,
      "step": 949
    },
    {
      "epoch": 0.24988081342780583,
      "grad_norm": 620.0,
      "learning_rate": 4.16812865497076e-07,
      "logits/chosen": -0.6460379958152771,
      "logits/rejected": -0.6534227132797241,
      "logps/chosen": -1272.5279541015625,
      "logps/rejected": -781.078125,
      "loss": 0.6986,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.020388126373291016,
      "rewards/margins": -0.01039428822696209,
      "rewards/rejected": -0.0099938390776515,
      "step": 950
    },
    {
      "epoch": 0.250143845862993,
      "grad_norm": 400.0,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -0.620230495929718,
      "logits/rejected": -0.6263192892074585,
      "logps/chosen": -690.1941528320312,
      "logps/rejected": -662.4988403320312,
      "loss": 0.6969,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.006712149828672409,
      "rewards/margins": -0.006974649615585804,
      "rewards/rejected": 0.00026249897200614214,
      "step": 951
    },
    {
      "epoch": 0.25040687829818015,
      "grad_norm": 596.0,
      "learning_rate": 4.165204678362573e-07,
      "logits/chosen": -0.6457705497741699,
      "logits/rejected": -0.6500723958015442,
      "logps/chosen": -1391.000244140625,
      "logps/rejected": -1173.8912353515625,
      "loss": 0.6894,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.013447855599224567,
      "rewards/margins": 0.008391574025154114,
      "rewards/rejected": 0.0050562843680381775,
      "step": 952
    },
    {
      "epoch": 0.2506699107333673,
      "grad_norm": 386.0,
      "learning_rate": 4.1637426900584796e-07,
      "logits/chosen": -0.643547773361206,
      "logits/rejected": -0.6508293747901917,
      "logps/chosen": -950.1175537109375,
      "logps/rejected": -765.178955078125,
      "loss": 0.6831,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.018361520022153854,
      "rewards/margins": 0.020774032920598984,
      "rewards/rejected": -0.0024125108029693365,
      "step": 953
    },
    {
      "epoch": 0.25093294316855447,
      "grad_norm": 496.0,
      "learning_rate": 4.162280701754386e-07,
      "logits/chosen": -0.6538925766944885,
      "logits/rejected": -0.6631028056144714,
      "logps/chosen": -920.0482788085938,
      "logps/rejected": -798.2728881835938,
      "loss": 0.6966,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.008536625653505325,
      "rewards/margins": -0.004946995060890913,
      "rewards/rejected": 0.013483620248734951,
      "step": 954
    },
    {
      "epoch": 0.25119597560374163,
      "grad_norm": 728.0,
      "learning_rate": 4.160818713450292e-07,
      "logits/chosen": -0.6592716574668884,
      "logits/rejected": -0.6587908267974854,
      "logps/chosen": -1589.419189453125,
      "logps/rejected": -940.9369506835938,
      "loss": 0.6898,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0037509917747229338,
      "rewards/margins": 0.007623385637998581,
      "rewards/rejected": -0.011374376714229584,
      "step": 955
    },
    {
      "epoch": 0.2514590080389288,
      "grad_norm": 464.0,
      "learning_rate": 4.1593567251461987e-07,
      "logits/chosen": -0.6479768753051758,
      "logits/rejected": NaN,
      "logps/chosen": -1037.12158203125,
      "logps/rejected": -611.2537231445312,
      "loss": 0.6971,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.019550994038581848,
      "rewards/margins": -0.007162143476307392,
      "rewards/rejected": -0.012388849630951881,
      "step": 956
    },
    {
      "epoch": 0.25172204047411595,
      "grad_norm": 492.0,
      "learning_rate": 4.1578947368421054e-07,
      "logits/chosen": -0.6315648555755615,
      "logits/rejected": -0.630164384841919,
      "logps/chosen": -879.68359375,
      "logps/rejected": -876.7628173828125,
      "loss": 0.6863,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.012038517743349075,
      "rewards/margins": 0.01416025124490261,
      "rewards/rejected": -0.002121733734384179,
      "step": 957
    },
    {
      "epoch": 0.2519850729093031,
      "grad_norm": 472.0,
      "learning_rate": 4.156432748538011e-07,
      "logits/chosen": -0.6305992603302002,
      "logits/rejected": -0.6375600099563599,
      "logps/chosen": -1262.648193359375,
      "logps/rejected": -969.32763671875,
      "loss": 0.6901,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01010208111256361,
      "rewards/margins": 0.007419013883918524,
      "rewards/rejected": 0.0026830676943063736,
      "step": 958
    },
    {
      "epoch": 0.25224810534449027,
      "grad_norm": 506.0,
      "learning_rate": 4.154970760233918e-07,
      "logits/chosen": -0.6745859980583191,
      "logits/rejected": -0.6577268242835999,
      "logps/chosen": -871.1368408203125,
      "logps/rejected": -632.828369140625,
      "loss": 0.6955,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01622776873409748,
      "rewards/margins": -0.003959321416914463,
      "rewards/rejected": 0.020187092944979668,
      "step": 959
    },
    {
      "epoch": 0.25251113777967743,
      "grad_norm": 564.0,
      "learning_rate": 4.1535087719298246e-07,
      "logits/chosen": -0.6361596584320068,
      "logits/rejected": -0.6389310359954834,
      "logps/chosen": -1262.8277587890625,
      "logps/rejected": -910.3294067382812,
      "loss": 0.6914,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.010638141073286533,
      "rewards/margins": 0.004257869906723499,
      "rewards/rejected": 0.006380271166563034,
      "step": 960
    },
    {
      "epoch": 0.25277417021486465,
      "grad_norm": 476.0,
      "learning_rate": 4.152046783625731e-07,
      "logits/chosen": -0.6224642395973206,
      "logits/rejected": -0.6200088858604431,
      "logps/chosen": -1048.4659423828125,
      "logps/rejected": -1066.3587646484375,
      "loss": 0.7052,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01302185095846653,
      "rewards/margins": -0.0228818878531456,
      "rewards/rejected": 0.009860038757324219,
      "step": 961
    },
    {
      "epoch": 0.2530372026500518,
      "grad_norm": 608.0,
      "learning_rate": 4.150584795321637e-07,
      "logits/chosen": -0.6411485075950623,
      "logits/rejected": -0.6446213722229004,
      "logps/chosen": -1224.052001953125,
      "logps/rejected": -916.55712890625,
      "loss": 0.688,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0044317240826785564,
      "rewards/margins": 0.011794091202318668,
      "rewards/rejected": -0.016225814819335938,
      "step": 962
    },
    {
      "epoch": 0.25330023508523897,
      "grad_norm": 708.0,
      "learning_rate": 4.1491228070175437e-07,
      "logits/chosen": -0.6540834903717041,
      "logits/rejected": -0.6457995176315308,
      "logps/chosen": -1022.19287109375,
      "logps/rejected": -1164.020751953125,
      "loss": 0.6813,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02131056785583496,
      "rewards/margins": 0.026399563997983932,
      "rewards/rejected": -0.005088995210826397,
      "step": 963
    },
    {
      "epoch": 0.2535632675204261,
      "grad_norm": 494.0,
      "learning_rate": 4.14766081871345e-07,
      "logits/chosen": -0.6423096656799316,
      "logits/rejected": -0.6565354466438293,
      "logps/chosen": -802.9921264648438,
      "logps/rejected": -713.1737670898438,
      "loss": 0.6907,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010305595584213734,
      "rewards/margins": 0.005396365188062191,
      "rewards/rejected": 0.004909229930490255,
      "step": 964
    },
    {
      "epoch": 0.2538262999556133,
      "grad_norm": 648.0,
      "learning_rate": 4.1461988304093566e-07,
      "logits/chosen": -0.6392073035240173,
      "logits/rejected": -0.6474464535713196,
      "logps/chosen": -1338.203369140625,
      "logps/rejected": -1060.070068359375,
      "loss": 0.7032,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.000610542600043118,
      "rewards/margins": -0.019461918622255325,
      "rewards/rejected": 0.018851375207304955,
      "step": 965
    },
    {
      "epoch": 0.25408933239080045,
      "grad_norm": 660.0,
      "learning_rate": 4.1447368421052633e-07,
      "logits/chosen": -0.6200531721115112,
      "logits/rejected": -0.6216959953308105,
      "logps/chosen": -1593.1424560546875,
      "logps/rejected": -1239.091796875,
      "loss": 0.684,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.019835472106933594,
      "rewards/margins": 0.019740009680390358,
      "rewards/rejected": 9.546324145048857e-05,
      "step": 966
    },
    {
      "epoch": 0.2543523648259876,
      "grad_norm": 568.0,
      "learning_rate": 4.143274853801169e-07,
      "logits/chosen": -0.6596719026565552,
      "logits/rejected": -0.6652796268463135,
      "logps/chosen": -1129.5001220703125,
      "logps/rejected": -934.3064575195312,
      "loss": 0.7056,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.004582023248076439,
      "rewards/margins": -0.023746203631162643,
      "rewards/rejected": 0.019164182245731354,
      "step": 967
    },
    {
      "epoch": 0.25461539726117477,
      "grad_norm": 516.0,
      "learning_rate": 4.141812865497076e-07,
      "logits/chosen": -0.648077666759491,
      "logits/rejected": -0.6608498692512512,
      "logps/chosen": -880.088134765625,
      "logps/rejected": -954.1636962890625,
      "loss": 0.6874,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.002792166545987129,
      "rewards/margins": 0.011858749203383923,
      "rewards/rejected": -0.009066580794751644,
      "step": 968
    },
    {
      "epoch": 0.25487842969636193,
      "grad_norm": 716.0,
      "learning_rate": 4.1403508771929825e-07,
      "logits/chosen": -0.6507014036178589,
      "logits/rejected": -0.6596258878707886,
      "logps/chosen": -1229.887451171875,
      "logps/rejected": -1136.353515625,
      "loss": 0.6831,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.015675164759159088,
      "rewards/margins": 0.021275237202644348,
      "rewards/rejected": -0.005600071512162685,
      "step": 969
    },
    {
      "epoch": 0.2551414621315491,
      "grad_norm": 764.0,
      "learning_rate": 4.1388888888888887e-07,
      "logits/chosen": -0.6581074595451355,
      "logits/rejected": -0.6665617227554321,
      "logps/chosen": -1636.98046875,
      "logps/rejected": -1087.7998046875,
      "loss": 0.6848,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02057952992618084,
      "rewards/margins": 0.017328549176454544,
      "rewards/rejected": 0.003250980516895652,
      "step": 970
    },
    {
      "epoch": 0.25540449456673625,
      "grad_norm": 510.0,
      "learning_rate": 4.137426900584795e-07,
      "logits/chosen": -0.6387740969657898,
      "logits/rejected": -0.6610140800476074,
      "logps/chosen": -1302.182373046875,
      "logps/rejected": -792.2884521484375,
      "loss": 0.6942,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008560944348573685,
      "rewards/margins": -0.0007077697664499283,
      "rewards/rejected": 0.009268713183701038,
      "step": 971
    },
    {
      "epoch": 0.2556675270019234,
      "grad_norm": 540.0,
      "learning_rate": 4.1359649122807016e-07,
      "logits/chosen": -0.6706705093383789,
      "logits/rejected": -0.6806513071060181,
      "logps/chosen": -1306.852783203125,
      "logps/rejected": -1093.7579345703125,
      "loss": 0.6797,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.014180564321577549,
      "rewards/margins": 0.028049372136592865,
      "rewards/rejected": -0.013868809677660465,
      "step": 972
    },
    {
      "epoch": 0.25593055943711057,
      "grad_norm": 556.0,
      "learning_rate": 4.1345029239766083e-07,
      "logits/chosen": -0.6520328521728516,
      "logits/rejected": -0.6579900979995728,
      "logps/chosen": -1151.3028564453125,
      "logps/rejected": -754.7607421875,
      "loss": 0.703,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0011653900146484375,
      "rewards/margins": -0.018252942711114883,
      "rewards/rejected": 0.017087554559111595,
      "step": 973
    },
    {
      "epoch": 0.25619359187229773,
      "grad_norm": 536.0,
      "learning_rate": 4.1330409356725145e-07,
      "logits/chosen": -0.6569728255271912,
      "logits/rejected": NaN,
      "logps/chosen": -918.9491577148438,
      "logps/rejected": -617.5560302734375,
      "loss": 0.6897,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.009088133461773396,
      "rewards/margins": 0.00722885224968195,
      "rewards/rejected": -0.016316985711455345,
      "step": 974
    },
    {
      "epoch": 0.2564566243074849,
      "grad_norm": 584.0,
      "learning_rate": 4.1315789473684207e-07,
      "logits/chosen": -0.6500560641288757,
      "logits/rejected": -0.6411870718002319,
      "logps/chosen": -887.0394287109375,
      "logps/rejected": -855.955810546875,
      "loss": 0.6941,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.024772070348262787,
      "rewards/margins": -0.001012324821203947,
      "rewards/rejected": -0.0237597469240427,
      "step": 975
    },
    {
      "epoch": 0.25671965674267205,
      "grad_norm": 632.0,
      "learning_rate": 4.1301169590643275e-07,
      "logits/chosen": -0.6693426370620728,
      "logits/rejected": -0.6649603843688965,
      "logps/chosen": -1253.2406005859375,
      "logps/rejected": -1052.33984375,
      "loss": 0.6936,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.015530203469097614,
      "rewards/margins": -0.00047102058306336403,
      "rewards/rejected": 0.016001224517822266,
      "step": 976
    },
    {
      "epoch": 0.25698268917785927,
      "grad_norm": 564.0,
      "learning_rate": 4.1286549707602337e-07,
      "logits/chosen": -0.6603196859359741,
      "logits/rejected": -0.6559270024299622,
      "logps/chosen": -1275.61572265625,
      "logps/rejected": -1119.7020263671875,
      "loss": 0.6976,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.008411120623350143,
      "rewards/margins": -0.008404635824263096,
      "rewards/rejected": -6.48485729470849e-06,
      "step": 977
    },
    {
      "epoch": 0.25724572161304643,
      "grad_norm": 640.0,
      "learning_rate": 4.1271929824561404e-07,
      "logits/chosen": -0.626673698425293,
      "logits/rejected": -0.6431872844696045,
      "logps/chosen": -1316.005615234375,
      "logps/rejected": -1038.4517822265625,
      "loss": 0.7027,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.000577878556214273,
      "rewards/margins": -0.01809105835855007,
      "rewards/rejected": 0.017513180151581764,
      "step": 978
    },
    {
      "epoch": 0.2575087540482336,
      "grad_norm": 450.0,
      "learning_rate": 4.1257309941520466e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6210249662399292,
      "logps/chosen": -992.8655395507812,
      "logps/rejected": -805.7991333007812,
      "loss": 0.6766,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0030942922458052635,
      "rewards/margins": 0.034136153757572174,
      "rewards/rejected": -0.031041860580444336,
      "step": 979
    },
    {
      "epoch": 0.25777178648342075,
      "grad_norm": 548.0,
      "learning_rate": 4.124269005847953e-07,
      "logits/chosen": -0.6815102100372314,
      "logits/rejected": -0.6813878417015076,
      "logps/chosen": -960.111572265625,
      "logps/rejected": -936.6555786132812,
      "loss": 0.7077,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.016501616686582565,
      "rewards/margins": -0.02765636146068573,
      "rewards/rejected": 0.01115474570542574,
      "step": 980
    },
    {
      "epoch": 0.2580348189186079,
      "grad_norm": 608.0,
      "learning_rate": 4.1228070175438595e-07,
      "logits/chosen": -0.6400889158248901,
      "logits/rejected": -0.6363367438316345,
      "logps/chosen": -1159.1328125,
      "logps/rejected": -1220.11767578125,
      "loss": 0.6844,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.017642593011260033,
      "rewards/margins": 0.01842193678021431,
      "rewards/rejected": -0.0007793419063091278,
      "step": 981
    },
    {
      "epoch": 0.25829785135379507,
      "grad_norm": 564.0,
      "learning_rate": 4.121345029239766e-07,
      "logits/chosen": -0.6269813776016235,
      "logits/rejected": -0.6276199817657471,
      "logps/chosen": -1027.215087890625,
      "logps/rejected": -769.8380737304688,
      "loss": 0.6934,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004740428179502487,
      "rewards/margins": -0.00013818731531500816,
      "rewards/rejected": 0.004878616891801357,
      "step": 982
    },
    {
      "epoch": 0.25856088378898223,
      "grad_norm": 580.0,
      "learning_rate": 4.1198830409356724e-07,
      "logits/chosen": -0.6419468522071838,
      "logits/rejected": -0.6533553600311279,
      "logps/chosen": -1286.263916015625,
      "logps/rejected": -695.2100830078125,
      "loss": 0.6916,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0021896364632993937,
      "rewards/margins": 0.004176665097475052,
      "rewards/rejected": -0.001987028867006302,
      "step": 983
    },
    {
      "epoch": 0.2588239162241694,
      "grad_norm": 588.0,
      "learning_rate": 4.1184210526315786e-07,
      "logits/chosen": -0.6577523946762085,
      "logits/rejected": -0.6614417433738708,
      "logps/chosen": -1209.311279296875,
      "logps/rejected": -995.1920166015625,
      "loss": 0.6844,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0005711548146791756,
      "rewards/margins": 0.017961597070097923,
      "rewards/rejected": -0.01739044114947319,
      "step": 984
    },
    {
      "epoch": 0.25908694865935655,
      "grad_norm": 664.0,
      "learning_rate": 4.1169590643274854e-07,
      "logits/chosen": -0.6458343267440796,
      "logits/rejected": -0.6421516537666321,
      "logps/chosen": -1286.3118896484375,
      "logps/rejected": -1004.1317138671875,
      "loss": 0.71,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0305925365537405,
      "rewards/margins": -0.032656606286764145,
      "rewards/rejected": 0.0020640676375478506,
      "step": 985
    },
    {
      "epoch": 0.2593499810945437,
      "grad_norm": 604.0,
      "learning_rate": 4.1154970760233916e-07,
      "logits/chosen": -0.663537859916687,
      "logits/rejected": -0.6765425205230713,
      "logps/chosen": -787.5771484375,
      "logps/rejected": -945.1334228515625,
      "loss": 0.7017,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.009326220490038395,
      "rewards/margins": -0.01582011952996254,
      "rewards/rejected": 0.006493901833891869,
      "step": 986
    },
    {
      "epoch": 0.2596130135297309,
      "grad_norm": 612.0,
      "learning_rate": 4.1140350877192983e-07,
      "logits/chosen": -0.6693596839904785,
      "logits/rejected": -0.6565718054771423,
      "logps/chosen": -1267.15234375,
      "logps/rejected": -814.3225708007812,
      "loss": 0.6835,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.007166958414018154,
      "rewards/margins": 0.019865607842803,
      "rewards/rejected": -0.02703256718814373,
      "step": 987
    },
    {
      "epoch": 0.25987604596491803,
      "grad_norm": 628.0,
      "learning_rate": 4.1125730994152045e-07,
      "logits/chosen": -0.6541004180908203,
      "logits/rejected": -0.6554379463195801,
      "logps/chosen": -1336.442138671875,
      "logps/rejected": -1255.2933349609375,
      "loss": 0.7036,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.023982618004083633,
      "rewards/margins": -0.019832896068692207,
      "rewards/rejected": -0.004149722866714001,
      "step": 988
    },
    {
      "epoch": 0.2601390784001052,
      "grad_norm": 540.0,
      "learning_rate": 4.1111111111111107e-07,
      "logits/chosen": -0.6340453624725342,
      "logits/rejected": -0.6422751545906067,
      "logps/chosen": -1144.8028564453125,
      "logps/rejected": -764.8045654296875,
      "loss": 0.7023,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.00642213923856616,
      "rewards/margins": -0.017829228192567825,
      "rewards/rejected": 0.011407088488340378,
      "step": 989
    },
    {
      "epoch": 0.26040211083529236,
      "grad_norm": 556.0,
      "learning_rate": 4.1096491228070174e-07,
      "logits/chosen": -0.6681361198425293,
      "logits/rejected": -0.667523980140686,
      "logps/chosen": -1274.7261962890625,
      "logps/rejected": -1192.0284423828125,
      "loss": 0.6915,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.015300942584872246,
      "rewards/margins": 0.004200411029160023,
      "rewards/rejected": 0.011100530624389648,
      "step": 990
    },
    {
      "epoch": 0.2606651432704795,
      "grad_norm": 498.0,
      "learning_rate": 4.108187134502924e-07,
      "logits/chosen": -0.652130663394928,
      "logits/rejected": -0.6541780233383179,
      "logps/chosen": -968.7282104492188,
      "logps/rejected": -652.9732055664062,
      "loss": 0.7063,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.018943022936582565,
      "rewards/margins": -0.02527771145105362,
      "rewards/rejected": 0.006334686651825905,
      "step": 991
    },
    {
      "epoch": 0.26092817570566673,
      "grad_norm": 480.0,
      "learning_rate": 4.10672514619883e-07,
      "logits/chosen": -0.6327903270721436,
      "logits/rejected": -0.6305159330368042,
      "logps/chosen": -1021.7764282226562,
      "logps/rejected": -852.796142578125,
      "loss": 0.6826,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.029436780139803886,
      "rewards/margins": 0.02347440831363201,
      "rewards/rejected": 0.005962372291833162,
      "step": 992
    },
    {
      "epoch": 0.2611912081408539,
      "grad_norm": 568.0,
      "learning_rate": 4.1052631578947365e-07,
      "logits/chosen": -0.6692930459976196,
      "logits/rejected": -0.6655673980712891,
      "logps/chosen": -1312.6904296875,
      "logps/rejected": -870.4715576171875,
      "loss": 0.6909,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.010732032358646393,
      "rewards/margins": 0.006722881458699703,
      "rewards/rejected": 0.00400915089994669,
      "step": 993
    },
    {
      "epoch": 0.26145424057604105,
      "grad_norm": 540.0,
      "learning_rate": 4.1038011695906433e-07,
      "logits/chosen": -0.6791431903839111,
      "logits/rejected": -0.6812678575515747,
      "logps/chosen": -1243.5205078125,
      "logps/rejected": -1108.3349609375,
      "loss": 0.6973,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.014312171377241611,
      "rewards/margins": -0.00788125954568386,
      "rewards/rejected": -0.006430912762880325,
      "step": 994
    },
    {
      "epoch": 0.2617172730112282,
      "grad_norm": 432.0,
      "learning_rate": 4.1023391812865495e-07,
      "logits/chosen": -0.6665586233139038,
      "logits/rejected": -0.6619126200675964,
      "logps/chosen": -712.2880859375,
      "logps/rejected": -782.8844604492188,
      "loss": 0.6973,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.03263607248663902,
      "rewards/margins": -0.007870102301239967,
      "rewards/rejected": -0.024765968322753906,
      "step": 995
    },
    {
      "epoch": 0.2619803054464154,
      "grad_norm": 408.0,
      "learning_rate": 4.1008771929824557e-07,
      "logits/chosen": -0.6555261611938477,
      "logits/rejected": -0.6497012376785278,
      "logps/chosen": -994.4965209960938,
      "logps/rejected": -864.79248046875,
      "loss": 0.6909,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01789226569235325,
      "rewards/margins": 0.004942560102790594,
      "rewards/rejected": 0.012949705123901367,
      "step": 996
    },
    {
      "epoch": 0.26224333788160253,
      "grad_norm": 502.0,
      "learning_rate": 4.0994152046783624e-07,
      "logits/chosen": -0.663555920124054,
      "logits/rejected": -0.6622716784477234,
      "logps/chosen": -1072.64306640625,
      "logps/rejected": -890.6778564453125,
      "loss": 0.6982,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0020089158788323402,
      "rewards/margins": -0.00962457712739706,
      "rewards/rejected": 0.007615662179887295,
      "step": 997
    },
    {
      "epoch": 0.2625063703167897,
      "grad_norm": 466.0,
      "learning_rate": 4.0979532163742686e-07,
      "logits/chosen": -0.6514309644699097,
      "logits/rejected": -0.660879373550415,
      "logps/chosen": -936.687255859375,
      "logps/rejected": -679.7059326171875,
      "loss": 0.6863,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0066817281767725945,
      "rewards/margins": 0.014472581446170807,
      "rewards/rejected": -0.0077908518724143505,
      "step": 998
    },
    {
      "epoch": 0.26276940275197685,
      "grad_norm": 652.0,
      "learning_rate": 4.0964912280701753e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6352875232696533,
      "logps/chosen": -976.9027709960938,
      "logps/rejected": -822.607421875,
      "loss": 0.7038,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.007833290845155716,
      "rewards/margins": -0.019905662164092064,
      "rewards/rejected": 0.012072373181581497,
      "step": 999
    },
    {
      "epoch": 0.263032435187164,
      "grad_norm": 588.0,
      "learning_rate": 4.095029239766082e-07,
      "logits/chosen": -0.6647536754608154,
      "logits/rejected": -0.6662343740463257,
      "logps/chosen": -1470.8970947265625,
      "logps/rejected": -1052.30126953125,
      "loss": 0.685,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.020780183374881744,
      "rewards/margins": 0.01772165484726429,
      "rewards/rejected": 0.003058528760448098,
      "step": 1000
    },
    {
      "epoch": 0.2632954676223512,
      "grad_norm": 584.0,
      "learning_rate": 4.0935672514619877e-07,
      "logits/chosen": -0.6572305560112,
      "logits/rejected": -0.653533935546875,
      "logps/chosen": -994.7073364257812,
      "logps/rejected": -973.357421875,
      "loss": 0.7017,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005534076597541571,
      "rewards/margins": -0.016376400366425514,
      "rewards/rejected": 0.01084232423454523,
      "step": 1001
    },
    {
      "epoch": 0.26355850005753834,
      "grad_norm": 624.0,
      "learning_rate": 4.0921052631578945e-07,
      "logits/chosen": -0.6255507469177246,
      "logits/rejected": -0.6289364695549011,
      "logps/chosen": -1189.9368896484375,
      "logps/rejected": -756.735595703125,
      "loss": 0.6973,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01393957156687975,
      "rewards/margins": -0.0075890542939305305,
      "rewards/rejected": -0.006350518669933081,
      "step": 1002
    },
    {
      "epoch": 0.2638215324927255,
      "grad_norm": 516.0,
      "learning_rate": 4.090643274853801e-07,
      "logits/chosen": -0.6516346335411072,
      "logits/rejected": -0.6479806900024414,
      "logps/chosen": -1137.131591796875,
      "logps/rejected": -799.2296142578125,
      "loss": 0.7076,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.02869124710559845,
      "rewards/margins": -0.028222128748893738,
      "rewards/rejected": -0.0004691123031079769,
      "step": 1003
    },
    {
      "epoch": 0.26408456492791266,
      "grad_norm": 450.0,
      "learning_rate": 4.0891812865497074e-07,
      "logits/chosen": -0.6531213521957397,
      "logits/rejected": -0.6580477356910706,
      "logps/chosen": -1383.140869140625,
      "logps/rejected": -1184.3951416015625,
      "loss": 0.7083,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.018806029111146927,
      "rewards/margins": -0.029477262869477272,
      "rewards/rejected": 0.010671235620975494,
      "step": 1004
    },
    {
      "epoch": 0.2643475973630998,
      "grad_norm": 628.0,
      "learning_rate": 4.0877192982456136e-07,
      "logits/chosen": -0.6613959074020386,
      "logits/rejected": -0.6611139178276062,
      "logps/chosen": -1183.5477294921875,
      "logps/rejected": -1010.1044311523438,
      "loss": 0.6907,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0021071434020996094,
      "rewards/margins": 0.005825996398925781,
      "rewards/rejected": -0.00793313980102539,
      "step": 1005
    },
    {
      "epoch": 0.264610629798287,
      "grad_norm": 510.0,
      "learning_rate": 4.0862573099415203e-07,
      "logits/chosen": -0.6478052139282227,
      "logits/rejected": -0.6593229174613953,
      "logps/chosen": -1046.8602294921875,
      "logps/rejected": -825.0645141601562,
      "loss": 0.711,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.01688213460147381,
      "rewards/margins": -0.03480091318488121,
      "rewards/rejected": 0.017918778583407402,
      "step": 1006
    },
    {
      "epoch": 0.26487366223347414,
      "grad_norm": 406.0,
      "learning_rate": 4.0847953216374265e-07,
      "logits/chosen": -0.6607606410980225,
      "logits/rejected": -0.6612842679023743,
      "logps/chosen": -846.4671630859375,
      "logps/rejected": -778.1599731445312,
      "loss": 0.6837,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.016026688739657402,
      "rewards/margins": 0.020276356488466263,
      "rewards/rejected": -0.004249668680131435,
      "step": 1007
    },
    {
      "epoch": 0.26513669466866135,
      "grad_norm": 504.0,
      "learning_rate": 4.083333333333333e-07,
      "logits/chosen": -0.6404139399528503,
      "logits/rejected": -0.6586987376213074,
      "logps/chosen": -1093.5750732421875,
      "logps/rejected": -911.3099365234375,
      "loss": 0.6961,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01020069234073162,
      "rewards/margins": -0.005050084553658962,
      "rewards/rejected": -0.00515060406178236,
      "step": 1008
    },
    {
      "epoch": 0.2653997271038485,
      "grad_norm": 490.0,
      "learning_rate": 4.0818713450292394e-07,
      "logits/chosen": -0.6611888408660889,
      "logits/rejected": -0.6531864404678345,
      "logps/chosen": -1180.1241455078125,
      "logps/rejected": -764.6567993164062,
      "loss": 0.6891,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.001316833309829235,
      "rewards/margins": 0.008646775037050247,
      "rewards/rejected": -0.009963605552911758,
      "step": 1009
    },
    {
      "epoch": 0.2656627595390357,
      "grad_norm": 442.0,
      "learning_rate": 4.0804093567251456e-07,
      "logits/chosen": -0.6305913925170898,
      "logits/rejected": -0.6443919539451599,
      "logps/chosen": -818.9306640625,
      "logps/rejected": -652.1432495117188,
      "loss": 0.7006,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0005535602103918791,
      "rewards/margins": -0.014216233044862747,
      "rewards/rejected": 0.01366267167031765,
      "step": 1010
    },
    {
      "epoch": 0.26592579197422284,
      "grad_norm": 688.0,
      "learning_rate": 4.0789473684210524e-07,
      "logits/chosen": -0.6642576456069946,
      "logits/rejected": -0.6740661859512329,
      "logps/chosen": -1363.777099609375,
      "logps/rejected": -1020.8844604492188,
      "loss": 0.7071,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.007761096581816673,
      "rewards/margins": -0.0265673641115427,
      "rewards/rejected": 0.01880626752972603,
      "step": 1011
    },
    {
      "epoch": 0.26618882440941,
      "grad_norm": 414.0,
      "learning_rate": 4.077485380116959e-07,
      "logits/chosen": -0.6662865281105042,
      "logits/rejected": -0.66348797082901,
      "logps/chosen": -868.6817626953125,
      "logps/rejected": -695.2415771484375,
      "loss": 0.6815,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.007106209173798561,
      "rewards/margins": 0.024123765528202057,
      "rewards/rejected": -0.017017554491758347,
      "step": 1012
    },
    {
      "epoch": 0.26645185684459716,
      "grad_norm": 516.0,
      "learning_rate": 4.076023391812866e-07,
      "logits/chosen": -0.671603798866272,
      "logits/rejected": -0.6694333553314209,
      "logps/chosen": -1067.6534423828125,
      "logps/rejected": -702.732421875,
      "loss": 0.6934,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.000249004689976573,
      "rewards/margins": 6.580725312232971e-06,
      "rewards/rejected": -0.000255584716796875,
      "step": 1013
    },
    {
      "epoch": 0.2667148892797843,
      "grad_norm": 396.0,
      "learning_rate": 4.0745614035087715e-07,
      "logits/chosen": -0.6939864158630371,
      "logits/rejected": -0.6942847967147827,
      "logps/chosen": -1222.3851318359375,
      "logps/rejected": -1062.3729248046875,
      "loss": 0.6862,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.03996744379401207,
      "rewards/margins": 0.014324522577226162,
      "rewards/rejected": 0.025642916560173035,
      "step": 1014
    },
    {
      "epoch": 0.2669779217149715,
      "grad_norm": 604.0,
      "learning_rate": 4.073099415204678e-07,
      "logits/chosen": -0.6638373732566833,
      "logits/rejected": -0.6700912117958069,
      "logps/chosen": -1134.1553955078125,
      "logps/rejected": -934.1598510742188,
      "loss": 0.7034,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.016888713464140892,
      "rewards/margins": -0.01870756223797798,
      "rewards/rejected": 0.001818846445530653,
      "step": 1015
    },
    {
      "epoch": 0.26724095415015864,
      "grad_norm": 588.0,
      "learning_rate": 4.071637426900585e-07,
      "logits/chosen": -0.652276337146759,
      "logits/rejected": -0.6528505682945251,
      "logps/chosen": -1265.5836181640625,
      "logps/rejected": -1093.1165771484375,
      "loss": 0.7017,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.006398678757250309,
      "rewards/margins": -0.016336727887392044,
      "rewards/rejected": 0.009938049130141735,
      "step": 1016
    },
    {
      "epoch": 0.2675039865853458,
      "grad_norm": 788.0,
      "learning_rate": 4.070175438596491e-07,
      "logits/chosen": -0.6436947584152222,
      "logits/rejected": -0.6476771235466003,
      "logps/chosen": -1017.9442138671875,
      "logps/rejected": -791.5717163085938,
      "loss": 0.6852,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009157419204711914,
      "rewards/margins": 0.016516733914613724,
      "rewards/rejected": -0.007359313778579235,
      "step": 1017
    },
    {
      "epoch": 0.26776701902053296,
      "grad_norm": 568.0,
      "learning_rate": 4.0687134502923973e-07,
      "logits/chosen": -0.6482601761817932,
      "logits/rejected": -0.6452585458755493,
      "logps/chosen": -1096.9281005859375,
      "logps/rejected": -1226.0303955078125,
      "loss": 0.6878,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0038145058788359165,
      "rewards/margins": 0.011468315497040749,
      "rewards/rejected": -0.015282820910215378,
      "step": 1018
    },
    {
      "epoch": 0.2680300514557201,
      "grad_norm": 510.0,
      "learning_rate": 4.067251461988304e-07,
      "logits/chosen": -0.6306020021438599,
      "logits/rejected": -0.6357886791229248,
      "logps/chosen": -907.8271484375,
      "logps/rejected": -713.8912963867188,
      "loss": 0.6915,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01126546785235405,
      "rewards/margins": 0.0040719034150242805,
      "rewards/rejected": 0.007193564437329769,
      "step": 1019
    },
    {
      "epoch": 0.2682930838909073,
      "grad_norm": 490.0,
      "learning_rate": 4.0657894736842103e-07,
      "logits/chosen": -0.6473131775856018,
      "logits/rejected": -0.654577910900116,
      "logps/chosen": -1087.4122314453125,
      "logps/rejected": -944.648193359375,
      "loss": 0.6916,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0011045439168810844,
      "rewards/margins": 0.00341720599681139,
      "rewards/rejected": -0.0023126602172851562,
      "step": 1020
    },
    {
      "epoch": 0.26855611632609444,
      "grad_norm": 576.0,
      "learning_rate": 4.064327485380117e-07,
      "logits/chosen": -0.6341025829315186,
      "logits/rejected": NaN,
      "logps/chosen": -1051.478759765625,
      "logps/rejected": -722.8448486328125,
      "loss": 0.7093,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.022113226354122162,
      "rewards/margins": -0.030350686982274055,
      "rewards/rejected": 0.008237458765506744,
      "step": 1021
    },
    {
      "epoch": 0.2688191487612816,
      "grad_norm": 544.0,
      "learning_rate": 4.062865497076023e-07,
      "logits/chosen": -0.6297544240951538,
      "logits/rejected": NaN,
      "logps/chosen": -1135.6759033203125,
      "logps/rejected": -804.2789306640625,
      "loss": 0.6946,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.016099167987704277,
      "rewards/margins": -0.0020067207515239716,
      "rewards/rejected": -0.014092444442212582,
      "step": 1022
    },
    {
      "epoch": 0.2690821811964688,
      "grad_norm": 532.0,
      "learning_rate": 4.0614035087719294e-07,
      "logits/chosen": -0.6543834209442139,
      "logits/rejected": -0.6592912077903748,
      "logps/chosen": -1258.0706787109375,
      "logps/rejected": -959.05126953125,
      "loss": 0.693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005361367017030716,
      "rewards/margins": 0.0010597226209938526,
      "rewards/rejected": -0.006421089172363281,
      "step": 1023
    },
    {
      "epoch": 0.269345213631656,
      "grad_norm": 506.0,
      "learning_rate": 4.059941520467836e-07,
      "logits/chosen": -0.6770811676979065,
      "logits/rejected": -0.665414571762085,
      "logps/chosen": -1224.322265625,
      "logps/rejected": -782.3323364257812,
      "loss": 0.6915,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0076469420455396175,
      "rewards/margins": 0.004037379287183285,
      "rewards/rejected": 0.0036095627583563328,
      "step": 1024
    },
    {
      "epoch": 0.26960824606684314,
      "grad_norm": 640.0,
      "learning_rate": 4.058479532163743e-07,
      "logits/chosen": -0.6459882259368896,
      "logits/rejected": -0.6583344340324402,
      "logps/chosen": -1651.23046875,
      "logps/rejected": -1213.6103515625,
      "loss": 0.7029,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.000999737298116088,
      "rewards/margins": -0.018663598224520683,
      "rewards/rejected": 0.017663858830928802,
      "step": 1025
    },
    {
      "epoch": 0.2698712785020303,
      "grad_norm": 560.0,
      "learning_rate": 4.0570175438596485e-07,
      "logits/chosen": -0.6399049162864685,
      "logits/rejected": -0.6419304609298706,
      "logps/chosen": -1423.80712890625,
      "logps/rejected": -1051.461669921875,
      "loss": 0.7078,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.01189050730317831,
      "rewards/margins": -0.028565123677253723,
      "rewards/rejected": 0.016674615442752838,
      "step": 1026
    },
    {
      "epoch": 0.27013431093721746,
      "grad_norm": 532.0,
      "learning_rate": 4.055555555555555e-07,
      "logits/chosen": -0.6651371717453003,
      "logits/rejected": -0.6697403192520142,
      "logps/chosen": -1137.0928955078125,
      "logps/rejected": -808.2890625,
      "loss": 0.7045,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.023144148290157318,
      "rewards/margins": -0.02153310738503933,
      "rewards/rejected": -0.0016110423021018505,
      "step": 1027
    },
    {
      "epoch": 0.2703973433724046,
      "grad_norm": 510.0,
      "learning_rate": 4.054093567251462e-07,
      "logits/chosen": -0.671063244342804,
      "logits/rejected": -0.6744346022605896,
      "logps/chosen": -1086.4698486328125,
      "logps/rejected": -842.3346557617188,
      "loss": 0.684,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.015654945746064186,
      "rewards/margins": 0.018865302205085754,
      "rewards/rejected": -0.0032103550620377064,
      "step": 1028
    },
    {
      "epoch": 0.2706603758075918,
      "grad_norm": 444.0,
      "learning_rate": 4.052631578947368e-07,
      "logits/chosen": -0.6387625932693481,
      "logits/rejected": -0.6432819962501526,
      "logps/chosen": -954.19140625,
      "logps/rejected": -764.9730224609375,
      "loss": 0.6956,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.015539407730102539,
      "rewards/margins": -0.004523373208940029,
      "rewards/rejected": 0.020062780007719994,
      "step": 1029
    },
    {
      "epoch": 0.27092340824277894,
      "grad_norm": 600.0,
      "learning_rate": 4.051169590643275e-07,
      "logits/chosen": -0.6484445333480835,
      "logits/rejected": -0.6709319353103638,
      "logps/chosen": -1171.1826171875,
      "logps/rejected": -957.633056640625,
      "loss": 0.6947,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.01604480668902397,
      "rewards/margins": -0.0023573869839310646,
      "rewards/rejected": -0.013687418773770332,
      "step": 1030
    },
    {
      "epoch": 0.2711864406779661,
      "grad_norm": 524.0,
      "learning_rate": 4.049707602339181e-07,
      "logits/chosen": -0.6443503499031067,
      "logits/rejected": -0.6526492834091187,
      "logps/chosen": -1342.5164794921875,
      "logps/rejected": -988.735595703125,
      "loss": 0.6988,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008691979572176933,
      "rewards/margins": -0.010394762270152569,
      "rewards/rejected": 0.0017027852591127157,
      "step": 1031
    },
    {
      "epoch": 0.27144947311315326,
      "grad_norm": 608.0,
      "learning_rate": 4.0482456140350873e-07,
      "logits/chosen": -0.6613576412200928,
      "logits/rejected": -0.6405103206634521,
      "logps/chosen": -1457.0675048828125,
      "logps/rejected": -983.2061767578125,
      "loss": 0.682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.008757686242461205,
      "rewards/margins": 0.024219799786806107,
      "rewards/rejected": -0.015462113544344902,
      "step": 1032
    },
    {
      "epoch": 0.2717125055483404,
      "grad_norm": 474.0,
      "learning_rate": 4.046783625730994e-07,
      "logits/chosen": -0.6843777298927307,
      "logits/rejected": -0.6832714080810547,
      "logps/chosen": -1142.8621826171875,
      "logps/rejected": -985.6036376953125,
      "loss": 0.6927,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.016361139714717865,
      "rewards/margins": 0.0014130594208836555,
      "rewards/rejected": -0.017774201929569244,
      "step": 1033
    },
    {
      "epoch": 0.2719755379835276,
      "grad_norm": 548.0,
      "learning_rate": 4.045321637426901e-07,
      "logits/chosen": -0.6453338861465454,
      "logits/rejected": -0.6428977847099304,
      "logps/chosen": -1425.409423828125,
      "logps/rejected": -1174.4830322265625,
      "loss": 0.6852,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0036478994879871607,
      "rewards/margins": 0.01706547848880291,
      "rewards/rejected": -0.020713377743959427,
      "step": 1034
    },
    {
      "epoch": 0.27223857041871474,
      "grad_norm": 780.0,
      "learning_rate": 4.0438596491228064e-07,
      "logits/chosen": -0.6549749970436096,
      "logits/rejected": -0.6607565879821777,
      "logps/chosen": -2038.29248046875,
      "logps/rejected": -1377.051025390625,
      "loss": 0.6994,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01269226148724556,
      "rewards/margins": -0.011702727526426315,
      "rewards/rejected": 0.024394989013671875,
      "step": 1035
    },
    {
      "epoch": 0.2725016028539019,
      "grad_norm": 544.0,
      "learning_rate": 4.042397660818713e-07,
      "logits/chosen": -0.6573097705841064,
      "logits/rejected": -0.6563469171524048,
      "logps/chosen": -1120.4471435546875,
      "logps/rejected": -755.455078125,
      "loss": 0.6842,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.014469431713223457,
      "rewards/margins": 0.019102096557617188,
      "rewards/rejected": -0.0046326639130711555,
      "step": 1036
    },
    {
      "epoch": 0.27276463528908906,
      "grad_norm": 560.0,
      "learning_rate": 4.04093567251462e-07,
      "logits/chosen": -0.6248221397399902,
      "logits/rejected": -0.6468011736869812,
      "logps/chosen": -1280.330078125,
      "logps/rejected": -824.321044921875,
      "loss": 0.6973,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.006325912661850452,
      "rewards/margins": -0.007974051870405674,
      "rewards/rejected": 0.0016481396742165089,
      "step": 1037
    },
    {
      "epoch": 0.2730276677242762,
      "grad_norm": 552.0,
      "learning_rate": 4.039473684210526e-07,
      "logits/chosen": -0.6455187797546387,
      "logits/rejected": -0.6577026844024658,
      "logps/chosen": -1116.836669921875,
      "logps/rejected": -979.5321655273438,
      "loss": 0.7037,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.004582976922392845,
      "rewards/margins": -0.02052764780819416,
      "rewards/rejected": 0.015944670885801315,
      "step": 1038
    },
    {
      "epoch": 0.27329070015946344,
      "grad_norm": 462.0,
      "learning_rate": 4.0380116959064323e-07,
      "logits/chosen": -0.6272568106651306,
      "logits/rejected": -0.6326680779457092,
      "logps/chosen": -912.9580078125,
      "logps/rejected": -836.4239501953125,
      "loss": 0.6897,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0023252489045262337,
      "rewards/margins": 0.007360459305346012,
      "rewards/rejected": -0.005035209935158491,
      "step": 1039
    },
    {
      "epoch": 0.2735537325946506,
      "grad_norm": 652.0,
      "learning_rate": 4.036549707602339e-07,
      "logits/chosen": -0.6198249459266663,
      "logits/rejected": NaN,
      "logps/chosen": -1095.0455322265625,
      "logps/rejected": -918.97802734375,
      "loss": 0.6917,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00013084517559036613,
      "rewards/margins": 0.0034241671673953533,
      "rewards/rejected": -0.0032933237962424755,
      "step": 1040
    },
    {
      "epoch": 0.27381676502983776,
      "grad_norm": 540.0,
      "learning_rate": 4.035087719298245e-07,
      "logits/chosen": -0.6300958395004272,
      "logits/rejected": -0.6291159987449646,
      "logps/chosen": -1301.055419921875,
      "logps/rejected": -1030.632568359375,
      "loss": 0.7129,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.041016578674316406,
      "rewards/margins": -0.0378628708422184,
      "rewards/rejected": -0.0031537057366222143,
      "step": 1041
    },
    {
      "epoch": 0.2740797974650249,
      "grad_norm": 520.0,
      "learning_rate": 4.033625730994152e-07,
      "logits/chosen": -0.6578852534294128,
      "logits/rejected": -0.6564586758613586,
      "logps/chosen": -1141.5133056640625,
      "logps/rejected": -1113.15234375,
      "loss": 0.6919,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0006036749109625816,
      "rewards/margins": 0.0028600695077329874,
      "rewards/rejected": -0.0022563934326171875,
      "step": 1042
    },
    {
      "epoch": 0.2743428299002121,
      "grad_norm": 532.0,
      "learning_rate": 4.0321637426900587e-07,
      "logits/chosen": -0.6477283239364624,
      "logits/rejected": -0.6497571468353271,
      "logps/chosen": -1108.3118896484375,
      "logps/rejected": -871.4352416992188,
      "loss": 0.6798,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.022542094811797142,
      "rewards/margins": 0.028749560937285423,
      "rewards/rejected": -0.006207465659826994,
      "step": 1043
    },
    {
      "epoch": 0.27460586233539924,
      "grad_norm": 426.0,
      "learning_rate": 4.0307017543859643e-07,
      "logits/chosen": -0.6572578549385071,
      "logits/rejected": -0.667364239692688,
      "logps/chosen": -1223.8636474609375,
      "logps/rejected": -991.6317138671875,
      "loss": 0.697,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.004730844404548407,
      "rewards/margins": -0.006937360391020775,
      "rewards/rejected": 0.002206516917794943,
      "step": 1044
    },
    {
      "epoch": 0.2748688947705864,
      "grad_norm": 608.0,
      "learning_rate": 4.029239766081871e-07,
      "logits/chosen": -0.6589837074279785,
      "logits/rejected": -0.652048647403717,
      "logps/chosen": -1467.8963623046875,
      "logps/rejected": -1086.802734375,
      "loss": 0.6915,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004387472290545702,
      "rewards/margins": 0.0039678579196333885,
      "rewards/rejected": 0.00041961669921875,
      "step": 1045
    },
    {
      "epoch": 0.27513192720577356,
      "grad_norm": 524.0,
      "learning_rate": 4.027777777777778e-07,
      "logits/chosen": -0.6680090427398682,
      "logits/rejected": -0.6828302145004272,
      "logps/chosen": -844.5335083007812,
      "logps/rejected": -874.327880859375,
      "loss": 0.7015,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.016024112701416016,
      "rewards/margins": -0.01646137237548828,
      "rewards/rejected": 0.0004372596740722656,
      "step": 1046
    },
    {
      "epoch": 0.2753949596409607,
      "grad_norm": 500.0,
      "learning_rate": 4.026315789473684e-07,
      "logits/chosen": -0.6499308943748474,
      "logits/rejected": -0.6490020751953125,
      "logps/chosen": -999.574951171875,
      "logps/rejected": -806.8080444335938,
      "loss": 0.6944,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0010422703344374895,
      "rewards/margins": -0.0019489277619868517,
      "rewards/rejected": 0.0009066583588719368,
      "step": 1047
    },
    {
      "epoch": 0.2756579920761479,
      "grad_norm": 496.0,
      "learning_rate": 4.02485380116959e-07,
      "logits/chosen": -0.6392037272453308,
      "logits/rejected": -0.6428118348121643,
      "logps/chosen": -1046.062255859375,
      "logps/rejected": -879.236572265625,
      "loss": 0.7044,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0017994889058172703,
      "rewards/margins": -0.02157135121524334,
      "rewards/rejected": 0.019771862775087357,
      "step": 1048
    },
    {
      "epoch": 0.27592102451133504,
      "grad_norm": 498.0,
      "learning_rate": 4.023391812865497e-07,
      "logits/chosen": -0.6390805840492249,
      "logits/rejected": -0.6519802808761597,
      "logps/chosen": -1466.98779296875,
      "logps/rejected": -1126.1260986328125,
      "loss": 0.6894,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.024661540985107422,
      "rewards/margins": 0.008351897820830345,
      "rewards/rejected": 0.016309643164277077,
      "step": 1049
    },
    {
      "epoch": 0.2761840569465222,
      "grad_norm": 428.0,
      "learning_rate": 4.021929824561403e-07,
      "logits/chosen": -0.6431736946105957,
      "logits/rejected": -0.643937349319458,
      "logps/chosen": -748.3729858398438,
      "logps/rejected": -633.919189453125,
      "loss": 0.6999,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0038834568113088608,
      "rewards/margins": -0.012446120381355286,
      "rewards/rejected": 0.016329577192664146,
      "step": 1050
    },
    {
      "epoch": 0.27644708938170937,
      "grad_norm": 604.0,
      "learning_rate": 4.02046783625731e-07,
      "logits/chosen": -0.6724680066108704,
      "logits/rejected": -0.6622713804244995,
      "logps/chosen": -1019.9321899414062,
      "logps/rejected": -995.2294311523438,
      "loss": 0.6929,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006949901580810547,
      "rewards/margins": 0.0015517231076955795,
      "rewards/rejected": 0.005398178938776255,
      "step": 1051
    },
    {
      "epoch": 0.2767101218168965,
      "grad_norm": 502.0,
      "learning_rate": 4.019005847953216e-07,
      "logits/chosen": -0.6348897218704224,
      "logits/rejected": -0.6536870002746582,
      "logps/chosen": -1055.6990966796875,
      "logps/rejected": -704.1910400390625,
      "loss": 0.6954,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0005188941722735763,
      "rewards/margins": -0.004099654965102673,
      "rewards/rejected": 0.004618549719452858,
      "step": 1052
    },
    {
      "epoch": 0.2769731542520837,
      "grad_norm": 648.0,
      "learning_rate": 4.017543859649123e-07,
      "logits/chosen": -0.6985424757003784,
      "logits/rejected": -0.6935781836509705,
      "logps/chosen": -1471.421630859375,
      "logps/rejected": -1011.7919311523438,
      "loss": 0.7093,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.041571810841560364,
      "rewards/margins": -0.03134122118353844,
      "rewards/rejected": -0.010230587795376778,
      "step": 1053
    },
    {
      "epoch": 0.27723618668727085,
      "grad_norm": 708.0,
      "learning_rate": 4.016081871345029e-07,
      "logits/chosen": -0.6696173548698425,
      "logits/rejected": -0.6578867435455322,
      "logps/chosen": -1362.9827880859375,
      "logps/rejected": -883.3330078125,
      "loss": 0.7088,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.010583782568573952,
      "rewards/margins": -0.029383087530732155,
      "rewards/rejected": 0.018799306824803352,
      "step": 1054
    },
    {
      "epoch": 0.27749921912245806,
      "grad_norm": 640.0,
      "learning_rate": 4.0146198830409357e-07,
      "logits/chosen": -0.6689469814300537,
      "logits/rejected": -0.6577019095420837,
      "logps/chosen": -1394.3807373046875,
      "logps/rejected": -1403.1754150390625,
      "loss": 0.6764,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01550913043320179,
      "rewards/margins": 0.03481416776776314,
      "rewards/rejected": -0.019305039197206497,
      "step": 1055
    },
    {
      "epoch": 0.2777622515576452,
      "grad_norm": 516.0,
      "learning_rate": 4.0131578947368424e-07,
      "logits/chosen": -0.6485460996627808,
      "logits/rejected": -0.6485535502433777,
      "logps/chosen": -1081.518310546875,
      "logps/rejected": -889.07373046875,
      "loss": 0.7088,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.03189220279455185,
      "rewards/margins": -0.030414771288633347,
      "rewards/rejected": -0.0014774315059185028,
      "step": 1056
    },
    {
      "epoch": 0.2780252839928324,
      "grad_norm": 548.0,
      "learning_rate": 4.011695906432748e-07,
      "logits/chosen": -0.6674666404724121,
      "logits/rejected": -0.670915961265564,
      "logps/chosen": -1526.540771484375,
      "logps/rejected": -1271.43505859375,
      "loss": 0.6915,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0017406472470611334,
      "rewards/margins": 0.004083204548805952,
      "rewards/rejected": -0.0023425575345754623,
      "step": 1057
    },
    {
      "epoch": 0.27828831642801954,
      "grad_norm": 660.0,
      "learning_rate": 4.010233918128655e-07,
      "logits/chosen": -0.6287864446640015,
      "logits/rejected": -0.6144223809242249,
      "logps/chosen": -1551.77880859375,
      "logps/rejected": -1243.47412109375,
      "loss": 0.6917,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.001873302273452282,
      "rewards/margins": 0.00358400447294116,
      "rewards/rejected": -0.0054573058150708675,
      "step": 1058
    },
    {
      "epoch": 0.2785513488632067,
      "grad_norm": 548.0,
      "learning_rate": 4.0087719298245616e-07,
      "logits/chosen": -0.6580462455749512,
      "logits/rejected": -0.6626896262168884,
      "logps/chosen": -1324.8314208984375,
      "logps/rejected": -982.5443115234375,
      "loss": 0.6785,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.002459621289744973,
      "rewards/margins": 0.030433937907218933,
      "rewards/rejected": -0.027974320575594902,
      "step": 1059
    },
    {
      "epoch": 0.27881438129839387,
      "grad_norm": 608.0,
      "learning_rate": 4.007309941520468e-07,
      "logits/chosen": -0.6486920118331909,
      "logits/rejected": -0.646702766418457,
      "logps/chosen": -1525.3629150390625,
      "logps/rejected": -1147.85888671875,
      "loss": 0.6865,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.002652360126376152,
      "rewards/margins": 0.015105819329619408,
      "rewards/rejected": -0.01775817945599556,
      "step": 1060
    },
    {
      "epoch": 0.279077413733581,
      "grad_norm": 736.0,
      "learning_rate": 4.005847953216374e-07,
      "logits/chosen": -0.6587094664573669,
      "logits/rejected": NaN,
      "logps/chosen": -1362.7982177734375,
      "logps/rejected": -1066.773193359375,
      "loss": 0.7059,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03515692055225372,
      "rewards/margins": -0.023712826892733574,
      "rewards/rejected": -0.011444092728197575,
      "step": 1061
    },
    {
      "epoch": 0.2793404461687682,
      "grad_norm": 612.0,
      "learning_rate": 4.0043859649122807e-07,
      "logits/chosen": -0.6622225642204285,
      "logits/rejected": NaN,
      "logps/chosen": -1364.10986328125,
      "logps/rejected": -719.5587158203125,
      "loss": 0.6988,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0019316681427881122,
      "rewards/margins": -0.010675334371626377,
      "rewards/rejected": 0.008743667975068092,
      "step": 1062
    },
    {
      "epoch": 0.27960347860395535,
      "grad_norm": 446.0,
      "learning_rate": 4.002923976608187e-07,
      "logits/chosen": -0.6447153091430664,
      "logits/rejected": -0.6519464254379272,
      "logps/chosen": -1077.30712890625,
      "logps/rejected": -972.4873657226562,
      "loss": 0.6864,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.00990209635347128,
      "rewards/margins": 0.014511967077851295,
      "rewards/rejected": -0.004609870724380016,
      "step": 1063
    },
    {
      "epoch": 0.2798665110391425,
      "grad_norm": 484.0,
      "learning_rate": 4.0014619883040936e-07,
      "logits/chosen": -0.6251311302185059,
      "logits/rejected": -0.6436154842376709,
      "logps/chosen": -1161.9310302734375,
      "logps/rejected": -800.1726684570312,
      "loss": 0.6902,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01748824119567871,
      "rewards/margins": 0.006567095406353474,
      "rewards/rejected": 0.010921146720647812,
      "step": 1064
    },
    {
      "epoch": 0.28012954347432967,
      "grad_norm": 652.0,
      "learning_rate": 4e-07,
      "logits/chosen": -0.6606945395469666,
      "logits/rejected": -0.6644086241722107,
      "logps/chosen": -1218.6329345703125,
      "logps/rejected": -1057.517578125,
      "loss": 0.6887,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005452441982924938,
      "rewards/margins": 0.010922527872025967,
      "rewards/rejected": -0.005470084957778454,
      "step": 1065
    },
    {
      "epoch": 0.28039257590951683,
      "grad_norm": 564.0,
      "learning_rate": 3.998538011695906e-07,
      "logits/chosen": -0.6540858149528503,
      "logits/rejected": -0.6499563455581665,
      "logps/chosen": -1694.9921875,
      "logps/rejected": -1344.332275390625,
      "loss": 0.6857,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01543431170284748,
      "rewards/margins": 0.01557936891913414,
      "rewards/rejected": -0.00014505372382700443,
      "step": 1066
    },
    {
      "epoch": 0.280655608344704,
      "grad_norm": 450.0,
      "learning_rate": 3.997076023391813e-07,
      "logits/chosen": -0.6505088806152344,
      "logits/rejected": NaN,
      "logps/chosen": -849.244384765625,
      "logps/rejected": -551.5653076171875,
      "loss": 0.6857,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.007631110958755016,
      "rewards/margins": 0.01577119715511799,
      "rewards/rejected": -0.008140088059008121,
      "step": 1067
    },
    {
      "epoch": 0.28091864077989115,
      "grad_norm": 668.0,
      "learning_rate": 3.9956140350877195e-07,
      "logits/chosen": -0.6533024311065674,
      "logits/rejected": -0.6514294743537903,
      "logps/chosen": -1018.693115234375,
      "logps/rejected": -1225.770263671875,
      "loss": 0.6856,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0031176097691059113,
      "rewards/margins": 0.01614232175052166,
      "rewards/rejected": -0.019259929656982422,
      "step": 1068
    },
    {
      "epoch": 0.2811816732150783,
      "grad_norm": 580.0,
      "learning_rate": 3.994152046783625e-07,
      "logits/chosen": -0.6461912393569946,
      "logits/rejected": -0.6499056816101074,
      "logps/chosen": -1033.8515625,
      "logps/rejected": -833.648681640625,
      "loss": 0.6943,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007407474331557751,
      "rewards/margins": -0.0019769666250795126,
      "rewards/rejected": 0.009384441189467907,
      "step": 1069
    },
    {
      "epoch": 0.2814447056502655,
      "grad_norm": 528.0,
      "learning_rate": 3.992690058479532e-07,
      "logits/chosen": -0.6559661626815796,
      "logits/rejected": -0.6630631685256958,
      "logps/chosen": -1128.34033203125,
      "logps/rejected": -806.8338012695312,
      "loss": 0.6764,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.013474847190082073,
      "rewards/margins": 0.0352293960750103,
      "rewards/rejected": -0.0217545498162508,
      "step": 1070
    },
    {
      "epoch": 0.2817077380854527,
      "grad_norm": 506.0,
      "learning_rate": 3.9912280701754386e-07,
      "logits/chosen": -0.6522818803787231,
      "logits/rejected": -0.6557843089103699,
      "logps/chosen": -963.8052978515625,
      "logps/rejected": -631.7890625,
      "loss": 0.6855,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01121444720774889,
      "rewards/margins": 0.016024446114897728,
      "rewards/rejected": -0.004809999838471413,
      "step": 1071
    },
    {
      "epoch": 0.28197077052063985,
      "grad_norm": 720.0,
      "learning_rate": 3.989766081871345e-07,
      "logits/chosen": -0.6444928050041199,
      "logits/rejected": -0.6444381475448608,
      "logps/chosen": -1321.5087890625,
      "logps/rejected": -840.2308349609375,
      "loss": 0.6983,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.016059687361121178,
      "rewards/margins": -0.009524965658783913,
      "rewards/rejected": -0.006534719839692116,
      "step": 1072
    },
    {
      "epoch": 0.282233802955827,
      "grad_norm": 502.0,
      "learning_rate": 3.9883040935672515e-07,
      "logits/chosen": -0.6765406131744385,
      "logits/rejected": -0.6774627566337585,
      "logps/chosen": -893.0484619140625,
      "logps/rejected": -597.6612548828125,
      "loss": 0.6907,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.024167824536561966,
      "rewards/margins": 0.005793476477265358,
      "rewards/rejected": -0.029961299151182175,
      "step": 1073
    },
    {
      "epoch": 0.28249683539101417,
      "grad_norm": 616.0,
      "learning_rate": 3.9868421052631577e-07,
      "logits/chosen": -0.6292068958282471,
      "logits/rejected": -0.6361256241798401,
      "logps/chosen": -1242.5374755859375,
      "logps/rejected": -798.916259765625,
      "loss": 0.691,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0031457911245524883,
      "rewards/margins": 0.004988383036106825,
      "rewards/rejected": -0.008134175091981888,
      "step": 1074
    },
    {
      "epoch": 0.2827598678262013,
      "grad_norm": 512.0,
      "learning_rate": 3.985380116959064e-07,
      "logits/chosen": -0.675474226474762,
      "logits/rejected": -0.6800715923309326,
      "logps/chosen": -1292.0389404296875,
      "logps/rejected": -1060.959716796875,
      "loss": 0.6933,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0027821557596325874,
      "rewards/margins": 0.00022268295288085938,
      "rewards/rejected": 0.002559470944106579,
      "step": 1075
    },
    {
      "epoch": 0.2830229002613885,
      "grad_norm": 568.0,
      "learning_rate": 3.9839181286549707e-07,
      "logits/chosen": -0.6448620557785034,
      "logits/rejected": -0.6470901370048523,
      "logps/chosen": -971.21630859375,
      "logps/rejected": -839.1351928710938,
      "loss": 0.7006,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.021428776904940605,
      "rewards/margins": -0.013625336810946465,
      "rewards/rejected": -0.007803440559655428,
      "step": 1076
    },
    {
      "epoch": 0.28328593269657565,
      "grad_norm": 490.0,
      "learning_rate": 3.9824561403508774e-07,
      "logits/chosen": -0.6475868225097656,
      "logits/rejected": -0.6536620855331421,
      "logps/chosen": -983.6138916015625,
      "logps/rejected": -753.2796630859375,
      "loss": 0.6856,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.02313385158777237,
      "rewards/margins": 0.01576995849609375,
      "rewards/rejected": 0.00736389122903347,
      "step": 1077
    },
    {
      "epoch": 0.2835489651317628,
      "grad_norm": 600.0,
      "learning_rate": 3.980994152046783e-07,
      "logits/chosen": -0.6549342274665833,
      "logits/rejected": -0.6608887910842896,
      "logps/chosen": -1590.4827880859375,
      "logps/rejected": -1010.6416015625,
      "loss": 0.6843,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.013540267944335938,
      "rewards/margins": 0.01871509477496147,
      "rewards/rejected": -0.03225536644458771,
      "step": 1078
    },
    {
      "epoch": 0.28381199756694997,
      "grad_norm": 516.0,
      "learning_rate": 3.97953216374269e-07,
      "logits/chosen": -0.6431609392166138,
      "logits/rejected": -0.6567973494529724,
      "logps/chosen": -1244.2508544921875,
      "logps/rejected": -638.1507568359375,
      "loss": 0.6851,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.00706100556999445,
      "rewards/margins": 0.01667804643511772,
      "rewards/rejected": -0.009617042727768421,
      "step": 1079
    },
    {
      "epoch": 0.28407503000213713,
      "grad_norm": 640.0,
      "learning_rate": 3.9780701754385965e-07,
      "logits/chosen": -0.6595245599746704,
      "logits/rejected": -0.6650901436805725,
      "logps/chosen": -1400.343017578125,
      "logps/rejected": -858.8358764648438,
      "loss": 0.6914,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0042306892573833466,
      "rewards/margins": 0.004261159338057041,
      "rewards/rejected": -3.047008067369461e-05,
      "step": 1080
    },
    {
      "epoch": 0.2843380624373243,
      "grad_norm": 464.0,
      "learning_rate": 3.9766081871345027e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6800839304924011,
      "logps/chosen": -888.3003540039062,
      "logps/rejected": -709.5090942382812,
      "loss": 0.6835,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.011880588717758656,
      "rewards/margins": 0.02020406909286976,
      "rewards/rejected": -0.03208465874195099,
      "step": 1081
    },
    {
      "epoch": 0.28460109487251145,
      "grad_norm": 500.0,
      "learning_rate": 3.975146198830409e-07,
      "logits/chosen": -0.641857922077179,
      "logits/rejected": -0.6529372930526733,
      "logps/chosen": -1324.8433837890625,
      "logps/rejected": -853.215576171875,
      "loss": 0.6907,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006160259246826172,
      "rewards/margins": 0.0053081512451171875,
      "rewards/rejected": 0.0008521080017089844,
      "step": 1082
    },
    {
      "epoch": 0.2848641273076986,
      "grad_norm": 482.0,
      "learning_rate": 3.9736842105263156e-07,
      "logits/chosen": -0.6511812806129456,
      "logits/rejected": -0.6514129042625427,
      "logps/chosen": -1105.843017578125,
      "logps/rejected": -995.6561889648438,
      "loss": 0.6904,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0018967627547681332,
      "rewards/margins": 0.006276320666074753,
      "rewards/rejected": -0.008173083886504173,
      "step": 1083
    },
    {
      "epoch": 0.28512715974288577,
      "grad_norm": 572.0,
      "learning_rate": 3.972222222222222e-07,
      "logits/chosen": -0.6821600794792175,
      "logits/rejected": -0.6799065470695496,
      "logps/chosen": -1416.904296875,
      "logps/rejected": -1085.01611328125,
      "loss": 0.6946,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.011042642407119274,
      "rewards/margins": -0.0021338933147490025,
      "rewards/rejected": 0.013176538050174713,
      "step": 1084
    },
    {
      "epoch": 0.28539019217807293,
      "grad_norm": 490.0,
      "learning_rate": 3.9707602339181286e-07,
      "logits/chosen": -0.6443455815315247,
      "logits/rejected": -0.6426514387130737,
      "logps/chosen": -902.3875732421875,
      "logps/rejected": -1017.5140991210938,
      "loss": 0.7002,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004138373304158449,
      "rewards/margins": -0.013414382003247738,
      "rewards/rejected": 0.017552759498357773,
      "step": 1085
    },
    {
      "epoch": 0.28565322461326015,
      "grad_norm": 540.0,
      "learning_rate": 3.9692982456140353e-07,
      "logits/chosen": -0.6394150853157043,
      "logits/rejected": -0.6582819223403931,
      "logps/chosen": -1546.87451171875,
      "logps/rejected": -1016.984619140625,
      "loss": 0.6915,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.02687063254415989,
      "rewards/margins": 0.003974056802690029,
      "rewards/rejected": 0.022896576672792435,
      "step": 1086
    },
    {
      "epoch": 0.2859162570484473,
      "grad_norm": 572.0,
      "learning_rate": 3.967836257309941e-07,
      "logits/chosen": -0.6577343940734863,
      "logits/rejected": -0.6749870777130127,
      "logps/chosen": -1248.0848388671875,
      "logps/rejected": -1015.6790161132812,
      "loss": 0.6935,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.002581881359219551,
      "rewards/margins": -0.00041046133264899254,
      "rewards/rejected": 0.00299234502017498,
      "step": 1087
    },
    {
      "epoch": 0.28617928948363447,
      "grad_norm": 592.0,
      "learning_rate": 3.9663742690058477e-07,
      "logits/chosen": -0.6763981580734253,
      "logits/rejected": -0.6729088425636292,
      "logps/chosen": -1507.36767578125,
      "logps/rejected": -1187.1527099609375,
      "loss": 0.6776,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.020608900114893913,
      "rewards/margins": 0.03188915178179741,
      "rewards/rejected": -0.011280251666903496,
      "step": 1088
    },
    {
      "epoch": 0.28644232191882163,
      "grad_norm": 552.0,
      "learning_rate": 3.9649122807017544e-07,
      "logits/chosen": -0.6562731862068176,
      "logits/rejected": -0.6523978114128113,
      "logps/chosen": -1092.10400390625,
      "logps/rejected": -1012.0548095703125,
      "loss": 0.6902,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02977742999792099,
      "rewards/margins": 0.006187010556459427,
      "rewards/rejected": 0.023590419441461563,
      "step": 1089
    },
    {
      "epoch": 0.2867053543540088,
      "grad_norm": 676.0,
      "learning_rate": 3.9634502923976606e-07,
      "logits/chosen": -0.6675021648406982,
      "logits/rejected": -0.6733320355415344,
      "logps/chosen": -1472.19580078125,
      "logps/rejected": -1305.9508056640625,
      "loss": 0.6848,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01312789972871542,
      "rewards/margins": 0.017726514488458633,
      "rewards/rejected": -0.0045986175537109375,
      "step": 1090
    },
    {
      "epoch": 0.28696838678919595,
      "grad_norm": 464.0,
      "learning_rate": 3.961988304093567e-07,
      "logits/chosen": -0.6479125022888184,
      "logits/rejected": NaN,
      "logps/chosen": -923.3173828125,
      "logps/rejected": -747.8340454101562,
      "loss": 0.7003,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.015176677145063877,
      "rewards/margins": -0.013687802478671074,
      "rewards/rejected": -0.0014888760633766651,
      "step": 1091
    },
    {
      "epoch": 0.2872314192243831,
      "grad_norm": 516.0,
      "learning_rate": 3.9605263157894735e-07,
      "logits/chosen": -0.6519966125488281,
      "logits/rejected": -0.6520751714706421,
      "logps/chosen": -1096.9451904296875,
      "logps/rejected": -902.6546630859375,
      "loss": 0.6943,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005916595458984375,
      "rewards/margins": -0.0018790247850120068,
      "rewards/rejected": -0.004037571605294943,
      "step": 1092
    },
    {
      "epoch": 0.28749445165957027,
      "grad_norm": 500.0,
      "learning_rate": 3.95906432748538e-07,
      "logits/chosen": -0.6587938070297241,
      "logits/rejected": -0.6647286415100098,
      "logps/chosen": -1068.1358642578125,
      "logps/rejected": -792.2522583007812,
      "loss": 0.6889,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0004973406903445721,
      "rewards/margins": 0.008889676071703434,
      "rewards/rejected": -0.009387016296386719,
      "step": 1093
    },
    {
      "epoch": 0.28775748409475743,
      "grad_norm": 540.0,
      "learning_rate": 3.9576023391812865e-07,
      "logits/chosen": -0.6244847774505615,
      "logits/rejected": -0.640779435634613,
      "logps/chosen": -1099.9847412109375,
      "logps/rejected": -693.1487426757812,
      "loss": 0.6845,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.004090690053999424,
      "rewards/margins": 0.01818551868200302,
      "rewards/rejected": -0.02227621152997017,
      "step": 1094
    },
    {
      "epoch": 0.2880205165299446,
      "grad_norm": 544.0,
      "learning_rate": 3.9561403508771927e-07,
      "logits/chosen": -0.64161217212677,
      "logits/rejected": -0.6395913362503052,
      "logps/chosen": -1026.69970703125,
      "logps/rejected": -741.8045043945312,
      "loss": 0.6899,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.002894497709348798,
      "rewards/margins": 0.006994294468313456,
      "rewards/rejected": -0.009888792410492897,
      "step": 1095
    },
    {
      "epoch": 0.28828354896513175,
      "grad_norm": 540.0,
      "learning_rate": 3.9546783625730994e-07,
      "logits/chosen": -0.6400676369667053,
      "logits/rejected": -0.6583520174026489,
      "logps/chosen": -919.5755004882812,
      "logps/rejected": -692.9663696289062,
      "loss": 0.6927,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.01418161392211914,
      "rewards/margins": 0.0015888204798102379,
      "rewards/rejected": 0.012592793442308903,
      "step": 1096
    },
    {
      "epoch": 0.2885465814003189,
      "grad_norm": 664.0,
      "learning_rate": 3.9532163742690056e-07,
      "logits/chosen": -0.6604154706001282,
      "logits/rejected": -0.6691626310348511,
      "logps/chosen": -1367.509521484375,
      "logps/rejected": -994.281005859375,
      "loss": 0.6991,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.001790046226233244,
      "rewards/margins": -0.010657502338290215,
      "rewards/rejected": 0.012447549030184746,
      "step": 1097
    },
    {
      "epoch": 0.2888096138355061,
      "grad_norm": 656.0,
      "learning_rate": 3.9517543859649123e-07,
      "logits/chosen": -0.6297842264175415,
      "logits/rejected": -0.6276374459266663,
      "logps/chosen": -936.4862670898438,
      "logps/rejected": -921.6547241210938,
      "loss": 0.6944,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.015913009643554688,
      "rewards/margins": -0.0020028119906783104,
      "rewards/rejected": -0.013910198584198952,
      "step": 1098
    },
    {
      "epoch": 0.28907264627069323,
      "grad_norm": 524.0,
      "learning_rate": 3.9502923976608185e-07,
      "logits/chosen": -0.6552160978317261,
      "logits/rejected": -0.6465417146682739,
      "logps/chosen": -1140.1976318359375,
      "logps/rejected": -699.2754516601562,
      "loss": 0.699,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.008876037783920765,
      "rewards/margins": -0.011013984680175781,
      "rewards/rejected": 0.0021379468962550163,
      "step": 1099
    },
    {
      "epoch": 0.2893356787058804,
      "grad_norm": 752.0,
      "learning_rate": 3.9488304093567247e-07,
      "logits/chosen": -0.6434788703918457,
      "logits/rejected": -0.6380988359451294,
      "logps/chosen": -1173.2109375,
      "logps/rejected": -1067.8865966796875,
      "loss": 0.6886,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.008067083545029163,
      "rewards/margins": 0.009949445724487305,
      "rewards/rejected": -0.0018823626451194286,
      "step": 1100
    },
    {
      "epoch": 0.28959871114106756,
      "grad_norm": 644.0,
      "learning_rate": 3.9473684210526315e-07,
      "logits/chosen": -0.6700547337532043,
      "logits/rejected": NaN,
      "logps/chosen": -1484.2979736328125,
      "logps/rejected": -1201.113037109375,
      "loss": 0.6896,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.023326683789491653,
      "rewards/margins": 0.007540607824921608,
      "rewards/rejected": 0.015786075964570045,
      "step": 1101
    },
    {
      "epoch": 0.28986174357625477,
      "grad_norm": 564.0,
      "learning_rate": 3.945906432748538e-07,
      "logits/chosen": -0.6502552628517151,
      "logits/rejected": -0.630970299243927,
      "logps/chosen": -942.251953125,
      "logps/rejected": -836.413818359375,
      "loss": 0.6997,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.006632041651755571,
      "rewards/margins": -0.012365818955004215,
      "rewards/rejected": 0.018997859209775925,
      "step": 1102
    },
    {
      "epoch": 0.29012477601144193,
      "grad_norm": 544.0,
      "learning_rate": 3.9444444444444444e-07,
      "logits/chosen": -0.6563892960548401,
      "logits/rejected": -0.664709746837616,
      "logps/chosen": -1101.8289794921875,
      "logps/rejected": -754.6497192382812,
      "loss": 0.6999,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.00865478441119194,
      "rewards/margins": -0.013065433129668236,
      "rewards/rejected": 0.0044106487184762955,
      "step": 1103
    },
    {
      "epoch": 0.2903878084466291,
      "grad_norm": 416.0,
      "learning_rate": 3.9429824561403506e-07,
      "logits/chosen": -0.6508435606956482,
      "logits/rejected": -0.6530235409736633,
      "logps/chosen": -679.16552734375,
      "logps/rejected": -681.4424438476562,
      "loss": 0.6868,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.010742474347352982,
      "rewards/margins": 0.013218117877840996,
      "rewards/rejected": -0.023960592225193977,
      "step": 1104
    },
    {
      "epoch": 0.29065084088181625,
      "grad_norm": 504.0,
      "learning_rate": 3.9415204678362573e-07,
      "logits/chosen": -0.6443397998809814,
      "logits/rejected": -0.641141414642334,
      "logps/chosen": -959.738037109375,
      "logps/rejected": -734.219482421875,
      "loss": 0.6924,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0036230087280273438,
      "rewards/margins": 0.0018666267860680819,
      "rewards/rejected": 0.0017563833389431238,
      "step": 1105
    },
    {
      "epoch": 0.2909138733170034,
      "grad_norm": 596.0,
      "learning_rate": 3.9400584795321635e-07,
      "logits/chosen": -0.6584608554840088,
      "logits/rejected": -0.6594254970550537,
      "logps/chosen": -1360.1788330078125,
      "logps/rejected": -892.3133544921875,
      "loss": 0.7057,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.011450959369540215,
      "rewards/margins": -0.02356114238500595,
      "rewards/rejected": 0.012110184878110886,
      "step": 1106
    },
    {
      "epoch": 0.2911769057521906,
      "grad_norm": 484.0,
      "learning_rate": 3.93859649122807e-07,
      "logits/chosen": -0.6713796854019165,
      "logits/rejected": -0.659874439239502,
      "logps/chosen": -1149.4864501953125,
      "logps/rejected": -632.2786865234375,
      "loss": 0.704,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.011684132739901543,
      "rewards/margins": -0.01953096315264702,
      "rewards/rejected": 0.007846832275390625,
      "step": 1107
    },
    {
      "epoch": 0.29143993818737773,
      "grad_norm": 480.0,
      "learning_rate": 3.9371345029239764e-07,
      "logits/chosen": -0.656561017036438,
      "logits/rejected": -0.6733439564704895,
      "logps/chosen": -820.1072387695312,
      "logps/rejected": -527.5010375976562,
      "loss": 0.6965,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.006224250886589289,
      "rewards/margins": -0.006452607922255993,
      "rewards/rejected": 0.00022835703566670418,
      "step": 1108
    },
    {
      "epoch": 0.2917029706225649,
      "grad_norm": 548.0,
      "learning_rate": 3.9356725146198826e-07,
      "logits/chosen": -0.6555590629577637,
      "logits/rejected": -0.6615183353424072,
      "logps/chosen": -1250.7855224609375,
      "logps/rejected": -930.3319702148438,
      "loss": 0.6846,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00491447513923049,
      "rewards/margins": 0.01793537102639675,
      "rewards/rejected": -0.02284984663128853,
      "step": 1109
    },
    {
      "epoch": 0.29196600305775205,
      "grad_norm": 572.0,
      "learning_rate": 3.9342105263157894e-07,
      "logits/chosen": -0.6529999375343323,
      "logits/rejected": -0.6603941917419434,
      "logps/chosen": -982.0807495117188,
      "logps/rejected": -799.5562133789062,
      "loss": 0.7032,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.002075863303616643,
      "rewards/margins": -0.018979262560606003,
      "rewards/rejected": 0.021055126562714577,
      "step": 1110
    },
    {
      "epoch": 0.2922290354929392,
      "grad_norm": 584.0,
      "learning_rate": 3.932748538011696e-07,
      "logits/chosen": -0.6269161105155945,
      "logits/rejected": -0.6351515054702759,
      "logps/chosen": -1283.7047119140625,
      "logps/rejected": -1072.7264404296875,
      "loss": 0.6915,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03722190856933594,
      "rewards/margins": 0.004316900856792927,
      "rewards/rejected": 0.03290500491857529,
      "step": 1111
    },
    {
      "epoch": 0.2924920679281264,
      "grad_norm": 608.0,
      "learning_rate": 3.931286549707602e-07,
      "logits/chosen": -0.6237962245941162,
      "logits/rejected": -0.630757212638855,
      "logps/chosen": -1195.387939453125,
      "logps/rejected": -1139.040283203125,
      "loss": 0.6805,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008668709546327591,
      "rewards/margins": 0.0270098689943552,
      "rewards/rejected": -0.01834115758538246,
      "step": 1112
    },
    {
      "epoch": 0.29275510036331354,
      "grad_norm": 700.0,
      "learning_rate": 3.9298245614035085e-07,
      "logits/chosen": -0.6535927653312683,
      "logits/rejected": -0.6556929349899292,
      "logps/chosen": -1159.30908203125,
      "logps/rejected": -1170.0860595703125,
      "loss": 0.7263,
      "rewards/accuracies": 0.0625,
      "rewards/chosen": -0.010592460632324219,
      "rewards/margins": -0.06438398361206055,
      "rewards/rejected": 0.053791530430316925,
      "step": 1113
    },
    {
      "epoch": 0.2930181327985007,
      "grad_norm": 496.0,
      "learning_rate": 3.928362573099415e-07,
      "logits/chosen": -0.6323816776275635,
      "logits/rejected": -0.6356412172317505,
      "logps/chosen": -1683.780029296875,
      "logps/rejected": -1435.7977294921875,
      "loss": 0.7053,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.04740419611334801,
      "rewards/margins": -0.02387542836368084,
      "rewards/rejected": -0.023528767749667168,
      "step": 1114
    },
    {
      "epoch": 0.29328116523368786,
      "grad_norm": 608.0,
      "learning_rate": 3.9269005847953214e-07,
      "logits/chosen": -0.644792914390564,
      "logits/rejected": -0.657893180847168,
      "logps/chosen": -1306.3402099609375,
      "logps/rejected": -982.0822143554688,
      "loss": 0.7047,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.010741805657744408,
      "rewards/margins": -0.02166614681482315,
      "rewards/rejected": 0.03240795060992241,
      "step": 1115
    },
    {
      "epoch": 0.293544197668875,
      "grad_norm": 576.0,
      "learning_rate": 3.9254385964912276e-07,
      "logits/chosen": -0.6639373302459717,
      "logits/rejected": -0.6648075580596924,
      "logps/chosen": -1283.3302001953125,
      "logps/rejected": -934.0421752929688,
      "loss": 0.6849,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.012777709402143955,
      "rewards/margins": 0.017915153875947,
      "rewards/rejected": -0.005137445405125618,
      "step": 1116
    },
    {
      "epoch": 0.29380723010406223,
      "grad_norm": 596.0,
      "learning_rate": 3.9239766081871343e-07,
      "logits/chosen": -0.6482964754104614,
      "logits/rejected": -0.6413993835449219,
      "logps/chosen": -1072.713134765625,
      "logps/rejected": -847.1229248046875,
      "loss": 0.6963,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0033528325147926807,
      "rewards/margins": -0.005156613886356354,
      "rewards/rejected": 0.0018037804402410984,
      "step": 1117
    },
    {
      "epoch": 0.2940702625392494,
      "grad_norm": 430.0,
      "learning_rate": 3.9225146198830405e-07,
      "logits/chosen": -0.6434556245803833,
      "logits/rejected": -0.6404415965080261,
      "logps/chosen": -942.6466064453125,
      "logps/rejected": -662.59326171875,
      "loss": 0.6898,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.000903034582734108,
      "rewards/margins": 0.0073323240503668785,
      "rewards/rejected": -0.008235358633100986,
      "step": 1118
    },
    {
      "epoch": 0.29433329497443655,
      "grad_norm": 516.0,
      "learning_rate": 3.9210526315789473e-07,
      "logits/chosen": -0.6325200796127319,
      "logits/rejected": -0.6410581469535828,
      "logps/chosen": -956.068359375,
      "logps/rejected": -686.601318359375,
      "loss": 0.6964,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0012072552926838398,
      "rewards/margins": -0.005416203755885363,
      "rewards/rejected": 0.006623459979891777,
      "step": 1119
    },
    {
      "epoch": 0.2945963274096237,
      "grad_norm": 580.0,
      "learning_rate": 3.919590643274854e-07,
      "logits/chosen": -0.6481190323829651,
      "logits/rejected": -0.6482767462730408,
      "logps/chosen": -1318.4796142578125,
      "logps/rejected": -900.9852294921875,
      "loss": 0.6852,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.01597280614078045,
      "rewards/margins": 0.017182637006044388,
      "rewards/rejected": -0.001209830865263939,
      "step": 1120
    },
    {
      "epoch": 0.2948593598448109,
      "grad_norm": 510.0,
      "learning_rate": 3.9181286549707597e-07,
      "logits/chosen": -0.6706191301345825,
      "logits/rejected": -0.6625929474830627,
      "logps/chosen": -859.0821533203125,
      "logps/rejected": -749.869140625,
      "loss": 0.6842,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.016448210924863815,
      "rewards/margins": 0.019282246008515358,
      "rewards/rejected": -0.002834034152328968,
      "step": 1121
    },
    {
      "epoch": 0.29512239227999804,
      "grad_norm": 376.0,
      "learning_rate": 3.9166666666666664e-07,
      "logits/chosen": -0.6445733308792114,
      "logits/rejected": -0.643030047416687,
      "logps/chosen": -696.7760009765625,
      "logps/rejected": -572.9459228515625,
      "loss": 0.6902,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.002401303965598345,
      "rewards/margins": 0.00618739053606987,
      "rewards/rejected": -0.0037860875017941,
      "step": 1122
    },
    {
      "epoch": 0.2953854247151852,
      "grad_norm": 528.0,
      "learning_rate": 3.915204678362573e-07,
      "logits/chosen": -0.6719378232955933,
      "logits/rejected": -0.6695493459701538,
      "logps/chosen": -829.9788818359375,
      "logps/rejected": -1021.5065307617188,
      "loss": 0.6945,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.023867223411798477,
      "rewards/margins": -0.0018125539645552635,
      "rewards/rejected": 0.025679778307676315,
      "step": 1123
    },
    {
      "epoch": 0.29564845715037236,
      "grad_norm": 520.0,
      "learning_rate": 3.9137426900584793e-07,
      "logits/chosen": -0.6795721054077148,
      "logits/rejected": NaN,
      "logps/chosen": -1248.7872314453125,
      "logps/rejected": -767.6151733398438,
      "loss": 0.6865,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.016365814954042435,
      "rewards/margins": 0.014356039464473724,
      "rewards/rejected": 0.002009773626923561,
      "step": 1124
    },
    {
      "epoch": 0.2959114895855595,
      "grad_norm": 424.0,
      "learning_rate": 3.9122807017543855e-07,
      "logits/chosen": -0.6520446538925171,
      "logits/rejected": -0.6492524743080139,
      "logps/chosen": -1069.59765625,
      "logps/rejected": -856.0582275390625,
      "loss": 0.6898,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.010400200262665749,
      "rewards/margins": 0.007144451141357422,
      "rewards/rejected": 0.0032557491213083267,
      "step": 1125
    },
    {
      "epoch": 0.2961745220207467,
      "grad_norm": 496.0,
      "learning_rate": 3.910818713450292e-07,
      "logits/chosen": -0.6612125039100647,
      "logits/rejected": -0.6636342406272888,
      "logps/chosen": -909.285888671875,
      "logps/rejected": -662.3291015625,
      "loss": 0.6914,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005121135618537664,
      "rewards/margins": 0.0046872603707015514,
      "rewards/rejected": 0.00043387478217482567,
      "step": 1126
    },
    {
      "epoch": 0.29643755445593384,
      "grad_norm": 564.0,
      "learning_rate": 3.9093567251461985e-07,
      "logits/chosen": -0.6513429880142212,
      "logits/rejected": -0.654640257358551,
      "logps/chosen": -1158.51513671875,
      "logps/rejected": -768.508056640625,
      "loss": 0.6851,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.012966345995664597,
      "rewards/margins": 0.01694631576538086,
      "rewards/rejected": -0.003979968838393688,
      "step": 1127
    },
    {
      "epoch": 0.296700586891121,
      "grad_norm": 664.0,
      "learning_rate": 3.907894736842105e-07,
      "logits/chosen": -0.6609973907470703,
      "logits/rejected": -0.6707051396369934,
      "logps/chosen": -933.2734985351562,
      "logps/rejected": -1047.1239013671875,
      "loss": 0.6859,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00842814426869154,
      "rewards/margins": 0.014987041242420673,
      "rewards/rejected": -0.023415185511112213,
      "step": 1128
    },
    {
      "epoch": 0.29696361932630816,
      "grad_norm": 548.0,
      "learning_rate": 3.9064327485380114e-07,
      "logits/chosen": -0.6654506325721741,
      "logits/rejected": -0.6614677309989929,
      "logps/chosen": -1330.3236083984375,
      "logps/rejected": -905.2978515625,
      "loss": 0.6966,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0007138253422454,
      "rewards/margins": -0.006465816870331764,
      "rewards/rejected": 0.005751991644501686,
      "step": 1129
    },
    {
      "epoch": 0.2972266517614953,
      "grad_norm": 564.0,
      "learning_rate": 3.9049707602339176e-07,
      "logits/chosen": -0.6411443948745728,
      "logits/rejected": -0.6406551599502563,
      "logps/chosen": -1559.545654296875,
      "logps/rejected": -1069.861083984375,
      "loss": 0.6884,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00392303429543972,
      "rewards/margins": 0.010322664864361286,
      "rewards/rejected": -0.006399631965905428,
      "step": 1130
    },
    {
      "epoch": 0.2974896841966825,
      "grad_norm": 480.0,
      "learning_rate": 3.9035087719298243e-07,
      "logits/chosen": -0.6755929589271545,
      "logits/rejected": NaN,
      "logps/chosen": -897.8643798828125,
      "logps/rejected": -531.2256469726562,
      "loss": 0.6854,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.014127444475889206,
      "rewards/margins": 0.01612367480993271,
      "rewards/rejected": -0.0019962310325354338,
      "step": 1131
    },
    {
      "epoch": 0.29775271663186964,
      "grad_norm": 512.0,
      "learning_rate": 3.902046783625731e-07,
      "logits/chosen": -0.6580691337585449,
      "logits/rejected": -0.6628553867340088,
      "logps/chosen": -1169.3916015625,
      "logps/rejected": -943.9203491210938,
      "loss": 0.6924,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.011739016510546207,
      "rewards/margins": 0.002826546085998416,
      "rewards/rejected": 0.008912469260394573,
      "step": 1132
    },
    {
      "epoch": 0.29801574906705686,
      "grad_norm": 548.0,
      "learning_rate": 3.900584795321637e-07,
      "logits/chosen": -0.651287317276001,
      "logits/rejected": -0.647782564163208,
      "logps/chosen": -898.607421875,
      "logps/rejected": -720.7930908203125,
      "loss": 0.6934,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00449028005823493,
      "rewards/margins": -0.00028352742083370686,
      "rewards/rejected": 0.0047738077118992805,
      "step": 1133
    },
    {
      "epoch": 0.298278781502244,
      "grad_norm": 520.0,
      "learning_rate": 3.8991228070175434e-07,
      "logits/chosen": -0.6436157822608948,
      "logits/rejected": NaN,
      "logps/chosen": -993.6171264648438,
      "logps/rejected": -629.6704711914062,
      "loss": 0.6998,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.00756769347935915,
      "rewards/margins": -0.012431717477738857,
      "rewards/rejected": 0.004864025395363569,
      "step": 1134
    },
    {
      "epoch": 0.2985418139374312,
      "grad_norm": 508.0,
      "learning_rate": 3.89766081871345e-07,
      "logits/chosen": -0.6702897548675537,
      "logits/rejected": -0.6672248244285583,
      "logps/chosen": -726.98779296875,
      "logps/rejected": -754.3872680664062,
      "loss": 0.7035,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.004550742916762829,
      "rewards/margins": -0.02004823461174965,
      "rewards/rejected": 0.015497494488954544,
      "step": 1135
    },
    {
      "epoch": 0.29880484637261834,
      "grad_norm": 516.0,
      "learning_rate": 3.896198830409357e-07,
      "logits/chosen": -0.6541873812675476,
      "logits/rejected": -0.6653061509132385,
      "logps/chosen": -1243.4552001953125,
      "logps/rejected": -905.9913940429688,
      "loss": 0.6905,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0061425212770700455,
      "rewards/margins": 0.0056986818090081215,
      "rewards/rejected": 0.000443839468061924,
      "step": 1136
    },
    {
      "epoch": 0.2990678788078055,
      "grad_norm": 540.0,
      "learning_rate": 3.894736842105263e-07,
      "logits/chosen": -0.6571702361106873,
      "logits/rejected": NaN,
      "logps/chosen": -1118.7451171875,
      "logps/rejected": -862.2950439453125,
      "loss": 0.7001,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.023633670061826706,
      "rewards/margins": -0.013030434027314186,
      "rewards/rejected": -0.010603236965835094,
      "step": 1137
    },
    {
      "epoch": 0.29933091124299266,
      "grad_norm": 536.0,
      "learning_rate": 3.8932748538011693e-07,
      "logits/chosen": -0.6615890860557556,
      "logits/rejected": -0.6616679430007935,
      "logps/chosen": -1209.2506103515625,
      "logps/rejected": -1084.643798828125,
      "loss": 0.6896,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012168502435088158,
      "rewards/margins": 0.008154009468853474,
      "rewards/rejected": 0.004014491569250822,
      "step": 1138
    },
    {
      "epoch": 0.2995939436781798,
      "grad_norm": 510.0,
      "learning_rate": 3.891812865497076e-07,
      "logits/chosen": -0.6581239104270935,
      "logits/rejected": -0.6576095223426819,
      "logps/chosen": -1121.97900390625,
      "logps/rejected": -903.9783935546875,
      "loss": 0.7006,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.011179447174072266,
      "rewards/margins": -0.013802524656057358,
      "rewards/rejected": 0.0026230793446302414,
      "step": 1139
    },
    {
      "epoch": 0.299856976113367,
      "grad_norm": 548.0,
      "learning_rate": 3.890350877192982e-07,
      "logits/chosen": -0.6523503065109253,
      "logits/rejected": -0.6611279845237732,
      "logps/chosen": -1296.4200439453125,
      "logps/rejected": -1148.367919921875,
      "loss": 0.6935,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.003563785459846258,
      "rewards/margins": -0.00010662044223863631,
      "rewards/rejected": 0.003670406062155962,
      "step": 1140
    },
    {
      "epoch": 0.30012000854855414,
      "grad_norm": 478.0,
      "learning_rate": 3.888888888888889e-07,
      "logits/chosen": -0.6635468006134033,
      "logits/rejected": -0.6640879511833191,
      "logps/chosen": -958.9033203125,
      "logps/rejected": -762.3034057617188,
      "loss": 0.6898,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.02027278020977974,
      "rewards/margins": 0.007772111799567938,
      "rewards/rejected": 0.012500667944550514,
      "step": 1141
    },
    {
      "epoch": 0.3003830409837413,
      "grad_norm": 524.0,
      "learning_rate": 3.887426900584795e-07,
      "logits/chosen": -0.6602208018302917,
      "logits/rejected": -0.6598796248435974,
      "logps/chosen": -1600.66748046875,
      "logps/rejected": -1498.167236328125,
      "loss": 0.6762,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.017840001732110977,
      "rewards/margins": 0.03581485524773598,
      "rewards/rejected": -0.017974853515625,
      "step": 1142
    },
    {
      "epoch": 0.30064607341892846,
      "grad_norm": 528.0,
      "learning_rate": 3.8859649122807013e-07,
      "logits/chosen": -0.6464124917984009,
      "logits/rejected": -0.6472681164741516,
      "logps/chosen": -1521.50634765625,
      "logps/rejected": -1200.51611328125,
      "loss": 0.688,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.005315875634551048,
      "rewards/margins": 0.011468984186649323,
      "rewards/rejected": -0.01678485982120037,
      "step": 1143
    },
    {
      "epoch": 0.3009091058541156,
      "grad_norm": 500.0,
      "learning_rate": 3.884502923976608e-07,
      "logits/chosen": -0.6447462439537048,
      "logits/rejected": -0.6522348523139954,
      "logps/chosen": -1267.154541015625,
      "logps/rejected": -959.1653442382812,
      "loss": 0.6945,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.000939464196562767,
      "rewards/margins": -0.002220725640654564,
      "rewards/rejected": 0.0012812614440917969,
      "step": 1144
    },
    {
      "epoch": 0.3011721382893028,
      "grad_norm": 488.0,
      "learning_rate": 3.883040935672515e-07,
      "logits/chosen": -0.6360741257667542,
      "logits/rejected": -0.638491690158844,
      "logps/chosen": -868.0792846679688,
      "logps/rejected": -650.4683837890625,
      "loss": 0.6877,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00986628606915474,
      "rewards/margins": 0.011380339041352272,
      "rewards/rejected": -0.001514054019935429,
      "step": 1145
    },
    {
      "epoch": 0.30143517072448994,
      "grad_norm": 644.0,
      "learning_rate": 3.8815789473684205e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6707696318626404,
      "logps/chosen": -948.4046630859375,
      "logps/rejected": -1136.9449462890625,
      "loss": 0.6934,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.00811004638671875,
      "rewards/margins": 0.0005287178792059422,
      "rewards/rejected": 0.00758132990449667,
      "step": 1146
    },
    {
      "epoch": 0.3016982031596771,
      "grad_norm": 604.0,
      "learning_rate": 3.880116959064327e-07,
      "logits/chosen": -0.6543450355529785,
      "logits/rejected": -0.6573212742805481,
      "logps/chosen": -1167.138427734375,
      "logps/rejected": -1157.0084228515625,
      "loss": 0.6951,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.001223134808242321,
      "rewards/margins": -0.0034404282923787832,
      "rewards/rejected": 0.004663563333451748,
      "step": 1147
    },
    {
      "epoch": 0.3019612355948643,
      "grad_norm": 568.0,
      "learning_rate": 3.878654970760234e-07,
      "logits/chosen": -0.668093204498291,
      "logits/rejected": -0.6771851181983948,
      "logps/chosen": -1294.3380126953125,
      "logps/rejected": -781.6842041015625,
      "loss": 0.696,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004342842381447554,
      "rewards/margins": -0.004989624954760075,
      "rewards/rejected": 0.009332465939223766,
      "step": 1148
    },
    {
      "epoch": 0.3022242680300515,
      "grad_norm": 604.0,
      "learning_rate": 3.87719298245614e-07,
      "logits/chosen": -0.6602059006690979,
      "logits/rejected": -0.6625915169715881,
      "logps/chosen": -1520.6837158203125,
      "logps/rejected": -1204.564697265625,
      "loss": 0.693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.018590452149510384,
      "rewards/margins": 0.0009278310462832451,
      "rewards/rejected": 0.017662620171904564,
      "step": 1149
    },
    {
      "epoch": 0.30248730046523864,
      "grad_norm": 474.0,
      "learning_rate": 3.875730994152047e-07,
      "logits/chosen": -0.6467640995979309,
      "logits/rejected": -0.6511926054954529,
      "logps/chosen": -1050.062255859375,
      "logps/rejected": -735.6846313476562,
      "loss": 0.6947,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.008597183972597122,
      "rewards/margins": -0.0025032521225512028,
      "rewards/rejected": 0.011100434698164463,
      "step": 1150
    },
    {
      "epoch": 0.3027503329004258,
      "grad_norm": 788.0,
      "learning_rate": 3.874269005847953e-07,
      "logits/chosen": -0.6367021799087524,
      "logits/rejected": -0.6423708200454712,
      "logps/chosen": -1364.0401611328125,
      "logps/rejected": -1073.5435791015625,
      "loss": 0.6876,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0021706579718738794,
      "rewards/margins": 0.011860370635986328,
      "rewards/rejected": -0.009689712896943092,
      "step": 1151
    },
    {
      "epoch": 0.30301336533561296,
      "grad_norm": 532.0,
      "learning_rate": 3.872807017543859e-07,
      "logits/chosen": -0.63132244348526,
      "logits/rejected": -0.6340920925140381,
      "logps/chosen": -987.8864135742188,
      "logps/rejected": -692.7664184570312,
      "loss": 0.7012,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.018015384674072266,
      "rewards/margins": -0.015236759558320045,
      "rewards/rejected": -0.002778626512736082,
      "step": 1152
    },
    {
      "epoch": 0.3032763977708001,
      "grad_norm": 552.0,
      "learning_rate": 3.871345029239766e-07,
      "logits/chosen": -0.6733959913253784,
      "logits/rejected": -0.6784306764602661,
      "logps/chosen": -1010.838623046875,
      "logps/rejected": -849.0982666015625,
      "loss": 0.7016,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006478595547378063,
      "rewards/margins": -0.015944957733154297,
      "rewards/rejected": 0.022423552349209785,
      "step": 1153
    },
    {
      "epoch": 0.3035394302059873,
      "grad_norm": 572.0,
      "learning_rate": 3.8698830409356727e-07,
      "logits/chosen": -0.6204798221588135,
      "logits/rejected": NaN,
      "logps/chosen": -1021.6737060546875,
      "logps/rejected": -701.7513427734375,
      "loss": 0.6952,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.010734938085079193,
      "rewards/margins": -0.0032109259627759457,
      "rewards/rejected": -0.007524014450609684,
      "step": 1154
    },
    {
      "epoch": 0.30380246264117444,
      "grad_norm": 524.0,
      "learning_rate": 3.8684210526315784e-07,
      "logits/chosen": -0.6355334520339966,
      "logits/rejected": -0.6426461935043335,
      "logps/chosen": -895.996337890625,
      "logps/rejected": -885.4797973632812,
      "loss": 0.6853,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.027762126177549362,
      "rewards/margins": 0.0171356201171875,
      "rewards/rejected": 0.010626506060361862,
      "step": 1155
    },
    {
      "epoch": 0.3040654950763616,
      "grad_norm": 480.0,
      "learning_rate": 3.866959064327485e-07,
      "logits/chosen": -0.6290016174316406,
      "logits/rejected": -0.6439146399497986,
      "logps/chosen": -857.0027465820312,
      "logps/rejected": -884.0283203125,
      "loss": 0.6973,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01792902871966362,
      "rewards/margins": -0.007560299709439278,
      "rewards/rejected": 0.025489332154393196,
      "step": 1156
    },
    {
      "epoch": 0.30432852751154876,
      "grad_norm": 564.0,
      "learning_rate": 3.865497076023392e-07,
      "logits/chosen": -0.6432619690895081,
      "logits/rejected": -0.6506809592247009,
      "logps/chosen": -994.8250732421875,
      "logps/rejected": -1004.9752807617188,
      "loss": 0.7112,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.026939107105135918,
      "rewards/margins": -0.03478851169347763,
      "rewards/rejected": 0.007849406450986862,
      "step": 1157
    },
    {
      "epoch": 0.3045915599467359,
      "grad_norm": 576.0,
      "learning_rate": 3.864035087719298e-07,
      "logits/chosen": -0.6637425422668457,
      "logits/rejected": -0.6499127149581909,
      "logps/chosen": -1283.18603515625,
      "logps/rejected": -1021.2158813476562,
      "loss": 0.6884,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.010556221008300781,
      "rewards/margins": 0.00987262837588787,
      "rewards/rejected": -0.020428849384188652,
      "step": 1158
    },
    {
      "epoch": 0.3048545923819231,
      "grad_norm": 506.0,
      "learning_rate": 3.862573099415204e-07,
      "logits/chosen": -0.6768254041671753,
      "logits/rejected": -0.67049241065979,
      "logps/chosen": -890.4217529296875,
      "logps/rejected": -807.7102661132812,
      "loss": 0.6884,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.003053949913010001,
      "rewards/margins": 0.010246370919048786,
      "rewards/rejected": -0.007192421238869429,
      "step": 1159
    },
    {
      "epoch": 0.30511762481711024,
      "grad_norm": 556.0,
      "learning_rate": 3.861111111111111e-07,
      "logits/chosen": -0.6807668805122375,
      "logits/rejected": -0.6749029755592346,
      "logps/chosen": -1185.550048828125,
      "logps/rejected": -1152.9173583984375,
      "loss": 0.6947,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.009061718359589577,
      "rewards/margins": -0.0026288037188351154,
      "rewards/rejected": -0.006432914640754461,
      "step": 1160
    },
    {
      "epoch": 0.3053806572522974,
      "grad_norm": 536.0,
      "learning_rate": 3.859649122807017e-07,
      "logits/chosen": -0.6595263481140137,
      "logits/rejected": -0.6532032489776611,
      "logps/chosen": -1337.203857421875,
      "logps/rejected": -939.091064453125,
      "loss": 0.6876,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00192184466868639,
      "rewards/margins": 0.011836623772978783,
      "rewards/rejected": -0.009914780035614967,
      "step": 1161
    },
    {
      "epoch": 0.30564368968748457,
      "grad_norm": 470.0,
      "learning_rate": 3.858187134502924e-07,
      "logits/chosen": -0.6421101093292236,
      "logits/rejected": -0.641124427318573,
      "logps/chosen": -895.6546630859375,
      "logps/rejected": -602.3424072265625,
      "loss": 0.6895,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007514095399528742,
      "rewards/margins": 0.008060455322265625,
      "rewards/rejected": -0.0005463608540594578,
      "step": 1162
    },
    {
      "epoch": 0.3059067221226717,
      "grad_norm": 548.0,
      "learning_rate": 3.8567251461988306e-07,
      "logits/chosen": -0.649497926235199,
      "logits/rejected": -0.6598175764083862,
      "logps/chosen": -1190.12451171875,
      "logps/rejected": -834.7523193359375,
      "loss": 0.6938,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008393381722271442,
      "rewards/margins": -0.0005982406437397003,
      "rewards/rejected": 0.008991622366011143,
      "step": 1163
    },
    {
      "epoch": 0.30616975455785894,
      "grad_norm": 520.0,
      "learning_rate": 3.8552631578947363e-07,
      "logits/chosen": -0.622612476348877,
      "logits/rejected": -0.6288926601409912,
      "logps/chosen": -981.1209716796875,
      "logps/rejected": -965.8173828125,
      "loss": 0.6957,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.010219954885542393,
      "rewards/margins": -0.004528425633907318,
      "rewards/rejected": -0.0056915283203125,
      "step": 1164
    },
    {
      "epoch": 0.3064327869930461,
      "grad_norm": 604.0,
      "learning_rate": 3.853801169590643e-07,
      "logits/chosen": -0.6303284168243408,
      "logits/rejected": -0.634311854839325,
      "logps/chosen": -1498.83837890625,
      "logps/rejected": -794.84521484375,
      "loss": 0.6906,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005730438511818647,
      "rewards/margins": 0.006093931850045919,
      "rewards/rejected": -0.011824369430541992,
      "step": 1165
    },
    {
      "epoch": 0.30669581942823326,
      "grad_norm": 556.0,
      "learning_rate": 3.85233918128655e-07,
      "logits/chosen": -0.6715361475944519,
      "logits/rejected": -0.6338512897491455,
      "logps/chosen": -847.5855712890625,
      "logps/rejected": -743.7952880859375,
      "loss": 0.6916,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008637428283691406,
      "rewards/margins": 0.003706454299390316,
      "rewards/rejected": 0.004930973052978516,
      "step": 1166
    },
    {
      "epoch": 0.3069588518634204,
      "grad_norm": 516.0,
      "learning_rate": 3.850877192982456e-07,
      "logits/chosen": -0.6239049434661865,
      "logits/rejected": -0.6293909549713135,
      "logps/chosen": -1162.829345703125,
      "logps/rejected": -626.071533203125,
      "loss": 0.7023,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.02380399778485298,
      "rewards/margins": -0.017320631071925163,
      "rewards/rejected": -0.00648336298763752,
      "step": 1167
    },
    {
      "epoch": 0.3072218842986076,
      "grad_norm": 608.0,
      "learning_rate": 3.849415204678362e-07,
      "logits/chosen": -0.6492030620574951,
      "logits/rejected": -0.648913562297821,
      "logps/chosen": -1390.8897705078125,
      "logps/rejected": -1008.2565307617188,
      "loss": 0.688,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004501341842114925,
      "rewards/margins": 0.01232890971004963,
      "rewards/rejected": -0.007827568799257278,
      "step": 1168
    },
    {
      "epoch": 0.30748491673379474,
      "grad_norm": 506.0,
      "learning_rate": 3.847953216374269e-07,
      "logits/chosen": -0.662439227104187,
      "logits/rejected": -0.6669598817825317,
      "logps/chosen": -1093.5712890625,
      "logps/rejected": -946.9263916015625,
      "loss": 0.6866,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01253891084343195,
      "rewards/margins": 0.013879681937396526,
      "rewards/rejected": -0.0013407698133960366,
      "step": 1169
    },
    {
      "epoch": 0.3077479491689819,
      "grad_norm": 528.0,
      "learning_rate": 3.846491228070175e-07,
      "logits/chosen": -0.6878072619438171,
      "logits/rejected": -0.6939217448234558,
      "logps/chosen": -1515.9439697265625,
      "logps/rejected": -1291.323974609375,
      "loss": 0.6881,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0021016125101596117,
      "rewards/margins": 0.010467912070453167,
      "rewards/rejected": -0.01256952341645956,
      "step": 1170
    },
    {
      "epoch": 0.30801098160416907,
      "grad_norm": 528.0,
      "learning_rate": 3.845029239766082e-07,
      "logits/chosen": -0.6482527852058411,
      "logits/rejected": -0.6476501226425171,
      "logps/chosen": -1143.560546875,
      "logps/rejected": -894.8755493164062,
      "loss": 0.6804,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.024029923602938652,
      "rewards/margins": 0.026764871552586555,
      "rewards/rejected": -0.0027349465526640415,
      "step": 1171
    },
    {
      "epoch": 0.3082740140393562,
      "grad_norm": 556.0,
      "learning_rate": 3.843567251461988e-07,
      "logits/chosen": -0.67725670337677,
      "logits/rejected": -0.6768910884857178,
      "logps/chosen": -1334.7449951171875,
      "logps/rejected": -1201.8487548828125,
      "loss": 0.6937,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.01140737347304821,
      "rewards/margins": -3.175809979438782e-05,
      "rewards/rejected": -0.011375617235898972,
      "step": 1172
    },
    {
      "epoch": 0.3085370464745434,
      "grad_norm": 584.0,
      "learning_rate": 3.842105263157894e-07,
      "logits/chosen": -0.6436478495597839,
      "logits/rejected": -0.6513664722442627,
      "logps/chosen": -1404.3289794921875,
      "logps/rejected": -1238.3673095703125,
      "loss": 0.6969,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0044586192816495895,
      "rewards/margins": -0.00692214909940958,
      "rewards/rejected": 0.0024635312147438526,
      "step": 1173
    },
    {
      "epoch": 0.30880007890973055,
      "grad_norm": 504.0,
      "learning_rate": 3.840643274853801e-07,
      "logits/chosen": -0.6399301886558533,
      "logits/rejected": -0.6436125636100769,
      "logps/chosen": -1156.8516845703125,
      "logps/rejected": -879.3160400390625,
      "loss": 0.6893,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010584449395537376,
      "rewards/margins": 0.008347034454345703,
      "rewards/rejected": 0.0022374149411916733,
      "step": 1174
    },
    {
      "epoch": 0.3090631113449177,
      "grad_norm": 460.0,
      "learning_rate": 3.8391812865497077e-07,
      "logits/chosen": -0.6483467221260071,
      "logits/rejected": -0.6521700024604797,
      "logps/chosen": -958.521240234375,
      "logps/rejected": -1013.7053833007812,
      "loss": 0.6894,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.014052201062440872,
      "rewards/margins": 0.00874318927526474,
      "rewards/rejected": -0.022795390337705612,
      "step": 1175
    },
    {
      "epoch": 0.30932614378010487,
      "grad_norm": 540.0,
      "learning_rate": 3.8377192982456144e-07,
      "logits/chosen": -0.6767364740371704,
      "logits/rejected": -0.6848412752151489,
      "logps/chosen": -1261.828125,
      "logps/rejected": -767.2779541015625,
      "loss": 0.6898,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.003861997975036502,
      "rewards/margins": 0.008665180765092373,
      "rewards/rejected": -0.004803180694580078,
      "step": 1176
    },
    {
      "epoch": 0.30958917621529203,
      "grad_norm": 664.0,
      "learning_rate": 3.83625730994152e-07,
      "logits/chosen": -0.6177428364753723,
      "logits/rejected": -0.6274251937866211,
      "logps/chosen": -826.048828125,
      "logps/rejected": -1041.4425048828125,
      "loss": 0.694,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0044144634157419205,
      "rewards/margins": -0.0009114257991313934,
      "rewards/rejected": 0.005325889680534601,
      "step": 1177
    },
    {
      "epoch": 0.3098522086504792,
      "grad_norm": 456.0,
      "learning_rate": 3.834795321637427e-07,
      "logits/chosen": -0.6419790387153625,
      "logits/rejected": -0.6421542763710022,
      "logps/chosen": -758.4639282226562,
      "logps/rejected": -634.4991455078125,
      "loss": 0.7003,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.014741325750946999,
      "rewards/margins": -0.013631917536258698,
      "rewards/rejected": -0.001109409611672163,
      "step": 1178
    },
    {
      "epoch": 0.31011524108566635,
      "grad_norm": 502.0,
      "learning_rate": 3.8333333333333335e-07,
      "logits/chosen": -0.651536762714386,
      "logits/rejected": -0.6601320505142212,
      "logps/chosen": -1290.454345703125,
      "logps/rejected": -945.5415649414062,
      "loss": 0.6906,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.020019054412841797,
      "rewards/margins": 0.005443238187581301,
      "rewards/rejected": 0.014575814828276634,
      "step": 1179
    },
    {
      "epoch": 0.31037827352085356,
      "grad_norm": 700.0,
      "learning_rate": 3.8318713450292397e-07,
      "logits/chosen": -0.6628106236457825,
      "logits/rejected": -0.6721311211585999,
      "logps/chosen": -1246.909423828125,
      "logps/rejected": -1096.095947265625,
      "loss": 0.7037,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.006278515327721834,
      "rewards/margins": -0.01961088366806507,
      "rewards/rejected": 0.013332366943359375,
      "step": 1180
    },
    {
      "epoch": 0.3106413059560407,
      "grad_norm": 608.0,
      "learning_rate": 3.830409356725146e-07,
      "logits/chosen": -0.6422219276428223,
      "logits/rejected": -0.6466472148895264,
      "logps/chosen": -1746.311767578125,
      "logps/rejected": -1581.3153076171875,
      "loss": 0.6967,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.016521833837032318,
      "rewards/margins": -0.006157682742923498,
      "rewards/rejected": -0.010364152491092682,
      "step": 1181
    },
    {
      "epoch": 0.3109043383912279,
      "grad_norm": 434.0,
      "learning_rate": 3.8289473684210526e-07,
      "logits/chosen": -0.652954638004303,
      "logits/rejected": -0.6639440655708313,
      "logps/chosen": -757.3783569335938,
      "logps/rejected": -583.646484375,
      "loss": 0.6874,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0018225194653496146,
      "rewards/margins": 0.01228861790150404,
      "rewards/rejected": -0.014111137948930264,
      "step": 1182
    },
    {
      "epoch": 0.31116737082641505,
      "grad_norm": 528.0,
      "learning_rate": 3.827485380116959e-07,
      "logits/chosen": -0.6417351365089417,
      "logits/rejected": -0.6473259925842285,
      "logps/chosen": -1105.0762939453125,
      "logps/rejected": -805.5738525390625,
      "loss": 0.6977,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0004937160992994905,
      "rewards/margins": -0.008423423394560814,
      "rewards/rejected": 0.007929707877337933,
      "step": 1183
    },
    {
      "epoch": 0.3114304032616022,
      "grad_norm": 760.0,
      "learning_rate": 3.8260233918128656e-07,
      "logits/chosen": -0.6291238069534302,
      "logits/rejected": -0.6408234238624573,
      "logps/chosen": -1472.34716796875,
      "logps/rejected": -1123.23779296875,
      "loss": 0.7018,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.003342723473906517,
      "rewards/margins": -0.016682051122188568,
      "rewards/rejected": 0.020024776458740234,
      "step": 1184
    },
    {
      "epoch": 0.31169343569678937,
      "grad_norm": 552.0,
      "learning_rate": 3.824561403508772e-07,
      "logits/chosen": -0.6523184180259705,
      "logits/rejected": -0.6578745245933533,
      "logps/chosen": -1212.36669921875,
      "logps/rejected": -729.6321411132812,
      "loss": 0.6988,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.015374183654785156,
      "rewards/margins": -0.010365772061049938,
      "rewards/rejected": -0.005008410662412643,
      "step": 1185
    },
    {
      "epoch": 0.3119564681319765,
      "grad_norm": 544.0,
      "learning_rate": 3.823099415204678e-07,
      "logits/chosen": -0.6432638764381409,
      "logits/rejected": -0.6481640338897705,
      "logps/chosen": -895.6925659179688,
      "logps/rejected": -994.4074096679688,
      "loss": 0.695,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.008071327582001686,
      "rewards/margins": -0.0030967718921601772,
      "rewards/rejected": -0.0049745566211640835,
      "step": 1186
    },
    {
      "epoch": 0.3122195005671637,
      "grad_norm": 556.0,
      "learning_rate": 3.8216374269005847e-07,
      "logits/chosen": -0.655906617641449,
      "logits/rejected": -0.6725975275039673,
      "logps/chosen": -893.271240234375,
      "logps/rejected": -617.68408203125,
      "loss": 0.7026,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.017841290682554245,
      "rewards/margins": -0.01851058006286621,
      "rewards/rejected": 0.000669288681820035,
      "step": 1187
    },
    {
      "epoch": 0.31248253300235085,
      "grad_norm": 418.0,
      "learning_rate": 3.8201754385964914e-07,
      "logits/chosen": -0.643943190574646,
      "logits/rejected": -0.6523507833480835,
      "logps/chosen": -756.453369140625,
      "logps/rejected": -705.1880493164062,
      "loss": 0.6927,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.014694355428218842,
      "rewards/margins": 0.0012773028574883938,
      "rewards/rejected": -0.015971658751368523,
      "step": 1188
    },
    {
      "epoch": 0.312745565437538,
      "grad_norm": 588.0,
      "learning_rate": 3.818713450292397e-07,
      "logits/chosen": -0.6368246078491211,
      "logits/rejected": -0.6466643810272217,
      "logps/chosen": -1345.628173828125,
      "logps/rejected": -1055.3450927734375,
      "loss": 0.6873,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.026209067553281784,
      "rewards/margins": 0.01219487190246582,
      "rewards/rejected": 0.014014197513461113,
      "step": 1189
    },
    {
      "epoch": 0.31300859787272517,
      "grad_norm": 536.0,
      "learning_rate": 3.817251461988304e-07,
      "logits/chosen": -0.6612446308135986,
      "logits/rejected": -0.6555891036987305,
      "logps/chosen": -1229.5045166015625,
      "logps/rejected": -793.5299682617188,
      "loss": 0.69,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.01345815695822239,
      "rewards/margins": 0.00716352416202426,
      "rewards/rejected": -0.020621681585907936,
      "step": 1190
    },
    {
      "epoch": 0.31327163030791233,
      "grad_norm": 484.0,
      "learning_rate": 3.8157894736842105e-07,
      "logits/chosen": -0.652843713760376,
      "logits/rejected": -0.6469486355781555,
      "logps/chosen": -1047.8712158203125,
      "logps/rejected": -863.017822265625,
      "loss": 0.6999,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.016292477026581764,
      "rewards/margins": -0.012770844623446465,
      "rewards/rejected": -0.0035216326359659433,
      "step": 1191
    },
    {
      "epoch": 0.3135346627430995,
      "grad_norm": 464.0,
      "learning_rate": 3.814327485380117e-07,
      "logits/chosen": -0.6757969260215759,
      "logits/rejected": -0.6692898869514465,
      "logps/chosen": -1003.7716674804688,
      "logps/rejected": -751.7215576171875,
      "loss": 0.6913,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.002661610022187233,
      "rewards/margins": 0.004722070414572954,
      "rewards/rejected": -0.0073836809024214745,
      "step": 1192
    },
    {
      "epoch": 0.31379769517828665,
      "grad_norm": 548.0,
      "learning_rate": 3.8128654970760235e-07,
      "logits/chosen": -0.6566632986068726,
      "logits/rejected": -0.6664397120475769,
      "logps/chosen": -989.964599609375,
      "logps/rejected": -801.9464721679688,
      "loss": 0.6906,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.014530754648149014,
      "rewards/margins": 0.00573148624971509,
      "rewards/rejected": 0.008799267932772636,
      "step": 1193
    },
    {
      "epoch": 0.3140607276134738,
      "grad_norm": 680.0,
      "learning_rate": 3.8114035087719297e-07,
      "logits/chosen": -0.6523885130882263,
      "logits/rejected": -0.6525888442993164,
      "logps/chosen": -1428.7032470703125,
      "logps/rejected": -1089.33056640625,
      "loss": 0.6978,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.005306625738739967,
      "rewards/margins": -0.007562064100056887,
      "rewards/rejected": 0.0022554395254701376,
      "step": 1194
    },
    {
      "epoch": 0.314323760048661,
      "grad_norm": 604.0,
      "learning_rate": 3.809941520467836e-07,
      "logits/chosen": -0.6506251096725464,
      "logits/rejected": -0.6441414952278137,
      "logps/chosen": -1088.8204345703125,
      "logps/rejected": -889.85595703125,
      "loss": 0.6899,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.001288319006562233,
      "rewards/margins": 0.0074613578617572784,
      "rewards/rejected": -0.008749676868319511,
      "step": 1195
    },
    {
      "epoch": 0.3145867924838482,
      "grad_norm": 544.0,
      "learning_rate": 3.8084795321637426e-07,
      "logits/chosen": -0.6685503721237183,
      "logits/rejected": -0.6699509620666504,
      "logps/chosen": -1340.383056640625,
      "logps/rejected": -948.8861083984375,
      "loss": 0.6965,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0007162094116210938,
      "rewards/margins": -0.005321551114320755,
      "rewards/rejected": 0.004605341702699661,
      "step": 1196
    },
    {
      "epoch": 0.31484982491903535,
      "grad_norm": 560.0,
      "learning_rate": 3.8070175438596493e-07,
      "logits/chosen": -0.6202337741851807,
      "logits/rejected": -0.6113705039024353,
      "logps/chosen": -1036.8271484375,
      "logps/rejected": -792.721923828125,
      "loss": 0.693,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.02263364940881729,
      "rewards/margins": 0.0010061729699373245,
      "rewards/rejected": -0.023639820516109467,
      "step": 1197
    },
    {
      "epoch": 0.3151128573542225,
      "grad_norm": 600.0,
      "learning_rate": 3.805555555555555e-07,
      "logits/chosen": -0.6541109085083008,
      "logits/rejected": -0.6473912000656128,
      "logps/chosen": -1020.9449462890625,
      "logps/rejected": -1139.93359375,
      "loss": 0.6905,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.017964744940400124,
      "rewards/margins": 0.005974484607577324,
      "rewards/rejected": 0.0119902603328228,
      "step": 1198
    },
    {
      "epoch": 0.31537588978940967,
      "grad_norm": 560.0,
      "learning_rate": 3.8040935672514617e-07,
      "logits/chosen": -0.6732209324836731,
      "logits/rejected": -0.6691810488700867,
      "logps/chosen": -930.4126586914062,
      "logps/rejected": -802.9664306640625,
      "loss": 0.6851,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0016370778903365135,
      "rewards/margins": 0.01707429625093937,
      "rewards/rejected": -0.01543722115457058,
      "step": 1199
    },
    {
      "epoch": 0.31563892222459683,
      "grad_norm": 568.0,
      "learning_rate": 3.8026315789473685e-07,
      "logits/chosen": -0.635676383972168,
      "logits/rejected": -0.6453359127044678,
      "logps/chosen": -1194.3753662109375,
      "logps/rejected": -805.7242431640625,
      "loss": 0.6966,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01656932942569256,
      "rewards/margins": -0.005886889062821865,
      "rewards/rejected": 0.02245621755719185,
      "step": 1200
    },
    {
      "epoch": 0.315901954659784,
      "grad_norm": 466.0,
      "learning_rate": 3.8011695906432747e-07,
      "logits/chosen": -0.6305369734764099,
      "logits/rejected": -0.630618691444397,
      "logps/chosen": -978.0216064453125,
      "logps/rejected": -749.595703125,
      "loss": 0.6897,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.015922928228974342,
      "rewards/margins": 0.007323455065488815,
      "rewards/rejected": 0.008599472232162952,
      "step": 1201
    },
    {
      "epoch": 0.31616498709497115,
      "grad_norm": 466.0,
      "learning_rate": 3.799707602339181e-07,
      "logits/chosen": -0.6320013403892517,
      "logits/rejected": -0.6451816558837891,
      "logps/chosen": -859.0306396484375,
      "logps/rejected": -515.63623046875,
      "loss": 0.691,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.006263064220547676,
      "rewards/margins": 0.004321623593568802,
      "rewards/rejected": -0.010584688745439053,
      "step": 1202
    },
    {
      "epoch": 0.3164280195301583,
      "grad_norm": 600.0,
      "learning_rate": 3.7982456140350876e-07,
      "logits/chosen": -0.6468057632446289,
      "logits/rejected": -0.6477053761482239,
      "logps/chosen": -1313.64208984375,
      "logps/rejected": -1135.8323974609375,
      "loss": 0.6888,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.014205265790224075,
      "rewards/margins": 0.0092614172026515,
      "rewards/rejected": 0.00494384765625,
      "step": 1203
    },
    {
      "epoch": 0.31669105196534547,
      "grad_norm": 580.0,
      "learning_rate": 3.796783625730994e-07,
      "logits/chosen": -0.6600494384765625,
      "logits/rejected": -0.6603704690933228,
      "logps/chosen": -1027.9356689453125,
      "logps/rejected": -1075.507568359375,
      "loss": 0.6961,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0069137574173510075,
      "rewards/margins": -0.005228615365922451,
      "rewards/rejected": 0.012142373248934746,
      "step": 1204
    },
    {
      "epoch": 0.31695408440053263,
      "grad_norm": 544.0,
      "learning_rate": 3.7953216374269005e-07,
      "logits/chosen": -0.6571281552314758,
      "logits/rejected": -0.6599420309066772,
      "logps/chosen": -1060.8294677734375,
      "logps/rejected": -799.7765502929688,
      "loss": 0.6986,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": 0.001250553410500288,
      "rewards/margins": -0.010224055498838425,
      "rewards/rejected": 0.011474610306322575,
      "step": 1205
    },
    {
      "epoch": 0.3172171168357198,
      "grad_norm": 524.0,
      "learning_rate": 3.793859649122807e-07,
      "logits/chosen": -0.682839572429657,
      "logits/rejected": -0.6873709559440613,
      "logps/chosen": -1021.0252075195312,
      "logps/rejected": -944.2435302734375,
      "loss": 0.6889,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.006485938560217619,
      "rewards/margins": 0.00962839275598526,
      "rewards/rejected": -0.016114331781864166,
      "step": 1206
    },
    {
      "epoch": 0.31748014927090695,
      "grad_norm": 536.0,
      "learning_rate": 3.792397660818713e-07,
      "logits/chosen": -0.6693794131278992,
      "logits/rejected": -0.6715419888496399,
      "logps/chosen": -1316.306396484375,
      "logps/rejected": -1175.4224853515625,
      "loss": 0.6924,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.012103940360248089,
      "rewards/margins": 0.0018007277976721525,
      "rewards/rejected": -0.01390466745942831,
      "step": 1207
    },
    {
      "epoch": 0.3177431817060941,
      "grad_norm": 588.0,
      "learning_rate": 3.7909356725146196e-07,
      "logits/chosen": -0.6465110778808594,
      "logits/rejected": NaN,
      "logps/chosen": -988.5225830078125,
      "logps/rejected": -832.9676513671875,
      "loss": 0.6949,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01695728302001953,
      "rewards/margins": -0.0028486251831054688,
      "rewards/rejected": -0.014108657836914062,
      "step": 1208
    },
    {
      "epoch": 0.3180062141412813,
      "grad_norm": 548.0,
      "learning_rate": 3.7894736842105264e-07,
      "logits/chosen": -0.6327305436134338,
      "logits/rejected": -0.6403325796127319,
      "logps/chosen": -1118.366455078125,
      "logps/rejected": -736.5094604492188,
      "loss": 0.6958,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.02717428281903267,
      "rewards/margins": -0.004264164250344038,
      "rewards/rejected": -0.022910118103027344,
      "step": 1209
    },
    {
      "epoch": 0.31826924657646843,
      "grad_norm": 704.0,
      "learning_rate": 3.7880116959064326e-07,
      "logits/chosen": -0.6380864977836609,
      "logits/rejected": -0.6421930193901062,
      "logps/chosen": -1376.808837890625,
      "logps/rejected": -1190.81884765625,
      "loss": 0.6951,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.009165859781205654,
      "rewards/margins": -0.0031311986967921257,
      "rewards/rejected": -0.006034660618752241,
      "step": 1210
    },
    {
      "epoch": 0.31853227901165565,
      "grad_norm": 452.0,
      "learning_rate": 3.786549707602339e-07,
      "logits/chosen": -0.626272976398468,
      "logits/rejected": -0.6279429793357849,
      "logps/chosen": -1076.854248046875,
      "logps/rejected": -684.7921752929688,
      "loss": 0.6842,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.020760059356689453,
      "rewards/margins": 0.018548298627138138,
      "rewards/rejected": 0.0022117618937045336,
      "step": 1211
    },
    {
      "epoch": 0.3187953114468428,
      "grad_norm": 428.0,
      "learning_rate": 3.7850877192982455e-07,
      "logits/chosen": -0.6805103421211243,
      "logits/rejected": -0.6774182319641113,
      "logps/chosen": -810.4495849609375,
      "logps/rejected": -660.2222900390625,
      "loss": 0.6939,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.01959528774023056,
      "rewards/margins": -0.0009486675262451172,
      "rewards/rejected": -0.018646622076630592,
      "step": 1212
    },
    {
      "epoch": 0.31905834388202997,
      "grad_norm": 580.0,
      "learning_rate": 3.7836257309941517e-07,
      "logits/chosen": -0.6533346176147461,
      "logits/rejected": -0.65491783618927,
      "logps/chosen": -1385.998779296875,
      "logps/rejected": -958.3995361328125,
      "loss": 0.68,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.019289590418338776,
      "rewards/margins": 0.027239371091127396,
      "rewards/rejected": -0.007949781604111195,
      "step": 1213
    },
    {
      "epoch": 0.31932137631721713,
      "grad_norm": 472.0,
      "learning_rate": 3.7821637426900584e-07,
      "logits/chosen": -0.630548357963562,
      "logits/rejected": -0.6387261748313904,
      "logps/chosen": -878.05712890625,
      "logps/rejected": -886.076171875,
      "loss": 0.6889,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005518436897546053,
      "rewards/margins": 0.009759997949004173,
      "rewards/rejected": -0.004241561517119408,
      "step": 1214
    },
    {
      "epoch": 0.3195844087524043,
      "grad_norm": 476.0,
      "learning_rate": 3.7807017543859646e-07,
      "logits/chosen": -0.6553307771682739,
      "logits/rejected": -0.6565780639648438,
      "logps/chosen": -1266.7725830078125,
      "logps/rejected": -1042.53759765625,
      "loss": 0.6889,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.003741359803825617,
      "rewards/margins": 0.008905126713216305,
      "rewards/rejected": -0.005163765978068113,
      "step": 1215
    },
    {
      "epoch": 0.31984744118759145,
      "grad_norm": 532.0,
      "learning_rate": 3.7792397660818713e-07,
      "logits/chosen": -0.6564500331878662,
      "logits/rejected": -0.6563724279403687,
      "logps/chosen": -1220.928466796875,
      "logps/rejected": -877.5025634765625,
      "loss": 0.6975,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.008989144116640091,
      "rewards/margins": -0.007760619278997183,
      "rewards/rejected": 0.01674976386129856,
      "step": 1216
    },
    {
      "epoch": 0.3201104736227786,
      "grad_norm": 604.0,
      "learning_rate": 3.7777777777777775e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6707172989845276,
      "logps/chosen": -1051.6842041015625,
      "logps/rejected": -988.2974243164062,
      "loss": 0.6822,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.013950681313872337,
      "rewards/margins": 0.022678758949041367,
      "rewards/rejected": -0.00872807577252388,
      "step": 1217
    },
    {
      "epoch": 0.3203735060579658,
      "grad_norm": 476.0,
      "learning_rate": 3.7763157894736843e-07,
      "logits/chosen": -0.6688739061355591,
      "logits/rejected": -0.672119140625,
      "logps/chosen": -1145.1126708984375,
      "logps/rejected": -907.8902587890625,
      "loss": 0.7045,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.02241792529821396,
      "rewards/margins": -0.021942520514130592,
      "rewards/rejected": -0.00047540594823658466,
      "step": 1218
    },
    {
      "epoch": 0.32063653849315293,
      "grad_norm": 604.0,
      "learning_rate": 3.7748538011695905e-07,
      "logits/chosen": -0.6508145928382874,
      "logits/rejected": -0.6538054347038269,
      "logps/chosen": -1084.9677734375,
      "logps/rejected": -1056.9830322265625,
      "loss": 0.6882,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0010979657527059317,
      "rewards/margins": 0.010661029256880283,
      "rewards/rejected": -0.011758995242416859,
      "step": 1219
    },
    {
      "epoch": 0.3208995709283401,
      "grad_norm": 482.0,
      "learning_rate": 3.7733918128654967e-07,
      "logits/chosen": -0.6584987640380859,
      "logits/rejected": -0.6602866649627686,
      "logps/chosen": -1281.4398193359375,
      "logps/rejected": -923.5222778320312,
      "loss": 0.7,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.009151412174105644,
      "rewards/margins": -0.012988042086362839,
      "rewards/rejected": 0.022139452397823334,
      "step": 1220
    },
    {
      "epoch": 0.32116260336352725,
      "grad_norm": 628.0,
      "learning_rate": 3.7719298245614034e-07,
      "logits/chosen": -0.6653870344161987,
      "logits/rejected": -0.6676673889160156,
      "logps/chosen": -1621.1151123046875,
      "logps/rejected": -1256.4066162109375,
      "loss": 0.6772,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.024993514642119408,
      "rewards/margins": 0.03253193199634552,
      "rewards/rejected": -0.007538415025919676,
      "step": 1221
    },
    {
      "epoch": 0.3214256357987144,
      "grad_norm": 552.0,
      "learning_rate": 3.77046783625731e-07,
      "logits/chosen": -0.6577020883560181,
      "logits/rejected": -0.6623193025588989,
      "logps/chosen": -1304.0316162109375,
      "logps/rejected": -1047.185546875,
      "loss": 0.6843,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.006333924829959869,
      "rewards/margins": 0.0184218417853117,
      "rewards/rejected": -0.02475576288998127,
      "step": 1222
    },
    {
      "epoch": 0.3216886682339016,
      "grad_norm": 472.0,
      "learning_rate": 3.7690058479532163e-07,
      "logits/chosen": -0.6601275205612183,
      "logits/rejected": -0.6596372127532959,
      "logps/chosen": -1030.2569580078125,
      "logps/rejected": -752.3972778320312,
      "loss": 0.6855,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007597685791552067,
      "rewards/margins": 0.01585879549384117,
      "rewards/rejected": -0.00826110877096653,
      "step": 1223
    },
    {
      "epoch": 0.32195170066908874,
      "grad_norm": 452.0,
      "learning_rate": 3.7675438596491225e-07,
      "logits/chosen": -0.6768001317977905,
      "logits/rejected": -0.6915352940559387,
      "logps/chosen": -1097.232421875,
      "logps/rejected": -947.5355224609375,
      "loss": 0.6927,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010125542059540749,
      "rewards/margins": 0.0017402656376361847,
      "rewards/rejected": 0.008385277353227139,
      "step": 1224
    },
    {
      "epoch": 0.3222147331042759,
      "grad_norm": 564.0,
      "learning_rate": 3.766081871345029e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6804713010787964,
      "logps/chosen": -933.8392944335938,
      "logps/rejected": -892.7415771484375,
      "loss": 0.6912,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.006310652941465378,
      "rewards/margins": 0.005104731302708387,
      "rewards/rejected": 0.001205921871587634,
      "step": 1225
    },
    {
      "epoch": 0.3224777655394631,
      "grad_norm": 584.0,
      "learning_rate": 3.7646198830409355e-07,
      "logits/chosen": -0.6700941324234009,
      "logits/rejected": -0.6773608326911926,
      "logps/chosen": -960.3145751953125,
      "logps/rejected": -676.0164184570312,
      "loss": 0.7034,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.013224124908447266,
      "rewards/margins": -0.01968221925199032,
      "rewards/rejected": 0.006458091549575329,
      "step": 1226
    },
    {
      "epoch": 0.3227407979746503,
      "grad_norm": 532.0,
      "learning_rate": 3.763157894736842e-07,
      "logits/chosen": -0.6270907521247864,
      "logits/rejected": -0.6308906078338623,
      "logps/chosen": -932.5294799804688,
      "logps/rejected": -840.285888671875,
      "loss": 0.6934,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0037756916135549545,
      "rewards/margins": -5.2545685321092606e-05,
      "rewards/rejected": -0.00372314453125,
      "step": 1227
    },
    {
      "epoch": 0.32300383040983743,
      "grad_norm": 452.0,
      "learning_rate": 3.7616959064327484e-07,
      "logits/chosen": -0.6618794202804565,
      "logits/rejected": -0.6633473634719849,
      "logps/chosen": -864.9869384765625,
      "logps/rejected": -543.8757934570312,
      "loss": 0.6979,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0015544408233836293,
      "rewards/margins": -0.009077738970518112,
      "rewards/rejected": 0.00752329733222723,
      "step": 1228
    },
    {
      "epoch": 0.3232668628450246,
      "grad_norm": 416.0,
      "learning_rate": 3.7602339181286546e-07,
      "logits/chosen": -0.6395652294158936,
      "logits/rejected": -0.6438323259353638,
      "logps/chosen": -747.9723510742188,
      "logps/rejected": -486.2598876953125,
      "loss": 0.6929,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006501101888716221,
      "rewards/margins": 0.0009040827862918377,
      "rewards/rejected": 0.0055970195680856705,
      "step": 1229
    },
    {
      "epoch": 0.32352989528021175,
      "grad_norm": 728.0,
      "learning_rate": 3.7587719298245613e-07,
      "logits/chosen": -0.6232985854148865,
      "logits/rejected": -0.6272353529930115,
      "logps/chosen": -1114.3896484375,
      "logps/rejected": -1000.2356567382812,
      "loss": 0.6958,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.005726337432861328,
      "rewards/margins": -0.004703999031335115,
      "rewards/rejected": -0.0010223388671875,
      "step": 1230
    },
    {
      "epoch": 0.3237929277153989,
      "grad_norm": 486.0,
      "learning_rate": 3.757309941520468e-07,
      "logits/chosen": -0.6519759893417358,
      "logits/rejected": -0.6490283608436584,
      "logps/chosen": -899.2621459960938,
      "logps/rejected": -659.6072387695312,
      "loss": 0.6925,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.010189344175159931,
      "rewards/margins": 0.0016777032287791371,
      "rewards/rejected": -0.011867046356201172,
      "step": 1231
    },
    {
      "epoch": 0.3240559601505861,
      "grad_norm": 676.0,
      "learning_rate": 3.7558479532163737e-07,
      "logits/chosen": -0.6698542237281799,
      "logits/rejected": -0.6610192060470581,
      "logps/chosen": -1387.1151123046875,
      "logps/rejected": -1397.9188232421875,
      "loss": 0.6915,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.007120036520063877,
      "rewards/margins": 0.003990365192294121,
      "rewards/rejected": 0.003129672259092331,
      "step": 1232
    },
    {
      "epoch": 0.32431899258577324,
      "grad_norm": 600.0,
      "learning_rate": 3.7543859649122804e-07,
      "logits/chosen": -0.6460720300674438,
      "logits/rejected": -0.6470980048179626,
      "logps/chosen": -1325.9083251953125,
      "logps/rejected": -1083.204833984375,
      "loss": 0.6914,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.011360644362866879,
      "rewards/margins": 0.004246472381055355,
      "rewards/rejected": -0.015607118606567383,
      "step": 1233
    },
    {
      "epoch": 0.3245820250209604,
      "grad_norm": 556.0,
      "learning_rate": 3.752923976608187e-07,
      "logits/chosen": -0.6179148554801941,
      "logits/rejected": -0.6193562150001526,
      "logps/chosen": -1230.4013671875,
      "logps/rejected": -1170.088134765625,
      "loss": 0.6887,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.009149741381406784,
      "rewards/margins": 0.009612752124667168,
      "rewards/rejected": -0.018762491643428802,
      "step": 1234
    },
    {
      "epoch": 0.32484505745614756,
      "grad_norm": 512.0,
      "learning_rate": 3.7514619883040934e-07,
      "logits/chosen": -0.6790138483047485,
      "logits/rejected": NaN,
      "logps/chosen": -1015.6578979492188,
      "logps/rejected": -677.1530151367188,
      "loss": 0.6964,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.010640335269272327,
      "rewards/margins": -0.005389357451349497,
      "rewards/rejected": -0.005250978749245405,
      "step": 1235
    },
    {
      "epoch": 0.3251080898913347,
      "grad_norm": 604.0,
      "learning_rate": 3.75e-07,
      "logits/chosen": -0.6299672722816467,
      "logits/rejected": -0.6281542778015137,
      "logps/chosen": -1274.664794921875,
      "logps/rejected": -1159.2242431640625,
      "loss": 0.7014,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.011155796237289906,
      "rewards/margins": -0.015888787806034088,
      "rewards/rejected": 0.004732991103082895,
      "step": 1236
    },
    {
      "epoch": 0.3253711223265219,
      "grad_norm": 576.0,
      "learning_rate": 3.7485380116959063e-07,
      "logits/chosen": -0.6722093224525452,
      "logits/rejected": -0.6743332743644714,
      "logps/chosen": -1048.5081787109375,
      "logps/rejected": -1047.6114501953125,
      "loss": 0.6832,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0034717561211436987,
      "rewards/margins": 0.020757388323545456,
      "rewards/rejected": -0.017285633832216263,
      "step": 1237
    },
    {
      "epoch": 0.32563415476170904,
      "grad_norm": 600.0,
      "learning_rate": 3.7470760233918125e-07,
      "logits/chosen": -0.6633172035217285,
      "logits/rejected": -0.6752429008483887,
      "logps/chosen": -1367.5535888671875,
      "logps/rejected": -1136.8056640625,
      "loss": 0.6951,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003338051028549671,
      "rewards/margins": -0.0026885494589805603,
      "rewards/rejected": 0.006026601418852806,
      "step": 1238
    },
    {
      "epoch": 0.3258971871968962,
      "grad_norm": 494.0,
      "learning_rate": 3.745614035087719e-07,
      "logits/chosen": -0.6475607752799988,
      "logits/rejected": -0.6601213216781616,
      "logps/chosen": -1109.720458984375,
      "logps/rejected": -874.16748046875,
      "loss": 0.6881,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0160980224609375,
      "rewards/margins": 0.010897158645093441,
      "rewards/rejected": 0.005200862884521484,
      "step": 1239
    },
    {
      "epoch": 0.32616021963208336,
      "grad_norm": 494.0,
      "learning_rate": 3.744152046783626e-07,
      "logits/chosen": -0.6318504810333252,
      "logits/rejected": -0.6373454928398132,
      "logps/chosen": -1246.038330078125,
      "logps/rejected": -895.9244384765625,
      "loss": 0.6967,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.004512882325798273,
      "rewards/margins": -0.0058879852294921875,
      "rewards/rejected": 0.0013751026708632708,
      "step": 1240
    },
    {
      "epoch": 0.3264232520672705,
      "grad_norm": 524.0,
      "learning_rate": 3.7426900584795316e-07,
      "logits/chosen": -0.639539361000061,
      "logits/rejected": -0.6479009389877319,
      "logps/chosen": -1027.42626953125,
      "logps/rejected": -851.9070434570312,
      "loss": 0.7013,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0075552938506007195,
      "rewards/margins": -0.01528167724609375,
      "rewards/rejected": 0.0077263833954930305,
      "step": 1241
    },
    {
      "epoch": 0.32668628450245774,
      "grad_norm": 468.0,
      "learning_rate": 3.7412280701754383e-07,
      "logits/chosen": -0.6335955262184143,
      "logits/rejected": -0.6482945084571838,
      "logps/chosen": -1171.57080078125,
      "logps/rejected": -808.2320556640625,
      "loss": 0.6886,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.015403365716338158,
      "rewards/margins": 0.009586621075868607,
      "rewards/rejected": 0.005816745106130838,
      "step": 1242
    },
    {
      "epoch": 0.3269493169376449,
      "grad_norm": 532.0,
      "learning_rate": 3.739766081871345e-07,
      "logits/chosen": -0.6577844619750977,
      "logits/rejected": -0.666083812713623,
      "logps/chosen": -1275.2294921875,
      "logps/rejected": -1044.636962890625,
      "loss": 0.6952,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.00866165105253458,
      "rewards/margins": -0.0032946597784757614,
      "rewards/rejected": 0.011956309899687767,
      "step": 1243
    },
    {
      "epoch": 0.32721234937283206,
      "grad_norm": 588.0,
      "learning_rate": 3.7383040935672513e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6509775519371033,
      "logps/chosen": -1010.2781982421875,
      "logps/rejected": -923.7470703125,
      "loss": 0.7055,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.02540597878396511,
      "rewards/margins": -0.02378988265991211,
      "rewards/rejected": -0.001616097055375576,
      "step": 1244
    },
    {
      "epoch": 0.3274753818080192,
      "grad_norm": 740.0,
      "learning_rate": 3.7368421052631575e-07,
      "logits/chosen": -0.6527620553970337,
      "logits/rejected": -0.6582884788513184,
      "logps/chosen": -1255.1463623046875,
      "logps/rejected": -1214.11572265625,
      "loss": 0.6866,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012075806967914104,
      "rewards/margins": 0.013622952625155449,
      "rewards/rejected": -0.00154714600648731,
      "step": 1245
    },
    {
      "epoch": 0.3277384142432064,
      "grad_norm": 484.0,
      "learning_rate": 3.735380116959064e-07,
      "logits/chosen": -0.6431283950805664,
      "logits/rejected": -0.6399978399276733,
      "logps/chosen": -1083.415771484375,
      "logps/rejected": -635.5335693359375,
      "loss": 0.6899,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009935189038515091,
      "rewards/margins": 0.007369424682110548,
      "rewards/rejected": 0.0025657645892351866,
      "step": 1246
    },
    {
      "epoch": 0.32800144667839354,
      "grad_norm": 596.0,
      "learning_rate": 3.7339181286549704e-07,
      "logits/chosen": -0.6575179696083069,
      "logits/rejected": NaN,
      "logps/chosen": -1689.306640625,
      "logps/rejected": -1184.2901611328125,
      "loss": 0.6795,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.021484756842255592,
      "rewards/margins": 0.028905486688017845,
      "rewards/rejected": -0.007420730777084827,
      "step": 1247
    },
    {
      "epoch": 0.3282644791135807,
      "grad_norm": 414.0,
      "learning_rate": 3.732456140350877e-07,
      "logits/chosen": -0.6426523327827454,
      "logits/rejected": -0.6491048336029053,
      "logps/chosen": -816.6695556640625,
      "logps/rejected": -677.4271850585938,
      "loss": 0.6952,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.007936572656035423,
      "rewards/margins": -0.003810024354606867,
      "rewards/rejected": -0.0041265483014285564,
      "step": 1248
    },
    {
      "epoch": 0.32852751154876786,
      "grad_norm": 672.0,
      "learning_rate": 3.7309941520467833e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6348952054977417,
      "logps/chosen": -1049.639404296875,
      "logps/rejected": -954.0166625976562,
      "loss": 0.6933,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006057548802345991,
      "rewards/margins": 0.0001704222522675991,
      "rewards/rejected": 0.005887126550078392,
      "step": 1249
    },
    {
      "epoch": 0.328790543983955,
      "grad_norm": 592.0,
      "learning_rate": 3.7295321637426895e-07,
      "logits/chosen": -0.6598955988883972,
      "logits/rejected": -0.6720227003097534,
      "logps/chosen": -1471.189208984375,
      "logps/rejected": -1421.16064453125,
      "loss": 0.6805,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.029271317645907402,
      "rewards/margins": 0.02690153568983078,
      "rewards/rejected": 0.0023697856813669205,
      "step": 1250
    },
    {
      "epoch": 0.3290535764191422,
      "grad_norm": 528.0,
      "learning_rate": 3.728070175438596e-07,
      "logits/chosen": -0.6610252261161804,
      "logits/rejected": -0.6659761667251587,
      "logps/chosen": -837.4346923828125,
      "logps/rejected": -567.82080078125,
      "loss": 0.6872,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.005460929125547409,
      "rewards/margins": 0.012629413045942783,
      "rewards/rejected": -0.007168483454734087,
      "step": 1251
    },
    {
      "epoch": 0.32931660885432934,
      "grad_norm": 576.0,
      "learning_rate": 3.726608187134503e-07,
      "logits/chosen": -0.6406247615814209,
      "logits/rejected": -0.6473864316940308,
      "logps/chosen": -1092.3001708984375,
      "logps/rejected": -981.0410766601562,
      "loss": 0.6913,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0013235090300440788,
      "rewards/margins": 0.004749013110995293,
      "rewards/rejected": -0.006072521209716797,
      "step": 1252
    },
    {
      "epoch": 0.3295796412895165,
      "grad_norm": 458.0,
      "learning_rate": 3.725146198830409e-07,
      "logits/chosen": -0.6590362787246704,
      "logits/rejected": -0.6530187726020813,
      "logps/chosen": -734.80126953125,
      "logps/rejected": -698.3148193359375,
      "loss": 0.6867,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.001401805318892002,
      "rewards/margins": 0.013407611288130283,
      "rewards/rejected": -0.012005805969238281,
      "step": 1253
    },
    {
      "epoch": 0.32984267372470366,
      "grad_norm": 486.0,
      "learning_rate": 3.7236842105263154e-07,
      "logits/chosen": -0.6515040993690491,
      "logits/rejected": -0.6503881812095642,
      "logps/chosen": -1427.359619140625,
      "logps/rejected": -1277.050537109375,
      "loss": 0.6943,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0043263910338282585,
      "rewards/margins": -0.0011369232088327408,
      "rewards/rejected": -0.003189468290656805,
      "step": 1254
    },
    {
      "epoch": 0.3301057061598908,
      "grad_norm": 608.0,
      "learning_rate": 3.722222222222222e-07,
      "logits/chosen": -0.6498206853866577,
      "logits/rejected": NaN,
      "logps/chosen": -1226.9642333984375,
      "logps/rejected": -614.2000732421875,
      "loss": 0.6918,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.014364529401063919,
      "rewards/margins": 0.003186130430549383,
      "rewards/rejected": -0.01755065843462944,
      "step": 1255
    },
    {
      "epoch": 0.330368738595078,
      "grad_norm": 532.0,
      "learning_rate": 3.7207602339181283e-07,
      "logits/chosen": -0.6406186819076538,
      "logits/rejected": -0.6468924283981323,
      "logps/chosen": -1210.95703125,
      "logps/rejected": -901.26953125,
      "loss": 0.6818,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.002126694191247225,
      "rewards/margins": 0.024167442694306374,
      "rewards/rejected": -0.022040747106075287,
      "step": 1256
    },
    {
      "epoch": 0.33063177103026514,
      "grad_norm": 580.0,
      "learning_rate": 3.719298245614035e-07,
      "logits/chosen": -0.6539303064346313,
      "logits/rejected": -0.6571341753005981,
      "logps/chosen": -1089.5767822265625,
      "logps/rejected": -1229.6507568359375,
      "loss": 0.6928,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.015852930024266243,
      "rewards/margins": 0.0013143522664904594,
      "rewards/rejected": 0.01453857496380806,
      "step": 1257
    },
    {
      "epoch": 0.33089480346545236,
      "grad_norm": 592.0,
      "learning_rate": 3.717836257309941e-07,
      "logits/chosen": -0.6736168265342712,
      "logits/rejected": -0.674079954624176,
      "logps/chosen": -1465.023681640625,
      "logps/rejected": -1586.1572265625,
      "loss": 0.6733,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02381591871380806,
      "rewards/margins": 0.04197263717651367,
      "rewards/rejected": -0.018156718462705612,
      "step": 1258
    },
    {
      "epoch": 0.3311578359006395,
      "grad_norm": 548.0,
      "learning_rate": 3.716374269005848e-07,
      "logits/chosen": -0.6546854972839355,
      "logits/rejected": -0.6648303866386414,
      "logps/chosen": -1200.4903564453125,
      "logps/rejected": -751.0581665039062,
      "loss": 0.6949,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.014605903998017311,
      "rewards/margins": -0.002555751707404852,
      "rewards/rejected": 0.01716165617108345,
      "step": 1259
    },
    {
      "epoch": 0.3314208683358267,
      "grad_norm": 3440.0,
      "learning_rate": 3.714912280701754e-07,
      "logits/chosen": -0.6155508756637573,
      "logits/rejected": NaN,
      "logps/chosen": -1060.160400390625,
      "logps/rejected": -618.5640869140625,
      "loss": 0.6918,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.015801239758729935,
      "rewards/margins": 0.00351791363209486,
      "rewards/rejected": -0.01931915245950222,
      "step": 1260
    },
    {
      "epoch": 0.33168390077101384,
      "grad_norm": 532.0,
      "learning_rate": 3.713450292397661e-07,
      "logits/chosen": -0.6231836676597595,
      "logits/rejected": -0.6420918703079224,
      "logps/chosen": -1001.56640625,
      "logps/rejected": -790.2869262695312,
      "loss": 0.6893,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.006185675971210003,
      "rewards/margins": 0.008421468548476696,
      "rewards/rejected": -0.002235793974250555,
      "step": 1261
    },
    {
      "epoch": 0.331946933206201,
      "grad_norm": 520.0,
      "learning_rate": 3.711988304093567e-07,
      "logits/chosen": -0.6663030385971069,
      "logits/rejected": -0.6614829301834106,
      "logps/chosen": -1077.3343505859375,
      "logps/rejected": -722.81689453125,
      "loss": 0.6922,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0003351685591042042,
      "rewards/margins": 0.0030565732158720493,
      "rewards/rejected": -0.0033917424734681845,
      "step": 1262
    },
    {
      "epoch": 0.33220996564138816,
      "grad_norm": 504.0,
      "learning_rate": 3.7105263157894733e-07,
      "logits/chosen": -0.6407740712165833,
      "logits/rejected": -0.6370534896850586,
      "logps/chosen": -997.4801025390625,
      "logps/rejected": -730.269775390625,
      "loss": 0.6954,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.017102528363466263,
      "rewards/margins": -0.0042542461305856705,
      "rewards/rejected": -0.012848282232880592,
      "step": 1263
    },
    {
      "epoch": 0.3324729980765753,
      "grad_norm": 764.0,
      "learning_rate": 3.70906432748538e-07,
      "logits/chosen": -0.641850471496582,
      "logits/rejected": NaN,
      "logps/chosen": -957.506103515625,
      "logps/rejected": -734.4739990234375,
      "loss": 0.6978,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.017904475331306458,
      "rewards/margins": -0.008511352352797985,
      "rewards/rejected": -0.009393120184540749,
      "step": 1264
    },
    {
      "epoch": 0.3327360305117625,
      "grad_norm": 520.0,
      "learning_rate": 3.707602339181287e-07,
      "logits/chosen": -0.6483097672462463,
      "logits/rejected": -0.6507972478866577,
      "logps/chosen": -938.6849365234375,
      "logps/rejected": -922.7664184570312,
      "loss": 0.6991,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.007590008899569511,
      "rewards/margins": -0.010791588574647903,
      "rewards/rejected": 0.003201580373570323,
      "step": 1265
    },
    {
      "epoch": 0.33299906294694964,
      "grad_norm": 428.0,
      "learning_rate": 3.7061403508771924e-07,
      "logits/chosen": -0.6386957168579102,
      "logits/rejected": -0.634488582611084,
      "logps/chosen": -881.3741455078125,
      "logps/rejected": -896.7469482421875,
      "loss": 0.6951,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.008624172769486904,
      "rewards/margins": -0.0033028600737452507,
      "rewards/rejected": 0.01192703191190958,
      "step": 1266
    },
    {
      "epoch": 0.3332620953821368,
      "grad_norm": 588.0,
      "learning_rate": 3.704678362573099e-07,
      "logits/chosen": -0.6300628185272217,
      "logits/rejected": -0.642663836479187,
      "logps/chosen": -1159.32421875,
      "logps/rejected": -938.4071044921875,
      "loss": 0.6912,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.003405475988984108,
      "rewards/margins": 0.00557708740234375,
      "rewards/rejected": -0.008982563391327858,
      "step": 1267
    },
    {
      "epoch": 0.33352512781732396,
      "grad_norm": 752.0,
      "learning_rate": 3.703216374269006e-07,
      "logits/chosen": -0.6591219305992126,
      "logits/rejected": -0.6703633666038513,
      "logps/chosen": -1642.255859375,
      "logps/rejected": -1252.1220703125,
      "loss": 0.7008,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0022262590937316418,
      "rewards/margins": -0.014269493520259857,
      "rewards/rejected": 0.012043237686157227,
      "step": 1268
    },
    {
      "epoch": 0.3337881602525111,
      "grad_norm": 504.0,
      "learning_rate": 3.701754385964912e-07,
      "logits/chosen": -0.6363828182220459,
      "logits/rejected": -0.6365097761154175,
      "logps/chosen": -849.6187133789062,
      "logps/rejected": -579.481689453125,
      "loss": 0.688,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009806156158447266,
      "rewards/margins": 0.011028384789824486,
      "rewards/rejected": -0.0012222290970385075,
      "step": 1269
    },
    {
      "epoch": 0.3340511926876983,
      "grad_norm": 500.0,
      "learning_rate": 3.700292397660819e-07,
      "logits/chosen": -0.6602532267570496,
      "logits/rejected": -0.662274956703186,
      "logps/chosen": -1214.5045166015625,
      "logps/rejected": -928.05517578125,
      "loss": 0.7051,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0014019007794559002,
      "rewards/margins": -0.023005295544862747,
      "rewards/rejected": 0.024407194927334785,
      "step": 1270
    },
    {
      "epoch": 0.33431422512288544,
      "grad_norm": 532.0,
      "learning_rate": 3.698830409356725e-07,
      "logits/chosen": -0.6484554409980774,
      "logits/rejected": -0.66072016954422,
      "logps/chosen": -1027.549560546875,
      "logps/rejected": -851.1241455078125,
      "loss": 0.6873,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0057535176165401936,
      "rewards/margins": 0.012841418385505676,
      "rewards/rejected": -0.007087897043675184,
      "step": 1271
    },
    {
      "epoch": 0.3345772575580726,
      "grad_norm": 544.0,
      "learning_rate": 3.697368421052631e-07,
      "logits/chosen": -0.6438239812850952,
      "logits/rejected": -0.6614159941673279,
      "logps/chosen": -1309.9600830078125,
      "logps/rejected": -866.21630859375,
      "loss": 0.6976,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0011451721657067537,
      "rewards/margins": -0.008025741204619408,
      "rewards/rejected": 0.009170914068818092,
      "step": 1272
    },
    {
      "epoch": 0.3348402899932598,
      "grad_norm": 510.0,
      "learning_rate": 3.695906432748538e-07,
      "logits/chosen": -0.6521553993225098,
      "logits/rejected": -0.6549531817436218,
      "logps/chosen": -1017.5234985351562,
      "logps/rejected": -842.080078125,
      "loss": 0.7,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.004271219950169325,
      "rewards/margins": -0.01277999859303236,
      "rewards/rejected": 0.008508777245879173,
      "step": 1273
    },
    {
      "epoch": 0.335103322428447,
      "grad_norm": 608.0,
      "learning_rate": 3.6944444444444447e-07,
      "logits/chosen": -0.6258636116981506,
      "logits/rejected": -0.646293580532074,
      "logps/chosen": -1108.7723388671875,
      "logps/rejected": -791.73876953125,
      "loss": 0.6818,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.009007930755615234,
      "rewards/margins": 0.023828648030757904,
      "rewards/rejected": -0.014820720069110394,
      "step": 1274
    },
    {
      "epoch": 0.33536635486363414,
      "grad_norm": 528.0,
      "learning_rate": 3.6929824561403503e-07,
      "logits/chosen": -0.6596442461013794,
      "logits/rejected": -0.6578451991081238,
      "logps/chosen": -1046.473876953125,
      "logps/rejected": -853.4508666992188,
      "loss": 0.6844,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0012286186683923006,
      "rewards/margins": 0.018611907958984375,
      "rewards/rejected": -0.019840527325868607,
      "step": 1275
    },
    {
      "epoch": 0.3356293872988213,
      "grad_norm": 608.0,
      "learning_rate": 3.691520467836257e-07,
      "logits/chosen": -0.6453493237495422,
      "logits/rejected": -0.6444498896598816,
      "logps/chosen": -1270.428466796875,
      "logps/rejected": -974.3535766601562,
      "loss": 0.702,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01594238355755806,
      "rewards/margins": -0.016275789588689804,
      "rewards/rejected": 0.00033340509980916977,
      "step": 1276
    },
    {
      "epoch": 0.33589241973400846,
      "grad_norm": 564.0,
      "learning_rate": 3.690058479532164e-07,
      "logits/chosen": -0.6670749187469482,
      "logits/rejected": -0.6720959544181824,
      "logps/chosen": -1217.381591796875,
      "logps/rejected": -1226.543212890625,
      "loss": 0.6984,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006543064024299383,
      "rewards/margins": -0.009732343256473541,
      "rewards/rejected": 0.016275404021143913,
      "step": 1277
    },
    {
      "epoch": 0.3361554521691956,
      "grad_norm": 478.0,
      "learning_rate": 3.68859649122807e-07,
      "logits/chosen": -0.6459900140762329,
      "logits/rejected": -0.6493703126907349,
      "logps/chosen": -1104.36767578125,
      "logps/rejected": -1246.79638671875,
      "loss": 0.6935,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.030999183654785156,
      "rewards/margins": -7.505354005843401e-05,
      "rewards/rejected": 0.03107423707842827,
      "step": 1278
    },
    {
      "epoch": 0.3364184846043828,
      "grad_norm": 604.0,
      "learning_rate": 3.687134502923976e-07,
      "logits/chosen": -0.6353607177734375,
      "logits/rejected": -0.6400195360183716,
      "logps/chosen": -1304.344482421875,
      "logps/rejected": -1005.9697875976562,
      "loss": 0.6888,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02442913129925728,
      "rewards/margins": 0.009822464548051357,
      "rewards/rejected": 0.014606666751205921,
      "step": 1279
    },
    {
      "epoch": 0.33668151703956994,
      "grad_norm": 616.0,
      "learning_rate": 3.685672514619883e-07,
      "logits/chosen": -0.6784083247184753,
      "logits/rejected": -0.6764259338378906,
      "logps/chosen": -1493.8243408203125,
      "logps/rejected": -1041.63134765625,
      "loss": 0.6976,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.013010597787797451,
      "rewards/margins": -0.007549954578280449,
      "rewards/rejected": -0.005460643675178289,
      "step": 1280
    },
    {
      "epoch": 0.3369445494747571,
      "grad_norm": 544.0,
      "learning_rate": 3.684210526315789e-07,
      "logits/chosen": -0.668932318687439,
      "logits/rejected": -0.6677504777908325,
      "logps/chosen": -1116.5184326171875,
      "logps/rejected": -1027.6494140625,
      "loss": 0.7052,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.014998340047895908,
      "rewards/margins": -0.023535536602139473,
      "rewards/rejected": 0.008537196554243565,
      "step": 1281
    },
    {
      "epoch": 0.33720758190994427,
      "grad_norm": 444.0,
      "learning_rate": 3.682748538011696e-07,
      "logits/chosen": -0.6675316691398621,
      "logits/rejected": -0.6623230576515198,
      "logps/chosen": -846.5130615234375,
      "logps/rejected": -528.2722778320312,
      "loss": 0.6953,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01822490617632866,
      "rewards/margins": -0.0035092360340058804,
      "rewards/rejected": -0.01471567153930664,
      "step": 1282
    },
    {
      "epoch": 0.3374706143451314,
      "grad_norm": 528.0,
      "learning_rate": 3.6812865497076026e-07,
      "logits/chosen": -0.6464096307754517,
      "logits/rejected": -0.6424044966697693,
      "logps/chosen": -1196.8681640625,
      "logps/rejected": -875.6204833984375,
      "loss": 0.6825,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.00791101437062025,
      "rewards/margins": 0.021924925968050957,
      "rewards/rejected": -0.014013910666108131,
      "step": 1283
    },
    {
      "epoch": 0.3377336467803186,
      "grad_norm": 470.0,
      "learning_rate": 3.679824561403508e-07,
      "logits/chosen": -0.6423569321632385,
      "logits/rejected": -0.6417719125747681,
      "logps/chosen": -1143.944091796875,
      "logps/rejected": -1038.86865234375,
      "loss": 0.6905,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.003018379444256425,
      "rewards/margins": 0.0056767938658595085,
      "rewards/rejected": -0.0026584151200950146,
      "step": 1284
    },
    {
      "epoch": 0.33799667921550575,
      "grad_norm": 516.0,
      "learning_rate": 3.678362573099415e-07,
      "logits/chosen": -0.6310661435127258,
      "logits/rejected": -0.637885570526123,
      "logps/chosen": -1236.0025634765625,
      "logps/rejected": -1127.5684814453125,
      "loss": 0.6852,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.006221866700798273,
      "rewards/margins": 0.017220498993992805,
      "rewards/rejected": -0.01099863089621067,
      "step": 1285
    },
    {
      "epoch": 0.3382597116506929,
      "grad_norm": 494.0,
      "learning_rate": 3.6769005847953217e-07,
      "logits/chosen": -0.6535565853118896,
      "logits/rejected": -0.6449140310287476,
      "logps/chosen": -882.5667724609375,
      "logps/rejected": -687.4800415039062,
      "loss": 0.6964,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0016257280949503183,
      "rewards/margins": -0.006140040699392557,
      "rewards/rejected": 0.00451431330293417,
      "step": 1286
    },
    {
      "epoch": 0.33852274408588007,
      "grad_norm": 434.0,
      "learning_rate": 3.675438596491228e-07,
      "logits/chosen": -0.634346067905426,
      "logits/rejected": -0.6310549378395081,
      "logps/chosen": -940.5855102539062,
      "logps/rejected": -845.8400268554688,
      "loss": 0.6944,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0009223935194313526,
      "rewards/margins": -0.0018983823247253895,
      "rewards/rejected": 0.0009759890381246805,
      "step": 1287
    },
    {
      "epoch": 0.33878577652106723,
      "grad_norm": 664.0,
      "learning_rate": 3.673976608187134e-07,
      "logits/chosen": -0.6535485982894897,
      "logits/rejected": -0.6589045524597168,
      "logps/chosen": -1498.88818359375,
      "logps/rejected": -1180.72216796875,
      "loss": 0.6934,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.003443240188062191,
      "rewards/margins": 0.0006148346001282334,
      "rewards/rejected": -0.00405807513743639,
      "step": 1288
    },
    {
      "epoch": 0.33904880895625444,
      "grad_norm": 704.0,
      "learning_rate": 3.672514619883041e-07,
      "logits/chosen": -0.6330310702323914,
      "logits/rejected": -0.6322528123855591,
      "logps/chosen": -1647.9345703125,
      "logps/rejected": -1292.18017578125,
      "loss": 0.7013,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0008707051165401936,
      "rewards/margins": -0.014222623780369759,
      "rewards/rejected": 0.015093325637280941,
      "step": 1289
    },
    {
      "epoch": 0.3393118413914416,
      "grad_norm": 564.0,
      "learning_rate": 3.671052631578947e-07,
      "logits/chosen": -0.6598241925239563,
      "logits/rejected": -0.6692287921905518,
      "logps/chosen": -1163.751708984375,
      "logps/rejected": -1287.1702880859375,
      "loss": 0.6878,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0073182107880711555,
      "rewards/margins": 0.011231042444705963,
      "rewards/rejected": -0.018549252301454544,
      "step": 1290
    },
    {
      "epoch": 0.33957487382662876,
      "grad_norm": 524.0,
      "learning_rate": 3.669590643274854e-07,
      "logits/chosen": -0.6224831342697144,
      "logits/rejected": -0.6311237215995789,
      "logps/chosen": -1301.443359375,
      "logps/rejected": -918.4521484375,
      "loss": 0.6866,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.013520432636141777,
      "rewards/margins": 0.013844394125044346,
      "rewards/rejected": -0.027364827692508698,
      "step": 1291
    },
    {
      "epoch": 0.3398379062618159,
      "grad_norm": 532.0,
      "learning_rate": 3.66812865497076e-07,
      "logits/chosen": -0.6621119976043701,
      "logits/rejected": -0.6606582403182983,
      "logps/chosen": -1150.2535400390625,
      "logps/rejected": -856.6397094726562,
      "loss": 0.695,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.007881355471909046,
      "rewards/margins": -0.0028349875938147306,
      "rewards/rejected": 0.01071634329855442,
      "step": 1292
    },
    {
      "epoch": 0.3401009386970031,
      "grad_norm": 576.0,
      "learning_rate": 3.666666666666666e-07,
      "logits/chosen": -0.6326344013214111,
      "logits/rejected": -0.6360766291618347,
      "logps/chosen": -1076.932373046875,
      "logps/rejected": -854.732666015625,
      "loss": 0.7031,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.021210098639130592,
      "rewards/margins": -0.019177721813321114,
      "rewards/rejected": -0.0020323749631643295,
      "step": 1293
    },
    {
      "epoch": 0.34036397113219025,
      "grad_norm": 334.0,
      "learning_rate": 3.665204678362573e-07,
      "logits/chosen": -0.6534469127655029,
      "logits/rejected": -0.6523584723472595,
      "logps/chosen": -445.84075927734375,
      "logps/rejected": -500.73736572265625,
      "loss": 0.6917,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.006244944408535957,
      "rewards/margins": 0.0032409667037427425,
      "rewards/rejected": -0.009485911577939987,
      "step": 1294
    },
    {
      "epoch": 0.3406270035673774,
      "grad_norm": 490.0,
      "learning_rate": 3.6637426900584796e-07,
      "logits/chosen": -0.6550513505935669,
      "logits/rejected": -0.6581655740737915,
      "logps/chosen": -901.6676635742188,
      "logps/rejected": -684.0162353515625,
      "loss": 0.6923,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.00162677769549191,
      "rewards/margins": 0.002211760962381959,
      "rewards/rejected": -0.0005849837325513363,
      "step": 1295
    },
    {
      "epoch": 0.34089003600256457,
      "grad_norm": 548.0,
      "learning_rate": 3.6622807017543853e-07,
      "logits/chosen": -0.6657907962799072,
      "logits/rejected": -0.6651989221572876,
      "logps/chosen": -1105.6500244140625,
      "logps/rejected": -765.2232666015625,
      "loss": 0.6977,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008363534696400166,
      "rewards/margins": -0.008254814893007278,
      "rewards/rejected": 0.01661834679543972,
      "step": 1296
    },
    {
      "epoch": 0.3411530684377517,
      "grad_norm": 492.0,
      "learning_rate": 3.660818713450292e-07,
      "logits/chosen": -0.6479700207710266,
      "logits/rejected": -0.6590862274169922,
      "logps/chosen": -1099.457275390625,
      "logps/rejected": -900.4182739257812,
      "loss": 0.6939,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.005044842604547739,
      "rewards/margins": -0.0010824198834598064,
      "rewards/rejected": 0.0061272624880075455,
      "step": 1297
    },
    {
      "epoch": 0.3414161008729389,
      "grad_norm": 560.0,
      "learning_rate": 3.6593567251461987e-07,
      "logits/chosen": -0.6745576858520508,
      "logits/rejected": -0.6760390996932983,
      "logps/chosen": -1107.0382080078125,
      "logps/rejected": -635.24609375,
      "loss": 0.7043,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.02026081085205078,
      "rewards/margins": -0.021650217473506927,
      "rewards/rejected": 0.0013894075527787209,
      "step": 1298
    },
    {
      "epoch": 0.34167913330812605,
      "grad_norm": 628.0,
      "learning_rate": 3.6578947368421055e-07,
      "logits/chosen": -0.6573056578636169,
      "logits/rejected": NaN,
      "logps/chosen": -1578.97119140625,
      "logps/rejected": -1108.087158203125,
      "loss": 0.6792,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.023104477673768997,
      "rewards/margins": 0.02890043333172798,
      "rewards/rejected": -0.005795956123620272,
      "step": 1299
    },
    {
      "epoch": 0.3419421657433132,
      "grad_norm": 544.0,
      "learning_rate": 3.6564327485380117e-07,
      "logits/chosen": -0.6239686608314514,
      "logits/rejected": -0.6273282766342163,
      "logps/chosen": -1092.4798583984375,
      "logps/rejected": -849.651123046875,
      "loss": 0.6985,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0068458556197583675,
      "rewards/margins": -0.01022186316549778,
      "rewards/rejected": 0.017067719250917435,
      "step": 1300
    },
    {
      "epoch": 0.34220519817850037,
      "grad_norm": 560.0,
      "learning_rate": 3.654970760233918e-07,
      "logits/chosen": -0.6638509631156921,
      "logits/rejected": -0.6690489649772644,
      "logps/chosen": -1187.62548828125,
      "logps/rejected": -913.02197265625,
      "loss": 0.69,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.012491321191191673,
      "rewards/margins": 0.006840181536972523,
      "rewards/rejected": -0.01933150365948677,
      "step": 1301
    },
    {
      "epoch": 0.34246823061368753,
      "grad_norm": 572.0,
      "learning_rate": 3.6535087719298246e-07,
      "logits/chosen": -0.6537299752235413,
      "logits/rejected": -0.6555546522140503,
      "logps/chosen": -1127.3897705078125,
      "logps/rejected": -738.2846069335938,
      "loss": 0.6941,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005984115414321423,
      "rewards/margins": -0.001587867271155119,
      "rewards/rejected": -0.004396247677505016,
      "step": 1302
    },
    {
      "epoch": 0.3427312630488747,
      "grad_norm": 524.0,
      "learning_rate": 3.652046783625731e-07,
      "logits/chosen": -0.639336109161377,
      "logits/rejected": -0.6552169919013977,
      "logps/chosen": -1122.843017578125,
      "logps/rejected": -670.11181640625,
      "loss": 0.6913,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007404995616525412,
      "rewards/margins": 0.004093408118933439,
      "rewards/rejected": 0.0033115865662693977,
      "step": 1303
    },
    {
      "epoch": 0.34299429548406185,
      "grad_norm": 484.0,
      "learning_rate": 3.6505847953216375e-07,
      "logits/chosen": -0.6243453025817871,
      "logits/rejected": -0.6326187252998352,
      "logps/chosen": -1135.6612548828125,
      "logps/rejected": -903.7587280273438,
      "loss": 0.7024,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.012100601568818092,
      "rewards/margins": -0.017577553167939186,
      "rewards/rejected": 0.005476953461766243,
      "step": 1304
    },
    {
      "epoch": 0.34325732791924907,
      "grad_norm": 544.0,
      "learning_rate": 3.6491228070175437e-07,
      "logits/chosen": -0.6369366645812988,
      "logits/rejected": NaN,
      "logps/chosen": -1242.7149658203125,
      "logps/rejected": -690.2888793945312,
      "loss": 0.682,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009689521044492722,
      "rewards/margins": 0.023429585620760918,
      "rewards/rejected": -0.013740062713623047,
      "step": 1305
    },
    {
      "epoch": 0.3435203603544362,
      "grad_norm": 536.0,
      "learning_rate": 3.64766081871345e-07,
      "logits/chosen": -0.6663172245025635,
      "logits/rejected": -0.6652175784111023,
      "logps/chosen": -1263.7298583984375,
      "logps/rejected": -1306.841064453125,
      "loss": 0.6833,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.013202190399169922,
      "rewards/margins": 0.020690251141786575,
      "rewards/rejected": -0.007488060276955366,
      "step": 1306
    },
    {
      "epoch": 0.3437833927896234,
      "grad_norm": 592.0,
      "learning_rate": 3.6461988304093566e-07,
      "logits/chosen": -0.6389211416244507,
      "logits/rejected": -0.6477199196815491,
      "logps/chosen": -1177.404296875,
      "logps/rejected": -854.6286010742188,
      "loss": 0.6852,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.007960700429975986,
      "rewards/margins": 0.016827965155243874,
      "rewards/rejected": -0.008867263793945312,
      "step": 1307
    },
    {
      "epoch": 0.34404642522481055,
      "grad_norm": 544.0,
      "learning_rate": 3.6447368421052634e-07,
      "logits/chosen": -0.6537418365478516,
      "logits/rejected": -0.6491364240646362,
      "logps/chosen": -1106.3681640625,
      "logps/rejected": -695.8341674804688,
      "loss": 0.6908,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.008302498608827591,
      "rewards/margins": 0.005485104396939278,
      "rewards/rejected": -0.013787603937089443,
      "step": 1308
    },
    {
      "epoch": 0.3443094576599977,
      "grad_norm": 700.0,
      "learning_rate": 3.643274853801169e-07,
      "logits/chosen": -0.656991720199585,
      "logits/rejected": -0.6410686373710632,
      "logps/chosen": -1318.58935546875,
      "logps/rejected": -964.6005859375,
      "loss": 0.7061,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.022222567349672318,
      "rewards/margins": -0.02431187778711319,
      "rewards/rejected": 0.0020893090404570103,
      "step": 1309
    },
    {
      "epoch": 0.34457249009518487,
      "grad_norm": 472.0,
      "learning_rate": 3.641812865497076e-07,
      "logits/chosen": -0.650141716003418,
      "logits/rejected": -0.6631180047988892,
      "logps/chosen": -786.0348510742188,
      "logps/rejected": -619.9840087890625,
      "loss": 0.6859,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.014320469461381435,
      "rewards/margins": 0.015591619536280632,
      "rewards/rejected": -0.0012711526360362768,
      "step": 1310
    },
    {
      "epoch": 0.34483552253037203,
      "grad_norm": 592.0,
      "learning_rate": 3.6403508771929825e-07,
      "logits/chosen": -0.6619382500648499,
      "logits/rejected": -0.6578173637390137,
      "logps/chosen": -1311.1915283203125,
      "logps/rejected": -949.3713989257812,
      "loss": 0.7053,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.009972669184207916,
      "rewards/margins": -0.023319724947214127,
      "rewards/rejected": 0.013347054831683636,
      "step": 1311
    },
    {
      "epoch": 0.3450985549655592,
      "grad_norm": 434.0,
      "learning_rate": 3.6388888888888887e-07,
      "logits/chosen": -0.6682755351066589,
      "logits/rejected": -0.6714462041854858,
      "logps/chosen": -871.2708740234375,
      "logps/rejected": -931.7677612304688,
      "loss": 0.6974,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.016429519280791283,
      "rewards/margins": -0.007845019921660423,
      "rewards/rejected": -0.00858449935913086,
      "step": 1312
    },
    {
      "epoch": 0.34536158740074635,
      "grad_norm": 596.0,
      "learning_rate": 3.6374269005847954e-07,
      "logits/chosen": -0.6633219718933105,
      "logits/rejected": NaN,
      "logps/chosen": -1396.3995361328125,
      "logps/rejected": -754.8648681640625,
      "loss": 0.692,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.000402450910769403,
      "rewards/margins": 0.0032750123646110296,
      "rewards/rejected": -0.0036774640902876854,
      "step": 1313
    },
    {
      "epoch": 0.3456246198359335,
      "grad_norm": 564.0,
      "learning_rate": 3.6359649122807016e-07,
      "logits/chosen": -0.6542778611183167,
      "logits/rejected": -0.661386251449585,
      "logps/chosen": -1138.64501953125,
      "logps/rejected": -1001.8331909179688,
      "loss": 0.7005,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.011720370501279831,
      "rewards/margins": -0.012378409504890442,
      "rewards/rejected": 0.024098776280879974,
      "step": 1314
    },
    {
      "epoch": 0.34588765227112067,
      "grad_norm": 502.0,
      "learning_rate": 3.634502923976608e-07,
      "logits/chosen": -0.6449838876724243,
      "logits/rejected": -0.6511586904525757,
      "logps/chosen": -1126.0234375,
      "logps/rejected": -857.4481201171875,
      "loss": 0.6872,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.011693381704390049,
      "rewards/margins": 0.012823009863495827,
      "rewards/rejected": -0.001129627344198525,
      "step": 1315
    },
    {
      "epoch": 0.34615068470630783,
      "grad_norm": 372.0,
      "learning_rate": 3.6330409356725145e-07,
      "logits/chosen": -0.646666944026947,
      "logits/rejected": -0.637140154838562,
      "logps/chosen": -620.97509765625,
      "logps/rejected": -532.4588623046875,
      "loss": 0.6915,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0026616090908646584,
      "rewards/margins": 0.0038456914480775595,
      "rewards/rejected": -0.0011840816587209702,
      "step": 1316
    },
    {
      "epoch": 0.346413717141495,
      "grad_norm": 464.0,
      "learning_rate": 3.6315789473684213e-07,
      "logits/chosen": -0.6446534395217896,
      "logits/rejected": -0.6521958112716675,
      "logps/chosen": -901.8157958984375,
      "logps/rejected": -953.0458984375,
      "loss": 0.7086,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.005483580287545919,
      "rewards/margins": -0.0296022891998291,
      "rewards/rejected": 0.024118708446621895,
      "step": 1317
    },
    {
      "epoch": 0.34667674957668215,
      "grad_norm": 516.0,
      "learning_rate": 3.630116959064327e-07,
      "logits/chosen": -0.6345188617706299,
      "logits/rejected": -0.6502469182014465,
      "logps/chosen": -907.2408447265625,
      "logps/rejected": -874.3869018554688,
      "loss": 0.6977,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01076593529433012,
      "rewards/margins": -0.008026599884033203,
      "rewards/rejected": 0.01879253424704075,
      "step": 1318
    },
    {
      "epoch": 0.3469397820118693,
      "grad_norm": 506.0,
      "learning_rate": 3.6286549707602337e-07,
      "logits/chosen": -0.6327176690101624,
      "logits/rejected": -0.6398833990097046,
      "logps/chosen": -758.2703247070312,
      "logps/rejected": -786.6882934570312,
      "loss": 0.6855,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.009884072467684746,
      "rewards/margins": 0.01588325761258602,
      "rewards/rejected": -0.005999183282256126,
      "step": 1319
    },
    {
      "epoch": 0.34720281444705653,
      "grad_norm": 426.0,
      "learning_rate": 3.6271929824561404e-07,
      "logits/chosen": -0.6503651142120361,
      "logits/rejected": -0.6592583656311035,
      "logps/chosen": -873.3109741210938,
      "logps/rejected": -667.8048095703125,
      "loss": 0.6892,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01838841289281845,
      "rewards/margins": 0.0082388399168849,
      "rewards/rejected": 0.010149573907256126,
      "step": 1320
    },
    {
      "epoch": 0.3474658468822437,
      "grad_norm": 480.0,
      "learning_rate": 3.6257309941520466e-07,
      "logits/chosen": -0.6709631085395813,
      "logits/rejected": -0.6826061606407166,
      "logps/chosen": -1040.87841796875,
      "logps/rejected": -755.8544921875,
      "loss": 0.6931,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0030746471602469683,
      "rewards/margins": 0.0004110343288630247,
      "rewards/rejected": 0.0026636114344000816,
      "step": 1321
    },
    {
      "epoch": 0.34772887931743085,
      "grad_norm": 508.0,
      "learning_rate": 3.624269005847953e-07,
      "logits/chosen": -0.6668371558189392,
      "logits/rejected": -0.6675282120704651,
      "logps/chosen": -1274.831298828125,
      "logps/rejected": -1074.23681640625,
      "loss": 0.6918,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007386112120002508,
      "rewards/margins": 0.0030816069338470697,
      "rewards/rejected": 0.004304504953324795,
      "step": 1322
    },
    {
      "epoch": 0.347991911752618,
      "grad_norm": 536.0,
      "learning_rate": 3.6228070175438595e-07,
      "logits/chosen": -0.634838879108429,
      "logits/rejected": -0.6437892317771912,
      "logps/chosen": -1080.4720458984375,
      "logps/rejected": -797.9248046875,
      "loss": 0.6964,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.021613312885165215,
      "rewards/margins": -0.005946349818259478,
      "rewards/rejected": 0.027559664100408554,
      "step": 1323
    },
    {
      "epoch": 0.34825494418780517,
      "grad_norm": 458.0,
      "learning_rate": 3.6213450292397657e-07,
      "logits/chosen": -0.6781330704689026,
      "logits/rejected": -0.6706657409667969,
      "logps/chosen": -1087.809814453125,
      "logps/rejected": -874.0473022460938,
      "loss": 0.6752,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02841339260339737,
      "rewards/margins": 0.03746052086353302,
      "rewards/rejected": -0.009047126397490501,
      "step": 1324
    },
    {
      "epoch": 0.34851797662299233,
      "grad_norm": 508.0,
      "learning_rate": 3.6198830409356725e-07,
      "logits/chosen": -0.6098615527153015,
      "logits/rejected": -0.6150752305984497,
      "logps/chosen": -1301.522216796875,
      "logps/rejected": -893.73095703125,
      "loss": 0.6854,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0036454196088016033,
      "rewards/margins": 0.017574071884155273,
      "rewards/rejected": -0.013928651809692383,
      "step": 1325
    },
    {
      "epoch": 0.3487810090581795,
      "grad_norm": 502.0,
      "learning_rate": 3.618421052631579e-07,
      "logits/chosen": -0.6450351476669312,
      "logits/rejected": NaN,
      "logps/chosen": -1108.7171630859375,
      "logps/rejected": -640.31298828125,
      "loss": 0.6923,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0033062933944165707,
      "rewards/margins": 0.0028899190947413445,
      "rewards/rejected": 0.0004163746489211917,
      "step": 1326
    },
    {
      "epoch": 0.34904404149336665,
      "grad_norm": 648.0,
      "learning_rate": 3.616959064327485e-07,
      "logits/chosen": -0.6198371648788452,
      "logits/rejected": NaN,
      "logps/chosen": -1355.359619140625,
      "logps/rejected": -985.7081298828125,
      "loss": 0.7156,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.02359485998749733,
      "rewards/margins": -0.043211936950683594,
      "rewards/rejected": 0.019617080688476562,
      "step": 1327
    },
    {
      "epoch": 0.3493070739285538,
      "grad_norm": 552.0,
      "learning_rate": 3.6154970760233916e-07,
      "logits/chosen": -0.6381522417068481,
      "logits/rejected": -0.6441096067428589,
      "logps/chosen": -1462.151611328125,
      "logps/rejected": -1139.8148193359375,
      "loss": 0.7033,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.012685108929872513,
      "rewards/margins": -0.01886777952313423,
      "rewards/rejected": 0.006182670593261719,
      "step": 1328
    },
    {
      "epoch": 0.349570106363741,
      "grad_norm": 508.0,
      "learning_rate": 3.6140350877192983e-07,
      "logits/chosen": -0.6385238170623779,
      "logits/rejected": -0.637847363948822,
      "logps/chosen": -1117.8343505859375,
      "logps/rejected": -780.4579467773438,
      "loss": 0.6982,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.019161034375429153,
      "rewards/margins": -0.009441852569580078,
      "rewards/rejected": -0.009719181805849075,
      "step": 1329
    },
    {
      "epoch": 0.34983313879892813,
      "grad_norm": 476.0,
      "learning_rate": 3.6125730994152045e-07,
      "logits/chosen": -0.6708453893661499,
      "logits/rejected": -0.6771742105484009,
      "logps/chosen": -1116.1129150390625,
      "logps/rejected": -671.5413818359375,
      "loss": 0.7016,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.02339630015194416,
      "rewards/margins": -0.01592264324426651,
      "rewards/rejected": -0.007473660632967949,
      "step": 1330
    },
    {
      "epoch": 0.3500961712341153,
      "grad_norm": 592.0,
      "learning_rate": 3.6111111111111107e-07,
      "logits/chosen": -0.6379939913749695,
      "logits/rejected": -0.6387461423873901,
      "logps/chosen": -1329.0811767578125,
      "logps/rejected": -1137.9078369140625,
      "loss": 0.6964,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0024348259903490543,
      "rewards/margins": -0.005910300649702549,
      "rewards/rejected": 0.003475475125014782,
      "step": 1331
    },
    {
      "epoch": 0.35035920366930245,
      "grad_norm": 548.0,
      "learning_rate": 3.6096491228070174e-07,
      "logits/chosen": -0.6406652927398682,
      "logits/rejected": -0.6515861749649048,
      "logps/chosen": -1100.626708984375,
      "logps/rejected": -895.8937377929688,
      "loss": 0.7021,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.003744125831872225,
      "rewards/margins": -0.01662292331457138,
      "rewards/rejected": 0.012878798879683018,
      "step": 1332
    },
    {
      "epoch": 0.3506222361044896,
      "grad_norm": 500.0,
      "learning_rate": 3.6081871345029236e-07,
      "logits/chosen": -0.6294634342193604,
      "logits/rejected": -0.6371097564697266,
      "logps/chosen": -844.8267211914062,
      "logps/rejected": -666.5530395507812,
      "loss": 0.6879,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007696247659623623,
      "rewards/margins": 0.011217022314667702,
      "rewards/rejected": -0.0035207755863666534,
      "step": 1333
    },
    {
      "epoch": 0.3508852685396768,
      "grad_norm": 482.0,
      "learning_rate": 3.6067251461988304e-07,
      "logits/chosen": -0.6401809453964233,
      "logits/rejected": -0.6434199810028076,
      "logps/chosen": -850.4658813476562,
      "logps/rejected": -696.5142211914062,
      "loss": 0.6947,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0032872201409190893,
      "rewards/margins": -0.002437114715576172,
      "rewards/rejected": -0.0008501054253429174,
      "step": 1334
    },
    {
      "epoch": 0.35114830097486394,
      "grad_norm": 484.0,
      "learning_rate": 3.6052631578947366e-07,
      "logits/chosen": -0.6419109106063843,
      "logits/rejected": -0.6665827035903931,
      "logps/chosen": -1103.388427734375,
      "logps/rejected": -687.6296997070312,
      "loss": 0.6893,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0017216680571436882,
      "rewards/margins": 0.007985543459653854,
      "rewards/rejected": -0.009707212448120117,
      "step": 1335
    },
    {
      "epoch": 0.35141133341005115,
      "grad_norm": 656.0,
      "learning_rate": 3.603801169590643e-07,
      "logits/chosen": -0.6444621086120605,
      "logits/rejected": -0.6571238040924072,
      "logps/chosen": -1379.0308837890625,
      "logps/rejected": -1301.24951171875,
      "loss": 0.6979,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.007498264778405428,
      "rewards/margins": -0.008837033063173294,
      "rewards/rejected": 0.0013387668877840042,
      "step": 1336
    },
    {
      "epoch": 0.3516743658452383,
      "grad_norm": 424.0,
      "learning_rate": 3.6023391812865495e-07,
      "logits/chosen": -0.6449524760246277,
      "logits/rejected": -0.6487862467765808,
      "logps/chosen": -1448.416259765625,
      "logps/rejected": -1338.24072265625,
      "loss": 0.6934,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0017938606906682253,
      "rewards/margins": -0.0001816756557673216,
      "rewards/rejected": 0.001975536346435547,
      "step": 1337
    },
    {
      "epoch": 0.3519373982804255,
      "grad_norm": 768.0,
      "learning_rate": 3.600877192982456e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6589342951774597,
      "logps/chosen": -1134.653564453125,
      "logps/rejected": -1252.6640625,
      "loss": 0.6861,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.004575920756906271,
      "rewards/margins": 0.015108205378055573,
      "rewards/rejected": -0.01053228322416544,
      "step": 1338
    },
    {
      "epoch": 0.35220043071561263,
      "grad_norm": 502.0,
      "learning_rate": 3.5994152046783624e-07,
      "logits/chosen": -0.6359072923660278,
      "logits/rejected": -0.6408352851867676,
      "logps/chosen": -1069.4405517578125,
      "logps/rejected": -563.7899169921875,
      "loss": 0.6928,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0013649938628077507,
      "rewards/margins": 0.0016913414001464844,
      "rewards/rejected": -0.00032634707167744637,
      "step": 1339
    },
    {
      "epoch": 0.3524634631507998,
      "grad_norm": 474.0,
      "learning_rate": 3.5979532163742686e-07,
      "logits/chosen": -0.6413196921348572,
      "logits/rejected": -0.6505141258239746,
      "logps/chosen": -1167.4539794921875,
      "logps/rejected": -935.0425415039062,
      "loss": 0.7017,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.012811660766601562,
      "rewards/margins": -0.016388608142733574,
      "rewards/rejected": 0.003576946211978793,
      "step": 1340
    },
    {
      "epoch": 0.35272649558598695,
      "grad_norm": 492.0,
      "learning_rate": 3.5964912280701754e-07,
      "logits/chosen": -0.6646548509597778,
      "logits/rejected": -0.661876916885376,
      "logps/chosen": -900.5973510742188,
      "logps/rejected": -715.3516845703125,
      "loss": 0.697,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003699016058817506,
      "rewards/margins": -0.007030106149613857,
      "rewards/rejected": 0.01072912197560072,
      "step": 1341
    },
    {
      "epoch": 0.3529895280211741,
      "grad_norm": 600.0,
      "learning_rate": 3.595029239766082e-07,
      "logits/chosen": -0.6689106225967407,
      "logits/rejected": -0.6738813519477844,
      "logps/chosen": -1227.65869140625,
      "logps/rejected": -1102.088134765625,
      "loss": 0.7065,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.02339639887213707,
      "rewards/margins": -0.025931738317012787,
      "rewards/rejected": 0.0025353431701660156,
      "step": 1342
    },
    {
      "epoch": 0.3532525604563613,
      "grad_norm": 544.0,
      "learning_rate": 3.5935672514619883e-07,
      "logits/chosen": -0.6460889577865601,
      "logits/rejected": -0.6636257171630859,
      "logps/chosen": -882.4815063476562,
      "logps/rejected": -918.8009033203125,
      "loss": 0.6869,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.004626465030014515,
      "rewards/margins": 0.01269674301147461,
      "rewards/rejected": -0.008070278912782669,
      "step": 1343
    },
    {
      "epoch": 0.35351559289154844,
      "grad_norm": 500.0,
      "learning_rate": 3.5921052631578945e-07,
      "logits/chosen": -0.6479208469390869,
      "logits/rejected": -0.6579192280769348,
      "logps/chosen": -939.7709350585938,
      "logps/rejected": -550.5693969726562,
      "loss": 0.6982,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.012286758050322533,
      "rewards/margins": -0.00945062655955553,
      "rewards/rejected": -0.002836132887750864,
      "step": 1344
    },
    {
      "epoch": 0.3537786253267356,
      "grad_norm": 502.0,
      "learning_rate": 3.590643274853801e-07,
      "logits/chosen": -0.665551483631134,
      "logits/rejected": -0.6724615097045898,
      "logps/chosen": -1036.260498046875,
      "logps/rejected": -687.5614624023438,
      "loss": 0.6871,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009218788705766201,
      "rewards/margins": 0.012862874194979668,
      "rewards/rejected": -0.003644084557890892,
      "step": 1345
    },
    {
      "epoch": 0.35404165776192276,
      "grad_norm": 668.0,
      "learning_rate": 3.5891812865497074e-07,
      "logits/chosen": -0.6828145384788513,
      "logits/rejected": -0.6784296035766602,
      "logps/chosen": -1788.3323974609375,
      "logps/rejected": -1448.3968505859375,
      "loss": 0.6883,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.017719553783535957,
      "rewards/margins": 0.011350346729159355,
      "rewards/rejected": 0.006369209382683039,
      "step": 1346
    },
    {
      "epoch": 0.3543046901971099,
      "grad_norm": 494.0,
      "learning_rate": 3.587719298245614e-07,
      "logits/chosen": -0.6395364999771118,
      "logits/rejected": -0.6479877829551697,
      "logps/chosen": -825.4178466796875,
      "logps/rejected": -661.7955322265625,
      "loss": 0.6919,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 5.054869689047337e-06,
      "rewards/margins": 0.0032501211389899254,
      "rewards/rejected": -0.003245068248361349,
      "step": 1347
    },
    {
      "epoch": 0.3545677226322971,
      "grad_norm": 454.0,
      "learning_rate": 3.5862573099415203e-07,
      "logits/chosen": -0.6488766074180603,
      "logits/rejected": -0.6534597277641296,
      "logps/chosen": -1447.435546875,
      "logps/rejected": -1151.4154052734375,
      "loss": 0.6835,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0006397260003723204,
      "rewards/margins": 0.02011261135339737,
      "rewards/rejected": -0.020752333104610443,
      "step": 1348
    },
    {
      "epoch": 0.35483075506748424,
      "grad_norm": 516.0,
      "learning_rate": 3.5847953216374265e-07,
      "logits/chosen": -0.6588243246078491,
      "logits/rejected": -0.6633944511413574,
      "logps/chosen": -1066.8408203125,
      "logps/rejected": -908.2868041992188,
      "loss": 0.6742,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.025150872766971588,
      "rewards/margins": 0.038801196962594986,
      "rewards/rejected": -0.013650322332978249,
      "step": 1349
    },
    {
      "epoch": 0.3550937875026714,
      "grad_norm": 536.0,
      "learning_rate": 3.583333333333333e-07,
      "logits/chosen": -0.6460524797439575,
      "logits/rejected": NaN,
      "logps/chosen": -1312.9312744140625,
      "logps/rejected": -787.45849609375,
      "loss": 0.6998,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.011430168524384499,
      "rewards/margins": -0.01258993148803711,
      "rewards/rejected": 0.0011597638949751854,
      "step": 1350
    },
    {
      "epoch": 0.3553568199378586,
      "grad_norm": 624.0,
      "learning_rate": 3.58187134502924e-07,
      "logits/chosen": -0.6571349501609802,
      "logits/rejected": -0.663463830947876,
      "logps/chosen": -1137.37646484375,
      "logps/rejected": -1121.766357421875,
      "loss": 0.695,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01705913431942463,
      "rewards/margins": -0.0031028748489916325,
      "rewards/rejected": -0.013956261798739433,
      "step": 1351
    },
    {
      "epoch": 0.3556198523730458,
      "grad_norm": 460.0,
      "learning_rate": 3.5804093567251457e-07,
      "logits/chosen": -0.6739209294319153,
      "logits/rejected": -0.6750917434692383,
      "logps/chosen": -925.6438598632812,
      "logps/rejected": -830.6806640625,
      "loss": 0.6839,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0069439890794456005,
      "rewards/margins": 0.01911001093685627,
      "rewards/rejected": -0.012166023254394531,
      "step": 1352
    },
    {
      "epoch": 0.35588288480823294,
      "grad_norm": 644.0,
      "learning_rate": 3.5789473684210524e-07,
      "logits/chosen": -0.6402905583381653,
      "logits/rejected": -0.6607736349105835,
      "logps/chosen": -1106.581787109375,
      "logps/rejected": -1175.761962890625,
      "loss": 0.6839,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004130745306611061,
      "rewards/margins": 0.01929492875933647,
      "rewards/rejected": -0.015164186246693134,
      "step": 1353
    },
    {
      "epoch": 0.3561459172434201,
      "grad_norm": 596.0,
      "learning_rate": 3.577485380116959e-07,
      "logits/chosen": -0.6773496270179749,
      "logits/rejected": NaN,
      "logps/chosen": -1084.465576171875,
      "logps/rejected": -692.35693359375,
      "loss": 0.6927,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.009547708556056023,
      "rewards/margins": 0.0015385157894343138,
      "rewards/rejected": -0.01108622644096613,
      "step": 1354
    },
    {
      "epoch": 0.35640894967860726,
      "grad_norm": 616.0,
      "learning_rate": 3.5760233918128653e-07,
      "logits/chosen": -0.6649175882339478,
      "logits/rejected": -0.6556999683380127,
      "logps/chosen": -1186.796875,
      "logps/rejected": -1127.774658203125,
      "loss": 0.7094,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.033887192606925964,
      "rewards/margins": -0.03153061866760254,
      "rewards/rejected": -0.002356575569137931,
      "step": 1355
    },
    {
      "epoch": 0.3566719821137944,
      "grad_norm": 500.0,
      "learning_rate": 3.574561403508772e-07,
      "logits/chosen": -0.6488007307052612,
      "logits/rejected": -0.6458926200866699,
      "logps/chosen": -728.1597290039062,
      "logps/rejected": -598.921875,
      "loss": 0.6878,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0037620540242642164,
      "rewards/margins": 0.011223601177334785,
      "rewards/rejected": -0.0074615478515625,
      "step": 1356
    },
    {
      "epoch": 0.3569350145489816,
      "grad_norm": 540.0,
      "learning_rate": 3.573099415204678e-07,
      "logits/chosen": -0.6407450437545776,
      "logits/rejected": -0.6475000977516174,
      "logps/chosen": -1322.679931640625,
      "logps/rejected": -1016.2112426757812,
      "loss": 0.6952,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004446553997695446,
      "rewards/margins": -0.0033957483246922493,
      "rewards/rejected": 0.00784230139106512,
      "step": 1357
    },
    {
      "epoch": 0.35719804698416874,
      "grad_norm": 484.0,
      "learning_rate": 3.5716374269005844e-07,
      "logits/chosen": -0.6437350511550903,
      "logits/rejected": -0.6496561169624329,
      "logps/chosen": -1112.8067626953125,
      "logps/rejected": -1022.2041015625,
      "loss": 0.6874,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.018245983868837357,
      "rewards/margins": 0.012091446667909622,
      "rewards/rejected": 0.00615453626960516,
      "step": 1358
    },
    {
      "epoch": 0.3574610794193559,
      "grad_norm": 512.0,
      "learning_rate": 3.570175438596491e-07,
      "logits/chosen": -0.6599694490432739,
      "logits/rejected": -0.6669167280197144,
      "logps/chosen": -1228.540771484375,
      "logps/rejected": -1071.2646484375,
      "loss": 0.7025,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.004718590062111616,
      "rewards/margins": -0.01759367063641548,
      "rewards/rejected": 0.012875080108642578,
      "step": 1359
    },
    {
      "epoch": 0.35772411185454306,
      "grad_norm": 516.0,
      "learning_rate": 3.568713450292398e-07,
      "logits/chosen": -0.646009087562561,
      "logits/rejected": -0.6447124481201172,
      "logps/chosen": -1133.465576171875,
      "logps/rejected": -1060.9083251953125,
      "loss": 0.6913,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.012189102359116077,
      "rewards/margins": 0.003819179255515337,
      "rewards/rejected": 0.008369922637939453,
      "step": 1360
    },
    {
      "epoch": 0.3579871442897302,
      "grad_norm": 436.0,
      "learning_rate": 3.5672514619883036e-07,
      "logits/chosen": -0.6335325837135315,
      "logits/rejected": -0.6482622027397156,
      "logps/chosen": -936.1229858398438,
      "logps/rejected": -885.650146484375,
      "loss": 0.6972,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.008586883544921875,
      "rewards/margins": -0.0076201423071324825,
      "rewards/rejected": -0.0009667397243902087,
      "step": 1361
    },
    {
      "epoch": 0.3582501767249174,
      "grad_norm": 490.0,
      "learning_rate": 3.5657894736842103e-07,
      "logits/chosen": -0.676108181476593,
      "logits/rejected": -0.6694931983947754,
      "logps/chosen": -890.84033203125,
      "logps/rejected": -480.82476806640625,
      "loss": 0.6893,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.011376666836440563,
      "rewards/margins": 0.008085489273071289,
      "rewards/rejected": 0.0032911784946918488,
      "step": 1362
    },
    {
      "epoch": 0.35851320916010454,
      "grad_norm": 672.0,
      "learning_rate": 3.564327485380117e-07,
      "logits/chosen": -0.6559972763061523,
      "logits/rejected": -0.6527930498123169,
      "logps/chosen": -1312.930908203125,
      "logps/rejected": -1083.7279052734375,
      "loss": 0.7009,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.013587570749223232,
      "rewards/margins": -0.014585685916244984,
      "rewards/rejected": 0.0009981144685298204,
      "step": 1363
    },
    {
      "epoch": 0.3587762415952917,
      "grad_norm": 612.0,
      "learning_rate": 3.562865497076023e-07,
      "logits/chosen": -0.6592758297920227,
      "logits/rejected": -0.6551997661590576,
      "logps/chosen": -1161.409423828125,
      "logps/rejected": -1119.3416748046875,
      "loss": 0.7011,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.006632233504205942,
      "rewards/margins": -0.015049981884658337,
      "rewards/rejected": 0.008417749777436256,
      "step": 1364
    },
    {
      "epoch": 0.35903927403047886,
      "grad_norm": 624.0,
      "learning_rate": 3.5614035087719294e-07,
      "logits/chosen": -0.6729813814163208,
      "logits/rejected": -0.6652224063873291,
      "logps/chosen": -1224.2816162109375,
      "logps/rejected": -933.5560302734375,
      "loss": 0.6932,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.00746841449290514,
      "rewards/margins": 0.001047896803356707,
      "rewards/rejected": -0.008516310714185238,
      "step": 1365
    },
    {
      "epoch": 0.359302306465666,
      "grad_norm": 608.0,
      "learning_rate": 3.559941520467836e-07,
      "logits/chosen": -0.6716737151145935,
      "logits/rejected": -0.6752448081970215,
      "logps/chosen": -1271.3355712890625,
      "logps/rejected": -973.1414794921875,
      "loss": 0.7064,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0018624302465468645,
      "rewards/margins": -0.025666140019893646,
      "rewards/rejected": 0.02380371280014515,
      "step": 1366
    },
    {
      "epoch": 0.35956533890085324,
      "grad_norm": 458.0,
      "learning_rate": 3.5584795321637423e-07,
      "logits/chosen": -0.6469441652297974,
      "logits/rejected": -0.6495229005813599,
      "logps/chosen": -645.6454467773438,
      "logps/rejected": -630.0075073242188,
      "loss": 0.6918,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006818294525146484,
      "rewards/margins": 0.0030581005848944187,
      "rewards/rejected": 0.0037601941730827093,
      "step": 1367
    },
    {
      "epoch": 0.3598283713360404,
      "grad_norm": 500.0,
      "learning_rate": 3.557017543859649e-07,
      "logits/chosen": -0.6284787654876709,
      "logits/rejected": -0.630110502243042,
      "logps/chosen": -913.41357421875,
      "logps/rejected": -935.6647338867188,
      "loss": 0.6955,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.004645919892936945,
      "rewards/margins": -0.004227066412568092,
      "rewards/rejected": -0.0004188541788607836,
      "step": 1368
    },
    {
      "epoch": 0.36009140377122756,
      "grad_norm": 616.0,
      "learning_rate": 3.5555555555555553e-07,
      "logits/chosen": -0.6804282665252686,
      "logits/rejected": -0.6892584562301636,
      "logps/chosen": -1054.3997802734375,
      "logps/rejected": -894.396484375,
      "loss": 0.6951,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.00023479503579437733,
      "rewards/margins": -0.003446578746661544,
      "rewards/rejected": 0.00368137308396399,
      "step": 1369
    },
    {
      "epoch": 0.3603544362064147,
      "grad_norm": 458.0,
      "learning_rate": 3.5540935672514615e-07,
      "logits/chosen": -0.6450144648551941,
      "logits/rejected": -0.6436710357666016,
      "logps/chosen": -1047.4290771484375,
      "logps/rejected": -856.901123046875,
      "loss": 0.6866,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0034763338044285774,
      "rewards/margins": 0.013948250561952591,
      "rewards/rejected": -0.010471916757524014,
      "step": 1370
    },
    {
      "epoch": 0.3606174686416019,
      "grad_norm": 480.0,
      "learning_rate": 3.552631578947368e-07,
      "logits/chosen": -0.6645822525024414,
      "logits/rejected": -0.6663259267807007,
      "logps/chosen": -1054.5562744140625,
      "logps/rejected": -974.9884643554688,
      "loss": 0.6862,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0115845687687397,
      "rewards/margins": 0.014351033605635166,
      "rewards/rejected": -0.002766466699540615,
      "step": 1371
    },
    {
      "epoch": 0.36088050107678904,
      "grad_norm": 512.0,
      "learning_rate": 3.551169590643275e-07,
      "logits/chosen": -0.6382738947868347,
      "logits/rejected": -0.6449803709983826,
      "logps/chosen": -1124.7919921875,
      "logps/rejected": -893.3270263671875,
      "loss": 0.6966,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0032890799921005964,
      "rewards/margins": -0.006041717249900103,
      "rewards/rejected": 0.0027526384219527245,
      "step": 1372
    },
    {
      "epoch": 0.3611435335119762,
      "grad_norm": 478.0,
      "learning_rate": 3.549707602339181e-07,
      "logits/chosen": -0.6509897708892822,
      "logits/rejected": -0.6507874131202698,
      "logps/chosen": -1127.964111328125,
      "logps/rejected": -959.5784301757812,
      "loss": 0.6914,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0011153211817145348,
      "rewards/margins": 0.0037949085235595703,
      "rewards/rejected": -0.004910230170935392,
      "step": 1373
    },
    {
      "epoch": 0.36140656594716336,
      "grad_norm": 532.0,
      "learning_rate": 3.5482456140350873e-07,
      "logits/chosen": -0.641910195350647,
      "logits/rejected": -0.6529531478881836,
      "logps/chosen": -1197.119873046875,
      "logps/rejected": -999.665283203125,
      "loss": 0.6906,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010266494937241077,
      "rewards/margins": 0.00608224980533123,
      "rewards/rejected": 0.004184246063232422,
      "step": 1374
    },
    {
      "epoch": 0.3616695983823505,
      "grad_norm": 536.0,
      "learning_rate": 3.546783625730994e-07,
      "logits/chosen": -0.6558746099472046,
      "logits/rejected": -0.6649460792541504,
      "logps/chosen": -1186.754150390625,
      "logps/rejected": -943.9686279296875,
      "loss": 0.6831,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0011013980256393552,
      "rewards/margins": 0.020730115473270416,
      "rewards/rejected": -0.021831512451171875,
      "step": 1375
    },
    {
      "epoch": 0.3619326308175377,
      "grad_norm": 596.0,
      "learning_rate": 3.5453216374269e-07,
      "logits/chosen": -0.6239414215087891,
      "logits/rejected": -0.6367314457893372,
      "logps/chosen": -1366.914794921875,
      "logps/rejected": -1010.2578125,
      "loss": 0.6874,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004749967250972986,
      "rewards/margins": 0.012479305267333984,
      "rewards/rejected": -0.00772933941334486,
      "step": 1376
    },
    {
      "epoch": 0.36219566325272484,
      "grad_norm": 708.0,
      "learning_rate": 3.543859649122807e-07,
      "logits/chosen": -0.6479495763778687,
      "logits/rejected": -0.6513209342956543,
      "logps/chosen": -1482.9234619140625,
      "logps/rejected": -1147.47314453125,
      "loss": 0.6971,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.01256637554615736,
      "rewards/margins": -0.006776714697480202,
      "rewards/rejected": 0.019343093037605286,
      "step": 1377
    },
    {
      "epoch": 0.362458695687912,
      "grad_norm": 652.0,
      "learning_rate": 3.542397660818713e-07,
      "logits/chosen": -0.6534637212753296,
      "logits/rejected": -0.6478027105331421,
      "logps/chosen": -1130.21337890625,
      "logps/rejected": -947.3778686523438,
      "loss": 0.6916,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.011113547720015049,
      "rewards/margins": 0.004000199027359486,
      "rewards/rejected": 0.007113351486623287,
      "step": 1378
    },
    {
      "epoch": 0.36272172812309916,
      "grad_norm": 560.0,
      "learning_rate": 3.54093567251462e-07,
      "logits/chosen": -0.6675056219100952,
      "logits/rejected": -0.6563869714736938,
      "logps/chosen": -1231.8426513671875,
      "logps/rejected": -922.991455078125,
      "loss": 0.7088,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.025238465517759323,
      "rewards/margins": -0.030377723276615143,
      "rewards/rejected": 0.0051392558962106705,
      "step": 1379
    },
    {
      "epoch": 0.3629847605582863,
      "grad_norm": 496.0,
      "learning_rate": 3.539473684210526e-07,
      "logits/chosen": -0.6532555818557739,
      "logits/rejected": -0.6649030447006226,
      "logps/chosen": -1221.816650390625,
      "logps/rejected": -1062.724609375,
      "loss": 0.6914,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.023682214319705963,
      "rewards/margins": 0.003849506378173828,
      "rewards/rejected": 0.019832707941532135,
      "step": 1380
    },
    {
      "epoch": 0.3632477929934735,
      "grad_norm": 434.0,
      "learning_rate": 3.538011695906433e-07,
      "logits/chosen": -0.6367974281311035,
      "logits/rejected": -0.6511608362197876,
      "logps/chosen": -956.2252197265625,
      "logps/rejected": -711.2340087890625,
      "loss": 0.6849,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.015186311677098274,
      "rewards/margins": 0.017171382904052734,
      "rewards/rejected": -0.0019850728567689657,
      "step": 1381
    },
    {
      "epoch": 0.36351082542866064,
      "grad_norm": 588.0,
      "learning_rate": 3.536549707602339e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6431409120559692,
      "logps/chosen": -1335.82861328125,
      "logps/rejected": -1297.36572265625,
      "loss": 0.6927,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00020952208433300257,
      "rewards/margins": 0.0018586637452244759,
      "rewards/rejected": -0.0020681857131421566,
      "step": 1382
    },
    {
      "epoch": 0.36377385786384786,
      "grad_norm": 532.0,
      "learning_rate": 3.535087719298245e-07,
      "logits/chosen": -0.661876916885376,
      "logits/rejected": -0.6670750975608826,
      "logps/chosen": -1030.054931640625,
      "logps/rejected": -629.630859375,
      "loss": 0.686,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.028192423284053802,
      "rewards/margins": 0.015834234654903412,
      "rewards/rejected": 0.01235818862915039,
      "step": 1383
    },
    {
      "epoch": 0.364036890299035,
      "grad_norm": 588.0,
      "learning_rate": 3.533625730994152e-07,
      "logits/chosen": -0.6736228466033936,
      "logits/rejected": -0.6769347786903381,
      "logps/chosen": -1290.2296142578125,
      "logps/rejected": -1022.85791015625,
      "loss": 0.6914,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007849598303437233,
      "rewards/margins": 0.0041041867807507515,
      "rewards/rejected": 0.0037454129196703434,
      "step": 1384
    },
    {
      "epoch": 0.3642999227342222,
      "grad_norm": 596.0,
      "learning_rate": 3.5321637426900587e-07,
      "logits/chosen": -0.6689372658729553,
      "logits/rejected": -0.6748322248458862,
      "logps/chosen": -1363.7833251953125,
      "logps/rejected": -830.4442749023438,
      "loss": 0.7126,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.024965189397335052,
      "rewards/margins": -0.037972260266542435,
      "rewards/rejected": 0.013007068075239658,
      "step": 1385
    },
    {
      "epoch": 0.36456295516940934,
      "grad_norm": 740.0,
      "learning_rate": 3.5307017543859644e-07,
      "logits/chosen": -0.6521790623664856,
      "logits/rejected": NaN,
      "logps/chosen": -1046.5992431640625,
      "logps/rejected": -966.0138549804688,
      "loss": 0.7018,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.022841645404696465,
      "rewards/margins": -0.016858387738466263,
      "rewards/rejected": -0.005983258131891489,
      "step": 1386
    },
    {
      "epoch": 0.3648259876045965,
      "grad_norm": 584.0,
      "learning_rate": 3.529239766081871e-07,
      "logits/chosen": -0.6570571660995483,
      "logits/rejected": -0.6630886197090149,
      "logps/chosen": -1445.607666015625,
      "logps/rejected": -1098.842041015625,
      "loss": 0.6814,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0340481773018837,
      "rewards/margins": 0.025349996984004974,
      "rewards/rejected": 0.008698177523911,
      "step": 1387
    },
    {
      "epoch": 0.36508902003978366,
      "grad_norm": 508.0,
      "learning_rate": 3.527777777777778e-07,
      "logits/chosen": -0.667077898979187,
      "logits/rejected": -0.6643998622894287,
      "logps/chosen": -1218.3643798828125,
      "logps/rejected": -892.87353515625,
      "loss": 0.7015,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.003627777798101306,
      "rewards/margins": -0.01600208505988121,
      "rewards/rejected": 0.012374306097626686,
      "step": 1388
    },
    {
      "epoch": 0.3653520524749708,
      "grad_norm": 540.0,
      "learning_rate": 3.526315789473684e-07,
      "logits/chosen": -0.6486114263534546,
      "logits/rejected": -0.6521527171134949,
      "logps/chosen": -1389.9608154296875,
      "logps/rejected": -1092.454833984375,
      "loss": 0.6819,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.014610862359404564,
      "rewards/margins": 0.02320890501141548,
      "rewards/rejected": -0.008598041720688343,
      "step": 1389
    },
    {
      "epoch": 0.365615084910158,
      "grad_norm": 560.0,
      "learning_rate": 3.524853801169591e-07,
      "logits/chosen": -0.6663261651992798,
      "logits/rejected": -0.6696833372116089,
      "logps/chosen": -1130.7838134765625,
      "logps/rejected": -874.9471435546875,
      "loss": 0.6945,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.015496208332479,
      "rewards/margins": -0.0021105287596583366,
      "rewards/rejected": -0.013385677710175514,
      "step": 1390
    },
    {
      "epoch": 0.36587811734534514,
      "grad_norm": 728.0,
      "learning_rate": 3.523391812865497e-07,
      "logits/chosen": -0.6601498126983643,
      "logits/rejected": -0.65851891040802,
      "logps/chosen": -1402.8505859375,
      "logps/rejected": -925.0546264648438,
      "loss": 0.6998,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.010727977380156517,
      "rewards/margins": -0.012265348806977272,
      "rewards/rejected": 0.0015373716596513987,
      "step": 1391
    },
    {
      "epoch": 0.3661411497805323,
      "grad_norm": 656.0,
      "learning_rate": 3.521929824561403e-07,
      "logits/chosen": -0.6350935697555542,
      "logits/rejected": -0.6342304944992065,
      "logps/chosen": -1111.9046630859375,
      "logps/rejected": -866.6410522460938,
      "loss": 0.69,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.006535054184496403,
      "rewards/margins": 0.007514286786317825,
      "rewards/rejected": -0.01404933724552393,
      "step": 1392
    },
    {
      "epoch": 0.36640418221571946,
      "grad_norm": 492.0,
      "learning_rate": 3.52046783625731e-07,
      "logits/chosen": -0.6584075689315796,
      "logits/rejected": -0.6564642786979675,
      "logps/chosen": -961.1216430664062,
      "logps/rejected": -693.9467163085938,
      "loss": 0.7003,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.011811161413788795,
      "rewards/margins": -0.013459491543471813,
      "rewards/rejected": 0.001648330595344305,
      "step": 1393
    },
    {
      "epoch": 0.3666672146509066,
      "grad_norm": 544.0,
      "learning_rate": 3.5190058479532166e-07,
      "logits/chosen": -0.6526809930801392,
      "logits/rejected": -0.6618905067443848,
      "logps/chosen": -1046.5875244140625,
      "logps/rejected": -738.02880859375,
      "loss": 0.6972,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.023501010611653328,
      "rewards/margins": -0.007534789852797985,
      "rewards/rejected": -0.015966223552823067,
      "step": 1394
    },
    {
      "epoch": 0.3669302470860938,
      "grad_norm": 656.0,
      "learning_rate": 3.5175438596491223e-07,
      "logits/chosen": -0.6479520201683044,
      "logits/rejected": -0.6476303339004517,
      "logps/chosen": -1244.8052978515625,
      "logps/rejected": -1195.447509765625,
      "loss": 0.6987,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010631753131747246,
      "rewards/margins": -0.010235692374408245,
      "rewards/rejected": 0.020867446437478065,
      "step": 1395
    },
    {
      "epoch": 0.36719327952128095,
      "grad_norm": 672.0,
      "learning_rate": 3.516081871345029e-07,
      "logits/chosen": -0.6623613834381104,
      "logits/rejected": NaN,
      "logps/chosen": -1376.680419921875,
      "logps/rejected": -1041.167724609375,
      "loss": 0.6954,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.002497387118637562,
      "rewards/margins": -0.0036324982065707445,
      "rewards/rejected": 0.006129885092377663,
      "step": 1396
    },
    {
      "epoch": 0.3674563119564681,
      "grad_norm": 580.0,
      "learning_rate": 3.5146198830409357e-07,
      "logits/chosen": -0.6573936343193054,
      "logits/rejected": -0.6545798182487488,
      "logps/chosen": -1281.0242919921875,
      "logps/rejected": -927.1917724609375,
      "loss": 0.6949,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.013811394572257996,
      "rewards/margins": -0.0026556970551609993,
      "rewards/rejected": 0.01646709442138672,
      "step": 1397
    },
    {
      "epoch": 0.3677193443916553,
      "grad_norm": 502.0,
      "learning_rate": 3.513157894736842e-07,
      "logits/chosen": -0.6742852330207825,
      "logits/rejected": -0.6806498169898987,
      "logps/chosen": -940.64013671875,
      "logps/rejected": -787.5245361328125,
      "loss": 0.6932,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010502146556973457,
      "rewards/margins": 0.0006115916185081005,
      "rewards/rejected": 0.009890556335449219,
      "step": 1398
    },
    {
      "epoch": 0.3679823768268425,
      "grad_norm": 492.0,
      "learning_rate": 3.511695906432748e-07,
      "logits/chosen": -0.6558747887611389,
      "logits/rejected": -0.6516993641853333,
      "logps/chosen": -708.9891357421875,
      "logps/rejected": -791.9132080078125,
      "loss": 0.6985,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00235672015696764,
      "rewards/margins": -0.009929658845067024,
      "rewards/rejected": 0.00757293775677681,
      "step": 1399
    },
    {
      "epoch": 0.36824540926202964,
      "grad_norm": 604.0,
      "learning_rate": 3.510233918128655e-07,
      "logits/chosen": -0.652825117111206,
      "logits/rejected": -0.6621367335319519,
      "logps/chosen": -1495.6478271484375,
      "logps/rejected": -1401.6778564453125,
      "loss": 0.6963,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.002488326746970415,
      "rewards/margins": -0.0037400219589471817,
      "rewards/rejected": 0.0012516975402832031,
      "step": 1400
    },
    {
      "epoch": 0.3685084416972168,
      "grad_norm": 520.0,
      "learning_rate": 3.508771929824561e-07,
      "logits/chosen": -0.6623619794845581,
      "logits/rejected": -0.669260561466217,
      "logps/chosen": -983.3527221679688,
      "logps/rejected": -734.6112670898438,
      "loss": 0.6932,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003595733316615224,
      "rewards/margins": 0.0006052018143236637,
      "rewards/rejected": 0.002990531735122204,
      "step": 1401
    },
    {
      "epoch": 0.36877147413240396,
      "grad_norm": 508.0,
      "learning_rate": 3.507309941520468e-07,
      "logits/chosen": -0.6373196244239807,
      "logits/rejected": -0.6397932767868042,
      "logps/chosen": -1031.00048828125,
      "logps/rejected": -826.2392578125,
      "loss": 0.6934,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.006413554307073355,
      "rewards/margins": -1.5640398487448692e-05,
      "rewards/rejected": 0.006429195404052734,
      "step": 1402
    },
    {
      "epoch": 0.3690345065675911,
      "grad_norm": 478.0,
      "learning_rate": 3.5058479532163745e-07,
      "logits/chosen": -0.6417333483695984,
      "logits/rejected": -0.625402569770813,
      "logps/chosen": -664.35888671875,
      "logps/rejected": -737.3248291015625,
      "loss": 0.7028,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.010564900003373623,
      "rewards/margins": -0.018169023096561432,
      "rewards/rejected": 0.007604121696203947,
      "step": 1403
    },
    {
      "epoch": 0.3692975390027783,
      "grad_norm": 496.0,
      "learning_rate": 3.50438596491228e-07,
      "logits/chosen": -0.6531339287757874,
      "logits/rejected": -0.6650183796882629,
      "logps/chosen": -1379.67626953125,
      "logps/rejected": -990.9039306640625,
      "loss": 0.698,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008325004950165749,
      "rewards/margins": -0.008489992469549179,
      "rewards/rejected": 0.0001649863552302122,
      "step": 1404
    },
    {
      "epoch": 0.36956057143796545,
      "grad_norm": 434.0,
      "learning_rate": 3.502923976608187e-07,
      "logits/chosen": -0.643478274345398,
      "logits/rejected": -0.6365252733230591,
      "logps/chosen": -799.7039184570312,
      "logps/rejected": -698.6820068359375,
      "loss": 0.6933,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0029263021424412727,
      "rewards/margins": -6.690016016364098e-05,
      "rewards/rejected": 0.0029932018369436264,
      "step": 1405
    },
    {
      "epoch": 0.3698236038731526,
      "grad_norm": 484.0,
      "learning_rate": 3.5014619883040936e-07,
      "logits/chosen": -0.6385207176208496,
      "logits/rejected": -0.6419087052345276,
      "logps/chosen": -977.113037109375,
      "logps/rejected": -868.0398559570312,
      "loss": 0.6973,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0022689818870276213,
      "rewards/margins": -0.007192229386419058,
      "rewards/rejected": 0.00946121197193861,
      "step": 1406
    },
    {
      "epoch": 0.37008663630833977,
      "grad_norm": 592.0,
      "learning_rate": 3.5e-07,
      "logits/chosen": -0.6776142120361328,
      "logits/rejected": NaN,
      "logps/chosen": -1302.9437255859375,
      "logps/rejected": -667.72216796875,
      "loss": 0.6804,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.022780515253543854,
      "rewards/margins": 0.026752423495054245,
      "rewards/rejected": -0.003971911501139402,
      "step": 1407
    },
    {
      "epoch": 0.3703496687435269,
      "grad_norm": 608.0,
      "learning_rate": 3.498538011695906e-07,
      "logits/chosen": -0.6617574691772461,
      "logits/rejected": -0.6585871577262878,
      "logps/chosen": -1144.863525390625,
      "logps/rejected": -1243.3277587890625,
      "loss": 0.7017,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.009338617324829102,
      "rewards/margins": -0.01635928265750408,
      "rewards/rejected": 0.025697899982333183,
      "step": 1408
    },
    {
      "epoch": 0.3706127011787141,
      "grad_norm": 580.0,
      "learning_rate": 3.497076023391813e-07,
      "logits/chosen": -0.6647626161575317,
      "logits/rejected": -0.6645058989524841,
      "logps/chosen": -1168.97900390625,
      "logps/rejected": -958.128662109375,
      "loss": 0.6947,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.014772034250199795,
      "rewards/margins": -0.0024472251534461975,
      "rewards/rejected": 0.017219258472323418,
      "step": 1409
    },
    {
      "epoch": 0.37087573361390125,
      "grad_norm": 552.0,
      "learning_rate": 3.495614035087719e-07,
      "logits/chosen": -0.6655471920967102,
      "logits/rejected": -0.6740649938583374,
      "logps/chosen": -1056.2867431640625,
      "logps/rejected": -848.2907104492188,
      "loss": 0.6944,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.017632102593779564,
      "rewards/margins": -0.0012419698759913445,
      "rewards/rejected": 0.018874073401093483,
      "step": 1410
    },
    {
      "epoch": 0.3711387660490884,
      "grad_norm": 482.0,
      "learning_rate": 3.4941520467836257e-07,
      "logits/chosen": -0.6402513384819031,
      "logits/rejected": -0.6379977464675903,
      "logps/chosen": -1143.156005859375,
      "logps/rejected": -983.1594848632812,
      "loss": 0.6909,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.006943512707948685,
      "rewards/margins": 0.005892372690141201,
      "rewards/rejected": 0.00105113978497684,
      "step": 1411
    },
    {
      "epoch": 0.37140179848427557,
      "grad_norm": 548.0,
      "learning_rate": 3.492690058479532e-07,
      "logits/chosen": -0.6515240669250488,
      "logits/rejected": -0.6620530486106873,
      "logps/chosen": -1385.8306884765625,
      "logps/rejected": -918.8182983398438,
      "loss": 0.6915,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.030002404004335403,
      "rewards/margins": 0.003614902961999178,
      "rewards/rejected": 0.026387501507997513,
      "step": 1412
    },
    {
      "epoch": 0.37166483091946273,
      "grad_norm": 544.0,
      "learning_rate": 3.491228070175438e-07,
      "logits/chosen": -0.6711568832397461,
      "logits/rejected": -0.675939679145813,
      "logps/chosen": -1507.6175537109375,
      "logps/rejected": -956.6016845703125,
      "loss": 0.6907,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.003905010409653187,
      "rewards/margins": 0.005321217235177755,
      "rewards/rejected": -0.0014162063598632812,
      "step": 1413
    },
    {
      "epoch": 0.37192786335464995,
      "grad_norm": 592.0,
      "learning_rate": 3.489766081871345e-07,
      "logits/chosen": -0.6540194749832153,
      "logits/rejected": NaN,
      "logps/chosen": -1320.6534423828125,
      "logps/rejected": -823.9541625976562,
      "loss": 0.7085,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.00164623255841434,
      "rewards/margins": -0.02974862977862358,
      "rewards/rejected": 0.03139486536383629,
      "step": 1414
    },
    {
      "epoch": 0.3721908957898371,
      "grad_norm": 484.0,
      "learning_rate": 3.4883040935672516e-07,
      "logits/chosen": -0.6324522495269775,
      "logits/rejected": -0.63787442445755,
      "logps/chosen": -912.7766723632812,
      "logps/rejected": -803.5211181640625,
      "loss": 0.6956,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.008306693285703659,
      "rewards/margins": -0.004168129526078701,
      "rewards/rejected": -0.004138564690947533,
      "step": 1415
    },
    {
      "epoch": 0.37245392822502427,
      "grad_norm": 556.0,
      "learning_rate": 3.486842105263157e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6710573434829712,
      "logps/chosen": -1140.3914794921875,
      "logps/rejected": -846.835693359375,
      "loss": 0.6857,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0014866830315440893,
      "rewards/margins": 0.015319728292524815,
      "rewards/rejected": -0.016806412488222122,
      "step": 1416
    },
    {
      "epoch": 0.3727169606602114,
      "grad_norm": 510.0,
      "learning_rate": 3.485380116959064e-07,
      "logits/chosen": -0.6580555438995361,
      "logits/rejected": -0.6611591577529907,
      "logps/chosen": -1011.6551513671875,
      "logps/rejected": -721.7452392578125,
      "loss": 0.6777,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.023494720458984375,
      "rewards/margins": 0.03199348226189613,
      "rewards/rejected": -0.008498763665556908,
      "step": 1417
    },
    {
      "epoch": 0.3729799930953986,
      "grad_norm": 486.0,
      "learning_rate": 3.4839181286549707e-07,
      "logits/chosen": -0.6750504970550537,
      "logits/rejected": -0.6781420111656189,
      "logps/chosen": -1108.329345703125,
      "logps/rejected": -1239.0738525390625,
      "loss": 0.7034,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.020937347784638405,
      "rewards/margins": -0.018780134618282318,
      "rewards/rejected": -0.0021572127006947994,
      "step": 1418
    },
    {
      "epoch": 0.37324302553058575,
      "grad_norm": 474.0,
      "learning_rate": 3.482456140350877e-07,
      "logits/chosen": -0.6572418808937073,
      "logits/rejected": -0.6687562465667725,
      "logps/chosen": -746.658935546875,
      "logps/rejected": -934.97216796875,
      "loss": 0.6965,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.00367469759657979,
      "rewards/margins": -0.006181240081787109,
      "rewards/rejected": 0.009855937212705612,
      "step": 1419
    },
    {
      "epoch": 0.3735060579657729,
      "grad_norm": 482.0,
      "learning_rate": 3.4809941520467836e-07,
      "logits/chosen": -0.646859884262085,
      "logits/rejected": -0.6480082273483276,
      "logps/chosen": -1009.0355224609375,
      "logps/rejected": -587.2435302734375,
      "loss": 0.6768,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02063160017132759,
      "rewards/margins": 0.03354492411017418,
      "rewards/rejected": -0.012913322076201439,
      "step": 1420
    },
    {
      "epoch": 0.37376909040096007,
      "grad_norm": 524.0,
      "learning_rate": 3.47953216374269e-07,
      "logits/chosen": -0.6618425846099854,
      "logits/rejected": -0.6598893404006958,
      "logps/chosen": -1259.435546875,
      "logps/rejected": -1092.86083984375,
      "loss": 0.697,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.016776464879512787,
      "rewards/margins": -0.006874752230942249,
      "rewards/rejected": 0.02365121990442276,
      "step": 1421
    },
    {
      "epoch": 0.37403212283614723,
      "grad_norm": 632.0,
      "learning_rate": 3.4780701754385965e-07,
      "logits/chosen": -0.6441406011581421,
      "logits/rejected": -0.6522976756095886,
      "logps/chosen": -1294.3411865234375,
      "logps/rejected": -1242.1068115234375,
      "loss": 0.6811,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.009559347294270992,
      "rewards/margins": 0.024770354852080345,
      "rewards/rejected": -0.015211011283099651,
      "step": 1422
    },
    {
      "epoch": 0.3742951552713344,
      "grad_norm": 478.0,
      "learning_rate": 3.4766081871345027e-07,
      "logits/chosen": -0.6522606015205383,
      "logits/rejected": -0.6570340394973755,
      "logps/chosen": -1051.8095703125,
      "logps/rejected": -1020.7888793945312,
      "loss": 0.6897,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.016132067888975143,
      "rewards/margins": 0.007229328621178865,
      "rewards/rejected": -0.02336139604449272,
      "step": 1423
    },
    {
      "epoch": 0.37455818770652155,
      "grad_norm": 620.0,
      "learning_rate": 3.4751461988304095e-07,
      "logits/chosen": -0.6437085866928101,
      "logits/rejected": -0.6451765894889832,
      "logps/chosen": -1423.4957275390625,
      "logps/rejected": -1257.5072021484375,
      "loss": 0.6947,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.004136371426284313,
      "rewards/margins": -0.002037525177001953,
      "rewards/rejected": -0.00209884624928236,
      "step": 1424
    },
    {
      "epoch": 0.3748212201417087,
      "grad_norm": 612.0,
      "learning_rate": 3.4736842105263157e-07,
      "logits/chosen": -0.6731237173080444,
      "logits/rejected": -0.6744900345802307,
      "logps/chosen": -1099.290771484375,
      "logps/rejected": -1029.1436767578125,
      "loss": 0.6976,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004141044802963734,
      "rewards/margins": -0.008219814859330654,
      "rewards/rejected": 0.012360858730971813,
      "step": 1425
    },
    {
      "epoch": 0.37508425257689587,
      "grad_norm": 520.0,
      "learning_rate": 3.472222222222222e-07,
      "logits/chosen": -0.6503170132637024,
      "logits/rejected": -0.6551337242126465,
      "logps/chosen": -1228.401611328125,
      "logps/rejected": -985.6558227539062,
      "loss": 0.6907,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.028598880395293236,
      "rewards/margins": 0.005387688055634499,
      "rewards/rejected": 0.023211192339658737,
      "step": 1426
    },
    {
      "epoch": 0.37534728501208303,
      "grad_norm": 692.0,
      "learning_rate": 3.4707602339181286e-07,
      "logits/chosen": -0.6424387693405151,
      "logits/rejected": -0.6525048017501831,
      "logps/chosen": -1384.0130615234375,
      "logps/rejected": -1105.831298828125,
      "loss": 0.6979,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01901273801922798,
      "rewards/margins": -0.007878780364990234,
      "rewards/rejected": -0.011133957654237747,
      "step": 1427
    },
    {
      "epoch": 0.3756103174472702,
      "grad_norm": 564.0,
      "learning_rate": 3.4692982456140353e-07,
      "logits/chosen": -0.6409892439842224,
      "logits/rejected": -0.646450936794281,
      "logps/chosen": -985.5391235351562,
      "logps/rejected": -997.6229858398438,
      "loss": 0.7031,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.005910110659897327,
      "rewards/margins": -0.018724441528320312,
      "rewards/rejected": 0.024634554982185364,
      "step": 1428
    },
    {
      "epoch": 0.3758733498824574,
      "grad_norm": 620.0,
      "learning_rate": 3.467836257309941e-07,
      "logits/chosen": -0.6470215916633606,
      "logits/rejected": -0.6491357088088989,
      "logps/chosen": -1576.9140625,
      "logps/rejected": -1305.5406494140625,
      "loss": 0.6845,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.035618968307971954,
      "rewards/margins": 0.017976853996515274,
      "rewards/rejected": 0.01764211617410183,
      "step": 1429
    },
    {
      "epoch": 0.37613638231764457,
      "grad_norm": 632.0,
      "learning_rate": 3.4663742690058477e-07,
      "logits/chosen": -0.6594193577766418,
      "logits/rejected": -0.6630119681358337,
      "logps/chosen": -922.3931884765625,
      "logps/rejected": -779.6610717773438,
      "loss": 0.6918,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.010847045108675957,
      "rewards/margins": 0.003531979862600565,
      "rewards/rejected": -0.014379025436937809,
      "step": 1430
    },
    {
      "epoch": 0.37639941475283173,
      "grad_norm": 480.0,
      "learning_rate": 3.4649122807017544e-07,
      "logits/chosen": -0.6481602787971497,
      "logits/rejected": NaN,
      "logps/chosen": -1237.276611328125,
      "logps/rejected": -868.2606201171875,
      "loss": 0.6848,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007169913500547409,
      "rewards/margins": 0.01779050938785076,
      "rewards/rejected": -0.010620594024658203,
      "step": 1431
    },
    {
      "epoch": 0.3766624471880189,
      "grad_norm": 468.0,
      "learning_rate": 3.4634502923976606e-07,
      "logits/chosen": -0.6563947200775146,
      "logits/rejected": -0.6613003015518188,
      "logps/chosen": -1068.3438720703125,
      "logps/rejected": -626.4989624023438,
      "loss": 0.6897,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0004010680131614208,
      "rewards/margins": 0.007607508450746536,
      "rewards/rejected": -0.007206440437585115,
      "step": 1432
    },
    {
      "epoch": 0.37692547962320605,
      "grad_norm": 552.0,
      "learning_rate": 3.4619883040935674e-07,
      "logits/chosen": -0.6685646176338196,
      "logits/rejected": -0.6610023975372314,
      "logps/chosen": -1378.30322265625,
      "logps/rejected": -1161.875732421875,
      "loss": 0.6977,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.012360762804746628,
      "rewards/margins": -0.007770060561597347,
      "rewards/rejected": -0.004590701777487993,
      "step": 1433
    },
    {
      "epoch": 0.3771885120583932,
      "grad_norm": 556.0,
      "learning_rate": 3.4605263157894736e-07,
      "logits/chosen": -0.6409745216369629,
      "logits/rejected": -0.6143701076507568,
      "logps/chosen": -856.7172241210938,
      "logps/rejected": -633.9810180664062,
      "loss": 0.6887,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.009550858289003372,
      "rewards/margins": 0.009511232376098633,
      "rewards/rejected": 3.962521441280842e-05,
      "step": 1434
    },
    {
      "epoch": 0.37745154449358037,
      "grad_norm": 506.0,
      "learning_rate": 3.45906432748538e-07,
      "logits/chosen": -0.6423207521438599,
      "logits/rejected": -0.6451884508132935,
      "logps/chosen": -1170.23779296875,
      "logps/rejected": -881.158203125,
      "loss": 0.6971,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02188105881214142,
      "rewards/margins": -0.0071926116943359375,
      "rewards/rejected": -0.014688445255160332,
      "step": 1435
    },
    {
      "epoch": 0.37771457692876753,
      "grad_norm": 488.0,
      "learning_rate": 3.4576023391812865e-07,
      "logits/chosen": -0.6566854119300842,
      "logits/rejected": NaN,
      "logps/chosen": -959.90625,
      "logps/rejected": -518.9138793945312,
      "loss": 0.6994,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.011470843106508255,
      "rewards/margins": -0.012024451047182083,
      "rewards/rejected": 0.0005536083481274545,
      "step": 1436
    },
    {
      "epoch": 0.3779776093639547,
      "grad_norm": 620.0,
      "learning_rate": 3.456140350877193e-07,
      "logits/chosen": -0.6254503726959229,
      "logits/rejected": NaN,
      "logps/chosen": -1350.01611328125,
      "logps/rejected": -725.8036499023438,
      "loss": 0.6992,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01586608588695526,
      "rewards/margins": -0.010683632455766201,
      "rewards/rejected": -0.0051824552938342094,
      "step": 1437
    },
    {
      "epoch": 0.37824064179914185,
      "grad_norm": 680.0,
      "learning_rate": 3.454678362573099e-07,
      "logits/chosen": -0.6657320261001587,
      "logits/rejected": -0.6840577125549316,
      "logps/chosen": -1469.25341796875,
      "logps/rejected": -1012.7411499023438,
      "loss": 0.7037,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.007836292497813702,
      "rewards/margins": -0.01868734508752823,
      "rewards/rejected": 0.026523636654019356,
      "step": 1438
    },
    {
      "epoch": 0.378503674234329,
      "grad_norm": 600.0,
      "learning_rate": 3.4532163742690056e-07,
      "logits/chosen": -0.6577680706977844,
      "logits/rejected": -0.6629502773284912,
      "logps/chosen": -1184.43212890625,
      "logps/rejected": -1119.61376953125,
      "loss": 0.6788,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01963353157043457,
      "rewards/margins": 0.030687853693962097,
      "rewards/rejected": -0.01105432491749525,
      "step": 1439
    },
    {
      "epoch": 0.3787667066695162,
      "grad_norm": 584.0,
      "learning_rate": 3.4517543859649124e-07,
      "logits/chosen": -0.6640162467956543,
      "logits/rejected": -0.6542878746986389,
      "logps/chosen": -1470.48779296875,
      "logps/rejected": -1042.33642578125,
      "loss": 0.699,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.019687844440340996,
      "rewards/margins": -0.011155511252582073,
      "rewards/rejected": -0.008532333187758923,
      "step": 1440
    },
    {
      "epoch": 0.37902973910470333,
      "grad_norm": 466.0,
      "learning_rate": 3.4502923976608186e-07,
      "logits/chosen": -0.674986720085144,
      "logits/rejected": -0.675905704498291,
      "logps/chosen": -775.4252319335938,
      "logps/rejected": -682.6349487304688,
      "loss": 0.7014,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.009233379736542702,
      "rewards/margins": -0.01541824359446764,
      "rewards/rejected": 0.0061848643235862255,
      "step": 1441
    },
    {
      "epoch": 0.3792927715398905,
      "grad_norm": 516.0,
      "learning_rate": 3.448830409356725e-07,
      "logits/chosen": -0.6595054268836975,
      "logits/rejected": -0.6722273826599121,
      "logps/chosen": -1167.0902099609375,
      "logps/rejected": -807.3331298828125,
      "loss": 0.6994,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.012303734198212624,
      "rewards/margins": -0.011294936761260033,
      "rewards/rejected": -0.0010087965056300163,
      "step": 1442
    },
    {
      "epoch": 0.37955580397507765,
      "grad_norm": 576.0,
      "learning_rate": 3.4473684210526315e-07,
      "logits/chosen": -0.6510381698608398,
      "logits/rejected": NaN,
      "logps/chosen": -983.9494018554688,
      "logps/rejected": -746.6290893554688,
      "loss": 0.6983,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01973876729607582,
      "rewards/margins": -0.009178255684673786,
      "rewards/rejected": -0.01056051254272461,
      "step": 1443
    },
    {
      "epoch": 0.3798188364102648,
      "grad_norm": 464.0,
      "learning_rate": 3.4459064327485377e-07,
      "logits/chosen": -0.6464637517929077,
      "logits/rejected": -0.6450154781341553,
      "logps/chosen": -786.2955322265625,
      "logps/rejected": -828.0525512695312,
      "loss": 0.6858,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.012906122952699661,
      "rewards/margins": 0.015581846237182617,
      "rewards/rejected": -0.02848796918988228,
      "step": 1444
    },
    {
      "epoch": 0.38008186884545203,
      "grad_norm": 568.0,
      "learning_rate": 3.4444444444444444e-07,
      "logits/chosen": -0.6611471176147461,
      "logits/rejected": -0.6668813228607178,
      "logps/chosen": -1328.3226318359375,
      "logps/rejected": -1148.8992919921875,
      "loss": 0.6877,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0055522434413433075,
      "rewards/margins": 0.01131587103009224,
      "rewards/rejected": -0.00576362619176507,
      "step": 1445
    },
    {
      "epoch": 0.3803449012806392,
      "grad_norm": 472.0,
      "learning_rate": 3.442982456140351e-07,
      "logits/chosen": -0.6387221813201904,
      "logits/rejected": -0.6339389681816101,
      "logps/chosen": -922.0442504882812,
      "logps/rejected": -967.2855834960938,
      "loss": 0.6907,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.009441088885068893,
      "rewards/margins": 0.0054610250517725945,
      "rewards/rejected": -0.014902114868164062,
      "step": 1446
    },
    {
      "epoch": 0.38060793371582635,
      "grad_norm": 490.0,
      "learning_rate": 3.441520467836257e-07,
      "logits/chosen": -0.6726828813552856,
      "logits/rejected": -0.6673673987388611,
      "logps/chosen": -1146.3218994140625,
      "logps/rejected": -935.2470092773438,
      "loss": 0.6849,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.016646910458803177,
      "rewards/margins": 0.01694946363568306,
      "rewards/rejected": -0.00030255329329520464,
      "step": 1447
    },
    {
      "epoch": 0.3808709661510135,
      "grad_norm": 756.0,
      "learning_rate": 3.4400584795321635e-07,
      "logits/chosen": -0.6461682319641113,
      "logits/rejected": -0.6650563478469849,
      "logps/chosen": -1193.9742431640625,
      "logps/rejected": -908.4417724609375,
      "loss": 0.696,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.014820958487689495,
      "rewards/margins": -0.005113030318170786,
      "rewards/rejected": -0.009707927703857422,
      "step": 1448
    },
    {
      "epoch": 0.3811339985862007,
      "grad_norm": 536.0,
      "learning_rate": 3.43859649122807e-07,
      "logits/chosen": -0.6651754379272461,
      "logits/rejected": -0.6723856925964355,
      "logps/chosen": -1245.20947265625,
      "logps/rejected": -1003.2911376953125,
      "loss": 0.6825,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02796030044555664,
      "rewards/margins": 0.021856117993593216,
      "rewards/rejected": 0.00610418152064085,
      "step": 1449
    },
    {
      "epoch": 0.38139703102138783,
      "grad_norm": 584.0,
      "learning_rate": 3.4371345029239765e-07,
      "logits/chosen": -0.629694938659668,
      "logits/rejected": -0.6364912986755371,
      "logps/chosen": -1394.356689453125,
      "logps/rejected": -1323.9720458984375,
      "loss": 0.6921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.016963863745331764,
      "rewards/margins": 0.0036097532138228416,
      "rewards/rejected": -0.02057361602783203,
      "step": 1450
    },
    {
      "epoch": 0.381660063456575,
      "grad_norm": 524.0,
      "learning_rate": 3.4356725146198827e-07,
      "logits/chosen": -0.6432716250419617,
      "logits/rejected": -0.6421748399734497,
      "logps/chosen": -1418.08251953125,
      "logps/rejected": -1043.86767578125,
      "loss": 0.68,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.023828981444239616,
      "rewards/margins": 0.027075769379734993,
      "rewards/rejected": -0.0032467846758663654,
      "step": 1451
    },
    {
      "epoch": 0.38192309589176215,
      "grad_norm": 362.0,
      "learning_rate": 3.4342105263157894e-07,
      "logits/chosen": -0.6531819105148315,
      "logits/rejected": -0.6564019322395325,
      "logps/chosen": -691.0771484375,
      "logps/rejected": -629.9965209960938,
      "loss": 0.6996,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0042647370137274265,
      "rewards/margins": -0.012232065200805664,
      "rewards/rejected": 0.007967328652739525,
      "step": 1452
    },
    {
      "epoch": 0.3821861283269493,
      "grad_norm": 502.0,
      "learning_rate": 3.4327485380116956e-07,
      "logits/chosen": -0.6626284122467041,
      "logits/rejected": -0.6708633899688721,
      "logps/chosen": -1163.6156005859375,
      "logps/rejected": -698.0291137695312,
      "loss": 0.6957,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0026128769386559725,
      "rewards/margins": -0.004420852288603783,
      "rewards/rejected": 0.007033729460090399,
      "step": 1453
    },
    {
      "epoch": 0.3824491607621365,
      "grad_norm": 676.0,
      "learning_rate": 3.4312865497076023e-07,
      "logits/chosen": -0.655305802822113,
      "logits/rejected": -0.6468291878700256,
      "logps/chosen": -1458.873291015625,
      "logps/rejected": -1380.8896484375,
      "loss": 0.675,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.008030509576201439,
      "rewards/margins": 0.03800821304321289,
      "rewards/rejected": -0.04603872448205948,
      "step": 1454
    },
    {
      "epoch": 0.38271219319732364,
      "grad_norm": 572.0,
      "learning_rate": 3.4298245614035085e-07,
      "logits/chosen": -0.675561785697937,
      "logits/rejected": -0.6738103628158569,
      "logps/chosen": -717.2552490234375,
      "logps/rejected": -876.2900390625,
      "loss": 0.6956,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011127186939120293,
      "rewards/margins": -0.004011821001768112,
      "rewards/rejected": -0.007115365006029606,
      "step": 1455
    },
    {
      "epoch": 0.3829752256325108,
      "grad_norm": 478.0,
      "learning_rate": 3.4283625730994147e-07,
      "logits/chosen": -0.6472570896148682,
      "logits/rejected": -0.6437224745750427,
      "logps/chosen": -1172.024658203125,
      "logps/rejected": -992.14892578125,
      "loss": 0.7009,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01739358901977539,
      "rewards/margins": -0.01505136676132679,
      "rewards/rejected": -0.002342223422601819,
      "step": 1456
    },
    {
      "epoch": 0.38323825806769796,
      "grad_norm": 478.0,
      "learning_rate": 3.4269005847953214e-07,
      "logits/chosen": -0.6462145447731018,
      "logits/rejected": -0.6541101336479187,
      "logps/chosen": -1275.3709716796875,
      "logps/rejected": -1257.43359375,
      "loss": 0.6964,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008657548576593399,
      "rewards/margins": -0.005342673510313034,
      "rewards/rejected": 0.014000223949551582,
      "step": 1457
    },
    {
      "epoch": 0.3835012905028851,
      "grad_norm": 394.0,
      "learning_rate": 3.425438596491228e-07,
      "logits/chosen": -0.6848607063293457,
      "logits/rejected": -0.6683999300003052,
      "logps/chosen": -819.9373168945312,
      "logps/rejected": -771.37841796875,
      "loss": 0.6987,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.010028935968875885,
      "rewards/margins": -0.010794354602694511,
      "rewards/rejected": 0.0007654192158952355,
      "step": 1458
    },
    {
      "epoch": 0.3837643229380723,
      "grad_norm": 390.0,
      "learning_rate": 3.423976608187134e-07,
      "logits/chosen": -0.6681801080703735,
      "logits/rejected": -0.6554883718490601,
      "logps/chosen": -968.7125854492188,
      "logps/rejected": -616.247314453125,
      "loss": 0.6978,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.007426261901855469,
      "rewards/margins": -0.008924247696995735,
      "rewards/rejected": 0.0014979843981564045,
      "step": 1459
    },
    {
      "epoch": 0.38402735537325944,
      "grad_norm": 450.0,
      "learning_rate": 3.4225146198830406e-07,
      "logits/chosen": -0.6406695246696472,
      "logits/rejected": -0.6428248882293701,
      "logps/chosen": -995.7289428710938,
      "logps/rejected": -823.8850708007812,
      "loss": 0.6888,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.016993524506688118,
      "rewards/margins": 0.00904226303100586,
      "rewards/rejected": 0.007951260544359684,
      "step": 1460
    },
    {
      "epoch": 0.38429038780844665,
      "grad_norm": 418.0,
      "learning_rate": 3.4210526315789473e-07,
      "logits/chosen": -0.6294195652008057,
      "logits/rejected": -0.645450234413147,
      "logps/chosen": -1624.4932861328125,
      "logps/rejected": -1240.3931884765625,
      "loss": 0.6896,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.003943539224565029,
      "rewards/margins": 0.008093168027698994,
      "rewards/rejected": -0.004149626009166241,
      "step": 1461
    },
    {
      "epoch": 0.3845534202436338,
      "grad_norm": 624.0,
      "learning_rate": 3.419590643274854e-07,
      "logits/chosen": -0.6471502780914307,
      "logits/rejected": -0.6455934643745422,
      "logps/chosen": -1011.3195190429688,
      "logps/rejected": -967.708740234375,
      "loss": 0.6952,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00041742308530956507,
      "rewards/margins": -0.0030789386946707964,
      "rewards/rejected": 0.003496360033750534,
      "step": 1462
    },
    {
      "epoch": 0.384816452678821,
      "grad_norm": 478.0,
      "learning_rate": 3.41812865497076e-07,
      "logits/chosen": -0.6389787197113037,
      "logits/rejected": -0.6484268307685852,
      "logps/chosen": -1058.769775390625,
      "logps/rejected": -738.01708984375,
      "loss": 0.7033,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.016965866088867188,
      "rewards/margins": -0.01943826675415039,
      "rewards/rejected": 0.002472400199621916,
      "step": 1463
    },
    {
      "epoch": 0.38507948511400814,
      "grad_norm": 572.0,
      "learning_rate": 3.4166666666666664e-07,
      "logits/chosen": -0.6350022554397583,
      "logits/rejected": -0.633411705493927,
      "logps/chosen": -1247.420654296875,
      "logps/rejected": -671.0526123046875,
      "loss": 0.699,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.008052586577832699,
      "rewards/margins": -0.010145999491214752,
      "rewards/rejected": 0.002093411050736904,
      "step": 1464
    },
    {
      "epoch": 0.3853425175491953,
      "grad_norm": 462.0,
      "learning_rate": 3.415204678362573e-07,
      "logits/chosen": -0.6349196434020996,
      "logits/rejected": -0.6510591506958008,
      "logps/chosen": -1088.2784423828125,
      "logps/rejected": -671.4498291015625,
      "loss": 0.6852,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.015261555090546608,
      "rewards/margins": 0.016310932114720345,
      "rewards/rejected": -0.0010493758600205183,
      "step": 1465
    },
    {
      "epoch": 0.38560554998438246,
      "grad_norm": 628.0,
      "learning_rate": 3.4137426900584794e-07,
      "logits/chosen": -0.6335353851318359,
      "logits/rejected": -0.6355789303779602,
      "logps/chosen": -1130.5576171875,
      "logps/rejected": -969.4520874023438,
      "loss": 0.709,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.015590859577059746,
      "rewards/margins": -0.028751280158758163,
      "rewards/rejected": 0.013160418719053268,
      "step": 1466
    },
    {
      "epoch": 0.3858685824195696,
      "grad_norm": 442.0,
      "learning_rate": 3.412280701754386e-07,
      "logits/chosen": -0.6407554149627686,
      "logits/rejected": -0.6566295027732849,
      "logps/chosen": -896.4446411132812,
      "logps/rejected": -745.5526123046875,
      "loss": 0.6899,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.003170489799231291,
      "rewards/margins": 0.007121897768229246,
      "rewards/rejected": -0.003951406572014093,
      "step": 1467
    },
    {
      "epoch": 0.3861316148547568,
      "grad_norm": 580.0,
      "learning_rate": 3.4108187134502923e-07,
      "logits/chosen": -0.6673235893249512,
      "logits/rejected": -0.6721889972686768,
      "logps/chosen": -1071.21435546875,
      "logps/rejected": -708.6773681640625,
      "loss": 0.6915,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.00671043386682868,
      "rewards/margins": 0.0037103649228811264,
      "rewards/rejected": 0.0030000684782862663,
      "step": 1468
    },
    {
      "epoch": 0.38639464728994394,
      "grad_norm": 592.0,
      "learning_rate": 3.4093567251461985e-07,
      "logits/chosen": -0.6672060489654541,
      "logits/rejected": -0.6769739985466003,
      "logps/chosen": -1333.67138671875,
      "logps/rejected": -942.4464111328125,
      "loss": 0.6957,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01978607103228569,
      "rewards/margins": -0.0046045295894145966,
      "rewards/rejected": -0.015181542374193668,
      "step": 1469
    },
    {
      "epoch": 0.3866576797251311,
      "grad_norm": 432.0,
      "learning_rate": 3.407894736842105e-07,
      "logits/chosen": -0.6597300171852112,
      "logits/rejected": -0.6632373332977295,
      "logps/chosen": -1027.490478515625,
      "logps/rejected": -1036.48486328125,
      "loss": 0.6963,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.007340240757912397,
      "rewards/margins": -0.005974721163511276,
      "rewards/rejected": 0.013314963318407536,
      "step": 1470
    },
    {
      "epoch": 0.38692071216031826,
      "grad_norm": 506.0,
      "learning_rate": 3.406432748538012e-07,
      "logits/chosen": -0.6477001905441284,
      "logits/rejected": -0.6445353627204895,
      "logps/chosen": -822.8078002929688,
      "logps/rejected": -833.9547119140625,
      "loss": 0.6983,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.006427002139389515,
      "rewards/margins": -0.009247872978448868,
      "rewards/rejected": 0.0028208736330270767,
      "step": 1471
    },
    {
      "epoch": 0.3871837445955054,
      "grad_norm": 696.0,
      "learning_rate": 3.4049707602339176e-07,
      "logits/chosen": -0.650501012802124,
      "logits/rejected": -0.6519038081169128,
      "logps/chosen": -1239.372314453125,
      "logps/rejected": -1069.5948486328125,
      "loss": 0.6943,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.011319350451231003,
      "rewards/margins": -0.0016489988192915916,
      "rewards/rejected": 0.012968350201845169,
      "step": 1472
    },
    {
      "epoch": 0.3874467770306926,
      "grad_norm": 584.0,
      "learning_rate": 3.4035087719298243e-07,
      "logits/chosen": -0.6627219915390015,
      "logits/rejected": -0.6584429144859314,
      "logps/chosen": -1212.334228515625,
      "logps/rejected": -890.7426147460938,
      "loss": 0.7015,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.008493042550981045,
      "rewards/margins": -0.016040325164794922,
      "rewards/rejected": 0.007547284476459026,
      "step": 1473
    },
    {
      "epoch": 0.38770980946587974,
      "grad_norm": 504.0,
      "learning_rate": 3.402046783625731e-07,
      "logits/chosen": -0.6403048038482666,
      "logits/rejected": -0.6401491761207581,
      "logps/chosen": -801.6808471679688,
      "logps/rejected": -881.8992309570312,
      "loss": 0.6911,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.005486059933900833,
      "rewards/margins": 0.004465056583285332,
      "rewards/rejected": 0.0010210032342001796,
      "step": 1474
    },
    {
      "epoch": 0.3879728419010669,
      "grad_norm": 652.0,
      "learning_rate": 3.400584795321637e-07,
      "logits/chosen": -0.6521607637405396,
      "logits/rejected": -0.667479395866394,
      "logps/chosen": -1357.99560546875,
      "logps/rejected": -1107.366455078125,
      "loss": 0.7004,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.014256668277084827,
      "rewards/margins": -0.012520931661128998,
      "rewards/rejected": 0.02677760273218155,
      "step": 1475
    },
    {
      "epoch": 0.3882358743362541,
      "grad_norm": 532.0,
      "learning_rate": 3.399122807017544e-07,
      "logits/chosen": -0.6482194066047668,
      "logits/rejected": -0.6522650718688965,
      "logps/chosen": -1252.239990234375,
      "logps/rejected": -1045.0452880859375,
      "loss": 0.6898,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01620807684957981,
      "rewards/margins": 0.0070326803252100945,
      "rewards/rejected": -0.02324075624346733,
      "step": 1476
    },
    {
      "epoch": 0.3884989067714413,
      "grad_norm": 648.0,
      "learning_rate": 3.39766081871345e-07,
      "logits/chosen": -0.6686497926712036,
      "logits/rejected": -0.6674495339393616,
      "logps/chosen": -1234.2991943359375,
      "logps/rejected": -778.916259765625,
      "loss": 0.6891,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005395793356001377,
      "rewards/margins": 0.009623052552342415,
      "rewards/rejected": -0.015018844045698643,
      "step": 1477
    },
    {
      "epoch": 0.38876193920662844,
      "grad_norm": 416.0,
      "learning_rate": 3.3961988304093564e-07,
      "logits/chosen": -0.6153169274330139,
      "logits/rejected": -0.6278780698776245,
      "logps/chosen": -754.09619140625,
      "logps/rejected": -709.6019897460938,
      "loss": 0.6908,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.007073592860251665,
      "rewards/margins": 0.005459211766719818,
      "rewards/rejected": 0.0016143800457939506,
      "step": 1478
    },
    {
      "epoch": 0.3890249716418156,
      "grad_norm": 472.0,
      "learning_rate": 3.394736842105263e-07,
      "logits/chosen": -0.6309705972671509,
      "logits/rejected": -0.6313449144363403,
      "logps/chosen": -1110.112548828125,
      "logps/rejected": -859.683837890625,
      "loss": 0.6818,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0141633041203022,
      "rewards/margins": 0.023761367425322533,
      "rewards/rejected": -0.009598064236342907,
      "step": 1479
    },
    {
      "epoch": 0.38928800407700276,
      "grad_norm": 472.0,
      "learning_rate": 3.39327485380117e-07,
      "logits/chosen": -0.6601778268814087,
      "logits/rejected": -0.6689079403877258,
      "logps/chosen": -1187.989501953125,
      "logps/rejected": -1128.9718017578125,
      "loss": 0.6865,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0034120080526918173,
      "rewards/margins": 0.013868904672563076,
      "rewards/rejected": -0.01045689545571804,
      "step": 1480
    },
    {
      "epoch": 0.3895510365121899,
      "grad_norm": 560.0,
      "learning_rate": 3.3918128654970755e-07,
      "logits/chosen": -0.653596043586731,
      "logits/rejected": -0.6526851654052734,
      "logps/chosen": -1050.826171875,
      "logps/rejected": -1010.5332641601562,
      "loss": 0.6918,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0008964543230831623,
      "rewards/margins": 0.0030752187594771385,
      "rewards/rejected": -0.002178763970732689,
      "step": 1481
    },
    {
      "epoch": 0.3898140689473771,
      "grad_norm": 466.0,
      "learning_rate": 3.390350877192982e-07,
      "logits/chosen": -0.6521125435829163,
      "logits/rejected": -0.6589213609695435,
      "logps/chosen": -877.935546875,
      "logps/rejected": -683.5722045898438,
      "loss": 0.6902,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.001198482234030962,
      "rewards/margins": 0.006799508351832628,
      "rewards/rejected": -0.007997989654541016,
      "step": 1482
    },
    {
      "epoch": 0.39007710138256424,
      "grad_norm": 552.0,
      "learning_rate": 3.388888888888889e-07,
      "logits/chosen": -0.6914211511611938,
      "logits/rejected": -0.6865216493606567,
      "logps/chosen": -1494.6148681640625,
      "logps/rejected": -1186.113525390625,
      "loss": 0.6933,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02642393298447132,
      "rewards/margins": 0.0007957457564771175,
      "rewards/rejected": -0.027219677343964577,
      "step": 1483
    },
    {
      "epoch": 0.3903401338177514,
      "grad_norm": 596.0,
      "learning_rate": 3.387426900584795e-07,
      "logits/chosen": -0.6457560062408447,
      "logits/rejected": -0.6523308753967285,
      "logps/chosen": -1306.0078125,
      "logps/rejected": -1119.9522705078125,
      "loss": 0.6967,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.009450340643525124,
      "rewards/margins": -0.0065898895263671875,
      "rewards/rejected": -0.002860450651496649,
      "step": 1484
    },
    {
      "epoch": 0.39060316625293856,
      "grad_norm": 676.0,
      "learning_rate": 3.3859649122807014e-07,
      "logits/chosen": -0.6631801128387451,
      "logits/rejected": -0.6478182673454285,
      "logps/chosen": -1478.7462158203125,
      "logps/rejected": -1273.9393310546875,
      "loss": 0.7069,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0033882141578942537,
      "rewards/margins": -0.02605275996029377,
      "rewards/rejected": 0.022664545103907585,
      "step": 1485
    },
    {
      "epoch": 0.3908661986881257,
      "grad_norm": 512.0,
      "learning_rate": 3.384502923976608e-07,
      "logits/chosen": -0.6476680040359497,
      "logits/rejected": -0.646785318851471,
      "logps/chosen": -919.1246948242188,
      "logps/rejected": -839.7377319335938,
      "loss": 0.6978,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.016899585723876953,
      "rewards/margins": -0.008214760571718216,
      "rewards/rejected": -0.008684826083481312,
      "step": 1486
    },
    {
      "epoch": 0.3911292311233129,
      "grad_norm": 482.0,
      "learning_rate": 3.3830409356725143e-07,
      "logits/chosen": -0.6560938358306885,
      "logits/rejected": -0.6675927639007568,
      "logps/chosen": -1105.88134765625,
      "logps/rejected": -1350.32763671875,
      "loss": 0.6892,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009891986846923828,
      "rewards/margins": 0.008041858673095703,
      "rewards/rejected": 0.0018501277081668377,
      "step": 1487
    },
    {
      "epoch": 0.39139226355850004,
      "grad_norm": 434.0,
      "learning_rate": 3.381578947368421e-07,
      "logits/chosen": -0.6397997736930847,
      "logits/rejected": -0.6393564343452454,
      "logps/chosen": -1198.5555419921875,
      "logps/rejected": -1176.43798828125,
      "loss": 0.6916,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.014479544013738632,
      "rewards/margins": 0.0035004618111997843,
      "rewards/rejected": 0.010979080572724342,
      "step": 1488
    },
    {
      "epoch": 0.3916552959936872,
      "grad_norm": 484.0,
      "learning_rate": 3.380116959064327e-07,
      "logits/chosen": -0.6744123697280884,
      "logits/rejected": -0.6769609451293945,
      "logps/chosen": -925.9090576171875,
      "logps/rejected": -599.4365234375,
      "loss": 0.7024,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0005318165058270097,
      "rewards/margins": -0.017969559878110886,
      "rewards/rejected": 0.01743774488568306,
      "step": 1489
    },
    {
      "epoch": 0.39191832842887436,
      "grad_norm": 632.0,
      "learning_rate": 3.3786549707602334e-07,
      "logits/chosen": -0.6317046284675598,
      "logits/rejected": -0.63212651014328,
      "logps/chosen": -1631.065185546875,
      "logps/rejected": -1051.6630859375,
      "loss": 0.6967,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.009541702456772327,
      "rewards/margins": -0.005889034830033779,
      "rewards/rejected": -0.003652668558061123,
      "step": 1490
    },
    {
      "epoch": 0.3921813608640615,
      "grad_norm": 568.0,
      "learning_rate": 3.37719298245614e-07,
      "logits/chosen": -0.6570738554000854,
      "logits/rejected": -0.6482739448547363,
      "logps/chosen": -1031.1392822265625,
      "logps/rejected": -1055.1883544921875,
      "loss": 0.7039,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.008017634972929955,
      "rewards/margins": -0.020877741277217865,
      "rewards/rejected": 0.01286010816693306,
      "step": 1491
    },
    {
      "epoch": 0.39244439329924874,
      "grad_norm": 512.0,
      "learning_rate": 3.375730994152047e-07,
      "logits/chosen": -0.645018458366394,
      "logits/rejected": -0.6440029740333557,
      "logps/chosen": -1063.1639404296875,
      "logps/rejected": -843.6893920898438,
      "loss": 0.7027,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.011400794610381126,
      "rewards/margins": -0.017663955688476562,
      "rewards/rejected": 0.006263160612434149,
      "step": 1492
    },
    {
      "epoch": 0.3927074257344359,
      "grad_norm": 532.0,
      "learning_rate": 3.374269005847953e-07,
      "logits/chosen": -0.642045259475708,
      "logits/rejected": -0.6411885023117065,
      "logps/chosen": -816.7265625,
      "logps/rejected": -838.0615234375,
      "loss": 0.6972,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.003443908877670765,
      "rewards/margins": -0.007568788714706898,
      "rewards/rejected": 0.011012697592377663,
      "step": 1493
    },
    {
      "epoch": 0.39297045816962306,
      "grad_norm": 444.0,
      "learning_rate": 3.3728070175438593e-07,
      "logits/chosen": -0.638299822807312,
      "logits/rejected": -0.6362855434417725,
      "logps/chosen": -822.9425048828125,
      "logps/rejected": -816.0430297851562,
      "loss": 0.7011,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.02025580406188965,
      "rewards/margins": -0.014417745172977448,
      "rewards/rejected": -0.005838060285896063,
      "step": 1494
    },
    {
      "epoch": 0.3932334906048102,
      "grad_norm": 552.0,
      "learning_rate": 3.371345029239766e-07,
      "logits/chosen": -0.6552926898002625,
      "logits/rejected": -0.6593828201293945,
      "logps/chosen": -1423.1800537109375,
      "logps/rejected": -1291.6031494140625,
      "loss": 0.6938,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0034822465386241674,
      "rewards/margins": -0.0005945209413766861,
      "rewards/rejected": 0.004076767712831497,
      "step": 1495
    },
    {
      "epoch": 0.3934965230399974,
      "grad_norm": 572.0,
      "learning_rate": 3.369883040935672e-07,
      "logits/chosen": -0.6506297588348389,
      "logits/rejected": -0.6572577357292175,
      "logps/chosen": -983.1588134765625,
      "logps/rejected": -988.5765991210938,
      "loss": 0.6854,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.012033747509121895,
      "rewards/margins": 0.01614055410027504,
      "rewards/rejected": -0.004106807056814432,
      "step": 1496
    },
    {
      "epoch": 0.39375955547518454,
      "grad_norm": 520.0,
      "learning_rate": 3.368421052631579e-07,
      "logits/chosen": -0.6640845537185669,
      "logits/rejected": -0.6832271814346313,
      "logps/chosen": -1190.394287109375,
      "logps/rejected": -994.0598754882812,
      "loss": 0.679,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.01558609027415514,
      "rewards/margins": 0.02942371368408203,
      "rewards/rejected": -0.013837622478604317,
      "step": 1497
    },
    {
      "epoch": 0.3940225879103717,
      "grad_norm": 636.0,
      "learning_rate": 3.366959064327485e-07,
      "logits/chosen": -0.639973521232605,
      "logits/rejected": -0.6516773700714111,
      "logps/chosen": -1180.624755859375,
      "logps/rejected": -563.7590942382812,
      "loss": 0.6947,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004377938341349363,
      "rewards/margins": -0.0020389556884765625,
      "rewards/rejected": 0.006416893098503351,
      "step": 1498
    },
    {
      "epoch": 0.39428562034555886,
      "grad_norm": 704.0,
      "learning_rate": 3.3654970760233913e-07,
      "logits/chosen": -0.6395966410636902,
      "logits/rejected": -0.6427875757217407,
      "logps/chosen": -1509.6138916015625,
      "logps/rejected": -947.513427734375,
      "loss": 0.6967,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01343383826315403,
      "rewards/margins": -0.006449318490922451,
      "rewards/rejected": -0.0069845207035541534,
      "step": 1499
    },
    {
      "epoch": 0.394548652780746,
      "grad_norm": 434.0,
      "learning_rate": 3.364035087719298e-07,
      "logits/chosen": -0.6600313782691956,
      "logits/rejected": -0.6609408855438232,
      "logps/chosen": -813.1091918945312,
      "logps/rejected": -665.2086791992188,
      "loss": 0.6885,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.015158699825406075,
      "rewards/margins": 0.00950245838612318,
      "rewards/rejected": 0.005656242370605469,
      "step": 1500
    },
    {
      "epoch": 0.3948116852159332,
      "grad_norm": 466.0,
      "learning_rate": 3.362573099415205e-07,
      "logits/chosen": -0.6498667597770691,
      "logits/rejected": -0.6574108004570007,
      "logps/chosen": -1149.275146484375,
      "logps/rejected": -780.3310546875,
      "loss": 0.6897,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.04091930389404297,
      "rewards/margins": 0.00769281480461359,
      "rewards/rejected": 0.03322649002075195,
      "step": 1501
    },
    {
      "epoch": 0.39507471765112034,
      "grad_norm": 572.0,
      "learning_rate": 3.361111111111111e-07,
      "logits/chosen": -0.6455311179161072,
      "logits/rejected": -0.6383956670761108,
      "logps/chosen": -1162.676025390625,
      "logps/rejected": -859.3262329101562,
      "loss": 0.6969,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0008916864171624184,
      "rewards/margins": -0.006910991854965687,
      "rewards/rejected": 0.006019306369125843,
      "step": 1502
    },
    {
      "epoch": 0.3953377500863075,
      "grad_norm": 480.0,
      "learning_rate": 3.359649122807017e-07,
      "logits/chosen": -0.6622380614280701,
      "logits/rejected": -0.6509875059127808,
      "logps/chosen": -1063.7105712890625,
      "logps/rejected": -652.822509765625,
      "loss": 0.6856,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0006936078425496817,
      "rewards/margins": 0.015427496284246445,
      "rewards/rejected": -0.01473388634622097,
      "step": 1503
    },
    {
      "epoch": 0.39560078252149466,
      "grad_norm": 568.0,
      "learning_rate": 3.358187134502924e-07,
      "logits/chosen": -0.6158092021942139,
      "logits/rejected": -0.6145615577697754,
      "logps/chosen": -1274.64404296875,
      "logps/rejected": -1208.4345703125,
      "loss": 0.6912,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00514144916087389,
      "rewards/margins": 0.005554201081395149,
      "rewards/rejected": -0.00041274912655353546,
      "step": 1504
    },
    {
      "epoch": 0.3958638149566818,
      "grad_norm": 544.0,
      "learning_rate": 3.3567251461988306e-07,
      "logits/chosen": -0.6587153673171997,
      "logits/rejected": -0.6522644758224487,
      "logps/chosen": -1141.8182373046875,
      "logps/rejected": -1231.046630859375,
      "loss": 0.6816,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.003332233289256692,
      "rewards/margins": 0.024474620819091797,
      "rewards/rejected": -0.0211423859000206,
      "step": 1505
    },
    {
      "epoch": 0.396126847391869,
      "grad_norm": 510.0,
      "learning_rate": 3.3552631578947363e-07,
      "logits/chosen": -0.6611145734786987,
      "logits/rejected": -0.6687253713607788,
      "logps/chosen": -1040.8919677734375,
      "logps/rejected": -914.370849609375,
      "loss": 0.6975,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.018057825043797493,
      "rewards/margins": -0.0079507352784276,
      "rewards/rejected": -0.010107088834047318,
      "step": 1506
    },
    {
      "epoch": 0.39638987982705615,
      "grad_norm": 764.0,
      "learning_rate": 3.353801169590643e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6545454859733582,
      "logps/chosen": -1223.85009765625,
      "logps/rejected": -1198.403564453125,
      "loss": 0.6902,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.009401512332260609,
      "rewards/margins": 0.007730866316705942,
      "rewards/rejected": -0.01713237725198269,
      "step": 1507
    },
    {
      "epoch": 0.39665291226224336,
      "grad_norm": 612.0,
      "learning_rate": 3.35233918128655e-07,
      "logits/chosen": -0.6474982500076294,
      "logits/rejected": -0.660403847694397,
      "logps/chosen": -1155.5546875,
      "logps/rejected": -774.149658203125,
      "loss": 0.6816,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.021232986822724342,
      "rewards/margins": 0.023893021047115326,
      "rewards/rejected": -0.002660035155713558,
      "step": 1508
    },
    {
      "epoch": 0.3969159446974305,
      "grad_norm": 474.0,
      "learning_rate": 3.350877192982456e-07,
      "logits/chosen": -0.6604600548744202,
      "logits/rejected": -0.666273832321167,
      "logps/chosen": -902.53466796875,
      "logps/rejected": -761.614501953125,
      "loss": 0.6901,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.004280948545783758,
      "rewards/margins": 0.006757261231541634,
      "rewards/rejected": -0.01103820838034153,
      "step": 1509
    },
    {
      "epoch": 0.3971789771326177,
      "grad_norm": 584.0,
      "learning_rate": 3.3494152046783627e-07,
      "logits/chosen": -0.6450804471969604,
      "logits/rejected": -0.6531645655632019,
      "logps/chosen": -1247.2540283203125,
      "logps/rejected": -1060.27392578125,
      "loss": 0.6846,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0037761698476970196,
      "rewards/margins": 0.018382741138339043,
      "rewards/rejected": -0.0221589095890522,
      "step": 1510
    },
    {
      "epoch": 0.39744200956780484,
      "grad_norm": 692.0,
      "learning_rate": 3.347953216374269e-07,
      "logits/chosen": -0.6574138402938843,
      "logits/rejected": -0.6512486934661865,
      "logps/chosen": -1823.14892578125,
      "logps/rejected": -1525.7919921875,
      "loss": 0.6736,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.026028823107481003,
      "rewards/margins": 0.03979520872235298,
      "rewards/rejected": -0.013766384683549404,
      "step": 1511
    },
    {
      "epoch": 0.397705042002992,
      "grad_norm": 648.0,
      "learning_rate": 3.346491228070175e-07,
      "logits/chosen": -0.6600540280342102,
      "logits/rejected": -0.658531129360199,
      "logps/chosen": -1758.9664306640625,
      "logps/rejected": -953.1635131835938,
      "loss": 0.6985,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.010785245336592197,
      "rewards/margins": -0.010141897015273571,
      "rewards/rejected": -0.0006433485541492701,
      "step": 1512
    },
    {
      "epoch": 0.39796807443817916,
      "grad_norm": 396.0,
      "learning_rate": 3.345029239766082e-07,
      "logits/chosen": -0.6448550224304199,
      "logits/rejected": -0.6446111798286438,
      "logps/chosen": -580.108154296875,
      "logps/rejected": -530.26416015625,
      "loss": 0.6888,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01574230194091797,
      "rewards/margins": 0.008915902115404606,
      "rewards/rejected": 0.0068264007568359375,
      "step": 1513
    },
    {
      "epoch": 0.3982311068733663,
      "grad_norm": 676.0,
      "learning_rate": 3.3435672514619886e-07,
      "logits/chosen": -0.6586689352989197,
      "logits/rejected": NaN,
      "logps/chosen": -1142.26708984375,
      "logps/rejected": -852.6962890625,
      "loss": 0.6969,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012744712643325329,
      "rewards/margins": -0.006842900533229113,
      "rewards/rejected": 0.019587615504860878,
      "step": 1514
    },
    {
      "epoch": 0.3984941393085535,
      "grad_norm": 612.0,
      "learning_rate": 3.342105263157894e-07,
      "logits/chosen": -0.6547446250915527,
      "logits/rejected": -0.6549131870269775,
      "logps/chosen": -1082.5709228515625,
      "logps/rejected": -1025.3140869140625,
      "loss": 0.6841,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.006093836389482021,
      "rewards/margins": 0.018904924392700195,
      "rewards/rejected": -0.024998759850859642,
      "step": 1515
    },
    {
      "epoch": 0.39875717174374065,
      "grad_norm": 422.0,
      "learning_rate": 3.340643274853801e-07,
      "logits/chosen": -0.6566891670227051,
      "logits/rejected": -0.6639472842216492,
      "logps/chosen": -977.0514526367188,
      "logps/rejected": -708.7017822265625,
      "loss": 0.6939,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -8.249212987720966e-05,
      "rewards/margins": -0.0007456782041117549,
      "rewards/rejected": 0.0006631857249885798,
      "step": 1516
    },
    {
      "epoch": 0.3990202041789278,
      "grad_norm": 568.0,
      "learning_rate": 3.3391812865497077e-07,
      "logits/chosen": -0.6532421112060547,
      "logits/rejected": -0.6403265595436096,
      "logps/chosen": -1277.640380859375,
      "logps/rejected": -965.0746459960938,
      "loss": 0.6982,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01283035334199667,
      "rewards/margins": -0.009192943572998047,
      "rewards/rejected": -0.003637409768998623,
      "step": 1517
    },
    {
      "epoch": 0.39928323661411497,
      "grad_norm": 484.0,
      "learning_rate": 3.337719298245614e-07,
      "logits/chosen": -0.6673189401626587,
      "logits/rejected": -0.672670841217041,
      "logps/chosen": -1020.0087280273438,
      "logps/rejected": -916.3733520507812,
      "loss": 0.6928,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0001767151989042759,
      "rewards/margins": 0.0011682980693876743,
      "rewards/rejected": -0.0009915824048221111,
      "step": 1518
    },
    {
      "epoch": 0.3995462690493021,
      "grad_norm": 644.0,
      "learning_rate": 3.33625730994152e-07,
      "logits/chosen": -0.6469868421554565,
      "logits/rejected": -0.6377601623535156,
      "logps/chosen": -1368.2535400390625,
      "logps/rejected": -884.025390625,
      "loss": 0.6836,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02687530592083931,
      "rewards/margins": 0.01999502256512642,
      "rewards/rejected": 0.006880283821374178,
      "step": 1519
    },
    {
      "epoch": 0.3998093014844893,
      "grad_norm": 588.0,
      "learning_rate": 3.334795321637427e-07,
      "logits/chosen": -0.6659542918205261,
      "logits/rejected": -0.6497420072555542,
      "logps/chosen": -1712.700927734375,
      "logps/rejected": -1102.017578125,
      "loss": 0.6921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.019505690783262253,
      "rewards/margins": 0.0026980391703546047,
      "rewards/rejected": 0.01680765114724636,
      "step": 1520
    },
    {
      "epoch": 0.40007233391967645,
      "grad_norm": 624.0,
      "learning_rate": 3.333333333333333e-07,
      "logits/chosen": -0.6562110781669617,
      "logits/rejected": -0.6576522588729858,
      "logps/chosen": -1199.199951171875,
      "logps/rejected": -933.720458984375,
      "loss": 0.6891,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.007318019401282072,
      "rewards/margins": 0.009121131151914597,
      "rewards/rejected": -0.0018031112849712372,
      "step": 1521
    },
    {
      "epoch": 0.4003353663548636,
      "grad_norm": 664.0,
      "learning_rate": 3.3318713450292397e-07,
      "logits/chosen": -0.650615930557251,
      "logits/rejected": -0.6505066156387329,
      "logps/chosen": -1528.7200927734375,
      "logps/rejected": -1276.4410400390625,
      "loss": 0.6969,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0068517690524458885,
      "rewards/margins": -0.006779288873076439,
      "rewards/rejected": -7.247878238558769e-05,
      "step": 1522
    },
    {
      "epoch": 0.4005983987900508,
      "grad_norm": 460.0,
      "learning_rate": 3.3304093567251465e-07,
      "logits/chosen": -0.6561241149902344,
      "logits/rejected": -0.6675561666488647,
      "logps/chosen": -1108.31640625,
      "logps/rejected": -703.6743774414062,
      "loss": 0.6876,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.016978265717625618,
      "rewards/margins": 0.011434650979936123,
      "rewards/rejected": 0.005543614272028208,
      "step": 1523
    },
    {
      "epoch": 0.400861431225238,
      "grad_norm": 572.0,
      "learning_rate": 3.328947368421052e-07,
      "logits/chosen": -0.6381561756134033,
      "logits/rejected": -0.6397653818130493,
      "logps/chosen": -1494.5162353515625,
      "logps/rejected": -1350.0504150390625,
      "loss": 0.7104,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01766795851290226,
      "rewards/margins": -0.03298225253820419,
      "rewards/rejected": 0.015314294025301933,
      "step": 1524
    },
    {
      "epoch": 0.40112446366042515,
      "grad_norm": 576.0,
      "learning_rate": 3.327485380116959e-07,
      "logits/chosen": -0.6538198590278625,
      "logits/rejected": -0.6576748490333557,
      "logps/chosen": -1121.90087890625,
      "logps/rejected": -884.2816772460938,
      "loss": 0.6818,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01815796084702015,
      "rewards/margins": 0.024036074057221413,
      "rewards/rejected": -0.005878114607185125,
      "step": 1525
    },
    {
      "epoch": 0.4013874960956123,
      "grad_norm": 430.0,
      "learning_rate": 3.3260233918128656e-07,
      "logits/chosen": -0.6450227499008179,
      "logits/rejected": -0.6455339193344116,
      "logps/chosen": -1029.613525390625,
      "logps/rejected": -762.3167724609375,
      "loss": 0.6995,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0021424293518066406,
      "rewards/margins": -0.011724662967026234,
      "rewards/rejected": 0.009582233615219593,
      "step": 1526
    },
    {
      "epoch": 0.40165052853079947,
      "grad_norm": 516.0,
      "learning_rate": 3.324561403508772e-07,
      "logits/chosen": -0.6511154174804688,
      "logits/rejected": -0.6667985320091248,
      "logps/chosen": -1216.9202880859375,
      "logps/rejected": -1135.3192138671875,
      "loss": 0.6977,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0009268763242289424,
      "rewards/margins": -0.008623981848359108,
      "rewards/rejected": 0.007697105873376131,
      "step": 1527
    },
    {
      "epoch": 0.4019135609659866,
      "grad_norm": 548.0,
      "learning_rate": 3.323099415204678e-07,
      "logits/chosen": -0.63193678855896,
      "logits/rejected": -0.639243483543396,
      "logps/chosen": -1302.7835693359375,
      "logps/rejected": -892.7556762695312,
      "loss": 0.6935,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.014818527735769749,
      "rewards/margins": 0.00021071359515190125,
      "rewards/rejected": 0.014607812277972698,
      "step": 1528
    },
    {
      "epoch": 0.4021765934011738,
      "grad_norm": 456.0,
      "learning_rate": 3.3216374269005847e-07,
      "logits/chosen": -0.6601677536964417,
      "logits/rejected": -0.6714317798614502,
      "logps/chosen": -868.87646484375,
      "logps/rejected": -617.584716796875,
      "loss": 0.6926,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0038336762227118015,
      "rewards/margins": 0.0015609734691679478,
      "rewards/rejected": -0.005394649226218462,
      "step": 1529
    },
    {
      "epoch": 0.40243962583636095,
      "grad_norm": 636.0,
      "learning_rate": 3.320175438596491e-07,
      "logits/chosen": -0.6658143401145935,
      "logits/rejected": -0.6668252348899841,
      "logps/chosen": -1461.96728515625,
      "logps/rejected": -1191.21435546875,
      "loss": 0.6977,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005499553866684437,
      "rewards/margins": -0.008335113525390625,
      "rewards/rejected": 0.013834666460752487,
      "step": 1530
    },
    {
      "epoch": 0.4027026582715481,
      "grad_norm": 556.0,
      "learning_rate": 3.3187134502923976e-07,
      "logits/chosen": -0.6529817581176758,
      "logits/rejected": -0.6565719246864319,
      "logps/chosen": -1262.17822265625,
      "logps/rejected": -659.3026123046875,
      "loss": 0.6978,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01281432993710041,
      "rewards/margins": -0.008765507489442825,
      "rewards/rejected": -0.004048823844641447,
      "step": 1531
    },
    {
      "epoch": 0.40296569070673527,
      "grad_norm": 648.0,
      "learning_rate": 3.317251461988304e-07,
      "logits/chosen": -0.6599384546279907,
      "logits/rejected": -0.6650593876838684,
      "logps/chosen": -1761.837158203125,
      "logps/rejected": -1549.657958984375,
      "loss": 0.6924,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0024362567346543074,
      "rewards/margins": 0.0030335900373756886,
      "rewards/rejected": -0.0005973349325358868,
      "step": 1532
    },
    {
      "epoch": 0.40322872314192243,
      "grad_norm": 482.0,
      "learning_rate": 3.31578947368421e-07,
      "logits/chosen": -0.6316864490509033,
      "logits/rejected": -0.6383981108665466,
      "logps/chosen": -1106.0947265625,
      "logps/rejected": -758.5191650390625,
      "loss": 0.7007,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.017466355115175247,
      "rewards/margins": -0.01337270624935627,
      "rewards/rejected": -0.0040936460718512535,
      "step": 1533
    },
    {
      "epoch": 0.4034917555771096,
      "grad_norm": 516.0,
      "learning_rate": 3.314327485380117e-07,
      "logits/chosen": -0.6545008420944214,
      "logits/rejected": -0.6578570604324341,
      "logps/chosen": -826.8756103515625,
      "logps/rejected": -896.7435913085938,
      "loss": 0.701,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0011490820907056332,
      "rewards/margins": -0.014548873528838158,
      "rewards/rejected": 0.013399792835116386,
      "step": 1534
    },
    {
      "epoch": 0.40375478801229675,
      "grad_norm": 564.0,
      "learning_rate": 3.3128654970760235e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6740866303443909,
      "logps/chosen": -1038.9056396484375,
      "logps/rejected": -969.618408203125,
      "loss": 0.705,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.006413841620087624,
      "rewards/margins": -0.02306227758526802,
      "rewards/rejected": 0.016648434102535248,
      "step": 1535
    },
    {
      "epoch": 0.4040178204474839,
      "grad_norm": 498.0,
      "learning_rate": 3.311403508771929e-07,
      "logits/chosen": -0.6544201374053955,
      "logits/rejected": -0.6609948873519897,
      "logps/chosen": -1121.9012451171875,
      "logps/rejected": -985.64697265625,
      "loss": 0.6884,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0016197208315134048,
      "rewards/margins": 0.009931756183505058,
      "rewards/rejected": -0.008312035351991653,
      "step": 1536
    },
    {
      "epoch": 0.40428085288267107,
      "grad_norm": 580.0,
      "learning_rate": 3.309941520467836e-07,
      "logits/chosen": -0.6507564187049866,
      "logits/rejected": -0.6488884091377258,
      "logps/chosen": -1037.58203125,
      "logps/rejected": -979.2318725585938,
      "loss": 0.6933,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.003007794264703989,
      "rewards/margins": 0.0009284005500376225,
      "rewards/rejected": -0.003936195746064186,
      "step": 1537
    },
    {
      "epoch": 0.40454388531785823,
      "grad_norm": 732.0,
      "learning_rate": 3.3084795321637426e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.628944456577301,
      "logps/chosen": -944.7982788085938,
      "logps/rejected": -1120.7664794921875,
      "loss": 0.684,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.006337642669677734,
      "rewards/margins": 0.020422838628292084,
      "rewards/rejected": -0.026760483160614967,
      "step": 1538
    },
    {
      "epoch": 0.40480691775304545,
      "grad_norm": 716.0,
      "learning_rate": 3.307017543859649e-07,
      "logits/chosen": -0.6620771288871765,
      "logits/rejected": -0.655751645565033,
      "logps/chosen": -1541.1849365234375,
      "logps/rejected": -1293.3057861328125,
      "loss": 0.7082,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.006550788879394531,
      "rewards/margins": -0.028837967664003372,
      "rewards/rejected": 0.02228717878460884,
      "step": 1539
    },
    {
      "epoch": 0.4050699501882326,
      "grad_norm": 556.0,
      "learning_rate": 3.3055555555555556e-07,
      "logits/chosen": -0.6710875630378723,
      "logits/rejected": -0.668194055557251,
      "logps/chosen": -1239.3768310546875,
      "logps/rejected": -942.8484497070312,
      "loss": 0.6899,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.007501410320401192,
      "rewards/margins": 0.007709789089858532,
      "rewards/rejected": -0.000208376906812191,
      "step": 1540
    },
    {
      "epoch": 0.40533298262341977,
      "grad_norm": 524.0,
      "learning_rate": 3.304093567251462e-07,
      "logits/chosen": -0.6549161672592163,
      "logits/rejected": -0.6539191007614136,
      "logps/chosen": -1089.6519775390625,
      "logps/rejected": -970.8359375,
      "loss": 0.689,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010476968251168728,
      "rewards/margins": 0.00983419455587864,
      "rewards/rejected": 0.0006427764892578125,
      "step": 1541
    },
    {
      "epoch": 0.40559601505860693,
      "grad_norm": 380.0,
      "learning_rate": 3.3026315789473685e-07,
      "logits/chosen": -0.6556523442268372,
      "logits/rejected": -0.6665278673171997,
      "logps/chosen": -884.3877563476562,
      "logps/rejected": -768.8030395507812,
      "loss": 0.6968,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009291457012295723,
      "rewards/margins": -0.005954359192401171,
      "rewards/rejected": -0.003337096655741334,
      "step": 1542
    },
    {
      "epoch": 0.4058590474937941,
      "grad_norm": 520.0,
      "learning_rate": 3.3011695906432747e-07,
      "logits/chosen": -0.6687331199645996,
      "logits/rejected": -0.6714181900024414,
      "logps/chosen": -1120.8509521484375,
      "logps/rejected": -864.2843017578125,
      "loss": 0.7005,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.005133725702762604,
      "rewards/margins": -0.014296055771410465,
      "rewards/rejected": 0.009162330999970436,
      "step": 1543
    },
    {
      "epoch": 0.40612207992898125,
      "grad_norm": 648.0,
      "learning_rate": 3.2997076023391814e-07,
      "logits/chosen": -0.6529435515403748,
      "logits/rejected": -0.6597723960876465,
      "logps/chosen": -1507.823974609375,
      "logps/rejected": -1376.8961181640625,
      "loss": 0.6843,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.012092208489775658,
      "rewards/margins": 0.019042015075683594,
      "rewards/rejected": -0.006949806120246649,
      "step": 1544
    },
    {
      "epoch": 0.4063851123641684,
      "grad_norm": 528.0,
      "learning_rate": 3.2982456140350876e-07,
      "logits/chosen": -0.6483952403068542,
      "logits/rejected": -0.6448502540588379,
      "logps/chosen": -1122.2340087890625,
      "logps/rejected": -965.9984741210938,
      "loss": 0.6841,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.007570171728730202,
      "rewards/margins": 0.019161226227879524,
      "rewards/rejected": -0.011591054499149323,
      "step": 1545
    },
    {
      "epoch": 0.40664814479935557,
      "grad_norm": 624.0,
      "learning_rate": 3.296783625730994e-07,
      "logits/chosen": -0.6226131916046143,
      "logits/rejected": -0.6258773803710938,
      "logps/chosen": -1164.7611083984375,
      "logps/rejected": -898.5610961914062,
      "loss": 0.6939,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.006247997283935547,
      "rewards/margins": -0.0007348540239036083,
      "rewards/rejected": -0.0055131432600319386,
      "step": 1546
    },
    {
      "epoch": 0.40691117723454273,
      "grad_norm": 584.0,
      "learning_rate": 3.2953216374269005e-07,
      "logits/chosen": -0.6317582726478577,
      "logits/rejected": -0.6431194543838501,
      "logps/chosen": -1183.567626953125,
      "logps/rejected": -955.8480224609375,
      "loss": 0.6958,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.010047626681625843,
      "rewards/margins": -0.0049323090352118015,
      "rewards/rejected": -0.005115318577736616,
      "step": 1547
    },
    {
      "epoch": 0.4071742096697299,
      "grad_norm": 512.0,
      "learning_rate": 3.293859649122807e-07,
      "logits/chosen": -0.6451389789581299,
      "logits/rejected": NaN,
      "logps/chosen": -1146.943603515625,
      "logps/rejected": -747.376220703125,
      "loss": 0.6924,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00450401334092021,
      "rewards/margins": 0.0019799722358584404,
      "rewards/rejected": 0.0025240418035537004,
      "step": 1548
    },
    {
      "epoch": 0.40743724210491705,
      "grad_norm": 506.0,
      "learning_rate": 3.292397660818713e-07,
      "logits/chosen": -0.6714915633201599,
      "logits/rejected": -0.6725939512252808,
      "logps/chosen": -1404.8599853515625,
      "logps/rejected": -1102.703125,
      "loss": 0.6892,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.022625543177127838,
      "rewards/margins": 0.00874862726777792,
      "rewards/rejected": 0.013876914978027344,
      "step": 1549
    },
    {
      "epoch": 0.4077002745401042,
      "grad_norm": 540.0,
      "learning_rate": 3.2909356725146197e-07,
      "logits/chosen": -0.6548647284507751,
      "logits/rejected": -0.664551854133606,
      "logps/chosen": -839.1068115234375,
      "logps/rejected": -669.0980224609375,
      "loss": 0.6986,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.011354446411132812,
      "rewards/margins": -0.009691334329545498,
      "rewards/rejected": 0.021045779809355736,
      "step": 1550
    },
    {
      "epoch": 0.4079633069752914,
      "grad_norm": 568.0,
      "learning_rate": 3.2894736842105264e-07,
      "logits/chosen": -0.650608479976654,
      "logits/rejected": -0.6517425179481506,
      "logps/chosen": -1200.7393798828125,
      "logps/rejected": -1079.760009765625,
      "loss": 0.6972,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00032272329553961754,
      "rewards/margins": -0.006825924385339022,
      "rewards/rejected": 0.006503200624138117,
      "step": 1551
    },
    {
      "epoch": 0.40822633941047853,
      "grad_norm": 552.0,
      "learning_rate": 3.2880116959064326e-07,
      "logits/chosen": -0.6575703620910645,
      "logits/rejected": -0.6520801782608032,
      "logps/chosen": -1361.781982421875,
      "logps/rejected": -1215.14208984375,
      "loss": 0.7058,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.019904330372810364,
      "rewards/margins": -0.024605466052889824,
      "rewards/rejected": 0.004701137542724609,
      "step": 1552
    },
    {
      "epoch": 0.4084893718456657,
      "grad_norm": 536.0,
      "learning_rate": 3.2865497076023393e-07,
      "logits/chosen": -0.6509449481964111,
      "logits/rejected": -0.6539117693901062,
      "logps/chosen": -787.6171875,
      "logps/rejected": -767.5914306640625,
      "loss": 0.6949,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0029974940698593855,
      "rewards/margins": -0.003168488619849086,
      "rewards/rejected": 0.000170994084328413,
      "step": 1553
    },
    {
      "epoch": 0.4087524042808529,
      "grad_norm": 544.0,
      "learning_rate": 3.2850877192982455e-07,
      "logits/chosen": -0.6388904452323914,
      "logits/rejected": -0.6381245851516724,
      "logps/chosen": -1162.075927734375,
      "logps/rejected": -1102.972900390625,
      "loss": 0.6947,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0026567457243800163,
      "rewards/margins": -0.00265846261754632,
      "rewards/rejected": 1.7165439203381538e-06,
      "step": 1554
    },
    {
      "epoch": 0.40901543671604007,
      "grad_norm": 564.0,
      "learning_rate": 3.2836257309941517e-07,
      "logits/chosen": -0.6479712724685669,
      "logits/rejected": -0.6426879167556763,
      "logps/chosen": -1015.4017944335938,
      "logps/rejected": -1004.4960327148438,
      "loss": 0.6992,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.007470321841537952,
      "rewards/margins": -0.011592864990234375,
      "rewards/rejected": 0.004122543148696423,
      "step": 1555
    },
    {
      "epoch": 0.40927846915122723,
      "grad_norm": 496.0,
      "learning_rate": 3.2821637426900584e-07,
      "logits/chosen": -0.6350692510604858,
      "logits/rejected": -0.6460055112838745,
      "logps/chosen": -1080.567626953125,
      "logps/rejected": -820.9005737304688,
      "loss": 0.6871,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.00218353234231472,
      "rewards/margins": 0.012725830078125,
      "rewards/rejected": -0.01490936428308487,
      "step": 1556
    },
    {
      "epoch": 0.4095415015864144,
      "grad_norm": 444.0,
      "learning_rate": 3.280701754385965e-07,
      "logits/chosen": -0.6553382873535156,
      "logits/rejected": -0.6660555601119995,
      "logps/chosen": -752.6923828125,
      "logps/rejected": -637.0059204101562,
      "loss": 0.7037,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.014263294637203217,
      "rewards/margins": -0.019971372559666634,
      "rewards/rejected": 0.005708075128495693,
      "step": 1557
    },
    {
      "epoch": 0.40980453402160155,
      "grad_norm": 636.0,
      "learning_rate": 3.279239766081871e-07,
      "logits/chosen": -0.6308872699737549,
      "logits/rejected": -0.6321490406990051,
      "logps/chosen": -1672.497314453125,
      "logps/rejected": -1382.98388671875,
      "loss": 0.6966,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0021560662426054478,
      "rewards/margins": -0.005368232261389494,
      "rewards/rejected": 0.007524300832301378,
      "step": 1558
    },
    {
      "epoch": 0.4100675664567887,
      "grad_norm": 584.0,
      "learning_rate": 3.2777777777777776e-07,
      "logits/chosen": -0.648322343826294,
      "logits/rejected": -0.6447035074234009,
      "logps/chosen": -1153.0360107421875,
      "logps/rejected": -1047.24462890625,
      "loss": 0.6921,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.017163755372166634,
      "rewards/margins": 0.0026827803812921047,
      "rewards/rejected": -0.019846534356474876,
      "step": 1559
    },
    {
      "epoch": 0.4103305988919759,
      "grad_norm": 556.0,
      "learning_rate": 3.2763157894736843e-07,
      "logits/chosen": -0.6600098609924316,
      "logits/rejected": -0.658012330532074,
      "logps/chosen": -995.2379150390625,
      "logps/rejected": -777.678955078125,
      "loss": 0.6949,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 2.9660062864422798e-05,
      "rewards/margins": -0.002252101432532072,
      "rewards/rejected": 0.0022817617282271385,
      "step": 1560
    },
    {
      "epoch": 0.41059363132716303,
      "grad_norm": 458.0,
      "learning_rate": 3.2748538011695905e-07,
      "logits/chosen": -0.6708433032035828,
      "logits/rejected": -0.6665191650390625,
      "logps/chosen": -1015.3009033203125,
      "logps/rejected": -858.5410766601562,
      "loss": 0.693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0006964672356843948,
      "rewards/margins": 0.0009209145791828632,
      "rewards/rejected": -0.0002244468778371811,
      "step": 1561
    },
    {
      "epoch": 0.4108566637623502,
      "grad_norm": 536.0,
      "learning_rate": 3.2733918128654967e-07,
      "logits/chosen": -0.6548566222190857,
      "logits/rejected": -0.6523477435112,
      "logps/chosen": -1323.6669921875,
      "logps/rejected": -678.2615356445312,
      "loss": 0.6929,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.010114860720932484,
      "rewards/margins": 0.0013076779432594776,
      "rewards/rejected": 0.008807183243334293,
      "step": 1562
    },
    {
      "epoch": 0.41111969619753735,
      "grad_norm": 572.0,
      "learning_rate": 3.2719298245614034e-07,
      "logits/chosen": -0.6500579118728638,
      "logits/rejected": -0.6489821672439575,
      "logps/chosen": -1288.6705322265625,
      "logps/rejected": -900.2091064453125,
      "loss": 0.6917,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008301163092255592,
      "rewards/margins": 0.00399627722799778,
      "rewards/rejected": 0.0043048858642578125,
      "step": 1563
    },
    {
      "epoch": 0.4113827286327245,
      "grad_norm": 494.0,
      "learning_rate": 3.2704678362573096e-07,
      "logits/chosen": -0.648944079875946,
      "logits/rejected": -0.6501427888870239,
      "logps/chosen": -1284.3671875,
      "logps/rejected": -1177.6168212890625,
      "loss": 0.6808,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008875846862792969,
      "rewards/margins": 0.02575216442346573,
      "rewards/rejected": -0.01687631569802761,
      "step": 1564
    },
    {
      "epoch": 0.4116457610679117,
      "grad_norm": 604.0,
      "learning_rate": 3.2690058479532164e-07,
      "logits/chosen": -0.6551206707954407,
      "logits/rejected": -0.6403259038925171,
      "logps/chosen": -1418.3494873046875,
      "logps/rejected": -999.5538330078125,
      "loss": 0.6857,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.011207390576601028,
      "rewards/margins": 0.015679743140935898,
      "rewards/rejected": -0.026887129992246628,
      "step": 1565
    },
    {
      "epoch": 0.41190879350309884,
      "grad_norm": 480.0,
      "learning_rate": 3.267543859649123e-07,
      "logits/chosen": -0.6412766575813293,
      "logits/rejected": -0.6449406743049622,
      "logps/chosen": -1069.7490234375,
      "logps/rejected": -1175.0511474609375,
      "loss": 0.6882,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0024391161277890205,
      "rewards/margins": 0.010983085259795189,
      "rewards/rejected": -0.008543968200683594,
      "step": 1566
    },
    {
      "epoch": 0.412171825938286,
      "grad_norm": 516.0,
      "learning_rate": 3.266081871345029e-07,
      "logits/chosen": -0.6711656451225281,
      "logits/rejected": -0.6729022264480591,
      "logps/chosen": -704.9417114257812,
      "logps/rejected": -797.1939697265625,
      "loss": 0.7039,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.010693836957216263,
      "rewards/margins": -0.021254397928714752,
      "rewards/rejected": 0.010560560040175915,
      "step": 1567
    },
    {
      "epoch": 0.41243485837347316,
      "grad_norm": 532.0,
      "learning_rate": 3.2646198830409355e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6509456634521484,
      "logps/chosen": -1552.159423828125,
      "logps/rejected": -1497.0689697265625,
      "loss": 0.6961,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.012696455232799053,
      "rewards/margins": -0.005259513854980469,
      "rewards/rejected": -0.0074369413778185844,
      "step": 1568
    },
    {
      "epoch": 0.4126978908086603,
      "grad_norm": 502.0,
      "learning_rate": 3.263157894736842e-07,
      "logits/chosen": -0.6621658802032471,
      "logits/rejected": -0.6671509742736816,
      "logps/chosen": -913.9652099609375,
      "logps/rejected": -675.2628173828125,
      "loss": 0.7015,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.004070615861564875,
      "rewards/margins": -0.016263816505670547,
      "rewards/rejected": 0.020334433764219284,
      "step": 1569
    },
    {
      "epoch": 0.41296092324384753,
      "grad_norm": 576.0,
      "learning_rate": 3.2616959064327484e-07,
      "logits/chosen": -0.6202718019485474,
      "logits/rejected": -0.6179651618003845,
      "logps/chosen": -895.74072265625,
      "logps/rejected": -745.089599609375,
      "loss": 0.6911,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.002281379187479615,
      "rewards/margins": 0.0044334400445222855,
      "rewards/rejected": -0.006714820396155119,
      "step": 1570
    },
    {
      "epoch": 0.4132239556790347,
      "grad_norm": 492.0,
      "learning_rate": 3.2602339181286546e-07,
      "logits/chosen": -0.6578431725502014,
      "logits/rejected": -0.659652590751648,
      "logps/chosen": -1038.9471435546875,
      "logps/rejected": -559.53857421875,
      "loss": 0.6973,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0003107076045125723,
      "rewards/margins": -0.007630969397723675,
      "rewards/rejected": 0.0073202610947191715,
      "step": 1571
    },
    {
      "epoch": 0.41348698811422185,
      "grad_norm": 430.0,
      "learning_rate": 3.2587719298245613e-07,
      "logits/chosen": -0.6317283511161804,
      "logits/rejected": -0.6282486915588379,
      "logps/chosen": -680.7391967773438,
      "logps/rejected": -564.2666625976562,
      "loss": 0.6829,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0011038780212402344,
      "rewards/margins": 0.020889757201075554,
      "rewards/rejected": -0.01978588104248047,
      "step": 1572
    },
    {
      "epoch": 0.413750020549409,
      "grad_norm": 506.0,
      "learning_rate": 3.2573099415204675e-07,
      "logits/chosen": -0.6428537964820862,
      "logits/rejected": -0.6468008756637573,
      "logps/chosen": -1053.2525634765625,
      "logps/rejected": -923.077880859375,
      "loss": 0.701,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.009312152862548828,
      "rewards/margins": -0.014781191013753414,
      "rewards/rejected": 0.005469036288559437,
      "step": 1573
    },
    {
      "epoch": 0.4140130529845962,
      "grad_norm": 488.0,
      "learning_rate": 3.255847953216374e-07,
      "logits/chosen": -0.6338471174240112,
      "logits/rejected": -0.6636881232261658,
      "logps/chosen": -624.8170166015625,
      "logps/rejected": -640.6160888671875,
      "loss": 0.6962,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.006654835306107998,
      "rewards/margins": -0.004780197516083717,
      "rewards/rejected": -0.0018746377900242805,
      "step": 1574
    },
    {
      "epoch": 0.41427608541978334,
      "grad_norm": 502.0,
      "learning_rate": 3.2543859649122805e-07,
      "logits/chosen": -0.6457715034484863,
      "logits/rejected": -0.6425861716270447,
      "logps/chosen": -1442.8704833984375,
      "logps/rejected": -1156.0848388671875,
      "loss": 0.6846,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02220907248556614,
      "rewards/margins": 0.01735086552798748,
      "rewards/rejected": 0.004858208820223808,
      "step": 1575
    },
    {
      "epoch": 0.4145391178549705,
      "grad_norm": 628.0,
      "learning_rate": 3.2529239766081867e-07,
      "logits/chosen": -0.6456426382064819,
      "logits/rejected": -0.6579955220222473,
      "logps/chosen": -1265.5556640625,
      "logps/rejected": -1015.8253173828125,
      "loss": 0.6943,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0050948141142725945,
      "rewards/margins": -0.0018743515247479081,
      "rewards/rejected": -0.003220462705940008,
      "step": 1576
    },
    {
      "epoch": 0.41480215029015766,
      "grad_norm": 410.0,
      "learning_rate": 3.2514619883040934e-07,
      "logits/chosen": -0.6307886838912964,
      "logits/rejected": -0.6342124342918396,
      "logps/chosen": -726.894287109375,
      "logps/rejected": -674.4141235351562,
      "loss": 0.6842,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.011048030108213425,
      "rewards/margins": 0.018221760168671608,
      "rewards/rejected": -0.007173729129135609,
      "step": 1577
    },
    {
      "epoch": 0.4150651827253448,
      "grad_norm": 504.0,
      "learning_rate": 3.25e-07,
      "logits/chosen": -0.6597302556037903,
      "logits/rejected": -0.6540552377700806,
      "logps/chosen": -1245.7855224609375,
      "logps/rejected": -1085.58642578125,
      "loss": 0.6895,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.009587050415575504,
      "rewards/margins": 0.008141374215483665,
      "rewards/rejected": -0.017728423699736595,
      "step": 1578
    },
    {
      "epoch": 0.415328215160532,
      "grad_norm": 612.0,
      "learning_rate": 3.248538011695906e-07,
      "logits/chosen": -0.6455063819885254,
      "logits/rejected": -0.6450760364532471,
      "logps/chosen": -1255.1263427734375,
      "logps/rejected": -931.7348022460938,
      "loss": 0.7077,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005420113913714886,
      "rewards/margins": -0.027613259851932526,
      "rewards/rejected": 0.022193146869540215,
      "step": 1579
    },
    {
      "epoch": 0.41559124759571914,
      "grad_norm": 628.0,
      "learning_rate": 3.2470760233918125e-07,
      "logits/chosen": -0.6538391709327698,
      "logits/rejected": -0.6459947824478149,
      "logps/chosen": -1210.0484619140625,
      "logps/rejected": -1107.2410888671875,
      "loss": 0.6929,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011827279813587666,
      "rewards/margins": 0.0016485208179801702,
      "rewards/rejected": -0.013475799933075905,
      "step": 1580
    },
    {
      "epoch": 0.4158542800309063,
      "grad_norm": 466.0,
      "learning_rate": 3.245614035087719e-07,
      "logits/chosen": -0.6716092824935913,
      "logits/rejected": -0.6751229763031006,
      "logps/chosen": -1232.0621337890625,
      "logps/rejected": -849.654541015625,
      "loss": 0.6918,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.002931309398263693,
      "rewards/margins": 0.0029965403955429792,
      "rewards/rejected": -0.005927849095314741,
      "step": 1581
    },
    {
      "epoch": 0.41611731246609346,
      "grad_norm": 604.0,
      "learning_rate": 3.2441520467836254e-07,
      "logits/chosen": -0.6860901117324829,
      "logits/rejected": -0.6761910915374756,
      "logps/chosen": -1470.23291015625,
      "logps/rejected": -1006.7007446289062,
      "loss": 0.699,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.00531392078846693,
      "rewards/margins": -0.011276865378022194,
      "rewards/rejected": 0.005962944123893976,
      "step": 1582
    },
    {
      "epoch": 0.4163803449012806,
      "grad_norm": 580.0,
      "learning_rate": 3.242690058479532e-07,
      "logits/chosen": -0.6552098393440247,
      "logits/rejected": -0.653523862361908,
      "logps/chosen": -1221.460693359375,
      "logps/rejected": -1036.552734375,
      "loss": 0.7001,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.015391351655125618,
      "rewards/margins": -0.012694456614553928,
      "rewards/rejected": -0.0026968957390636206,
      "step": 1583
    },
    {
      "epoch": 0.4166433773364678,
      "grad_norm": 676.0,
      "learning_rate": 3.2412280701754384e-07,
      "logits/chosen": -0.6550735235214233,
      "logits/rejected": -0.6546621322631836,
      "logps/chosen": -1086.650634765625,
      "logps/rejected": -648.9877319335938,
      "loss": 0.6991,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.01238412968814373,
      "rewards/margins": -0.011580515652894974,
      "rewards/rejected": -0.0008036143844947219,
      "step": 1584
    },
    {
      "epoch": 0.41690640977165494,
      "grad_norm": 540.0,
      "learning_rate": 3.239766081871345e-07,
      "logits/chosen": -0.6628119945526123,
      "logits/rejected": -0.6631358861923218,
      "logps/chosen": -981.0185546875,
      "logps/rejected": -893.7028198242188,
      "loss": 0.6847,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.015705585479736328,
      "rewards/margins": 0.01780247874557972,
      "rewards/rejected": -0.0020968918688595295,
      "step": 1585
    },
    {
      "epoch": 0.41716944220684216,
      "grad_norm": 500.0,
      "learning_rate": 3.2383040935672513e-07,
      "logits/chosen": -0.6737947463989258,
      "logits/rejected": -0.6732072830200195,
      "logps/chosen": -1415.2789306640625,
      "logps/rejected": -1110.448974609375,
      "loss": 0.6906,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.01176595687866211,
      "rewards/margins": 0.005771256983280182,
      "rewards/rejected": -0.017537211999297142,
      "step": 1586
    },
    {
      "epoch": 0.4174324746420293,
      "grad_norm": 560.0,
      "learning_rate": 3.236842105263158e-07,
      "logits/chosen": -0.6723660230636597,
      "logits/rejected": -0.6770533919334412,
      "logps/chosen": -1530.4737548828125,
      "logps/rejected": -1154.662841796875,
      "loss": 0.68,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01605205610394478,
      "rewards/margins": 0.027498628944158554,
      "rewards/rejected": -0.011446572840213776,
      "step": 1587
    },
    {
      "epoch": 0.4176955070772165,
      "grad_norm": 588.0,
      "learning_rate": 3.235380116959064e-07,
      "logits/chosen": -0.643976628780365,
      "logits/rejected": -0.6490916609764099,
      "logps/chosen": -1405.8955078125,
      "logps/rejected": -1169.0250244140625,
      "loss": 0.6884,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.008041763678193092,
      "rewards/margins": 0.010196303948760033,
      "rewards/rejected": -0.0021545407362282276,
      "step": 1588
    },
    {
      "epoch": 0.41795853951240364,
      "grad_norm": 510.0,
      "learning_rate": 3.2339181286549704e-07,
      "logits/chosen": -0.6240664720535278,
      "logits/rejected": -0.6112319827079773,
      "logps/chosen": -971.709228515625,
      "logps/rejected": -965.8057250976562,
      "loss": 0.7016,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02777567133307457,
      "rewards/margins": -0.01642436906695366,
      "rewards/rejected": -0.011351300403475761,
      "step": 1589
    },
    {
      "epoch": 0.4182215719475908,
      "grad_norm": 572.0,
      "learning_rate": 3.232456140350877e-07,
      "logits/chosen": -0.6536038517951965,
      "logits/rejected": -0.6555669903755188,
      "logps/chosen": -1437.3319091796875,
      "logps/rejected": -1514.423095703125,
      "loss": 0.6934,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.021340085193514824,
      "rewards/margins": 0.0005455974023789167,
      "rewards/rejected": 0.02079448476433754,
      "step": 1590
    },
    {
      "epoch": 0.41848460438277796,
      "grad_norm": 560.0,
      "learning_rate": 3.230994152046784e-07,
      "logits/chosen": -0.6415539383888245,
      "logits/rejected": -0.6570103168487549,
      "logps/chosen": -1464.4442138671875,
      "logps/rejected": -1069.04150390625,
      "loss": 0.6913,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.017736196517944336,
      "rewards/margins": 0.003942918963730335,
      "rewards/rejected": 0.013793278485536575,
      "step": 1591
    },
    {
      "epoch": 0.4187476368179651,
      "grad_norm": 608.0,
      "learning_rate": 3.2295321637426896e-07,
      "logits/chosen": -0.6469296813011169,
      "logits/rejected": -0.6480126976966858,
      "logps/chosen": -1500.2552490234375,
      "logps/rejected": -762.601806640625,
      "loss": 0.6871,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.001943302107974887,
      "rewards/margins": 0.012975024059414864,
      "rewards/rejected": -0.011031723581254482,
      "step": 1592
    },
    {
      "epoch": 0.4190106692531523,
      "grad_norm": 506.0,
      "learning_rate": 3.2280701754385963e-07,
      "logits/chosen": -0.653252363204956,
      "logits/rejected": -0.6537808775901794,
      "logps/chosen": -867.0054931640625,
      "logps/rejected": -815.5643310546875,
      "loss": 0.694,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.000597000471316278,
      "rewards/margins": -0.001020527444779873,
      "rewards/rejected": 0.0016175275668501854,
      "step": 1593
    },
    {
      "epoch": 0.41927370168833944,
      "grad_norm": 584.0,
      "learning_rate": 3.226608187134503e-07,
      "logits/chosen": -0.6742738485336304,
      "logits/rejected": -0.6709766387939453,
      "logps/chosen": -968.6350708007812,
      "logps/rejected": -820.7222900390625,
      "loss": 0.6944,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0019237520173192024,
      "rewards/margins": -0.0017363551305606961,
      "rewards/rejected": -0.0001873960718512535,
      "step": 1594
    },
    {
      "epoch": 0.4195367341235266,
      "grad_norm": 468.0,
      "learning_rate": 3.225146198830409e-07,
      "logits/chosen": -0.654341995716095,
      "logits/rejected": -0.6486029624938965,
      "logps/chosen": -1195.455322265625,
      "logps/rejected": -984.1376953125,
      "loss": 0.6786,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03887844458222389,
      "rewards/margins": 0.030130766332149506,
      "rewards/rejected": 0.008747673593461514,
      "step": 1595
    },
    {
      "epoch": 0.41979976655871376,
      "grad_norm": 536.0,
      "learning_rate": 3.223684210526316e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.632801353931427,
      "logps/chosen": -888.1112670898438,
      "logps/rejected": -936.7611083984375,
      "loss": 0.6913,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.011432650499045849,
      "rewards/margins": 0.004205035045742989,
      "rewards/rejected": 0.0072276112623512745,
      "step": 1596
    },
    {
      "epoch": 0.4200627989939009,
      "grad_norm": 600.0,
      "learning_rate": 3.222222222222222e-07,
      "logits/chosen": -0.6513303518295288,
      "logits/rejected": -0.6516244411468506,
      "logps/chosen": -1126.659423828125,
      "logps/rejected": -1094.8482666015625,
      "loss": 0.6954,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0013626104919239879,
      "rewards/margins": -0.004251670092344284,
      "rewards/rejected": 0.0028890613466501236,
      "step": 1597
    },
    {
      "epoch": 0.4203258314290881,
      "grad_norm": 540.0,
      "learning_rate": 3.2207602339181283e-07,
      "logits/chosen": -0.6492689251899719,
      "logits/rejected": -0.643698513507843,
      "logps/chosen": -980.3973388671875,
      "logps/rejected": -838.4694213867188,
      "loss": 0.6898,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.013225365430116653,
      "rewards/margins": 0.0074402811005711555,
      "rewards/rejected": -0.020665645599365234,
      "step": 1598
    },
    {
      "epoch": 0.42058886386427524,
      "grad_norm": 596.0,
      "learning_rate": 3.219298245614035e-07,
      "logits/chosen": -0.6367884278297424,
      "logits/rejected": -0.6464820504188538,
      "logps/chosen": -1282.6248779296875,
      "logps/rejected": -910.4740600585938,
      "loss": 0.6917,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.002184772863984108,
      "rewards/margins": 0.004249857272952795,
      "rewards/rejected": -0.006434631068259478,
      "step": 1599
    },
    {
      "epoch": 0.4208518962994624,
      "grad_norm": 568.0,
      "learning_rate": 3.217836257309942e-07,
      "logits/chosen": -0.6512848734855652,
      "logits/rejected": -0.6524915099143982,
      "logps/chosen": -1203.0081787109375,
      "logps/rejected": -957.721435546875,
      "loss": 0.6974,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.012894248589873314,
      "rewards/margins": -0.007166767958551645,
      "rewards/rejected": -0.0057274820283055305,
      "step": 1600
    },
    {
      "epoch": 0.4211149287346496,
      "grad_norm": 400.0,
      "learning_rate": 3.2163742690058475e-07,
      "logits/chosen": -0.6422144174575806,
      "logits/rejected": -0.6425589919090271,
      "logps/chosen": -565.5540771484375,
      "logps/rejected": -599.5645751953125,
      "loss": 0.7015,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0022763251326978207,
      "rewards/margins": -0.01611018367111683,
      "rewards/rejected": 0.018386509269475937,
      "step": 1601
    },
    {
      "epoch": 0.4213779611698368,
      "grad_norm": 576.0,
      "learning_rate": 3.214912280701754e-07,
      "logits/chosen": -0.6745222806930542,
      "logits/rejected": -0.6575595140457153,
      "logps/chosen": -1567.1815185546875,
      "logps/rejected": -1163.0882568359375,
      "loss": 0.6953,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006857491098344326,
      "rewards/margins": -0.003695676103234291,
      "rewards/rejected": 0.010553169064223766,
      "step": 1602
    },
    {
      "epoch": 0.42164099360502394,
      "grad_norm": 668.0,
      "learning_rate": 3.213450292397661e-07,
      "logits/chosen": -0.6689974665641785,
      "logits/rejected": -0.6796356439590454,
      "logps/chosen": -1379.7161865234375,
      "logps/rejected": -1055.96240234375,
      "loss": 0.6977,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.002711009234189987,
      "rewards/margins": -0.007835772819817066,
      "rewards/rejected": 0.01054677926003933,
      "step": 1603
    },
    {
      "epoch": 0.4219040260402111,
      "grad_norm": 452.0,
      "learning_rate": 3.211988304093567e-07,
      "logits/chosen": -0.6272083520889282,
      "logits/rejected": -0.6368023753166199,
      "logps/chosen": -1171.959716796875,
      "logps/rejected": -935.1819458007812,
      "loss": 0.6905,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.002434635302051902,
      "rewards/margins": 0.0066626062616705894,
      "rewards/rejected": -0.00909724272787571,
      "step": 1604
    },
    {
      "epoch": 0.42216705847539826,
      "grad_norm": 564.0,
      "learning_rate": 3.2105263157894733e-07,
      "logits/chosen": -0.6577216982841492,
      "logits/rejected": -0.6644573211669922,
      "logps/chosen": -1280.8619384765625,
      "logps/rejected": -1085.88427734375,
      "loss": 0.6922,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.013082219287753105,
      "rewards/margins": 0.0026158341206610203,
      "rewards/rejected": -0.015698052942752838,
      "step": 1605
    },
    {
      "epoch": 0.4224300909105854,
      "grad_norm": 540.0,
      "learning_rate": 3.20906432748538e-07,
      "logits/chosen": -0.6262326240539551,
      "logits/rejected": -0.6313226222991943,
      "logps/chosen": -1601.6693115234375,
      "logps/rejected": -1246.7288818359375,
      "loss": 0.7018,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.012104988098144531,
      "rewards/margins": -0.01692819595336914,
      "rewards/rejected": 0.029033184051513672,
      "step": 1606
    },
    {
      "epoch": 0.4226931233457726,
      "grad_norm": 504.0,
      "learning_rate": 3.207602339181286e-07,
      "logits/chosen": -0.6576041579246521,
      "logits/rejected": -0.6594152450561523,
      "logps/chosen": -1259.4617919921875,
      "logps/rejected": -923.5267333984375,
      "loss": 0.6827,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.018964290618896484,
      "rewards/margins": 0.02193431742489338,
      "rewards/rejected": -0.0029700282029807568,
      "step": 1607
    },
    {
      "epoch": 0.42295615578095974,
      "grad_norm": 516.0,
      "learning_rate": 3.206140350877193e-07,
      "logits/chosen": -0.6745821833610535,
      "logits/rejected": -0.6804152727127075,
      "logps/chosen": -904.2158203125,
      "logps/rejected": -740.3509521484375,
      "loss": 0.698,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.009869290515780449,
      "rewards/margins": -0.00886530801653862,
      "rewards/rejected": 0.01873459853231907,
      "step": 1608
    },
    {
      "epoch": 0.4232191882161469,
      "grad_norm": 576.0,
      "learning_rate": 3.204678362573099e-07,
      "logits/chosen": -0.6570254564285278,
      "logits/rejected": -0.6548641920089722,
      "logps/chosen": -988.8473510742188,
      "logps/rejected": -705.7557983398438,
      "loss": 0.6925,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.006217288784682751,
      "rewards/margins": 0.0016461852937936783,
      "rewards/rejected": -0.007863475009799004,
      "step": 1609
    },
    {
      "epoch": 0.42348222065133406,
      "grad_norm": 712.0,
      "learning_rate": 3.2032163742690054e-07,
      "logits/chosen": -0.6357124447822571,
      "logits/rejected": -0.6389849781990051,
      "logps/chosen": -1259.921630859375,
      "logps/rejected": -1199.419189453125,
      "loss": 0.6946,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.008217811584472656,
      "rewards/margins": -0.002273750025779009,
      "rewards/rejected": 0.010491561144590378,
      "step": 1610
    },
    {
      "epoch": 0.4237452530865212,
      "grad_norm": 528.0,
      "learning_rate": 3.201754385964912e-07,
      "logits/chosen": -0.6304892897605896,
      "logits/rejected": -0.6318364143371582,
      "logps/chosen": -1042.8697509765625,
      "logps/rejected": -872.8433227539062,
      "loss": 0.6988,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01826167106628418,
      "rewards/margins": -0.009997749701142311,
      "rewards/rejected": -0.008263921365141869,
      "step": 1611
    },
    {
      "epoch": 0.4240082855217084,
      "grad_norm": 532.0,
      "learning_rate": 3.200292397660819e-07,
      "logits/chosen": -0.6406459212303162,
      "logits/rejected": -0.6396692991256714,
      "logps/chosen": -1059.8387451171875,
      "logps/rejected": -910.06787109375,
      "loss": 0.6904,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.007208347320556641,
      "rewards/margins": 0.005856419447809458,
      "rewards/rejected": 0.0013519290369004011,
      "step": 1612
    },
    {
      "epoch": 0.42427131795689554,
      "grad_norm": 520.0,
      "learning_rate": 3.198830409356725e-07,
      "logits/chosen": -0.678979218006134,
      "logits/rejected": -0.6736536622047424,
      "logps/chosen": -1266.3040771484375,
      "logps/rejected": -1155.0081787109375,
      "loss": 0.7009,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.015428353101015091,
      "rewards/margins": -0.015098094940185547,
      "rewards/rejected": -0.00033025728771463037,
      "step": 1613
    },
    {
      "epoch": 0.4245343503920827,
      "grad_norm": 458.0,
      "learning_rate": 3.197368421052631e-07,
      "logits/chosen": -0.6451931595802307,
      "logits/rejected": -0.6443077325820923,
      "logps/chosen": -1052.48486328125,
      "logps/rejected": -831.7582397460938,
      "loss": 0.7046,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.005402088165283203,
      "rewards/margins": -0.022264480590820312,
      "rewards/rejected": 0.01686239056289196,
      "step": 1614
    },
    {
      "epoch": 0.42479738282726986,
      "grad_norm": 488.0,
      "learning_rate": 3.195906432748538e-07,
      "logits/chosen": -0.6494143009185791,
      "logits/rejected": -0.6558723449707031,
      "logps/chosen": -1262.076171875,
      "logps/rejected": -746.2010498046875,
      "loss": 0.6936,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.020429611206054688,
      "rewards/margins": 0.00021591153927147388,
      "rewards/rejected": 0.020213700830936432,
      "step": 1615
    },
    {
      "epoch": 0.425060415262457,
      "grad_norm": 392.0,
      "learning_rate": 3.194444444444444e-07,
      "logits/chosen": -0.6389672756195068,
      "logits/rejected": -0.6454582214355469,
      "logps/chosen": -643.4962158203125,
      "logps/rejected": -607.8225708007812,
      "loss": 0.6894,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01670236513018608,
      "rewards/margins": 0.008601474575698376,
      "rewards/rejected": 0.008100890554487705,
      "step": 1616
    },
    {
      "epoch": 0.42532344769764424,
      "grad_norm": 608.0,
      "learning_rate": 3.192982456140351e-07,
      "logits/chosen": -0.6511952877044678,
      "logits/rejected": -0.6510287523269653,
      "logps/chosen": -1337.1846923828125,
      "logps/rejected": -1125.4749755859375,
      "loss": 0.7129,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.030867960304021835,
      "rewards/margins": -0.0372861884534359,
      "rewards/rejected": 0.0064182281494140625,
      "step": 1617
    },
    {
      "epoch": 0.4255864801328314,
      "grad_norm": 664.0,
      "learning_rate": 3.191520467836257e-07,
      "logits/chosen": -0.6753067970275879,
      "logits/rejected": -0.6736682057380676,
      "logps/chosen": -1548.033935546875,
      "logps/rejected": -924.1786499023438,
      "loss": 0.6921,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0016434667631983757,
      "rewards/margins": 0.003002071287482977,
      "rewards/rejected": -0.00464553851634264,
      "step": 1618
    },
    {
      "epoch": 0.42584951256801856,
      "grad_norm": 624.0,
      "learning_rate": 3.1900584795321633e-07,
      "logits/chosen": -0.6773258447647095,
      "logits/rejected": -0.6793254613876343,
      "logps/chosen": -1268.6976318359375,
      "logps/rejected": -1101.5416259765625,
      "loss": 0.6942,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00879211351275444,
      "rewards/margins": -0.0012773035559803247,
      "rewards/rejected": -0.0075148120522499084,
      "step": 1619
    },
    {
      "epoch": 0.4261125450032057,
      "grad_norm": 490.0,
      "learning_rate": 3.18859649122807e-07,
      "logits/chosen": -0.6553754210472107,
      "logits/rejected": -0.6618182063102722,
      "logps/chosen": -1224.568115234375,
      "logps/rejected": -991.1240234375,
      "loss": 0.7029,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.00649919593706727,
      "rewards/margins": -0.01876554638147354,
      "rewards/rejected": 0.012266350910067558,
      "step": 1620
    },
    {
      "epoch": 0.4263755774383929,
      "grad_norm": 632.0,
      "learning_rate": 3.187134502923977e-07,
      "logits/chosen": -0.6419009566307068,
      "logits/rejected": -0.6485466957092285,
      "logps/chosen": -1186.484619140625,
      "logps/rejected": -975.9483642578125,
      "loss": 0.6991,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.025165939703583717,
      "rewards/margins": -0.011285973712801933,
      "rewards/rejected": -0.013879964128136635,
      "step": 1621
    },
    {
      "epoch": 0.42663860987358004,
      "grad_norm": 504.0,
      "learning_rate": 3.1856725146198824e-07,
      "logits/chosen": -0.6649930477142334,
      "logits/rejected": -0.6468183994293213,
      "logps/chosen": -1217.4359130859375,
      "logps/rejected": -600.69140625,
      "loss": 0.7025,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0074242595583200455,
      "rewards/margins": -0.018100213259458542,
      "rewards/rejected": 0.010675953701138496,
      "step": 1622
    },
    {
      "epoch": 0.4269016423087672,
      "grad_norm": 532.0,
      "learning_rate": 3.184210526315789e-07,
      "logits/chosen": -0.6527882218360901,
      "logits/rejected": -0.6562166213989258,
      "logps/chosen": -1222.7572021484375,
      "logps/rejected": -1215.0428466796875,
      "loss": 0.6953,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.015390397049486637,
      "rewards/margins": -0.003499604295939207,
      "rewards/rejected": 0.018890000879764557,
      "step": 1623
    },
    {
      "epoch": 0.42716467474395436,
      "grad_norm": 612.0,
      "learning_rate": 3.182748538011696e-07,
      "logits/chosen": -0.6332823038101196,
      "logits/rejected": -0.635166585445404,
      "logps/chosen": -1389.0421142578125,
      "logps/rejected": -1079.68017578125,
      "loss": 0.6759,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.022480010986328125,
      "rewards/margins": 0.036130812019109726,
      "rewards/rejected": -0.013650799170136452,
      "step": 1624
    },
    {
      "epoch": 0.4274277071791415,
      "grad_norm": 492.0,
      "learning_rate": 3.1812865497076026e-07,
      "logits/chosen": -0.6646883487701416,
      "logits/rejected": -0.6750717163085938,
      "logps/chosen": -879.973876953125,
      "logps/rejected": -716.9735717773438,
      "loss": 0.6859,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.003131389617919922,
      "rewards/margins": 0.015443944372236729,
      "rewards/rejected": -0.018575334921479225,
      "step": 1625
    },
    {
      "epoch": 0.4276907396143287,
      "grad_norm": 486.0,
      "learning_rate": 3.179824561403508e-07,
      "logits/chosen": -0.6713522672653198,
      "logits/rejected": -0.6813910007476807,
      "logps/chosen": -1032.5543212890625,
      "logps/rejected": -759.9085083007812,
      "loss": 0.6969,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.00857839547097683,
      "rewards/margins": -0.007093381602317095,
      "rewards/rejected": -0.0014850145671516657,
      "step": 1626
    },
    {
      "epoch": 0.42795377204951585,
      "grad_norm": 464.0,
      "learning_rate": 3.178362573099415e-07,
      "logits/chosen": -0.6522696018218994,
      "logits/rejected": -0.6459854245185852,
      "logps/chosen": -905.415283203125,
      "logps/rejected": -893.8826293945312,
      "loss": 0.6827,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.01661548763513565,
      "rewards/margins": 0.02163849025964737,
      "rewards/rejected": -0.005023002624511719,
      "step": 1627
    },
    {
      "epoch": 0.428216804484703,
      "grad_norm": 456.0,
      "learning_rate": 3.1769005847953217e-07,
      "logits/chosen": -0.6228148937225342,
      "logits/rejected": -0.6266052722930908,
      "logps/chosen": -1076.432373046875,
      "logps/rejected": -930.9612426757812,
      "loss": 0.6945,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.013086508959531784,
      "rewards/margins": -0.002351666335016489,
      "rewards/rejected": -0.010734844021499157,
      "step": 1628
    },
    {
      "epoch": 0.42847983691989017,
      "grad_norm": 696.0,
      "learning_rate": 3.175438596491228e-07,
      "logits/chosen": -0.5947161316871643,
      "logits/rejected": -0.600211501121521,
      "logps/chosen": -1174.380126953125,
      "logps/rejected": -913.4715576171875,
      "loss": 0.6885,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0058557502925395966,
      "rewards/margins": 0.009804057888686657,
      "rewards/rejected": -0.015659809112548828,
      "step": 1629
    },
    {
      "epoch": 0.4287428693550773,
      "grad_norm": 446.0,
      "learning_rate": 3.1739766081871346e-07,
      "logits/chosen": -0.6599218845367432,
      "logits/rejected": -0.6636722683906555,
      "logps/chosen": -811.1275024414062,
      "logps/rejected": -896.449462890625,
      "loss": 0.6955,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.012828066013753414,
      "rewards/margins": -0.004242515657097101,
      "rewards/rejected": -0.008585548959672451,
      "step": 1630
    },
    {
      "epoch": 0.4290059017902645,
      "grad_norm": 588.0,
      "learning_rate": 3.172514619883041e-07,
      "logits/chosen": -0.6349276900291443,
      "logits/rejected": -0.6375880241394043,
      "logps/chosen": -1375.783447265625,
      "logps/rejected": -1135.2935791015625,
      "loss": 0.6868,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03471317142248154,
      "rewards/margins": 0.013417244888842106,
      "rewards/rejected": 0.021295929327607155,
      "step": 1631
    },
    {
      "epoch": 0.4292689342254517,
      "grad_norm": 588.0,
      "learning_rate": 3.171052631578947e-07,
      "logits/chosen": -0.6729130148887634,
      "logits/rejected": -0.6652853488922119,
      "logps/chosen": -860.79931640625,
      "logps/rejected": -977.221923828125,
      "loss": 0.6984,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.002602100372314453,
      "rewards/margins": -0.009908200241625309,
      "rewards/rejected": 0.007306098937988281,
      "step": 1632
    },
    {
      "epoch": 0.42953196666063886,
      "grad_norm": 532.0,
      "learning_rate": 3.169590643274854e-07,
      "logits/chosen": -0.6394110918045044,
      "logits/rejected": -0.6422995328903198,
      "logps/chosen": -1104.656982421875,
      "logps/rejected": -1213.7572021484375,
      "loss": 0.7009,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.005420875735580921,
      "rewards/margins": -0.014577103778719902,
      "rewards/rejected": 0.009156227111816406,
      "step": 1633
    },
    {
      "epoch": 0.429794999095826,
      "grad_norm": 484.0,
      "learning_rate": 3.1681286549707605e-07,
      "logits/chosen": -0.6451383233070374,
      "logits/rejected": -0.6631438136100769,
      "logps/chosen": -884.326416015625,
      "logps/rejected": -854.800537109375,
      "loss": 0.7079,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.01505889929831028,
      "rewards/margins": -0.029166078194975853,
      "rewards/rejected": 0.014107179827988148,
      "step": 1634
    },
    {
      "epoch": 0.4300580315310132,
      "grad_norm": 372.0,
      "learning_rate": 3.166666666666666e-07,
      "logits/chosen": -0.6486830115318298,
      "logits/rejected": -0.6539771556854248,
      "logps/chosen": -947.7405395507812,
      "logps/rejected": -717.568359375,
      "loss": 0.697,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.009324170649051666,
      "rewards/margins": -0.0070501333102583885,
      "rewards/rejected": 0.01637430302798748,
      "step": 1635
    },
    {
      "epoch": 0.43032106396620035,
      "grad_norm": 580.0,
      "learning_rate": 3.165204678362573e-07,
      "logits/chosen": -0.6413591504096985,
      "logits/rejected": -0.6450040936470032,
      "logps/chosen": -1309.673095703125,
      "logps/rejected": -828.1405029296875,
      "loss": 0.6853,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01495971530675888,
      "rewards/margins": 0.01613759994506836,
      "rewards/rejected": -0.0011778827756643295,
      "step": 1636
    },
    {
      "epoch": 0.4305840964013875,
      "grad_norm": 624.0,
      "learning_rate": 3.1637426900584796e-07,
      "logits/chosen": -0.6595262885093689,
      "logits/rejected": -0.6561727523803711,
      "logps/chosen": -1273.10986328125,
      "logps/rejected": -865.6551513671875,
      "loss": 0.7035,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.006996345706284046,
      "rewards/margins": -0.019909095019102097,
      "rewards/rejected": 0.0129127511754632,
      "step": 1637
    },
    {
      "epoch": 0.43084712883657467,
      "grad_norm": 580.0,
      "learning_rate": 3.162280701754386e-07,
      "logits/chosen": -0.6799883842468262,
      "logits/rejected": -0.6928859949111938,
      "logps/chosen": -1259.1641845703125,
      "logps/rejected": -950.16748046875,
      "loss": 0.6853,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0032366756349802017,
      "rewards/margins": 0.01633157767355442,
      "rewards/rejected": -0.019568253308534622,
      "step": 1638
    },
    {
      "epoch": 0.4311101612717618,
      "grad_norm": 540.0,
      "learning_rate": 3.160818713450292e-07,
      "logits/chosen": -0.6758103370666504,
      "logits/rejected": -0.6765021681785583,
      "logps/chosen": -1245.3785400390625,
      "logps/rejected": -1150.684814453125,
      "loss": 0.7028,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.005535316187888384,
      "rewards/margins": -0.018209457397460938,
      "rewards/rejected": 0.023744774982333183,
      "step": 1639
    },
    {
      "epoch": 0.431373193706949,
      "grad_norm": 528.0,
      "learning_rate": 3.159356725146199e-07,
      "logits/chosen": -0.6578555107116699,
      "logits/rejected": -0.6694664359092712,
      "logps/chosen": -1139.1002197265625,
      "logps/rejected": -806.959228515625,
      "loss": 0.6799,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0352289192378521,
      "rewards/margins": 0.027866363525390625,
      "rewards/rejected": 0.007362556643784046,
      "step": 1640
    },
    {
      "epoch": 0.43163622614213615,
      "grad_norm": 624.0,
      "learning_rate": 3.157894736842105e-07,
      "logits/chosen": -0.654135525226593,
      "logits/rejected": NaN,
      "logps/chosen": -990.1948852539062,
      "logps/rejected": -742.8507080078125,
      "loss": 0.6928,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.014261245727539062,
      "rewards/margins": 0.0013525958638638258,
      "rewards/rejected": -0.015613842755556107,
      "step": 1641
    },
    {
      "epoch": 0.4318992585773233,
      "grad_norm": 616.0,
      "learning_rate": 3.1564327485380117e-07,
      "logits/chosen": -0.6778225302696228,
      "logits/rejected": -0.6805363893508911,
      "logps/chosen": -1590.81103515625,
      "logps/rejected": -1017.7843017578125,
      "loss": 0.6835,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.027044344693422318,
      "rewards/margins": 0.021016214042901993,
      "rewards/rejected": 0.006028127856552601,
      "step": 1642
    },
    {
      "epoch": 0.43216229101251047,
      "grad_norm": 572.0,
      "learning_rate": 3.1549707602339184e-07,
      "logits/chosen": -0.671022355556488,
      "logits/rejected": -0.6785374879837036,
      "logps/chosen": -1234.7568359375,
      "logps/rejected": -1040.64013671875,
      "loss": 0.6944,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01188678853213787,
      "rewards/margins": -0.0011373512679710984,
      "rewards/rejected": 0.013024138286709785,
      "step": 1643
    },
    {
      "epoch": 0.43242532344769763,
      "grad_norm": 568.0,
      "learning_rate": 3.153508771929824e-07,
      "logits/chosen": -0.6515763998031616,
      "logits/rejected": -0.6695284247398376,
      "logps/chosen": -1503.9127197265625,
      "logps/rejected": -948.15625,
      "loss": 0.6841,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.01617717742919922,
      "rewards/margins": 0.01906898245215416,
      "rewards/rejected": -0.0028918026946485043,
      "step": 1644
    },
    {
      "epoch": 0.4326883558828848,
      "grad_norm": 436.0,
      "learning_rate": 3.152046783625731e-07,
      "logits/chosen": -0.6464923024177551,
      "logits/rejected": -0.6585800647735596,
      "logps/chosen": -876.876953125,
      "logps/rejected": -628.8714599609375,
      "loss": 0.6797,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0010229117469862103,
      "rewards/margins": 0.02871561236679554,
      "rewards/rejected": -0.027692701667547226,
      "step": 1645
    },
    {
      "epoch": 0.43295138831807195,
      "grad_norm": 486.0,
      "learning_rate": 3.1505847953216375e-07,
      "logits/chosen": -0.6381278038024902,
      "logits/rejected": -0.662238597869873,
      "logps/chosen": -834.558349609375,
      "logps/rejected": -635.0447998046875,
      "loss": 0.7001,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.0012041092850267887,
      "rewards/margins": -0.013173865154385567,
      "rewards/rejected": 0.014377974905073643,
      "step": 1646
    },
    {
      "epoch": 0.4332144207532591,
      "grad_norm": 652.0,
      "learning_rate": 3.149122807017544e-07,
      "logits/chosen": -0.6317840814590454,
      "logits/rejected": -0.6300394535064697,
      "logps/chosen": -1333.1348876953125,
      "logps/rejected": -849.545654296875,
      "loss": 0.6929,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.017273902893066406,
      "rewards/margins": 0.002194356406107545,
      "rewards/rejected": 0.01507954578846693,
      "step": 1647
    },
    {
      "epoch": 0.4334774531884463,
      "grad_norm": 430.0,
      "learning_rate": 3.14766081871345e-07,
      "logits/chosen": -0.6385202407836914,
      "logits/rejected": -0.6450071334838867,
      "logps/chosen": -875.2591552734375,
      "logps/rejected": -660.3148193359375,
      "loss": 0.6841,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.01178064290434122,
      "rewards/margins": 0.019138813018798828,
      "rewards/rejected": -0.007358170114457607,
      "step": 1648
    },
    {
      "epoch": 0.4337404856236335,
      "grad_norm": 580.0,
      "learning_rate": 3.1461988304093567e-07,
      "logits/chosen": -0.6596041321754456,
      "logits/rejected": -0.6457624435424805,
      "logps/chosen": -1138.5345458984375,
      "logps/rejected": -859.2091064453125,
      "loss": 0.6996,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.037130214273929596,
      "rewards/margins": -0.011912442743778229,
      "rewards/rejected": -0.025217773392796516,
      "step": 1649
    },
    {
      "epoch": 0.43400351805882065,
      "grad_norm": 476.0,
      "learning_rate": 3.144736842105263e-07,
      "logits/chosen": -0.664070725440979,
      "logits/rejected": -0.6838306784629822,
      "logps/chosen": -1251.1053466796875,
      "logps/rejected": -948.8714599609375,
      "loss": 0.6877,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.005218889098614454,
      "rewards/margins": 0.011226655915379524,
      "rewards/rejected": -0.006007766351103783,
      "step": 1650
    },
    {
      "epoch": 0.4342665504940078,
      "grad_norm": 450.0,
      "learning_rate": 3.1432748538011696e-07,
      "logits/chosen": -0.6598318219184875,
      "logits/rejected": -0.66217440366745,
      "logps/chosen": -1180.3681640625,
      "logps/rejected": -1119.8106689453125,
      "loss": 0.6976,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.007672118954360485,
      "rewards/margins": -0.0086212158203125,
      "rewards/rejected": 0.01629333570599556,
      "step": 1651
    },
    {
      "epoch": 0.43452958292919497,
      "grad_norm": 604.0,
      "learning_rate": 3.141812865497076e-07,
      "logits/chosen": -0.6221746802330017,
      "logits/rejected": -0.6194616556167603,
      "logps/chosen": -1181.91845703125,
      "logps/rejected": -966.2799072265625,
      "loss": 0.7061,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009218026883900166,
      "rewards/margins": -0.024189617484807968,
      "rewards/rejected": 0.014971591532230377,
      "step": 1652
    },
    {
      "epoch": 0.43479261536438213,
      "grad_norm": 412.0,
      "learning_rate": 3.140350877192982e-07,
      "logits/chosen": -0.6500826478004456,
      "logits/rejected": -0.6640623807907104,
      "logps/chosen": -791.3848876953125,
      "logps/rejected": -827.04638671875,
      "loss": 0.6992,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.019428158178925514,
      "rewards/margins": -0.011545658111572266,
      "rewards/rejected": -0.007882499136030674,
      "step": 1653
    },
    {
      "epoch": 0.4350556477995693,
      "grad_norm": 632.0,
      "learning_rate": 3.1388888888888887e-07,
      "logits/chosen": -0.6556202173233032,
      "logits/rejected": -0.6461870670318604,
      "logps/chosen": -1042.9891357421875,
      "logps/rejected": -789.1712646484375,
      "loss": 0.6951,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0013088228879496455,
      "rewards/margins": -0.0032137385569512844,
      "rewards/rejected": 0.004522562026977539,
      "step": 1654
    },
    {
      "epoch": 0.43531868023475645,
      "grad_norm": 536.0,
      "learning_rate": 3.1374269005847954e-07,
      "logits/chosen": -0.6395612955093384,
      "logits/rejected": -0.6421056985855103,
      "logps/chosen": -1331.0086669921875,
      "logps/rejected": -971.2442626953125,
      "loss": 0.694,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.007996369153261185,
      "rewards/margins": -0.0006313314661383629,
      "rewards/rejected": 0.008627701550722122,
      "step": 1655
    },
    {
      "epoch": 0.4355817126699436,
      "grad_norm": 560.0,
      "learning_rate": 3.135964912280701e-07,
      "logits/chosen": -0.6351000070571899,
      "logits/rejected": -0.6363069415092468,
      "logps/chosen": -1196.8160400390625,
      "logps/rejected": -1032.3148193359375,
      "loss": 0.6866,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.009446239098906517,
      "rewards/margins": 0.014385412447154522,
      "rewards/rejected": -0.0049391742795705795,
      "step": 1656
    },
    {
      "epoch": 0.43584474510513077,
      "grad_norm": 458.0,
      "learning_rate": 3.134502923976608e-07,
      "logits/chosen": -0.6499266624450684,
      "logits/rejected": -0.6426411271095276,
      "logps/chosen": -866.9169921875,
      "logps/rejected": -784.5182495117188,
      "loss": 0.6955,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006294155493378639,
      "rewards/margins": -0.004313373938202858,
      "rewards/rejected": 0.010607529431581497,
      "step": 1657
    },
    {
      "epoch": 0.43610777754031793,
      "grad_norm": 444.0,
      "learning_rate": 3.1330409356725146e-07,
      "logits/chosen": -0.6405975818634033,
      "logits/rejected": -0.6373448371887207,
      "logps/chosen": -1067.1888427734375,
      "logps/rejected": -904.7736206054688,
      "loss": 0.6901,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01896371692419052,
      "rewards/margins": 0.006993675604462624,
      "rewards/rejected": 0.011970045045018196,
      "step": 1658
    },
    {
      "epoch": 0.4363708099755051,
      "grad_norm": 486.0,
      "learning_rate": 3.131578947368421e-07,
      "logits/chosen": -0.6513969302177429,
      "logits/rejected": -0.6597293615341187,
      "logps/chosen": -935.078857421875,
      "logps/rejected": -823.6011962890625,
      "loss": 0.698,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.007558346726000309,
      "rewards/margins": -0.00850839726626873,
      "rewards/rejected": 0.0009500496089458466,
      "step": 1659
    },
    {
      "epoch": 0.43663384241069225,
      "grad_norm": 720.0,
      "learning_rate": 3.1301169590643275e-07,
      "logits/chosen": -0.6356493234634399,
      "logits/rejected": NaN,
      "logps/chosen": -1578.7562255859375,
      "logps/rejected": -1371.2783203125,
      "loss": 0.6912,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0003569603431969881,
      "rewards/margins": 0.005068682134151459,
      "rewards/rejected": -0.004711722955107689,
      "step": 1660
    },
    {
      "epoch": 0.4368968748458794,
      "grad_norm": 600.0,
      "learning_rate": 3.1286549707602337e-07,
      "logits/chosen": -0.652977466583252,
      "logits/rejected": NaN,
      "logps/chosen": -1198.43603515625,
      "logps/rejected": -1010.1912841796875,
      "loss": 0.6801,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.025354955345392227,
      "rewards/margins": 0.027465248480439186,
      "rewards/rejected": -0.002110290341079235,
      "step": 1661
    },
    {
      "epoch": 0.4371599072810666,
      "grad_norm": 446.0,
      "learning_rate": 3.12719298245614e-07,
      "logits/chosen": -0.6700387597084045,
      "logits/rejected": -0.6717919111251831,
      "logps/chosen": -907.561767578125,
      "logps/rejected": -609.9851684570312,
      "loss": 0.6857,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.02189049869775772,
      "rewards/margins": 0.015440416522324085,
      "rewards/rejected": 0.006450082175433636,
      "step": 1662
    },
    {
      "epoch": 0.43742293971625373,
      "grad_norm": 596.0,
      "learning_rate": 3.1257309941520466e-07,
      "logits/chosen": -0.6645525693893433,
      "logits/rejected": -0.6649974584579468,
      "logps/chosen": -1134.205078125,
      "logps/rejected": -971.17724609375,
      "loss": 0.7034,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.001305198296904564,
      "rewards/margins": -0.01895800046622753,
      "rewards/rejected": 0.020263196900486946,
      "step": 1663
    },
    {
      "epoch": 0.43768597215144095,
      "grad_norm": 552.0,
      "learning_rate": 3.1242690058479534e-07,
      "logits/chosen": -0.6590147614479065,
      "logits/rejected": -0.6652899980545044,
      "logps/chosen": -1415.6717529296875,
      "logps/rejected": -1247.23291015625,
      "loss": 0.6867,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.026501178741455078,
      "rewards/margins": 0.014459704048931599,
      "rewards/rejected": 0.012041473761200905,
      "step": 1664
    },
    {
      "epoch": 0.4379490045866281,
      "grad_norm": 568.0,
      "learning_rate": 3.1228070175438596e-07,
      "logits/chosen": -0.6602726578712463,
      "logits/rejected": -0.6653050780296326,
      "logps/chosen": -1344.174072265625,
      "logps/rejected": -1179.99560546875,
      "loss": 0.6862,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005819845944643021,
      "rewards/margins": 0.014261294156312943,
      "rewards/rejected": -0.008441448211669922,
      "step": 1665
    },
    {
      "epoch": 0.43821203702181527,
      "grad_norm": 584.0,
      "learning_rate": 3.121345029239766e-07,
      "logits/chosen": -0.6477345824241638,
      "logits/rejected": -0.644006073474884,
      "logps/chosen": -1255.692626953125,
      "logps/rejected": -912.881591796875,
      "loss": 0.6935,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.020385267212986946,
      "rewards/margins": 0.0002958318218588829,
      "rewards/rejected": -0.020681096240878105,
      "step": 1666
    },
    {
      "epoch": 0.43847506945700243,
      "grad_norm": 510.0,
      "learning_rate": 3.1198830409356725e-07,
      "logits/chosen": -0.6565583944320679,
      "logits/rejected": -0.6503223776817322,
      "logps/chosen": -912.2530517578125,
      "logps/rejected": -830.5139770507812,
      "loss": 0.6914,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.006817817687988281,
      "rewards/margins": 0.003991795238107443,
      "rewards/rejected": 0.0028260224498808384,
      "step": 1667
    },
    {
      "epoch": 0.4387381018921896,
      "grad_norm": 660.0,
      "learning_rate": 3.118421052631579e-07,
      "logits/chosen": -0.649249255657196,
      "logits/rejected": -0.6453031301498413,
      "logps/chosen": -1242.005615234375,
      "logps/rejected": -1232.1600341796875,
      "loss": 0.6938,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.007870292291045189,
      "rewards/margins": -0.0005970965139567852,
      "rewards/rejected": 0.008467389270663261,
      "step": 1668
    },
    {
      "epoch": 0.43900113432737675,
      "grad_norm": 636.0,
      "learning_rate": 3.116959064327485e-07,
      "logits/chosen": -0.6190549731254578,
      "logits/rejected": NaN,
      "logps/chosen": -1116.254150390625,
      "logps/rejected": -826.6668090820312,
      "loss": 0.6847,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.003792953211814165,
      "rewards/margins": 0.01783137395977974,
      "rewards/rejected": -0.014038419350981712,
      "step": 1669
    },
    {
      "epoch": 0.4392641667625639,
      "grad_norm": 520.0,
      "learning_rate": 3.1154970760233916e-07,
      "logits/chosen": -0.6356655359268188,
      "logits/rejected": -0.6360117197036743,
      "logps/chosen": -1082.0926513671875,
      "logps/rejected": -907.4857177734375,
      "loss": 0.6824,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03647356107831001,
      "rewards/margins": 0.02233314886689186,
      "rewards/rejected": 0.01414041593670845,
      "step": 1670
    },
    {
      "epoch": 0.4395271991977511,
      "grad_norm": 548.0,
      "learning_rate": 3.1140350877192983e-07,
      "logits/chosen": -0.6474497318267822,
      "logits/rejected": -0.6538083553314209,
      "logps/chosen": -1274.4248046875,
      "logps/rejected": -930.5399169921875,
      "loss": 0.6951,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.01250448264181614,
      "rewards/margins": -0.0031825073529034853,
      "rewards/rejected": -0.009321975521743298,
      "step": 1671
    },
    {
      "epoch": 0.43979023163293823,
      "grad_norm": 524.0,
      "learning_rate": 3.1125730994152045e-07,
      "logits/chosen": -0.6503868103027344,
      "logits/rejected": -0.6636390686035156,
      "logps/chosen": -1106.818603515625,
      "logps/rejected": -839.3562622070312,
      "loss": 0.7054,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.004350758157670498,
      "rewards/margins": -0.023655224591493607,
      "rewards/rejected": 0.019304467365145683,
      "step": 1672
    },
    {
      "epoch": 0.4400532640681254,
      "grad_norm": 636.0,
      "learning_rate": 3.111111111111111e-07,
      "logits/chosen": -0.6877526044845581,
      "logits/rejected": -0.6882307529449463,
      "logps/chosen": -1785.96240234375,
      "logps/rejected": -1387.1658935546875,
      "loss": 0.6782,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.022579288110136986,
      "rewards/margins": 0.031446170061826706,
      "rewards/rejected": -0.00886688195168972,
      "step": 1673
    },
    {
      "epoch": 0.44031629650331255,
      "grad_norm": 506.0,
      "learning_rate": 3.1096491228070175e-07,
      "logits/chosen": -0.6545761823654175,
      "logits/rejected": -0.6454638242721558,
      "logps/chosen": -924.8802490234375,
      "logps/rejected": -909.040283203125,
      "loss": 0.699,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0054017058573663235,
      "rewards/margins": -0.010851670056581497,
      "rewards/rejected": 0.005449961870908737,
      "step": 1674
    },
    {
      "epoch": 0.4405793289384997,
      "grad_norm": 664.0,
      "learning_rate": 3.1081871345029237e-07,
      "logits/chosen": -0.6442337036132812,
      "logits/rejected": -0.6473708152770996,
      "logps/chosen": -1077.1419677734375,
      "logps/rejected": -800.7092895507812,
      "loss": 0.6922,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.002382945967838168,
      "rewards/margins": 0.0028128623962402344,
      "rewards/rejected": -0.0004299161955714226,
      "step": 1675
    },
    {
      "epoch": 0.4408423613736869,
      "grad_norm": 540.0,
      "learning_rate": 3.1067251461988304e-07,
      "logits/chosen": -0.6440128684043884,
      "logits/rejected": -0.6405496597290039,
      "logps/chosen": -1128.9993896484375,
      "logps/rejected": -987.0584716796875,
      "loss": 0.7074,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01223049033433199,
      "rewards/margins": -0.026931097730994225,
      "rewards/rejected": 0.014700601808726788,
      "step": 1676
    },
    {
      "epoch": 0.44110539380887404,
      "grad_norm": 448.0,
      "learning_rate": 3.105263157894737e-07,
      "logits/chosen": -0.6382442116737366,
      "logits/rejected": -0.65610671043396,
      "logps/chosen": -1038.30126953125,
      "logps/rejected": -812.2364501953125,
      "loss": 0.696,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.010854341089725494,
      "rewards/margins": -0.004983140155673027,
      "rewards/rejected": -0.005871201399713755,
      "step": 1677
    },
    {
      "epoch": 0.4413684262440612,
      "grad_norm": 460.0,
      "learning_rate": 3.103801169590643e-07,
      "logits/chosen": -0.6378395557403564,
      "logits/rejected": -0.632445216178894,
      "logps/chosen": -734.3924560546875,
      "logps/rejected": -801.249267578125,
      "loss": 0.6867,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006818485911935568,
      "rewards/margins": 0.01335983257740736,
      "rewards/rejected": -0.006541348062455654,
      "step": 1678
    },
    {
      "epoch": 0.4416314586792484,
      "grad_norm": 608.0,
      "learning_rate": 3.1023391812865495e-07,
      "logits/chosen": -0.6665700078010559,
      "logits/rejected": -0.6680166125297546,
      "logps/chosen": -1164.69091796875,
      "logps/rejected": -1115.33056640625,
      "loss": 0.6871,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.035176753997802734,
      "rewards/margins": 0.012804223224520683,
      "rewards/rejected": 0.0223725326359272,
      "step": 1679
    },
    {
      "epoch": 0.4418944911144356,
      "grad_norm": 612.0,
      "learning_rate": 3.100877192982456e-07,
      "logits/chosen": -0.6297393441200256,
      "logits/rejected": -0.6172467470169067,
      "logps/chosen": -1280.65771484375,
      "logps/rejected": -852.9609985351562,
      "loss": 0.6994,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009798813611268997,
      "rewards/margins": -0.011869048699736595,
      "rewards/rejected": 0.0020702360197901726,
      "step": 1680
    },
    {
      "epoch": 0.44215752354962273,
      "grad_norm": 536.0,
      "learning_rate": 3.0994152046783624e-07,
      "logits/chosen": -0.6643472909927368,
      "logits/rejected": -0.666724681854248,
      "logps/chosen": -1134.3707275390625,
      "logps/rejected": -856.7861328125,
      "loss": 0.6941,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.015385150909423828,
      "rewards/margins": -0.00043783197179436684,
      "rewards/rejected": 0.015822982415556908,
      "step": 1681
    },
    {
      "epoch": 0.4424205559848099,
      "grad_norm": 548.0,
      "learning_rate": 3.0979532163742686e-07,
      "logits/chosen": -0.6482046246528625,
      "logits/rejected": -0.6565896272659302,
      "logps/chosen": -1741.40673828125,
      "logps/rejected": -1396.283447265625,
      "loss": 0.6829,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.024617576971650124,
      "rewards/margins": 0.021394923329353333,
      "rewards/rejected": 0.0032226566690951586,
      "step": 1682
    },
    {
      "epoch": 0.44268358841999705,
      "grad_norm": 576.0,
      "learning_rate": 3.0964912280701754e-07,
      "logits/chosen": -0.6805831789970398,
      "logits/rejected": -0.679329514503479,
      "logps/chosen": -1464.246826171875,
      "logps/rejected": -1188.87109375,
      "loss": 0.6952,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0038138385862112045,
      "rewards/margins": -0.0034821517765522003,
      "rewards/rejected": -0.0003316877409815788,
      "step": 1683
    },
    {
      "epoch": 0.4429466208551842,
      "grad_norm": 556.0,
      "learning_rate": 3.0950292397660816e-07,
      "logits/chosen": -0.6598581075668335,
      "logits/rejected": -0.6630271077156067,
      "logps/chosen": -1037.8822021484375,
      "logps/rejected": -883.1322631835938,
      "loss": 0.6999,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.005487728863954544,
      "rewards/margins": -0.012804700061678886,
      "rewards/rejected": 0.007316972129046917,
      "step": 1684
    },
    {
      "epoch": 0.4432096532903714,
      "grad_norm": 462.0,
      "learning_rate": 3.0935672514619883e-07,
      "logits/chosen": -0.6752532720565796,
      "logits/rejected": -0.6690900325775146,
      "logps/chosen": -806.64453125,
      "logps/rejected": -581.4154663085938,
      "loss": 0.6927,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008766556158661842,
      "rewards/margins": 0.0012296682689338923,
      "rewards/rejected": 0.007536888122558594,
      "step": 1685
    },
    {
      "epoch": 0.44347268572555854,
      "grad_norm": 490.0,
      "learning_rate": 3.092105263157895e-07,
      "logits/chosen": -0.6334673762321472,
      "logits/rejected": -0.649170994758606,
      "logps/chosen": -889.6781005859375,
      "logps/rejected": -742.1400146484375,
      "loss": 0.7,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.020105456933379173,
      "rewards/margins": -0.013094616122543812,
      "rewards/rejected": 0.03320007398724556,
      "step": 1686
    },
    {
      "epoch": 0.4437357181607457,
      "grad_norm": 466.0,
      "learning_rate": 3.0906432748538007e-07,
      "logits/chosen": -0.6659504771232605,
      "logits/rejected": -0.6778014898300171,
      "logps/chosen": -1110.390625,
      "logps/rejected": -776.9226684570312,
      "loss": 0.6923,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0011690615210682154,
      "rewards/margins": 0.0018329614540562034,
      "rewards/rejected": -0.0006638993509113789,
      "step": 1687
    },
    {
      "epoch": 0.44399875059593286,
      "grad_norm": 452.0,
      "learning_rate": 3.0891812865497074e-07,
      "logits/chosen": -0.6501567363739014,
      "logits/rejected": -0.6468501091003418,
      "logps/chosen": -796.149658203125,
      "logps/rejected": -646.7780151367188,
      "loss": 0.6924,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.00029344484210014343,
      "rewards/margins": 0.002586556598544121,
      "rewards/rejected": -0.002293109893798828,
      "step": 1688
    },
    {
      "epoch": 0.44426178303112,
      "grad_norm": 580.0,
      "learning_rate": 3.087719298245614e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6362201571464539,
      "logps/chosen": -1159.6790771484375,
      "logps/rejected": -958.6465454101562,
      "loss": 0.697,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.007056237198412418,
      "rewards/margins": -0.006789684761315584,
      "rewards/rejected": -0.00026655150577425957,
      "step": 1689
    },
    {
      "epoch": 0.4445248154663072,
      "grad_norm": 612.0,
      "learning_rate": 3.0862573099415204e-07,
      "logits/chosen": -0.6496284008026123,
      "logits/rejected": -0.6609758138656616,
      "logps/chosen": -1854.3585205078125,
      "logps/rejected": -1622.7388916015625,
      "loss": 0.6892,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.023008918389678,
      "rewards/margins": 0.009053231216967106,
      "rewards/rejected": 0.013955689035356045,
      "step": 1690
    },
    {
      "epoch": 0.44478784790149434,
      "grad_norm": 608.0,
      "learning_rate": 3.0847953216374266e-07,
      "logits/chosen": -0.6653954386711121,
      "logits/rejected": -0.6650487184524536,
      "logps/chosen": -1368.3084716796875,
      "logps/rejected": -1192.93896484375,
      "loss": 0.6914,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.010422086343169212,
      "rewards/margins": 0.004530000500380993,
      "rewards/rejected": -0.01495208777487278,
      "step": 1691
    },
    {
      "epoch": 0.4450508803366815,
      "grad_norm": 560.0,
      "learning_rate": 3.0833333333333333e-07,
      "logits/chosen": -0.6410005688667297,
      "logits/rejected": -0.6464231014251709,
      "logps/chosen": -1401.02392578125,
      "logps/rejected": -1187.1220703125,
      "loss": 0.6993,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.010240362025797367,
      "rewards/margins": -0.011527061462402344,
      "rewards/rejected": 0.0012866971082985401,
      "step": 1692
    },
    {
      "epoch": 0.44531391277186866,
      "grad_norm": 436.0,
      "learning_rate": 3.0818713450292395e-07,
      "logits/chosen": -0.6267809271812439,
      "logits/rejected": -0.6149469017982483,
      "logps/chosen": -700.0847778320312,
      "logps/rejected": -659.509033203125,
      "loss": 0.7042,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.005325794219970703,
      "rewards/margins": -0.021202756091952324,
      "rewards/rejected": 0.01587696000933647,
      "step": 1693
    },
    {
      "epoch": 0.4455769452070558,
      "grad_norm": 480.0,
      "learning_rate": 3.080409356725146e-07,
      "logits/chosen": -0.638190507888794,
      "logits/rejected": -0.6471596360206604,
      "logps/chosen": -1027.155517578125,
      "logps/rejected": -731.2658081054688,
      "loss": 0.702,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.028059866279363632,
      "rewards/margins": -0.016432905569672585,
      "rewards/rejected": -0.011626958847045898,
      "step": 1694
    },
    {
      "epoch": 0.44583997764224303,
      "grad_norm": 510.0,
      "learning_rate": 3.0789473684210524e-07,
      "logits/chosen": -0.6498557329177856,
      "logits/rejected": -0.6532900929450989,
      "logps/chosen": -1006.0451049804688,
      "logps/rejected": -573.8547973632812,
      "loss": 0.7016,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.0007902145152911544,
      "rewards/margins": -0.015859508886933327,
      "rewards/rejected": 0.016649724915623665,
      "step": 1695
    },
    {
      "epoch": 0.4461030100774302,
      "grad_norm": 660.0,
      "learning_rate": 3.0774853801169586e-07,
      "logits/chosen": -0.6717749238014221,
      "logits/rejected": -0.6665346622467041,
      "logps/chosen": -1212.457763671875,
      "logps/rejected": -922.0748291015625,
      "loss": 0.6918,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0002359393984079361,
      "rewards/margins": 0.004175520036369562,
      "rewards/rejected": -0.0044114612974226475,
      "step": 1696
    },
    {
      "epoch": 0.44636604251261736,
      "grad_norm": 648.0,
      "learning_rate": 3.0760233918128653e-07,
      "logits/chosen": -0.6134018898010254,
      "logits/rejected": -0.6183191537857056,
      "logps/chosen": -1309.8988037109375,
      "logps/rejected": -908.765380859375,
      "loss": 0.6953,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012436865828931332,
      "rewards/margins": -0.003831862937659025,
      "rewards/rejected": 0.01626873016357422,
      "step": 1697
    },
    {
      "epoch": 0.4466290749478045,
      "grad_norm": 596.0,
      "learning_rate": 3.074561403508772e-07,
      "logits/chosen": -0.6840379238128662,
      "logits/rejected": -0.6824891567230225,
      "logps/chosen": -1126.024169921875,
      "logps/rejected": -798.1461181640625,
      "loss": 0.6895,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.012042521499097347,
      "rewards/margins": 0.008565378375351429,
      "rewards/rejected": 0.003477145219221711,
      "step": 1698
    },
    {
      "epoch": 0.4468921073829917,
      "grad_norm": 552.0,
      "learning_rate": 3.0730994152046777e-07,
      "logits/chosen": -0.645940899848938,
      "logits/rejected": -0.6454678773880005,
      "logps/chosen": -892.0560302734375,
      "logps/rejected": -944.3673095703125,
      "loss": 0.7014,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.013563442975282669,
      "rewards/margins": -0.015216254629194736,
      "rewards/rejected": 0.0016528135165572166,
      "step": 1699
    },
    {
      "epoch": 0.44715513981817884,
      "grad_norm": 572.0,
      "learning_rate": 3.0716374269005845e-07,
      "logits/chosen": -0.6586173176765442,
      "logits/rejected": -0.6648781895637512,
      "logps/chosen": -1092.7642822265625,
      "logps/rejected": -827.532958984375,
      "loss": 0.6887,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.009739875793457031,
      "rewards/margins": 0.009426688775420189,
      "rewards/rejected": 0.00031318608671426773,
      "step": 1700
    },
    {
      "epoch": 0.447418172253366,
      "grad_norm": 560.0,
      "learning_rate": 3.070175438596491e-07,
      "logits/chosen": -0.6431764364242554,
      "logits/rejected": -0.6509057283401489,
      "logps/chosen": -1073.144287109375,
      "logps/rejected": -816.9907836914062,
      "loss": 0.6891,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.028294850140810013,
      "rewards/margins": 0.009382056072354317,
      "rewards/rejected": 0.018912794068455696,
      "step": 1701
    },
    {
      "epoch": 0.44768120468855316,
      "grad_norm": 524.0,
      "learning_rate": 3.0687134502923974e-07,
      "logits/chosen": -0.6617210507392883,
      "logits/rejected": -0.6637781858444214,
      "logps/chosen": -990.3735961914062,
      "logps/rejected": -922.7333374023438,
      "loss": 0.6856,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0163453109562397,
      "rewards/margins": 0.015876198187470436,
      "rewards/rejected": 0.00046911253593862057,
      "step": 1702
    },
    {
      "epoch": 0.4479442371237403,
      "grad_norm": 320.0,
      "learning_rate": 3.067251461988304e-07,
      "logits/chosen": -0.6494088172912598,
      "logits/rejected": -0.6618737578392029,
      "logps/chosen": -779.65380859375,
      "logps/rejected": -601.81689453125,
      "loss": 0.6903,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.00897836685180664,
      "rewards/margins": 0.00596275320276618,
      "rewards/rejected": -0.014941120520234108,
      "step": 1703
    },
    {
      "epoch": 0.4482072695589275,
      "grad_norm": 568.0,
      "learning_rate": 3.0657894736842103e-07,
      "logits/chosen": -0.6775528192520142,
      "logits/rejected": -0.6741320490837097,
      "logps/chosen": -1269.1295166015625,
      "logps/rejected": -736.5784912109375,
      "loss": 0.7077,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.03977222368121147,
      "rewards/margins": -0.0276412945240736,
      "rewards/rejected": -0.012130928225815296,
      "step": 1704
    },
    {
      "epoch": 0.44847030199411464,
      "grad_norm": 556.0,
      "learning_rate": 3.064327485380117e-07,
      "logits/chosen": -0.6503993272781372,
      "logits/rejected": -0.6457400321960449,
      "logps/chosen": -1274.35009765625,
      "logps/rejected": -1068.006103515625,
      "loss": 0.6968,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0016345027834177017,
      "rewards/margins": -0.006820201873779297,
      "rewards/rejected": 0.005185699090361595,
      "step": 1705
    },
    {
      "epoch": 0.4487333344293018,
      "grad_norm": 432.0,
      "learning_rate": 3.062865497076023e-07,
      "logits/chosen": -0.6471582651138306,
      "logits/rejected": -0.6553775668144226,
      "logps/chosen": -876.1822509765625,
      "logps/rejected": -495.20782470703125,
      "loss": 0.6923,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.019024087116122246,
      "rewards/margins": 0.002316283294931054,
      "rewards/rejected": 0.016707802191376686,
      "step": 1706
    },
    {
      "epoch": 0.44899636686448896,
      "grad_norm": 564.0,
      "learning_rate": 3.06140350877193e-07,
      "logits/chosen": -0.6411586403846741,
      "logits/rejected": -0.6503449082374573,
      "logps/chosen": -1169.9853515625,
      "logps/rejected": -791.621337890625,
      "loss": 0.6983,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00950861070305109,
      "rewards/margins": -0.00980224646627903,
      "rewards/rejected": 0.0002936370437964797,
      "step": 1707
    },
    {
      "epoch": 0.4492593992996761,
      "grad_norm": 440.0,
      "learning_rate": 3.059941520467836e-07,
      "logits/chosen": -0.6439459323883057,
      "logits/rejected": -0.643218457698822,
      "logps/chosen": -703.5490112304688,
      "logps/rejected": -735.4305419921875,
      "loss": 0.6925,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.011021328158676624,
      "rewards/margins": 0.0015752788167446852,
      "rewards/rejected": -0.012596606276929379,
      "step": 1708
    },
    {
      "epoch": 0.4495224317348633,
      "grad_norm": 402.0,
      "learning_rate": 3.0584795321637424e-07,
      "logits/chosen": -0.6715153455734253,
      "logits/rejected": -0.6780632734298706,
      "logps/chosen": -895.27783203125,
      "logps/rejected": -703.3065795898438,
      "loss": 0.6961,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.023967649787664413,
      "rewards/margins": -0.005287886597216129,
      "rewards/rejected": -0.01867976225912571,
      "step": 1709
    },
    {
      "epoch": 0.44978546417005044,
      "grad_norm": 444.0,
      "learning_rate": 3.057017543859649e-07,
      "logits/chosen": -0.6341972947120667,
      "logits/rejected": -0.637699544429779,
      "logps/chosen": -1004.4105224609375,
      "logps/rejected": -708.7257080078125,
      "loss": 0.6884,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.022588538005948067,
      "rewards/margins": 0.009747695177793503,
      "rewards/rejected": 0.012840843759477139,
      "step": 1710
    },
    {
      "epoch": 0.45004849660523766,
      "grad_norm": 524.0,
      "learning_rate": 3.055555555555556e-07,
      "logits/chosen": -0.6540046334266663,
      "logits/rejected": -0.6422790288925171,
      "logps/chosen": -1147.661865234375,
      "logps/rejected": -841.6793212890625,
      "loss": 0.6821,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.014385033398866653,
      "rewards/margins": 0.022594451904296875,
      "rewards/rejected": -0.008209419436752796,
      "step": 1711
    },
    {
      "epoch": 0.4503115290404248,
      "grad_norm": 524.0,
      "learning_rate": 3.0540935672514615e-07,
      "logits/chosen": -0.6472917199134827,
      "logits/rejected": -0.6480578780174255,
      "logps/chosen": -929.3366088867188,
      "logps/rejected": -989.4315795898438,
      "loss": 0.7026,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.00045957579277455807,
      "rewards/margins": -0.017456628382205963,
      "rewards/rejected": 0.01699705235660076,
      "step": 1712
    },
    {
      "epoch": 0.450574561475612,
      "grad_norm": 412.0,
      "learning_rate": 3.052631578947368e-07,
      "logits/chosen": -0.6673761010169983,
      "logits/rejected": -0.6730131506919861,
      "logps/chosen": -797.0183715820312,
      "logps/rejected": -749.8277587890625,
      "loss": 0.6891,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0157944206148386,
      "rewards/margins": 0.009238003753125668,
      "rewards/rejected": -0.025032421573996544,
      "step": 1713
    },
    {
      "epoch": 0.45083759391079914,
      "grad_norm": 490.0,
      "learning_rate": 3.051169590643275e-07,
      "logits/chosen": -0.6788685321807861,
      "logits/rejected": -0.6855385899543762,
      "logps/chosen": -1093.2459716796875,
      "logps/rejected": -897.6698608398438,
      "loss": 0.6822,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01043558120727539,
      "rewards/margins": 0.023700619116425514,
      "rewards/rejected": -0.013265037909150124,
      "step": 1714
    },
    {
      "epoch": 0.4511006263459863,
      "grad_norm": 426.0,
      "learning_rate": 3.049707602339181e-07,
      "logits/chosen": -0.6673814058303833,
      "logits/rejected": -0.6653293967247009,
      "logps/chosen": -904.9301147460938,
      "logps/rejected": -794.9391479492188,
      "loss": 0.7005,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.008138084784150124,
      "rewards/margins": -0.01401214674115181,
      "rewards/rejected": 0.005874061491340399,
      "step": 1715
    },
    {
      "epoch": 0.45136365878117346,
      "grad_norm": 564.0,
      "learning_rate": 3.048245614035088e-07,
      "logits/chosen": -0.6541430950164795,
      "logits/rejected": -0.6557521224021912,
      "logps/chosen": -1675.430908203125,
      "logps/rejected": -1271.22021484375,
      "loss": 0.6921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0021358502563089132,
      "rewards/margins": 0.002610110677778721,
      "rewards/rejected": -0.0004742625169456005,
      "step": 1716
    },
    {
      "epoch": 0.4516266912163606,
      "grad_norm": 434.0,
      "learning_rate": 3.046783625730994e-07,
      "logits/chosen": -0.6423463821411133,
      "logits/rejected": -0.653298556804657,
      "logps/chosen": -886.1837158203125,
      "logps/rejected": -679.4110107421875,
      "loss": 0.7017,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0018912306986749172,
      "rewards/margins": -0.016409777104854584,
      "rewards/rejected": 0.01830100826919079,
      "step": 1717
    },
    {
      "epoch": 0.4518897236515478,
      "grad_norm": 636.0,
      "learning_rate": 3.0453216374269003e-07,
      "logits/chosen": -0.640284538269043,
      "logits/rejected": -0.649090051651001,
      "logps/chosen": -1286.729736328125,
      "logps/rejected": -1123.9320068359375,
      "loss": 0.6941,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.002771948929876089,
      "rewards/margins": -0.0005383477546274662,
      "rewards/rejected": -0.0022336007095873356,
      "step": 1718
    },
    {
      "epoch": 0.45215275608673494,
      "grad_norm": 588.0,
      "learning_rate": 3.043859649122807e-07,
      "logits/chosen": -0.6281501054763794,
      "logits/rejected": -0.6341792345046997,
      "logps/chosen": -1130.63671875,
      "logps/rejected": -1131.5755615234375,
      "loss": 0.6963,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.007808972150087357,
      "rewards/margins": -0.005105685442686081,
      "rewards/rejected": 0.012914658524096012,
      "step": 1719
    },
    {
      "epoch": 0.4524157885219221,
      "grad_norm": 488.0,
      "learning_rate": 3.042397660818714e-07,
      "logits/chosen": -0.6389180421829224,
      "logits/rejected": -0.6395503282546997,
      "logps/chosen": -1207.51611328125,
      "logps/rejected": -1007.904296875,
      "loss": 0.6982,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0278491023927927,
      "rewards/margins": -0.008940314874053001,
      "rewards/rejected": -0.0189087875187397,
      "step": 1720
    },
    {
      "epoch": 0.45267882095710926,
      "grad_norm": 416.0,
      "learning_rate": 3.0409356725146194e-07,
      "logits/chosen": -0.6553063988685608,
      "logits/rejected": -0.6648125648498535,
      "logps/chosen": -1008.8172607421875,
      "logps/rejected": -879.88525390625,
      "loss": 0.6929,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.010692549869418144,
      "rewards/margins": 0.0007701400318183005,
      "rewards/rejected": -0.011462688446044922,
      "step": 1721
    },
    {
      "epoch": 0.4529418533922964,
      "grad_norm": 576.0,
      "learning_rate": 3.039473684210526e-07,
      "logits/chosen": -0.6606734991073608,
      "logits/rejected": -0.6544660925865173,
      "logps/chosen": -1297.03369140625,
      "logps/rejected": -1069.58251953125,
      "loss": 0.6982,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.004383659455925226,
      "rewards/margins": -0.009509611874818802,
      "rewards/rejected": 0.01389327086508274,
      "step": 1722
    },
    {
      "epoch": 0.4532048858274836,
      "grad_norm": 386.0,
      "learning_rate": 3.038011695906433e-07,
      "logits/chosen": -0.6509073972702026,
      "logits/rejected": -0.6501578688621521,
      "logps/chosen": -732.0377197265625,
      "logps/rejected": -467.094482421875,
      "loss": 0.6904,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.010977650061249733,
      "rewards/margins": 0.006247901823371649,
      "rewards/rejected": 0.004729747772216797,
      "step": 1723
    },
    {
      "epoch": 0.45346791826267074,
      "grad_norm": 452.0,
      "learning_rate": 3.036549707602339e-07,
      "logits/chosen": -0.658290684223175,
      "logits/rejected": -0.6534606218338013,
      "logps/chosen": -1072.8916015625,
      "logps/rejected": -1094.802490234375,
      "loss": 0.6912,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.014860630966722965,
      "rewards/margins": 0.004209422972053289,
      "rewards/rejected": -0.019070053473114967,
      "step": 1724
    },
    {
      "epoch": 0.4537309506978579,
      "grad_norm": 556.0,
      "learning_rate": 3.035087719298245e-07,
      "logits/chosen": -0.635404646396637,
      "logits/rejected": -0.6325827836990356,
      "logps/chosen": -1448.123046875,
      "logps/rejected": -1365.2503662109375,
      "loss": 0.6824,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.000355816213414073,
      "rewards/margins": 0.022566795349121094,
      "rewards/rejected": -0.022922612726688385,
      "step": 1725
    },
    {
      "epoch": 0.4539939831330451,
      "grad_norm": 560.0,
      "learning_rate": 3.033625730994152e-07,
      "logits/chosen": -0.6434142589569092,
      "logits/rejected": -0.6504065990447998,
      "logps/chosen": -1065.751220703125,
      "logps/rejected": -1022.7496337890625,
      "loss": 0.7022,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.007491970434784889,
      "rewards/margins": -0.017195699736475945,
      "rewards/rejected": 0.024687672033905983,
      "step": 1726
    },
    {
      "epoch": 0.4542570155682323,
      "grad_norm": 604.0,
      "learning_rate": 3.032163742690058e-07,
      "logits/chosen": -0.6415596008300781,
      "logits/rejected": -0.6470178365707397,
      "logps/chosen": -1379.06982421875,
      "logps/rejected": -1106.03857421875,
      "loss": 0.6818,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02778778038918972,
      "rewards/margins": 0.02358102612197399,
      "rewards/rejected": 0.004206752870231867,
      "step": 1727
    },
    {
      "epoch": 0.45452004800341944,
      "grad_norm": 506.0,
      "learning_rate": 3.030701754385965e-07,
      "logits/chosen": -0.6736142635345459,
      "logits/rejected": -0.6694549918174744,
      "logps/chosen": -1163.279052734375,
      "logps/rejected": -1240.9674072265625,
      "loss": 0.6889,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0023347854148596525,
      "rewards/margins": 0.009015179239213467,
      "rewards/rejected": -0.011349964886903763,
      "step": 1728
    },
    {
      "epoch": 0.4547830804386066,
      "grad_norm": 528.0,
      "learning_rate": 3.029239766081871e-07,
      "logits/chosen": -0.6637045741081238,
      "logits/rejected": -0.6650904417037964,
      "logps/chosen": -979.197265625,
      "logps/rejected": -805.050048828125,
      "loss": 0.697,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01757650263607502,
      "rewards/margins": -0.006972885224968195,
      "rewards/rejected": -0.010603617876768112,
      "step": 1729
    },
    {
      "epoch": 0.45504611287379376,
      "grad_norm": 390.0,
      "learning_rate": 3.0277777777777773e-07,
      "logits/chosen": -0.6494971513748169,
      "logits/rejected": -0.6542210578918457,
      "logps/chosen": -890.7572021484375,
      "logps/rejected": -592.157958984375,
      "loss": 0.6905,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.002525138668715954,
      "rewards/margins": 0.005691433325409889,
      "rewards/rejected": -0.008216572925448418,
      "step": 1730
    },
    {
      "epoch": 0.4553091453089809,
      "grad_norm": 592.0,
      "learning_rate": 3.026315789473684e-07,
      "logits/chosen": -0.6518363952636719,
      "logits/rejected": -0.6455237865447998,
      "logps/chosen": -1335.312255859375,
      "logps/rejected": -900.8556518554688,
      "loss": 0.6825,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.020668888464570045,
      "rewards/margins": 0.022655202075839043,
      "rewards/rejected": -0.0019863129127770662,
      "step": 1731
    },
    {
      "epoch": 0.4555721777441681,
      "grad_norm": 596.0,
      "learning_rate": 3.024853801169591e-07,
      "logits/chosen": -0.6437175273895264,
      "logits/rejected": -0.6335432529449463,
      "logps/chosen": -833.6568603515625,
      "logps/rejected": -759.444580078125,
      "loss": 0.6952,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.007153701968491077,
      "rewards/margins": -0.0034143447410315275,
      "rewards/rejected": -0.0037393569946289062,
      "step": 1732
    },
    {
      "epoch": 0.45583521017935524,
      "grad_norm": 496.0,
      "learning_rate": 3.023391812865497e-07,
      "logits/chosen": -0.6523363590240479,
      "logits/rejected": -0.6519489288330078,
      "logps/chosen": -830.25146484375,
      "logps/rejected": -894.2445068359375,
      "loss": 0.7008,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.009254884906113148,
      "rewards/margins": -0.014511918649077415,
      "rewards/rejected": 0.023766804486513138,
      "step": 1733
    },
    {
      "epoch": 0.4560982426145424,
      "grad_norm": 536.0,
      "learning_rate": 3.021929824561403e-07,
      "logits/chosen": -0.675830602645874,
      "logits/rejected": -0.6727815270423889,
      "logps/chosen": -1257.7977294921875,
      "logps/rejected": -865.153564453125,
      "loss": 0.6983,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0035573968198150396,
      "rewards/margins": -0.009480285458266735,
      "rewards/rejected": 0.005922888871282339,
      "step": 1734
    },
    {
      "epoch": 0.45636127504972956,
      "grad_norm": 636.0,
      "learning_rate": 3.02046783625731e-07,
      "logits/chosen": -0.6662041544914246,
      "logits/rejected": -0.6744341254234314,
      "logps/chosen": -1351.9556884765625,
      "logps/rejected": -1279.89013671875,
      "loss": 0.6997,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.002027510665357113,
      "rewards/margins": -0.011936950497329235,
      "rewards/rejected": 0.009909438900649548,
      "step": 1735
    },
    {
      "epoch": 0.4566243074849167,
      "grad_norm": 500.0,
      "learning_rate": 3.019005847953216e-07,
      "logits/chosen": -0.6750423312187195,
      "logits/rejected": -0.6811108589172363,
      "logps/chosen": -1402.054443359375,
      "logps/rejected": -1285.8353271484375,
      "loss": 0.694,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.011128711514174938,
      "rewards/margins": -0.0009154323488473892,
      "rewards/rejected": -0.010213281027972698,
      "step": 1736
    },
    {
      "epoch": 0.4568873399201039,
      "grad_norm": 446.0,
      "learning_rate": 3.017543859649123e-07,
      "logits/chosen": -0.632288932800293,
      "logits/rejected": -0.6297182440757751,
      "logps/chosen": -903.5993041992188,
      "logps/rejected": -833.08642578125,
      "loss": 0.6963,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.004986191168427467,
      "rewards/margins": -0.005942535120993853,
      "rewards/rejected": 0.0009563441853970289,
      "step": 1737
    },
    {
      "epoch": 0.45715037235529105,
      "grad_norm": 648.0,
      "learning_rate": 3.016081871345029e-07,
      "logits/chosen": -0.6528468132019043,
      "logits/rejected": -0.6563240885734558,
      "logps/chosen": -1253.7606201171875,
      "logps/rejected": -1146.2821044921875,
      "loss": 0.6912,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.011789512820541859,
      "rewards/margins": 0.005391214974224567,
      "rewards/rejected": 0.006398297380656004,
      "step": 1738
    },
    {
      "epoch": 0.4574134047904782,
      "grad_norm": 432.0,
      "learning_rate": 3.014619883040935e-07,
      "logits/chosen": -0.6544378995895386,
      "logits/rejected": -0.6597378849983215,
      "logps/chosen": -814.0961303710938,
      "logps/rejected": -553.0447998046875,
      "loss": 0.6982,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0026259412989020348,
      "rewards/margins": -0.009357498958706856,
      "rewards/rejected": 0.011983443051576614,
      "step": 1739
    },
    {
      "epoch": 0.45767643722566537,
      "grad_norm": 516.0,
      "learning_rate": 3.013157894736842e-07,
      "logits/chosen": -0.6549522280693054,
      "logits/rejected": -0.660409688949585,
      "logps/chosen": -1303.8248291015625,
      "logps/rejected": -974.7491455078125,
      "loss": 0.7063,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.012568949721753597,
      "rewards/margins": -0.02549009397625923,
      "rewards/rejected": 0.038059040904045105,
      "step": 1740
    },
    {
      "epoch": 0.4579394696608525,
      "grad_norm": 496.0,
      "learning_rate": 3.0116959064327487e-07,
      "logits/chosen": -0.6398788094520569,
      "logits/rejected": -0.6443174481391907,
      "logps/chosen": -1379.773193359375,
      "logps/rejected": -1236.1800537109375,
      "loss": 0.698,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.009272098541259766,
      "rewards/margins": -0.00923614390194416,
      "rewards/rejected": -3.5954173654317856e-05,
      "step": 1741
    },
    {
      "epoch": 0.45820250209603974,
      "grad_norm": 544.0,
      "learning_rate": 3.0102339181286544e-07,
      "logits/chosen": -0.660286009311676,
      "logits/rejected": -0.6533069014549255,
      "logps/chosen": -913.7987060546875,
      "logps/rejected": -1012.110595703125,
      "loss": 0.6952,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.009936332702636719,
      "rewards/margins": -0.003940295893698931,
      "rewards/rejected": 0.013876628130674362,
      "step": 1742
    },
    {
      "epoch": 0.4584655345312269,
      "grad_norm": 520.0,
      "learning_rate": 3.008771929824561e-07,
      "logits/chosen": -0.6456508636474609,
      "logits/rejected": -0.6367610096931458,
      "logps/chosen": -625.5216674804688,
      "logps/rejected": -599.7916870117188,
      "loss": 0.6896,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.013556099496781826,
      "rewards/margins": 0.00730743445456028,
      "rewards/rejected": 0.006248665042221546,
      "step": 1743
    },
    {
      "epoch": 0.45872856696641406,
      "grad_norm": 592.0,
      "learning_rate": 3.007309941520468e-07,
      "logits/chosen": -0.623127281665802,
      "logits/rejected": -0.6308258771896362,
      "logps/chosen": -1021.142822265625,
      "logps/rejected": -1010.0066528320312,
      "loss": 0.6917,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0005653382977470756,
      "rewards/margins": 0.003516578348353505,
      "rewards/rejected": -0.0040819174610078335,
      "step": 1744
    },
    {
      "epoch": 0.4589915994016012,
      "grad_norm": 500.0,
      "learning_rate": 3.0058479532163745e-07,
      "logits/chosen": -0.6340764760971069,
      "logits/rejected": -0.6378899216651917,
      "logps/chosen": -1015.3348388671875,
      "logps/rejected": -896.026123046875,
      "loss": 0.6909,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004788875579833984,
      "rewards/margins": 0.005141162313520908,
      "rewards/rejected": -0.0003522872575558722,
      "step": 1745
    },
    {
      "epoch": 0.4592546318367884,
      "grad_norm": 580.0,
      "learning_rate": 3.00438596491228e-07,
      "logits/chosen": -0.6164220571517944,
      "logits/rejected": NaN,
      "logps/chosen": -1527.886474609375,
      "logps/rejected": -915.8790893554688,
      "loss": 0.6899,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008973408490419388,
      "rewards/margins": 0.007957268506288528,
      "rewards/rejected": 0.0010161387035623193,
      "step": 1746
    },
    {
      "epoch": 0.45951766427197555,
      "grad_norm": 572.0,
      "learning_rate": 3.002923976608187e-07,
      "logits/chosen": -0.6433882713317871,
      "logits/rejected": -0.6741899847984314,
      "logps/chosen": -1393.55419921875,
      "logps/rejected": -1275.0194091796875,
      "loss": 0.7036,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01494893990457058,
      "rewards/margins": -0.020003987476229668,
      "rewards/rejected": 0.005055047105997801,
      "step": 1747
    },
    {
      "epoch": 0.4597806967071627,
      "grad_norm": 478.0,
      "learning_rate": 3.0014619883040937e-07,
      "logits/chosen": -0.6646685004234314,
      "logits/rejected": -0.6586941480636597,
      "logps/chosen": -1228.493408203125,
      "logps/rejected": -728.2088623046875,
      "loss": 0.6871,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.007871724665164948,
      "rewards/margins": 0.012575101107358932,
      "rewards/rejected": -0.02044682577252388,
      "step": 1748
    },
    {
      "epoch": 0.46004372914234987,
      "grad_norm": 528.0,
      "learning_rate": 3e-07,
      "logits/chosen": -0.6341022849082947,
      "logits/rejected": -0.6337418556213379,
      "logps/chosen": -1170.3336181640625,
      "logps/rejected": -861.438720703125,
      "loss": 0.6893,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0012903215829283,
      "rewards/margins": 0.008230305276811123,
      "rewards/rejected": -0.006939982529729605,
      "step": 1749
    },
    {
      "epoch": 0.460306761577537,
      "grad_norm": 544.0,
      "learning_rate": 2.9985380116959066e-07,
      "logits/chosen": -0.6591903567314148,
      "logits/rejected": -0.6772468090057373,
      "logps/chosen": -1476.8399658203125,
      "logps/rejected": -1099.1444091796875,
      "loss": 0.6968,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0014403333188965917,
      "rewards/margins": -0.006867502816021442,
      "rewards/rejected": 0.00542717007920146,
      "step": 1750
    },
    {
      "epoch": 0.4605697940127242,
      "grad_norm": 470.0,
      "learning_rate": 2.997076023391813e-07,
      "logits/chosen": -0.6373677849769592,
      "logits/rejected": -0.6454620361328125,
      "logps/chosen": -933.8858032226562,
      "logps/rejected": -682.0732421875,
      "loss": 0.7097,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.012125970795750618,
      "rewards/margins": -0.031935691833496094,
      "rewards/rejected": 0.019809722900390625,
      "step": 1751
    },
    {
      "epoch": 0.46083282644791135,
      "grad_norm": 572.0,
      "learning_rate": 2.995614035087719e-07,
      "logits/chosen": -0.6543930172920227,
      "logits/rejected": -0.6503053307533264,
      "logps/chosen": -1014.7701416015625,
      "logps/rejected": -662.73876953125,
      "loss": 0.7013,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.01586160808801651,
      "rewards/margins": -0.015031099319458008,
      "rewards/rejected": -0.0008305074879899621,
      "step": 1752
    },
    {
      "epoch": 0.4610958588830985,
      "grad_norm": 620.0,
      "learning_rate": 2.9941520467836257e-07,
      "logits/chosen": -0.649701714515686,
      "logits/rejected": NaN,
      "logps/chosen": -728.6732788085938,
      "logps/rejected": -460.42840576171875,
      "loss": 0.6911,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011536120437085629,
      "rewards/margins": 0.0047243135049939156,
      "rewards/rejected": -0.016260433942079544,
      "step": 1753
    },
    {
      "epoch": 0.46135889131828567,
      "grad_norm": 592.0,
      "learning_rate": 2.9926900584795324e-07,
      "logits/chosen": -0.6530405282974243,
      "logits/rejected": -0.661197304725647,
      "logps/chosen": -1445.0029296875,
      "logps/rejected": -901.1421508789062,
      "loss": 0.6887,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.004249667748808861,
      "rewards/margins": 0.00952615961432457,
      "rewards/rejected": -0.013775826431810856,
      "step": 1754
    },
    {
      "epoch": 0.46162192375347283,
      "grad_norm": 478.0,
      "learning_rate": 2.991228070175438e-07,
      "logits/chosen": -0.6342275738716125,
      "logits/rejected": -0.6238793134689331,
      "logps/chosen": -851.0466918945312,
      "logps/rejected": -547.6431884765625,
      "loss": 0.6964,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0035321232862770557,
      "rewards/margins": -0.005553911905735731,
      "rewards/rejected": 0.0020217886194586754,
      "step": 1755
    },
    {
      "epoch": 0.46188495618866,
      "grad_norm": 512.0,
      "learning_rate": 2.989766081871345e-07,
      "logits/chosen": -0.6599900126457214,
      "logits/rejected": -0.6596477031707764,
      "logps/chosen": -1061.2886962890625,
      "logps/rejected": -809.7825317382812,
      "loss": 0.6995,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.002085925079882145,
      "rewards/margins": -0.011908960528671741,
      "rewards/rejected": 0.009823037311434746,
      "step": 1756
    },
    {
      "epoch": 0.4621479886238472,
      "grad_norm": 668.0,
      "learning_rate": 2.9883040935672516e-07,
      "logits/chosen": -0.6627508997917175,
      "logits/rejected": NaN,
      "logps/chosen": -1405.9552001953125,
      "logps/rejected": -988.4425048828125,
      "loss": 0.7067,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.002218531910330057,
      "rewards/margins": -0.025867603719234467,
      "rewards/rejected": 0.028086138889193535,
      "step": 1757
    },
    {
      "epoch": 0.46241102105903437,
      "grad_norm": 680.0,
      "learning_rate": 2.986842105263158e-07,
      "logits/chosen": -0.6655709147453308,
      "logits/rejected": -0.6674914360046387,
      "logps/chosen": -1397.959228515625,
      "logps/rejected": -855.7401733398438,
      "loss": 0.7068,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.02249918133020401,
      "rewards/margins": -0.026303671300411224,
      "rewards/rejected": 0.003804492997005582,
      "step": 1758
    },
    {
      "epoch": 0.4626740534942215,
      "grad_norm": 600.0,
      "learning_rate": 2.985380116959064e-07,
      "logits/chosen": -0.6511760950088501,
      "logits/rejected": -0.6676902770996094,
      "logps/chosen": -1273.389892578125,
      "logps/rejected": -702.5653076171875,
      "loss": 0.6972,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006392288021743298,
      "rewards/margins": -0.007636453025043011,
      "rewards/rejected": 0.014028739184141159,
      "step": 1759
    },
    {
      "epoch": 0.4629370859294087,
      "grad_norm": 564.0,
      "learning_rate": 2.9839181286549707e-07,
      "logits/chosen": -0.6471531391143799,
      "logits/rejected": -0.6372582316398621,
      "logps/chosen": -1335.56787109375,
      "logps/rejected": -1113.5517578125,
      "loss": 0.688,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004444217775017023,
      "rewards/margins": 0.011105346493422985,
      "rewards/rejected": -0.006661129184067249,
      "step": 1760
    },
    {
      "epoch": 0.46320011836459585,
      "grad_norm": 608.0,
      "learning_rate": 2.982456140350877e-07,
      "logits/chosen": -0.6298132538795471,
      "logits/rejected": -0.6292035579681396,
      "logps/chosen": -1276.5740966796875,
      "logps/rejected": -938.439453125,
      "loss": 0.6903,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.010683631524443626,
      "rewards/margins": 0.006179093383252621,
      "rewards/rejected": -0.01686272583901882,
      "step": 1761
    },
    {
      "epoch": 0.463463150799783,
      "grad_norm": 632.0,
      "learning_rate": 2.9809941520467836e-07,
      "logits/chosen": -0.6481651067733765,
      "logits/rejected": -0.6440125703811646,
      "logps/chosen": -1039.7562255859375,
      "logps/rejected": -812.409912109375,
      "loss": 0.6934,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010751534253358841,
      "rewards/margins": 0.00026025716215372086,
      "rewards/rejected": 0.010491276159882545,
      "step": 1762
    },
    {
      "epoch": 0.46372618323497017,
      "grad_norm": 386.0,
      "learning_rate": 2.9795321637426904e-07,
      "logits/chosen": -0.6533280611038208,
      "logits/rejected": -0.6568551659584045,
      "logps/chosen": -786.69384765625,
      "logps/rejected": -749.238525390625,
      "loss": 0.6793,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02293219603598118,
      "rewards/margins": 0.029535960406064987,
      "rewards/rejected": -0.006603766232728958,
      "step": 1763
    },
    {
      "epoch": 0.46398921567015733,
      "grad_norm": 544.0,
      "learning_rate": 2.978070175438596e-07,
      "logits/chosen": -0.642772376537323,
      "logits/rejected": -0.6487798690795898,
      "logps/chosen": -1057.9833984375,
      "logps/rejected": -887.0198974609375,
      "loss": 0.6879,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.004872702062129974,
      "rewards/margins": 0.011089133098721504,
      "rewards/rejected": -0.006216429173946381,
      "step": 1764
    },
    {
      "epoch": 0.4642522481053445,
      "grad_norm": 528.0,
      "learning_rate": 2.976608187134503e-07,
      "logits/chosen": -0.6633506417274475,
      "logits/rejected": -0.6542232632637024,
      "logps/chosen": -1251.70849609375,
      "logps/rejected": -1175.127685546875,
      "loss": 0.6836,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.005678653717041016,
      "rewards/margins": 0.020075131207704544,
      "rewards/rejected": -0.014396477490663528,
      "step": 1765
    },
    {
      "epoch": 0.46451528054053165,
      "grad_norm": 536.0,
      "learning_rate": 2.9751461988304095e-07,
      "logits/chosen": -0.6296244263648987,
      "logits/rejected": -0.634250283241272,
      "logps/chosen": -794.181640625,
      "logps/rejected": -1003.61572265625,
      "loss": 0.6854,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008261394686996937,
      "rewards/margins": 0.016489412635564804,
      "rewards/rejected": -0.008228017017245293,
      "step": 1766
    },
    {
      "epoch": 0.4647783129757188,
      "grad_norm": 532.0,
      "learning_rate": 2.9736842105263157e-07,
      "logits/chosen": -0.6497272253036499,
      "logits/rejected": -0.649389386177063,
      "logps/chosen": -1277.15185546875,
      "logps/rejected": -1073.6402587890625,
      "loss": 0.6946,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03226327896118164,
      "rewards/margins": -0.002444172278046608,
      "rewards/rejected": -0.029819106683135033,
      "step": 1767
    },
    {
      "epoch": 0.46504134541090597,
      "grad_norm": 500.0,
      "learning_rate": 2.972222222222222e-07,
      "logits/chosen": -0.6526975035667419,
      "logits/rejected": -0.6559621095657349,
      "logps/chosen": -1230.981201171875,
      "logps/rejected": -1088.984619140625,
      "loss": 0.6999,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.007756043691188097,
      "rewards/margins": -0.012718869373202324,
      "rewards/rejected": 0.004962827078998089,
      "step": 1768
    },
    {
      "epoch": 0.46530437784609313,
      "grad_norm": 564.0,
      "learning_rate": 2.9707602339181286e-07,
      "logits/chosen": -0.653294026851654,
      "logits/rejected": -0.650312602519989,
      "logps/chosen": -1279.3577880859375,
      "logps/rejected": -987.2052612304688,
      "loss": 0.7025,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.006270407699048519,
      "rewards/margins": -0.017652226611971855,
      "rewards/rejected": 0.0239226333796978,
      "step": 1769
    },
    {
      "epoch": 0.4655674102812803,
      "grad_norm": 564.0,
      "learning_rate": 2.969298245614035e-07,
      "logits/chosen": -0.6761909127235413,
      "logits/rejected": -0.6787368059158325,
      "logps/chosen": -1329.7982177734375,
      "logps/rejected": -872.6871948242188,
      "loss": 0.6976,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.02441740222275257,
      "rewards/margins": -0.007583902683109045,
      "rewards/rejected": -0.01683349721133709,
      "step": 1770
    },
    {
      "epoch": 0.46583044271646745,
      "grad_norm": 492.0,
      "learning_rate": 2.9678362573099415e-07,
      "logits/chosen": -0.6399356126785278,
      "logits/rejected": -0.6574835181236267,
      "logps/chosen": -1331.8065185546875,
      "logps/rejected": -1142.7822265625,
      "loss": 0.7,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01920595020055771,
      "rewards/margins": -0.013117790222167969,
      "rewards/rejected": -0.006088161841034889,
      "step": 1771
    },
    {
      "epoch": 0.4660934751516546,
      "grad_norm": 520.0,
      "learning_rate": 2.966374269005848e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6440368890762329,
      "logps/chosen": -844.5037841796875,
      "logps/rejected": -891.59521484375,
      "loss": 0.7056,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.007184791378676891,
      "rewards/margins": -0.024128150194883347,
      "rewards/rejected": 0.01694335974752903,
      "step": 1772
    },
    {
      "epoch": 0.46635650758684183,
      "grad_norm": 544.0,
      "learning_rate": 2.964912280701754e-07,
      "logits/chosen": -0.6521964073181152,
      "logits/rejected": -0.6447716355323792,
      "logps/chosen": -1209.3875732421875,
      "logps/rejected": -859.3702392578125,
      "loss": 0.6816,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00044126459397375584,
      "rewards/margins": 0.024045657366514206,
      "rewards/rejected": -0.023604393005371094,
      "step": 1773
    },
    {
      "epoch": 0.466619540022029,
      "grad_norm": 684.0,
      "learning_rate": 2.9634502923976607e-07,
      "logits/chosen": -0.65088951587677,
      "logits/rejected": -0.6576842665672302,
      "logps/chosen": -1109.8505859375,
      "logps/rejected": -1013.5914306640625,
      "loss": 0.6989,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.001929569523781538,
      "rewards/margins": -0.010953331366181374,
      "rewards/rejected": 0.012882900424301624,
      "step": 1774
    },
    {
      "epoch": 0.46688257245721615,
      "grad_norm": 556.0,
      "learning_rate": 2.9619883040935674e-07,
      "logits/chosen": -0.6423773765563965,
      "logits/rejected": -0.6484628915786743,
      "logps/chosen": -980.0735473632812,
      "logps/rejected": -862.7178955078125,
      "loss": 0.693,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.006611347198486328,
      "rewards/margins": 0.0007079122588038445,
      "rewards/rejected": -0.007319260388612747,
      "step": 1775
    },
    {
      "epoch": 0.4671456048924033,
      "grad_norm": 648.0,
      "learning_rate": 2.960526315789473e-07,
      "logits/chosen": -0.6309809684753418,
      "logits/rejected": NaN,
      "logps/chosen": -841.2230834960938,
      "logps/rejected": -593.9633178710938,
      "loss": 0.6907,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.005535125732421875,
      "rewards/margins": 0.005116940010339022,
      "rewards/rejected": 0.00041818595491349697,
      "step": 1776
    },
    {
      "epoch": 0.46740863732759047,
      "grad_norm": 532.0,
      "learning_rate": 2.95906432748538e-07,
      "logits/chosen": -0.6454187035560608,
      "logits/rejected": -0.6433311104774475,
      "logps/chosen": -1150.236328125,
      "logps/rejected": -1104.3626708984375,
      "loss": 0.6963,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.018450496718287468,
      "rewards/margins": -0.005487919319421053,
      "rewards/rejected": -0.012962580658495426,
      "step": 1777
    },
    {
      "epoch": 0.46767166976277763,
      "grad_norm": 490.0,
      "learning_rate": 2.9576023391812865e-07,
      "logits/chosen": -0.6372265815734863,
      "logits/rejected": NaN,
      "logps/chosen": -1241.6126708984375,
      "logps/rejected": -633.3027954101562,
      "loss": 0.6895,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01631774939596653,
      "rewards/margins": 0.00812234915792942,
      "rewards/rejected": 0.00819540023803711,
      "step": 1778
    },
    {
      "epoch": 0.4679347021979648,
      "grad_norm": 478.0,
      "learning_rate": 2.9561403508771927e-07,
      "logits/chosen": -0.6677201986312866,
      "logits/rejected": -0.6808623671531677,
      "logps/chosen": -986.15625,
      "logps/rejected": -559.191162109375,
      "loss": 0.6784,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.029324911534786224,
      "rewards/margins": 0.03057389333844185,
      "rewards/rejected": -0.0012489802902564406,
      "step": 1779
    },
    {
      "epoch": 0.46819773463315195,
      "grad_norm": 712.0,
      "learning_rate": 2.9546783625730994e-07,
      "logits/chosen": -0.6474325656890869,
      "logits/rejected": -0.6471437215805054,
      "logps/chosen": -1407.760986328125,
      "logps/rejected": -1232.107666015625,
      "loss": 0.7045,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.005175399594008923,
      "rewards/margins": -0.021302413195371628,
      "rewards/rejected": 0.026477813720703125,
      "step": 1780
    },
    {
      "epoch": 0.4684607670683391,
      "grad_norm": 480.0,
      "learning_rate": 2.9532163742690056e-07,
      "logits/chosen": -0.663853645324707,
      "logits/rejected": -0.6708099842071533,
      "logps/chosen": -1119.2904052734375,
      "logps/rejected": -909.5594482421875,
      "loss": 0.6876,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.02930288389325142,
      "rewards/margins": 0.011736487969756126,
      "rewards/rejected": 0.017566395923495293,
      "step": 1781
    },
    {
      "epoch": 0.4687237995035263,
      "grad_norm": 612.0,
      "learning_rate": 2.951754385964912e-07,
      "logits/chosen": -0.6675325632095337,
      "logits/rejected": -0.6606196165084839,
      "logps/chosen": -1074.317626953125,
      "logps/rejected": -1235.976806640625,
      "loss": 0.674,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.022995375096797943,
      "rewards/margins": 0.03957729786634445,
      "rewards/rejected": -0.01658191718161106,
      "step": 1782
    },
    {
      "epoch": 0.46898683193871343,
      "grad_norm": 588.0,
      "learning_rate": 2.9502923976608186e-07,
      "logits/chosen": -0.6504598259925842,
      "logits/rejected": -0.6579604148864746,
      "logps/chosen": -1300.008544921875,
      "logps/rejected": -1021.7842407226562,
      "loss": 0.6979,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.011673451401293278,
      "rewards/margins": -0.007756137289106846,
      "rewards/rejected": -0.003917313646525145,
      "step": 1783
    },
    {
      "epoch": 0.4692498643739006,
      "grad_norm": 568.0,
      "learning_rate": 2.9488304093567253e-07,
      "logits/chosen": -0.6591647863388062,
      "logits/rejected": -0.665442943572998,
      "logps/chosen": -1342.1409912109375,
      "logps/rejected": -1385.978759765625,
      "loss": 0.6808,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0064184172078967094,
      "rewards/margins": 0.025343608111143112,
      "rewards/rejected": -0.031762029975652695,
      "step": 1784
    },
    {
      "epoch": 0.46951289680908775,
      "grad_norm": 604.0,
      "learning_rate": 2.947368421052631e-07,
      "logits/chosen": -0.6734040379524231,
      "logits/rejected": NaN,
      "logps/chosen": -1251.65087890625,
      "logps/rejected": -924.3553466796875,
      "loss": 0.6947,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.007087946869432926,
      "rewards/margins": -0.00205378420650959,
      "rewards/rejected": -0.005034160800278187,
      "step": 1785
    },
    {
      "epoch": 0.4697759292442749,
      "grad_norm": 450.0,
      "learning_rate": 2.9459064327485377e-07,
      "logits/chosen": -0.6456972360610962,
      "logits/rejected": -0.6505695581436157,
      "logps/chosen": -1338.6202392578125,
      "logps/rejected": -1065.3253173828125,
      "loss": 0.6983,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01780528947710991,
      "rewards/margins": -0.01000881101936102,
      "rewards/rejected": -0.007796478923410177,
      "step": 1786
    },
    {
      "epoch": 0.4700389616794621,
      "grad_norm": 568.0,
      "learning_rate": 2.9444444444444444e-07,
      "logits/chosen": -0.6338911056518555,
      "logits/rejected": NaN,
      "logps/chosen": -1419.0115966796875,
      "logps/rejected": -1071.7493896484375,
      "loss": 0.7001,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.015894509851932526,
      "rewards/margins": -0.013010216876864433,
      "rewards/rejected": -0.002884292509406805,
      "step": 1787
    },
    {
      "epoch": 0.47030199411464924,
      "grad_norm": 528.0,
      "learning_rate": 2.942982456140351e-07,
      "logits/chosen": -0.6642760038375854,
      "logits/rejected": -0.672111988067627,
      "logps/chosen": -1173.1505126953125,
      "logps/rejected": -1093.6329345703125,
      "loss": 0.6846,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.02199420891702175,
      "rewards/margins": 0.01856827922165394,
      "rewards/rejected": 0.003425931092351675,
      "step": 1788
    },
    {
      "epoch": 0.47056502654983645,
      "grad_norm": 448.0,
      "learning_rate": 2.941520467836257e-07,
      "logits/chosen": -0.6500630974769592,
      "logits/rejected": -0.6550520062446594,
      "logps/chosen": -931.7738037109375,
      "logps/rejected": -856.9249267578125,
      "loss": 0.6981,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.016500089317560196,
      "rewards/margins": -0.009417915716767311,
      "rewards/rejected": -0.007082178257405758,
      "step": 1789
    },
    {
      "epoch": 0.4708280589850236,
      "grad_norm": 528.0,
      "learning_rate": 2.9400584795321636e-07,
      "logits/chosen": -0.6149068474769592,
      "logits/rejected": -0.6289595365524292,
      "logps/chosen": -945.0551147460938,
      "logps/rejected": -787.9215698242188,
      "loss": 0.6977,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0035647391341626644,
      "rewards/margins": -0.008606623858213425,
      "rewards/rejected": 0.0050418865866959095,
      "step": 1790
    },
    {
      "epoch": 0.4710910914202108,
      "grad_norm": 608.0,
      "learning_rate": 2.9385964912280703e-07,
      "logits/chosen": -0.6366475820541382,
      "logits/rejected": -0.6452856659889221,
      "logps/chosen": -1277.370849609375,
      "logps/rejected": -874.3472900390625,
      "loss": 0.7017,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.02467336691915989,
      "rewards/margins": -0.016373446211218834,
      "rewards/rejected": -0.00829992350190878,
      "step": 1791
    },
    {
      "epoch": 0.47135412385539793,
      "grad_norm": 540.0,
      "learning_rate": 2.9371345029239765e-07,
      "logits/chosen": -0.6390343904495239,
      "logits/rejected": -0.6450300216674805,
      "logps/chosen": -849.2640380859375,
      "logps/rejected": -976.1331787109375,
      "loss": 0.6933,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0015403754077851772,
      "rewards/margins": 0.0003262516111135483,
      "rewards/rejected": 0.0012141242623329163,
      "step": 1792
    },
    {
      "epoch": 0.4716171562905851,
      "grad_norm": 532.0,
      "learning_rate": 2.935672514619883e-07,
      "logits/chosen": -0.6517982482910156,
      "logits/rejected": -0.6473969221115112,
      "logps/chosen": -967.3961791992188,
      "logps/rejected": -812.4448852539062,
      "loss": 0.6982,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.004604817368090153,
      "rewards/margins": -0.008590985089540482,
      "rewards/rejected": 0.01319580152630806,
      "step": 1793
    },
    {
      "epoch": 0.47188018872577225,
      "grad_norm": 840.0,
      "learning_rate": 2.9342105263157894e-07,
      "logits/chosen": -0.6706224679946899,
      "logits/rejected": -0.6612798571586609,
      "logps/chosen": -1703.2120361328125,
      "logps/rejected": -1282.379150390625,
      "loss": 0.7053,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.012968636117875576,
      "rewards/margins": -0.022313402965664864,
      "rewards/rejected": 0.009344767779111862,
      "step": 1794
    },
    {
      "epoch": 0.4721432211609594,
      "grad_norm": 504.0,
      "learning_rate": 2.9327485380116956e-07,
      "logits/chosen": -0.6475629210472107,
      "logits/rejected": -0.6557979583740234,
      "logps/chosen": -1117.9208984375,
      "logps/rejected": -978.5523681640625,
      "loss": 0.7009,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.013166428543627262,
      "rewards/margins": -0.014499379321932793,
      "rewards/rejected": 0.0013329512439668179,
      "step": 1795
    },
    {
      "epoch": 0.4724062535961466,
      "grad_norm": 548.0,
      "learning_rate": 2.9312865497076023e-07,
      "logits/chosen": -0.635291576385498,
      "logits/rejected": -0.645921528339386,
      "logps/chosen": -1072.59716796875,
      "logps/rejected": -881.6173095703125,
      "loss": 0.6898,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0068543441593647,
      "rewards/margins": 0.007274341303855181,
      "rewards/rejected": -0.0004199983086436987,
      "step": 1796
    },
    {
      "epoch": 0.47266928603133374,
      "grad_norm": 484.0,
      "learning_rate": 2.929824561403509e-07,
      "logits/chosen": -0.6863420009613037,
      "logits/rejected": NaN,
      "logps/chosen": -1321.62890625,
      "logps/rejected": -1023.9210815429688,
      "loss": 0.6868,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.005558870267122984,
      "rewards/margins": 0.01336364634335041,
      "rewards/rejected": -0.0078047760762274265,
      "step": 1797
    },
    {
      "epoch": 0.4729323184665209,
      "grad_norm": 648.0,
      "learning_rate": 2.928362573099415e-07,
      "logits/chosen": -0.6455439329147339,
      "logits/rejected": -0.6408949494361877,
      "logps/chosen": -1698.8209228515625,
      "logps/rejected": -1098.5445556640625,
      "loss": 0.6871,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009779835119843483,
      "rewards/margins": 0.012622738257050514,
      "rewards/rejected": -0.022402575239539146,
      "step": 1798
    },
    {
      "epoch": 0.47319535090170806,
      "grad_norm": 636.0,
      "learning_rate": 2.9269005847953215e-07,
      "logits/chosen": -0.6542583703994751,
      "logits/rejected": -0.653791606426239,
      "logps/chosen": -1318.850341796875,
      "logps/rejected": -1280.7203369140625,
      "loss": 0.6956,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.013795185834169388,
      "rewards/margins": -0.0042349835857748985,
      "rewards/rejected": -0.009560203179717064,
      "step": 1799
    },
    {
      "epoch": 0.4734583833368952,
      "grad_norm": 488.0,
      "learning_rate": 2.925438596491228e-07,
      "logits/chosen": -0.6350343227386475,
      "logits/rejected": -0.6284515857696533,
      "logps/chosen": -1122.866455078125,
      "logps/rejected": -983.68212890625,
      "loss": 0.6936,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.01066126860678196,
      "rewards/margins": 0.0007036690367385745,
      "rewards/rejected": -0.011364937759935856,
      "step": 1800
    },
    {
      "epoch": 0.4737214157720824,
      "grad_norm": 446.0,
      "learning_rate": 2.9239766081871344e-07,
      "logits/chosen": -0.6348352432250977,
      "logits/rejected": -0.6270288228988647,
      "logps/chosen": -785.98779296875,
      "logps/rejected": -734.1560668945312,
      "loss": 0.6947,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.015825368463993073,
      "rewards/margins": -0.0029264455661177635,
      "rewards/rejected": 0.018751811236143112,
      "step": 1801
    },
    {
      "epoch": 0.47398444820726954,
      "grad_norm": 536.0,
      "learning_rate": 2.9225146198830406e-07,
      "logits/chosen": -0.6568096876144409,
      "logits/rejected": -0.6591812372207642,
      "logps/chosen": -927.66845703125,
      "logps/rejected": -925.4268798828125,
      "loss": 0.7076,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.01824512518942356,
      "rewards/margins": -0.0283387191593647,
      "rewards/rejected": 0.01009359396994114,
      "step": 1802
    },
    {
      "epoch": 0.4742474806424567,
      "grad_norm": 400.0,
      "learning_rate": 2.9210526315789473e-07,
      "logits/chosen": -0.6376814842224121,
      "logits/rejected": -0.6326507925987244,
      "logps/chosen": -765.2332153320312,
      "logps/rejected": -788.14990234375,
      "loss": 0.6976,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.0008561133872717619,
      "rewards/margins": -0.008592700585722923,
      "rewards/rejected": 0.009448814205825329,
      "step": 1803
    },
    {
      "epoch": 0.4745105130776439,
      "grad_norm": 454.0,
      "learning_rate": 2.9195906432748535e-07,
      "logits/chosen": -0.6694150567054749,
      "logits/rejected": -0.6746282577514648,
      "logps/chosen": -1063.0855712890625,
      "logps/rejected": -749.8662109375,
      "loss": 0.7005,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.005515670869499445,
      "rewards/margins": -0.01404027920216322,
      "rewards/rejected": 0.019555948674678802,
      "step": 1804
    },
    {
      "epoch": 0.4747735455128311,
      "grad_norm": 560.0,
      "learning_rate": 2.91812865497076e-07,
      "logits/chosen": -0.6439559459686279,
      "logits/rejected": -0.6556494235992432,
      "logps/chosen": -1203.4208984375,
      "logps/rejected": -945.3441772460938,
      "loss": 0.7119,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.005289220251142979,
      "rewards/margins": -0.03597264364361763,
      "rewards/rejected": 0.03068341687321663,
      "step": 1805
    },
    {
      "epoch": 0.47503657794801823,
      "grad_norm": 648.0,
      "learning_rate": 2.916666666666667e-07,
      "logits/chosen": -0.6568448543548584,
      "logits/rejected": -0.6576229333877563,
      "logps/chosen": -1423.32275390625,
      "logps/rejected": -1158.80810546875,
      "loss": 0.7088,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.010723496787250042,
      "rewards/margins": -0.02970409393310547,
      "rewards/rejected": 0.01898059993982315,
      "step": 1806
    },
    {
      "epoch": 0.4752996103832054,
      "grad_norm": 448.0,
      "learning_rate": 2.9152046783625726e-07,
      "logits/chosen": -0.6260783076286316,
      "logits/rejected": -0.6419054865837097,
      "logps/chosen": -855.2042236328125,
      "logps/rejected": -748.2398071289062,
      "loss": 0.7003,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.009462546557188034,
      "rewards/margins": -0.013411044143140316,
      "rewards/rejected": 0.003948497120290995,
      "step": 1807
    },
    {
      "epoch": 0.47556264281839256,
      "grad_norm": 564.0,
      "learning_rate": 2.9137426900584794e-07,
      "logits/chosen": -0.6540557146072388,
      "logits/rejected": -0.6598162651062012,
      "logps/chosen": -1491.27001953125,
      "logps/rejected": -1161.3045654296875,
      "loss": 0.6945,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008018016815185547,
      "rewards/margins": -0.0019026759546250105,
      "rewards/rejected": 0.009920692071318626,
      "step": 1808
    },
    {
      "epoch": 0.4758256752535797,
      "grad_norm": 532.0,
      "learning_rate": 2.912280701754386e-07,
      "logits/chosen": -0.648090124130249,
      "logits/rejected": -0.6596598625183105,
      "logps/chosen": -1036.73681640625,
      "logps/rejected": -503.96185302734375,
      "loss": 0.6822,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.030985068529844284,
      "rewards/margins": 0.022560786455869675,
      "rewards/rejected": 0.008424281142652035,
      "step": 1809
    },
    {
      "epoch": 0.4760887076887669,
      "grad_norm": 490.0,
      "learning_rate": 2.9108187134502923e-07,
      "logits/chosen": -0.6641566157341003,
      "logits/rejected": -0.6671048998832703,
      "logps/chosen": -1087.9404296875,
      "logps/rejected": -886.5636596679688,
      "loss": 0.6893,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01932506449520588,
      "rewards/margins": 0.008591176010668278,
      "rewards/rejected": -0.027916239574551582,
      "step": 1810
    },
    {
      "epoch": 0.47635174012395404,
      "grad_norm": 660.0,
      "learning_rate": 2.9093567251461985e-07,
      "logits/chosen": -0.6495614647865295,
      "logits/rejected": -0.6518712043762207,
      "logps/chosen": -1046.674072265625,
      "logps/rejected": -780.0623168945312,
      "loss": 0.702,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.008997535333037376,
      "rewards/margins": -0.01642579957842827,
      "rewards/rejected": 0.007428263314068317,
      "step": 1811
    },
    {
      "epoch": 0.4766147725591412,
      "grad_norm": 456.0,
      "learning_rate": 2.907894736842105e-07,
      "logits/chosen": -0.6655745506286621,
      "logits/rejected": -0.6665443778038025,
      "logps/chosen": -944.050048828125,
      "logps/rejected": -648.169921875,
      "loss": 0.691,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.001863956800661981,
      "rewards/margins": 0.004950810223817825,
      "rewards/rejected": -0.0030868526082485914,
      "step": 1812
    },
    {
      "epoch": 0.47687780499432836,
      "grad_norm": 528.0,
      "learning_rate": 2.9064327485380114e-07,
      "logits/chosen": -0.6348856687545776,
      "logits/rejected": -0.6384962797164917,
      "logps/chosen": -1108.181396484375,
      "logps/rejected": -667.9494018554688,
      "loss": 0.7033,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.014425944536924362,
      "rewards/margins": -0.019651222974061966,
      "rewards/rejected": 0.005225277505815029,
      "step": 1813
    },
    {
      "epoch": 0.4771408374295155,
      "grad_norm": 516.0,
      "learning_rate": 2.904970760233918e-07,
      "logits/chosen": -0.6367051005363464,
      "logits/rejected": -0.6591222882270813,
      "logps/chosen": -1234.4622802734375,
      "logps/rejected": -937.5419921875,
      "loss": 0.6837,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01282491721212864,
      "rewards/margins": 0.01985015906393528,
      "rewards/rejected": -0.007025241386145353,
      "step": 1814
    },
    {
      "epoch": 0.4774038698647027,
      "grad_norm": 556.0,
      "learning_rate": 2.9035087719298244e-07,
      "logits/chosen": -0.6475429534912109,
      "logits/rejected": -0.6708630919456482,
      "logps/chosen": -1402.761962890625,
      "logps/rejected": -1144.0294189453125,
      "loss": 0.6851,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.010376452468335629,
      "rewards/margins": 0.017254160717129707,
      "rewards/rejected": -0.006877708714455366,
      "step": 1815
    },
    {
      "epoch": 0.47766690229988984,
      "grad_norm": 608.0,
      "learning_rate": 2.9020467836257306e-07,
      "logits/chosen": -0.6447799801826477,
      "logits/rejected": -0.6523940563201904,
      "logps/chosen": -1225.972900390625,
      "logps/rejected": -823.6788330078125,
      "loss": 0.7033,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01624922640621662,
      "rewards/margins": -0.01924281008541584,
      "rewards/rejected": 0.0029935836791992188,
      "step": 1816
    },
    {
      "epoch": 0.477929934735077,
      "grad_norm": 608.0,
      "learning_rate": 2.9005847953216373e-07,
      "logits/chosen": -0.6330475807189941,
      "logits/rejected": -0.6446561813354492,
      "logps/chosen": -1241.1241455078125,
      "logps/rejected": -1030.1220703125,
      "loss": 0.7159,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.020184136927127838,
      "rewards/margins": -0.04428844526410103,
      "rewards/rejected": 0.02410431019961834,
      "step": 1817
    },
    {
      "epoch": 0.47819296717026416,
      "grad_norm": 600.0,
      "learning_rate": 2.899122807017544e-07,
      "logits/chosen": -0.6412649154663086,
      "logits/rejected": -0.6694531440734863,
      "logps/chosen": -1154.665283203125,
      "logps/rejected": -723.9741821289062,
      "loss": 0.6897,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006085729226469994,
      "rewards/margins": 0.008241796866059303,
      "rewards/rejected": -0.0021560676395893097,
      "step": 1818
    },
    {
      "epoch": 0.4784559996054513,
      "grad_norm": 434.0,
      "learning_rate": 2.8976608187134497e-07,
      "logits/chosen": -0.6318967342376709,
      "logits/rejected": -0.6357293725013733,
      "logps/chosen": -872.426513671875,
      "logps/rejected": -598.3617553710938,
      "loss": 0.6843,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0002758018672466278,
      "rewards/margins": 0.01843252405524254,
      "rewards/rejected": -0.018708325922489166,
      "step": 1819
    },
    {
      "epoch": 0.47871903204063854,
      "grad_norm": 490.0,
      "learning_rate": 2.8961988304093564e-07,
      "logits/chosen": -0.6578152179718018,
      "logits/rejected": -0.6548578143119812,
      "logps/chosen": -1566.534912109375,
      "logps/rejected": -1127.1920166015625,
      "loss": 0.6989,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0033665660303086042,
      "rewards/margins": -0.011114406399428844,
      "rewards/rejected": 0.014480972662568092,
      "step": 1820
    },
    {
      "epoch": 0.4789820644758257,
      "grad_norm": 480.0,
      "learning_rate": 2.894736842105263e-07,
      "logits/chosen": -0.6625577211380005,
      "logits/rejected": -0.6582701802253723,
      "logps/chosen": -983.522705078125,
      "logps/rejected": -876.6278686523438,
      "loss": 0.6979,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007325839251279831,
      "rewards/margins": -0.008950901217758656,
      "rewards/rejected": 0.01627674140036106,
      "step": 1821
    },
    {
      "epoch": 0.47924509691101286,
      "grad_norm": 356.0,
      "learning_rate": 2.8932748538011693e-07,
      "logits/chosen": -0.6587927341461182,
      "logits/rejected": -0.6655193567276001,
      "logps/chosen": -742.9324951171875,
      "logps/rejected": -568.9067993164062,
      "loss": 0.6963,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 6.0797203332185745e-05,
      "rewards/margins": -0.005740785971283913,
      "rewards/rejected": 0.005801583640277386,
      "step": 1822
    },
    {
      "epoch": 0.4795081293462,
      "grad_norm": 584.0,
      "learning_rate": 2.891812865497076e-07,
      "logits/chosen": -0.6768029928207397,
      "logits/rejected": -0.6880939602851868,
      "logps/chosen": -1362.857177734375,
      "logps/rejected": -834.6278686523438,
      "loss": 0.6952,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.013925360515713692,
      "rewards/margins": -0.002866936381906271,
      "rewards/rejected": 0.0167922955006361,
      "step": 1823
    },
    {
      "epoch": 0.4797711617813872,
      "grad_norm": 492.0,
      "learning_rate": 2.8903508771929823e-07,
      "logits/chosen": -0.6684175729751587,
      "logits/rejected": NaN,
      "logps/chosen": -1033.9263916015625,
      "logps/rejected": -679.5321044921875,
      "loss": 0.6924,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0006616124883294106,
      "rewards/margins": 0.003054190892726183,
      "rewards/rejected": -0.002392576541751623,
      "step": 1824
    },
    {
      "epoch": 0.48003419421657434,
      "grad_norm": 596.0,
      "learning_rate": 2.8888888888888885e-07,
      "logits/chosen": -0.6606877446174622,
      "logits/rejected": -0.6663134098052979,
      "logps/chosen": -1011.3245239257812,
      "logps/rejected": -1024.3466796875,
      "loss": 0.6842,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.014401149936020374,
      "rewards/margins": 0.019011594355106354,
      "rewards/rejected": -0.00461044255644083,
      "step": 1825
    },
    {
      "epoch": 0.4802972266517615,
      "grad_norm": 480.0,
      "learning_rate": 2.887426900584795e-07,
      "logits/chosen": -0.6469895839691162,
      "logits/rejected": -0.6458110809326172,
      "logps/chosen": -1257.2843017578125,
      "logps/rejected": -1079.842041015625,
      "loss": 0.6843,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01253213919699192,
      "rewards/margins": 0.0187714584171772,
      "rewards/rejected": -0.00623931922018528,
      "step": 1826
    },
    {
      "epoch": 0.48056025908694866,
      "grad_norm": 500.0,
      "learning_rate": 2.885964912280702e-07,
      "logits/chosen": -0.6693687438964844,
      "logits/rejected": -0.6756365299224854,
      "logps/chosen": -733.4949951171875,
      "logps/rejected": -785.3759765625,
      "loss": 0.6959,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.009588337503373623,
      "rewards/margins": -0.0048645976930856705,
      "rewards/rejected": -0.004723739810287952,
      "step": 1827
    },
    {
      "epoch": 0.4808232915221358,
      "grad_norm": 468.0,
      "learning_rate": 2.884502923976608e-07,
      "logits/chosen": -0.6513949632644653,
      "logits/rejected": -0.6574758291244507,
      "logps/chosen": -1017.4915771484375,
      "logps/rejected": -947.55126953125,
      "loss": 0.6945,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009684229269623756,
      "rewards/margins": -0.0023901460226625204,
      "rewards/rejected": 0.01207437552511692,
      "step": 1828
    },
    {
      "epoch": 0.481086323957323,
      "grad_norm": 580.0,
      "learning_rate": 2.8830409356725143e-07,
      "logits/chosen": -0.675247311592102,
      "logits/rejected": -0.6770396828651428,
      "logps/chosen": -1070.1204833984375,
      "logps/rejected": -1162.3677978515625,
      "loss": 0.7022,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01634197309613228,
      "rewards/margins": -0.01693897321820259,
      "rewards/rejected": 0.0005969996564090252,
      "step": 1829
    },
    {
      "epoch": 0.48134935639251014,
      "grad_norm": 540.0,
      "learning_rate": 2.881578947368421e-07,
      "logits/chosen": -0.6468369960784912,
      "logits/rejected": -0.6512441635131836,
      "logps/chosen": -1224.7374267578125,
      "logps/rejected": -971.797119140625,
      "loss": 0.6937,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.022718334570527077,
      "rewards/margins": -0.0003162387292832136,
      "rewards/rejected": -0.022402096539735794,
      "step": 1830
    },
    {
      "epoch": 0.4816123888276973,
      "grad_norm": 544.0,
      "learning_rate": 2.880116959064328e-07,
      "logits/chosen": -0.6464173793792725,
      "logits/rejected": -0.6544342637062073,
      "logps/chosen": -1231.3577880859375,
      "logps/rejected": -899.8817138671875,
      "loss": 0.6937,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010151004418730736,
      "rewards/margins": 0.0003923424519598484,
      "rewards/rejected": 0.00975866336375475,
      "step": 1831
    },
    {
      "epoch": 0.48187542126288446,
      "grad_norm": 568.0,
      "learning_rate": 2.8786549707602334e-07,
      "logits/chosen": -0.6221058964729309,
      "logits/rejected": -0.6369892954826355,
      "logps/chosen": -1257.1185302734375,
      "logps/rejected": -787.44970703125,
      "loss": 0.6869,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0008512968197464943,
      "rewards/margins": 0.01378483884036541,
      "rewards/rejected": -0.012933540157973766,
      "step": 1832
    },
    {
      "epoch": 0.4821384536980716,
      "grad_norm": 524.0,
      "learning_rate": 2.87719298245614e-07,
      "logits/chosen": -0.6617661714553833,
      "logits/rejected": -0.666296124458313,
      "logps/chosen": -944.096435546875,
      "logps/rejected": -936.6666259765625,
      "loss": 0.6888,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.006727791391313076,
      "rewards/margins": 0.009414957836270332,
      "rewards/rejected": -0.002687167376279831,
      "step": 1833
    },
    {
      "epoch": 0.4824014861332588,
      "grad_norm": 510.0,
      "learning_rate": 2.875730994152047e-07,
      "logits/chosen": -0.6857980489730835,
      "logits/rejected": -0.6815484166145325,
      "logps/chosen": -1330.6309814453125,
      "logps/rejected": -967.6096801757812,
      "loss": 0.6876,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.003025483340024948,
      "rewards/margins": 0.011648607440292835,
      "rewards/rejected": -0.014674090780317783,
      "step": 1834
    },
    {
      "epoch": 0.48266451856844594,
      "grad_norm": 444.0,
      "learning_rate": 2.874269005847953e-07,
      "logits/chosen": -0.6460019946098328,
      "logits/rejected": -0.6520271301269531,
      "logps/chosen": -964.9868774414062,
      "logps/rejected": -722.1511840820312,
      "loss": 0.6897,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.002847766736522317,
      "rewards/margins": 0.007451247423887253,
      "rewards/rejected": -0.004603480454534292,
      "step": 1835
    },
    {
      "epoch": 0.48292755100363316,
      "grad_norm": 548.0,
      "learning_rate": 2.87280701754386e-07,
      "logits/chosen": -0.6505786776542664,
      "logits/rejected": -0.6478260159492493,
      "logps/chosen": -1134.0208740234375,
      "logps/rejected": -1048.8807373046875,
      "loss": 0.6893,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.001256656600162387,
      "rewards/margins": 0.008631658740341663,
      "rewards/rejected": -0.009888315573334694,
      "step": 1836
    },
    {
      "epoch": 0.4831905834388203,
      "grad_norm": 544.0,
      "learning_rate": 2.871345029239766e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6695863008499146,
      "logps/chosen": -869.2807006835938,
      "logps/rejected": -913.5459594726562,
      "loss": 0.6929,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009634542278945446,
      "rewards/margins": 0.0007258889963850379,
      "rewards/rejected": -0.010360432788729668,
      "step": 1837
    },
    {
      "epoch": 0.4834536158740075,
      "grad_norm": 470.0,
      "learning_rate": 2.869883040935672e-07,
      "logits/chosen": -0.6432141661643982,
      "logits/rejected": -0.6603997349739075,
      "logps/chosen": -916.1726684570312,
      "logps/rejected": -611.5117797851562,
      "loss": 0.6949,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.010651016607880592,
      "rewards/margins": -0.003296805080026388,
      "rewards/rejected": -0.0073542119935154915,
      "step": 1838
    },
    {
      "epoch": 0.48371664830919464,
      "grad_norm": 544.0,
      "learning_rate": 2.868421052631579e-07,
      "logits/chosen": -0.6592707633972168,
      "logits/rejected": -0.6612541675567627,
      "logps/chosen": -1276.962158203125,
      "logps/rejected": -1019.193359375,
      "loss": 0.6961,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.015533828176558018,
      "rewards/margins": -0.004879762418568134,
      "rewards/rejected": -0.010654067620635033,
      "step": 1839
    },
    {
      "epoch": 0.4839796807443818,
      "grad_norm": 608.0,
      "learning_rate": 2.8669590643274857e-07,
      "logits/chosen": -0.6544253826141357,
      "logits/rejected": -0.6544740796089172,
      "logps/chosen": -1413.8111572265625,
      "logps/rejected": -1262.355712890625,
      "loss": 0.6902,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009442711248993874,
      "rewards/margins": 0.006187725812196732,
      "rewards/rejected": 0.003254985436797142,
      "step": 1840
    },
    {
      "epoch": 0.48424271317956896,
      "grad_norm": 576.0,
      "learning_rate": 2.8654970760233914e-07,
      "logits/chosen": -0.6828075647354126,
      "logits/rejected": -0.7015039920806885,
      "logps/chosen": -1184.2152099609375,
      "logps/rejected": -668.2781372070312,
      "loss": 0.6912,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.011234188452363014,
      "rewards/margins": 0.004710197448730469,
      "rewards/rejected": -0.015944385901093483,
      "step": 1841
    },
    {
      "epoch": 0.4845057456147561,
      "grad_norm": 502.0,
      "learning_rate": 2.864035087719298e-07,
      "logits/chosen": -0.6436514258384705,
      "logits/rejected": -0.6404848098754883,
      "logps/chosen": -968.651611328125,
      "logps/rejected": -807.428955078125,
      "loss": 0.7012,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.017619038000702858,
      "rewards/margins": -0.015605449676513672,
      "rewards/rejected": -0.002013587858527899,
      "step": 1842
    },
    {
      "epoch": 0.4847687780499433,
      "grad_norm": 520.0,
      "learning_rate": 2.862573099415205e-07,
      "logits/chosen": -0.6591982841491699,
      "logits/rejected": -0.6532934308052063,
      "logps/chosen": -1076.441162109375,
      "logps/rejected": -922.07763671875,
      "loss": 0.6936,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.001002215314656496,
      "rewards/margins": -0.00042762700468301773,
      "rewards/rejected": -0.000574588542804122,
      "step": 1843
    },
    {
      "epoch": 0.48503181048513044,
      "grad_norm": 544.0,
      "learning_rate": 2.861111111111111e-07,
      "logits/chosen": -0.6621166467666626,
      "logits/rejected": -0.6593062877655029,
      "logps/chosen": -1099.60595703125,
      "logps/rejected": -777.6648559570312,
      "loss": 0.7147,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0367252342402935,
      "rewards/margins": -0.04089012369513512,
      "rewards/rejected": 0.00416488666087389,
      "step": 1844
    },
    {
      "epoch": 0.4852948429203176,
      "grad_norm": 688.0,
      "learning_rate": 2.859649122807017e-07,
      "logits/chosen": -0.6578283905982971,
      "logits/rejected": -0.6675417423248291,
      "logps/chosen": -1528.9700927734375,
      "logps/rejected": -1144.3790283203125,
      "loss": 0.6832,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011565208435058594,
      "rewards/margins": 0.021598244085907936,
      "rewards/rejected": -0.03316344693303108,
      "step": 1845
    },
    {
      "epoch": 0.48555787535550476,
      "grad_norm": 624.0,
      "learning_rate": 2.858187134502924e-07,
      "logits/chosen": -0.6528226137161255,
      "logits/rejected": -0.6420987844467163,
      "logps/chosen": -1479.0802001953125,
      "logps/rejected": -1205.3095703125,
      "loss": 0.7038,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.02280445024371147,
      "rewards/margins": -0.020318223163485527,
      "rewards/rejected": -0.002486229408532381,
      "step": 1846
    },
    {
      "epoch": 0.4858209077906919,
      "grad_norm": 600.0,
      "learning_rate": 2.85672514619883e-07,
      "logits/chosen": -0.6683857440948486,
      "logits/rejected": -0.6778947710990906,
      "logps/chosen": -1261.699951171875,
      "logps/rejected": -841.406005859375,
      "loss": 0.7078,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.021288014948368073,
      "rewards/margins": -0.028952838853001595,
      "rewards/rejected": 0.007664823904633522,
      "step": 1847
    },
    {
      "epoch": 0.4860839402258791,
      "grad_norm": 486.0,
      "learning_rate": 2.855263157894737e-07,
      "logits/chosen": -0.6405225992202759,
      "logits/rejected": -0.6511330604553223,
      "logps/chosen": -994.65087890625,
      "logps/rejected": -740.9539794921875,
      "loss": 0.6945,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.013758469372987747,
      "rewards/margins": -0.002098178956657648,
      "rewards/rejected": -0.011660289950668812,
      "step": 1848
    },
    {
      "epoch": 0.48634697266106625,
      "grad_norm": 684.0,
      "learning_rate": 2.853801169590643e-07,
      "logits/chosen": -0.6524016857147217,
      "logits/rejected": -0.659895658493042,
      "logps/chosen": -1407.100830078125,
      "logps/rejected": -778.1799926757812,
      "loss": 0.6927,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.006706619169563055,
      "rewards/margins": 0.0017669673543423414,
      "rewards/rejected": 0.00493965158239007,
      "step": 1849
    },
    {
      "epoch": 0.4866100050962534,
      "grad_norm": 820.0,
      "learning_rate": 2.852339181286549e-07,
      "logits/chosen": -0.6592417359352112,
      "logits/rejected": -0.6649070382118225,
      "logps/chosen": -1824.054931640625,
      "logps/rejected": -1266.971923828125,
      "loss": 0.6856,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.015205572359263897,
      "rewards/margins": 0.016460226848721504,
      "rewards/rejected": -0.001254654722288251,
      "step": 1850
    },
    {
      "epoch": 0.4868730375314406,
      "grad_norm": 604.0,
      "learning_rate": 2.850877192982456e-07,
      "logits/chosen": -0.6461887955665588,
      "logits/rejected": -0.6537363529205322,
      "logps/chosen": -1398.5296630859375,
      "logps/rejected": -1670.7130126953125,
      "loss": 0.6829,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004562235437333584,
      "rewards/margins": 0.021549846976995468,
      "rewards/rejected": -0.01698761060833931,
      "step": 1851
    },
    {
      "epoch": 0.4871360699666278,
      "grad_norm": 474.0,
      "learning_rate": 2.8494152046783627e-07,
      "logits/chosen": -0.66927170753479,
      "logits/rejected": -0.6640247106552124,
      "logps/chosen": -830.29931640625,
      "logps/rejected": -809.27783203125,
      "loss": 0.6883,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01867647096514702,
      "rewards/margins": 0.010559463873505592,
      "rewards/rejected": 0.008117008954286575,
      "step": 1852
    },
    {
      "epoch": 0.48739910240181494,
      "grad_norm": 680.0,
      "learning_rate": 2.847953216374269e-07,
      "logits/chosen": -0.6509287357330322,
      "logits/rejected": -0.6506156921386719,
      "logps/chosen": -1155.3275146484375,
      "logps/rejected": -1312.209716796875,
      "loss": 0.7153,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.03143711015582085,
      "rewards/margins": -0.04219990223646164,
      "rewards/rejected": 0.01076278742402792,
      "step": 1853
    },
    {
      "epoch": 0.4876621348370021,
      "grad_norm": 612.0,
      "learning_rate": 2.846491228070175e-07,
      "logits/chosen": -0.6670321822166443,
      "logits/rejected": -0.6817113757133484,
      "logps/chosen": -1595.10302734375,
      "logps/rejected": -909.8359375,
      "loss": 0.7017,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.018448732793331146,
      "rewards/margins": -0.016172505915164948,
      "rewards/rejected": -0.002276229439303279,
      "step": 1854
    },
    {
      "epoch": 0.48792516727218926,
      "grad_norm": 700.0,
      "learning_rate": 2.845029239766082e-07,
      "logits/chosen": -0.6099696159362793,
      "logits/rejected": -0.613822877407074,
      "logps/chosen": -1338.67333984375,
      "logps/rejected": -1051.46240234375,
      "loss": 0.6909,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0036758435890078545,
      "rewards/margins": 0.005530547350645065,
      "rewards/rejected": -0.009206390008330345,
      "step": 1855
    },
    {
      "epoch": 0.4881881997073764,
      "grad_norm": 612.0,
      "learning_rate": 2.843567251461988e-07,
      "logits/chosen": -0.6492022275924683,
      "logits/rejected": -0.6591724753379822,
      "logps/chosen": -1438.731689453125,
      "logps/rejected": -1209.1927490234375,
      "loss": 0.694,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.012506198137998581,
      "rewards/margins": -0.0012257578782737255,
      "rewards/rejected": -0.011280441656708717,
      "step": 1856
    },
    {
      "epoch": 0.4884512321425636,
      "grad_norm": 472.0,
      "learning_rate": 2.842105263157895e-07,
      "logits/chosen": -0.6490651369094849,
      "logits/rejected": -0.6445680856704712,
      "logps/chosen": -876.1414794921875,
      "logps/rejected": -641.4680786132812,
      "loss": 0.6945,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02111797407269478,
      "rewards/margins": -0.0022550588473677635,
      "rewards/rejected": -0.01886291615664959,
      "step": 1857
    },
    {
      "epoch": 0.48871426457775075,
      "grad_norm": 572.0,
      "learning_rate": 2.840643274853801e-07,
      "logits/chosen": -0.6644777059555054,
      "logits/rejected": -0.6657312512397766,
      "logps/chosen": -945.9951782226562,
      "logps/rejected": -934.7796020507812,
      "loss": 0.6879,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.015122605487704277,
      "rewards/margins": 0.011476661078631878,
      "rewards/rejected": 0.003645944409072399,
      "step": 1858
    },
    {
      "epoch": 0.4889772970129379,
      "grad_norm": 724.0,
      "learning_rate": 2.839181286549707e-07,
      "logits/chosen": -0.6298064589500427,
      "logits/rejected": -0.631138026714325,
      "logps/chosen": -1239.59130859375,
      "logps/rejected": -1199.579833984375,
      "loss": 0.7056,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01740284077823162,
      "rewards/margins": -0.023708535358309746,
      "rewards/rejected": 0.006305694580078125,
      "step": 1859
    },
    {
      "epoch": 0.48924032944812507,
      "grad_norm": 504.0,
      "learning_rate": 2.837719298245614e-07,
      "logits/chosen": -0.6474272012710571,
      "logits/rejected": -0.6503764390945435,
      "logps/chosen": -1220.0452880859375,
      "logps/rejected": -700.068115234375,
      "loss": 0.6959,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.007969953119754791,
      "rewards/margins": -0.004921627230942249,
      "rewards/rejected": -0.003048325190320611,
      "step": 1860
    },
    {
      "epoch": 0.4895033618833122,
      "grad_norm": 426.0,
      "learning_rate": 2.8362573099415206e-07,
      "logits/chosen": -0.6331998705863953,
      "logits/rejected": -0.639175295829773,
      "logps/chosen": -745.5870971679688,
      "logps/rejected": -676.851806640625,
      "loss": 0.7032,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.009558296762406826,
      "rewards/margins": -0.0191282257437706,
      "rewards/rejected": 0.009569931775331497,
      "step": 1861
    },
    {
      "epoch": 0.4897663943184994,
      "grad_norm": 596.0,
      "learning_rate": 2.8347953216374263e-07,
      "logits/chosen": -0.6607807874679565,
      "logits/rejected": -0.6574020981788635,
      "logps/chosen": -1008.5682983398438,
      "logps/rejected": -1015.0611572265625,
      "loss": 0.6988,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.027696993201971054,
      "rewards/margins": -0.00875706598162651,
      "rewards/rejected": -0.018939925357699394,
      "step": 1862
    },
    {
      "epoch": 0.49002942675368655,
      "grad_norm": 494.0,
      "learning_rate": 2.833333333333333e-07,
      "logits/chosen": -0.6327956914901733,
      "logits/rejected": -0.6452597379684448,
      "logps/chosen": -989.2886352539062,
      "logps/rejected": -791.8114624023438,
      "loss": 0.7066,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.034468840807676315,
      "rewards/margins": -0.02620086446404457,
      "rewards/rejected": -0.008267974480986595,
      "step": 1863
    },
    {
      "epoch": 0.4902924591888737,
      "grad_norm": 676.0,
      "learning_rate": 2.83187134502924e-07,
      "logits/chosen": -0.6465547680854797,
      "logits/rejected": -0.6447956562042236,
      "logps/chosen": -1280.0582275390625,
      "logps/rejected": -1186.129150390625,
      "loss": 0.696,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.007663726806640625,
      "rewards/margins": -0.004915524274110794,
      "rewards/rejected": 0.012579251080751419,
      "step": 1864
    },
    {
      "epoch": 0.49055549162406087,
      "grad_norm": 680.0,
      "learning_rate": 2.830409356725146e-07,
      "logits/chosen": -0.6446011662483215,
      "logits/rejected": -0.6548316478729248,
      "logps/chosen": -1540.9075927734375,
      "logps/rejected": -889.8660888671875,
      "loss": 0.7057,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.013847971335053444,
      "rewards/margins": -0.023973705247044563,
      "rewards/rejected": 0.01012573204934597,
      "step": 1865
    },
    {
      "epoch": 0.49081852405924803,
      "grad_norm": 486.0,
      "learning_rate": 2.828947368421052e-07,
      "logits/chosen": -0.6575090885162354,
      "logits/rejected": -0.6609853506088257,
      "logps/chosen": -768.6578369140625,
      "logps/rejected": -804.997314453125,
      "loss": 0.7041,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.012637997046113014,
      "rewards/margins": -0.021014072000980377,
      "rewards/rejected": 0.008376073092222214,
      "step": 1866
    },
    {
      "epoch": 0.49108155649443525,
      "grad_norm": 628.0,
      "learning_rate": 2.827485380116959e-07,
      "logits/chosen": -0.6465340256690979,
      "logits/rejected": -0.6466158628463745,
      "logps/chosen": -1372.8017578125,
      "logps/rejected": -1025.6575927734375,
      "loss": 0.695,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02895507961511612,
      "rewards/margins": -0.002958584576845169,
      "rewards/rejected": -0.02599649503827095,
      "step": 1867
    },
    {
      "epoch": 0.4913445889296224,
      "grad_norm": 600.0,
      "learning_rate": 2.8260233918128656e-07,
      "logits/chosen": -0.6191537976264954,
      "logits/rejected": -0.6352415084838867,
      "logps/chosen": -1612.9990234375,
      "logps/rejected": -908.3760986328125,
      "loss": 0.688,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.023151397705078125,
      "rewards/margins": 0.011627959087491035,
      "rewards/rejected": 0.01152343861758709,
      "step": 1868
    },
    {
      "epoch": 0.49160762136480957,
      "grad_norm": 596.0,
      "learning_rate": 2.824561403508772e-07,
      "logits/chosen": -0.6646291613578796,
      "logits/rejected": -0.6678712964057922,
      "logps/chosen": -1224.210205078125,
      "logps/rejected": -912.1141357421875,
      "loss": 0.7061,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.013546179048717022,
      "rewards/margins": -0.02377901040017605,
      "rewards/rejected": 0.010232829488813877,
      "step": 1869
    },
    {
      "epoch": 0.4918706537999967,
      "grad_norm": 440.0,
      "learning_rate": 2.8230994152046785e-07,
      "logits/chosen": -0.6548486351966858,
      "logits/rejected": -0.6561270952224731,
      "logps/chosen": -841.7383422851562,
      "logps/rejected": -738.1104125976562,
      "loss": 0.6965,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0016043659998103976,
      "rewards/margins": -0.005844020284712315,
      "rewards/rejected": 0.00423965509980917,
      "step": 1870
    },
    {
      "epoch": 0.4921336862351839,
      "grad_norm": 462.0,
      "learning_rate": 2.821637426900585e-07,
      "logits/chosen": -0.6619755625724792,
      "logits/rejected": -0.6600256562232971,
      "logps/chosen": -1160.8955078125,
      "logps/rejected": -917.0265502929688,
      "loss": 0.6793,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.013917256146669388,
      "rewards/margins": 0.028673553839325905,
      "rewards/rejected": -0.014756298623979092,
      "step": 1871
    },
    {
      "epoch": 0.49239671867037105,
      "grad_norm": 540.0,
      "learning_rate": 2.820175438596491e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6472163200378418,
      "logps/chosen": -879.931884765625,
      "logps/rejected": -941.4465942382812,
      "loss": 0.6963,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.01513976976275444,
      "rewards/margins": -0.00550770852714777,
      "rewards/rejected": -0.009632062166929245,
      "step": 1872
    },
    {
      "epoch": 0.4926597511055582,
      "grad_norm": 424.0,
      "learning_rate": 2.8187134502923977e-07,
      "logits/chosen": -0.6752392649650574,
      "logits/rejected": -0.6785920858383179,
      "logps/chosen": -845.8470458984375,
      "logps/rejected": -740.473876953125,
      "loss": 0.6961,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.026928042992949486,
      "rewards/margins": -0.005372285842895508,
      "rewards/rejected": -0.021555759012699127,
      "step": 1873
    },
    {
      "epoch": 0.49292278354074537,
      "grad_norm": 412.0,
      "learning_rate": 2.8172514619883044e-07,
      "logits/chosen": -0.6417914628982544,
      "logits/rejected": -0.6366674900054932,
      "logps/chosen": -1230.6566162109375,
      "logps/rejected": -864.2032470703125,
      "loss": 0.6996,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0043433187529444695,
      "rewards/margins": -0.012382222339510918,
      "rewards/rejected": 0.016725540161132812,
      "step": 1874
    },
    {
      "epoch": 0.49318581597593253,
      "grad_norm": 604.0,
      "learning_rate": 2.81578947368421e-07,
      "logits/chosen": -0.652578592300415,
      "logits/rejected": -0.6621028780937195,
      "logps/chosen": -1388.911376953125,
      "logps/rejected": -838.713623046875,
      "loss": 0.6966,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0030132292304188013,
      "rewards/margins": -0.006015203893184662,
      "rewards/rejected": 0.009028434753417969,
      "step": 1875
    },
    {
      "epoch": 0.4934488484111197,
      "grad_norm": 504.0,
      "learning_rate": 2.814327485380117e-07,
      "logits/chosen": -0.6561107039451599,
      "logits/rejected": -0.6500544548034668,
      "logps/chosen": -1068.5196533203125,
      "logps/rejected": -1084.959228515625,
      "loss": 0.6879,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0033840169198811054,
      "rewards/margins": 0.010601234622299671,
      "rewards/rejected": -0.007217216771095991,
      "step": 1876
    },
    {
      "epoch": 0.49371188084630685,
      "grad_norm": 536.0,
      "learning_rate": 2.8128654970760235e-07,
      "logits/chosen": -0.6510651707649231,
      "logits/rejected": -0.6550567746162415,
      "logps/chosen": -928.8916625976562,
      "logps/rejected": -730.2677001953125,
      "loss": 0.6854,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004449843429028988,
      "rewards/margins": 0.0171845443546772,
      "rewards/rejected": -0.012734698131680489,
      "step": 1877
    },
    {
      "epoch": 0.493974913281494,
      "grad_norm": 592.0,
      "learning_rate": 2.8114035087719297e-07,
      "logits/chosen": -0.6756385564804077,
      "logits/rejected": -0.676039457321167,
      "logps/chosen": -1484.6573486328125,
      "logps/rejected": -1439.380615234375,
      "loss": 0.6956,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.010302351787686348,
      "rewards/margins": -0.004354763776063919,
      "rewards/rejected": 0.014657115563750267,
      "step": 1878
    },
    {
      "epoch": 0.49423794571668117,
      "grad_norm": 596.0,
      "learning_rate": 2.809941520467836e-07,
      "logits/chosen": -0.6614649891853333,
      "logits/rejected": -0.6770615577697754,
      "logps/chosen": -1016.07763671875,
      "logps/rejected": -1116.6197509765625,
      "loss": 0.6873,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.003593254368752241,
      "rewards/margins": 0.013012789189815521,
      "rewards/rejected": -0.016606047749519348,
      "step": 1879
    },
    {
      "epoch": 0.49450097815186833,
      "grad_norm": 490.0,
      "learning_rate": 2.8084795321637426e-07,
      "logits/chosen": -0.6561257243156433,
      "logits/rejected": -0.6582883596420288,
      "logps/chosen": -1271.000732421875,
      "logps/rejected": -962.6669921875,
      "loss": 0.6816,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.013942909426987171,
      "rewards/margins": 0.023793600499629974,
      "rewards/rejected": -0.009850692935287952,
      "step": 1880
    },
    {
      "epoch": 0.4947640105870555,
      "grad_norm": 476.0,
      "learning_rate": 2.807017543859649e-07,
      "logits/chosen": -0.6436923742294312,
      "logits/rejected": -0.6423931121826172,
      "logps/chosen": -903.0349731445312,
      "logps/rejected": -727.59130859375,
      "loss": 0.6924,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03095393255352974,
      "rewards/margins": 0.001864290446974337,
      "rewards/rejected": 0.02908964455127716,
      "step": 1881
    },
    {
      "epoch": 0.4950270430222427,
      "grad_norm": 512.0,
      "learning_rate": 2.8055555555555556e-07,
      "logits/chosen": -0.652387261390686,
      "logits/rejected": -0.6563988924026489,
      "logps/chosen": -1127.1405029296875,
      "logps/rejected": -1026.870361328125,
      "loss": 0.6885,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.007864571176469326,
      "rewards/margins": 0.00958337727934122,
      "rewards/rejected": -0.017447948455810547,
      "step": 1882
    },
    {
      "epoch": 0.49529007545742987,
      "grad_norm": 448.0,
      "learning_rate": 2.8040935672514623e-07,
      "logits/chosen": -0.6768661737442017,
      "logits/rejected": -0.6855399012565613,
      "logps/chosen": -891.2470092773438,
      "logps/rejected": -907.0704345703125,
      "loss": 0.7097,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.028179552406072617,
      "rewards/margins": -0.0319613479077816,
      "rewards/rejected": 0.0037817959673702717,
      "step": 1883
    },
    {
      "epoch": 0.49555310789261703,
      "grad_norm": 490.0,
      "learning_rate": 2.802631578947368e-07,
      "logits/chosen": -0.6434841752052307,
      "logits/rejected": -0.6354549527168274,
      "logps/chosen": -905.8961181640625,
      "logps/rejected": -953.07763671875,
      "loss": 0.6956,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.010273647494614124,
      "rewards/margins": -0.004655836150050163,
      "rewards/rejected": -0.005617809481918812,
      "step": 1884
    },
    {
      "epoch": 0.4958161403278042,
      "grad_norm": 604.0,
      "learning_rate": 2.8011695906432747e-07,
      "logits/chosen": -0.6555191278457642,
      "logits/rejected": -0.6544124484062195,
      "logps/chosen": -1347.1483154296875,
      "logps/rejected": -1053.0050048828125,
      "loss": 0.6885,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012018396519124508,
      "rewards/margins": 0.010669517330825329,
      "rewards/rejected": 0.0013488773256540298,
      "step": 1885
    },
    {
      "epoch": 0.49607917276299135,
      "grad_norm": 544.0,
      "learning_rate": 2.7997076023391814e-07,
      "logits/chosen": -0.6615043878555298,
      "logits/rejected": -0.6667029857635498,
      "logps/chosen": -905.4093017578125,
      "logps/rejected": -659.3038940429688,
      "loss": 0.6884,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0036216748412698507,
      "rewards/margins": 0.0099671371281147,
      "rewards/rejected": -0.01358881127089262,
      "step": 1886
    },
    {
      "epoch": 0.4963422051981785,
      "grad_norm": 470.0,
      "learning_rate": 2.7982456140350876e-07,
      "logits/chosen": -0.6385102272033691,
      "logits/rejected": -0.6416151523590088,
      "logps/chosen": -929.3580322265625,
      "logps/rejected": -735.7632446289062,
      "loss": 0.6891,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02163085900247097,
      "rewards/margins": 0.009125137701630592,
      "rewards/rejected": 0.012505722232162952,
      "step": 1887
    },
    {
      "epoch": 0.49660523763336567,
      "grad_norm": 528.0,
      "learning_rate": 2.796783625730994e-07,
      "logits/chosen": -0.6366258859634399,
      "logits/rejected": -0.6524959206581116,
      "logps/chosen": -1077.729248046875,
      "logps/rejected": -1121.4029541015625,
      "loss": 0.6996,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.008256340399384499,
      "rewards/margins": -0.011772489175200462,
      "rewards/rejected": 0.003516149939969182,
      "step": 1888
    },
    {
      "epoch": 0.49686827006855283,
      "grad_norm": 580.0,
      "learning_rate": 2.7953216374269006e-07,
      "logits/chosen": -0.6507888436317444,
      "logits/rejected": -0.6567156910896301,
      "logps/chosen": -1340.0606689453125,
      "logps/rejected": -905.9859008789062,
      "loss": 0.6979,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.009480667300522327,
      "rewards/margins": -0.008621596731245518,
      "rewards/rejected": 0.018102265894412994,
      "step": 1889
    },
    {
      "epoch": 0.49713130250374,
      "grad_norm": 636.0,
      "learning_rate": 2.793859649122807e-07,
      "logits/chosen": -0.6506417393684387,
      "logits/rejected": -0.6531330943107605,
      "logps/chosen": -1401.50146484375,
      "logps/rejected": -925.153076171875,
      "loss": 0.7044,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.01408691331744194,
      "rewards/margins": -0.021419668570160866,
      "rewards/rejected": 0.007332754321396351,
      "step": 1890
    },
    {
      "epoch": 0.49739433493892715,
      "grad_norm": 520.0,
      "learning_rate": 2.7923976608187135e-07,
      "logits/chosen": -0.6757326722145081,
      "logits/rejected": -0.662689745426178,
      "logps/chosen": -885.5735473632812,
      "logps/rejected": -638.7526245117188,
      "loss": 0.6918,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0018529891967773438,
      "rewards/margins": 0.004175853915512562,
      "rewards/rejected": -0.006028842180967331,
      "step": 1891
    },
    {
      "epoch": 0.4976573673741143,
      "grad_norm": 608.0,
      "learning_rate": 2.7909356725146197e-07,
      "logits/chosen": -0.6402465105056763,
      "logits/rejected": -0.6464112997055054,
      "logps/chosen": -1397.0921630859375,
      "logps/rejected": -1248.050048828125,
      "loss": 0.6853,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0075168609619140625,
      "rewards/margins": 0.017019078135490417,
      "rewards/rejected": -0.009502220898866653,
      "step": 1892
    },
    {
      "epoch": 0.4979203998093015,
      "grad_norm": 540.0,
      "learning_rate": 2.789473684210526e-07,
      "logits/chosen": -0.6591407060623169,
      "logits/rejected": NaN,
      "logps/chosen": -1190.00439453125,
      "logps/rejected": -896.3494873046875,
      "loss": 0.6991,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.007326602004468441,
      "rewards/margins": -0.010904120281338692,
      "rewards/rejected": 0.0035775182768702507,
      "step": 1893
    },
    {
      "epoch": 0.49818343224448863,
      "grad_norm": 584.0,
      "learning_rate": 2.7880116959064326e-07,
      "logits/chosen": -0.6585073471069336,
      "logits/rejected": -0.6566205024719238,
      "logps/chosen": -925.9328002929688,
      "logps/rejected": -967.4778442382812,
      "loss": 0.703,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.005050087347626686,
      "rewards/margins": -0.01863069459795952,
      "rewards/rejected": 0.023680781945586205,
      "step": 1894
    },
    {
      "epoch": 0.4984464646796758,
      "grad_norm": 576.0,
      "learning_rate": 2.7865497076023393e-07,
      "logits/chosen": -0.6546610593795776,
      "logits/rejected": -0.6510019302368164,
      "logps/chosen": -1067.8392333984375,
      "logps/rejected": -986.32421875,
      "loss": 0.6989,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.005437755491584539,
      "rewards/margins": -0.011150168254971504,
      "rewards/rejected": 0.005712414626032114,
      "step": 1895
    },
    {
      "epoch": 0.49870949711486295,
      "grad_norm": 492.0,
      "learning_rate": 2.785087719298245e-07,
      "logits/chosen": -0.6481940150260925,
      "logits/rejected": -0.6672602891921997,
      "logps/chosen": -1058.578369140625,
      "logps/rejected": -909.558349609375,
      "loss": 0.6873,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0017938616219908,
      "rewards/margins": 0.012479686178267002,
      "rewards/rejected": -0.01068582572042942,
      "step": 1896
    },
    {
      "epoch": 0.4989725295500501,
      "grad_norm": 584.0,
      "learning_rate": 2.783625730994152e-07,
      "logits/chosen": -0.6674405336380005,
      "logits/rejected": NaN,
      "logps/chosen": -1496.6466064453125,
      "logps/rejected": -953.2020874023438,
      "loss": 0.6946,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011246107518672943,
      "rewards/margins": -0.0021714214235544205,
      "rewards/rejected": -0.009074687957763672,
      "step": 1897
    },
    {
      "epoch": 0.49923556198523733,
      "grad_norm": 572.0,
      "learning_rate": 2.7821637426900585e-07,
      "logits/chosen": -0.6754599809646606,
      "logits/rejected": -0.6955508589744568,
      "logps/chosen": -1078.1348876953125,
      "logps/rejected": -614.0076904296875,
      "loss": 0.6924,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.006834316998720169,
      "rewards/margins": 0.0024425508454442024,
      "rewards/rejected": 0.004391765687614679,
      "step": 1898
    },
    {
      "epoch": 0.4994985944204245,
      "grad_norm": 572.0,
      "learning_rate": 2.7807017543859647e-07,
      "logits/chosen": -0.6771205067634583,
      "logits/rejected": -0.6761838793754578,
      "logps/chosen": -1213.24365234375,
      "logps/rejected": -772.5914306640625,
      "loss": 0.6922,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0008996967226266861,
      "rewards/margins": 0.002879332983866334,
      "rewards/rejected": -0.001979636028409004,
      "step": 1899
    },
    {
      "epoch": 0.49976162685561165,
      "grad_norm": 532.0,
      "learning_rate": 2.7792397660818714e-07,
      "logits/chosen": -0.6606839299201965,
      "logits/rejected": -0.672735333442688,
      "logps/chosen": -1298.3768310546875,
      "logps/rejected": -935.056884765625,
      "loss": 0.6863,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00270423898473382,
      "rewards/margins": 0.014222240075469017,
      "rewards/rejected": -0.011518002487719059,
      "step": 1900
    },
    {
      "epoch": 0.5000246592907988,
      "grad_norm": 580.0,
      "learning_rate": 2.7777777777777776e-07,
      "logits/chosen": -0.6607822775840759,
      "logits/rejected": -0.6656407713890076,
      "logps/chosen": -1117.3250732421875,
      "logps/rejected": -996.4796142578125,
      "loss": 0.7003,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.007667399011552334,
      "rewards/margins": -0.013790512457489967,
      "rewards/rejected": 0.021457912400364876,
      "step": 1901
    },
    {
      "epoch": 0.500287691725986,
      "grad_norm": 532.0,
      "learning_rate": 2.776315789473684e-07,
      "logits/chosen": -0.6603703498840332,
      "logits/rejected": -0.6612821817398071,
      "logps/chosen": -1322.3504638671875,
      "logps/rejected": -1101.1923828125,
      "loss": 0.6938,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.004679870791733265,
      "rewards/margins": -0.0005035400390625,
      "rewards/rejected": -0.004176330752670765,
      "step": 1902
    },
    {
      "epoch": 0.5005507241611731,
      "grad_norm": 556.0,
      "learning_rate": 2.7748538011695905e-07,
      "logits/chosen": -0.6556367874145508,
      "logits/rejected": -0.6616585850715637,
      "logps/chosen": -1047.15966796875,
      "logps/rejected": -721.9739990234375,
      "loss": 0.6935,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0060933125205338,
      "rewards/margins": 0.0001628883183002472,
      "rewards/rejected": 0.005930422805249691,
      "step": 1903
    },
    {
      "epoch": 0.5008137565963603,
      "grad_norm": 532.0,
      "learning_rate": 2.773391812865497e-07,
      "logits/chosen": -0.6430690884590149,
      "logits/rejected": -0.6480003595352173,
      "logps/chosen": -1061.3348388671875,
      "logps/rejected": -980.3656005859375,
      "loss": 0.6964,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0041609760373830795,
      "rewards/margins": -0.006338499486446381,
      "rewards/rejected": 0.0021775243803858757,
      "step": 1904
    },
    {
      "epoch": 0.5010767890315474,
      "grad_norm": 472.0,
      "learning_rate": 2.771929824561403e-07,
      "logits/chosen": -0.652133584022522,
      "logits/rejected": -0.6609929203987122,
      "logps/chosen": -1008.4940185546875,
      "logps/rejected": -849.9945068359375,
      "loss": 0.688,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0009932524990290403,
      "rewards/margins": 0.010678673163056374,
      "rewards/rejected": -0.009685421362519264,
      "step": 1905
    },
    {
      "epoch": 0.5013398214667346,
      "grad_norm": 474.0,
      "learning_rate": 2.7704678362573096e-07,
      "logits/chosen": -0.6883870959281921,
      "logits/rejected": -0.6803078651428223,
      "logps/chosen": -1083.9072265625,
      "logps/rejected": -995.77587890625,
      "loss": 0.6837,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00789098720997572,
      "rewards/margins": 0.019473837688565254,
      "rewards/rejected": -0.011582852341234684,
      "step": 1906
    },
    {
      "epoch": 0.5016028539019218,
      "grad_norm": 580.0,
      "learning_rate": 2.7690058479532164e-07,
      "logits/chosen": -0.6607139706611633,
      "logits/rejected": NaN,
      "logps/chosen": -1690.74365234375,
      "logps/rejected": -1118.7100830078125,
      "loss": 0.6948,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0017347324173897505,
      "rewards/margins": -0.0029383643995970488,
      "rewards/rejected": 0.0012036319822072983,
      "step": 1907
    },
    {
      "epoch": 0.5018658863371089,
      "grad_norm": 510.0,
      "learning_rate": 2.767543859649123e-07,
      "logits/chosen": -0.6640268564224243,
      "logits/rejected": NaN,
      "logps/chosen": -1107.2308349609375,
      "logps/rejected": -783.7469482421875,
      "loss": 0.6964,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.02666015736758709,
      "rewards/margins": -0.006208132952451706,
      "rewards/rejected": -0.020452022552490234,
      "step": 1908
    },
    {
      "epoch": 0.5021289187722962,
      "grad_norm": 592.0,
      "learning_rate": 2.766081871345029e-07,
      "logits/chosen": -0.658214807510376,
      "logits/rejected": -0.6749385595321655,
      "logps/chosen": -1325.728515625,
      "logps/rejected": -984.0426025390625,
      "loss": 0.6871,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.019716741517186165,
      "rewards/margins": 0.012782859615981579,
      "rewards/rejected": 0.0069338795728981495,
      "step": 1909
    },
    {
      "epoch": 0.5023919512074833,
      "grad_norm": 600.0,
      "learning_rate": 2.7646198830409355e-07,
      "logits/chosen": -0.645201563835144,
      "logits/rejected": -0.653975784778595,
      "logps/chosen": -944.699951171875,
      "logps/rejected": -734.710693359375,
      "loss": 0.6727,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.039148423820734024,
      "rewards/margins": 0.041932009160518646,
      "rewards/rejected": -0.0027835846412926912,
      "step": 1910
    },
    {
      "epoch": 0.5026549836426705,
      "grad_norm": 520.0,
      "learning_rate": 2.763157894736842e-07,
      "logits/chosen": -0.6493076086044312,
      "logits/rejected": -0.6480732560157776,
      "logps/chosen": -1348.994873046875,
      "logps/rejected": -901.6287841796875,
      "loss": 0.6927,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0056160916574299335,
      "rewards/margins": 0.001592158921994269,
      "rewards/rejected": 0.004023933783173561,
      "step": 1911
    },
    {
      "epoch": 0.5029180160778576,
      "grad_norm": 524.0,
      "learning_rate": 2.7616959064327484e-07,
      "logits/chosen": -0.6109946370124817,
      "logits/rejected": -0.6237181425094604,
      "logps/chosen": -994.0099487304688,
      "logps/rejected": -790.8946533203125,
      "loss": 0.6908,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.009189317934215069,
      "rewards/margins": 0.005948927253484726,
      "rewards/rejected": -0.01513824425637722,
      "step": 1912
    },
    {
      "epoch": 0.5031810485130448,
      "grad_norm": 494.0,
      "learning_rate": 2.760233918128655e-07,
      "logits/chosen": -0.6708120703697205,
      "logits/rejected": -0.6640723943710327,
      "logps/chosen": -994.5240478515625,
      "logps/rejected": -778.47265625,
      "loss": 0.6943,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0028402330353856087,
      "rewards/margins": -0.0018263820093125105,
      "rewards/rejected": 0.0046666148118674755,
      "step": 1913
    },
    {
      "epoch": 0.5034440809482319,
      "grad_norm": 482.0,
      "learning_rate": 2.7587719298245614e-07,
      "logits/chosen": -0.6314698457717896,
      "logits/rejected": -0.6225214600563049,
      "logps/chosen": -1084.545654296875,
      "logps/rejected": -724.6240234375,
      "loss": 0.6885,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0023210514336824417,
      "rewards/margins": 0.010311889462172985,
      "rewards/rejected": -0.012632942758500576,
      "step": 1914
    },
    {
      "epoch": 0.5037071133834191,
      "grad_norm": 628.0,
      "learning_rate": 2.7573099415204676e-07,
      "logits/chosen": -0.6608688831329346,
      "logits/rejected": -0.6645888090133667,
      "logps/chosen": -1606.1763916015625,
      "logps/rejected": -799.3816528320312,
      "loss": 0.6929,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -7.228786125779152e-05,
      "rewards/margins": 0.0016242028214037418,
      "rewards/rejected": -0.0016964912647381425,
      "step": 1915
    },
    {
      "epoch": 0.5039701458186062,
      "grad_norm": 516.0,
      "learning_rate": 2.7558479532163743e-07,
      "logits/chosen": -0.6637864112854004,
      "logits/rejected": -0.6562219858169556,
      "logps/chosen": -944.6165161132812,
      "logps/rejected": -1263.990478515625,
      "loss": 0.7013,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0034149172715842724,
      "rewards/margins": -0.014959147199988365,
      "rewards/rejected": 0.018374061211943626,
      "step": 1916
    },
    {
      "epoch": 0.5042331782537934,
      "grad_norm": 584.0,
      "learning_rate": 2.754385964912281e-07,
      "logits/chosen": -0.6442443132400513,
      "logits/rejected": -0.6341903209686279,
      "logps/chosen": -994.673583984375,
      "logps/rejected": -982.5239868164062,
      "loss": 0.6996,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0040658945217728615,
      "rewards/margins": -0.011987877078354359,
      "rewards/rejected": 0.007921982556581497,
      "step": 1917
    },
    {
      "epoch": 0.5044962106889805,
      "grad_norm": 568.0,
      "learning_rate": 2.7529239766081867e-07,
      "logits/chosen": -0.6839747428894043,
      "logits/rejected": -0.6942832469940186,
      "logps/chosen": -985.7747802734375,
      "logps/rejected": -685.62451171875,
      "loss": 0.7058,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.014484167098999023,
      "rewards/margins": -0.024450447410345078,
      "rewards/rejected": 0.00996627938002348,
      "step": 1918
    },
    {
      "epoch": 0.5047592431241678,
      "grad_norm": 700.0,
      "learning_rate": 2.7514619883040934e-07,
      "logits/chosen": -0.66895592212677,
      "logits/rejected": -0.6484256982803345,
      "logps/chosen": -1126.36083984375,
      "logps/rejected": -1190.010498046875,
      "loss": 0.6952,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.018608378246426582,
      "rewards/margins": -0.0027329432778060436,
      "rewards/rejected": -0.01587543450295925,
      "step": 1919
    },
    {
      "epoch": 0.5050222755593549,
      "grad_norm": 454.0,
      "learning_rate": 2.75e-07,
      "logits/chosen": -0.629137396812439,
      "logits/rejected": -0.6397041082382202,
      "logps/chosen": -1089.0537109375,
      "logps/rejected": -901.2989501953125,
      "loss": 0.694,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0007368088699877262,
      "rewards/margins": -0.0010590562596917152,
      "rewards/rejected": 0.00032224738970398903,
      "step": 1920
    },
    {
      "epoch": 0.5052853079945421,
      "grad_norm": 436.0,
      "learning_rate": 2.7485380116959063e-07,
      "logits/chosen": -0.6648294925689697,
      "logits/rejected": -0.6572656035423279,
      "logps/chosen": -849.4884033203125,
      "logps/rejected": -655.7454833984375,
      "loss": 0.6945,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0004249574849382043,
      "rewards/margins": -0.0023167619947344065,
      "rewards/rejected": 0.0018918045097962022,
      "step": 1921
    },
    {
      "epoch": 0.5055483404297293,
      "grad_norm": 482.0,
      "learning_rate": 2.7470760233918125e-07,
      "logits/chosen": -0.6601352691650391,
      "logits/rejected": -0.6530523896217346,
      "logps/chosen": -894.5704345703125,
      "logps/rejected": -783.649169921875,
      "loss": 0.7009,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.031233597546815872,
      "rewards/margins": -0.01502928789705038,
      "rewards/rejected": -0.016204310581088066,
      "step": 1922
    },
    {
      "epoch": 0.5058113728649164,
      "grad_norm": 524.0,
      "learning_rate": 2.7456140350877193e-07,
      "logits/chosen": -0.6836310625076294,
      "logits/rejected": -0.6636703014373779,
      "logps/chosen": -836.985595703125,
      "logps/rejected": -937.7974243164062,
      "loss": 0.6957,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00860443152487278,
      "rewards/margins": -0.0041678426787257195,
      "rewards/rejected": 0.012772275134921074,
      "step": 1923
    },
    {
      "epoch": 0.5060744053001036,
      "grad_norm": 848.0,
      "learning_rate": 2.7441520467836255e-07,
      "logits/chosen": -0.6390922665596008,
      "logits/rejected": -0.6446767449378967,
      "logps/chosen": -1748.5389404296875,
      "logps/rejected": -1409.2415771484375,
      "loss": 0.6948,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.013598251156508923,
      "rewards/margins": -0.0026644710451364517,
      "rewards/rejected": 0.0162627212703228,
      "step": 1924
    },
    {
      "epoch": 0.5063374377352907,
      "grad_norm": 552.0,
      "learning_rate": 2.742690058479532e-07,
      "logits/chosen": -0.6418286561965942,
      "logits/rejected": -0.661898672580719,
      "logps/chosen": -1089.16064453125,
      "logps/rejected": -851.6502685546875,
      "loss": 0.6757,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03417358174920082,
      "rewards/margins": 0.03616971895098686,
      "rewards/rejected": -0.001996135339140892,
      "step": 1925
    },
    {
      "epoch": 0.5066004701704779,
      "grad_norm": 520.0,
      "learning_rate": 2.741228070175439e-07,
      "logits/chosen": -0.6301001906394958,
      "logits/rejected": -0.6294184327125549,
      "logps/chosen": -1061.2933349609375,
      "logps/rejected": -890.4588012695312,
      "loss": 0.6948,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012718772515654564,
      "rewards/margins": -0.0031967165414243937,
      "rewards/rejected": 0.015915488824248314,
      "step": 1926
    },
    {
      "epoch": 0.506863502605665,
      "grad_norm": 488.0,
      "learning_rate": 2.7397660818713446e-07,
      "logits/chosen": -0.669888973236084,
      "logits/rejected": -0.6658363342285156,
      "logps/chosen": -895.8511352539062,
      "logps/rejected": -512.9290771484375,
      "loss": 0.6924,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.6117584891617298e-05,
      "rewards/margins": 0.0017157546244561672,
      "rewards/rejected": -0.0017318730242550373,
      "step": 1927
    },
    {
      "epoch": 0.5071265350408523,
      "grad_norm": 548.0,
      "learning_rate": 2.7383040935672513e-07,
      "logits/chosen": -0.6622811555862427,
      "logits/rejected": -0.6602020263671875,
      "logps/chosen": -1059.6826171875,
      "logps/rejected": -871.2845458984375,
      "loss": 0.6935,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.012931013479828835,
      "rewards/margins": 2.050423063337803e-06,
      "rewards/rejected": -0.012933064252138138,
      "step": 1928
    },
    {
      "epoch": 0.5073895674760394,
      "grad_norm": 556.0,
      "learning_rate": 2.736842105263158e-07,
      "logits/chosen": -0.6491297483444214,
      "logits/rejected": -0.6530522704124451,
      "logps/chosen": -1327.42236328125,
      "logps/rejected": -1046.3525390625,
      "loss": 0.6949,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0038731577806174755,
      "rewards/margins": -0.0028641698881983757,
      "rewards/rejected": 0.006737328600138426,
      "step": 1929
    },
    {
      "epoch": 0.5076525999112266,
      "grad_norm": 400.0,
      "learning_rate": 2.735380116959064e-07,
      "logits/chosen": -0.6644228100776672,
      "logits/rejected": -0.6700035333633423,
      "logps/chosen": -851.9392700195312,
      "logps/rejected": -929.1608276367188,
      "loss": 0.6965,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.001315021887421608,
      "rewards/margins": -0.0060482025146484375,
      "rewards/rejected": 0.004733181092888117,
      "step": 1930
    },
    {
      "epoch": 0.5079156323464137,
      "grad_norm": 548.0,
      "learning_rate": 2.7339181286549704e-07,
      "logits/chosen": -0.6000823378562927,
      "logits/rejected": -0.6082191467285156,
      "logps/chosen": -979.2525634765625,
      "logps/rejected": -926.636474609375,
      "loss": 0.6865,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.005847168155014515,
      "rewards/margins": 0.013814256526529789,
      "rewards/rejected": -0.007967090234160423,
      "step": 1931
    },
    {
      "epoch": 0.5081786647816009,
      "grad_norm": 616.0,
      "learning_rate": 2.732456140350877e-07,
      "logits/chosen": -0.6517011523246765,
      "logits/rejected": -0.6590681076049805,
      "logps/chosen": -1297.1895751953125,
      "logps/rejected": -1009.4881591796875,
      "loss": 0.7003,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.013228225521743298,
      "rewards/margins": -0.013679981231689453,
      "rewards/rejected": 0.00045175570994615555,
      "step": 1932
    },
    {
      "epoch": 0.508441697216788,
      "grad_norm": 408.0,
      "learning_rate": 2.7309941520467834e-07,
      "logits/chosen": -0.6696532368659973,
      "logits/rejected": -0.6765496730804443,
      "logps/chosen": -744.36572265625,
      "logps/rejected": -848.9701538085938,
      "loss": 0.6911,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.001983070746064186,
      "rewards/margins": 0.0047171590849757195,
      "rewards/rejected": -0.002734088571742177,
      "step": 1933
    },
    {
      "epoch": 0.5087047296519752,
      "grad_norm": 556.0,
      "learning_rate": 2.72953216374269e-07,
      "logits/chosen": -0.6366410255432129,
      "logits/rejected": -0.6505538821220398,
      "logps/chosen": -1475.1060791015625,
      "logps/rejected": -1517.615966796875,
      "loss": 0.6943,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.000882625812664628,
      "rewards/margins": -0.0019344785250723362,
      "rewards/rejected": 0.0028171075973659754,
      "step": 1934
    },
    {
      "epoch": 0.5089677620871623,
      "grad_norm": 592.0,
      "learning_rate": 2.7280701754385963e-07,
      "logits/chosen": -0.6672852039337158,
      "logits/rejected": -0.6670788526535034,
      "logps/chosen": -1169.1385498046875,
      "logps/rejected": -841.2139892578125,
      "loss": 0.6956,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01088347565382719,
      "rewards/margins": -0.004444837104529142,
      "rewards/rejected": -0.006438637617975473,
      "step": 1935
    },
    {
      "epoch": 0.5092307945223495,
      "grad_norm": 580.0,
      "learning_rate": 2.7266081871345025e-07,
      "logits/chosen": -0.6319125294685364,
      "logits/rejected": NaN,
      "logps/chosen": -1277.2391357421875,
      "logps/rejected": -763.0700073242188,
      "loss": 0.6783,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0008518211543560028,
      "rewards/margins": 0.030454255640506744,
      "rewards/rejected": -0.02960243448615074,
      "step": 1936
    },
    {
      "epoch": 0.5094938269575368,
      "grad_norm": 496.0,
      "learning_rate": 2.725146198830409e-07,
      "logits/chosen": -0.651087760925293,
      "logits/rejected": -0.6569855213165283,
      "logps/chosen": -1288.840576171875,
      "logps/rejected": -852.5359497070312,
      "loss": 0.6833,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.007937814109027386,
      "rewards/margins": 0.020096398890018463,
      "rewards/rejected": -0.012158584780991077,
      "step": 1937
    },
    {
      "epoch": 0.5097568593927239,
      "grad_norm": 504.0,
      "learning_rate": 2.723684210526316e-07,
      "logits/chosen": -0.6489721536636353,
      "logits/rejected": -0.6644414663314819,
      "logps/chosen": -1181.29541015625,
      "logps/rejected": -1266.3017578125,
      "loss": 0.6958,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.013936328701674938,
      "rewards/margins": -0.003906157799065113,
      "rewards/rejected": 0.017842484638094902,
      "step": 1938
    },
    {
      "epoch": 0.5100198918279111,
      "grad_norm": 612.0,
      "learning_rate": 2.7222222222222216e-07,
      "logits/chosen": -0.660522997379303,
      "logits/rejected": -0.6488562226295471,
      "logps/chosen": -1369.170654296875,
      "logps/rejected": -1058.505126953125,
      "loss": 0.6859,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.006043052766472101,
      "rewards/margins": 0.015647411346435547,
      "rewards/rejected": -0.02169046550989151,
      "step": 1939
    },
    {
      "epoch": 0.5102829242630982,
      "grad_norm": 492.0,
      "learning_rate": 2.7207602339181284e-07,
      "logits/chosen": -0.6426509022712708,
      "logits/rejected": -0.6540607810020447,
      "logps/chosen": -1005.2059326171875,
      "logps/rejected": -698.3607788085938,
      "loss": 0.7027,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.003396988846361637,
      "rewards/margins": -0.01831207424402237,
      "rewards/rejected": 0.021709062159061432,
      "step": 1940
    },
    {
      "epoch": 0.5105459566982854,
      "grad_norm": 556.0,
      "learning_rate": 2.719298245614035e-07,
      "logits/chosen": -0.649806022644043,
      "logits/rejected": NaN,
      "logps/chosen": -1274.0826416015625,
      "logps/rejected": -865.0418701171875,
      "loss": 0.6914,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.012111853808164597,
      "rewards/margins": 0.005164335947483778,
      "rewards/rejected": -0.01727619208395481,
      "step": 1941
    },
    {
      "epoch": 0.5108089891334725,
      "grad_norm": 528.0,
      "learning_rate": 2.7178362573099413e-07,
      "logits/chosen": -0.6531878113746643,
      "logits/rejected": -0.6731276512145996,
      "logps/chosen": -987.2538452148438,
      "logps/rejected": -733.863037109375,
      "loss": 0.696,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0020599369890987873,
      "rewards/margins": -0.005229472182691097,
      "rewards/rejected": 0.003169536590576172,
      "step": 1942
    },
    {
      "epoch": 0.5110720215686597,
      "grad_norm": 632.0,
      "learning_rate": 2.716374269005848e-07,
      "logits/chosen": -0.6382675766944885,
      "logits/rejected": -0.6391232013702393,
      "logps/chosen": -1447.264404296875,
      "logps/rejected": -982.8138427734375,
      "loss": 0.7018,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.011252310127019882,
      "rewards/margins": -0.015898801386356354,
      "rewards/rejected": 0.004646492190659046,
      "step": 1943
    },
    {
      "epoch": 0.5113350540038468,
      "grad_norm": 660.0,
      "learning_rate": 2.714912280701754e-07,
      "logits/chosen": -0.646570086479187,
      "logits/rejected": -0.6393513083457947,
      "logps/chosen": -1697.959716796875,
      "logps/rejected": -1498.502197265625,
      "loss": 0.6866,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005060672760009766,
      "rewards/margins": 0.013924884609878063,
      "rewards/rejected": -0.008864212781190872,
      "step": 1944
    },
    {
      "epoch": 0.511598086439034,
      "grad_norm": 560.0,
      "learning_rate": 2.7134502923976604e-07,
      "logits/chosen": -0.6410630345344543,
      "logits/rejected": -0.6455520391464233,
      "logps/chosen": -1175.6656494140625,
      "logps/rejected": -721.3416748046875,
      "loss": 0.6888,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009671308100223541,
      "rewards/margins": 0.009722423739731312,
      "rewards/rejected": -0.01939372904598713,
      "step": 1945
    },
    {
      "epoch": 0.5118611188742211,
      "grad_norm": 628.0,
      "learning_rate": 2.711988304093567e-07,
      "logits/chosen": -0.6498444080352783,
      "logits/rejected": -0.647308886051178,
      "logps/chosen": -1290.314208984375,
      "logps/rejected": -913.948974609375,
      "loss": 0.6951,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.022472763434052467,
      "rewards/margins": -0.003432844765484333,
      "rewards/rejected": -0.01903991959989071,
      "step": 1946
    },
    {
      "epoch": 0.5121241513094084,
      "grad_norm": 414.0,
      "learning_rate": 2.710526315789474e-07,
      "logits/chosen": -0.6357147693634033,
      "logits/rejected": -0.6467241644859314,
      "logps/chosen": -1137.08740234375,
      "logps/rejected": -1047.8228759765625,
      "loss": 0.6925,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.010047243908047676,
      "rewards/margins": 0.001612663036212325,
      "rewards/rejected": 0.008434580639004707,
      "step": 1947
    },
    {
      "epoch": 0.5123871837445955,
      "grad_norm": 502.0,
      "learning_rate": 2.7090643274853795e-07,
      "logits/chosen": -0.6580315828323364,
      "logits/rejected": -0.65535968542099,
      "logps/chosen": -853.2113037109375,
      "logps/rejected": -653.0968627929688,
      "loss": 0.7002,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0017374048475176096,
      "rewards/margins": -0.013229083269834518,
      "rewards/rejected": 0.014966487884521484,
      "step": 1948
    },
    {
      "epoch": 0.5126502161797827,
      "grad_norm": 516.0,
      "learning_rate": 2.7076023391812863e-07,
      "logits/chosen": -0.6506156921386719,
      "logits/rejected": -0.6607378721237183,
      "logps/chosen": -986.3169555664062,
      "logps/rejected": -1061.94873046875,
      "loss": 0.6785,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.015205860137939453,
      "rewards/margins": 0.02998514100909233,
      "rewards/rejected": -0.014779281802475452,
      "step": 1949
    },
    {
      "epoch": 0.5129132486149698,
      "grad_norm": 552.0,
      "learning_rate": 2.706140350877193e-07,
      "logits/chosen": -0.6498463749885559,
      "logits/rejected": -0.6490477323532104,
      "logps/chosen": -1170.8505859375,
      "logps/rejected": -780.8360595703125,
      "loss": 0.6885,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01373748853802681,
      "rewards/margins": 0.010622788220643997,
      "rewards/rejected": 0.0031147003173828125,
      "step": 1950
    },
    {
      "epoch": 0.513176281050157,
      "grad_norm": 484.0,
      "learning_rate": 2.7046783625730997e-07,
      "logits/chosen": -0.6445609331130981,
      "logits/rejected": -0.6506455540657043,
      "logps/chosen": -1172.504638671875,
      "logps/rejected": -1054.2982177734375,
      "loss": 0.6957,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0008954042568802834,
      "rewards/margins": -0.004653454292565584,
      "rewards/rejected": 0.0055488585494458675,
      "step": 1951
    },
    {
      "epoch": 0.5134393134853441,
      "grad_norm": 440.0,
      "learning_rate": 2.7032163742690054e-07,
      "logits/chosen": -0.6732948422431946,
      "logits/rejected": -0.6842491030693054,
      "logps/chosen": -915.8494873046875,
      "logps/rejected": -604.4825439453125,
      "loss": 0.7093,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.01973242685198784,
      "rewards/margins": -0.031454898416996,
      "rewards/rejected": 0.011722471565008163,
      "step": 1952
    },
    {
      "epoch": 0.5137023459205313,
      "grad_norm": 584.0,
      "learning_rate": 2.701754385964912e-07,
      "logits/chosen": -0.6778047680854797,
      "logits/rejected": -0.6651496887207031,
      "logps/chosen": -1627.5682373046875,
      "logps/rejected": -1326.105712890625,
      "loss": 0.7008,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.009531212039291859,
      "rewards/margins": -0.01436548214405775,
      "rewards/rejected": 0.004834271967411041,
      "step": 1953
    },
    {
      "epoch": 0.5139653783557185,
      "grad_norm": 412.0,
      "learning_rate": 2.700292397660819e-07,
      "logits/chosen": -0.6394535303115845,
      "logits/rejected": -0.6427717208862305,
      "logps/chosen": -758.7022094726562,
      "logps/rejected": -574.5660400390625,
      "loss": 0.7082,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.006431198678910732,
      "rewards/margins": -0.029392529278993607,
      "rewards/rejected": 0.02296133153140545,
      "step": 1954
    },
    {
      "epoch": 0.5142284107909056,
      "grad_norm": 580.0,
      "learning_rate": 2.698830409356725e-07,
      "logits/chosen": -0.64130699634552,
      "logits/rejected": -0.6544835567474365,
      "logps/chosen": -1279.7685546875,
      "logps/rejected": -939.5267944335938,
      "loss": 0.6908,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00929403305053711,
      "rewards/margins": 0.005868719425052404,
      "rewards/rejected": 0.0034253131598234177,
      "step": 1955
    },
    {
      "epoch": 0.5144914432260929,
      "grad_norm": 524.0,
      "learning_rate": 2.697368421052632e-07,
      "logits/chosen": -0.661187469959259,
      "logits/rejected": -0.6707997918128967,
      "logps/chosen": -1276.3204345703125,
      "logps/rejected": -916.1123046875,
      "loss": 0.6908,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007108927704393864,
      "rewards/margins": 0.0059343320317566395,
      "rewards/rejected": 0.001174593809992075,
      "step": 1956
    },
    {
      "epoch": 0.51475447566128,
      "grad_norm": 576.0,
      "learning_rate": 2.695906432748538e-07,
      "logits/chosen": -0.6078104972839355,
      "logits/rejected": -0.6129421591758728,
      "logps/chosen": -1117.273681640625,
      "logps/rejected": -867.004150390625,
      "loss": 0.6879,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.025759121403098106,
      "rewards/margins": 0.011167430318892002,
      "rewards/rejected": 0.014591695740818977,
      "step": 1957
    },
    {
      "epoch": 0.5150175080964672,
      "grad_norm": 466.0,
      "learning_rate": 2.694444444444444e-07,
      "logits/chosen": -0.656832218170166,
      "logits/rejected": -0.6503811478614807,
      "logps/chosen": -992.8277587890625,
      "logps/rejected": -855.2951049804688,
      "loss": 0.6934,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.007450391072779894,
      "rewards/margins": -4.4442713260650635e-05,
      "rewards/rejected": -0.007405946962535381,
      "step": 1958
    },
    {
      "epoch": 0.5152805405316543,
      "grad_norm": 568.0,
      "learning_rate": 2.692982456140351e-07,
      "logits/chosen": -0.6534103155136108,
      "logits/rejected": -0.6591341495513916,
      "logps/chosen": -1094.7432861328125,
      "logps/rejected": -666.5662841796875,
      "loss": 0.6977,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.007513236720114946,
      "rewards/margins": -0.008388519287109375,
      "rewards/rejected": 0.0008752822177484632,
      "step": 1959
    },
    {
      "epoch": 0.5155435729668415,
      "grad_norm": 612.0,
      "learning_rate": 2.6915204678362576e-07,
      "logits/chosen": -0.6522630453109741,
      "logits/rejected": -0.644982635974884,
      "logps/chosen": -1241.7001953125,
      "logps/rejected": -1143.585693359375,
      "loss": 0.7022,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.002412986010313034,
      "rewards/margins": -0.017066622152924538,
      "rewards/rejected": 0.01947960816323757,
      "step": 1960
    },
    {
      "epoch": 0.5158066054020286,
      "grad_norm": 632.0,
      "learning_rate": 2.6900584795321633e-07,
      "logits/chosen": -0.6812131404876709,
      "logits/rejected": -0.6906218528747559,
      "logps/chosen": -1308.733154296875,
      "logps/rejected": -994.4481201171875,
      "loss": 0.6927,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.1362677216529846e-05,
      "rewards/margins": 0.0017785541713237762,
      "rewards/rejected": -0.0017999178962782025,
      "step": 1961
    },
    {
      "epoch": 0.5160696378372158,
      "grad_norm": 456.0,
      "learning_rate": 2.68859649122807e-07,
      "logits/chosen": -0.6601186394691467,
      "logits/rejected": -0.6588069796562195,
      "logps/chosen": -1049.9903564453125,
      "logps/rejected": -817.920654296875,
      "loss": 0.6919,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005004310980439186,
      "rewards/margins": 0.0027558314613997936,
      "rewards/rejected": 0.002248478587716818,
      "step": 1962
    },
    {
      "epoch": 0.5163326702724029,
      "grad_norm": 660.0,
      "learning_rate": 2.687134502923977e-07,
      "logits/chosen": -0.6575765609741211,
      "logits/rejected": -0.6638635396957397,
      "logps/chosen": -1320.9912109375,
      "logps/rejected": -1313.8837890625,
      "loss": 0.7056,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02972092665731907,
      "rewards/margins": -0.023721076548099518,
      "rewards/rejected": -0.005999851040542126,
      "step": 1963
    },
    {
      "epoch": 0.5165957027075901,
      "grad_norm": 426.0,
      "learning_rate": 2.685672514619883e-07,
      "logits/chosen": -0.661746084690094,
      "logits/rejected": -0.6699345707893372,
      "logps/chosen": -837.142578125,
      "logps/rejected": -571.90771484375,
      "loss": 0.6981,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0016958238556981087,
      "rewards/margins": -0.009701727889478207,
      "rewards/rejected": 0.008005904965102673,
      "step": 1964
    },
    {
      "epoch": 0.5168587351427772,
      "grad_norm": 572.0,
      "learning_rate": 2.684210526315789e-07,
      "logits/chosen": -0.6738949418067932,
      "logits/rejected": -0.6802046298980713,
      "logps/chosen": -1211.6141357421875,
      "logps/rejected": -819.8423461914062,
      "loss": 0.6921,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.006052876356989145,
      "rewards/margins": 0.0033700456842780113,
      "rewards/rejected": -0.00942292157560587,
      "step": 1965
    },
    {
      "epoch": 0.5171217675779645,
      "grad_norm": 404.0,
      "learning_rate": 2.682748538011696e-07,
      "logits/chosen": -0.6522868871688843,
      "logits/rejected": -0.6506860256195068,
      "logps/chosen": -868.0413208007812,
      "logps/rejected": -676.557861328125,
      "loss": 0.6823,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0223567932844162,
      "rewards/margins": 0.022856712341308594,
      "rewards/rejected": -0.0004999161465093493,
      "step": 1966
    },
    {
      "epoch": 0.5173848000131516,
      "grad_norm": 484.0,
      "learning_rate": 2.681286549707602e-07,
      "logits/chosen": -0.6628242135047913,
      "logits/rejected": -0.6690526604652405,
      "logps/chosen": -1123.056396484375,
      "logps/rejected": -827.6143798828125,
      "loss": 0.6833,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0025694852229207754,
      "rewards/margins": 0.020283173769712448,
      "rewards/rejected": -0.01771369017660618,
      "step": 1967
    },
    {
      "epoch": 0.5176478324483388,
      "grad_norm": 560.0,
      "learning_rate": 2.679824561403509e-07,
      "logits/chosen": -0.6611493825912476,
      "logits/rejected": -0.6628329157829285,
      "logps/chosen": -1288.98291015625,
      "logps/rejected": -1196.515869140625,
      "loss": 0.6845,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02606048621237278,
      "rewards/margins": 0.018885038793087006,
      "rewards/rejected": 0.007175445556640625,
      "step": 1968
    },
    {
      "epoch": 0.517910864883526,
      "grad_norm": 596.0,
      "learning_rate": 2.678362573099415e-07,
      "logits/chosen": -0.6574552059173584,
      "logits/rejected": -0.6556031703948975,
      "logps/chosen": -1028.59814453125,
      "logps/rejected": -942.943115234375,
      "loss": 0.7055,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.02019543945789337,
      "rewards/margins": -0.02392749674618244,
      "rewards/rejected": 0.0037320612464100122,
      "step": 1969
    },
    {
      "epoch": 0.5181738973187131,
      "grad_norm": 592.0,
      "learning_rate": 2.676900584795321e-07,
      "logits/chosen": -0.6500288248062134,
      "logits/rejected": NaN,
      "logps/chosen": -1344.737060546875,
      "logps/rejected": -1226.03173828125,
      "loss": 0.7032,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.013890266418457031,
      "rewards/margins": -0.01958761177957058,
      "rewards/rejected": 0.005697345361113548,
      "step": 1970
    },
    {
      "epoch": 0.5184369297539003,
      "grad_norm": 484.0,
      "learning_rate": 2.675438596491228e-07,
      "logits/chosen": -0.668375551700592,
      "logits/rejected": -0.6614587306976318,
      "logps/chosen": -888.7539672851562,
      "logps/rejected": -506.2745666503906,
      "loss": 0.6945,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.007772158831357956,
      "rewards/margins": -0.0017552380450069904,
      "rewards/rejected": 0.009527397342026234,
      "step": 1971
    },
    {
      "epoch": 0.5186999621890874,
      "grad_norm": 446.0,
      "learning_rate": 2.6739766081871347e-07,
      "logits/chosen": -0.6369709968566895,
      "logits/rejected": -0.6500731110572815,
      "logps/chosen": -938.5117797851562,
      "logps/rejected": -919.624755859375,
      "loss": 0.6901,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.004902458284050226,
      "rewards/margins": 0.006320476066321135,
      "rewards/rejected": -0.01122293621301651,
      "step": 1972
    },
    {
      "epoch": 0.5189629946242746,
      "grad_norm": 490.0,
      "learning_rate": 2.672514619883041e-07,
      "logits/chosen": -0.662494421005249,
      "logits/rejected": -0.6662045121192932,
      "logps/chosen": -1211.0797119140625,
      "logps/rejected": -732.93359375,
      "loss": 0.6923,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003814124735072255,
      "rewards/margins": 0.002433109562844038,
      "rewards/rejected": 0.0013810154050588608,
      "step": 1973
    },
    {
      "epoch": 0.5192260270594617,
      "grad_norm": 482.0,
      "learning_rate": 2.671052631578947e-07,
      "logits/chosen": -0.6525269746780396,
      "logits/rejected": NaN,
      "logps/chosen": -823.2041015625,
      "logps/rejected": -751.130615234375,
      "loss": 0.6941,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.015303612686693668,
      "rewards/margins": -0.0013569826260209084,
      "rewards/rejected": 0.016660595312714577,
      "step": 1974
    },
    {
      "epoch": 0.519489059494649,
      "grad_norm": 486.0,
      "learning_rate": 2.669590643274854e-07,
      "logits/chosen": -0.6479438543319702,
      "logits/rejected": -0.656160295009613,
      "logps/chosen": -1252.3231201171875,
      "logps/rejected": -926.5983276367188,
      "loss": 0.6868,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.003131866455078125,
      "rewards/margins": 0.013357640244066715,
      "rewards/rejected": -0.016489507630467415,
      "step": 1975
    },
    {
      "epoch": 0.5197520919298361,
      "grad_norm": 506.0,
      "learning_rate": 2.66812865497076e-07,
      "logits/chosen": -0.6275262832641602,
      "logits/rejected": -0.635825514793396,
      "logps/chosen": -1366.0906982421875,
      "logps/rejected": -1148.4652099609375,
      "loss": 0.6763,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02084674872457981,
      "rewards/margins": 0.03470649942755699,
      "rewards/rejected": -0.013859748840332031,
      "step": 1976
    },
    {
      "epoch": 0.5200151243650233,
      "grad_norm": 592.0,
      "learning_rate": 2.6666666666666667e-07,
      "logits/chosen": -0.6524511575698853,
      "logits/rejected": -0.6523795127868652,
      "logps/chosen": -1535.2235107421875,
      "logps/rejected": -1299.1422119140625,
      "loss": 0.6778,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.005327604711055756,
      "rewards/margins": 0.03204965591430664,
      "rewards/rejected": -0.026722049340605736,
      "step": 1977
    },
    {
      "epoch": 0.5202781568002104,
      "grad_norm": 472.0,
      "learning_rate": 2.665204678362573e-07,
      "logits/chosen": -0.654116153717041,
      "logits/rejected": -0.6703193187713623,
      "logps/chosen": -835.162841796875,
      "logps/rejected": -693.052490234375,
      "loss": 0.6867,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.004315948113799095,
      "rewards/margins": 0.01349944993853569,
      "rewards/rejected": -0.017815399914979935,
      "step": 1978
    },
    {
      "epoch": 0.5205411892353976,
      "grad_norm": 492.0,
      "learning_rate": 2.663742690058479e-07,
      "logits/chosen": -0.6517537236213684,
      "logits/rejected": -0.6467518210411072,
      "logps/chosen": -1186.9146728515625,
      "logps/rejected": -702.73046875,
      "loss": 0.6958,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0019201275426894426,
      "rewards/margins": -0.004458046052604914,
      "rewards/rejected": 0.006378173828125,
      "step": 1979
    },
    {
      "epoch": 0.5208042216705847,
      "grad_norm": 588.0,
      "learning_rate": 2.662280701754386e-07,
      "logits/chosen": -0.6350906491279602,
      "logits/rejected": NaN,
      "logps/chosen": -930.0919799804688,
      "logps/rejected": -945.365478515625,
      "loss": 0.695,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.005989075638353825,
      "rewards/margins": -0.0028911586850881577,
      "rewards/rejected": 0.008880234323441982,
      "step": 1980
    },
    {
      "epoch": 0.5210672541057719,
      "grad_norm": 456.0,
      "learning_rate": 2.6608187134502926e-07,
      "logits/chosen": -0.6468313932418823,
      "logits/rejected": -0.6536762118339539,
      "logps/chosen": -1217.420654296875,
      "logps/rejected": -714.6575927734375,
      "loss": 0.6863,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0038367747329175472,
      "rewards/margins": 0.014471961185336113,
      "rewards/rejected": -0.010635185055434704,
      "step": 1981
    },
    {
      "epoch": 0.521330286540959,
      "grad_norm": 592.0,
      "learning_rate": 2.659356725146198e-07,
      "logits/chosen": -0.6572943925857544,
      "logits/rejected": -0.6570364832878113,
      "logps/chosen": -1161.579345703125,
      "logps/rejected": -1027.021728515625,
      "loss": 0.6919,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.009078598581254482,
      "rewards/margins": 0.0027050022035837173,
      "rewards/rejected": 0.006373596377670765,
      "step": 1982
    },
    {
      "epoch": 0.5215933189761462,
      "grad_norm": 572.0,
      "learning_rate": 2.657894736842105e-07,
      "logits/chosen": -0.6490082740783691,
      "logits/rejected": NaN,
      "logps/chosen": -1259.3365478515625,
      "logps/rejected": -1094.9288330078125,
      "loss": 0.6868,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0028345107566565275,
      "rewards/margins": 0.01332626398652792,
      "rewards/rejected": -0.010491752997040749,
      "step": 1983
    },
    {
      "epoch": 0.5218563514113335,
      "grad_norm": 504.0,
      "learning_rate": 2.6564327485380117e-07,
      "logits/chosen": -0.6659319400787354,
      "logits/rejected": -0.6656390428543091,
      "logps/chosen": -965.4232177734375,
      "logps/rejected": -797.9454956054688,
      "loss": 0.6844,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0014902586117386818,
      "rewards/margins": 0.01885838620364666,
      "rewards/rejected": -0.020348645746707916,
      "step": 1984
    },
    {
      "epoch": 0.5221193838465206,
      "grad_norm": 2400.0,
      "learning_rate": 2.654970760233918e-07,
      "logits/chosen": -0.6586664319038391,
      "logits/rejected": -0.669733464717865,
      "logps/chosen": -1396.239990234375,
      "logps/rejected": -746.1431274414062,
      "loss": 0.6935,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0016311637591570616,
      "rewards/margins": 1.1206604540348053e-05,
      "rewards/rejected": 0.0016199585516005754,
      "step": 1985
    },
    {
      "epoch": 0.5223824162817078,
      "grad_norm": 724.0,
      "learning_rate": 2.653508771929824e-07,
      "logits/chosen": -0.6540647149085999,
      "logits/rejected": -0.6594731211662292,
      "logps/chosen": -1576.9267578125,
      "logps/rejected": -1264.4556884765625,
      "loss": 0.6948,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007557106204330921,
      "rewards/margins": -0.00289230328053236,
      "rewards/rejected": 0.010449410416185856,
      "step": 1986
    },
    {
      "epoch": 0.5226454487168949,
      "grad_norm": 508.0,
      "learning_rate": 2.652046783625731e-07,
      "logits/chosen": -0.6582899689674377,
      "logits/rejected": -0.6629778742790222,
      "logps/chosen": -762.9781494140625,
      "logps/rejected": -916.7410888671875,
      "loss": 0.6903,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.019875910133123398,
      "rewards/margins": 0.005936813540756702,
      "rewards/rejected": 0.013939094729721546,
      "step": 1987
    },
    {
      "epoch": 0.5229084811520821,
      "grad_norm": 568.0,
      "learning_rate": 2.650584795321637e-07,
      "logits/chosen": -0.6646980047225952,
      "logits/rejected": -0.667870044708252,
      "logps/chosen": -1008.0806884765625,
      "logps/rejected": -1003.7807006835938,
      "loss": 0.6985,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.005993175320327282,
      "rewards/margins": -0.009832002222537994,
      "rewards/rejected": 0.0158251766115427,
      "step": 1988
    },
    {
      "epoch": 0.5231715135872692,
      "grad_norm": 428.0,
      "learning_rate": 2.649122807017544e-07,
      "logits/chosen": -0.6630300283432007,
      "logits/rejected": -0.6609824895858765,
      "logps/chosen": -888.6239624023438,
      "logps/rejected": -985.62451171875,
      "loss": 0.688,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01621704176068306,
      "rewards/margins": 0.010677911341190338,
      "rewards/rejected": 0.0055391304194927216,
      "step": 1989
    },
    {
      "epoch": 0.5234345460224564,
      "grad_norm": 984.0,
      "learning_rate": 2.6476608187134505e-07,
      "logits/chosen": -0.66612708568573,
      "logits/rejected": -0.6589748859405518,
      "logps/chosen": -1446.4222412109375,
      "logps/rejected": -1062.39111328125,
      "loss": 0.6854,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.019670965149998665,
      "rewards/margins": 0.01680450513958931,
      "rewards/rejected": 0.0028664579149335623,
      "step": 1990
    },
    {
      "epoch": 0.5236975784576435,
      "grad_norm": 568.0,
      "learning_rate": 2.6461988304093567e-07,
      "logits/chosen": -0.6675102114677429,
      "logits/rejected": -0.6690680384635925,
      "logps/chosen": -1004.7508544921875,
      "logps/rejected": -861.7905883789062,
      "loss": 0.6831,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.024287033826112747,
      "rewards/margins": 0.021387862041592598,
      "rewards/rejected": 0.002899170620366931,
      "step": 1991
    },
    {
      "epoch": 0.5239606108928307,
      "grad_norm": 832.0,
      "learning_rate": 2.644736842105263e-07,
      "logits/chosen": -0.6341084241867065,
      "logits/rejected": -0.6493899822235107,
      "logps/chosen": -1135.6591796875,
      "logps/rejected": -841.9097290039062,
      "loss": 0.7092,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.003046464640647173,
      "rewards/margins": -0.02922673150897026,
      "rewards/rejected": 0.026180271059274673,
      "step": 1992
    },
    {
      "epoch": 0.5242236433280179,
      "grad_norm": 904.0,
      "learning_rate": 2.6432748538011696e-07,
      "logits/chosen": -0.6253660917282104,
      "logits/rejected": -0.6546262502670288,
      "logps/chosen": -1120.8458251953125,
      "logps/rejected": -1157.574951171875,
      "loss": 0.6993,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0034737589303404093,
      "rewards/margins": -0.011769772507250309,
      "rewards/rejected": 0.008296013809740543,
      "step": 1993
    },
    {
      "epoch": 0.5244866757632051,
      "grad_norm": 644.0,
      "learning_rate": 2.6418128654970763e-07,
      "logits/chosen": -0.6582355499267578,
      "logits/rejected": -0.6598194241523743,
      "logps/chosen": -1264.9478759765625,
      "logps/rejected": -956.6683349609375,
      "loss": 0.6938,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.013468551449477673,
      "rewards/margins": -0.0008059507235884666,
      "rewards/rejected": 0.01427450217306614,
      "step": 1994
    },
    {
      "epoch": 0.5247497081983922,
      "grad_norm": 552.0,
      "learning_rate": 2.640350877192982e-07,
      "logits/chosen": -0.6614875197410583,
      "logits/rejected": -0.6754339337348938,
      "logps/chosen": -1163.66015625,
      "logps/rejected": -971.7057495117188,
      "loss": 0.6949,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0007831580005586147,
      "rewards/margins": -0.002920722123235464,
      "rewards/rejected": 0.002137565053999424,
      "step": 1995
    },
    {
      "epoch": 0.5250127406335794,
      "grad_norm": 476.0,
      "learning_rate": 2.638888888888889e-07,
      "logits/chosen": -0.6488953828811646,
      "logits/rejected": -0.6470944285392761,
      "logps/chosen": -603.9679565429688,
      "logps/rejected": -756.3970947265625,
      "loss": 0.6974,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005485057830810547,
      "rewards/margins": -0.008109664544463158,
      "rewards/rejected": 0.0026246074121445417,
      "step": 1996
    },
    {
      "epoch": 0.5252757730687665,
      "grad_norm": 490.0,
      "learning_rate": 2.6374269005847955e-07,
      "logits/chosen": -0.6272384524345398,
      "logits/rejected": -0.6429992318153381,
      "logps/chosen": -1028.5118408203125,
      "logps/rejected": -664.603759765625,
      "loss": 0.6962,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.008814144879579544,
      "rewards/margins": -0.0051323892548680305,
      "rewards/rejected": -0.003681754693388939,
      "step": 1997
    },
    {
      "epoch": 0.5255388055039537,
      "grad_norm": 470.0,
      "learning_rate": 2.6359649122807017e-07,
      "logits/chosen": -0.6692617535591125,
      "logits/rejected": -0.6690961122512817,
      "logps/chosen": -1163.7467041015625,
      "logps/rejected": -1004.5628051757812,
      "loss": 0.6864,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0018238076008856297,
      "rewards/margins": 0.01395187433809042,
      "rewards/rejected": -0.015775680541992188,
      "step": 1998
    },
    {
      "epoch": 0.5258018379391408,
      "grad_norm": 644.0,
      "learning_rate": 2.634502923976608e-07,
      "logits/chosen": -0.6599964499473572,
      "logits/rejected": -0.6718497276306152,
      "logps/chosen": -1242.50439453125,
      "logps/rejected": -1078.77001953125,
      "loss": 0.7068,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.002269553719088435,
      "rewards/margins": -0.025827789679169655,
      "rewards/rejected": 0.028097346425056458,
      "step": 1999
    },
    {
      "epoch": 0.526064870374328,
      "grad_norm": 442.0,
      "learning_rate": 2.6330409356725146e-07,
      "logits/chosen": -0.6433997750282288,
      "logits/rejected": -0.6444616913795471,
      "logps/chosen": -1183.36669921875,
      "logps/rejected": -1033.1260986328125,
      "loss": 0.6981,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.007510948460549116,
      "rewards/margins": -0.009602404199540615,
      "rewards/rejected": 0.002091455738991499,
      "step": 2000
    },
    {
      "epoch": 0.5263279028095152,
      "grad_norm": 680.0,
      "learning_rate": 2.631578947368421e-07,
      "logits/chosen": -0.6034098267555237,
      "logits/rejected": -0.6052515506744385,
      "logps/chosen": -1502.409912109375,
      "logps/rejected": -1261.4998779296875,
      "loss": 0.6874,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.004806710407137871,
      "rewards/margins": 0.012519551441073418,
      "rewards/rejected": -0.007712841033935547,
      "step": 2001
    },
    {
      "epoch": 0.5265909352447024,
      "grad_norm": 704.0,
      "learning_rate": 2.6301169590643275e-07,
      "logits/chosen": -0.6449110507965088,
      "logits/rejected": NaN,
      "logps/chosen": -1970.5457763671875,
      "logps/rejected": -1523.361572265625,
      "loss": 0.6872,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.010406209155917168,
      "rewards/margins": 0.013016605749726295,
      "rewards/rejected": -0.023422814905643463,
      "step": 2002
    },
    {
      "epoch": 0.5268539676798896,
      "grad_norm": 484.0,
      "learning_rate": 2.628654970760234e-07,
      "logits/chosen": -0.6485067009925842,
      "logits/rejected": -0.657439649105072,
      "logps/chosen": -1007.30029296875,
      "logps/rejected": -927.9636840820312,
      "loss": 0.6927,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02118377760052681,
      "rewards/margins": 0.001889992505311966,
      "rewards/rejected": 0.019293785095214844,
      "step": 2003
    },
    {
      "epoch": 0.5271170001150767,
      "grad_norm": 508.0,
      "learning_rate": 2.62719298245614e-07,
      "logits/chosen": -0.6639008522033691,
      "logits/rejected": -0.6652726531028748,
      "logps/chosen": -1106.8858642578125,
      "logps/rejected": -849.6041259765625,
      "loss": 0.6921,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.00796966627240181,
      "rewards/margins": 0.002404499799013138,
      "rewards/rejected": 0.005565166007727385,
      "step": 2004
    },
    {
      "epoch": 0.5273800325502639,
      "grad_norm": 484.0,
      "learning_rate": 2.6257309941520466e-07,
      "logits/chosen": -0.666877269744873,
      "logits/rejected": -0.6568950414657593,
      "logps/chosen": -897.3137817382812,
      "logps/rejected": -708.7716674804688,
      "loss": 0.6914,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0007566451095044613,
      "rewards/margins": 0.004166746977716684,
      "rewards/rejected": -0.003410100704059005,
      "step": 2005
    },
    {
      "epoch": 0.527643064985451,
      "grad_norm": 580.0,
      "learning_rate": 2.6242690058479534e-07,
      "logits/chosen": -0.6215512752532959,
      "logits/rejected": -0.6396120190620422,
      "logps/chosen": -1005.7701416015625,
      "logps/rejected": -830.4246215820312,
      "loss": 0.6984,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.0004881857894361019,
      "rewards/margins": -0.009698677808046341,
      "rewards/rejected": 0.01018686406314373,
      "step": 2006
    },
    {
      "epoch": 0.5279060974206382,
      "grad_norm": 504.0,
      "learning_rate": 2.6228070175438596e-07,
      "logits/chosen": -0.6450603604316711,
      "logits/rejected": -0.6529671549797058,
      "logps/chosen": -999.9849853515625,
      "logps/rejected": -846.4415283203125,
      "loss": 0.6907,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0063812267035245895,
      "rewards/margins": 0.005613327492028475,
      "rewards/rejected": -0.011994552798569202,
      "step": 2007
    },
    {
      "epoch": 0.5281691298558253,
      "grad_norm": 532.0,
      "learning_rate": 2.621345029239766e-07,
      "logits/chosen": -0.6481261849403381,
      "logits/rejected": -0.6491267085075378,
      "logps/chosen": -867.1863403320312,
      "logps/rejected": -629.10400390625,
      "loss": 0.6975,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0027977945283055305,
      "rewards/margins": -0.008145426400005817,
      "rewards/rejected": 0.010943222790956497,
      "step": 2008
    },
    {
      "epoch": 0.5284321622910125,
      "grad_norm": 612.0,
      "learning_rate": 2.6198830409356725e-07,
      "logits/chosen": -0.6325732469558716,
      "logits/rejected": -0.6287708282470703,
      "logps/chosen": -1512.6971435546875,
      "logps/rejected": -1288.913818359375,
      "loss": 0.6895,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.025706864893436432,
      "rewards/margins": 0.007890511304140091,
      "rewards/rejected": 0.01781635358929634,
      "step": 2009
    },
    {
      "epoch": 0.5286951947261996,
      "grad_norm": 564.0,
      "learning_rate": 2.6184210526315787e-07,
      "logits/chosen": -0.6520823240280151,
      "logits/rejected": -0.6622433662414551,
      "logps/chosen": -835.8027954101562,
      "logps/rejected": -767.3124389648438,
      "loss": 0.6918,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.009352302178740501,
      "rewards/margins": 0.0032129285391420126,
      "rewards/rejected": 0.006139373406767845,
      "step": 2010
    },
    {
      "epoch": 0.5289582271613869,
      "grad_norm": 640.0,
      "learning_rate": 2.6169590643274854e-07,
      "logits/chosen": -0.652039110660553,
      "logits/rejected": NaN,
      "logps/chosen": -1524.5028076171875,
      "logps/rejected": -863.9175415039062,
      "loss": 0.7002,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.010251998901367188,
      "rewards/margins": -0.012978171929717064,
      "rewards/rejected": 0.023230168968439102,
      "step": 2011
    },
    {
      "epoch": 0.529221259596574,
      "grad_norm": 592.0,
      "learning_rate": 2.6154970760233916e-07,
      "logits/chosen": -0.6656234860420227,
      "logits/rejected": -0.671114981174469,
      "logps/chosen": -1303.2803955078125,
      "logps/rejected": -1010.9310913085938,
      "loss": 0.7006,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.011097908020019531,
      "rewards/margins": -0.013933374546468258,
      "rewards/rejected": 0.025031283497810364,
      "step": 2012
    },
    {
      "epoch": 0.5294842920317612,
      "grad_norm": 580.0,
      "learning_rate": 2.614035087719298e-07,
      "logits/chosen": -0.626711905002594,
      "logits/rejected": -0.6402908563613892,
      "logps/chosen": -943.6401977539062,
      "logps/rejected": -742.4570922851562,
      "loss": 0.6986,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.010281085968017578,
      "rewards/margins": -0.010039232671260834,
      "rewards/rejected": -0.00024185213260352612,
      "step": 2013
    },
    {
      "epoch": 0.5297473244669483,
      "grad_norm": 508.0,
      "learning_rate": 2.6125730994152046e-07,
      "logits/chosen": -0.687469482421875,
      "logits/rejected": NaN,
      "logps/chosen": -1065.403564453125,
      "logps/rejected": -990.7166137695312,
      "loss": 0.6919,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.007737063802778721,
      "rewards/margins": 0.003231382928788662,
      "rewards/rejected": -0.010968447662889957,
      "step": 2014
    },
    {
      "epoch": 0.5300103569021355,
      "grad_norm": 644.0,
      "learning_rate": 2.6111111111111113e-07,
      "logits/chosen": -0.6731030941009521,
      "logits/rejected": NaN,
      "logps/chosen": -1255.205810546875,
      "logps/rejected": -843.9288940429688,
      "loss": 0.6962,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0007636072114109993,
      "rewards/margins": -0.005050087813287973,
      "rewards/rejected": 0.005813694559037685,
      "step": 2015
    },
    {
      "epoch": 0.5302733893373227,
      "grad_norm": 480.0,
      "learning_rate": 2.609649122807017e-07,
      "logits/chosen": -0.6535898447036743,
      "logits/rejected": -0.6574090123176575,
      "logps/chosen": -768.0809326171875,
      "logps/rejected": -927.444091796875,
      "loss": 0.6849,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.008924007415771484,
      "rewards/margins": 0.016924381256103516,
      "rewards/rejected": -0.008000372909009457,
      "step": 2016
    },
    {
      "epoch": 0.5305364217725098,
      "grad_norm": 584.0,
      "learning_rate": 2.6081871345029237e-07,
      "logits/chosen": -0.6397665143013,
      "logits/rejected": -0.6440280079841614,
      "logps/chosen": -1302.1468505859375,
      "logps/rejected": -915.1913452148438,
      "loss": 0.6996,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.017871569842100143,
      "rewards/margins": -0.012559127062559128,
      "rewards/rejected": 0.03043069876730442,
      "step": 2017
    },
    {
      "epoch": 0.530799454207697,
      "grad_norm": 548.0,
      "learning_rate": 2.6067251461988304e-07,
      "logits/chosen": -0.652898371219635,
      "logits/rejected": -0.6711244583129883,
      "logps/chosen": -942.2081298828125,
      "logps/rejected": -937.0556030273438,
      "loss": 0.6982,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.016394803300499916,
      "rewards/margins": -0.009596443735063076,
      "rewards/rejected": -0.006798361893743277,
      "step": 2018
    },
    {
      "epoch": 0.5310624866428841,
      "grad_norm": 432.0,
      "learning_rate": 2.6052631578947366e-07,
      "logits/chosen": -0.6646480560302734,
      "logits/rejected": -0.662615954875946,
      "logps/chosen": -840.6588745117188,
      "logps/rejected": -446.89569091796875,
      "loss": 0.6967,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0053958892822265625,
      "rewards/margins": -0.0068048471584916115,
      "rewards/rejected": 0.012200737372040749,
      "step": 2019
    },
    {
      "epoch": 0.5313255190780714,
      "grad_norm": 448.0,
      "learning_rate": 2.6038011695906433e-07,
      "logits/chosen": -0.6596494913101196,
      "logits/rejected": -0.65712970495224,
      "logps/chosen": -626.386962890625,
      "logps/rejected": -684.917724609375,
      "loss": 0.6872,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008015109226107597,
      "rewards/margins": 0.012722352519631386,
      "rewards/rejected": -0.004707240499556065,
      "step": 2020
    },
    {
      "epoch": 0.5315885515132585,
      "grad_norm": 636.0,
      "learning_rate": 2.6023391812865495e-07,
      "logits/chosen": -0.67961585521698,
      "logits/rejected": -0.6762871742248535,
      "logps/chosen": -1281.493896484375,
      "logps/rejected": -1036.031494140625,
      "loss": 0.6912,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02151961252093315,
      "rewards/margins": 0.005198717582970858,
      "rewards/rejected": -0.02671833150088787,
      "step": 2021
    },
    {
      "epoch": 0.5318515839484457,
      "grad_norm": 528.0,
      "learning_rate": 2.600877192982456e-07,
      "logits/chosen": -0.6386237144470215,
      "logits/rejected": -0.6589130163192749,
      "logps/chosen": -1253.444580078125,
      "logps/rejected": -808.1363525390625,
      "loss": 0.6871,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010932637378573418,
      "rewards/margins": 0.013111782260239124,
      "rewards/rejected": -0.002179145347326994,
      "step": 2022
    },
    {
      "epoch": 0.5321146163836328,
      "grad_norm": 580.0,
      "learning_rate": 2.5994152046783625e-07,
      "logits/chosen": -0.6453545093536377,
      "logits/rejected": -0.6464083790779114,
      "logps/chosen": -1266.3048095703125,
      "logps/rejected": -981.954833984375,
      "loss": 0.6947,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0014492985792458057,
      "rewards/margins": -0.002131557324901223,
      "rewards/rejected": 0.0006822581635788083,
      "step": 2023
    },
    {
      "epoch": 0.53237764881882,
      "grad_norm": 636.0,
      "learning_rate": 2.597953216374269e-07,
      "logits/chosen": -0.6553102731704712,
      "logits/rejected": -0.65827876329422,
      "logps/chosen": -1411.461669921875,
      "logps/rejected": -1116.484619140625,
      "loss": 0.694,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004073524381965399,
      "rewards/margins": -0.0010450356639921665,
      "rewards/rejected": 0.00511856097728014,
      "step": 2024
    },
    {
      "epoch": 0.5326406812540071,
      "grad_norm": 644.0,
      "learning_rate": 2.596491228070175e-07,
      "logits/chosen": -0.6879883408546448,
      "logits/rejected": -0.6888246536254883,
      "logps/chosen": -1157.0101318359375,
      "logps/rejected": -1024.8541259765625,
      "loss": 0.6908,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.014966774731874466,
      "rewards/margins": 0.005181502550840378,
      "rewards/rejected": 0.009785271249711514,
      "step": 2025
    },
    {
      "epoch": 0.5329037136891943,
      "grad_norm": 652.0,
      "learning_rate": 2.5950292397660816e-07,
      "logits/chosen": -0.6584295034408569,
      "logits/rejected": NaN,
      "logps/chosen": -1053.253662109375,
      "logps/rejected": -982.4423217773438,
      "loss": 0.7013,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.007211780641227961,
      "rewards/margins": -0.015918826684355736,
      "rewards/rejected": 0.023130610585212708,
      "step": 2026
    },
    {
      "epoch": 0.5331667461243814,
      "grad_norm": 572.0,
      "learning_rate": 2.5935672514619883e-07,
      "logits/chosen": -0.6511206030845642,
      "logits/rejected": -0.6558772921562195,
      "logps/chosen": -1251.58837890625,
      "logps/rejected": -1208.9166259765625,
      "loss": 0.6844,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.009667444974184036,
      "rewards/margins": 0.017999600619077682,
      "rewards/rejected": -0.02766704559326172,
      "step": 2027
    },
    {
      "epoch": 0.5334297785595686,
      "grad_norm": 584.0,
      "learning_rate": 2.5921052631578945e-07,
      "logits/chosen": -0.6491626501083374,
      "logits/rejected": -0.6503750085830688,
      "logps/chosen": -1353.21484375,
      "logps/rejected": -1007.9444580078125,
      "loss": 0.6912,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.018172644078731537,
      "rewards/margins": 0.005348872393369675,
      "rewards/rejected": 0.012823773548007011,
      "step": 2028
    },
    {
      "epoch": 0.5336928109947557,
      "grad_norm": 572.0,
      "learning_rate": 2.5906432748538007e-07,
      "logits/chosen": -0.6628384590148926,
      "logits/rejected": NaN,
      "logps/chosen": -1515.6068115234375,
      "logps/rejected": -794.468017578125,
      "loss": 0.691,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.008052730932831764,
      "rewards/margins": 0.005446054041385651,
      "rewards/rejected": 0.0026066782884299755,
      "step": 2029
    },
    {
      "epoch": 0.533955843429943,
      "grad_norm": 516.0,
      "learning_rate": 2.5891812865497074e-07,
      "logits/chosen": -0.671928346157074,
      "logits/rejected": -0.6597479581832886,
      "logps/chosen": -1116.0577392578125,
      "logps/rejected": -866.7285766601562,
      "loss": 0.6978,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.01257181167602539,
      "rewards/margins": -0.008826927281916142,
      "rewards/rejected": 0.02139873616397381,
      "step": 2030
    },
    {
      "epoch": 0.5342188758651302,
      "grad_norm": 454.0,
      "learning_rate": 2.587719298245614e-07,
      "logits/chosen": -0.6381630897521973,
      "logits/rejected": -0.6522036194801331,
      "logps/chosen": -933.7434692382812,
      "logps/rejected": -768.5172119140625,
      "loss": 0.6975,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.011250877752900124,
      "rewards/margins": -0.007846640422940254,
      "rewards/rejected": 0.019097520038485527,
      "step": 2031
    },
    {
      "epoch": 0.5344819083003173,
      "grad_norm": 516.0,
      "learning_rate": 2.5862573099415204e-07,
      "logits/chosen": -0.6698979735374451,
      "logits/rejected": -0.6665815711021423,
      "logps/chosen": -1194.60791015625,
      "logps/rejected": -903.79736328125,
      "loss": 0.7027,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02280283160507679,
      "rewards/margins": -0.017675112932920456,
      "rewards/rejected": -0.0051277149468660355,
      "step": 2032
    },
    {
      "epoch": 0.5347449407355045,
      "grad_norm": 536.0,
      "learning_rate": 2.584795321637427e-07,
      "logits/chosen": -0.6406868696212769,
      "logits/rejected": NaN,
      "logps/chosen": -1233.7359619140625,
      "logps/rejected": -683.0427856445312,
      "loss": 0.6808,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.015240001492202282,
      "rewards/margins": 0.02607135847210884,
      "rewards/rejected": -0.010831356048583984,
      "step": 2033
    },
    {
      "epoch": 0.5350079731706916,
      "grad_norm": 504.0,
      "learning_rate": 2.5833333333333333e-07,
      "logits/chosen": -0.6291343569755554,
      "logits/rejected": -0.6253518462181091,
      "logps/chosen": -1161.4183349609375,
      "logps/rejected": -914.31982421875,
      "loss": 0.6755,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.023760603740811348,
      "rewards/margins": 0.03710803762078285,
      "rewards/rejected": -0.013347433879971504,
      "step": 2034
    },
    {
      "epoch": 0.5352710056058788,
      "grad_norm": 584.0,
      "learning_rate": 2.5818713450292395e-07,
      "logits/chosen": -0.6607261300086975,
      "logits/rejected": NaN,
      "logps/chosen": -1092.3726806640625,
      "logps/rejected": -940.4671020507812,
      "loss": 0.6867,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0057333954609930515,
      "rewards/margins": 0.013532066717743874,
      "rewards/rejected": -0.007798672653734684,
      "step": 2035
    },
    {
      "epoch": 0.5355340380410659,
      "grad_norm": 636.0,
      "learning_rate": 2.580409356725146e-07,
      "logits/chosen": -0.6608579754829407,
      "logits/rejected": -0.6661797761917114,
      "logps/chosen": -931.9480590820312,
      "logps/rejected": -729.6242065429688,
      "loss": 0.6923,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0004837033338844776,
      "rewards/margins": 0.002381800441071391,
      "rewards/rejected": -0.0018980973400175571,
      "step": 2036
    },
    {
      "epoch": 0.5357970704762531,
      "grad_norm": 536.0,
      "learning_rate": 2.578947368421053e-07,
      "logits/chosen": -0.6616108417510986,
      "logits/rejected": NaN,
      "logps/chosen": -1265.923583984375,
      "logps/rejected": -909.3123168945312,
      "loss": 0.6987,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01968269608914852,
      "rewards/margins": -0.009238054044544697,
      "rewards/rejected": -0.01044464111328125,
      "step": 2037
    },
    {
      "epoch": 0.5360601029114402,
      "grad_norm": 508.0,
      "learning_rate": 2.5774853801169586e-07,
      "logits/chosen": -0.6711745262145996,
      "logits/rejected": -0.6716340780258179,
      "logps/chosen": -904.4338989257812,
      "logps/rejected": -763.799560546875,
      "loss": 0.7041,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.022008752450346947,
      "rewards/margins": -0.021331358700990677,
      "rewards/rejected": -0.0006773949135094881,
      "step": 2038
    },
    {
      "epoch": 0.5363231353466275,
      "grad_norm": 452.0,
      "learning_rate": 2.5760233918128654e-07,
      "logits/chosen": -0.6469131708145142,
      "logits/rejected": -0.6497589945793152,
      "logps/chosen": -970.0980224609375,
      "logps/rejected": -709.885498046875,
      "loss": 0.6781,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0269620418548584,
      "rewards/margins": 0.031273745000362396,
      "rewards/rejected": -0.004311704076826572,
      "step": 2039
    },
    {
      "epoch": 0.5365861677818146,
      "grad_norm": 464.0,
      "learning_rate": 2.574561403508772e-07,
      "logits/chosen": -0.6501954793930054,
      "logits/rejected": -0.6496890187263489,
      "logps/chosen": -748.4427490234375,
      "logps/rejected": -932.7449951171875,
      "loss": 0.6901,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.029114246368408203,
      "rewards/margins": 0.006773756816983223,
      "rewards/rejected": 0.02234048955142498,
      "step": 2040
    },
    {
      "epoch": 0.5368492002170018,
      "grad_norm": 732.0,
      "learning_rate": 2.5730994152046783e-07,
      "logits/chosen": -0.6258752346038818,
      "logits/rejected": -0.6389397382736206,
      "logps/chosen": -898.711181640625,
      "logps/rejected": -798.106689453125,
      "loss": 0.6959,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0014630320947617292,
      "rewards/margins": -0.005230521783232689,
      "rewards/rejected": 0.006693554576486349,
      "step": 2041
    },
    {
      "epoch": 0.5371122326521889,
      "grad_norm": 516.0,
      "learning_rate": 2.5716374269005845e-07,
      "logits/chosen": -0.6353044509887695,
      "logits/rejected": -0.6488763093948364,
      "logps/chosen": -1170.58056640625,
      "logps/rejected": -882.1693115234375,
      "loss": 0.6877,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010907935909926891,
      "rewards/margins": 0.012245751917362213,
      "rewards/rejected": -0.0013378142612054944,
      "step": 2042
    },
    {
      "epoch": 0.5373752650873761,
      "grad_norm": 488.0,
      "learning_rate": 2.570175438596491e-07,
      "logits/chosen": -0.6444466710090637,
      "logits/rejected": -0.6532811522483826,
      "logps/chosen": -1006.960205078125,
      "logps/rejected": -724.4728393554688,
      "loss": 0.6874,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01789531484246254,
      "rewards/margins": 0.011850356124341488,
      "rewards/rejected": 0.006044959649443626,
      "step": 2043
    },
    {
      "epoch": 0.5376382975225632,
      "grad_norm": 572.0,
      "learning_rate": 2.5687134502923974e-07,
      "logits/chosen": -0.6485925316810608,
      "logits/rejected": -0.6451088190078735,
      "logps/chosen": -1113.0291748046875,
      "logps/rejected": -915.1234130859375,
      "loss": 0.709,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.010516928508877754,
      "rewards/margins": -0.030741121619939804,
      "rewards/rejected": 0.020224187523126602,
      "step": 2044
    },
    {
      "epoch": 0.5379013299577504,
      "grad_norm": 612.0,
      "learning_rate": 2.567251461988304e-07,
      "logits/chosen": -0.6474430561065674,
      "logits/rejected": -0.6567508578300476,
      "logps/chosen": -1228.0560302734375,
      "logps/rejected": -1198.325927734375,
      "loss": 0.6945,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.011737537570297718,
      "rewards/margins": -0.0020289418753236532,
      "rewards/rejected": 0.013766479678452015,
      "step": 2045
    },
    {
      "epoch": 0.5381643623929376,
      "grad_norm": 584.0,
      "learning_rate": 2.565789473684211e-07,
      "logits/chosen": -0.6637958288192749,
      "logits/rejected": -0.6641354560852051,
      "logps/chosen": -1362.1856689453125,
      "logps/rejected": -1007.0184326171875,
      "loss": 0.691,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.020270537585020065,
      "rewards/margins": 0.005010128021240234,
      "rewards/rejected": -0.0252806656062603,
      "step": 2046
    },
    {
      "epoch": 0.5384273948281247,
      "grad_norm": 512.0,
      "learning_rate": 2.5643274853801165e-07,
      "logits/chosen": -0.6267355680465698,
      "logits/rejected": -0.6384762525558472,
      "logps/chosen": -1045.2266845703125,
      "logps/rejected": -1003.9536743164062,
      "loss": 0.6975,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.009076405316591263,
      "rewards/margins": -0.00800953060388565,
      "rewards/rejected": -0.0010668756440281868,
      "step": 2047
    },
    {
      "epoch": 0.538690427263312,
      "grad_norm": 624.0,
      "learning_rate": 2.5628654970760233e-07,
      "logits/chosen": -0.6547138690948486,
      "logits/rejected": -0.6616547107696533,
      "logps/chosen": -1176.9974365234375,
      "logps/rejected": -1184.4613037109375,
      "loss": 0.7111,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.025040818378329277,
      "rewards/margins": -0.03480644151568413,
      "rewards/rejected": 0.009765625,
      "step": 2048
    },
    {
      "epoch": 0.5389534596984991,
      "grad_norm": 468.0,
      "learning_rate": 2.56140350877193e-07,
      "logits/chosen": -0.6383483409881592,
      "logits/rejected": -0.6391074061393738,
      "logps/chosen": -1207.5433349609375,
      "logps/rejected": -988.6400146484375,
      "loss": 0.6931,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0007257469114847481,
      "rewards/margins": 0.001391078345477581,
      "rewards/rejected": -0.0006653306190855801,
      "step": 2049
    },
    {
      "epoch": 0.5392164921336863,
      "grad_norm": 776.0,
      "learning_rate": 2.559941520467836e-07,
      "logits/chosen": -0.6450375318527222,
      "logits/rejected": NaN,
      "logps/chosen": -1267.353759765625,
      "logps/rejected": -1094.9927978515625,
      "loss": 0.7031,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.008515167981386185,
      "rewards/margins": -0.018715761601924896,
      "rewards/rejected": 0.02723092958331108,
      "step": 2050
    },
    {
      "epoch": 0.5394795245688734,
      "grad_norm": 716.0,
      "learning_rate": 2.5584795321637424e-07,
      "logits/chosen": -0.6817431449890137,
      "logits/rejected": -0.6668984293937683,
      "logps/chosen": -1498.78759765625,
      "logps/rejected": -1306.0350341796875,
      "loss": 0.6865,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0036191223189234734,
      "rewards/margins": 0.014210437424480915,
      "rewards/rejected": -0.010591315105557442,
      "step": 2051
    },
    {
      "epoch": 0.5397425570040606,
      "grad_norm": 452.0,
      "learning_rate": 2.557017543859649e-07,
      "logits/chosen": -0.6614714860916138,
      "logits/rejected": -0.6517232656478882,
      "logps/chosen": -950.5807495117188,
      "logps/rejected": -618.783447265625,
      "loss": 0.6869,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00901050679385662,
      "rewards/margins": 0.012822533026337624,
      "rewards/rejected": -0.0038120269309729338,
      "step": 2052
    },
    {
      "epoch": 0.5400055894392477,
      "grad_norm": 568.0,
      "learning_rate": 2.5555555555555553e-07,
      "logits/chosen": -0.6357442140579224,
      "logits/rejected": NaN,
      "logps/chosen": -1169.5107421875,
      "logps/rejected": -748.765869140625,
      "loss": 0.6871,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.001180171617306769,
      "rewards/margins": 0.012861013412475586,
      "rewards/rejected": -0.011680841445922852,
      "step": 2053
    },
    {
      "epoch": 0.5402686218744349,
      "grad_norm": 796.0,
      "learning_rate": 2.554093567251462e-07,
      "logits/chosen": -0.6543492674827576,
      "logits/rejected": -0.659142017364502,
      "logps/chosen": -1178.896484375,
      "logps/rejected": -963.8231811523438,
      "loss": 0.6846,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.017774201929569244,
      "rewards/margins": 0.017586709931492805,
      "rewards/rejected": 0.0001874924637377262,
      "step": 2054
    },
    {
      "epoch": 0.540531654309622,
      "grad_norm": 676.0,
      "learning_rate": 2.552631578947368e-07,
      "logits/chosen": -0.6128685474395752,
      "logits/rejected": -0.6257098913192749,
      "logps/chosen": -1231.5537109375,
      "logps/rejected": -1001.2732543945312,
      "loss": 0.6912,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0009699815418571234,
      "rewards/margins": 0.004302883055061102,
      "rewards/rejected": -0.005272865295410156,
      "step": 2055
    },
    {
      "epoch": 0.5407946867448092,
      "grad_norm": 580.0,
      "learning_rate": 2.5511695906432744e-07,
      "logits/chosen": -0.6352294087409973,
      "logits/rejected": -0.640727162361145,
      "logps/chosen": -1421.032958984375,
      "logps/rejected": -927.01708984375,
      "loss": 0.6968,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.014508439227938652,
      "rewards/margins": -0.006845569238066673,
      "rewards/rejected": -0.007662868592888117,
      "step": 2056
    },
    {
      "epoch": 0.5410577191799963,
      "grad_norm": 568.0,
      "learning_rate": 2.549707602339181e-07,
      "logits/chosen": -0.6488056182861328,
      "logits/rejected": -0.6392849087715149,
      "logps/chosen": -1046.2611083984375,
      "logps/rejected": -875.9793090820312,
      "loss": 0.6833,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.014600562863051891,
      "rewards/margins": 0.020360374823212624,
      "rewards/rejected": -0.005759811494499445,
      "step": 2057
    },
    {
      "epoch": 0.5413207516151836,
      "grad_norm": 500.0,
      "learning_rate": 2.548245614035088e-07,
      "logits/chosen": -0.6385636925697327,
      "logits/rejected": -0.6528871059417725,
      "logps/chosen": -1090.6082763671875,
      "logps/rejected": -764.0452270507812,
      "loss": 0.6867,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0066543580032885075,
      "rewards/margins": 0.013349294662475586,
      "rewards/rejected": -0.0066949366591870785,
      "step": 2058
    },
    {
      "epoch": 0.5415837840503707,
      "grad_norm": 540.0,
      "learning_rate": 2.5467836257309936e-07,
      "logits/chosen": -0.6356393098831177,
      "logits/rejected": -0.6360955238342285,
      "logps/chosen": -1224.072998046875,
      "logps/rejected": -954.64892578125,
      "loss": 0.6935,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.001865195925347507,
      "rewards/margins": -0.000325583852827549,
      "rewards/rejected": 0.0021907822228968143,
      "step": 2059
    },
    {
      "epoch": 0.5418468164855579,
      "grad_norm": 404.0,
      "learning_rate": 2.5453216374269003e-07,
      "logits/chosen": -0.6237249374389648,
      "logits/rejected": -0.6261767148971558,
      "logps/chosen": -883.5442504882812,
      "logps/rejected": -580.947021484375,
      "loss": 0.6853,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.012080811895430088,
      "rewards/margins": 0.016324138268828392,
      "rewards/rejected": -0.028404951095581055,
      "step": 2060
    },
    {
      "epoch": 0.542109848920745,
      "grad_norm": 536.0,
      "learning_rate": 2.543859649122807e-07,
      "logits/chosen": -0.6578695774078369,
      "logits/rejected": -0.6669278144836426,
      "logps/chosen": -1275.99609375,
      "logps/rejected": -964.6803588867188,
      "loss": 0.6932,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0003777500241994858,
      "rewards/margins": 0.00022854795679450035,
      "rewards/rejected": -0.0006062984466552734,
      "step": 2061
    },
    {
      "epoch": 0.5423728813559322,
      "grad_norm": 484.0,
      "learning_rate": 2.542397660818713e-07,
      "logits/chosen": -0.6763983964920044,
      "logits/rejected": -0.6654254794120789,
      "logps/chosen": -870.3554077148438,
      "logps/rejected": -1036.0810546875,
      "loss": 0.6914,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010549163445830345,
      "rewards/margins": 0.004953147377818823,
      "rewards/rejected": 0.005596017464995384,
      "step": 2062
    },
    {
      "epoch": 0.5426359137911194,
      "grad_norm": 500.0,
      "learning_rate": 2.54093567251462e-07,
      "logits/chosen": -0.6586225032806396,
      "logits/rejected": -0.666810154914856,
      "logps/chosen": -1175.810546875,
      "logps/rejected": -954.5195922851562,
      "loss": 0.6917,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0018543255282565951,
      "rewards/margins": 0.0035755163989961147,
      "rewards/rejected": -0.0017211916856467724,
      "step": 2063
    },
    {
      "epoch": 0.5428989462263065,
      "grad_norm": 552.0,
      "learning_rate": 2.539473684210526e-07,
      "logits/chosen": -0.6443628668785095,
      "logits/rejected": -0.6329385042190552,
      "logps/chosen": -1004.2349243164062,
      "logps/rejected": -897.9661254882812,
      "loss": 0.6895,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.010365201160311699,
      "rewards/margins": 0.007630730047821999,
      "rewards/rejected": 0.0027344697155058384,
      "step": 2064
    },
    {
      "epoch": 0.5431619786614937,
      "grad_norm": 580.0,
      "learning_rate": 2.5380116959064324e-07,
      "logits/chosen": -0.6543387174606323,
      "logits/rejected": -0.6441087126731873,
      "logps/chosen": -1094.74560546875,
      "logps/rejected": -725.447509765625,
      "loss": 0.6907,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00558099802583456,
      "rewards/margins": 0.0056455605663359165,
      "rewards/rejected": -6.456440314650536e-05,
      "step": 2065
    },
    {
      "epoch": 0.5434250110966808,
      "grad_norm": 616.0,
      "learning_rate": 2.536549707602339e-07,
      "logits/chosen": -0.6692767143249512,
      "logits/rejected": -0.671180784702301,
      "logps/chosen": -1321.36767578125,
      "logps/rejected": -1134.7076416015625,
      "loss": 0.6855,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.009852123446762562,
      "rewards/margins": 0.015758514404296875,
      "rewards/rejected": -0.005906390957534313,
      "step": 2066
    },
    {
      "epoch": 0.5436880435318681,
      "grad_norm": 588.0,
      "learning_rate": 2.535087719298246e-07,
      "logits/chosen": -0.6544338464736938,
      "logits/rejected": -0.6533443331718445,
      "logps/chosen": -1617.062744140625,
      "logps/rejected": -1007.1168823242188,
      "loss": 0.6896,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.015534592792391777,
      "rewards/margins": 0.007906151004135609,
      "rewards/rejected": 0.007628441788256168,
      "step": 2067
    },
    {
      "epoch": 0.5439510759670552,
      "grad_norm": 880.0,
      "learning_rate": 2.5336257309941515e-07,
      "logits/chosen": -0.6806665658950806,
      "logits/rejected": -0.680922269821167,
      "logps/chosen": -1857.337158203125,
      "logps/rejected": -1411.9329833984375,
      "loss": 0.7073,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.02002725750207901,
      "rewards/margins": -0.02734527736902237,
      "rewards/rejected": 0.007318019401282072,
      "step": 2068
    },
    {
      "epoch": 0.5442141084022424,
      "grad_norm": 520.0,
      "learning_rate": 2.532163742690058e-07,
      "logits/chosen": -0.6400355100631714,
      "logits/rejected": -0.6422351002693176,
      "logps/chosen": -1341.2362060546875,
      "logps/rejected": -1214.2994384765625,
      "loss": 0.6859,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02538261190056801,
      "rewards/margins": 0.0172092467546463,
      "rewards/rejected": 0.008173370733857155,
      "step": 2069
    },
    {
      "epoch": 0.5444771408374295,
      "grad_norm": 592.0,
      "learning_rate": 2.530701754385965e-07,
      "logits/chosen": -0.6525245308876038,
      "logits/rejected": -0.6443248987197876,
      "logps/chosen": -1140.9150390625,
      "logps/rejected": -890.05712890625,
      "loss": 0.6832,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.02288074418902397,
      "rewards/margins": 0.020295429974794388,
      "rewards/rejected": 0.0025853165425360203,
      "step": 2070
    },
    {
      "epoch": 0.5447401732726167,
      "grad_norm": 464.0,
      "learning_rate": 2.5292397660818717e-07,
      "logits/chosen": -0.6709978580474854,
      "logits/rejected": -0.6575424671173096,
      "logps/chosen": -967.2338256835938,
      "logps/rejected": -903.9949951171875,
      "loss": 0.6893,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0015330321621149778,
      "rewards/margins": 0.008303355425596237,
      "rewards/rejected": -0.006770324427634478,
      "step": 2071
    },
    {
      "epoch": 0.5450032057078038,
      "grad_norm": 540.0,
      "learning_rate": 2.5277777777777773e-07,
      "logits/chosen": -0.657374382019043,
      "logits/rejected": -0.6578062176704407,
      "logps/chosen": -1256.3585205078125,
      "logps/rejected": -737.0791015625,
      "loss": 0.6961,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.011233569122850895,
      "rewards/margins": -0.005289411172270775,
      "rewards/rejected": 0.016522981226444244,
      "step": 2072
    },
    {
      "epoch": 0.545266238142991,
      "grad_norm": 498.0,
      "learning_rate": 2.526315789473684e-07,
      "logits/chosen": -0.6656746864318848,
      "logits/rejected": -0.6746309399604797,
      "logps/chosen": -1123.196533203125,
      "logps/rejected": -750.6597900390625,
      "loss": 0.6934,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.007451439276337624,
      "rewards/margins": 1.7642509192228317e-05,
      "rewards/rejected": -0.007469081785529852,
      "step": 2073
    },
    {
      "epoch": 0.5455292705781781,
      "grad_norm": 536.0,
      "learning_rate": 2.524853801169591e-07,
      "logits/chosen": -0.6515939235687256,
      "logits/rejected": -0.6663641929626465,
      "logps/chosen": -1066.4537353515625,
      "logps/rejected": -895.4622802734375,
      "loss": 0.6871,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.013193322345614433,
      "rewards/margins": 0.013770485296845436,
      "rewards/rejected": -0.0005771629512310028,
      "step": 2074
    },
    {
      "epoch": 0.5457923030133653,
      "grad_norm": 624.0,
      "learning_rate": 2.523391812865497e-07,
      "logits/chosen": -0.6721249222755432,
      "logits/rejected": -0.6742486953735352,
      "logps/chosen": -1095.161376953125,
      "logps/rejected": -981.4461059570312,
      "loss": 0.6968,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.006241799332201481,
      "rewards/margins": -0.006508063990622759,
      "rewards/rejected": 0.012749862857162952,
      "step": 2075
    },
    {
      "epoch": 0.5460553354485524,
      "grad_norm": 516.0,
      "learning_rate": 2.5219298245614037e-07,
      "logits/chosen": -0.6552441716194153,
      "logits/rejected": -0.6675000786781311,
      "logps/chosen": -1306.6944580078125,
      "logps/rejected": -880.5905151367188,
      "loss": 0.6982,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.007732105441391468,
      "rewards/margins": -0.009510993957519531,
      "rewards/rejected": 0.0017788882832974195,
      "step": 2076
    },
    {
      "epoch": 0.5463183678837397,
      "grad_norm": 636.0,
      "learning_rate": 2.52046783625731e-07,
      "logits/chosen": -0.688215970993042,
      "logits/rejected": -0.6807788014411926,
      "logps/chosen": -1301.3525390625,
      "logps/rejected": -881.2572631835938,
      "loss": 0.6967,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01094741839915514,
      "rewards/margins": -0.005436942912638187,
      "rewards/rejected": -0.005510473623871803,
      "step": 2077
    },
    {
      "epoch": 0.5465814003189269,
      "grad_norm": 580.0,
      "learning_rate": 2.519005847953216e-07,
      "logits/chosen": -0.6143941283226013,
      "logits/rejected": -0.6223238706588745,
      "logps/chosen": -1295.77490234375,
      "logps/rejected": -859.6782836914062,
      "loss": 0.6939,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.01791982352733612,
      "rewards/margins": -0.0009469036012887955,
      "rewards/rejected": 0.018866730853915215,
      "step": 2078
    },
    {
      "epoch": 0.546844432754114,
      "grad_norm": 512.0,
      "learning_rate": 2.517543859649123e-07,
      "logits/chosen": -0.639282763004303,
      "logits/rejected": -0.6353256702423096,
      "logps/chosen": -900.3781127929688,
      "logps/rejected": -1078.6549072265625,
      "loss": 0.6891,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.012849998660385609,
      "rewards/margins": 0.008932113647460938,
      "rewards/rejected": -0.02178211137652397,
      "step": 2079
    },
    {
      "epoch": 0.5471074651893012,
      "grad_norm": 456.0,
      "learning_rate": 2.5160818713450296e-07,
      "logits/chosen": -0.662697434425354,
      "logits/rejected": -0.6640721559524536,
      "logps/chosen": -865.4141235351562,
      "logps/rejected": -569.3323364257812,
      "loss": 0.6909,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009276581928133965,
      "rewards/margins": 0.005370808765292168,
      "rewards/rejected": 0.0039057733956724405,
      "step": 2080
    },
    {
      "epoch": 0.5473704976244883,
      "grad_norm": 450.0,
      "learning_rate": 2.514619883040935e-07,
      "logits/chosen": -0.6570497751235962,
      "logits/rejected": -0.6520885229110718,
      "logps/chosen": -904.108642578125,
      "logps/rejected": -658.6132202148438,
      "loss": 0.6769,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02866191789507866,
      "rewards/margins": 0.033807944506406784,
      "rewards/rejected": -0.005146026611328125,
      "step": 2081
    },
    {
      "epoch": 0.5476335300596755,
      "grad_norm": 494.0,
      "learning_rate": 2.513157894736842e-07,
      "logits/chosen": -0.6476973295211792,
      "logits/rejected": -0.6597459316253662,
      "logps/chosen": -971.3399047851562,
      "logps/rejected": -892.6032104492188,
      "loss": 0.7016,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.019800281152129173,
      "rewards/margins": -0.015817642211914062,
      "rewards/rejected": -0.003982640337198973,
      "step": 2082
    },
    {
      "epoch": 0.5478965624948626,
      "grad_norm": 640.0,
      "learning_rate": 2.5116959064327487e-07,
      "logits/chosen": -0.6579195261001587,
      "logits/rejected": -0.6484495401382446,
      "logps/chosen": -1176.302001953125,
      "logps/rejected": -1082.756591796875,
      "loss": 0.707,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.02235136181116104,
      "rewards/margins": -0.027108576148748398,
      "rewards/rejected": 0.004757214803248644,
      "step": 2083
    },
    {
      "epoch": 0.5481595949300498,
      "grad_norm": 502.0,
      "learning_rate": 2.510233918128655e-07,
      "logits/chosen": -0.6612834334373474,
      "logits/rejected": -0.6593655943870544,
      "logps/chosen": -988.9846801757812,
      "logps/rejected": -842.0853881835938,
      "loss": 0.6828,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.018616773188114166,
      "rewards/margins": 0.02177267149090767,
      "rewards/rejected": -0.0031558985356241465,
      "step": 2084
    },
    {
      "epoch": 0.548422627365237,
      "grad_norm": 428.0,
      "learning_rate": 2.508771929824561e-07,
      "logits/chosen": -0.6503183245658875,
      "logits/rejected": -0.6619398593902588,
      "logps/chosen": -839.416259765625,
      "logps/rejected": -526.9024658203125,
      "loss": 0.6935,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.005315686110407114,
      "rewards/margins": -0.000324249267578125,
      "rewards/rejected": -0.004991436842828989,
      "step": 2085
    },
    {
      "epoch": 0.5486856598004242,
      "grad_norm": 532.0,
      "learning_rate": 2.507309941520468e-07,
      "logits/chosen": -0.6634956002235413,
      "logits/rejected": -0.6718443632125854,
      "logps/chosen": -1498.2984619140625,
      "logps/rejected": -1081.372314453125,
      "loss": 0.6936,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -7.238425314426422e-05,
      "rewards/margins": -0.0006447798223234713,
      "rewards/rejected": 0.0005723950453102589,
      "step": 2086
    },
    {
      "epoch": 0.5489486922356113,
      "grad_norm": 540.0,
      "learning_rate": 2.505847953216374e-07,
      "logits/chosen": -0.660183310508728,
      "logits/rejected": -0.6533533334732056,
      "logps/chosen": -933.6494140625,
      "logps/rejected": -863.5472412109375,
      "loss": 0.6959,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007175446022301912,
      "rewards/margins": -0.004560280125588179,
      "rewards/rejected": 0.011735724285244942,
      "step": 2087
    },
    {
      "epoch": 0.5492117246707985,
      "grad_norm": 494.0,
      "learning_rate": 2.504385964912281e-07,
      "logits/chosen": -0.6637248396873474,
      "logits/rejected": -0.6741417050361633,
      "logps/chosen": -1281.0277099609375,
      "logps/rejected": -899.1395874023438,
      "loss": 0.702,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.013267850503325462,
      "rewards/margins": -0.01711726002395153,
      "rewards/rejected": 0.0038494111504405737,
      "step": 2088
    },
    {
      "epoch": 0.5494747571059856,
      "grad_norm": 604.0,
      "learning_rate": 2.502923976608187e-07,
      "logits/chosen": -0.6500038504600525,
      "logits/rejected": NaN,
      "logps/chosen": -1309.8697509765625,
      "logps/rejected": -978.6716918945312,
      "loss": 0.6779,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.006625956390053034,
      "rewards/margins": 0.031221792101860046,
      "rewards/rejected": -0.024595832452178,
      "step": 2089
    },
    {
      "epoch": 0.5497377895411728,
      "grad_norm": 604.0,
      "learning_rate": 2.501461988304093e-07,
      "logits/chosen": -0.6511635780334473,
      "logits/rejected": -0.6547532081604004,
      "logps/chosen": -1188.41845703125,
      "logps/rejected": -1170.4193115234375,
      "loss": 0.6873,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007772063836455345,
      "rewards/margins": 0.012100791558623314,
      "rewards/rejected": -0.004328727722167969,
      "step": 2090
    },
    {
      "epoch": 0.5500008219763599,
      "grad_norm": 500.0,
      "learning_rate": 2.5e-07,
      "logits/chosen": -0.6424809694290161,
      "logits/rejected": -0.656071662902832,
      "logps/chosen": -1147.351806640625,
      "logps/rejected": -1119.4542236328125,
      "loss": 0.6969,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.023607924580574036,
      "rewards/margins": -0.006562138441950083,
      "rewards/rejected": -0.01704578287899494,
      "step": 2091
    },
    {
      "epoch": 0.5502638544115471,
      "grad_norm": 482.0,
      "learning_rate": 2.498538011695906e-07,
      "logits/chosen": -0.6438482999801636,
      "logits/rejected": -0.651241660118103,
      "logps/chosen": -1017.0836791992188,
      "logps/rejected": -758.5673217773438,
      "loss": 0.6973,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0171419158577919,
      "rewards/margins": -0.00667314138263464,
      "rewards/rejected": -0.010468769818544388,
      "step": 2092
    },
    {
      "epoch": 0.5505268868467343,
      "grad_norm": 612.0,
      "learning_rate": 2.497076023391813e-07,
      "logits/chosen": -0.6450794339179993,
      "logits/rejected": -0.6575045585632324,
      "logps/chosen": -1481.6826171875,
      "logps/rejected": -1176.04345703125,
      "loss": 0.6979,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0002474784851074219,
      "rewards/margins": -0.00870437640696764,
      "rewards/rejected": 0.008951854892075062,
      "step": 2093
    },
    {
      "epoch": 0.5507899192819214,
      "grad_norm": 584.0,
      "learning_rate": 2.495614035087719e-07,
      "logits/chosen": -0.6626890897750854,
      "logits/rejected": -0.6815874576568604,
      "logps/chosen": -1311.09521484375,
      "logps/rejected": -978.0084228515625,
      "loss": 0.7005,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.021292688325047493,
      "rewards/margins": -0.013706399127840996,
      "rewards/rejected": -0.007586287800222635,
      "step": 2094
    },
    {
      "epoch": 0.5510529517171087,
      "grad_norm": 672.0,
      "learning_rate": 2.494152046783626e-07,
      "logits/chosen": -0.6638946533203125,
      "logits/rejected": NaN,
      "logps/chosen": -1620.7548828125,
      "logps/rejected": -964.228515625,
      "loss": 0.6894,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009763861075043678,
      "rewards/margins": 0.008528996258974075,
      "rewards/rejected": 0.001234865514561534,
      "step": 2095
    },
    {
      "epoch": 0.5513159841522958,
      "grad_norm": 596.0,
      "learning_rate": 2.492690058479532e-07,
      "logits/chosen": -0.6257883310317993,
      "logits/rejected": -0.6368396282196045,
      "logps/chosen": -1307.927978515625,
      "logps/rejected": -975.64208984375,
      "loss": 0.693,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0202102642506361,
      "rewards/margins": 0.0016018853057175875,
      "rewards/rejected": 0.01860838197171688,
      "step": 2096
    },
    {
      "epoch": 0.551579016587483,
      "grad_norm": 556.0,
      "learning_rate": 2.4912280701754387e-07,
      "logits/chosen": -0.6640546321868896,
      "logits/rejected": -0.6666496396064758,
      "logps/chosen": -1249.804931640625,
      "logps/rejected": -1025.545654296875,
      "loss": 0.6861,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.012216760776937008,
      "rewards/margins": 0.015326401218771935,
      "rewards/rejected": -0.003109646262601018,
      "step": 2097
    },
    {
      "epoch": 0.5518420490226701,
      "grad_norm": 544.0,
      "learning_rate": 2.489766081871345e-07,
      "logits/chosen": -0.6382324695587158,
      "logits/rejected": -0.6325074434280396,
      "logps/chosen": -1097.919677734375,
      "logps/rejected": -880.9474487304688,
      "loss": 0.6896,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.015164757147431374,
      "rewards/margins": 0.008364249020814896,
      "rewards/rejected": 0.006800508126616478,
      "step": 2098
    },
    {
      "epoch": 0.5521050814578573,
      "grad_norm": 604.0,
      "learning_rate": 2.4883040935672516e-07,
      "logits/chosen": -0.6422406435012817,
      "logits/rejected": -0.6508408784866333,
      "logps/chosen": -1440.7686767578125,
      "logps/rejected": -1082.11572265625,
      "loss": 0.6805,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.011650847271084785,
      "rewards/margins": 0.026743698865175247,
      "rewards/rejected": -0.015092849731445312,
      "step": 2099
    },
    {
      "epoch": 0.5523681138930444,
      "grad_norm": 484.0,
      "learning_rate": 2.486842105263158e-07,
      "logits/chosen": -0.6495321989059448,
      "logits/rejected": -0.6496573090553284,
      "logps/chosen": -1270.7313232421875,
      "logps/rejected": -837.55126953125,
      "loss": 0.6968,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.005994510371237993,
      "rewards/margins": -0.006331349723041058,
      "rewards/rejected": 0.0003368381876498461,
      "step": 2100
    },
    {
      "epoch": 0.5526311463282316,
      "grad_norm": 544.0,
      "learning_rate": 2.485380116959064e-07,
      "logits/chosen": -0.6638900637626648,
      "logits/rejected": -0.6573318243026733,
      "logps/chosen": -1296.032470703125,
      "logps/rejected": -997.5059204101562,
      "loss": 0.6999,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004212379455566406,
      "rewards/margins": -0.012440967373549938,
      "rewards/rejected": 0.01665334776043892,
      "step": 2101
    },
    {
      "epoch": 0.5528941787634187,
      "grad_norm": 396.0,
      "learning_rate": 2.4839181286549707e-07,
      "logits/chosen": -0.6596291661262512,
      "logits/rejected": -0.6636720299720764,
      "logps/chosen": -866.1340942382812,
      "logps/rejected": -745.0870361328125,
      "loss": 0.69,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.007443046662956476,
      "rewards/margins": 0.006827354431152344,
      "rewards/rejected": 0.0006156919989734888,
      "step": 2102
    },
    {
      "epoch": 0.553157211198606,
      "grad_norm": 560.0,
      "learning_rate": 2.482456140350877e-07,
      "logits/chosen": -0.6594558954238892,
      "logits/rejected": -0.6640085577964783,
      "logps/chosen": -1112.624755859375,
      "logps/rejected": -1049.90185546875,
      "loss": 0.6987,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.015031338669359684,
      "rewards/margins": -0.01069407444447279,
      "rewards/rejected": -0.00433726329356432,
      "step": 2103
    },
    {
      "epoch": 0.553420243633793,
      "grad_norm": 860.0,
      "learning_rate": 2.4809941520467837e-07,
      "logits/chosen": -0.6668755412101746,
      "logits/rejected": -0.6840710043907166,
      "logps/chosen": -971.2101440429688,
      "logps/rejected": -1222.37939453125,
      "loss": 0.6923,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006983185186982155,
      "rewards/margins": 0.0022718445397913456,
      "rewards/rejected": 0.0047113411128520966,
      "step": 2104
    },
    {
      "epoch": 0.5536832760689803,
      "grad_norm": 576.0,
      "learning_rate": 2.47953216374269e-07,
      "logits/chosen": -0.6677412986755371,
      "logits/rejected": -0.6844832897186279,
      "logps/chosen": -1023.978515625,
      "logps/rejected": -1055.959716796875,
      "loss": 0.6883,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00510988337919116,
      "rewards/margins": 0.010746289044618607,
      "rewards/rejected": -0.005636405665427446,
      "step": 2105
    },
    {
      "epoch": 0.5539463085041674,
      "grad_norm": 700.0,
      "learning_rate": 2.478070175438596e-07,
      "logits/chosen": -0.6420900225639343,
      "logits/rejected": -0.657090425491333,
      "logps/chosen": -1091.73291015625,
      "logps/rejected": -917.14501953125,
      "loss": 0.6889,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003966236487030983,
      "rewards/margins": 0.00918436050415039,
      "rewards/rejected": -0.005218124948441982,
      "step": 2106
    },
    {
      "epoch": 0.5542093409393546,
      "grad_norm": 612.0,
      "learning_rate": 2.476608187134503e-07,
      "logits/chosen": -0.6504570245742798,
      "logits/rejected": -0.657531201839447,
      "logps/chosen": -1371.7615966796875,
      "logps/rejected": -937.1257934570312,
      "loss": 0.7018,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.027631187811493874,
      "rewards/margins": -0.015924740582704544,
      "rewards/rejected": -0.01170644722878933,
      "step": 2107
    },
    {
      "epoch": 0.5544723733745417,
      "grad_norm": 494.0,
      "learning_rate": 2.4751461988304095e-07,
      "logits/chosen": -0.6390912532806396,
      "logits/rejected": -0.6345586776733398,
      "logps/chosen": -867.8247680664062,
      "logps/rejected": -847.51708984375,
      "loss": 0.6969,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00996789988130331,
      "rewards/margins": -0.006855201907455921,
      "rewards/rejected": 0.01682310178875923,
      "step": 2108
    },
    {
      "epoch": 0.5547354058097289,
      "grad_norm": 416.0,
      "learning_rate": 2.4736842105263157e-07,
      "logits/chosen": -0.6430820226669312,
      "logits/rejected": -0.6431914567947388,
      "logps/chosen": -734.6245727539062,
      "logps/rejected": -714.474609375,
      "loss": 0.7026,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.007832622155547142,
      "rewards/margins": -0.018352413550019264,
      "rewards/rejected": 0.026185033842921257,
      "step": 2109
    },
    {
      "epoch": 0.5549984382449161,
      "grad_norm": 680.0,
      "learning_rate": 2.4722222222222224e-07,
      "logits/chosen": -0.66390061378479,
      "logits/rejected": -0.6686225533485413,
      "logps/chosen": -1025.307373046875,
      "logps/rejected": -1019.9459838867188,
      "loss": 0.6897,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0041005611419677734,
      "rewards/margins": 0.007967853918671608,
      "rewards/rejected": -0.0038672927767038345,
      "step": 2110
    },
    {
      "epoch": 0.5552614706801032,
      "grad_norm": 548.0,
      "learning_rate": 2.4707602339181286e-07,
      "logits/chosen": NaN,
      "logits/rejected": NaN,
      "logps/chosen": -923.60205078125,
      "logps/rejected": -789.0018310546875,
      "loss": 0.699,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00811061728745699,
      "rewards/margins": -0.009888552129268646,
      "rewards/rejected": 0.01799917221069336,
      "step": 2111
    },
    {
      "epoch": 0.5555245031152904,
      "grad_norm": 450.0,
      "learning_rate": 2.469298245614035e-07,
      "logits/chosen": -0.6397450566291809,
      "logits/rejected": -0.6417497992515564,
      "logps/chosen": -1039.00390625,
      "logps/rejected": -895.175537109375,
      "loss": 0.6802,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02609548717737198,
      "rewards/margins": 0.0264605525881052,
      "rewards/rejected": -0.0003650668077170849,
      "step": 2112
    },
    {
      "epoch": 0.5557875355504776,
      "grad_norm": 478.0,
      "learning_rate": 2.4678362573099416e-07,
      "logits/chosen": -0.6507079601287842,
      "logits/rejected": -0.6412315368652344,
      "logps/chosen": -1184.132080078125,
      "logps/rejected": -1040.3541259765625,
      "loss": 0.6881,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02095813676714897,
      "rewards/margins": 0.010799692943692207,
      "rewards/rejected": 0.01015844289213419,
      "step": 2113
    },
    {
      "epoch": 0.5560505679856648,
      "grad_norm": 502.0,
      "learning_rate": 2.466374269005848e-07,
      "logits/chosen": -0.6608831882476807,
      "logits/rejected": -0.6706328392028809,
      "logps/chosen": -1246.78076171875,
      "logps/rejected": -1164.04150390625,
      "loss": 0.7005,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0306683536618948,
      "rewards/margins": -0.01416158676147461,
      "rewards/rejected": -0.016506768763065338,
      "step": 2114
    },
    {
      "epoch": 0.5563136004208519,
      "grad_norm": 552.0,
      "learning_rate": 2.464912280701754e-07,
      "logits/chosen": -0.6810302138328552,
      "logits/rejected": -0.6825247406959534,
      "logps/chosen": -1207.7398681640625,
      "logps/rejected": -849.8790283203125,
      "loss": 0.6846,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01983671262860298,
      "rewards/margins": 0.017732907086610794,
      "rewards/rejected": 0.002103806007653475,
      "step": 2115
    },
    {
      "epoch": 0.5565766328560391,
      "grad_norm": 628.0,
      "learning_rate": 2.4634502923976607e-07,
      "logits/chosen": -0.6579961180686951,
      "logits/rejected": -0.6529919505119324,
      "logps/chosen": -1108.065673828125,
      "logps/rejected": -1081.2496337890625,
      "loss": 0.6929,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0020042420364916325,
      "rewards/margins": 0.0014355664607137442,
      "rewards/rejected": 0.0005686766235157847,
      "step": 2116
    },
    {
      "epoch": 0.5568396652912262,
      "grad_norm": 544.0,
      "learning_rate": 2.461988304093567e-07,
      "logits/chosen": -0.6547476053237915,
      "logits/rejected": -0.6545785069465637,
      "logps/chosen": -1090.1866455078125,
      "logps/rejected": -789.9824829101562,
      "loss": 0.7048,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.009809495881199837,
      "rewards/margins": -0.02231893315911293,
      "rewards/rejected": 0.012509441003203392,
      "step": 2117
    },
    {
      "epoch": 0.5571026977264134,
      "grad_norm": 568.0,
      "learning_rate": 2.4605263157894736e-07,
      "logits/chosen": -0.6573109030723572,
      "logits/rejected": NaN,
      "logps/chosen": -1296.5435791015625,
      "logps/rejected": -808.1644897460938,
      "loss": 0.7003,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0013715741224586964,
      "rewards/margins": -0.012585639022290707,
      "rewards/rejected": 0.01395721361041069,
      "step": 2118
    },
    {
      "epoch": 0.5573657301616005,
      "grad_norm": 540.0,
      "learning_rate": 2.45906432748538e-07,
      "logits/chosen": -0.6653050184249878,
      "logits/rejected": -0.6686209440231323,
      "logps/chosen": -1177.71875,
      "logps/rejected": -768.078857421875,
      "loss": 0.6896,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.004893493372946978,
      "rewards/margins": 0.00799169484525919,
      "rewards/rejected": -0.01288518961519003,
      "step": 2119
    },
    {
      "epoch": 0.5576287625967877,
      "grad_norm": 532.0,
      "learning_rate": 2.4576023391812865e-07,
      "logits/chosen": -0.6381896138191223,
      "logits/rejected": -0.6389558911323547,
      "logps/chosen": -1175.8544921875,
      "logps/rejected": -748.31005859375,
      "loss": 0.6877,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01205434836447239,
      "rewards/margins": 0.011772632598876953,
      "rewards/rejected": 0.0002817139611579478,
      "step": 2120
    },
    {
      "epoch": 0.5578917950319748,
      "grad_norm": 536.0,
      "learning_rate": 2.456140350877193e-07,
      "logits/chosen": -0.652552604675293,
      "logits/rejected": -0.6603278517723083,
      "logps/chosen": -1151.552490234375,
      "logps/rejected": -892.8589477539062,
      "loss": 0.703,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0019866949878633022,
      "rewards/margins": -0.0193604938685894,
      "rewards/rejected": 0.01737379841506481,
      "step": 2121
    },
    {
      "epoch": 0.558154827467162,
      "grad_norm": 540.0,
      "learning_rate": 2.4546783625730995e-07,
      "logits/chosen": -0.644977331161499,
      "logits/rejected": -0.6583385467529297,
      "logps/chosen": -1235.791259765625,
      "logps/rejected": -591.8731079101562,
      "loss": 0.696,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.006083392538130283,
      "rewards/margins": -0.005071544088423252,
      "rewards/rejected": -0.001011848682537675,
      "step": 2122
    },
    {
      "epoch": 0.5584178599023492,
      "grad_norm": 616.0,
      "learning_rate": 2.4532163742690057e-07,
      "logits/chosen": -0.6520483493804932,
      "logits/rejected": -0.6461396813392639,
      "logps/chosen": -1396.8897705078125,
      "logps/rejected": -1108.138671875,
      "loss": 0.7032,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.014698314480483532,
      "rewards/margins": -0.01867198944091797,
      "rewards/rejected": 0.003973674960434437,
      "step": 2123
    },
    {
      "epoch": 0.5586808923375364,
      "grad_norm": 572.0,
      "learning_rate": 2.4517543859649124e-07,
      "logits/chosen": -0.6638011336326599,
      "logits/rejected": -0.6640823483467102,
      "logps/chosen": -1281.60693359375,
      "logps/rejected": -863.496337890625,
      "loss": 0.6829,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0063849459402263165,
      "rewards/margins": 0.021581459790468216,
      "rewards/rejected": -0.015196515247225761,
      "step": 2124
    },
    {
      "epoch": 0.5589439247727236,
      "grad_norm": 580.0,
      "learning_rate": 2.4502923976608186e-07,
      "logits/chosen": -0.6315814256668091,
      "logits/rejected": -0.641100287437439,
      "logps/chosen": -1533.3218994140625,
      "logps/rejected": -1311.88232421875,
      "loss": 0.68,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.022547533735632896,
      "rewards/margins": 0.02786111831665039,
      "rewards/rejected": -0.005313586909323931,
      "step": 2125
    },
    {
      "epoch": 0.5592069572079107,
      "grad_norm": 596.0,
      "learning_rate": 2.448830409356725e-07,
      "logits/chosen": -0.6618365049362183,
      "logits/rejected": -0.6555001139640808,
      "logps/chosen": -997.8831176757812,
      "logps/rejected": -1063.85693359375,
      "loss": 0.7094,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0184478759765625,
      "rewards/margins": -0.031077291816473007,
      "rewards/rejected": 0.012629413045942783,
      "step": 2126
    },
    {
      "epoch": 0.5594699896430979,
      "grad_norm": 612.0,
      "learning_rate": 2.4473684210526315e-07,
      "logits/chosen": -0.6430874466896057,
      "logits/rejected": -0.6532052755355835,
      "logps/chosen": -946.235107421875,
      "logps/rejected": -846.02880859375,
      "loss": 0.6989,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0017672537360340357,
      "rewards/margins": -0.011154461652040482,
      "rewards/rejected": 0.00938720814883709,
      "step": 2127
    },
    {
      "epoch": 0.559733022078285,
      "grad_norm": 520.0,
      "learning_rate": 2.4459064327485377e-07,
      "logits/chosen": -0.6400495171546936,
      "logits/rejected": -0.6393948793411255,
      "logps/chosen": -1147.2860107421875,
      "logps/rejected": -1035.60693359375,
      "loss": 0.6955,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008390570059418678,
      "rewards/margins": -0.0034790528006851673,
      "rewards/rejected": -0.004911518655717373,
      "step": 2128
    },
    {
      "epoch": 0.5599960545134722,
      "grad_norm": 492.0,
      "learning_rate": 2.4444444444444445e-07,
      "logits/chosen": -0.657325267791748,
      "logits/rejected": -0.6591063141822815,
      "logps/chosen": -1240.052001953125,
      "logps/rejected": -721.352294921875,
      "loss": 0.6977,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.021812820807099342,
      "rewards/margins": -0.007996941916644573,
      "rewards/rejected": -0.013815879821777344,
      "step": 2129
    },
    {
      "epoch": 0.5602590869486593,
      "grad_norm": 472.0,
      "learning_rate": 2.4429824561403507e-07,
      "logits/chosen": -0.6421915292739868,
      "logits/rejected": -0.6497732996940613,
      "logps/chosen": -903.7676391601562,
      "logps/rejected": -824.787353515625,
      "loss": 0.6915,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0047891149297356606,
      "rewards/margins": 0.0038727286737412214,
      "rewards/rejected": 0.0009163855575025082,
      "step": 2130
    },
    {
      "epoch": 0.5605221193838466,
      "grad_norm": 556.0,
      "learning_rate": 2.4415204678362574e-07,
      "logits/chosen": -0.6520218253135681,
      "logits/rejected": -0.6526110172271729,
      "logps/chosen": -1035.0361328125,
      "logps/rejected": -753.1517333984375,
      "loss": 0.6982,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00660057133063674,
      "rewards/margins": -0.00946884136646986,
      "rewards/rejected": 0.0028682714328169823,
      "step": 2131
    },
    {
      "epoch": 0.5607851518190337,
      "grad_norm": 632.0,
      "learning_rate": 2.4400584795321636e-07,
      "logits/chosen": -0.6253756284713745,
      "logits/rejected": -0.6203560829162598,
      "logps/chosen": -898.7314453125,
      "logps/rejected": -1064.0611572265625,
      "loss": 0.6962,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005872631445527077,
      "rewards/margins": -0.00503387488424778,
      "rewards/rejected": 0.010906506329774857,
      "step": 2132
    },
    {
      "epoch": 0.5610481842542209,
      "grad_norm": 536.0,
      "learning_rate": 2.4385964912280703e-07,
      "logits/chosen": -0.6409765481948853,
      "logits/rejected": -0.6444759964942932,
      "logps/chosen": -1266.97900390625,
      "logps/rejected": -714.9635009765625,
      "loss": 0.6851,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.029922202229499817,
      "rewards/margins": 0.01694631762802601,
      "rewards/rejected": 0.012975884601473808,
      "step": 2133
    },
    {
      "epoch": 0.561311216689408,
      "grad_norm": 456.0,
      "learning_rate": 2.4371345029239765e-07,
      "logits/chosen": -0.6362314224243164,
      "logits/rejected": -0.6417262554168701,
      "logps/chosen": -855.811279296875,
      "logps/rejected": -596.6444091796875,
      "loss": 0.7005,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01501159742474556,
      "rewards/margins": -0.013778496533632278,
      "rewards/rejected": -0.0012331008911132812,
      "step": 2134
    },
    {
      "epoch": 0.5615742491245952,
      "grad_norm": 628.0,
      "learning_rate": 2.4356725146198827e-07,
      "logits/chosen": -0.6308629512786865,
      "logits/rejected": -0.6306989789009094,
      "logps/chosen": -881.0838012695312,
      "logps/rejected": -811.6629638671875,
      "loss": 0.6883,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00411539152264595,
      "rewards/margins": 0.010259057395160198,
      "rewards/rejected": -0.014374447055161,
      "step": 2135
    },
    {
      "epoch": 0.5618372815597823,
      "grad_norm": 512.0,
      "learning_rate": 2.4342105263157894e-07,
      "logits/chosen": -0.6694864630699158,
      "logits/rejected": -0.6679006218910217,
      "logps/chosen": -1181.41064453125,
      "logps/rejected": -877.888916015625,
      "loss": 0.6926,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0035699857398867607,
      "rewards/margins": 0.0017956718802452087,
      "rewards/rejected": -0.0053656576201319695,
      "step": 2136
    },
    {
      "epoch": 0.5621003139949695,
      "grad_norm": 446.0,
      "learning_rate": 2.4327485380116956e-07,
      "logits/chosen": -0.6447292566299438,
      "logits/rejected": -0.6510711908340454,
      "logps/chosen": -841.1030883789062,
      "logps/rejected": -723.8323974609375,
      "loss": 0.6862,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.016318989917635918,
      "rewards/margins": 0.01434855442494154,
      "rewards/rejected": 0.0019704345613718033,
      "step": 2137
    },
    {
      "epoch": 0.5623633464301566,
      "grad_norm": 668.0,
      "learning_rate": 2.4312865497076024e-07,
      "logits/chosen": -0.6558911800384521,
      "logits/rejected": -0.6469764113426208,
      "logps/chosen": -965.3228759765625,
      "logps/rejected": -898.6708984375,
      "loss": 0.7094,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.008848095312714577,
      "rewards/margins": -0.030585862696170807,
      "rewards/rejected": 0.02173776552081108,
      "step": 2138
    },
    {
      "epoch": 0.5626263788653438,
      "grad_norm": 624.0,
      "learning_rate": 2.4298245614035086e-07,
      "logits/chosen": -0.6599411964416504,
      "logits/rejected": -0.6716910004615784,
      "logps/chosen": -1108.9761962890625,
      "logps/rejected": -1311.4947509765625,
      "loss": 0.7035,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.002173136919736862,
      "rewards/margins": -0.01978016085922718,
      "rewards/rejected": 0.021953295916318893,
      "step": 2139
    },
    {
      "epoch": 0.562889411300531,
      "grad_norm": 480.0,
      "learning_rate": 2.4283625730994153e-07,
      "logits/chosen": -0.6374880075454712,
      "logits/rejected": -0.6454769968986511,
      "logps/chosen": -1084.0606689453125,
      "logps/rejected": -1019.1649780273438,
      "loss": 0.7022,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.019260788336396217,
      "rewards/margins": -0.017410945147275925,
      "rewards/rejected": -0.0018498431891202927,
      "step": 2140
    },
    {
      "epoch": 0.5631524437357182,
      "grad_norm": 644.0,
      "learning_rate": 2.4269005847953215e-07,
      "logits/chosen": -0.6551333665847778,
      "logits/rejected": NaN,
      "logps/chosen": -1063.8697509765625,
      "logps/rejected": -707.6849365234375,
      "loss": 0.7019,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.011213683523237705,
      "rewards/margins": -0.016394425183534622,
      "rewards/rejected": 0.005180740728974342,
      "step": 2141
    },
    {
      "epoch": 0.5634154761709054,
      "grad_norm": 556.0,
      "learning_rate": 2.425438596491228e-07,
      "logits/chosen": -0.6318691968917847,
      "logits/rejected": -0.6397624015808105,
      "logps/chosen": -1074.833740234375,
      "logps/rejected": -757.6280517578125,
      "loss": 0.6917,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006368447560817003,
      "rewards/margins": 0.0034205433912575245,
      "rewards/rejected": 0.0029479030054062605,
      "step": 2142
    },
    {
      "epoch": 0.5636785086060925,
      "grad_norm": 478.0,
      "learning_rate": 2.4239766081871344e-07,
      "logits/chosen": -0.6484956741333008,
      "logits/rejected": -0.6567071676254272,
      "logps/chosen": -1221.2152099609375,
      "logps/rejected": -1061.7232666015625,
      "loss": 0.6973,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.014363384805619717,
      "rewards/margins": -0.007877825759351254,
      "rewards/rejected": -0.006485557649284601,
      "step": 2143
    },
    {
      "epoch": 0.5639415410412797,
      "grad_norm": 540.0,
      "learning_rate": 2.422514619883041e-07,
      "logits/chosen": -0.637662947177887,
      "logits/rejected": -0.6406792402267456,
      "logps/chosen": -984.1138916015625,
      "logps/rejected": -814.132568359375,
      "loss": 0.693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005940579809248447,
      "rewards/margins": 0.0011557108955457807,
      "rewards/rejected": -0.0070962910540401936,
      "step": 2144
    },
    {
      "epoch": 0.5642045734764668,
      "grad_norm": 430.0,
      "learning_rate": 2.4210526315789473e-07,
      "logits/chosen": -0.6442935466766357,
      "logits/rejected": -0.6535082459449768,
      "logps/chosen": -923.3742065429688,
      "logps/rejected": -1023.0274047851562,
      "loss": 0.6993,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.013150598853826523,
      "rewards/margins": -0.011923788115382195,
      "rewards/rejected": -0.0012268073623999953,
      "step": 2145
    },
    {
      "epoch": 0.564467605911654,
      "grad_norm": 560.0,
      "learning_rate": 2.4195906432748535e-07,
      "logits/chosen": -0.6347697377204895,
      "logits/rejected": -0.6481051445007324,
      "logps/chosen": -1272.6531982421875,
      "logps/rejected": -940.665283203125,
      "loss": 0.6807,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0021084786858409643,
      "rewards/margins": 0.025690842419862747,
      "rewards/rejected": -0.02358236536383629,
      "step": 2146
    },
    {
      "epoch": 0.5647306383468411,
      "grad_norm": 454.0,
      "learning_rate": 2.4181286549707603e-07,
      "logits/chosen": -0.642135500907898,
      "logits/rejected": -0.6476993560791016,
      "logps/chosen": -1052.0325927734375,
      "logps/rejected": -805.8076782226562,
      "loss": 0.6889,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.009138775058090687,
      "rewards/margins": 0.009363602846860886,
      "rewards/rejected": -0.01850237511098385,
      "step": 2147
    },
    {
      "epoch": 0.5649936707820283,
      "grad_norm": 524.0,
      "learning_rate": 2.4166666666666665e-07,
      "logits/chosen": -0.6697397232055664,
      "logits/rejected": -0.6722185015678406,
      "logps/chosen": -1561.208251953125,
      "logps/rejected": -1353.8306884765625,
      "loss": 0.6817,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01221847627311945,
      "rewards/margins": 0.024131394922733307,
      "rewards/rejected": -0.011912917718291283,
      "step": 2148
    },
    {
      "epoch": 0.5652567032172154,
      "grad_norm": 640.0,
      "learning_rate": 2.4152046783625727e-07,
      "logits/chosen": -0.676861584186554,
      "logits/rejected": -0.6744500994682312,
      "logps/chosen": -1636.0869140625,
      "logps/rejected": -1136.4002685546875,
      "loss": 0.6949,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.007403658237308264,
      "rewards/margins": -0.0024972916580736637,
      "rewards/rejected": -0.004906368907541037,
      "step": 2149
    },
    {
      "epoch": 0.5655197356524027,
      "grad_norm": 1360.0,
      "learning_rate": 2.4137426900584794e-07,
      "logits/chosen": -0.6341429352760315,
      "logits/rejected": NaN,
      "logps/chosen": -1353.227783203125,
      "logps/rejected": -943.3031616210938,
      "loss": 0.6943,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0033719069324433804,
      "rewards/margins": -0.000262546818703413,
      "rewards/rejected": -0.00310935964807868,
      "step": 2150
    },
    {
      "epoch": 0.5657827680875898,
      "grad_norm": 524.0,
      "learning_rate": 2.412280701754386e-07,
      "logits/chosen": -0.6481389999389648,
      "logits/rejected": -0.6476120352745056,
      "logps/chosen": -1227.1475830078125,
      "logps/rejected": -1021.0249633789062,
      "loss": 0.682,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.007068539038300514,
      "rewards/margins": 0.023631669580936432,
      "rewards/rejected": -0.016563130542635918,
      "step": 2151
    },
    {
      "epoch": 0.566045800522777,
      "grad_norm": 432.0,
      "learning_rate": 2.4108187134502923e-07,
      "logits/chosen": -0.6359405517578125,
      "logits/rejected": -0.6405717730522156,
      "logps/chosen": -1241.38525390625,
      "logps/rejected": -1012.519287109375,
      "loss": 0.6947,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.009540843777358532,
      "rewards/margins": -0.0028072360437363386,
      "rewards/rejected": -0.006733607966452837,
      "step": 2152
    },
    {
      "epoch": 0.5663088329579641,
      "grad_norm": 400.0,
      "learning_rate": 2.409356725146199e-07,
      "logits/chosen": -0.6547942161560059,
      "logits/rejected": -0.6600444316864014,
      "logps/chosen": -685.1347045898438,
      "logps/rejected": -560.6275634765625,
      "loss": 0.6816,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.03041868284344673,
      "rewards/margins": 0.02400193177163601,
      "rewards/rejected": 0.00641674967482686,
      "step": 2153
    },
    {
      "epoch": 0.5665718653931513,
      "grad_norm": 540.0,
      "learning_rate": 2.407894736842105e-07,
      "logits/chosen": -0.6474095582962036,
      "logits/rejected": -0.6455131769180298,
      "logps/chosen": -1597.21533203125,
      "logps/rejected": -1103.750732421875,
      "loss": 0.6928,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.016506290063261986,
      "rewards/margins": 0.001774884294718504,
      "rewards/rejected": 0.014731409028172493,
      "step": 2154
    },
    {
      "epoch": 0.5668348978283384,
      "grad_norm": 604.0,
      "learning_rate": 2.4064327485380115e-07,
      "logits/chosen": -0.6649461984634399,
      "logits/rejected": -0.6728691458702087,
      "logps/chosen": -1286.1876220703125,
      "logps/rejected": -748.7074584960938,
      "loss": 0.7041,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.020629215985536575,
      "rewards/margins": -0.020899390801787376,
      "rewards/rejected": 0.00027017516549676657,
      "step": 2155
    },
    {
      "epoch": 0.5670979302635256,
      "grad_norm": 592.0,
      "learning_rate": 2.404970760233918e-07,
      "logits/chosen": -0.6506103873252869,
      "logits/rejected": -0.6577775478363037,
      "logps/chosen": -1367.287353515625,
      "logps/rejected": -969.0352783203125,
      "loss": 0.6929,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.013464165851473808,
      "rewards/margins": 0.0013998504728078842,
      "rewards/rejected": 0.012064315378665924,
      "step": 2156
    },
    {
      "epoch": 0.5673609626987128,
      "grad_norm": 492.0,
      "learning_rate": 2.4035087719298244e-07,
      "logits/chosen": -0.6355314254760742,
      "logits/rejected": -0.6260172724723816,
      "logps/chosen": -764.7261962890625,
      "logps/rejected": -672.5595703125,
      "loss": 0.6878,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0038727764040231705,
      "rewards/margins": 0.01130666770040989,
      "rewards/rejected": -0.007433889899402857,
      "step": 2157
    },
    {
      "epoch": 0.5676239951338999,
      "grad_norm": 504.0,
      "learning_rate": 2.4020467836257306e-07,
      "logits/chosen": -0.6647000312805176,
      "logits/rejected": -0.6662982106208801,
      "logps/chosen": -1400.9766845703125,
      "logps/rejected": -1354.58642578125,
      "loss": 0.6877,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.011877059936523438,
      "rewards/margins": 0.011672019958496094,
      "rewards/rejected": 0.00020504044368863106,
      "step": 2158
    },
    {
      "epoch": 0.5678870275690872,
      "grad_norm": 540.0,
      "learning_rate": 2.4005847953216373e-07,
      "logits/chosen": -0.647086501121521,
      "logits/rejected": -0.6500549912452698,
      "logps/chosen": -1113.0732421875,
      "logps/rejected": -737.1220703125,
      "loss": 0.6923,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.014542675577104092,
      "rewards/margins": 0.0027442926075309515,
      "rewards/rejected": 0.011798381805419922,
      "step": 2159
    },
    {
      "epoch": 0.5681500600042743,
      "grad_norm": 644.0,
      "learning_rate": 2.3991228070175435e-07,
      "logits/chosen": -0.6471233367919922,
      "logits/rejected": -0.6377661228179932,
      "logps/chosen": -1174.74755859375,
      "logps/rejected": -984.2760009765625,
      "loss": 0.7007,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.00627097999677062,
      "rewards/margins": -0.01372451800853014,
      "rewards/rejected": 0.01999550126492977,
      "step": 2160
    },
    {
      "epoch": 0.5684130924394615,
      "grad_norm": 568.0,
      "learning_rate": 2.39766081871345e-07,
      "logits/chosen": -0.6523156762123108,
      "logits/rejected": -0.6522361040115356,
      "logps/chosen": -1333.0064697265625,
      "logps/rejected": -1077.8292236328125,
      "loss": 0.6926,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0041099549271166325,
      "rewards/margins": 0.0014878263464197516,
      "rewards/rejected": 0.0026221275329589844,
      "step": 2161
    },
    {
      "epoch": 0.5686761248746486,
      "grad_norm": 672.0,
      "learning_rate": 2.3961988304093564e-07,
      "logits/chosen": -0.6855335235595703,
      "logits/rejected": -0.6798518896102905,
      "logps/chosen": -1205.5479736328125,
      "logps/rejected": -1023.9984741210938,
      "loss": 0.6926,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005303764715790749,
      "rewards/margins": 0.0013581269886344671,
      "rewards/rejected": 0.003945636563003063,
      "step": 2162
    },
    {
      "epoch": 0.5689391573098358,
      "grad_norm": 376.0,
      "learning_rate": 2.394736842105263e-07,
      "logits/chosen": -0.6502763628959656,
      "logits/rejected": -0.6530694365501404,
      "logps/chosen": -764.7730712890625,
      "logps/rejected": -582.9896850585938,
      "loss": 0.6845,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00951767060905695,
      "rewards/margins": 0.01800370216369629,
      "rewards/rejected": -0.027521369978785515,
      "step": 2163
    },
    {
      "epoch": 0.5692021897450229,
      "grad_norm": 596.0,
      "learning_rate": 2.3932748538011694e-07,
      "logits/chosen": -0.6609914898872375,
      "logits/rejected": -0.6721763014793396,
      "logps/chosen": -1207.891845703125,
      "logps/rejected": -914.9449462890625,
      "loss": 0.6956,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.004249955527484417,
      "rewards/margins": -0.004045582842081785,
      "rewards/rejected": 0.00829553697258234,
      "step": 2164
    },
    {
      "epoch": 0.5694652221802101,
      "grad_norm": 604.0,
      "learning_rate": 2.391812865497076e-07,
      "logits/chosen": -0.6759874820709229,
      "logits/rejected": -0.6683996319770813,
      "logps/chosen": -1308.3782958984375,
      "logps/rejected": -1042.17138671875,
      "loss": 0.6886,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.015303803607821465,
      "rewards/margins": 0.010834790766239166,
      "rewards/rejected": 0.004469012841582298,
      "step": 2165
    },
    {
      "epoch": 0.5697282546153972,
      "grad_norm": 552.0,
      "learning_rate": 2.3903508771929823e-07,
      "logits/chosen": -0.6499719619750977,
      "logits/rejected": -0.6502106189727783,
      "logps/chosen": -996.2389526367188,
      "logps/rejected": -1081.30078125,
      "loss": 0.7078,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.039195917546749115,
      "rewards/margins": -0.027056071907281876,
      "rewards/rejected": -0.01213984563946724,
      "step": 2166
    },
    {
      "epoch": 0.5699912870505844,
      "grad_norm": 520.0,
      "learning_rate": 2.388888888888889e-07,
      "logits/chosen": -0.6571409702301025,
      "logits/rejected": -0.6568465232849121,
      "logps/chosen": -1155.8248291015625,
      "logps/rejected": -1330.748046875,
      "loss": 0.7077,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.012252996675670147,
      "rewards/margins": -0.02835111878812313,
      "rewards/rejected": 0.01609811745584011,
      "step": 2167
    },
    {
      "epoch": 0.5702543194857715,
      "grad_norm": 564.0,
      "learning_rate": 2.387426900584795e-07,
      "logits/chosen": -0.6632673144340515,
      "logits/rejected": -0.6617965698242188,
      "logps/chosen": -1086.8037109375,
      "logps/rejected": -919.613525390625,
      "loss": 0.6987,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0030932421796023846,
      "rewards/margins": -0.009668447077274323,
      "rewards/rejected": 0.006575203500688076,
      "step": 2168
    },
    {
      "epoch": 0.5705173519209588,
      "grad_norm": 540.0,
      "learning_rate": 2.3859649122807014e-07,
      "logits/chosen": -0.6314207315444946,
      "logits/rejected": -0.6263116598129272,
      "logps/chosen": -1362.1739501953125,
      "logps/rejected": -982.690185546875,
      "loss": 0.7145,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.05024976283311844,
      "rewards/margins": -0.04088296741247177,
      "rewards/rejected": -0.009366797283291817,
      "step": 2169
    },
    {
      "epoch": 0.5707803843561459,
      "grad_norm": 568.0,
      "learning_rate": 2.384502923976608e-07,
      "logits/chosen": -0.6813554167747498,
      "logits/rejected": -0.6833376884460449,
      "logps/chosen": -1295.386474609375,
      "logps/rejected": -1015.1577758789062,
      "loss": 0.7036,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006739807315170765,
      "rewards/margins": -0.020293522626161575,
      "rewards/rejected": 0.027033329010009766,
      "step": 2170
    },
    {
      "epoch": 0.5710434167913331,
      "grad_norm": 500.0,
      "learning_rate": 2.3830409356725146e-07,
      "logits/chosen": -0.6274068355560303,
      "logits/rejected": -0.6375195980072021,
      "logps/chosen": -920.7529296875,
      "logps/rejected": -754.959716796875,
      "loss": 0.6911,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008297253400087357,
      "rewards/margins": 0.004872942343354225,
      "rewards/rejected": 0.0034243110567331314,
      "step": 2171
    },
    {
      "epoch": 0.5713064492265203,
      "grad_norm": 560.0,
      "learning_rate": 2.3815789473684208e-07,
      "logits/chosen": -0.6346030235290527,
      "logits/rejected": -0.6338847875595093,
      "logps/chosen": -1258.989013671875,
      "logps/rejected": -904.7838134765625,
      "loss": 0.685,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.020917700603604317,
      "rewards/margins": 0.01731882244348526,
      "rewards/rejected": 0.003598881885409355,
      "step": 2172
    },
    {
      "epoch": 0.5715694816617074,
      "grad_norm": 596.0,
      "learning_rate": 2.3801169590643275e-07,
      "logits/chosen": -0.6514549851417542,
      "logits/rejected": -0.6502580046653748,
      "logps/chosen": -1304.78662109375,
      "logps/rejected": -914.86865234375,
      "loss": 0.6979,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0011856085620820522,
      "rewards/margins": -0.007778117433190346,
      "rewards/rejected": 0.00896372552961111,
      "step": 2173
    },
    {
      "epoch": 0.5718325140968946,
      "grad_norm": 512.0,
      "learning_rate": 2.3786549707602337e-07,
      "logits/chosen": -0.6543897390365601,
      "logits/rejected": -0.6685929894447327,
      "logps/chosen": -1075.5675048828125,
      "logps/rejected": -813.255615234375,
      "loss": 0.6969,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01290130615234375,
      "rewards/margins": -0.005650999024510384,
      "rewards/rejected": 0.018552303314208984,
      "step": 2174
    },
    {
      "epoch": 0.5720955465320817,
      "grad_norm": 506.0,
      "learning_rate": 2.3771929824561402e-07,
      "logits/chosen": -0.6654921174049377,
      "logits/rejected": -0.681544840335846,
      "logps/chosen": -943.96337890625,
      "logps/rejected": -923.1876831054688,
      "loss": 0.6915,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.007730960845947266,
      "rewards/margins": 0.004160499665886164,
      "rewards/rejected": -0.011891460046172142,
      "step": 2175
    },
    {
      "epoch": 0.5723585789672689,
      "grad_norm": 444.0,
      "learning_rate": 2.3757309941520467e-07,
      "logits/chosen": -0.6644759774208069,
      "logits/rejected": -0.6564151644706726,
      "logps/chosen": -852.7631225585938,
      "logps/rejected": -994.6610107421875,
      "loss": 0.6903,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.009480285458266735,
      "rewards/margins": 0.006504582706838846,
      "rewards/rejected": 0.002975702751427889,
      "step": 2176
    },
    {
      "epoch": 0.572621611402456,
      "grad_norm": 470.0,
      "learning_rate": 2.374269005847953e-07,
      "logits/chosen": -0.652703046798706,
      "logits/rejected": -0.6615961194038391,
      "logps/chosen": -1022.7913818359375,
      "logps/rejected": -1030.3544921875,
      "loss": 0.6896,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.5067635104060173e-05,
      "rewards/margins": 0.007622814271599054,
      "rewards/rejected": -0.007637883070856333,
      "step": 2177
    },
    {
      "epoch": 0.5728846438376433,
      "grad_norm": 564.0,
      "learning_rate": 2.3728070175438593e-07,
      "logits/chosen": -0.65519118309021,
      "logits/rejected": -0.6597062945365906,
      "logps/chosen": -1230.002685546875,
      "logps/rejected": -953.2792358398438,
      "loss": 0.6935,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0013191228499636054,
      "rewards/margins": 0.00010609778109937906,
      "rewards/rejected": 0.0012130264658480883,
      "step": 2178
    },
    {
      "epoch": 0.5731476762728304,
      "grad_norm": 304.0,
      "learning_rate": 2.371345029239766e-07,
      "logits/chosen": -0.6646789908409119,
      "logits/rejected": -0.6674570441246033,
      "logps/chosen": -614.1493530273438,
      "logps/rejected": -651.6494140625,
      "loss": 0.6927,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.020400049164891243,
      "rewards/margins": 0.0010344511829316616,
      "rewards/rejected": 0.019365597516298294,
      "step": 2179
    },
    {
      "epoch": 0.5734107087080176,
      "grad_norm": 462.0,
      "learning_rate": 2.3698830409356723e-07,
      "logits/chosen": -0.6565907597541809,
      "logits/rejected": -0.6568731069564819,
      "logps/chosen": -1560.7357177734375,
      "logps/rejected": -1294.8896484375,
      "loss": 0.6869,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01991434395313263,
      "rewards/margins": 0.013572598807513714,
      "rewards/rejected": 0.006341743282973766,
      "step": 2180
    },
    {
      "epoch": 0.5736737411432047,
      "grad_norm": 466.0,
      "learning_rate": 2.3684210526315787e-07,
      "logits/chosen": -0.6632953882217407,
      "logits/rejected": -0.65852952003479,
      "logps/chosen": -933.52001953125,
      "logps/rejected": -1008.7120971679688,
      "loss": 0.69,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.008429622277617455,
      "rewards/margins": 0.006635952740907669,
      "rewards/rejected": -0.015065575949847698,
      "step": 2181
    },
    {
      "epoch": 0.5739367735783919,
      "grad_norm": 478.0,
      "learning_rate": 2.3669590643274854e-07,
      "logits/chosen": -0.6234129071235657,
      "logits/rejected": -0.6225883960723877,
      "logps/chosen": -1057.2784423828125,
      "logps/rejected": -938.2288208007812,
      "loss": 0.6991,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.009141732007265091,
      "rewards/margins": -0.011207962408661842,
      "rewards/rejected": 0.0020662315655499697,
      "step": 2182
    },
    {
      "epoch": 0.574199806013579,
      "grad_norm": 420.0,
      "learning_rate": 2.3654970760233916e-07,
      "logits/chosen": -0.642356276512146,
      "logits/rejected": -0.6533743739128113,
      "logps/chosen": -831.9619140625,
      "logps/rejected": -551.1555786132812,
      "loss": 0.6954,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.005184887908399105,
      "rewards/margins": -0.004146908409893513,
      "rewards/rejected": -0.0010379799641668797,
      "step": 2183
    },
    {
      "epoch": 0.5744628384487662,
      "grad_norm": 648.0,
      "learning_rate": 2.3640350877192984e-07,
      "logits/chosen": -0.6770877838134766,
      "logits/rejected": -0.680869460105896,
      "logps/chosen": -1037.48095703125,
      "logps/rejected": -992.0927734375,
      "loss": 0.7007,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.008318090811371803,
      "rewards/margins": -0.014813518151640892,
      "rewards/rejected": 0.006495428271591663,
      "step": 2184
    },
    {
      "epoch": 0.5747258708839533,
      "grad_norm": 494.0,
      "learning_rate": 2.3625730994152046e-07,
      "logits/chosen": -0.6370013952255249,
      "logits/rejected": -0.6485295295715332,
      "logps/chosen": -925.4473876953125,
      "logps/rejected": -781.2327880859375,
      "loss": 0.6875,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0003211977891623974,
      "rewards/margins": 0.011945152655243874,
      "rewards/rejected": -0.012266349047422409,
      "step": 2185
    },
    {
      "epoch": 0.5749889033191405,
      "grad_norm": 540.0,
      "learning_rate": 2.361111111111111e-07,
      "logits/chosen": -0.6667673587799072,
      "logits/rejected": -0.6627007126808167,
      "logps/chosen": -1136.3692626953125,
      "logps/rejected": -1006.308349609375,
      "loss": 0.6819,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.016126634553074837,
      "rewards/margins": 0.023740578442811966,
      "rewards/rejected": -0.0076139443553984165,
      "step": 2186
    },
    {
      "epoch": 0.5752519357543278,
      "grad_norm": 528.0,
      "learning_rate": 2.3596491228070175e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6734999418258667,
      "logps/chosen": -966.35400390625,
      "logps/rejected": -1143.130859375,
      "loss": 0.7062,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.007134055718779564,
      "rewards/margins": -0.025529813021421432,
      "rewards/rejected": 0.018395759165287018,
      "step": 2187
    },
    {
      "epoch": 0.5755149681895149,
      "grad_norm": 460.0,
      "learning_rate": 2.358187134502924e-07,
      "logits/chosen": -0.6474238634109497,
      "logits/rejected": -0.6390416622161865,
      "logps/chosen": -859.2962036132812,
      "logps/rejected": -945.5958251953125,
      "loss": 0.702,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.015214539133012295,
      "rewards/margins": -0.01695084571838379,
      "rewards/rejected": 0.0017363072838634253,
      "step": 2188
    },
    {
      "epoch": 0.5757780006247021,
      "grad_norm": 544.0,
      "learning_rate": 2.3567251461988302e-07,
      "logits/chosen": -0.6560945510864258,
      "logits/rejected": -0.6514638066291809,
      "logps/chosen": -1027.6068115234375,
      "logps/rejected": -793.8199462890625,
      "loss": 0.6955,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.006899450905621052,
      "rewards/margins": -0.0034113391302525997,
      "rewards/rejected": -0.003488111775368452,
      "step": 2189
    },
    {
      "epoch": 0.5760410330598892,
      "grad_norm": 564.0,
      "learning_rate": 2.355263157894737e-07,
      "logits/chosen": -0.6689100861549377,
      "logits/rejected": -0.6705539226531982,
      "logps/chosen": -1171.693603515625,
      "logps/rejected": -855.5753784179688,
      "loss": 0.7025,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.007303046993911266,
      "rewards/margins": -0.018050003796815872,
      "rewards/rejected": 0.010746955871582031,
      "step": 2190
    },
    {
      "epoch": 0.5763040654950764,
      "grad_norm": 876.0,
      "learning_rate": 2.353801169590643e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6357529163360596,
      "logps/chosen": -1572.0882568359375,
      "logps/rejected": -1287.5289306640625,
      "loss": 0.6897,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.018671799451112747,
      "rewards/margins": 0.008306313306093216,
      "rewards/rejected": 0.010365486145019531,
      "step": 2191
    },
    {
      "epoch": 0.5765670979302635,
      "grad_norm": 596.0,
      "learning_rate": 2.3523391812865496e-07,
      "logits/chosen": -0.6493340730667114,
      "logits/rejected": -0.6547349691390991,
      "logps/chosen": -1216.3212890625,
      "logps/rejected": -992.8072509765625,
      "loss": 0.7085,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.030548669397830963,
      "rewards/margins": -0.03007493168115616,
      "rewards/rejected": -0.00047373760025948286,
      "step": 2192
    },
    {
      "epoch": 0.5768301303654507,
      "grad_norm": 494.0,
      "learning_rate": 2.350877192982456e-07,
      "logits/chosen": -0.6496682167053223,
      "logits/rejected": -0.6542263627052307,
      "logps/chosen": -1097.65625,
      "logps/rejected": -857.2293701171875,
      "loss": 0.69,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.012165402062237263,
      "rewards/margins": 0.00760273914784193,
      "rewards/rejected": -0.01976814493536949,
      "step": 2193
    },
    {
      "epoch": 0.5770931628006378,
      "grad_norm": 620.0,
      "learning_rate": 2.3494152046783625e-07,
      "logits/chosen": -0.6513859629631042,
      "logits/rejected": -0.6565351486206055,
      "logps/chosen": -993.7134399414062,
      "logps/rejected": -922.845703125,
      "loss": 0.6943,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0014834403991699219,
      "rewards/margins": -0.0011816021287813783,
      "rewards/rejected": 0.0026650428771972656,
      "step": 2194
    },
    {
      "epoch": 0.577356195235825,
      "grad_norm": 524.0,
      "learning_rate": 2.3479532163742687e-07,
      "logits/chosen": -0.6575300693511963,
      "logits/rejected": -0.6551388502120972,
      "logps/chosen": -986.2056884765625,
      "logps/rejected": -859.7833862304688,
      "loss": 0.6798,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0011770252604037523,
      "rewards/margins": 0.02709512785077095,
      "rewards/rejected": -0.025918101891875267,
      "step": 2195
    },
    {
      "epoch": 0.5776192276710121,
      "grad_norm": 528.0,
      "learning_rate": 2.3464912280701754e-07,
      "logits/chosen": -0.6413154602050781,
      "logits/rejected": -0.632749080657959,
      "logps/chosen": -1399.11083984375,
      "logps/rejected": -979.6348266601562,
      "loss": 0.6947,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0023865706752985716,
      "rewards/margins": -0.0024249081034213305,
      "rewards/rejected": 3.833835944533348e-05,
      "step": 2196
    },
    {
      "epoch": 0.5778822601061994,
      "grad_norm": 568.0,
      "learning_rate": 2.345029239766082e-07,
      "logits/chosen": -0.6589479446411133,
      "logits/rejected": -0.6520906090736389,
      "logps/chosen": -1562.15478515625,
      "logps/rejected": -1450.5469970703125,
      "loss": 0.7008,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006933593191206455,
      "rewards/margins": -0.01461257878690958,
      "rewards/rejected": 0.021546173840761185,
      "step": 2197
    },
    {
      "epoch": 0.5781452925413865,
      "grad_norm": 584.0,
      "learning_rate": 2.343567251461988e-07,
      "logits/chosen": -0.6719545125961304,
      "logits/rejected": -0.6560062766075134,
      "logps/chosen": -1166.769775390625,
      "logps/rejected": -1076.9976806640625,
      "loss": 0.7009,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01305387169122696,
      "rewards/margins": -0.013286376371979713,
      "rewards/rejected": 0.00023250561207532883,
      "step": 2198
    },
    {
      "epoch": 0.5784083249765737,
      "grad_norm": 364.0,
      "learning_rate": 2.3421052631578948e-07,
      "logits/chosen": -0.6499006152153015,
      "logits/rejected": -0.6531028747558594,
      "logps/chosen": -904.9320678710938,
      "logps/rejected": -817.57177734375,
      "loss": 0.689,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.000957203097641468,
      "rewards/margins": 0.008579825982451439,
      "rewards/rejected": -0.007622623350471258,
      "step": 2199
    },
    {
      "epoch": 0.5786713574117608,
      "grad_norm": 572.0,
      "learning_rate": 2.340643274853801e-07,
      "logits/chosen": -0.6637308597564697,
      "logits/rejected": -0.6607815623283386,
      "logps/chosen": -1337.916259765625,
      "logps/rejected": -1162.2252197265625,
      "loss": 0.7035,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.004387378692626953,
      "rewards/margins": -0.01957111619412899,
      "rewards/rejected": 0.023958493024110794,
      "step": 2200
    },
    {
      "epoch": 0.578934389846948,
      "grad_norm": 560.0,
      "learning_rate": 2.3391812865497075e-07,
      "logits/chosen": -0.6713331341743469,
      "logits/rejected": -0.6704230904579163,
      "logps/chosen": -1364.7664794921875,
      "logps/rejected": -826.1878662109375,
      "loss": 0.6839,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.009393024258315563,
      "rewards/margins": 0.019411755725741386,
      "rewards/rejected": -0.028804780915379524,
      "step": 2201
    },
    {
      "epoch": 0.5791974222821351,
      "grad_norm": 716.0,
      "learning_rate": 2.337719298245614e-07,
      "logits/chosen": -0.6440658569335938,
      "logits/rejected": -0.6498790383338928,
      "logps/chosen": -1074.732177734375,
      "logps/rejected": -1234.16455078125,
      "loss": 0.7063,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 2.1171988919377327e-05,
      "rewards/margins": -0.025133896619081497,
      "rewards/rejected": 0.025155067443847656,
      "step": 2202
    },
    {
      "epoch": 0.5794604547173223,
      "grad_norm": 560.0,
      "learning_rate": 2.3362573099415204e-07,
      "logits/chosen": -0.6501711010932922,
      "logits/rejected": -0.6583856344223022,
      "logps/chosen": -1399.7327880859375,
      "logps/rejected": -1140.941162109375,
      "loss": 0.7087,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.03089933656156063,
      "rewards/margins": -0.030177688226103783,
      "rewards/rejected": -0.0007216454832814634,
      "step": 2203
    },
    {
      "epoch": 0.5797234871525095,
      "grad_norm": 568.0,
      "learning_rate": 2.3347953216374269e-07,
      "logits/chosen": -0.6418564319610596,
      "logits/rejected": -0.6571084856987,
      "logps/chosen": -1273.750732421875,
      "logps/rejected": -1178.0322265625,
      "loss": 0.6996,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.02282085455954075,
      "rewards/margins": -0.012227201834321022,
      "rewards/rejected": -0.010593652725219727,
      "step": 2204
    },
    {
      "epoch": 0.5799865195876966,
      "grad_norm": 556.0,
      "learning_rate": 2.3333333333333333e-07,
      "logits/chosen": -0.651595950126648,
      "logits/rejected": -0.6582589149475098,
      "logps/chosen": -1139.10888671875,
      "logps/rejected": -711.9615478515625,
      "loss": 0.6858,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0048719411715865135,
      "rewards/margins": 0.015230465680360794,
      "rewards/rejected": -0.01035852450877428,
      "step": 2205
    },
    {
      "epoch": 0.5802495520228839,
      "grad_norm": 544.0,
      "learning_rate": 2.3318713450292395e-07,
      "logits/chosen": -0.6395997405052185,
      "logits/rejected": -0.6392065286636353,
      "logps/chosen": -1471.9539794921875,
      "logps/rejected": -1268.92431640625,
      "loss": 0.6894,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0027151107788085938,
      "rewards/margins": 0.009145830757915974,
      "rewards/rejected": -0.011860943399369717,
      "step": 2206
    },
    {
      "epoch": 0.580512584458071,
      "grad_norm": 520.0,
      "learning_rate": 2.3304093567251462e-07,
      "logits/chosen": -0.6475396156311035,
      "logits/rejected": -0.6548736095428467,
      "logps/chosen": -1009.0516357421875,
      "logps/rejected": -856.8760375976562,
      "loss": 0.6848,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01137466449290514,
      "rewards/margins": 0.018633365631103516,
      "rewards/rejected": -0.007258703000843525,
      "step": 2207
    },
    {
      "epoch": 0.5807756168932582,
      "grad_norm": 724.0,
      "learning_rate": 2.3289473684210524e-07,
      "logits/chosen": -0.6659828424453735,
      "logits/rejected": -0.6656773090362549,
      "logps/chosen": -1485.923095703125,
      "logps/rejected": -1201.3111572265625,
      "loss": 0.6968,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.010557366535067558,
      "rewards/margins": -0.006182383745908737,
      "rewards/rejected": -0.004374981392174959,
      "step": 2208
    },
    {
      "epoch": 0.5810386493284453,
      "grad_norm": 688.0,
      "learning_rate": 2.327485380116959e-07,
      "logits/chosen": -0.666491687297821,
      "logits/rejected": -0.6499218940734863,
      "logps/chosen": -1473.3809814453125,
      "logps/rejected": -1068.169677734375,
      "loss": 0.7007,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005493353120982647,
      "rewards/margins": -0.013089272193610668,
      "rewards/rejected": 0.007595920003950596,
      "step": 2209
    },
    {
      "epoch": 0.5813016817636325,
      "grad_norm": 664.0,
      "learning_rate": 2.3260233918128654e-07,
      "logits/chosen": -0.6617000102996826,
      "logits/rejected": NaN,
      "logps/chosen": -1917.5797119140625,
      "logps/rejected": -1127.6131591796875,
      "loss": 0.6968,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.034004781395196915,
      "rewards/margins": -0.006125735118985176,
      "rewards/rejected": 0.04013051837682724,
      "step": 2210
    },
    {
      "epoch": 0.5815647141988196,
      "grad_norm": 476.0,
      "learning_rate": 2.3245614035087718e-07,
      "logits/chosen": -0.641451358795166,
      "logits/rejected": -0.644831657409668,
      "logps/chosen": -1186.019287109375,
      "logps/rejected": -958.0337524414062,
      "loss": 0.6906,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03303432837128639,
      "rewards/margins": 0.005817698314785957,
      "rewards/rejected": 0.027216624468564987,
      "step": 2211
    },
    {
      "epoch": 0.5818277466340068,
      "grad_norm": 604.0,
      "learning_rate": 2.3230994152046783e-07,
      "logits/chosen": -0.6357482671737671,
      "logits/rejected": NaN,
      "logps/chosen": -1439.0477294921875,
      "logps/rejected": -1072.73974609375,
      "loss": 0.6803,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.00042409898014739156,
      "rewards/margins": 0.026553917676210403,
      "rewards/rejected": -0.026978017762303352,
      "step": 2212
    },
    {
      "epoch": 0.5820907790691939,
      "grad_norm": 600.0,
      "learning_rate": 2.3216374269005848e-07,
      "logits/chosen": -0.6760803461074829,
      "logits/rejected": -0.6568293571472168,
      "logps/chosen": -1360.3612060546875,
      "logps/rejected": -857.57177734375,
      "loss": 0.6859,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010235786437988281,
      "rewards/margins": 0.015245627611875534,
      "rewards/rejected": -0.0050098407082259655,
      "step": 2213
    },
    {
      "epoch": 0.5823538115043811,
      "grad_norm": 872.0,
      "learning_rate": 2.3201754385964912e-07,
      "logits/chosen": -0.6287403106689453,
      "logits/rejected": -0.6391606330871582,
      "logps/chosen": -1372.8984375,
      "logps/rejected": -1081.9151611328125,
      "loss": 0.7025,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004041480831801891,
      "rewards/margins": -0.016748618334531784,
      "rewards/rejected": 0.02079010009765625,
      "step": 2214
    },
    {
      "epoch": 0.5826168439395683,
      "grad_norm": 474.0,
      "learning_rate": 2.3187134502923974e-07,
      "logits/chosen": -0.6355750560760498,
      "logits/rejected": -0.6394596695899963,
      "logps/chosen": -1112.6845703125,
      "logps/rejected": -969.3973388671875,
      "loss": 0.6914,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0035218235570937395,
      "rewards/margins": 0.004170131869614124,
      "rewards/rejected": -0.00769195519387722,
      "step": 2215
    },
    {
      "epoch": 0.5828798763747555,
      "grad_norm": 454.0,
      "learning_rate": 2.3172514619883042e-07,
      "logits/chosen": -0.648465633392334,
      "logits/rejected": -0.6565027236938477,
      "logps/chosen": -814.8988647460938,
      "logps/rejected": -580.141845703125,
      "loss": 0.6918,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01637129858136177,
      "rewards/margins": 0.00295443506911397,
      "rewards/rejected": -0.019325735047459602,
      "step": 2216
    },
    {
      "epoch": 0.5831429088099426,
      "grad_norm": 516.0,
      "learning_rate": 2.3157894736842104e-07,
      "logits/chosen": -0.6772180795669556,
      "logits/rejected": -0.6775317788124084,
      "logps/chosen": -1002.9127197265625,
      "logps/rejected": -669.8361206054688,
      "loss": 0.7011,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.014686205424368382,
      "rewards/margins": -0.015480902045965195,
      "rewards/rejected": 0.0007946963887661695,
      "step": 2217
    },
    {
      "epoch": 0.5834059412451298,
      "grad_norm": 536.0,
      "learning_rate": 2.3143274853801168e-07,
      "logits/chosen": -0.6507624387741089,
      "logits/rejected": -0.6481515169143677,
      "logps/chosen": -825.6376953125,
      "logps/rejected": -1023.093505859375,
      "loss": 0.6901,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.002196931978687644,
      "rewards/margins": 0.006524037569761276,
      "rewards/rejected": -0.008720969781279564,
      "step": 2218
    },
    {
      "epoch": 0.583668973680317,
      "grad_norm": 588.0,
      "learning_rate": 2.3128654970760233e-07,
      "logits/chosen": -0.6682206988334656,
      "logits/rejected": -0.6746277809143066,
      "logps/chosen": -1174.1658935546875,
      "logps/rejected": -918.8323974609375,
      "loss": 0.6716,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.029999732971191406,
      "rewards/margins": 0.04476499557495117,
      "rewards/rejected": -0.014765264466404915,
      "step": 2219
    },
    {
      "epoch": 0.5839320061155041,
      "grad_norm": 732.0,
      "learning_rate": 2.3114035087719297e-07,
      "logits/chosen": -0.6663205027580261,
      "logits/rejected": -0.6711661219596863,
      "logps/chosen": -999.4321899414062,
      "logps/rejected": -954.175048828125,
      "loss": 0.6846,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004127931781113148,
      "rewards/margins": 0.018229153007268906,
      "rewards/rejected": -0.014101221226155758,
      "step": 2220
    },
    {
      "epoch": 0.5841950385506913,
      "grad_norm": 556.0,
      "learning_rate": 2.309941520467836e-07,
      "logits/chosen": -0.6742037534713745,
      "logits/rejected": -0.6776483058929443,
      "logps/chosen": -1190.541748046875,
      "logps/rejected": -840.3558959960938,
      "loss": 0.6857,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00678176898509264,
      "rewards/margins": 0.015872955322265625,
      "rewards/rejected": -0.02265472523868084,
      "step": 2221
    },
    {
      "epoch": 0.5844580709858784,
      "grad_norm": 540.0,
      "learning_rate": 2.3084795321637427e-07,
      "logits/chosen": -0.6299594640731812,
      "logits/rejected": -0.6435942649841309,
      "logps/chosen": -1442.9251708984375,
      "logps/rejected": -1096.721923828125,
      "loss": 0.6976,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.002132225316017866,
      "rewards/margins": -0.007636452093720436,
      "rewards/rejected": 0.00976867787539959,
      "step": 2222
    },
    {
      "epoch": 0.5847211034210656,
      "grad_norm": 528.0,
      "learning_rate": 2.307017543859649e-07,
      "logits/chosen": -0.6485918164253235,
      "logits/rejected": -0.6569357514381409,
      "logps/chosen": -1114.0941162109375,
      "logps/rejected": -802.6260986328125,
      "loss": 0.6854,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009965324774384499,
      "rewards/margins": 0.01583881489932537,
      "rewards/rejected": -0.005873489193618298,
      "step": 2223
    },
    {
      "epoch": 0.5849841358562528,
      "grad_norm": 572.0,
      "learning_rate": 2.3055555555555556e-07,
      "logits/chosen": -0.6534619927406311,
      "logits/rejected": -0.6730203628540039,
      "logps/chosen": -1220.7340087890625,
      "logps/rejected": -864.6672973632812,
      "loss": 0.7049,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.009238434955477715,
      "rewards/margins": -0.02244577184319496,
      "rewards/rejected": 0.01320733968168497,
      "step": 2224
    },
    {
      "epoch": 0.58524716829144,
      "grad_norm": 552.0,
      "learning_rate": 2.3040935672514618e-07,
      "logits/chosen": -0.6612931489944458,
      "logits/rejected": -0.673904538154602,
      "logps/chosen": -1281.5279541015625,
      "logps/rejected": -776.154541015625,
      "loss": 0.6896,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006369209382683039,
      "rewards/margins": 0.00797262229025364,
      "rewards/rejected": -0.0016034129075706005,
      "step": 2225
    },
    {
      "epoch": 0.5855102007266271,
      "grad_norm": 600.0,
      "learning_rate": 2.3026315789473683e-07,
      "logits/chosen": -0.6654859781265259,
      "logits/rejected": -0.6606422662734985,
      "logps/chosen": -985.2529907226562,
      "logps/rejected": -809.1864013671875,
      "loss": 0.6898,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.011980820447206497,
      "rewards/margins": 0.007332992274314165,
      "rewards/rejected": 0.004647827707231045,
      "step": 2226
    },
    {
      "epoch": 0.5857732331618143,
      "grad_norm": 462.0,
      "learning_rate": 2.301169590643275e-07,
      "logits/chosen": -0.658723771572113,
      "logits/rejected": -0.6632359623908997,
      "logps/chosen": -1128.0137939453125,
      "logps/rejected": -877.0723876953125,
      "loss": 0.6813,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0015712736640125513,
      "rewards/margins": 0.02456517145037651,
      "rewards/rejected": -0.026136446744203568,
      "step": 2227
    },
    {
      "epoch": 0.5860362655970014,
      "grad_norm": 628.0,
      "learning_rate": 2.2997076023391812e-07,
      "logits/chosen": -0.6395001411437988,
      "logits/rejected": -0.6464925408363342,
      "logps/chosen": -1205.0263671875,
      "logps/rejected": -946.617431640625,
      "loss": 0.6968,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.017710495740175247,
      "rewards/margins": -0.006819820962846279,
      "rewards/rejected": -0.010890674777328968,
      "step": 2228
    },
    {
      "epoch": 0.5862992980321886,
      "grad_norm": 552.0,
      "learning_rate": 2.2982456140350877e-07,
      "logits/chosen": -0.6504367589950562,
      "logits/rejected": -0.6609867811203003,
      "logps/chosen": -1297.198974609375,
      "logps/rejected": -1032.3455810546875,
      "loss": 0.6872,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.009356498718261719,
      "rewards/margins": 0.012857580557465553,
      "rewards/rejected": -0.003501081606373191,
      "step": 2229
    },
    {
      "epoch": 0.5865623304673757,
      "grad_norm": 576.0,
      "learning_rate": 2.296783625730994e-07,
      "logits/chosen": -0.666456937789917,
      "logits/rejected": -0.6507732272148132,
      "logps/chosen": -955.0012817382812,
      "logps/rejected": -927.516357421875,
      "loss": 0.6964,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.011736012995243073,
      "rewards/margins": -0.005806732922792435,
      "rewards/rejected": -0.005929278209805489,
      "step": 2230
    },
    {
      "epoch": 0.5868253629025629,
      "grad_norm": 478.0,
      "learning_rate": 2.2953216374269006e-07,
      "logits/chosen": -0.6461895704269409,
      "logits/rejected": -0.6404513716697693,
      "logps/chosen": -1349.7322998046875,
      "logps/rejected": -1022.2696533203125,
      "loss": 0.686,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0011990556959062815,
      "rewards/margins": 0.014665411785244942,
      "rewards/rejected": -0.013466359116137028,
      "step": 2231
    },
    {
      "epoch": 0.58708839533775,
      "grad_norm": 528.0,
      "learning_rate": 2.2938596491228068e-07,
      "logits/chosen": -0.636856734752655,
      "logits/rejected": -0.6393846869468689,
      "logps/chosen": -931.28271484375,
      "logps/rejected": -867.169677734375,
      "loss": 0.6844,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.007162285037338734,
      "rewards/margins": 0.018492411822080612,
      "rewards/rejected": -0.011330127716064453,
      "step": 2232
    },
    {
      "epoch": 0.5873514277729373,
      "grad_norm": 468.0,
      "learning_rate": 2.2923976608187135e-07,
      "logits/chosen": -0.6419047117233276,
      "logits/rejected": -0.646403431892395,
      "logps/chosen": -772.3143310546875,
      "logps/rejected": -830.4486694335938,
      "loss": 0.7007,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.017665576189756393,
      "rewards/margins": -0.014528179541230202,
      "rewards/rejected": 0.032193757593631744,
      "step": 2233
    },
    {
      "epoch": 0.5876144602081245,
      "grad_norm": 486.0,
      "learning_rate": 2.2909356725146197e-07,
      "logits/chosen": -0.6294780373573303,
      "logits/rejected": -0.6370201110839844,
      "logps/chosen": -912.1368408203125,
      "logps/rejected": -475.7125244140625,
      "loss": 0.6853,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.025368116796016693,
      "rewards/margins": 0.016405249014496803,
      "rewards/rejected": 0.008962870575487614,
      "step": 2234
    },
    {
      "epoch": 0.5878774926433116,
      "grad_norm": 540.0,
      "learning_rate": 2.2894736842105262e-07,
      "logits/chosen": -0.6596845388412476,
      "logits/rejected": -0.6591838598251343,
      "logps/chosen": -1098.2542724609375,
      "logps/rejected": -753.9907836914062,
      "loss": 0.7013,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.009634303860366344,
      "rewards/margins": -0.015997696667909622,
      "rewards/rejected": 0.006363392807543278,
      "step": 2235
    },
    {
      "epoch": 0.5881405250784988,
      "grad_norm": 576.0,
      "learning_rate": 2.2880116959064326e-07,
      "logits/chosen": -0.6410818099975586,
      "logits/rejected": -0.6399692893028259,
      "logps/chosen": -1220.43115234375,
      "logps/rejected": -919.5992431640625,
      "loss": 0.6975,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008507919497787952,
      "rewards/margins": -0.00762634351849556,
      "rewards/rejected": -0.0008815769106149673,
      "step": 2236
    },
    {
      "epoch": 0.5884035575136859,
      "grad_norm": 422.0,
      "learning_rate": 2.286549707602339e-07,
      "logits/chosen": -0.6502215266227722,
      "logits/rejected": -0.651094377040863,
      "logps/chosen": -692.443359375,
      "logps/rejected": -643.2903442382812,
      "loss": 0.6957,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.013651705347001553,
      "rewards/margins": -0.004598476458340883,
      "rewards/rejected": -0.009053229354321957,
      "step": 2237
    },
    {
      "epoch": 0.5886665899488731,
      "grad_norm": 568.0,
      "learning_rate": 2.2850877192982453e-07,
      "logits/chosen": -0.6788442730903625,
      "logits/rejected": -0.6672525405883789,
      "logps/chosen": -1328.8004150390625,
      "logps/rejected": -893.0750732421875,
      "loss": 0.7065,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.014915753155946732,
      "rewards/margins": -0.025777578353881836,
      "rewards/rejected": 0.010861825197935104,
      "step": 2238
    },
    {
      "epoch": 0.5889296223840602,
      "grad_norm": 532.0,
      "learning_rate": 2.283625730994152e-07,
      "logits/chosen": -0.629798412322998,
      "logits/rejected": -0.6337906718254089,
      "logps/chosen": -1147.3206787109375,
      "logps/rejected": -931.96142578125,
      "loss": 0.695,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.015283203683793545,
      "rewards/margins": -0.002786919940263033,
      "rewards/rejected": 0.018070127815008163,
      "step": 2239
    },
    {
      "epoch": 0.5891926548192474,
      "grad_norm": 580.0,
      "learning_rate": 2.2821637426900582e-07,
      "logits/chosen": -0.6226959228515625,
      "logits/rejected": NaN,
      "logps/chosen": -980.39697265625,
      "logps/rejected": -783.9244995117188,
      "loss": 0.6919,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.019237902015447617,
      "rewards/margins": 0.0030388841405510902,
      "rewards/rejected": -0.022276783362030983,
      "step": 2240
    },
    {
      "epoch": 0.5894556872544345,
      "grad_norm": 524.0,
      "learning_rate": 2.2807017543859647e-07,
      "logits/chosen": -0.6704712510108948,
      "logits/rejected": -0.6766953468322754,
      "logps/chosen": -1096.3353271484375,
      "logps/rejected": -773.2465209960938,
      "loss": 0.7062,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.010158825665712357,
      "rewards/margins": -0.025089643895626068,
      "rewards/rejected": 0.035248469561338425,
      "step": 2241
    },
    {
      "epoch": 0.5897187196896218,
      "grad_norm": 536.0,
      "learning_rate": 2.2792397660818714e-07,
      "logits/chosen": -0.6600750088691711,
      "logits/rejected": -0.6629973649978638,
      "logps/chosen": -1094.0745849609375,
      "logps/rejected": -932.341796875,
      "loss": 0.6922,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008915710262954235,
      "rewards/margins": 0.0029294504784047604,
      "rewards/rejected": 0.005986261181533337,
      "step": 2242
    },
    {
      "epoch": 0.5899817521248089,
      "grad_norm": 648.0,
      "learning_rate": 2.2777777777777776e-07,
      "logits/chosen": -0.6372975707054138,
      "logits/rejected": -0.6613441109657288,
      "logps/chosen": -1321.8360595703125,
      "logps/rejected": -1041.511962890625,
      "loss": 0.6991,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00663413992151618,
      "rewards/margins": -0.010987091809511185,
      "rewards/rejected": 0.004352951422333717,
      "step": 2243
    },
    {
      "epoch": 0.5902447845599961,
      "grad_norm": 596.0,
      "learning_rate": 2.2763157894736843e-07,
      "logits/chosen": -0.6650123596191406,
      "logits/rejected": NaN,
      "logps/chosen": -1215.253173828125,
      "logps/rejected": -652.3922119140625,
      "loss": 0.6979,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.010481736622750759,
      "rewards/margins": -0.008736132644116879,
      "rewards/rejected": -0.0017456056084483862,
      "step": 2244
    },
    {
      "epoch": 0.5905078169951832,
      "grad_norm": 608.0,
      "learning_rate": 2.2748538011695905e-07,
      "logits/chosen": -0.65166175365448,
      "logits/rejected": -0.6573853492736816,
      "logps/chosen": -1731.6923828125,
      "logps/rejected": -1507.1134033203125,
      "loss": 0.6983,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.004138658754527569,
      "rewards/margins": -0.00890064425766468,
      "rewards/rejected": 0.013039303943514824,
      "step": 2245
    },
    {
      "epoch": 0.5907708494303704,
      "grad_norm": 588.0,
      "learning_rate": 2.273391812865497e-07,
      "logits/chosen": -0.6618717908859253,
      "logits/rejected": -0.6831504702568054,
      "logps/chosen": -1308.7618408203125,
      "logps/rejected": -861.06640625,
      "loss": 0.6943,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.002137375995516777,
      "rewards/margins": -0.0017150393687188625,
      "rewards/rejected": 0.0038524148985743523,
      "step": 2246
    },
    {
      "epoch": 0.5910338818655575,
      "grad_norm": 2432.0,
      "learning_rate": 2.2719298245614035e-07,
      "logits/chosen": -0.6680673360824585,
      "logits/rejected": -0.6695066690444946,
      "logps/chosen": -1265.772216796875,
      "logps/rejected": -1013.3030395507812,
      "loss": 0.6977,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.004587841685861349,
      "rewards/margins": -0.008127069100737572,
      "rewards/rejected": 0.012714911252260208,
      "step": 2247
    },
    {
      "epoch": 0.5912969143007447,
      "grad_norm": 572.0,
      "learning_rate": 2.27046783625731e-07,
      "logits/chosen": -0.6626126766204834,
      "logits/rejected": -0.6684597730636597,
      "logps/chosen": -1419.772216796875,
      "logps/rejected": -1257.1107177734375,
      "loss": 0.6916,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.009483147412538528,
      "rewards/margins": 0.004224683158099651,
      "rewards/rejected": 0.005258465185761452,
      "step": 2248
    },
    {
      "epoch": 0.5915599467359319,
      "grad_norm": 552.0,
      "learning_rate": 2.2690058479532161e-07,
      "logits/chosen": -0.6615610122680664,
      "logits/rejected": -0.6668124794960022,
      "logps/chosen": -1424.6986083984375,
      "logps/rejected": -950.8784790039062,
      "loss": 0.695,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0003688810393214226,
      "rewards/margins": -0.0027040482964366674,
      "rewards/rejected": 0.002335167024284601,
      "step": 2249
    },
    {
      "epoch": 0.591822979171119,
      "grad_norm": 448.0,
      "learning_rate": 2.2675438596491229e-07,
      "logits/chosen": -0.645630955696106,
      "logits/rejected": -0.6432108879089355,
      "logps/chosen": -999.781494140625,
      "logps/rejected": -688.6973876953125,
      "loss": 0.6833,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.017805766314268112,
      "rewards/margins": 0.02021150477230549,
      "rewards/rejected": -0.0024057391565293074,
      "step": 2250
    },
    {
      "epoch": 0.5920860116063063,
      "grad_norm": 564.0,
      "learning_rate": 2.266081871345029e-07,
      "logits/chosen": -0.6382496356964111,
      "logits/rejected": -0.6379877924919128,
      "logps/chosen": -1073.1646728515625,
      "logps/rejected": -895.0267944335938,
      "loss": 0.7009,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.012295200489461422,
      "rewards/margins": -0.01458883285522461,
      "rewards/rejected": 0.0022936342284083366,
      "step": 2251
    },
    {
      "epoch": 0.5923490440414934,
      "grad_norm": 524.0,
      "learning_rate": 2.2646198830409355e-07,
      "logits/chosen": -0.6512689590454102,
      "logits/rejected": -0.6550430059432983,
      "logps/chosen": -1065.0732421875,
      "logps/rejected": -976.0645751953125,
      "loss": 0.6943,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.011666917242109776,
      "rewards/margins": -0.00181832246016711,
      "rewards/rejected": -0.009848594665527344,
      "step": 2252
    },
    {
      "epoch": 0.5926120764766806,
      "grad_norm": 524.0,
      "learning_rate": 2.263157894736842e-07,
      "logits/chosen": -0.6753146648406982,
      "logits/rejected": -0.6836787462234497,
      "logps/chosen": -1297.705810546875,
      "logps/rejected": -973.4315795898438,
      "loss": 0.6924,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.004837704822421074,
      "rewards/margins": 0.0022669800091534853,
      "rewards/rejected": -0.007104684133082628,
      "step": 2253
    },
    {
      "epoch": 0.5928751089118677,
      "grad_norm": 604.0,
      "learning_rate": 2.2616959064327485e-07,
      "logits/chosen": -0.6567413210868835,
      "logits/rejected": -0.6626213788986206,
      "logps/chosen": -1120.7254638671875,
      "logps/rejected": -1144.6160888671875,
      "loss": 0.6911,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004438495263457298,
      "rewards/margins": 0.004893206991255283,
      "rewards/rejected": -0.0004547118442133069,
      "step": 2254
    },
    {
      "epoch": 0.5931381413470549,
      "grad_norm": 458.0,
      "learning_rate": 2.2602339181286547e-07,
      "logits/chosen": -0.6400833129882812,
      "logits/rejected": -0.6445139050483704,
      "logps/chosen": -964.31787109375,
      "logps/rejected": -983.931640625,
      "loss": 0.6875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.00786371249705553,
      "rewards/margins": 0.012100791558623314,
      "rewards/rejected": -0.004237078595906496,
      "step": 2255
    },
    {
      "epoch": 0.593401173782242,
      "grad_norm": 540.0,
      "learning_rate": 2.2587719298245614e-07,
      "logits/chosen": -0.6512026190757751,
      "logits/rejected": -0.6589572429656982,
      "logps/chosen": -1011.9142456054688,
      "logps/rejected": -1045.79052734375,
      "loss": 0.6883,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01993703655898571,
      "rewards/margins": 0.010771369561553001,
      "rewards/rejected": 0.009165667928755283,
      "step": 2256
    },
    {
      "epoch": 0.5936642062174292,
      "grad_norm": 468.0,
      "learning_rate": 2.2573099415204678e-07,
      "logits/chosen": -0.6635352969169617,
      "logits/rejected": -0.6664034724235535,
      "logps/chosen": -1017.529052734375,
      "logps/rejected": -681.5687866210938,
      "loss": 0.7017,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.006196403875946999,
      "rewards/margins": -0.015863418579101562,
      "rewards/rejected": 0.009667014703154564,
      "step": 2257
    },
    {
      "epoch": 0.5939272386526163,
      "grad_norm": 512.0,
      "learning_rate": 2.255847953216374e-07,
      "logits/chosen": -0.6714010238647461,
      "logits/rejected": -0.6768745183944702,
      "logps/chosen": -1067.5711669921875,
      "logps/rejected": -851.7088623046875,
      "loss": 0.6899,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.010232734493911266,
      "rewards/margins": 0.006954431999474764,
      "rewards/rejected": 0.0032783029600977898,
      "step": 2258
    },
    {
      "epoch": 0.5941902710878035,
      "grad_norm": 462.0,
      "learning_rate": 2.2543859649122808e-07,
      "logits/chosen": -0.6848406791687012,
      "logits/rejected": -0.6897676587104797,
      "logps/chosen": -1045.984130859375,
      "logps/rejected": -858.3267822265625,
      "loss": 0.6939,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.001458359183743596,
      "rewards/margins": -0.0007455830927938223,
      "rewards/rejected": 0.0022039411123842,
      "step": 2259
    },
    {
      "epoch": 0.5944533035229906,
      "grad_norm": 612.0,
      "learning_rate": 2.252923976608187e-07,
      "logits/chosen": -0.6351069211959839,
      "logits/rejected": -0.6225795745849609,
      "logps/chosen": -1147.8543701171875,
      "logps/rejected": -1310.295654296875,
      "loss": 0.6917,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.00434722937643528,
      "rewards/margins": 0.0035815241280943155,
      "rewards/rejected": -0.007928753271698952,
      "step": 2260
    },
    {
      "epoch": 0.5947163359581779,
      "grad_norm": 560.0,
      "learning_rate": 2.2514619883040934e-07,
      "logits/chosen": -0.6476957201957703,
      "logits/rejected": -0.677409291267395,
      "logps/chosen": -1266.2662353515625,
      "logps/rejected": -952.6426391601562,
      "loss": 0.6958,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004344749264419079,
      "rewards/margins": -0.0038637155666947365,
      "rewards/rejected": 0.008208464831113815,
      "step": 2261
    },
    {
      "epoch": 0.594979368393365,
      "grad_norm": 580.0,
      "learning_rate": 2.25e-07,
      "logits/chosen": -0.6599774360656738,
      "logits/rejected": -0.653195858001709,
      "logps/chosen": -1028.035888671875,
      "logps/rejected": -838.8869018554688,
      "loss": 0.6997,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0050373077392578125,
      "rewards/margins": -0.012084772810339928,
      "rewards/rejected": 0.01712207868695259,
      "step": 2262
    },
    {
      "epoch": 0.5952424008285522,
      "grad_norm": 378.0,
      "learning_rate": 2.2485380116959064e-07,
      "logits/chosen": -0.636330783367157,
      "logits/rejected": -0.6411451101303101,
      "logps/chosen": -813.6362915039062,
      "logps/rejected": -856.563720703125,
      "loss": 0.6853,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01882619969546795,
      "rewards/margins": 0.016250325366854668,
      "rewards/rejected": 0.0025758747942745686,
      "step": 2263
    },
    {
      "epoch": 0.5955054332637393,
      "grad_norm": 524.0,
      "learning_rate": 2.2470760233918126e-07,
      "logits/chosen": -0.6496321558952332,
      "logits/rejected": -0.6576182842254639,
      "logps/chosen": -1136.311279296875,
      "logps/rejected": -781.073974609375,
      "loss": 0.7079,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.027704669162631035,
      "rewards/margins": -0.028491828590631485,
      "rewards/rejected": 0.0007871624547988176,
      "step": 2264
    },
    {
      "epoch": 0.5957684656989265,
      "grad_norm": 596.0,
      "learning_rate": 2.2456140350877193e-07,
      "logits/chosen": -0.6520965099334717,
      "logits/rejected": -0.6519982814788818,
      "logps/chosen": -1430.7177734375,
      "logps/rejected": -1310.0198974609375,
      "loss": 0.6814,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.033200453966856,
      "rewards/margins": 0.023793412372469902,
      "rewards/rejected": 0.00940704345703125,
      "step": 2265
    },
    {
      "epoch": 0.5960314981341137,
      "grad_norm": 430.0,
      "learning_rate": 2.2441520467836255e-07,
      "logits/chosen": -0.6634746789932251,
      "logits/rejected": -0.673223614692688,
      "logps/chosen": -912.5943603515625,
      "logps/rejected": -810.9327392578125,
      "loss": 0.6904,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.004543972201645374,
      "rewards/margins": 0.0059320442378520966,
      "rewards/rejected": -0.010476017370820045,
      "step": 2266
    },
    {
      "epoch": 0.5962945305693008,
      "grad_norm": 524.0,
      "learning_rate": 2.2426900584795322e-07,
      "logits/chosen": -0.6485470533370972,
      "logits/rejected": -0.6447204947471619,
      "logps/chosen": -1072.54443359375,
      "logps/rejected": -790.864501953125,
      "loss": 0.7052,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.004030132200568914,
      "rewards/margins": -0.023492911830544472,
      "rewards/rejected": 0.027523040771484375,
      "step": 2267
    },
    {
      "epoch": 0.596557563004488,
      "grad_norm": 508.0,
      "learning_rate": 2.2412280701754384e-07,
      "logits/chosen": -0.6313591599464417,
      "logits/rejected": -0.6362928748130798,
      "logps/chosen": -1051.8817138671875,
      "logps/rejected": -612.6459350585938,
      "loss": 0.684,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.024021627381443977,
      "rewards/margins": 0.018717478960752487,
      "rewards/rejected": 0.005304146558046341,
      "step": 2268
    },
    {
      "epoch": 0.5968205954396751,
      "grad_norm": 420.0,
      "learning_rate": 2.239766081871345e-07,
      "logits/chosen": -0.6712552905082703,
      "logits/rejected": -0.6720257997512817,
      "logps/chosen": -969.964599609375,
      "logps/rejected": -994.8373413085938,
      "loss": 0.6932,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00400543212890625,
      "rewards/margins": 0.0005116453394293785,
      "rewards/rejected": -0.004517078399658203,
      "step": 2269
    },
    {
      "epoch": 0.5970836278748624,
      "grad_norm": 450.0,
      "learning_rate": 2.2383040935672513e-07,
      "logits/chosen": -0.6703519225120544,
      "logits/rejected": -0.6750322580337524,
      "logps/chosen": -932.79150390625,
      "logps/rejected": -862.5929565429688,
      "loss": 0.6968,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.022196101024746895,
      "rewards/margins": -0.006284618750214577,
      "rewards/rejected": -0.015911484137177467,
      "step": 2270
    },
    {
      "epoch": 0.5973466603100495,
      "grad_norm": 620.0,
      "learning_rate": 2.2368421052631578e-07,
      "logits/chosen": -0.6571004390716553,
      "logits/rejected": -0.6613839268684387,
      "logps/chosen": -1214.898681640625,
      "logps/rejected": -998.4442749023438,
      "loss": 0.6881,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00037574832094833255,
      "rewards/margins": 0.010540484450757504,
      "rewards/rejected": -0.010916233994066715,
      "step": 2271
    },
    {
      "epoch": 0.5976096927452367,
      "grad_norm": 496.0,
      "learning_rate": 2.2353801169590643e-07,
      "logits/chosen": -0.6490243673324585,
      "logits/rejected": -0.6591761112213135,
      "logps/chosen": -1285.2874755859375,
      "logps/rejected": -1195.526611328125,
      "loss": 0.7036,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.003711128607392311,
      "rewards/margins": -0.019994162023067474,
      "rewards/rejected": 0.016283035278320312,
      "step": 2272
    },
    {
      "epoch": 0.5978727251804238,
      "grad_norm": 592.0,
      "learning_rate": 2.2339181286549707e-07,
      "logits/chosen": -0.6365290880203247,
      "logits/rejected": -0.6265351176261902,
      "logps/chosen": -1199.0250244140625,
      "logps/rejected": -675.0494995117188,
      "loss": 0.6965,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0028501502238214016,
      "rewards/margins": -0.005530644208192825,
      "rewards/rejected": 0.008380794897675514,
      "step": 2273
    },
    {
      "epoch": 0.598135757615611,
      "grad_norm": 656.0,
      "learning_rate": 2.2324561403508772e-07,
      "logits/chosen": -0.6454261541366577,
      "logits/rejected": -0.651033341884613,
      "logps/chosen": -1517.8056640625,
      "logps/rejected": -1062.9483642578125,
      "loss": 0.6939,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005820656195282936,
      "rewards/margins": -0.00044841691851615906,
      "rewards/rejected": 0.0062690735794603825,
      "step": 2274
    },
    {
      "epoch": 0.5983987900507981,
      "grad_norm": 502.0,
      "learning_rate": 2.2309941520467834e-07,
      "logits/chosen": -0.6382817029953003,
      "logits/rejected": -0.6501376628875732,
      "logps/chosen": -1063.6717529296875,
      "logps/rejected": -799.1516723632812,
      "loss": 0.703,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.012645244598388672,
      "rewards/margins": -0.019208623096346855,
      "rewards/rejected": 0.006563378032296896,
      "step": 2275
    },
    {
      "epoch": 0.5986618224859853,
      "grad_norm": 564.0,
      "learning_rate": 2.22953216374269e-07,
      "logits/chosen": -0.6463431119918823,
      "logits/rejected": -0.6538259983062744,
      "logps/chosen": -1296.7991943359375,
      "logps/rejected": -891.5261840820312,
      "loss": 0.7012,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.00243701902218163,
      "rewards/margins": -0.015300273895263672,
      "rewards/rejected": 0.01286325417459011,
      "step": 2276
    },
    {
      "epoch": 0.5989248549211724,
      "grad_norm": 620.0,
      "learning_rate": 2.2280701754385963e-07,
      "logits/chosen": -0.6454955339431763,
      "logits/rejected": -0.6467558145523071,
      "logps/chosen": -1286.1783447265625,
      "logps/rejected": -1327.0103759765625,
      "loss": 0.6902,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0019424444762989879,
      "rewards/margins": 0.006590653210878372,
      "rewards/rejected": -0.008533095940947533,
      "step": 2277
    },
    {
      "epoch": 0.5991878873563596,
      "grad_norm": 498.0,
      "learning_rate": 2.2266081871345028e-07,
      "logits/chosen": -0.6667273044586182,
      "logits/rejected": -0.6599315404891968,
      "logps/chosen": -1082.3223876953125,
      "logps/rejected": -936.9593505859375,
      "loss": 0.6931,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.001159476232714951,
      "rewards/margins": 0.0012600896880030632,
      "rewards/rejected": -0.002419566735625267,
      "step": 2278
    },
    {
      "epoch": 0.5994509197915467,
      "grad_norm": 572.0,
      "learning_rate": 2.2251461988304093e-07,
      "logits/chosen": -0.6697176098823547,
      "logits/rejected": NaN,
      "logps/chosen": -1631.5889892578125,
      "logps/rejected": -952.1215209960938,
      "loss": 0.7074,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.01597604900598526,
      "rewards/margins": -0.02701892890036106,
      "rewards/rejected": 0.011042880825698376,
      "step": 2279
    },
    {
      "epoch": 0.599713952226734,
      "grad_norm": 488.0,
      "learning_rate": 2.2236842105263157e-07,
      "logits/chosen": -0.6434754133224487,
      "logits/rejected": -0.6390355825424194,
      "logps/chosen": -1192.1947021484375,
      "logps/rejected": -971.9594116210938,
      "loss": 0.6916,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.021895408630371094,
      "rewards/margins": 0.004149818327277899,
      "rewards/rejected": 0.017745589837431908,
      "step": 2280
    },
    {
      "epoch": 0.5999769846619212,
      "grad_norm": 504.0,
      "learning_rate": 2.222222222222222e-07,
      "logits/chosen": -0.6377279758453369,
      "logits/rejected": -0.6394221782684326,
      "logps/chosen": -1209.116943359375,
      "logps/rejected": -913.6373291015625,
      "loss": 0.6866,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.019040871411561966,
      "rewards/margins": 0.014064311049878597,
      "rewards/rejected": 0.004976559430360794,
      "step": 2281
    },
    {
      "epoch": 0.6002400170971083,
      "grad_norm": 712.0,
      "learning_rate": 2.2207602339181286e-07,
      "logits/chosen": -0.6399545669555664,
      "logits/rejected": -0.6474955677986145,
      "logps/chosen": -1543.6422119140625,
      "logps/rejected": -756.377197265625,
      "loss": 0.6786,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.005597591400146484,
      "rewards/margins": 0.03025808185338974,
      "rewards/rejected": -0.024660494178533554,
      "step": 2282
    },
    {
      "epoch": 0.6005030495322955,
      "grad_norm": 540.0,
      "learning_rate": 2.2192982456140348e-07,
      "logits/chosen": -0.6655070781707764,
      "logits/rejected": -0.6755620837211609,
      "logps/chosen": -1722.6533203125,
      "logps/rejected": -1183.4923095703125,
      "loss": 0.7075,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.005342578981071711,
      "rewards/margins": -0.02773313596844673,
      "rewards/rejected": 0.02239055559039116,
      "step": 2283
    },
    {
      "epoch": 0.6007660819674826,
      "grad_norm": 572.0,
      "learning_rate": 2.2178362573099413e-07,
      "logits/chosen": -0.6629101037979126,
      "logits/rejected": -0.6589374542236328,
      "logps/chosen": -1039.9569091796875,
      "logps/rejected": -812.4078369140625,
      "loss": 0.6946,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.002373314928263426,
      "rewards/margins": -0.0020481105893850327,
      "rewards/rejected": 0.0044214241206645966,
      "step": 2284
    },
    {
      "epoch": 0.6010291144026698,
      "grad_norm": 450.0,
      "learning_rate": 2.2163742690058478e-07,
      "logits/chosen": -0.6389980316162109,
      "logits/rejected": -0.6428239941596985,
      "logps/chosen": -774.4331665039062,
      "logps/rejected": -716.0704956054688,
      "loss": 0.6968,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.009997176006436348,
      "rewards/margins": -0.0069609652273356915,
      "rewards/rejected": 0.016958141699433327,
      "step": 2285
    },
    {
      "epoch": 0.6012921468378569,
      "grad_norm": 474.0,
      "learning_rate": 2.2149122807017542e-07,
      "logits/chosen": -0.6261014342308044,
      "logits/rejected": -0.6416071653366089,
      "logps/chosen": -1060.2130126953125,
      "logps/rejected": -707.40771484375,
      "loss": 0.6738,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02486458048224449,
      "rewards/margins": 0.04044938087463379,
      "rewards/rejected": -0.015584803186357021,
      "step": 2286
    },
    {
      "epoch": 0.6015551792730441,
      "grad_norm": 560.0,
      "learning_rate": 2.213450292397661e-07,
      "logits/chosen": -0.6363581418991089,
      "logits/rejected": -0.6394193768501282,
      "logps/chosen": -939.4083862304688,
      "logps/rejected": -740.5606689453125,
      "loss": 0.6981,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.004092407412827015,
      "rewards/margins": -0.009374618530273438,
      "rewards/rejected": 0.005282210651785135,
      "step": 2287
    },
    {
      "epoch": 0.6018182117082312,
      "grad_norm": 428.0,
      "learning_rate": 2.2119883040935672e-07,
      "logits/chosen": -0.6442164182662964,
      "logits/rejected": -0.6489943265914917,
      "logps/chosen": -678.7261962890625,
      "logps/rejected": -577.291015625,
      "loss": 0.6877,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.007303666789084673,
      "rewards/margins": 0.01124720461666584,
      "rewards/rejected": -0.003943539224565029,
      "step": 2288
    },
    {
      "epoch": 0.6020812441434185,
      "grad_norm": 560.0,
      "learning_rate": 2.2105263157894736e-07,
      "logits/chosen": -0.6451142430305481,
      "logits/rejected": -0.6462133526802063,
      "logps/chosen": -1257.723876953125,
      "logps/rejected": -1236.266845703125,
      "loss": 0.6908,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.004935074131935835,
      "rewards/margins": 0.005598735064268112,
      "rewards/rejected": -0.010533809661865234,
      "step": 2289
    },
    {
      "epoch": 0.6023442765786056,
      "grad_norm": 468.0,
      "learning_rate": 2.20906432748538e-07,
      "logits/chosen": -0.6564833521842957,
      "logits/rejected": -0.6511625647544861,
      "logps/chosen": -1149.6666259765625,
      "logps/rejected": -1088.962890625,
      "loss": 0.6863,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.015072634443640709,
      "rewards/margins": 0.01472263503819704,
      "rewards/rejected": -0.029795266687870026,
      "step": 2290
    },
    {
      "epoch": 0.6026073090137928,
      "grad_norm": 482.0,
      "learning_rate": 2.2076023391812866e-07,
      "logits/chosen": -0.6514440774917603,
      "logits/rejected": -0.6587567925453186,
      "logps/chosen": -1012.8821411132812,
      "logps/rejected": -751.3463745117188,
      "loss": 0.6946,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.015067864209413528,
      "rewards/margins": -0.0018071176018565893,
      "rewards/rejected": 0.01687498204410076,
      "step": 2291
    },
    {
      "epoch": 0.6028703414489799,
      "grad_norm": 564.0,
      "learning_rate": 2.2061403508771928e-07,
      "logits/chosen": -0.6369065046310425,
      "logits/rejected": -0.6407644152641296,
      "logps/chosen": -924.6771240234375,
      "logps/rejected": -828.60498046875,
      "loss": 0.7037,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.010123920626938343,
      "rewards/margins": -0.0203991886228323,
      "rewards/rejected": 0.01027526892721653,
      "step": 2292
    },
    {
      "epoch": 0.6031333738841671,
      "grad_norm": 498.0,
      "learning_rate": 2.2046783625730995e-07,
      "logits/chosen": -0.6497144103050232,
      "logits/rejected": -0.6531537771224976,
      "logps/chosen": -771.7914428710938,
      "logps/rejected": -805.8975830078125,
      "loss": 0.6959,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01576819457113743,
      "rewards/margins": -0.004939412698149681,
      "rewards/rejected": -0.010828780941665173,
      "step": 2293
    },
    {
      "epoch": 0.6033964063193542,
      "grad_norm": 424.0,
      "learning_rate": 2.2032163742690057e-07,
      "logits/chosen": -0.6381365656852722,
      "logits/rejected": -0.6394147872924805,
      "logps/chosen": -920.233154296875,
      "logps/rejected": -617.6300659179688,
      "loss": 0.682,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.8514303267002106e-05,
      "rewards/margins": 0.023013876751065254,
      "rewards/rejected": -0.02304239384829998,
      "step": 2294
    },
    {
      "epoch": 0.6036594387545414,
      "grad_norm": 472.0,
      "learning_rate": 2.2017543859649121e-07,
      "logits/chosen": -0.6554661989212036,
      "logits/rejected": -0.6619917750358582,
      "logps/chosen": -1229.323974609375,
      "logps/rejected": -1031.9512939453125,
      "loss": 0.6767,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.026076126843690872,
      "rewards/margins": 0.03359346091747284,
      "rewards/rejected": -0.007517337333410978,
      "step": 2295
    },
    {
      "epoch": 0.6039224711897286,
      "grad_norm": 640.0,
      "learning_rate": 2.2002923976608186e-07,
      "logits/chosen": -0.6353873014450073,
      "logits/rejected": -0.6529735326766968,
      "logps/chosen": -1662.2506103515625,
      "logps/rejected": -1281.8785400390625,
      "loss": 0.6887,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004649352747946978,
      "rewards/margins": 0.009523391723632812,
      "rewards/rejected": -0.004874038975685835,
      "step": 2296
    },
    {
      "epoch": 0.6041855036249157,
      "grad_norm": 540.0,
      "learning_rate": 2.198830409356725e-07,
      "logits/chosen": -0.6601430773735046,
      "logits/rejected": -0.6627526879310608,
      "logps/chosen": -1486.108642578125,
      "logps/rejected": -1313.6571044921875,
      "loss": 0.6849,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01764822006225586,
      "rewards/margins": 0.01722259633243084,
      "rewards/rejected": 0.00042562466114759445,
      "step": 2297
    },
    {
      "epoch": 0.604448536060103,
      "grad_norm": 420.0,
      "learning_rate": 2.1973684210526313e-07,
      "logits/chosen": -0.6764863133430481,
      "logits/rejected": -0.6694476008415222,
      "logps/chosen": -715.1788330078125,
      "logps/rejected": -715.1881103515625,
      "loss": 0.7038,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.003299904055893421,
      "rewards/margins": -0.020493507385253906,
      "rewards/rejected": 0.01719360426068306,
      "step": 2298
    },
    {
      "epoch": 0.6047115684952901,
      "grad_norm": 432.0,
      "learning_rate": 2.195906432748538e-07,
      "logits/chosen": -0.6578142642974854,
      "logits/rejected": -0.6597893834114075,
      "logps/chosen": -763.7015991210938,
      "logps/rejected": -842.9200439453125,
      "loss": 0.6949,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.013463211245834827,
      "rewards/margins": -0.003319167997688055,
      "rewards/rejected": -0.010144044645130634,
      "step": 2299
    },
    {
      "epoch": 0.6049746009304773,
      "grad_norm": 504.0,
      "learning_rate": 2.1944444444444442e-07,
      "logits/chosen": -0.6339885592460632,
      "logits/rejected": -0.6472615003585815,
      "logps/chosen": -1073.7884521484375,
      "logps/rejected": -919.8585205078125,
      "loss": 0.6916,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0027181627228856087,
      "rewards/margins": 0.0038758274167776108,
      "rewards/rejected": -0.006593991070985794,
      "step": 2300
    },
    {
      "epoch": 0.6052376333656644,
      "grad_norm": 512.0,
      "learning_rate": 2.1929824561403507e-07,
      "logits/chosen": -0.6139378547668457,
      "logits/rejected": -0.6152337789535522,
      "logps/chosen": -1122.0609130859375,
      "logps/rejected": -808.0708618164062,
      "loss": 0.6998,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.020904159173369408,
      "rewards/margins": -0.01251764316111803,
      "rewards/rejected": -0.008386516012251377,
      "step": 2301
    },
    {
      "epoch": 0.6055006658008516,
      "grad_norm": 656.0,
      "learning_rate": 2.1915204678362574e-07,
      "logits/chosen": -0.6306737661361694,
      "logits/rejected": -0.6174444556236267,
      "logps/chosen": -1241.8885498046875,
      "logps/rejected": -858.4098510742188,
      "loss": 0.695,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.005067253019660711,
      "rewards/margins": -0.0030651099514216185,
      "rewards/rejected": -0.00200214353390038,
      "step": 2302
    },
    {
      "epoch": 0.6057636982360387,
      "grad_norm": 532.0,
      "learning_rate": 2.1900584795321636e-07,
      "logits/chosen": -0.6599708795547485,
      "logits/rejected": -0.6707993745803833,
      "logps/chosen": -982.379150390625,
      "logps/rejected": -795.9701538085938,
      "loss": 0.686,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.018665121868252754,
      "rewards/margins": 0.015424869954586029,
      "rewards/rejected": 0.0032402516808360815,
      "step": 2303
    },
    {
      "epoch": 0.6060267306712259,
      "grad_norm": 532.0,
      "learning_rate": 2.18859649122807e-07,
      "logits/chosen": -0.6799353957176208,
      "logits/rejected": -0.6764771938323975,
      "logps/chosen": -1144.308349609375,
      "logps/rejected": -823.12548828125,
      "loss": 0.678,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.03031759336590767,
      "rewards/margins": 0.03158111497759819,
      "rewards/rejected": -0.0012635234743356705,
      "step": 2304
    },
    {
      "epoch": 0.606289763106413,
      "grad_norm": 604.0,
      "learning_rate": 2.1871345029239765e-07,
      "logits/chosen": -0.667569637298584,
      "logits/rejected": -0.6744455099105835,
      "logps/chosen": -1436.5743408203125,
      "logps/rejected": -872.5663452148438,
      "loss": 0.7009,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.017065001651644707,
      "rewards/margins": -0.014946316368877888,
      "rewards/rejected": -0.0021186829544603825,
      "step": 2305
    },
    {
      "epoch": 0.6065527955416002,
      "grad_norm": 632.0,
      "learning_rate": 2.185672514619883e-07,
      "logits/chosen": -0.6566082239151001,
      "logits/rejected": -0.6664537787437439,
      "logps/chosen": -1719.862060546875,
      "logps/rejected": -1391.1685791015625,
      "loss": 0.6913,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.018896104767918587,
      "rewards/margins": 0.0044672018848359585,
      "rewards/rejected": 0.014428900554776192,
      "step": 2306
    },
    {
      "epoch": 0.6068158279767873,
      "grad_norm": 604.0,
      "learning_rate": 2.1842105263157894e-07,
      "logits/chosen": -0.6523323059082031,
      "logits/rejected": -0.6562436819076538,
      "logps/chosen": -1572.0023193359375,
      "logps/rejected": -1077.88720703125,
      "loss": 0.6864,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.00993442628532648,
      "rewards/margins": 0.014430142007768154,
      "rewards/rejected": -0.004495715722441673,
      "step": 2307
    },
    {
      "epoch": 0.6070788604119746,
      "grad_norm": 310.0,
      "learning_rate": 2.182748538011696e-07,
      "logits/chosen": -0.6444135904312134,
      "logits/rejected": -0.6441490650177002,
      "logps/chosen": -791.4536743164062,
      "logps/rejected": -702.2420043945312,
      "loss": 0.6943,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.014923285692930222,
      "rewards/margins": -0.002112150192260742,
      "rewards/rejected": 0.017035435885190964,
      "step": 2308
    },
    {
      "epoch": 0.6073418928471617,
      "grad_norm": 524.0,
      "learning_rate": 2.181286549707602e-07,
      "logits/chosen": -0.6540353298187256,
      "logits/rejected": -0.6614155769348145,
      "logps/chosen": -1002.4044189453125,
      "logps/rejected": -979.0084228515625,
      "loss": 0.6891,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.009067201055586338,
      "rewards/margins": 0.009014083072543144,
      "rewards/rejected": -0.018081283196806908,
      "step": 2309
    },
    {
      "epoch": 0.6076049252823489,
      "grad_norm": 580.0,
      "learning_rate": 2.1798245614035088e-07,
      "logits/chosen": -0.6527329683303833,
      "logits/rejected": -0.6515753865242004,
      "logps/chosen": -1225.64111328125,
      "logps/rejected": -1130.28662109375,
      "loss": 0.679,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.007430267054587603,
      "rewards/margins": 0.03047199547290802,
      "rewards/rejected": -0.023041723296046257,
      "step": 2310
    },
    {
      "epoch": 0.607867957717536,
      "grad_norm": 676.0,
      "learning_rate": 2.178362573099415e-07,
      "logits/chosen": -0.6410969495773315,
      "logits/rejected": -0.6516053676605225,
      "logps/chosen": -1229.6116943359375,
      "logps/rejected": -1237.348876953125,
      "loss": 0.6996,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.005004500970244408,
      "rewards/margins": -0.011909483931958675,
      "rewards/rejected": 0.006904983427375555,
      "step": 2311
    },
    {
      "epoch": 0.6081309901527232,
      "grad_norm": 556.0,
      "learning_rate": 2.1769005847953215e-07,
      "logits/chosen": -0.6641777753829956,
      "logits/rejected": -0.6730693578720093,
      "logps/chosen": -1196.9034423828125,
      "logps/rejected": -1128.50341796875,
      "loss": 0.707,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.02230091020464897,
      "rewards/margins": -0.026661206036806107,
      "rewards/rejected": 0.004360293969511986,
      "step": 2312
    },
    {
      "epoch": 0.6083940225879104,
      "grad_norm": 520.0,
      "learning_rate": 2.175438596491228e-07,
      "logits/chosen": -0.643561840057373,
      "logits/rejected": -0.6542521715164185,
      "logps/chosen": -1104.00732421875,
      "logps/rejected": -896.0236206054688,
      "loss": 0.6986,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.025548934936523438,
      "rewards/margins": -0.010526848025619984,
      "rewards/rejected": -0.015022087842226028,
      "step": 2313
    },
    {
      "epoch": 0.6086570550230975,
      "grad_norm": 488.0,
      "learning_rate": 2.1739766081871344e-07,
      "logits/chosen": -0.6447792053222656,
      "logits/rejected": -0.635398805141449,
      "logps/chosen": -1080.328369140625,
      "logps/rejected": -1027.6363525390625,
      "loss": 0.6877,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.015304564498364925,
      "rewards/margins": 0.011815357021987438,
      "rewards/rejected": 0.0034892093390226364,
      "step": 2314
    },
    {
      "epoch": 0.6089200874582847,
      "grad_norm": 476.0,
      "learning_rate": 2.1725146198830406e-07,
      "logits/chosen": -0.6487351059913635,
      "logits/rejected": -0.6488304138183594,
      "logps/chosen": -1020.1358032226562,
      "logps/rejected": -1048.4921875,
      "loss": 0.6852,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008239364251494408,
      "rewards/margins": 0.01658029481768608,
      "rewards/rejected": -0.008340930566191673,
      "step": 2315
    },
    {
      "epoch": 0.6091831198934718,
      "grad_norm": 464.0,
      "learning_rate": 2.1710526315789474e-07,
      "logits/chosen": -0.6763243079185486,
      "logits/rejected": -0.669409990310669,
      "logps/chosen": -862.3241577148438,
      "logps/rejected": -663.179443359375,
      "loss": 0.6943,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.003504705848172307,
      "rewards/margins": -0.0018503665924072266,
      "rewards/rejected": 0.00535507220774889,
      "step": 2316
    },
    {
      "epoch": 0.6094461523286591,
      "grad_norm": 568.0,
      "learning_rate": 2.1695906432748538e-07,
      "logits/chosen": -0.6568195819854736,
      "logits/rejected": -0.6581493020057678,
      "logps/chosen": -1200.09130859375,
      "logps/rejected": -1048.505126953125,
      "loss": 0.6864,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.01977406069636345,
      "rewards/margins": 0.013944149948656559,
      "rewards/rejected": 0.005829904228448868,
      "step": 2317
    },
    {
      "epoch": 0.6097091847638462,
      "grad_norm": 660.0,
      "learning_rate": 2.16812865497076e-07,
      "logits/chosen": -0.6636393070220947,
      "logits/rejected": -0.6694166660308838,
      "logps/chosen": -1466.981201171875,
      "logps/rejected": -1258.76708984375,
      "loss": 0.7073,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01761798746883869,
      "rewards/margins": -0.027194881811738014,
      "rewards/rejected": 0.009576892480254173,
      "step": 2318
    },
    {
      "epoch": 0.6099722171990334,
      "grad_norm": 540.0,
      "learning_rate": 2.1666666666666667e-07,
      "logits/chosen": -0.6365056037902832,
      "logits/rejected": -0.6484338641166687,
      "logps/chosen": -1132.4031982421875,
      "logps/rejected": -985.4927368164062,
      "loss": 0.6962,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.014814662747085094,
      "rewards/margins": -0.005953073501586914,
      "rewards/rejected": -0.00886158924549818,
      "step": 2319
    },
    {
      "epoch": 0.6102352496342205,
      "grad_norm": 552.0,
      "learning_rate": 2.165204678362573e-07,
      "logits/chosen": -0.6540794968605042,
      "logits/rejected": -0.6645479798316956,
      "logps/chosen": -869.9873657226562,
      "logps/rejected": -710.997314453125,
      "loss": 0.6954,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.011762714013457298,
      "rewards/margins": -0.003942202776670456,
      "rewards/rejected": -0.007820512168109417,
      "step": 2320
    },
    {
      "epoch": 0.6104982820694077,
      "grad_norm": 560.0,
      "learning_rate": 2.1637426900584794e-07,
      "logits/chosen": -0.6289263963699341,
      "logits/rejected": -0.6353657841682434,
      "logps/chosen": -990.6599731445312,
      "logps/rejected": -829.8740844726562,
      "loss": 0.6813,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.018503570929169655,
      "rewards/margins": 0.024980641901493073,
      "rewards/rejected": -0.006477068178355694,
      "step": 2321
    },
    {
      "epoch": 0.6107613145045948,
      "grad_norm": 596.0,
      "learning_rate": 2.162280701754386e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6321125626564026,
      "logps/chosen": -737.5405883789062,
      "logps/rejected": -837.7752685546875,
      "loss": 0.6784,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.020102214068174362,
      "rewards/margins": 0.030892468988895416,
      "rewards/rejected": -0.01079025212675333,
      "step": 2322
    },
    {
      "epoch": 0.611024346939782,
      "grad_norm": 600.0,
      "learning_rate": 2.1608187134502923e-07,
      "logits/chosen": -0.642228364944458,
      "logits/rejected": -0.6396000385284424,
      "logps/chosen": -1350.7071533203125,
      "logps/rejected": -1130.633056640625,
      "loss": 0.6884,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0013099673669785261,
      "rewards/margins": 0.010715105570852757,
      "rewards/rejected": -0.012025070376694202,
      "step": 2323
    },
    {
      "epoch": 0.6112873793749691,
      "grad_norm": 486.0,
      "learning_rate": 2.1593567251461985e-07,
      "logits/chosen": -0.6507499814033508,
      "logits/rejected": -0.6554633378982544,
      "logps/chosen": -946.5870361328125,
      "logps/rejected": -604.490966796875,
      "loss": 0.7004,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.010009384714066982,
      "rewards/margins": -0.013302327133715153,
      "rewards/rejected": 0.0032929428853094578,
      "step": 2324
    },
    {
      "epoch": 0.6115504118101563,
      "grad_norm": 556.0,
      "learning_rate": 2.1578947368421053e-07,
      "logits/chosen": -0.6803808212280273,
      "logits/rejected": -0.6814324855804443,
      "logps/chosen": -1649.3603515625,
      "logps/rejected": -1281.2283935546875,
      "loss": 0.7003,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.007397078443318605,
      "rewards/margins": -0.01350393332540989,
      "rewards/rejected": 0.006106851622462273,
      "step": 2325
    },
    {
      "epoch": 0.6118134442453435,
      "grad_norm": 488.0,
      "learning_rate": 2.1564327485380115e-07,
      "logits/chosen": -0.6534985303878784,
      "logits/rejected": -0.6699833869934082,
      "logps/chosen": -832.7158813476562,
      "logps/rejected": -672.0584716796875,
      "loss": 0.6935,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.008392619900405407,
      "rewards/margins": -7.82012939453125e-05,
      "rewards/rejected": -0.008314418606460094,
      "step": 2326
    },
    {
      "epoch": 0.6120764766805307,
      "grad_norm": 580.0,
      "learning_rate": 2.1549707602339182e-07,
      "logits/chosen": -0.6110532283782959,
      "logits/rejected": -0.6229729056358337,
      "logps/chosen": -1096.18408203125,
      "logps/rejected": -777.7682495117188,
      "loss": 0.6962,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.017029380425810814,
      "rewards/margins": -0.005069160368293524,
      "rewards/rejected": -0.011960221454501152,
      "step": 2327
    },
    {
      "epoch": 0.6123395091157179,
      "grad_norm": 456.0,
      "learning_rate": 2.1535087719298244e-07,
      "logits/chosen": -0.6734393835067749,
      "logits/rejected": -0.664073646068573,
      "logps/chosen": -723.42138671875,
      "logps/rejected": -656.8562622070312,
      "loss": 0.6755,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0290966983884573,
      "rewards/margins": 0.036822035908699036,
      "rewards/rejected": -0.0077253347262740135,
      "step": 2328
    },
    {
      "epoch": 0.612602541550905,
      "grad_norm": 510.0,
      "learning_rate": 2.1520467836257309e-07,
      "logits/chosen": -0.6436260342597961,
      "logits/rejected": -0.6528536081314087,
      "logps/chosen": -1070.17529296875,
      "logps/rejected": -934.2296142578125,
      "loss": 0.6968,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0190492644906044,
      "rewards/margins": -0.006432152818888426,
      "rewards/rejected": -0.012617111206054688,
      "step": 2329
    },
    {
      "epoch": 0.6128655739860922,
      "grad_norm": 492.0,
      "learning_rate": 2.1505847953216373e-07,
      "logits/chosen": -0.6729048490524292,
      "logits/rejected": -0.6725853085517883,
      "logps/chosen": -882.6712036132812,
      "logps/rejected": -794.8883056640625,
      "loss": 0.694,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.005923653021454811,
      "rewards/margins": -0.0010395043063908815,
      "rewards/rejected": 0.006963157095015049,
      "step": 2330
    },
    {
      "epoch": 0.6131286064212793,
      "grad_norm": 382.0,
      "learning_rate": 2.1491228070175438e-07,
      "logits/chosen": -0.6404862403869629,
      "logits/rejected": -0.6354591250419617,
      "logps/chosen": -717.9664916992188,
      "logps/rejected": -622.26025390625,
      "loss": 0.6894,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.008510876446962357,
      "rewards/margins": 0.008086586371064186,
      "rewards/rejected": -0.016597462818026543,
      "step": 2331
    },
    {
      "epoch": 0.6133916388564665,
      "grad_norm": 470.0,
      "learning_rate": 2.1476608187134502e-07,
      "logits/chosen": -0.6640379428863525,
      "logits/rejected": -0.669413685798645,
      "logps/chosen": -1010.7733764648438,
      "logps/rejected": -875.77294921875,
      "loss": 0.6913,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.003916931804269552,
      "rewards/margins": 0.004986668936908245,
      "rewards/rejected": -0.0010697375982999802,
      "step": 2332
    },
    {
      "epoch": 0.6136546712916536,
      "grad_norm": 664.0,
      "learning_rate": 2.1461988304093567e-07,
      "logits/chosen": -0.6482572555541992,
      "logits/rejected": -0.6667571067810059,
      "logps/chosen": -1825.7371826171875,
      "logps/rejected": -1191.2115478515625,
      "loss": 0.7024,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.007745074573904276,
      "rewards/margins": -0.017541218549013138,
      "rewards/rejected": 0.009796143509447575,
      "step": 2333
    },
    {
      "epoch": 0.6139177037268408,
      "grad_norm": 572.0,
      "learning_rate": 2.1447368421052632e-07,
      "logits/chosen": -0.6373530626296997,
      "logits/rejected": -0.634766697883606,
      "logps/chosen": -1150.0361328125,
      "logps/rejected": -934.0209350585938,
      "loss": 0.6852,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.023993590846657753,
      "rewards/margins": 0.01679697073996067,
      "rewards/rejected": 0.007196618244051933,
      "step": 2334
    },
    {
      "epoch": 0.614180736162028,
      "grad_norm": 568.0,
      "learning_rate": 2.1432748538011694e-07,
      "logits/chosen": -0.6320626139640808,
      "logits/rejected": -0.632708728313446,
      "logps/chosen": -1129.56982421875,
      "logps/rejected": -995.5462646484375,
      "loss": 0.6878,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005767154507339001,
      "rewards/margins": 0.011768913827836514,
      "rewards/rejected": -0.006001759320497513,
      "step": 2335
    },
    {
      "epoch": 0.6144437685972152,
      "grad_norm": 520.0,
      "learning_rate": 2.141812865497076e-07,
      "logits/chosen": -0.6475306749343872,
      "logits/rejected": -0.6465603709220886,
      "logps/chosen": -1009.43603515625,
      "logps/rejected": -743.5230712890625,
      "loss": 0.6987,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003165912814438343,
      "rewards/margins": -0.010155963711440563,
      "rewards/rejected": 0.01332187745720148,
      "step": 2336
    },
    {
      "epoch": 0.6147068010324023,
      "grad_norm": 572.0,
      "learning_rate": 2.1403508771929823e-07,
      "logits/chosen": -0.6491643786430359,
      "logits/rejected": -0.6639767289161682,
      "logps/chosen": -1214.852294921875,
      "logps/rejected": -826.4765014648438,
      "loss": 0.7105,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.016346074640750885,
      "rewards/margins": -0.0328831672668457,
      "rewards/rejected": 0.016537094488739967,
      "step": 2337
    },
    {
      "epoch": 0.6149698334675895,
      "grad_norm": 372.0,
      "learning_rate": 2.1388888888888888e-07,
      "logits/chosen": -0.6493688821792603,
      "logits/rejected": -0.65561842918396,
      "logps/chosen": -1003.0426025390625,
      "logps/rejected": -858.5020141601562,
      "loss": 0.6952,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.01526937447488308,
      "rewards/margins": -0.003807544708251953,
      "rewards/rejected": -0.011461829766631126,
      "step": 2338
    },
    {
      "epoch": 0.6152328659027766,
      "grad_norm": 560.0,
      "learning_rate": 2.1374269005847952e-07,
      "logits/chosen": -0.6681265830993652,
      "logits/rejected": NaN,
      "logps/chosen": -1490.297607421875,
      "logps/rejected": -1018.70361328125,
      "loss": 0.6972,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.009825517423450947,
      "rewards/margins": -0.0075272563844919205,
      "rewards/rejected": -0.0022982601076364517,
      "step": 2339
    },
    {
      "epoch": 0.6154958983379638,
      "grad_norm": 592.0,
      "learning_rate": 2.1359649122807017e-07,
      "logits/chosen": -0.6764253973960876,
      "logits/rejected": -0.6637426614761353,
      "logps/chosen": -1483.6761474609375,
      "logps/rejected": -952.2650146484375,
      "loss": 0.6922,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.015911292284727097,
      "rewards/margins": 0.0034971237182617188,
      "rewards/rejected": 0.012414168566465378,
      "step": 2340
    },
    {
      "epoch": 0.6157589307731509,
      "grad_norm": 556.0,
      "learning_rate": 2.134502923976608e-07,
      "logits/chosen": -0.6600728034973145,
      "logits/rejected": -0.6568987369537354,
      "logps/chosen": -1149.8515625,
      "logps/rejected": -935.4176025390625,
      "loss": 0.6895,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02864832989871502,
      "rewards/margins": 0.008057165890932083,
      "rewards/rejected": 0.020591165870428085,
      "step": 2341
    },
    {
      "epoch": 0.6160219632083381,
      "grad_norm": 616.0,
      "learning_rate": 2.1330409356725146e-07,
      "logits/chosen": -0.6365247368812561,
      "logits/rejected": -0.6351812481880188,
      "logps/chosen": -1644.9345703125,
      "logps/rejected": -1176.3514404296875,
      "loss": 0.6826,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.033541489392519,
      "rewards/margins": 0.021562958136200905,
      "rewards/rejected": 0.011978529393672943,
      "step": 2342
    },
    {
      "epoch": 0.6162849956435253,
      "grad_norm": 556.0,
      "learning_rate": 2.1315789473684208e-07,
      "logits/chosen": -0.6807726621627808,
      "logits/rejected": -0.6686906814575195,
      "logps/chosen": -1024.6441650390625,
      "logps/rejected": -597.1281127929688,
      "loss": 0.6904,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.011770820245146751,
      "rewards/margins": 0.006196069996803999,
      "rewards/rejected": -0.017966892570257187,
      "step": 2343
    },
    {
      "epoch": 0.6165480280787125,
      "grad_norm": 472.0,
      "learning_rate": 2.1301169590643273e-07,
      "logits/chosen": -0.6405764818191528,
      "logits/rejected": -0.645415186882019,
      "logps/chosen": -1168.3887939453125,
      "logps/rejected": -934.404296875,
      "loss": 0.685,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008822060190141201,
      "rewards/margins": 0.017676357179880142,
      "rewards/rejected": -0.008854292333126068,
      "step": 2344
    },
    {
      "epoch": 0.6168110605138997,
      "grad_norm": 556.0,
      "learning_rate": 2.1286549707602337e-07,
      "logits/chosen": -0.6214299201965332,
      "logits/rejected": -0.6279311776161194,
      "logps/chosen": -1154.619140625,
      "logps/rejected": -900.0028686523438,
      "loss": 0.6913,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0053920745849609375,
      "rewards/margins": 0.004565049894154072,
      "rewards/rejected": 0.0008270259713754058,
      "step": 2345
    },
    {
      "epoch": 0.6170740929490868,
      "grad_norm": 584.0,
      "learning_rate": 2.1271929824561402e-07,
      "logits/chosen": -0.6264079809188843,
      "logits/rejected": -0.6375543475151062,
      "logps/chosen": -1475.3507080078125,
      "logps/rejected": -1144.2587890625,
      "loss": 0.7,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.018186569213867188,
      "rewards/margins": -0.012476444244384766,
      "rewards/rejected": -0.005710124969482422,
      "step": 2346
    },
    {
      "epoch": 0.617337125384274,
      "grad_norm": 464.0,
      "learning_rate": 2.125730994152047e-07,
      "logits/chosen": -0.6359666585922241,
      "logits/rejected": -0.646614134311676,
      "logps/chosen": -1014.7291870117188,
      "logps/rejected": -1086.949462890625,
      "loss": 0.6959,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.003343201009556651,
      "rewards/margins": -0.0050203315913677216,
      "rewards/rejected": 0.001677132211625576,
      "step": 2347
    },
    {
      "epoch": 0.6176001578194611,
      "grad_norm": 612.0,
      "learning_rate": 2.1242690058479531e-07,
      "logits/chosen": -0.6476885676383972,
      "logits/rejected": -0.646675169467926,
      "logps/chosen": -1151.2603759765625,
      "logps/rejected": -1148.6766357421875,
      "loss": 0.6952,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.006274891085922718,
      "rewards/margins": -0.003419114276766777,
      "rewards/rejected": 0.00969400443136692,
      "step": 2348
    },
    {
      "epoch": 0.6178631902546483,
      "grad_norm": 668.0,
      "learning_rate": 2.1228070175438596e-07,
      "logits/chosen": -0.6184571385383606,
      "logits/rejected": -0.6200077533721924,
      "logps/chosen": -1076.1412353515625,
      "logps/rejected": -953.6043701171875,
      "loss": 0.6936,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0038059235084801912,
      "rewards/margins": -0.000271512457402423,
      "rewards/rejected": 0.0040774354711174965,
      "step": 2349
    },
    {
      "epoch": 0.6181262226898354,
      "grad_norm": 532.0,
      "learning_rate": 2.121345029239766e-07,
      "logits/chosen": -0.652042031288147,
      "logits/rejected": -0.6593759655952454,
      "logps/chosen": -1013.4854125976562,
      "logps/rejected": -663.0535888671875,
      "loss": 0.7033,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.004638482350856066,
      "rewards/margins": -0.0194960106164217,
      "rewards/rejected": 0.014857531525194645,
      "step": 2350
    },
    {
      "epoch": 0.6183892551250226,
      "grad_norm": 600.0,
      "learning_rate": 2.1198830409356725e-07,
      "logits/chosen": -0.6313027739524841,
      "logits/rejected": -0.6349718570709229,
      "logps/chosen": -1217.060791015625,
      "logps/rejected": -1331.6031494140625,
      "loss": 0.6842,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0017057422082871199,
      "rewards/margins": 0.019707871600985527,
      "rewards/rejected": -0.018002130091190338,
      "step": 2351
    },
    {
      "epoch": 0.6186522875602097,
      "grad_norm": 512.0,
      "learning_rate": 2.1184210526315787e-07,
      "logits/chosen": -0.6485707759857178,
      "logits/rejected": -0.6517980694770813,
      "logps/chosen": -938.2669677734375,
      "logps/rejected": -915.7496337890625,
      "loss": 0.6886,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.009192181751132011,
      "rewards/margins": 0.009698962792754173,
      "rewards/rejected": -0.0005067822057753801,
      "step": 2352
    },
    {
      "epoch": 0.618915319995397,
      "grad_norm": 632.0,
      "learning_rate": 2.1169590643274855e-07,
      "logits/chosen": -0.63877934217453,
      "logits/rejected": -0.6346046328544617,
      "logps/chosen": -1188.6080322265625,
      "logps/rejected": -802.48876953125,
      "loss": 0.6918,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00950393732637167,
      "rewards/margins": 0.003740978427231312,
      "rewards/rejected": 0.005762957967817783,
      "step": 2353
    },
    {
      "epoch": 0.6191783524305841,
      "grad_norm": 624.0,
      "learning_rate": 2.1154970760233917e-07,
      "logits/chosen": -0.6344829201698303,
      "logits/rejected": -0.6473495364189148,
      "logps/chosen": -1288.125,
      "logps/rejected": -909.32763671875,
      "loss": 0.6982,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.026066016405820847,
      "rewards/margins": -0.008427048102021217,
      "rewards/rejected": -0.01763896830379963,
      "step": 2354
    },
    {
      "epoch": 0.6194413848657713,
      "grad_norm": 504.0,
      "learning_rate": 2.114035087719298e-07,
      "logits/chosen": -0.652863085269928,
      "logits/rejected": -0.6577721238136292,
      "logps/chosen": -952.5972900390625,
      "logps/rejected": -872.7371215820312,
      "loss": 0.6909,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.004162121098488569,
      "rewards/margins": 0.005228138528764248,
      "rewards/rejected": -0.00939025916159153,
      "step": 2355
    },
    {
      "epoch": 0.6197044173009584,
      "grad_norm": 448.0,
      "learning_rate": 2.1125730994152046e-07,
      "logits/chosen": -0.6556655168533325,
      "logits/rejected": -0.6661707162857056,
      "logps/chosen": -978.0469970703125,
      "logps/rejected": -711.111328125,
      "loss": 0.6939,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0023465158883482218,
      "rewards/margins": -0.0009755135979503393,
      "rewards/rejected": -0.0013710022903978825,
      "step": 2356
    },
    {
      "epoch": 0.6199674497361456,
      "grad_norm": 656.0,
      "learning_rate": 2.111111111111111e-07,
      "logits/chosen": -0.6440525054931641,
      "logits/rejected": -0.6514356732368469,
      "logps/chosen": -1097.7763671875,
      "logps/rejected": -748.5435791015625,
      "loss": 0.7048,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.016301153227686882,
      "rewards/margins": -0.022928528487682343,
      "rewards/rejected": 0.0066273692063987255,
      "step": 2357
    },
    {
      "epoch": 0.6202304821713327,
      "grad_norm": 588.0,
      "learning_rate": 2.1096491228070172e-07,
      "logits/chosen": -0.6261002421379089,
      "logits/rejected": -0.6349457502365112,
      "logps/chosen": -1009.726806640625,
      "logps/rejected": -930.0875244140625,
      "loss": 0.696,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.011858177371323109,
      "rewards/margins": -0.004673768300563097,
      "rewards/rejected": 0.016531944274902344,
      "step": 2358
    },
    {
      "epoch": 0.6204935146065199,
      "grad_norm": 620.0,
      "learning_rate": 2.108187134502924e-07,
      "logits/chosen": -0.6268709897994995,
      "logits/rejected": -0.6200591921806335,
      "logps/chosen": -1118.2332763671875,
      "logps/rejected": -1145.9517822265625,
      "loss": 0.6906,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.01395964715629816,
      "rewards/margins": 0.006166028790175915,
      "rewards/rejected": 0.0077936165034770966,
      "step": 2359
    },
    {
      "epoch": 0.6207565470417071,
      "grad_norm": 512.0,
      "learning_rate": 2.1067251461988302e-07,
      "logits/chosen": -0.6657134294509888,
      "logits/rejected": -0.6591719388961792,
      "logps/chosen": -1289.48388671875,
      "logps/rejected": -893.7942504882812,
      "loss": 0.6877,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01222915668040514,
      "rewards/margins": 0.012385036796331406,
      "rewards/rejected": -0.0001558787189424038,
      "step": 2360
    },
    {
      "epoch": 0.6210195794768942,
      "grad_norm": 5248.0,
      "learning_rate": 2.1052631578947366e-07,
      "logits/chosen": -0.6706064939498901,
      "logits/rejected": -0.6749849319458008,
      "logps/chosen": -1260.4327392578125,
      "logps/rejected": -866.42822265625,
      "loss": 0.6727,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02599954605102539,
      "rewards/margins": 0.042275432497262955,
      "rewards/rejected": -0.016275884583592415,
      "step": 2361
    },
    {
      "epoch": 0.6212826119120815,
      "grad_norm": 588.0,
      "learning_rate": 2.1038011695906434e-07,
      "logits/chosen": -0.6304076910018921,
      "logits/rejected": -0.6467604637145996,
      "logps/chosen": -1568.879638671875,
      "logps/rejected": -1214.2520751953125,
      "loss": 0.6963,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0039732931181788445,
      "rewards/margins": -0.0051901815459132195,
      "rewards/rejected": 0.009163476526737213,
      "step": 2362
    },
    {
      "epoch": 0.6215456443472686,
      "grad_norm": 592.0,
      "learning_rate": 2.1023391812865496e-07,
      "logits/chosen": -0.6399202346801758,
      "logits/rejected": -0.6296294927597046,
      "logps/chosen": -1075.2132568359375,
      "logps/rejected": -843.073486328125,
      "loss": 0.6923,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.009711552411317825,
      "rewards/margins": 0.00205574044957757,
      "rewards/rejected": -0.011767291463911533,
      "step": 2363
    },
    {
      "epoch": 0.6218086767824558,
      "grad_norm": 576.0,
      "learning_rate": 2.100877192982456e-07,
      "logits/chosen": -0.6672226190567017,
      "logits/rejected": -0.6633692979812622,
      "logps/chosen": -1223.3740234375,
      "logps/rejected": -847.1124267578125,
      "loss": 0.6812,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.019896887242794037,
      "rewards/margins": 0.025764942169189453,
      "rewards/rejected": -0.005868052132427692,
      "step": 2364
    },
    {
      "epoch": 0.6220717092176429,
      "grad_norm": 532.0,
      "learning_rate": 2.0994152046783625e-07,
      "logits/chosen": -0.6500728130340576,
      "logits/rejected": -0.6612182259559631,
      "logps/chosen": -1041.1402587890625,
      "logps/rejected": -811.83056640625,
      "loss": 0.6831,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.017408084124326706,
      "rewards/margins": 0.020752431824803352,
      "rewards/rejected": -0.0033443449065089226,
      "step": 2365
    },
    {
      "epoch": 0.6223347416528301,
      "grad_norm": 724.0,
      "learning_rate": 2.097953216374269e-07,
      "logits/chosen": -0.6469152569770813,
      "logits/rejected": -0.6494884490966797,
      "logps/chosen": -1387.0296630859375,
      "logps/rejected": -1107.83154296875,
      "loss": 0.7048,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.013237381353974342,
      "rewards/margins": -0.02166900783777237,
      "rewards/rejected": 0.008431625552475452,
      "step": 2366
    },
    {
      "epoch": 0.6225977740880172,
      "grad_norm": 524.0,
      "learning_rate": 2.0964912280701754e-07,
      "logits/chosen": -0.6616843938827515,
      "logits/rejected": -0.6717959046363831,
      "logps/chosen": -1056.33349609375,
      "logps/rejected": -891.958984375,
      "loss": 0.679,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.019596481695771217,
      "rewards/margins": 0.029553890228271484,
      "rewards/rejected": -0.009957408532500267,
      "step": 2367
    },
    {
      "epoch": 0.6228608065232044,
      "grad_norm": 540.0,
      "learning_rate": 2.095029239766082e-07,
      "logits/chosen": -0.6563310623168945,
      "logits/rejected": -0.6630014181137085,
      "logps/chosen": -1191.778076171875,
      "logps/rejected": -1068.7645263671875,
      "loss": 0.6941,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.002344513311982155,
      "rewards/margins": -0.0009591099806129932,
      "rewards/rejected": 0.0033036228269338608,
      "step": 2368
    },
    {
      "epoch": 0.6231238389583915,
      "grad_norm": 470.0,
      "learning_rate": 2.093567251461988e-07,
      "logits/chosen": -0.6522793173789978,
      "logits/rejected": -0.6526330709457397,
      "logps/chosen": -888.5318603515625,
      "logps/rejected": -875.2164306640625,
      "loss": 0.6952,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.006776905152946711,
      "rewards/margins": -0.003891564207151532,
      "rewards/rejected": -0.002885342575609684,
      "step": 2369
    },
    {
      "epoch": 0.6233868713935787,
      "grad_norm": 568.0,
      "learning_rate": 2.0921052631578948e-07,
      "logits/chosen": -0.676264226436615,
      "logits/rejected": -0.6829925775527954,
      "logps/chosen": -1272.3641357421875,
      "logps/rejected": -1482.7476806640625,
      "loss": 0.6965,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0032851221039891243,
      "rewards/margins": -0.006249141413718462,
      "rewards/rejected": 0.0029640195425599813,
      "step": 2370
    },
    {
      "epoch": 0.6236499038287658,
      "grad_norm": 580.0,
      "learning_rate": 2.090643274853801e-07,
      "logits/chosen": -0.6594712734222412,
      "logits/rejected": -0.6514507532119751,
      "logps/chosen": -1519.07861328125,
      "logps/rejected": -1017.5139770507812,
      "loss": 0.6849,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03219032287597656,
      "rewards/margins": 0.017442893236875534,
      "rewards/rejected": 0.01474742777645588,
      "step": 2371
    },
    {
      "epoch": 0.623912936263953,
      "grad_norm": 576.0,
      "learning_rate": 2.0891812865497075e-07,
      "logits/chosen": -0.6307961344718933,
      "logits/rejected": -0.6306915879249573,
      "logps/chosen": -1439.955810546875,
      "logps/rejected": -1117.3623046875,
      "loss": 0.7017,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.018124770373106003,
      "rewards/margins": -0.016139699146151543,
      "rewards/rejected": -0.001985073322430253,
      "step": 2372
    },
    {
      "epoch": 0.6241759686991402,
      "grad_norm": 512.0,
      "learning_rate": 2.087719298245614e-07,
      "logits/chosen": -0.6547097563743591,
      "logits/rejected": -0.6391438245773315,
      "logps/chosen": -1043.0521240234375,
      "logps/rejected": -843.9295043945312,
      "loss": 0.6927,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.009566688910126686,
      "rewards/margins": 0.0014387129340320826,
      "rewards/rejected": -0.01100540068000555,
      "step": 2373
    },
    {
      "epoch": 0.6244390011343274,
      "grad_norm": 576.0,
      "learning_rate": 2.0862573099415204e-07,
      "logits/chosen": -0.6679317951202393,
      "logits/rejected": -0.6655787229537964,
      "logps/chosen": -1269.5235595703125,
      "logps/rejected": -1162.919677734375,
      "loss": 0.6892,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01584768295288086,
      "rewards/margins": 0.008513163775205612,
      "rewards/rejected": 0.007334519177675247,
      "step": 2374
    },
    {
      "epoch": 0.6247020335695146,
      "grad_norm": 544.0,
      "learning_rate": 2.0847953216374266e-07,
      "logits/chosen": -0.6711554527282715,
      "logits/rejected": -0.667935311794281,
      "logps/chosen": -1262.3046875,
      "logps/rejected": -1112.346435546875,
      "loss": 0.6861,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006704762112349272,
      "rewards/margins": 0.015559340827167034,
      "rewards/rejected": -0.008854580111801624,
      "step": 2375
    },
    {
      "epoch": 0.6249650660047017,
      "grad_norm": 484.0,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -0.6596821546554565,
      "logits/rejected": -0.6663756966590881,
      "logps/chosen": -911.5975952148438,
      "logps/rejected": -812.2467041015625,
      "loss": 0.6924,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003849125001579523,
      "rewards/margins": 0.0021389005705714226,
      "rewards/rejected": 0.0017102230340242386,
      "step": 2376
    },
    {
      "epoch": 0.6252280984398889,
      "grad_norm": 564.0,
      "learning_rate": 2.0818713450292398e-07,
      "logits/chosen": -0.659935712814331,
      "logits/rejected": -0.660727858543396,
      "logps/chosen": -1224.5972900390625,
      "logps/rejected": -1428.7110595703125,
      "loss": 0.6877,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.001714086625725031,
      "rewards/margins": 0.011529302224516869,
      "rewards/rejected": -0.0098152169957757,
      "step": 2377
    },
    {
      "epoch": 0.625491130875076,
      "grad_norm": 452.0,
      "learning_rate": 2.080409356725146e-07,
      "logits/chosen": -0.643600344657898,
      "logits/rejected": -0.6560097932815552,
      "logps/chosen": -853.6458740234375,
      "logps/rejected": -823.79833984375,
      "loss": 0.6939,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.018668364733457565,
      "rewards/margins": -0.0008907301817089319,
      "rewards/rejected": -0.017777632921934128,
      "step": 2378
    },
    {
      "epoch": 0.6257541633102632,
      "grad_norm": 628.0,
      "learning_rate": 2.0789473684210527e-07,
      "logits/chosen": -0.662464439868927,
      "logits/rejected": -0.6730395555496216,
      "logps/chosen": -1089.3448486328125,
      "logps/rejected": -1099.6802978515625,
      "loss": 0.7043,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0038166046142578125,
      "rewards/margins": -0.020432665944099426,
      "rewards/rejected": 0.02424926683306694,
      "step": 2379
    },
    {
      "epoch": 0.6260171957454503,
      "grad_norm": 512.0,
      "learning_rate": 2.077485380116959e-07,
      "logits/chosen": -0.643608808517456,
      "logits/rejected": -0.6572858095169067,
      "logps/chosen": -1298.388427734375,
      "logps/rejected": -761.4056396484375,
      "loss": 0.6931,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005151796154677868,
      "rewards/margins": 0.000624943058937788,
      "rewards/rejected": -0.005776741076260805,
      "step": 2380
    },
    {
      "epoch": 0.6262802281806376,
      "grad_norm": 708.0,
      "learning_rate": 2.0760233918128654e-07,
      "logits/chosen": -0.6663441061973572,
      "logits/rejected": -0.6488066911697388,
      "logps/chosen": -1325.459716796875,
      "logps/rejected": -967.9281005859375,
      "loss": 0.6973,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011963177472352982,
      "rewards/margins": -0.0075436607003211975,
      "rewards/rejected": -0.004419517703354359,
      "step": 2381
    },
    {
      "epoch": 0.6265432606158247,
      "grad_norm": 616.0,
      "learning_rate": 2.0745614035087718e-07,
      "logits/chosen": -0.6438345909118652,
      "logits/rejected": -0.6412562727928162,
      "logps/chosen": -1110.5767822265625,
      "logps/rejected": -884.1642456054688,
      "loss": 0.6914,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0007801530882716179,
      "rewards/margins": 0.004248237702995539,
      "rewards/rejected": -0.003468084614723921,
      "step": 2382
    },
    {
      "epoch": 0.6268062930510119,
      "grad_norm": 464.0,
      "learning_rate": 2.0730994152046783e-07,
      "logits/chosen": -0.6391203999519348,
      "logits/rejected": -0.6447045207023621,
      "logps/chosen": -1241.154296875,
      "logps/rejected": -1123.5693359375,
      "loss": 0.6972,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006677055731415749,
      "rewards/margins": -0.007893658243119717,
      "rewards/rejected": 0.01457071304321289,
      "step": 2383
    },
    {
      "epoch": 0.627069325486199,
      "grad_norm": 596.0,
      "learning_rate": 2.0716374269005845e-07,
      "logits/chosen": -0.6477143168449402,
      "logits/rejected": -0.6574144959449768,
      "logps/chosen": -1254.238037109375,
      "logps/rejected": -895.2053833007812,
      "loss": 0.6759,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0235077366232872,
      "rewards/margins": 0.035979364067316055,
      "rewards/rejected": -0.012471628375351429,
      "step": 2384
    },
    {
      "epoch": 0.6273323579213862,
      "grad_norm": 636.0,
      "learning_rate": 2.0701754385964912e-07,
      "logits/chosen": -0.6360023021697998,
      "logits/rejected": -0.6423729062080383,
      "logps/chosen": -1263.97314453125,
      "logps/rejected": -1090.658935546875,
      "loss": 0.6999,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.008084488101303577,
      "rewards/margins": -0.01301279105246067,
      "rewards/rejected": 0.021097278222441673,
      "step": 2385
    },
    {
      "epoch": 0.6275953903565733,
      "grad_norm": 536.0,
      "learning_rate": 2.0687134502923974e-07,
      "logits/chosen": -0.6315830945968628,
      "logits/rejected": -0.646452784538269,
      "logps/chosen": -1227.8621826171875,
      "logps/rejected": -1144.1697998046875,
      "loss": 0.6927,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01030874252319336,
      "rewards/margins": 0.0018993369303643703,
      "rewards/rejected": 0.008409406058490276,
      "step": 2386
    },
    {
      "epoch": 0.6278584227917605,
      "grad_norm": 544.0,
      "learning_rate": 2.0672514619883042e-07,
      "logits/chosen": -0.6616744995117188,
      "logits/rejected": -0.6814316511154175,
      "logps/chosen": -991.4951171875,
      "logps/rejected": -696.716796875,
      "loss": 0.6923,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.010066796094179153,
      "rewards/margins": 0.0023667337372899055,
      "rewards/rejected": -0.012433529831469059,
      "step": 2387
    },
    {
      "epoch": 0.6281214552269476,
      "grad_norm": 556.0,
      "learning_rate": 2.0657894736842104e-07,
      "logits/chosen": -0.6683040261268616,
      "logits/rejected": -0.6770749688148499,
      "logps/chosen": -1033.258544921875,
      "logps/rejected": -904.580322265625,
      "loss": 0.6956,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0057006822898983955,
      "rewards/margins": -0.003603077959269285,
      "rewards/rejected": 0.009303760714828968,
      "step": 2388
    },
    {
      "epoch": 0.6283844876621348,
      "grad_norm": 584.0,
      "learning_rate": 2.0643274853801168e-07,
      "logits/chosen": -0.6417069435119629,
      "logits/rejected": -0.6397731304168701,
      "logps/chosen": -1567.5496826171875,
      "logps/rejected": -1207.3060302734375,
      "loss": 0.6978,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.008086872287094593,
      "rewards/margins": -0.008432483300566673,
      "rewards/rejected": 0.00034561147913336754,
      "step": 2389
    },
    {
      "epoch": 0.628647520097322,
      "grad_norm": 508.0,
      "learning_rate": 2.0628654970760233e-07,
      "logits/chosen": -0.6441807746887207,
      "logits/rejected": -0.6503152251243591,
      "logps/chosen": -1109.243408203125,
      "logps/rejected": -746.7449951171875,
      "loss": 0.7004,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01030816975980997,
      "rewards/margins": -0.013409899547696114,
      "rewards/rejected": 0.0031017307192087173,
      "step": 2390
    },
    {
      "epoch": 0.6289105525325092,
      "grad_norm": 568.0,
      "learning_rate": 2.0614035087719298e-07,
      "logits/chosen": -0.6742460131645203,
      "logits/rejected": -0.6698209047317505,
      "logps/chosen": -1083.8699951171875,
      "logps/rejected": -956.3475341796875,
      "loss": 0.6895,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.007263755891472101,
      "rewards/margins": 0.008008099161088467,
      "rewards/rejected": -0.0007443432696163654,
      "step": 2391
    },
    {
      "epoch": 0.6291735849676964,
      "grad_norm": 516.0,
      "learning_rate": 2.0599415204678362e-07,
      "logits/chosen": -0.6606893539428711,
      "logits/rejected": -0.6721242070198059,
      "logps/chosen": -943.978271484375,
      "logps/rejected": -845.21533203125,
      "loss": 0.6959,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.009145975112915039,
      "rewards/margins": -0.00496096583083272,
      "rewards/rejected": -0.004185009282082319,
      "step": 2392
    },
    {
      "epoch": 0.6294366174028835,
      "grad_norm": 592.0,
      "learning_rate": 2.0584795321637427e-07,
      "logits/chosen": -0.643884539604187,
      "logits/rejected": -0.6384831070899963,
      "logps/chosen": -975.8154296875,
      "logps/rejected": -790.8865966796875,
      "loss": 0.7053,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.015782738104462624,
      "rewards/margins": -0.022756576538085938,
      "rewards/rejected": 0.006973839830607176,
      "step": 2393
    },
    {
      "epoch": 0.6296996498380707,
      "grad_norm": 452.0,
      "learning_rate": 2.0570175438596491e-07,
      "logits/chosen": -0.6293568015098572,
      "logits/rejected": -0.6295035481452942,
      "logps/chosen": -1006.8536987304688,
      "logps/rejected": -863.0630493164062,
      "loss": 0.6988,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01636657677590847,
      "rewards/margins": -0.010386848822236061,
      "rewards/rejected": 0.02675342559814453,
      "step": 2394
    },
    {
      "epoch": 0.6299626822732578,
      "grad_norm": 576.0,
      "learning_rate": 2.0555555555555553e-07,
      "logits/chosen": -0.6619591116905212,
      "logits/rejected": -0.6644609570503235,
      "logps/chosen": -1043.48876953125,
      "logps/rejected": -882.84814453125,
      "loss": 0.6988,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011042309924960136,
      "rewards/margins": -0.009697631001472473,
      "rewards/rejected": -0.0013446798548102379,
      "step": 2395
    },
    {
      "epoch": 0.630225714708445,
      "grad_norm": 414.0,
      "learning_rate": 2.054093567251462e-07,
      "logits/chosen": -0.6575624942779541,
      "logits/rejected": -0.6733388900756836,
      "logps/chosen": -858.96337890625,
      "logps/rejected": -467.7987976074219,
      "loss": 0.6858,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00663909874856472,
      "rewards/margins": 0.015363980084657669,
      "rewards/rejected": -0.008724880404770374,
      "step": 2396
    },
    {
      "epoch": 0.6304887471436321,
      "grad_norm": 516.0,
      "learning_rate": 2.0526315789473683e-07,
      "logits/chosen": -0.6542561650276184,
      "logits/rejected": -0.6542028188705444,
      "logps/chosen": -1196.2388916015625,
      "logps/rejected": -952.6029052734375,
      "loss": 0.6865,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007368756458163261,
      "rewards/margins": 0.013862992636859417,
      "rewards/rejected": -0.006494234781712294,
      "step": 2397
    },
    {
      "epoch": 0.6307517795788193,
      "grad_norm": 584.0,
      "learning_rate": 2.0511695906432747e-07,
      "logits/chosen": -0.6697317957878113,
      "logits/rejected": -0.6667272448539734,
      "logps/chosen": -1220.067626953125,
      "logps/rejected": -936.8177490234375,
      "loss": 0.7061,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.017821691930294037,
      "rewards/margins": -0.023356245830655098,
      "rewards/rejected": 0.005534553434699774,
      "step": 2398
    },
    {
      "epoch": 0.6310148120140064,
      "grad_norm": 536.0,
      "learning_rate": 2.0497076023391812e-07,
      "logits/chosen": -0.6596409678459167,
      "logits/rejected": -0.6635767817497253,
      "logps/chosen": -1475.18701171875,
      "logps/rejected": -1222.7069091796875,
      "loss": 0.6849,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010396766476333141,
      "rewards/margins": 0.018125439062714577,
      "rewards/rejected": -0.007728673052042723,
      "step": 2399
    },
    {
      "epoch": 0.6312778444491937,
      "grad_norm": 668.0,
      "learning_rate": 2.0482456140350877e-07,
      "logits/chosen": -0.6490381360054016,
      "logits/rejected": -0.6556553840637207,
      "logps/chosen": -1370.88916015625,
      "logps/rejected": -1130.5360107421875,
      "loss": 0.7022,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": 0.008494377136230469,
      "rewards/margins": -0.017109394073486328,
      "rewards/rejected": 0.025603771209716797,
      "step": 2400
    },
    {
      "epoch": 0.6315408768843808,
      "grad_norm": 492.0,
      "learning_rate": 2.0467836257309939e-07,
      "logits/chosen": -0.6398760080337524,
      "logits/rejected": -0.6535860300064087,
      "logps/chosen": -890.23193359375,
      "logps/rejected": -740.2337036132812,
      "loss": 0.7024,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.004837035667151213,
      "rewards/margins": -0.017836952582001686,
      "rewards/rejected": 0.022673986852169037,
      "step": 2401
    },
    {
      "epoch": 0.631803909319568,
      "grad_norm": 536.0,
      "learning_rate": 2.0453216374269006e-07,
      "logits/chosen": -0.6353153586387634,
      "logits/rejected": -0.6426400542259216,
      "logps/chosen": -1208.6258544921875,
      "logps/rejected": -1114.062744140625,
      "loss": 0.6929,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0159638412296772,
      "rewards/margins": 0.0009254459291696548,
      "rewards/rejected": 0.015038395300507545,
      "step": 2402
    },
    {
      "epoch": 0.6320669417547551,
      "grad_norm": 528.0,
      "learning_rate": 2.0438596491228068e-07,
      "logits/chosen": -0.6414791941642761,
      "logits/rejected": -0.6495314240455627,
      "logps/chosen": -1040.2733154296875,
      "logps/rejected": -792.6778564453125,
      "loss": 0.6862,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01191720925271511,
      "rewards/margins": 0.014884233474731445,
      "rewards/rejected": -0.0029670235235244036,
      "step": 2403
    },
    {
      "epoch": 0.6323299741899423,
      "grad_norm": 736.0,
      "learning_rate": 2.0423976608187133e-07,
      "logits/chosen": -0.6777032017707825,
      "logits/rejected": -0.6731885671615601,
      "logps/chosen": -1590.9169921875,
      "logps/rejected": -902.9234619140625,
      "loss": 0.69,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.011856651864945889,
      "rewards/margins": 0.007399463094770908,
      "rewards/rejected": 0.004457188304513693,
      "step": 2404
    },
    {
      "epoch": 0.6325930066251294,
      "grad_norm": 460.0,
      "learning_rate": 2.0409356725146197e-07,
      "logits/chosen": -0.641350507736206,
      "logits/rejected": -0.6345451474189758,
      "logps/chosen": -946.2098388671875,
      "logps/rejected": -798.2261962890625,
      "loss": 0.6954,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005511092953383923,
      "rewards/margins": -0.004010391421616077,
      "rewards/rejected": 0.009521485306322575,
      "step": 2405
    },
    {
      "epoch": 0.6328560390603166,
      "grad_norm": 464.0,
      "learning_rate": 2.0394736842105262e-07,
      "logits/chosen": -0.6473276019096375,
      "logits/rejected": -0.6533592939376831,
      "logps/chosen": -1256.939697265625,
      "logps/rejected": -1022.1309814453125,
      "loss": 0.7027,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.029343225061893463,
      "rewards/margins": -0.01849842071533203,
      "rewards/rejected": -0.010844802483916283,
      "step": 2406
    },
    {
      "epoch": 0.6331190714955038,
      "grad_norm": 596.0,
      "learning_rate": 2.038011695906433e-07,
      "logits/chosen": -0.6512542963027954,
      "logits/rejected": -0.6402112245559692,
      "logps/chosen": -1092.057861328125,
      "logps/rejected": -1024.534912109375,
      "loss": 0.6984,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.018216991797089577,
      "rewards/margins": -0.009299278259277344,
      "rewards/rejected": -0.008917711675167084,
      "step": 2407
    },
    {
      "epoch": 0.6333821039306909,
      "grad_norm": 600.0,
      "learning_rate": 2.036549707602339e-07,
      "logits/chosen": -0.6395214796066284,
      "logits/rejected": -0.6257408857345581,
      "logps/chosen": -1139.0400390625,
      "logps/rejected": -858.912353515625,
      "loss": 0.6894,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.024512864649295807,
      "rewards/margins": 0.008761979639530182,
      "rewards/rejected": 0.015750885009765625,
      "step": 2408
    },
    {
      "epoch": 0.6336451363658782,
      "grad_norm": 564.0,
      "learning_rate": 2.0350877192982456e-07,
      "logits/chosen": -0.6462354063987732,
      "logits/rejected": -0.6481842994689941,
      "logps/chosen": -1544.237548828125,
      "logps/rejected": -1079.0552978515625,
      "loss": 0.6981,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.021213531494140625,
      "rewards/margins": -0.008437537588179111,
      "rewards/rejected": 0.02965107001364231,
      "step": 2409
    },
    {
      "epoch": 0.6339081688010653,
      "grad_norm": 498.0,
      "learning_rate": 2.033625730994152e-07,
      "logits/chosen": -0.6109200716018677,
      "logits/rejected": -0.6146712303161621,
      "logps/chosen": -942.8727416992188,
      "logps/rejected": -1054.6361083984375,
      "loss": 0.6946,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0030086522456258535,
      "rewards/margins": -0.0020043374970555305,
      "rewards/rejected": -0.001004314748570323,
      "step": 2410
    },
    {
      "epoch": 0.6341712012362525,
      "grad_norm": 572.0,
      "learning_rate": 2.0321637426900585e-07,
      "logits/chosen": -0.6782251596450806,
      "logits/rejected": -0.6715468764305115,
      "logps/chosen": -1759.7696533203125,
      "logps/rejected": -1448.5537109375,
      "loss": 0.6725,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.033084869384765625,
      "rewards/margins": 0.04301939159631729,
      "rewards/rejected": -0.009934519417583942,
      "step": 2411
    },
    {
      "epoch": 0.6344342336714396,
      "grad_norm": 532.0,
      "learning_rate": 2.0307017543859647e-07,
      "logits/chosen": -0.6615597009658813,
      "logits/rejected": -0.667773425579071,
      "logps/chosen": -1248.3369140625,
      "logps/rejected": -1031.7864990234375,
      "loss": 0.7029,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.037072278559207916,
      "rewards/margins": -0.0189330093562603,
      "rewards/rejected": -0.018139267340302467,
      "step": 2412
    },
    {
      "epoch": 0.6346972661066268,
      "grad_norm": 556.0,
      "learning_rate": 2.0292397660818714e-07,
      "logits/chosen": -0.6210433840751648,
      "logits/rejected": -0.6153314709663391,
      "logps/chosen": -1015.1267700195312,
      "logps/rejected": -937.164794921875,
      "loss": 0.6975,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010689258575439453,
      "rewards/margins": -0.008409214206039906,
      "rewards/rejected": 0.019098471850156784,
      "step": 2413
    },
    {
      "epoch": 0.6349602985418139,
      "grad_norm": 506.0,
      "learning_rate": 2.0277777777777776e-07,
      "logits/chosen": -0.6488004922866821,
      "logits/rejected": -0.6585991978645325,
      "logps/chosen": -980.166015625,
      "logps/rejected": -648.568115234375,
      "loss": 0.6958,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005159949883818626,
      "rewards/margins": -0.004330349154770374,
      "rewards/rejected": -0.0008296011947095394,
      "step": 2414
    },
    {
      "epoch": 0.6352233309770011,
      "grad_norm": 422.0,
      "learning_rate": 2.026315789473684e-07,
      "logits/chosen": -0.6429774761199951,
      "logits/rejected": -0.6573836207389832,
      "logps/chosen": -732.9256591796875,
      "logps/rejected": -649.88330078125,
      "loss": 0.6866,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00016984972171485424,
      "rewards/margins": 0.013700629584491253,
      "rewards/rejected": -0.013870478607714176,
      "step": 2415
    },
    {
      "epoch": 0.6354863634121882,
      "grad_norm": 528.0,
      "learning_rate": 2.0248538011695906e-07,
      "logits/chosen": -0.6176669597625732,
      "logits/rejected": -0.6192634105682373,
      "logps/chosen": -1055.386474609375,
      "logps/rejected": -748.5631103515625,
      "loss": 0.681,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.007181548047810793,
      "rewards/margins": 0.024956321343779564,
      "rewards/rejected": -0.01777477376163006,
      "step": 2416
    },
    {
      "epoch": 0.6357493958473754,
      "grad_norm": 556.0,
      "learning_rate": 2.023391812865497e-07,
      "logits/chosen": -0.6363253593444824,
      "logits/rejected": -0.6378846764564514,
      "logps/chosen": -965.7362670898438,
      "logps/rejected": -748.4033813476562,
      "loss": 0.6854,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01894369162619114,
      "rewards/margins": 0.01623215526342392,
      "rewards/rejected": 0.0027115349657833576,
      "step": 2417
    },
    {
      "epoch": 0.6360124282825625,
      "grad_norm": 564.0,
      "learning_rate": 2.0219298245614032e-07,
      "logits/chosen": -0.645711362361908,
      "logits/rejected": -0.6661881804466248,
      "logps/chosen": -1458.7010498046875,
      "logps/rejected": -1052.042236328125,
      "loss": 0.6927,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.013014602474868298,
      "rewards/margins": 0.0018429760821163654,
      "rewards/rejected": -0.01485757902264595,
      "step": 2418
    },
    {
      "epoch": 0.6362754607177498,
      "grad_norm": 616.0,
      "learning_rate": 2.02046783625731e-07,
      "logits/chosen": -0.6551544070243835,
      "logits/rejected": -0.6562240123748779,
      "logps/chosen": -1420.96044921875,
      "logps/rejected": -1479.785400390625,
      "loss": 0.6811,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02609262615442276,
      "rewards/margins": 0.025508785620331764,
      "rewards/rejected": 0.0005838391371071339,
      "step": 2419
    },
    {
      "epoch": 0.6365384931529369,
      "grad_norm": 552.0,
      "learning_rate": 2.0190058479532161e-07,
      "logits/chosen": -0.6544402241706848,
      "logits/rejected": -0.6517308950424194,
      "logps/chosen": -1070.4854736328125,
      "logps/rejected": -720.427490234375,
      "loss": 0.6847,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.019461344927549362,
      "rewards/margins": 0.01862645335495472,
      "rewards/rejected": 0.0008348925039172173,
      "step": 2420
    },
    {
      "epoch": 0.6368015255881241,
      "grad_norm": 576.0,
      "learning_rate": 2.0175438596491226e-07,
      "logits/chosen": -0.6410530209541321,
      "logits/rejected": -0.6457737684249878,
      "logps/chosen": -1379.595458984375,
      "logps/rejected": -1313.146484375,
      "loss": 0.694,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.02710742875933647,
      "rewards/margins": -0.00046977971214801073,
      "rewards/rejected": 0.02757721021771431,
      "step": 2421
    },
    {
      "epoch": 0.6370645580233113,
      "grad_norm": 528.0,
      "learning_rate": 2.0160818713450293e-07,
      "logits/chosen": -0.6777678728103638,
      "logits/rejected": -0.6762036681175232,
      "logps/chosen": -1259.9774169921875,
      "logps/rejected": -905.8472900390625,
      "loss": 0.7053,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.00347747839987278,
      "rewards/margins": -0.023108482360839844,
      "rewards/rejected": 0.019631003960967064,
      "step": 2422
    },
    {
      "epoch": 0.6373275904584984,
      "grad_norm": 612.0,
      "learning_rate": 2.0146198830409355e-07,
      "logits/chosen": -0.6372083425521851,
      "logits/rejected": -0.6430807709693909,
      "logps/chosen": -872.423095703125,
      "logps/rejected": -965.8032836914062,
      "loss": 0.6846,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0022353166714310646,
      "rewards/margins": 0.017610928043723106,
      "rewards/rejected": -0.015375614166259766,
      "step": 2423
    },
    {
      "epoch": 0.6375906228936856,
      "grad_norm": 584.0,
      "learning_rate": 2.013157894736842e-07,
      "logits/chosen": -0.6346732974052429,
      "logits/rejected": -0.6345435976982117,
      "logps/chosen": -1189.394775390625,
      "logps/rejected": -684.8128662109375,
      "loss": 0.6804,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.031099796295166016,
      "rewards/margins": 0.026507949456572533,
      "rewards/rejected": 0.004591846838593483,
      "step": 2424
    },
    {
      "epoch": 0.6378536553288727,
      "grad_norm": 656.0,
      "learning_rate": 2.0116959064327485e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6453825235366821,
      "logps/chosen": -990.4991455078125,
      "logps/rejected": -695.5975341796875,
      "loss": 0.689,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.005082129966467619,
      "rewards/margins": 0.008611870929598808,
      "rewards/rejected": -0.013693999499082565,
      "step": 2425
    },
    {
      "epoch": 0.6381166877640599,
      "grad_norm": 664.0,
      "learning_rate": 2.010233918128655e-07,
      "logits/chosen": -0.6669120788574219,
      "logits/rejected": -0.6666615605354309,
      "logps/chosen": -1286.7381591796875,
      "logps/rejected": -1254.4005126953125,
      "loss": 0.6915,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012339210137724876,
      "rewards/margins": 0.004184437450021505,
      "rewards/rejected": 0.008154774084687233,
      "step": 2426
    },
    {
      "epoch": 0.638379720199247,
      "grad_norm": 474.0,
      "learning_rate": 2.0087719298245614e-07,
      "logits/chosen": -0.6468674540519714,
      "logits/rejected": -0.6496248841285706,
      "logps/chosen": -1124.6513671875,
      "logps/rejected": -846.8048706054688,
      "loss": 0.6938,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.010646341368556023,
      "rewards/margins": -0.00046892277896404266,
      "rewards/rejected": -0.010177421383559704,
      "step": 2427
    },
    {
      "epoch": 0.6386427526344343,
      "grad_norm": 612.0,
      "learning_rate": 2.0073099415204679e-07,
      "logits/chosen": -0.6603735089302063,
      "logits/rejected": -0.6642783284187317,
      "logps/chosen": -1220.74462890625,
      "logps/rejected": -1010.68212890625,
      "loss": 0.6785,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.020780468359589577,
      "rewards/margins": 0.03006286732852459,
      "rewards/rejected": -0.009282398968935013,
      "step": 2428
    },
    {
      "epoch": 0.6389057850696214,
      "grad_norm": 510.0,
      "learning_rate": 2.005847953216374e-07,
      "logits/chosen": -0.6554582118988037,
      "logits/rejected": -0.662199854850769,
      "logps/chosen": -886.9076538085938,
      "logps/rejected": -619.7872314453125,
      "loss": 0.6897,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0031913742423057556,
      "rewards/margins": 0.008148480206727982,
      "rewards/rejected": -0.011339856311678886,
      "step": 2429
    },
    {
      "epoch": 0.6391688175048086,
      "grad_norm": 406.0,
      "learning_rate": 2.0043859649122808e-07,
      "logits/chosen": -0.6225589513778687,
      "logits/rejected": -0.6247717142105103,
      "logps/chosen": -808.1151733398438,
      "logps/rejected": -599.6525268554688,
      "loss": 0.694,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.03194484859704971,
      "rewards/margins": -0.0009153853170573711,
      "rewards/rejected": 0.03286023065447807,
      "step": 2430
    },
    {
      "epoch": 0.6394318499399957,
      "grad_norm": 544.0,
      "learning_rate": 2.002923976608187e-07,
      "logits/chosen": -0.674921452999115,
      "logits/rejected": -0.6668782234191895,
      "logps/chosen": -774.739501953125,
      "logps/rejected": -977.3129272460938,
      "loss": 0.6945,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006430434994399548,
      "rewards/margins": -0.002020787913352251,
      "rewards/rejected": 0.00845122430473566,
      "step": 2431
    },
    {
      "epoch": 0.6396948823751829,
      "grad_norm": 442.0,
      "learning_rate": 2.0014619883040934e-07,
      "logits/chosen": -0.6331616640090942,
      "logits/rejected": -0.640434205532074,
      "logps/chosen": -883.19482421875,
      "logps/rejected": -791.2267456054688,
      "loss": 0.6953,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012349032796919346,
      "rewards/margins": -0.00343489833176136,
      "rewards/rejected": 0.01578393206000328,
      "step": 2432
    },
    {
      "epoch": 0.63995791481037,
      "grad_norm": 520.0,
      "learning_rate": 2e-07,
      "logits/chosen": -0.647162139415741,
      "logits/rejected": -0.6478341817855835,
      "logps/chosen": -1051.4805908203125,
      "logps/rejected": -821.7486572265625,
      "loss": 0.6908,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008019639179110527,
      "rewards/margins": 0.005646609701216221,
      "rewards/rejected": 0.0023730280809104443,
      "step": 2433
    },
    {
      "epoch": 0.6402209472455572,
      "grad_norm": 494.0,
      "learning_rate": 1.9985380116959064e-07,
      "logits/chosen": -0.65129154920578,
      "logits/rejected": -0.6457209587097168,
      "logps/chosen": -1037.8819580078125,
      "logps/rejected": -805.7305908203125,
      "loss": 0.693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005487252026796341,
      "rewards/margins": 0.0015995983267202973,
      "rewards/rejected": 0.003887653350830078,
      "step": 2434
    },
    {
      "epoch": 0.6404839796807443,
      "grad_norm": 528.0,
      "learning_rate": 1.9970760233918126e-07,
      "logits/chosen": -0.6557469367980957,
      "logits/rejected": -0.6581280827522278,
      "logps/chosen": -1023.3870849609375,
      "logps/rejected": -985.5267333984375,
      "loss": 0.6914,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00028762780129909515,
      "rewards/margins": 0.0041893962770700455,
      "rewards/rejected": -0.004477025009691715,
      "step": 2435
    },
    {
      "epoch": 0.6407470121159315,
      "grad_norm": 510.0,
      "learning_rate": 1.9956140350877193e-07,
      "logits/chosen": -0.659478485584259,
      "logits/rejected": -0.6573915481567383,
      "logps/chosen": -715.1646118164062,
      "logps/rejected": -829.879638671875,
      "loss": 0.708,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0056913383305072784,
      "rewards/margins": -0.028365183621644974,
      "rewards/rejected": 0.022673845291137695,
      "step": 2436
    },
    {
      "epoch": 0.6410100445511188,
      "grad_norm": 510.0,
      "learning_rate": 1.9941520467836258e-07,
      "logits/chosen": -0.655931830406189,
      "logits/rejected": -0.6493110656738281,
      "logps/chosen": -1298.647216796875,
      "logps/rejected": -919.36474609375,
      "loss": 0.6914,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003980923444032669,
      "rewards/margins": 0.003996753133833408,
      "rewards/rejected": -1.583038829267025e-05,
      "step": 2437
    },
    {
      "epoch": 0.6412730769863059,
      "grad_norm": 668.0,
      "learning_rate": 1.992690058479532e-07,
      "logits/chosen": -0.6547083258628845,
      "logits/rejected": -0.6552219986915588,
      "logps/chosen": -990.9443969726562,
      "logps/rejected": -982.468017578125,
      "loss": 0.6896,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.011348631232976913,
      "rewards/margins": 0.007776642683893442,
      "rewards/rejected": 0.0035719869192689657,
      "step": 2438
    },
    {
      "epoch": 0.6415361094214931,
      "grad_norm": 592.0,
      "learning_rate": 1.9912280701754387e-07,
      "logits/chosen": -0.641472339630127,
      "logits/rejected": -0.6315276622772217,
      "logps/chosen": -1267.1873779296875,
      "logps/rejected": -665.3665161132812,
      "loss": 0.7001,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.016167068853974342,
      "rewards/margins": -0.012288331054151058,
      "rewards/rejected": -0.003878736635670066,
      "step": 2439
    },
    {
      "epoch": 0.6417991418566802,
      "grad_norm": 462.0,
      "learning_rate": 1.989766081871345e-07,
      "logits/chosen": -0.6496891975402832,
      "logits/rejected": -0.6505287885665894,
      "logps/chosen": -947.8828735351562,
      "logps/rejected": -923.427490234375,
      "loss": 0.6956,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00339088449254632,
      "rewards/margins": -0.004150438122451305,
      "rewards/rejected": 0.007541323080658913,
      "step": 2440
    },
    {
      "epoch": 0.6420621742918674,
      "grad_norm": 446.0,
      "learning_rate": 1.9883040935672514e-07,
      "logits/chosen": -0.6551352143287659,
      "logits/rejected": -0.6613792777061462,
      "logps/chosen": -875.0409545898438,
      "logps/rejected": -620.1327514648438,
      "loss": 0.6869,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004285716917365789,
      "rewards/margins": 0.013359500095248222,
      "rewards/rejected": -0.009073782712221146,
      "step": 2441
    },
    {
      "epoch": 0.6423252067270545,
      "grad_norm": 588.0,
      "learning_rate": 1.9868421052631578e-07,
      "logits/chosen": -0.6486873626708984,
      "logits/rejected": -0.6624647974967957,
      "logps/chosen": -1059.0765380859375,
      "logps/rejected": -1013.9463500976562,
      "loss": 0.7021,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.004833316896110773,
      "rewards/margins": -0.016841983422636986,
      "rewards/rejected": 0.0120086669921875,
      "step": 2442
    },
    {
      "epoch": 0.6425882391622417,
      "grad_norm": 524.0,
      "learning_rate": 1.9853801169590643e-07,
      "logits/chosen": -0.6471864581108093,
      "logits/rejected": -0.6445327401161194,
      "logps/chosen": -910.014892578125,
      "logps/rejected": -968.4625244140625,
      "loss": 0.6955,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.008363056927919388,
      "rewards/margins": -0.0040503512136638165,
      "rewards/rejected": -0.004312704782932997,
      "step": 2443
    },
    {
      "epoch": 0.6428512715974288,
      "grad_norm": 462.0,
      "learning_rate": 1.9839181286549705e-07,
      "logits/chosen": -0.6501427888870239,
      "logits/rejected": -0.6547475457191467,
      "logps/chosen": -989.545654296875,
      "logps/rejected": -1043.6407470703125,
      "loss": 0.6972,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.027550125494599342,
      "rewards/margins": -0.00783929880708456,
      "rewards/rejected": -0.019710825756192207,
      "step": 2444
    },
    {
      "epoch": 0.643114304032616,
      "grad_norm": 556.0,
      "learning_rate": 1.9824561403508772e-07,
      "logits/chosen": -0.6380305886268616,
      "logits/rejected": -0.6675969362258911,
      "logps/chosen": -1211.1546630859375,
      "logps/rejected": -651.1776123046875,
      "loss": 0.7047,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.004978753626346588,
      "rewards/margins": -0.02228250727057457,
      "rewards/rejected": 0.01730375364422798,
      "step": 2445
    },
    {
      "epoch": 0.6433773364678032,
      "grad_norm": 652.0,
      "learning_rate": 1.9809941520467834e-07,
      "logits/chosen": -0.6311845183372498,
      "logits/rejected": -0.6402206420898438,
      "logps/chosen": -1379.59814453125,
      "logps/rejected": -1249.232666015625,
      "loss": 0.6957,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0068502421490848064,
      "rewards/margins": -0.004257298074662685,
      "rewards/rejected": 0.01110753882676363,
      "step": 2446
    },
    {
      "epoch": 0.6436403689029904,
      "grad_norm": 652.0,
      "learning_rate": 1.97953216374269e-07,
      "logits/chosen": -0.6177358627319336,
      "logits/rejected": -0.6238604784011841,
      "logps/chosen": -1364.5699462890625,
      "logps/rejected": -1323.8392333984375,
      "loss": 0.7043,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.008206367492675781,
      "rewards/margins": -0.020643139258027077,
      "rewards/rejected": 0.028849506750702858,
      "step": 2447
    },
    {
      "epoch": 0.6439034013381775,
      "grad_norm": 510.0,
      "learning_rate": 1.9780701754385963e-07,
      "logits/chosen": -0.6353910565376282,
      "logits/rejected": -0.6291096806526184,
      "logps/chosen": -1186.7193603515625,
      "logps/rejected": -1118.149658203125,
      "loss": 0.682,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.016618728637695312,
      "rewards/margins": 0.022888660430908203,
      "rewards/rejected": -0.006269931327551603,
      "step": 2448
    },
    {
      "epoch": 0.6441664337733647,
      "grad_norm": 524.0,
      "learning_rate": 1.9766081871345028e-07,
      "logits/chosen": NaN,
      "logits/rejected": NaN,
      "logps/chosen": -964.1339111328125,
      "logps/rejected": -815.47900390625,
      "loss": 0.685,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.020404435694217682,
      "rewards/margins": 0.01692819595336914,
      "rewards/rejected": 0.0034762388095259666,
      "step": 2449
    },
    {
      "epoch": 0.6444294662085518,
      "grad_norm": 556.0,
      "learning_rate": 1.9751461988304093e-07,
      "logits/chosen": -0.646198034286499,
      "logits/rejected": -0.655738890171051,
      "logps/chosen": -853.9755859375,
      "logps/rejected": -899.4666137695312,
      "loss": 0.6904,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008541584014892578,
      "rewards/margins": 0.006265782751142979,
      "rewards/rejected": 0.002275800798088312,
      "step": 2450
    },
    {
      "epoch": 0.644692498643739,
      "grad_norm": 648.0,
      "learning_rate": 1.9736842105263157e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6474409103393555,
      "logps/chosen": -1238.0623779296875,
      "logps/rejected": -993.6275634765625,
      "loss": 0.6987,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.014142228290438652,
      "rewards/margins": -0.009900188073515892,
      "rewards/rejected": 0.024042414501309395,
      "step": 2451
    },
    {
      "epoch": 0.6449555310789262,
      "grad_norm": 924.0,
      "learning_rate": 1.9722222222222222e-07,
      "logits/chosen": -0.6843039989471436,
      "logits/rejected": -0.6860421895980835,
      "logps/chosen": -918.3191528320312,
      "logps/rejected": -628.5120239257812,
      "loss": 0.6816,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.00259833293966949,
      "rewards/margins": 0.023648308590054512,
      "rewards/rejected": -0.02624664269387722,
      "step": 2452
    },
    {
      "epoch": 0.6452185635141133,
      "grad_norm": 532.0,
      "learning_rate": 1.9707602339181287e-07,
      "logits/chosen": -0.6440694332122803,
      "logits/rejected": -0.6402490139007568,
      "logps/chosen": -1359.965087890625,
      "logps/rejected": -1040.3641357421875,
      "loss": 0.7006,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.004342651925981045,
      "rewards/margins": -0.014070702716708183,
      "rewards/rejected": 0.009728050790727139,
      "step": 2453
    },
    {
      "epoch": 0.6454815959493005,
      "grad_norm": 480.0,
      "learning_rate": 1.969298245614035e-07,
      "logits/chosen": -0.6375161409378052,
      "logits/rejected": -0.6456345319747925,
      "logps/chosen": -1193.2130126953125,
      "logps/rejected": -1099.969970703125,
      "loss": 0.6785,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.017533589154481888,
      "rewards/margins": 0.030277254059910774,
      "rewards/rejected": -0.012743664905428886,
      "step": 2454
    },
    {
      "epoch": 0.6457446283844877,
      "grad_norm": 544.0,
      "learning_rate": 1.9678362573099413e-07,
      "logits/chosen": -0.6526318192481995,
      "logits/rejected": -0.646861732006073,
      "logps/chosen": -1106.66845703125,
      "logps/rejected": -1219.50732421875,
      "loss": 0.6886,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0013806342612951994,
      "rewards/margins": 0.009774254634976387,
      "rewards/rejected": -0.008393622934818268,
      "step": 2455
    },
    {
      "epoch": 0.6460076608196749,
      "grad_norm": 576.0,
      "learning_rate": 1.966374269005848e-07,
      "logits/chosen": -0.6579135656356812,
      "logits/rejected": -0.6489805579185486,
      "logps/chosen": -1070.415771484375,
      "logps/rejected": -956.0302124023438,
      "loss": 0.6861,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00848245620727539,
      "rewards/margins": 0.01490645483136177,
      "rewards/rejected": -0.006423997692763805,
      "step": 2456
    },
    {
      "epoch": 0.646270693254862,
      "grad_norm": 532.0,
      "learning_rate": 1.9649122807017542e-07,
      "logits/chosen": -0.6403825879096985,
      "logits/rejected": -0.6408226490020752,
      "logps/chosen": -1157.1231689453125,
      "logps/rejected": -784.3365478515625,
      "loss": 0.6793,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.011984444223344326,
      "rewards/margins": 0.02849912829697132,
      "rewards/rejected": -0.01651468127965927,
      "step": 2457
    },
    {
      "epoch": 0.6465337256900492,
      "grad_norm": 502.0,
      "learning_rate": 1.9634502923976607e-07,
      "logits/chosen": -0.6412931084632874,
      "logits/rejected": -0.6474970579147339,
      "logps/chosen": -976.4659423828125,
      "logps/rejected": -902.3522338867188,
      "loss": 0.6962,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0020379056222736835,
      "rewards/margins": -0.0057059284299612045,
      "rewards/rejected": 0.0036680232733488083,
      "step": 2458
    },
    {
      "epoch": 0.6467967581252363,
      "grad_norm": 588.0,
      "learning_rate": 1.9619883040935672e-07,
      "logits/chosen": -0.6459706425666809,
      "logits/rejected": -0.6554992198944092,
      "logps/chosen": -1470.726806640625,
      "logps/rejected": -946.3988647460938,
      "loss": 0.7074,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.007668018341064453,
      "rewards/margins": -0.026073360815644264,
      "rewards/rejected": 0.01840534433722496,
      "step": 2459
    },
    {
      "epoch": 0.6470597905604235,
      "grad_norm": 498.0,
      "learning_rate": 1.9605263157894736e-07,
      "logits/chosen": -0.6566760540008545,
      "logits/rejected": -0.6643236875534058,
      "logps/chosen": -1289.1815185546875,
      "logps/rejected": -846.9974975585938,
      "loss": 0.6944,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.011036396957933903,
      "rewards/margins": -0.001935766078531742,
      "rewards/rejected": -0.009100628085434437,
      "step": 2460
    },
    {
      "epoch": 0.6473228229956106,
      "grad_norm": 416.0,
      "learning_rate": 1.9590643274853798e-07,
      "logits/chosen": -0.630719780921936,
      "logits/rejected": -0.6363896131515503,
      "logps/chosen": -597.0174560546875,
      "logps/rejected": -661.7099609375,
      "loss": 0.6964,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.003620338626205921,
      "rewards/margins": -0.005946348886936903,
      "rewards/rejected": 0.0023260114248842,
      "step": 2461
    },
    {
      "epoch": 0.6475858554307978,
      "grad_norm": 516.0,
      "learning_rate": 1.9576023391812866e-07,
      "logits/chosen": -0.6515212059020996,
      "logits/rejected": -0.6532692909240723,
      "logps/chosen": -1154.5225830078125,
      "logps/rejected": -911.1237182617188,
      "loss": 0.6876,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.014095592312514782,
      "rewards/margins": 0.011782550252974033,
      "rewards/rejected": 0.0023130420595407486,
      "step": 2462
    },
    {
      "epoch": 0.6478488878659849,
      "grad_norm": 462.0,
      "learning_rate": 1.9561403508771928e-07,
      "logits/chosen": -0.6702843308448792,
      "logits/rejected": -0.6754359006881714,
      "logps/chosen": -1305.2799072265625,
      "logps/rejected": -1048.397216796875,
      "loss": 0.693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.016588928177952766,
      "rewards/margins": 0.0007321833400055766,
      "rewards/rejected": 0.01585674285888672,
      "step": 2463
    },
    {
      "epoch": 0.6481119203011722,
      "grad_norm": 528.0,
      "learning_rate": 1.9546783625730992e-07,
      "logits/chosen": -0.6581461429595947,
      "logits/rejected": -0.6669906377792358,
      "logps/chosen": -1030.017822265625,
      "logps/rejected": -1186.4339599609375,
      "loss": 0.6742,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0030777929350733757,
      "rewards/margins": 0.03957195207476616,
      "rewards/rejected": -0.04264974966645241,
      "step": 2464
    },
    {
      "epoch": 0.6483749527363593,
      "grad_norm": 560.0,
      "learning_rate": 1.9532163742690057e-07,
      "logits/chosen": -0.6403710246086121,
      "logits/rejected": -0.629951000213623,
      "logps/chosen": -1102.6673583984375,
      "logps/rejected": -604.6776123046875,
      "loss": 0.682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.004433248192071915,
      "rewards/margins": 0.02364521101117134,
      "rewards/rejected": -0.019211959093809128,
      "step": 2465
    },
    {
      "epoch": 0.6486379851715465,
      "grad_norm": 600.0,
      "learning_rate": 1.9517543859649122e-07,
      "logits/chosen": -0.6613543629646301,
      "logits/rejected": -0.6467583179473877,
      "logps/chosen": -1482.843994140625,
      "logps/rejected": -1053.1263427734375,
      "loss": 0.6933,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010568523779511452,
      "rewards/margins": 0.0003372207283973694,
      "rewards/rejected": 0.010231303051114082,
      "step": 2466
    },
    {
      "epoch": 0.6489010176067336,
      "grad_norm": 464.0,
      "learning_rate": 1.9502923976608186e-07,
      "logits/chosen": -0.6179444193840027,
      "logits/rejected": -0.6337894201278687,
      "logps/chosen": -899.259765625,
      "logps/rejected": -601.0985107421875,
      "loss": 0.6851,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0011693008709698915,
      "rewards/margins": 0.01669926755130291,
      "rewards/rejected": -0.015529966913163662,
      "step": 2467
    },
    {
      "epoch": 0.6491640500419208,
      "grad_norm": 588.0,
      "learning_rate": 1.948830409356725e-07,
      "logits/chosen": -0.6261348128318787,
      "logits/rejected": -0.6374227404594421,
      "logps/chosen": -1154.123291015625,
      "logps/rejected": -939.2626953125,
      "loss": 0.6823,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006172943860292435,
      "rewards/margins": 0.022817231714725494,
      "rewards/rejected": -0.01664428785443306,
      "step": 2468
    },
    {
      "epoch": 0.649427082477108,
      "grad_norm": 612.0,
      "learning_rate": 1.9473684210526315e-07,
      "logits/chosen": -0.6278479695320129,
      "logits/rejected": -0.6344054937362671,
      "logps/chosen": -1316.4766845703125,
      "logps/rejected": -930.3444213867188,
      "loss": 0.7057,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01109485886991024,
      "rewards/margins": -0.023420238867402077,
      "rewards/rejected": 0.012325381860136986,
      "step": 2469
    },
    {
      "epoch": 0.6496901149122951,
      "grad_norm": 528.0,
      "learning_rate": 1.945906432748538e-07,
      "logits/chosen": -0.6672361493110657,
      "logits/rejected": NaN,
      "logps/chosen": -1166.27978515625,
      "logps/rejected": -768.0828857421875,
      "loss": 0.6884,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.003067874349653721,
      "rewards/margins": 0.009942150674760342,
      "rewards/rejected": -0.006874275393784046,
      "step": 2470
    },
    {
      "epoch": 0.6499531473474823,
      "grad_norm": 434.0,
      "learning_rate": 1.9444444444444445e-07,
      "logits/chosen": -0.661939263343811,
      "logits/rejected": -0.6781541109085083,
      "logps/chosen": -947.9467163085938,
      "logps/rejected": -614.8844604492188,
      "loss": 0.6927,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006598187610507011,
      "rewards/margins": 0.0013518333435058594,
      "rewards/rejected": 0.00524635287001729,
      "step": 2471
    },
    {
      "epoch": 0.6502161797826694,
      "grad_norm": 506.0,
      "learning_rate": 1.9429824561403507e-07,
      "logits/chosen": -0.64494389295578,
      "logits/rejected": NaN,
      "logps/chosen": -1257.8135986328125,
      "logps/rejected": -1058.5701904296875,
      "loss": 0.6877,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.007003973238170147,
      "rewards/margins": 0.011668968945741653,
      "rewards/rejected": -0.018672943115234375,
      "step": 2472
    },
    {
      "epoch": 0.6504792122178567,
      "grad_norm": 604.0,
      "learning_rate": 1.9415204678362574e-07,
      "logits/chosen": -0.6269286274909973,
      "logits/rejected": -0.635598361492157,
      "logps/chosen": -1860.367919921875,
      "logps/rejected": -1264.503662109375,
      "loss": 0.6969,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.013005543500185013,
      "rewards/margins": -0.0071742055006325245,
      "rewards/rejected": 0.0201797503978014,
      "step": 2473
    },
    {
      "epoch": 0.6507422446530438,
      "grad_norm": 472.0,
      "learning_rate": 1.9400584795321636e-07,
      "logits/chosen": -0.6601656079292297,
      "logits/rejected": -0.6609313488006592,
      "logps/chosen": -1232.410400390625,
      "logps/rejected": -940.9686889648438,
      "loss": 0.6907,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.019061852246522903,
      "rewards/margins": 0.00559845007956028,
      "rewards/rejected": 0.013463401235640049,
      "step": 2474
    },
    {
      "epoch": 0.651005277088231,
      "grad_norm": 446.0,
      "learning_rate": 1.93859649122807e-07,
      "logits/chosen": -0.6611047983169556,
      "logits/rejected": -0.6678599715232849,
      "logps/chosen": -915.9116821289062,
      "logps/rejected": -588.786865234375,
      "loss": 0.6916,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.002382183214649558,
      "rewards/margins": 0.003573798341676593,
      "rewards/rejected": -0.005955982953310013,
      "step": 2475
    },
    {
      "epoch": 0.6512683095234181,
      "grad_norm": 632.0,
      "learning_rate": 1.9371345029239765e-07,
      "logits/chosen": -0.6524093151092529,
      "logits/rejected": -0.6581680178642273,
      "logps/chosen": -1246.683837890625,
      "logps/rejected": -981.5948486328125,
      "loss": 0.6941,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006769942585378885,
      "rewards/margins": -0.0001493450254201889,
      "rewards/rejected": 0.0069192880764603615,
      "step": 2476
    },
    {
      "epoch": 0.6515313419586053,
      "grad_norm": 440.0,
      "learning_rate": 1.935672514619883e-07,
      "logits/chosen": -0.677272379398346,
      "logits/rejected": -0.6718651056289673,
      "logps/chosen": -1013.9969482421875,
      "logps/rejected": -783.663330078125,
      "loss": 0.69,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.014321424067020416,
      "rewards/margins": 0.006606864742934704,
      "rewards/rejected": -0.020928287878632545,
      "step": 2477
    },
    {
      "epoch": 0.6517943743937924,
      "grad_norm": 584.0,
      "learning_rate": 1.9342105263157892e-07,
      "logits/chosen": -0.651123046875,
      "logits/rejected": -0.6513775587081909,
      "logps/chosen": -1478.0484619140625,
      "logps/rejected": -1211.3955078125,
      "loss": 0.6973,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.014140984043478966,
      "rewards/margins": -0.007505321875214577,
      "rewards/rejected": 0.02164630964398384,
      "step": 2478
    },
    {
      "epoch": 0.6520574068289796,
      "grad_norm": 564.0,
      "learning_rate": 1.932748538011696e-07,
      "logits/chosen": -0.6506376266479492,
      "logits/rejected": -0.6545106172561646,
      "logps/chosen": -1313.887939453125,
      "logps/rejected": -1090.30322265625,
      "loss": 0.6912,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0019266128074377775,
      "rewards/margins": 0.004222536459565163,
      "rewards/rejected": -0.002295923652127385,
      "step": 2479
    },
    {
      "epoch": 0.6523204392641667,
      "grad_norm": 512.0,
      "learning_rate": 1.931286549707602e-07,
      "logits/chosen": -0.6502139568328857,
      "logits/rejected": -0.6516965627670288,
      "logps/chosen": -1248.4332275390625,
      "logps/rejected": -1141.4959716796875,
      "loss": 0.6954,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.022381020709872246,
      "rewards/margins": -0.003652382642030716,
      "rewards/rejected": -0.01872863993048668,
      "step": 2480
    },
    {
      "epoch": 0.6525834716993539,
      "grad_norm": 640.0,
      "learning_rate": 1.9298245614035086e-07,
      "logits/chosen": -0.6508334875106812,
      "logits/rejected": -0.6632416248321533,
      "logps/chosen": -1434.0018310546875,
      "logps/rejected": -1109.0665283203125,
      "loss": 0.7017,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.003550910856574774,
      "rewards/margins": -0.016492750495672226,
      "rewards/rejected": 0.020043659955263138,
      "step": 2481
    },
    {
      "epoch": 0.652846504134541,
      "grad_norm": 640.0,
      "learning_rate": 1.9283625730994153e-07,
      "logits/chosen": -0.6591958999633789,
      "logits/rejected": -0.6716814041137695,
      "logps/chosen": -1233.89599609375,
      "logps/rejected": -905.51025390625,
      "loss": 0.6847,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.002791309729218483,
      "rewards/margins": 0.01807851903140545,
      "rewards/rejected": -0.015287207439541817,
      "step": 2482
    },
    {
      "epoch": 0.6531095365697283,
      "grad_norm": 540.0,
      "learning_rate": 1.9269005847953215e-07,
      "logits/chosen": -0.655335545539856,
      "logits/rejected": -0.6469780206680298,
      "logps/chosen": -1144.577880859375,
      "logps/rejected": -920.5737915039062,
      "loss": 0.6802,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.011737825348973274,
      "rewards/margins": 0.027379892766475677,
      "rewards/rejected": -0.0156420711427927,
      "step": 2483
    },
    {
      "epoch": 0.6533725690049155,
      "grad_norm": 520.0,
      "learning_rate": 1.925438596491228e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6693537831306458,
      "logps/chosen": -984.8690795898438,
      "logps/rejected": -1017.6137084960938,
      "loss": 0.6855,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.015050887130200863,
      "rewards/margins": 0.016196729615330696,
      "rewards/rejected": -0.001145839923992753,
      "step": 2484
    },
    {
      "epoch": 0.6536356014401026,
      "grad_norm": 580.0,
      "learning_rate": 1.9239766081871344e-07,
      "logits/chosen": -0.6535071134567261,
      "logits/rejected": -0.6605018377304077,
      "logps/chosen": -1281.69970703125,
      "logps/rejected": -915.487060546875,
      "loss": 0.6948,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -5.1402952522039413e-05,
      "rewards/margins": -0.0019553182646632195,
      "rewards/rejected": 0.00190391531214118,
      "step": 2485
    },
    {
      "epoch": 0.6538986338752898,
      "grad_norm": 442.0,
      "learning_rate": 1.922514619883041e-07,
      "logits/chosen": -0.6572045087814331,
      "logits/rejected": -0.6493358016014099,
      "logps/chosen": -918.861328125,
      "logps/rejected": -832.1693115234375,
      "loss": 0.6897,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0157686248421669,
      "rewards/margins": 0.007861185818910599,
      "rewards/rejected": 0.007907437160611153,
      "step": 2486
    },
    {
      "epoch": 0.6541616663104769,
      "grad_norm": 500.0,
      "learning_rate": 1.921052631578947e-07,
      "logits/chosen": -0.6615506410598755,
      "logits/rejected": -0.666367769241333,
      "logps/chosen": -1382.4940185546875,
      "logps/rejected": -884.8426513671875,
      "loss": 0.6927,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.004424096085131168,
      "rewards/margins": 0.0019365306943655014,
      "rewards/rejected": -0.0063606263138353825,
      "step": 2487
    },
    {
      "epoch": 0.6544246987456641,
      "grad_norm": 436.0,
      "learning_rate": 1.9195906432748538e-07,
      "logits/chosen": -0.6732100248336792,
      "logits/rejected": -0.6794488430023193,
      "logps/chosen": -1146.345947265625,
      "logps/rejected": -1059.5242919921875,
      "loss": 0.6908,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.027275562286376953,
      "rewards/margins": 0.00511617586016655,
      "rewards/rejected": -0.0323917418718338,
      "step": 2488
    },
    {
      "epoch": 0.6546877311808512,
      "grad_norm": 564.0,
      "learning_rate": 1.91812865497076e-07,
      "logits/chosen": -0.6350886225700378,
      "logits/rejected": -0.6458371877670288,
      "logps/chosen": -1076.7078857421875,
      "logps/rejected": -812.3658447265625,
      "loss": 0.6972,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.003469611518085003,
      "rewards/margins": -0.006425238214433193,
      "rewards/rejected": 0.0029556271620094776,
      "step": 2489
    },
    {
      "epoch": 0.6549507636160384,
      "grad_norm": 536.0,
      "learning_rate": 1.9166666666666668e-07,
      "logits/chosen": -0.6641272902488708,
      "logits/rejected": NaN,
      "logps/chosen": -1257.4178466796875,
      "logps/rejected": -521.2056274414062,
      "loss": 0.6939,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.005161285400390625,
      "rewards/margins": -0.0009291649330407381,
      "rewards/rejected": -0.004232121631503105,
      "step": 2490
    },
    {
      "epoch": 0.6552137960512255,
      "grad_norm": 628.0,
      "learning_rate": 1.915204678362573e-07,
      "logits/chosen": -0.662407398223877,
      "logits/rejected": -0.6754022240638733,
      "logps/chosen": -1388.1982421875,
      "logps/rejected": -952.9306030273438,
      "loss": 0.6902,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0008929252508096397,
      "rewards/margins": 0.007606412284076214,
      "rewards/rejected": -0.006713487207889557,
      "step": 2491
    },
    {
      "epoch": 0.6554768284864128,
      "grad_norm": 422.0,
      "learning_rate": 1.9137426900584794e-07,
      "logits/chosen": -0.6523133516311646,
      "logits/rejected": -0.6528911590576172,
      "logps/chosen": -981.0147094726562,
      "logps/rejected": -744.8521728515625,
      "loss": 0.691,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.014536857604980469,
      "rewards/margins": 0.005046750418841839,
      "rewards/rejected": 0.00949010904878378,
      "step": 2492
    },
    {
      "epoch": 0.6557398609215999,
      "grad_norm": 560.0,
      "learning_rate": 1.912280701754386e-07,
      "logits/chosen": -0.6466803550720215,
      "logits/rejected": -0.6347830295562744,
      "logps/chosen": -891.31005859375,
      "logps/rejected": -999.7974853515625,
      "loss": 0.6925,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008709955960512161,
      "rewards/margins": 0.0024310583248734474,
      "rewards/rejected": 0.006278896238654852,
      "step": 2493
    },
    {
      "epoch": 0.6560028933567871,
      "grad_norm": 644.0,
      "learning_rate": 1.9108187134502923e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6717228293418884,
      "logps/chosen": -981.2135620117188,
      "logps/rejected": -1261.609375,
      "loss": 0.6983,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01727294921875,
      "rewards/margins": -0.009645270183682442,
      "rewards/rejected": -0.007627679035067558,
      "step": 2494
    },
    {
      "epoch": 0.6562659257919742,
      "grad_norm": 504.0,
      "learning_rate": 1.9093567251461985e-07,
      "logits/chosen": -0.6600304841995239,
      "logits/rejected": -0.6724786162376404,
      "logps/chosen": -1200.31982421875,
      "logps/rejected": -869.3751220703125,
      "loss": 0.6872,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.001288986997678876,
      "rewards/margins": 0.012668037787079811,
      "rewards/rejected": -0.011379051953554153,
      "step": 2495
    },
    {
      "epoch": 0.6565289582271614,
      "grad_norm": 700.0,
      "learning_rate": 1.9078947368421053e-07,
      "logits/chosen": -0.6396923661231995,
      "logits/rejected": -0.6349676847457886,
      "logps/chosen": -1347.026123046875,
      "logps/rejected": -1002.2728271484375,
      "loss": 0.6852,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.022398758679628372,
      "rewards/margins": 0.016684627160429955,
      "rewards/rejected": 0.005714130587875843,
      "step": 2496
    },
    {
      "epoch": 0.6567919906623485,
      "grad_norm": 696.0,
      "learning_rate": 1.9064327485380117e-07,
      "logits/chosen": -0.6426030397415161,
      "logits/rejected": -0.6438584327697754,
      "logps/chosen": -1608.1676025390625,
      "logps/rejected": -1328.04833984375,
      "loss": 0.6815,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.010639571584761143,
      "rewards/margins": 0.024233436211943626,
      "rewards/rejected": -0.013593863695859909,
      "step": 2497
    },
    {
      "epoch": 0.6570550230975357,
      "grad_norm": 508.0,
      "learning_rate": 1.904970760233918e-07,
      "logits/chosen": -0.6396567225456238,
      "logits/rejected": -0.6445601582527161,
      "logps/chosen": -930.4628295898438,
      "logps/rejected": -630.6199340820312,
      "loss": 0.704,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.014994621276855469,
      "rewards/margins": -0.02056727558374405,
      "rewards/rejected": 0.005572652444243431,
      "step": 2498
    },
    {
      "epoch": 0.6573180555327229,
      "grad_norm": 454.0,
      "learning_rate": 1.9035087719298247e-07,
      "logits/chosen": -0.6501558423042297,
      "logits/rejected": -0.651871383190155,
      "logps/chosen": -1068.5574951171875,
      "logps/rejected": -872.2774658203125,
      "loss": 0.6905,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0023023607209324837,
      "rewards/margins": 0.005661393515765667,
      "rewards/rejected": -0.0033590318635106087,
      "step": 2499
    },
    {
      "epoch": 0.65758108796791,
      "grad_norm": 498.0,
      "learning_rate": 1.9020467836257309e-07,
      "logits/chosen": -0.6504734754562378,
      "logits/rejected": -0.6457260847091675,
      "logps/chosen": -1225.0166015625,
      "logps/rejected": -1243.3323974609375,
      "loss": 0.6946,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.024126434698700905,
      "rewards/margins": -0.0025291433557868004,
      "rewards/rejected": 0.02665558084845543,
      "step": 2500
    },
    {
      "epoch": 0.6578441204030973,
      "grad_norm": 442.0,
      "learning_rate": 1.9005847953216373e-07,
      "logits/chosen": -0.6708792448043823,
      "logits/rejected": -0.6643751859664917,
      "logps/chosen": -990.1689453125,
      "logps/rejected": -1007.6812133789062,
      "loss": 0.7025,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.001379108289256692,
      "rewards/margins": -0.017892934381961823,
      "rewards/rejected": 0.016513826325535774,
      "step": 2501
    },
    {
      "epoch": 0.6581071528382844,
      "grad_norm": 506.0,
      "learning_rate": 1.8991228070175438e-07,
      "logits/chosen": -0.6463927030563354,
      "logits/rejected": -0.6355507373809814,
      "logps/chosen": -979.13525390625,
      "logps/rejected": -922.2996215820312,
      "loss": 0.6812,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.012142276391386986,
      "rewards/margins": 0.02486124075949192,
      "rewards/rejected": -0.01271896343678236,
      "step": 2502
    },
    {
      "epoch": 0.6583701852734716,
      "grad_norm": 548.0,
      "learning_rate": 1.8976608187134503e-07,
      "logits/chosen": -0.6503900289535522,
      "logits/rejected": -0.6443361639976501,
      "logps/chosen": -1107.652099609375,
      "logps/rejected": -827.5894165039062,
      "loss": 0.7036,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.004863167181611061,
      "rewards/margins": -0.019930172711610794,
      "rewards/rejected": 0.015067005529999733,
      "step": 2503
    },
    {
      "epoch": 0.6586332177086587,
      "grad_norm": 498.0,
      "learning_rate": 1.8961988304093565e-07,
      "logits/chosen": -0.6729932427406311,
      "logits/rejected": -0.6824114322662354,
      "logps/chosen": -786.2803344726562,
      "logps/rejected": -807.202392578125,
      "loss": 0.6765,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.03017129749059677,
      "rewards/margins": 0.03455314785242081,
      "rewards/rejected": -0.004381846636533737,
      "step": 2504
    },
    {
      "epoch": 0.6588962501438459,
      "grad_norm": 672.0,
      "learning_rate": 1.8947368421052632e-07,
      "logits/chosen": -0.6195228099822998,
      "logits/rejected": -0.6341606378555298,
      "logps/chosen": -997.98876953125,
      "logps/rejected": -784.2424926757812,
      "loss": 0.6933,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004493808373808861,
      "rewards/margins": 0.00026164017617702484,
      "rewards/rejected": 0.004232167731970549,
      "step": 2505
    },
    {
      "epoch": 0.659159282579033,
      "grad_norm": 384.0,
      "learning_rate": 1.8932748538011694e-07,
      "logits/chosen": -0.648426353931427,
      "logits/rejected": -0.6658165454864502,
      "logps/chosen": -907.2271118164062,
      "logps/rejected": -582.4832763671875,
      "loss": 0.6971,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0040933601558208466,
      "rewards/margins": -0.0070146555081009865,
      "rewards/rejected": 0.011108016595244408,
      "step": 2506
    },
    {
      "epoch": 0.6594223150142202,
      "grad_norm": 612.0,
      "learning_rate": 1.8918128654970758e-07,
      "logits/chosen": -0.6583497524261475,
      "logits/rejected": -0.6597508788108826,
      "logps/chosen": -1210.48681640625,
      "logps/rejected": -932.75927734375,
      "loss": 0.6921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0032500261440873146,
      "rewards/margins": 0.002760028000921011,
      "rewards/rejected": 0.0004899981431663036,
      "step": 2507
    },
    {
      "epoch": 0.6596853474494073,
      "grad_norm": 552.0,
      "learning_rate": 1.8903508771929823e-07,
      "logits/chosen": -0.6667113304138184,
      "logits/rejected": -0.6519515514373779,
      "logps/chosen": -897.7548828125,
      "logps/rejected": -947.4315185546875,
      "loss": 0.7096,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.022159479558467865,
      "rewards/margins": -0.030646702274680138,
      "rewards/rejected": 0.008487223647534847,
      "step": 2508
    },
    {
      "epoch": 0.6599483798845945,
      "grad_norm": 632.0,
      "learning_rate": 1.8888888888888888e-07,
      "logits/chosen": -0.6671730875968933,
      "logits/rejected": -0.672652006149292,
      "logps/chosen": -1389.5093994140625,
      "logps/rejected": -1246.67138671875,
      "loss": 0.6889,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.013346862979233265,
      "rewards/margins": 0.00892476923763752,
      "rewards/rejected": 0.0044220928102731705,
      "step": 2509
    },
    {
      "epoch": 0.6602114123197816,
      "grad_norm": 620.0,
      "learning_rate": 1.8874269005847952e-07,
      "logits/chosen": -0.6431910991668701,
      "logits/rejected": -0.6297283172607422,
      "logps/chosen": -1334.388427734375,
      "logps/rejected": -1164.57275390625,
      "loss": 0.6959,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.005238676909357309,
      "rewards/margins": -0.004463148768991232,
      "rewards/rejected": 0.009701823815703392,
      "step": 2510
    },
    {
      "epoch": 0.6604744447549689,
      "grad_norm": 576.0,
      "learning_rate": 1.8859649122807017e-07,
      "logits/chosen": -0.6661077737808228,
      "logits/rejected": -0.6688177585601807,
      "logps/chosen": -1198.141845703125,
      "logps/rejected": -1031.60693359375,
      "loss": 0.6892,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.016471005976200104,
      "rewards/margins": 0.008910275995731354,
      "rewards/rejected": 0.00756072998046875,
      "step": 2511
    },
    {
      "epoch": 0.660737477190156,
      "grad_norm": 532.0,
      "learning_rate": 1.8845029239766082e-07,
      "logits/chosen": -0.6604509353637695,
      "logits/rejected": -0.668798565864563,
      "logps/chosen": -1520.295654296875,
      "logps/rejected": -1097.711669921875,
      "loss": 0.6911,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0079529769718647,
      "rewards/margins": 0.004745197482407093,
      "rewards/rejected": 0.00320777902379632,
      "step": 2512
    },
    {
      "epoch": 0.6610005096253432,
      "grad_norm": 490.0,
      "learning_rate": 1.8830409356725146e-07,
      "logits/chosen": -0.6465259194374084,
      "logits/rejected": -0.6416101455688477,
      "logps/chosen": -1041.4337158203125,
      "logps/rejected": -851.0355224609375,
      "loss": 0.6893,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0015178676694631577,
      "rewards/margins": 0.009083748795092106,
      "rewards/rejected": -0.010601615533232689,
      "step": 2513
    },
    {
      "epoch": 0.6612635420605303,
      "grad_norm": 434.0,
      "learning_rate": 1.881578947368421e-07,
      "logits/chosen": -0.654207170009613,
      "logits/rejected": -0.6593817472457886,
      "logps/chosen": -845.1986694335938,
      "logps/rejected": -497.919189453125,
      "loss": 0.6922,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.007134866900742054,
      "rewards/margins": 0.002342748688533902,
      "rewards/rejected": 0.004792117979377508,
      "step": 2514
    },
    {
      "epoch": 0.6615265744957175,
      "grad_norm": 580.0,
      "learning_rate": 1.8801169590643273e-07,
      "logits/chosen": -0.6828283667564392,
      "logits/rejected": -0.6840769052505493,
      "logps/chosen": -1316.984619140625,
      "logps/rejected": -1374.301025390625,
      "loss": 0.6836,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.023125268518924713,
      "rewards/margins": 0.020230675116181374,
      "rewards/rejected": 0.0028945929370820522,
      "step": 2515
    },
    {
      "epoch": 0.6617896069309047,
      "grad_norm": 556.0,
      "learning_rate": 1.878654970760234e-07,
      "logits/chosen": -0.6451125144958496,
      "logits/rejected": -0.6573176383972168,
      "logps/chosen": -1237.0550537109375,
      "logps/rejected": -869.47900390625,
      "loss": 0.6931,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0008544919546693563,
      "rewards/margins": 0.0008163449820131063,
      "rewards/rejected": 3.814708907157183e-05,
      "step": 2516
    },
    {
      "epoch": 0.6620526393660918,
      "grad_norm": 552.0,
      "learning_rate": 1.8771929824561402e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.663253903388977,
      "logps/chosen": -1263.6829833984375,
      "logps/rejected": -923.4156494140625,
      "loss": 0.682,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.014463996514678001,
      "rewards/margins": 0.02296466939151287,
      "rewards/rejected": -0.00850067101418972,
      "step": 2517
    },
    {
      "epoch": 0.662315671801279,
      "grad_norm": 536.0,
      "learning_rate": 1.8757309941520467e-07,
      "logits/chosen": -0.640764594078064,
      "logits/rejected": -0.6338332891464233,
      "logps/chosen": -1198.7366943359375,
      "logps/rejected": -973.80224609375,
      "loss": 0.6948,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.004545212257653475,
      "rewards/margins": -0.00261278310790658,
      "rewards/rejected": -0.0019324300810694695,
      "step": 2518
    },
    {
      "epoch": 0.6625787042364661,
      "grad_norm": 568.0,
      "learning_rate": 1.8742690058479531e-07,
      "logits/chosen": -0.6561108827590942,
      "logits/rejected": -0.6571424603462219,
      "logps/chosen": -1165.06982421875,
      "logps/rejected": -822.2874145507812,
      "loss": 0.6995,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.006224155891686678,
      "rewards/margins": -0.01158599928021431,
      "rewards/rejected": 0.00536184199154377,
      "step": 2519
    },
    {
      "epoch": 0.6628417366716534,
      "grad_norm": 506.0,
      "learning_rate": 1.8728070175438596e-07,
      "logits/chosen": -0.6185482144355774,
      "logits/rejected": -0.6268329620361328,
      "logps/chosen": -1352.61962890625,
      "logps/rejected": -1038.50390625,
      "loss": 0.6886,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.000845431350171566,
      "rewards/margins": 0.009984780102968216,
      "rewards/rejected": -0.0091393468901515,
      "step": 2520
    },
    {
      "epoch": 0.6631047691068405,
      "grad_norm": 656.0,
      "learning_rate": 1.8713450292397658e-07,
      "logits/chosen": -0.6580456495285034,
      "logits/rejected": -0.6611415147781372,
      "logps/chosen": -1388.2005615234375,
      "logps/rejected": -1021.0821533203125,
      "loss": 0.7068,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.016228867694735527,
      "rewards/margins": -0.02625599130988121,
      "rewards/rejected": 0.010027121752500534,
      "step": 2521
    },
    {
      "epoch": 0.6633678015420277,
      "grad_norm": 508.0,
      "learning_rate": 1.8698830409356725e-07,
      "logits/chosen": -0.6497337818145752,
      "logits/rejected": -0.6541049480438232,
      "logps/chosen": -712.3563232421875,
      "logps/rejected": -742.744384765625,
      "loss": 0.6969,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.010876083746552467,
      "rewards/margins": -0.006185436621308327,
      "rewards/rejected": 0.017061518505215645,
      "step": 2522
    },
    {
      "epoch": 0.6636308339772148,
      "grad_norm": 506.0,
      "learning_rate": 1.8684210526315787e-07,
      "logits/chosen": -0.6443954706192017,
      "logits/rejected": -0.6509280204772949,
      "logps/chosen": -1430.9908447265625,
      "logps/rejected": -1229.6177978515625,
      "loss": 0.6985,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.015484049916267395,
      "rewards/margins": -0.010097886435687542,
      "rewards/rejected": -0.005386161617934704,
      "step": 2523
    },
    {
      "epoch": 0.663893866412402,
      "grad_norm": 544.0,
      "learning_rate": 1.8669590643274852e-07,
      "logits/chosen": -0.6501508355140686,
      "logits/rejected": -0.6410433053970337,
      "logps/chosen": -1046.2655029296875,
      "logps/rejected": -1060.63818359375,
      "loss": 0.6967,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0031458851881325245,
      "rewards/margins": -0.0060686590149998665,
      "rewards/rejected": 0.002922773826867342,
      "step": 2524
    },
    {
      "epoch": 0.6641568988475891,
      "grad_norm": 588.0,
      "learning_rate": 1.8654970760233917e-07,
      "logits/chosen": -0.6382898092269897,
      "logits/rejected": -0.6522223949432373,
      "logps/chosen": -1080.25048828125,
      "logps/rejected": -988.47314453125,
      "loss": 0.7019,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.01907949522137642,
      "rewards/margins": -0.016753720119595528,
      "rewards/rejected": -0.002325773239135742,
      "step": 2525
    },
    {
      "epoch": 0.6644199312827763,
      "grad_norm": 540.0,
      "learning_rate": 1.864035087719298e-07,
      "logits/chosen": -0.6497178077697754,
      "logits/rejected": -0.6573899388313293,
      "logps/chosen": -1103.685791015625,
      "logps/rejected": -623.020751953125,
      "loss": 0.7021,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.0017644884064793587,
      "rewards/margins": -0.0167065616697073,
      "rewards/rejected": 0.01847105287015438,
      "step": 2526
    },
    {
      "epoch": 0.6646829637179634,
      "grad_norm": 552.0,
      "learning_rate": 1.8625730994152046e-07,
      "logits/chosen": -0.6505095362663269,
      "logits/rejected": -0.6454840302467346,
      "logps/chosen": -912.871337890625,
      "logps/rejected": -834.9413452148438,
      "loss": 0.6974,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.004074429161846638,
      "rewards/margins": -0.008006047457456589,
      "rewards/rejected": 0.003931618295609951,
      "step": 2527
    },
    {
      "epoch": 0.6649459961531506,
      "grad_norm": 536.0,
      "learning_rate": 1.861111111111111e-07,
      "logits/chosen": -0.6576568484306335,
      "logits/rejected": -0.6506586074829102,
      "logps/chosen": -1243.87646484375,
      "logps/rejected": -942.1370849609375,
      "loss": 0.6914,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.015025710687041283,
      "rewards/margins": 0.0037799831479787827,
      "rewards/rejected": 0.0112457275390625,
      "step": 2528
    },
    {
      "epoch": 0.6652090285883377,
      "grad_norm": 676.0,
      "learning_rate": 1.8596491228070175e-07,
      "logits/chosen": -0.67039954662323,
      "logits/rejected": -0.665473222732544,
      "logps/chosen": -1253.242431640625,
      "logps/rejected": -1104.917236328125,
      "loss": 0.7059,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.014831161126494408,
      "rewards/margins": -0.02395925670862198,
      "rewards/rejected": 0.009128093719482422,
      "step": 2529
    },
    {
      "epoch": 0.665472061023525,
      "grad_norm": 608.0,
      "learning_rate": 1.858187134502924e-07,
      "logits/chosen": -0.6513347625732422,
      "logits/rejected": -0.6517698168754578,
      "logps/chosen": -1381.9847412109375,
      "logps/rejected": -1061.888427734375,
      "loss": 0.6901,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.007837962359189987,
      "rewards/margins": 0.006923102308064699,
      "rewards/rejected": 0.0009148595854640007,
      "step": 2530
    },
    {
      "epoch": 0.6657350934587122,
      "grad_norm": 656.0,
      "learning_rate": 1.8567251461988304e-07,
      "logits/chosen": -0.6673515439033508,
      "logits/rejected": -0.6618804931640625,
      "logps/chosen": -1203.986083984375,
      "logps/rejected": -1043.6436767578125,
      "loss": 0.7019,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.007509900256991386,
      "rewards/margins": -0.016689110547304153,
      "rewards/rejected": 0.02419901080429554,
      "step": 2531
    },
    {
      "epoch": 0.6659981258938993,
      "grad_norm": 516.0,
      "learning_rate": 1.8552631578947366e-07,
      "logits/chosen": -0.6532301902770996,
      "logits/rejected": -0.651068389415741,
      "logps/chosen": -1146.99755859375,
      "logps/rejected": -685.36865234375,
      "loss": 0.688,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008217239752411842,
      "rewards/margins": 0.010897446423768997,
      "rewards/rejected": -0.002680206671357155,
      "step": 2532
    },
    {
      "epoch": 0.6662611583290865,
      "grad_norm": 628.0,
      "learning_rate": 1.8538011695906434e-07,
      "logits/chosen": -0.6451030969619751,
      "logits/rejected": -0.6602153182029724,
      "logps/chosen": -1298.562744140625,
      "logps/rejected": -689.6367797851562,
      "loss": 0.6869,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.001480007078498602,
      "rewards/margins": 0.014782813377678394,
      "rewards/rejected": -0.01626281812787056,
      "step": 2533
    },
    {
      "epoch": 0.6665241907642736,
      "grad_norm": 528.0,
      "learning_rate": 1.8523391812865496e-07,
      "logits/chosen": -0.662319004535675,
      "logits/rejected": -0.6681361198425293,
      "logps/chosen": -906.49560546875,
      "logps/rejected": -794.0001831054688,
      "loss": 0.6991,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0043111806735396385,
      "rewards/margins": -0.011232092045247555,
      "rewards/rejected": 0.00692090904340148,
      "step": 2534
    },
    {
      "epoch": 0.6667872231994608,
      "grad_norm": 552.0,
      "learning_rate": 1.850877192982456e-07,
      "logits/chosen": -0.6341415643692017,
      "logits/rejected": -0.6363887786865234,
      "logps/chosen": -1206.8712158203125,
      "logps/rejected": -955.5877685546875,
      "loss": 0.6894,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.006136894226074219,
      "rewards/margins": 0.008242512121796608,
      "rewards/rejected": -0.0021056169643998146,
      "step": 2535
    },
    {
      "epoch": 0.6670502556346479,
      "grad_norm": 408.0,
      "learning_rate": 1.8494152046783625e-07,
      "logits/chosen": -0.6680382490158081,
      "logits/rejected": -0.6687607169151306,
      "logps/chosen": -867.33251953125,
      "logps/rejected": -873.78564453125,
      "loss": 0.7093,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.02713184431195259,
      "rewards/margins": -0.03119387850165367,
      "rewards/rejected": 0.004062032327055931,
      "step": 2536
    },
    {
      "epoch": 0.6673132880698351,
      "grad_norm": 556.0,
      "learning_rate": 1.847953216374269e-07,
      "logits/chosen": -0.6524431109428406,
      "logits/rejected": -0.6370692849159241,
      "logps/chosen": -1006.6990966796875,
      "logps/rejected": -841.7559814453125,
      "loss": 0.6986,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.001230048481374979,
      "rewards/margins": -0.010249519720673561,
      "rewards/rejected": 0.011479568667709827,
      "step": 2537
    },
    {
      "epoch": 0.6675763205050222,
      "grad_norm": 572.0,
      "learning_rate": 1.8464912280701752e-07,
      "logits/chosen": -0.6331483125686646,
      "logits/rejected": -0.6413981914520264,
      "logps/chosen": -1282.5921630859375,
      "logps/rejected": -938.7388305664062,
      "loss": 0.6846,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.024839401245117188,
      "rewards/margins": 0.018565557897090912,
      "rewards/rejected": 0.006273842416703701,
      "step": 2538
    },
    {
      "epoch": 0.6678393529402095,
      "grad_norm": 536.0,
      "learning_rate": 1.845029239766082e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6460387110710144,
      "logps/chosen": -1001.36767578125,
      "logps/rejected": -878.8737182617188,
      "loss": 0.6927,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.002414894523099065,
      "rewards/margins": 0.0018464098684489727,
      "rewards/rejected": -0.004261303227394819,
      "step": 2539
    },
    {
      "epoch": 0.6681023853753966,
      "grad_norm": 524.0,
      "learning_rate": 1.843567251461988e-07,
      "logits/chosen": -0.6741699576377869,
      "logits/rejected": -0.6701077818870544,
      "logps/chosen": -1131.4923095703125,
      "logps/rejected": -841.6107177734375,
      "loss": 0.6986,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.017333984375,
      "rewards/margins": -0.01009283121675253,
      "rewards/rejected": -0.0072411540895700455,
      "step": 2540
    },
    {
      "epoch": 0.6683654178105838,
      "grad_norm": 640.0,
      "learning_rate": 1.8421052631578946e-07,
      "logits/chosen": -0.6894676089286804,
      "logits/rejected": -0.6850627660751343,
      "logps/chosen": -1305.29541015625,
      "logps/rejected": -1174.3248291015625,
      "loss": 0.7066,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.019202232360839844,
      "rewards/margins": -0.025506019592285156,
      "rewards/rejected": 0.006303786300122738,
      "step": 2541
    },
    {
      "epoch": 0.6686284502457709,
      "grad_norm": 544.0,
      "learning_rate": 1.8406432748538013e-07,
      "logits/chosen": -0.6379556059837341,
      "logits/rejected": -0.630673885345459,
      "logps/chosen": -1197.2197265625,
      "logps/rejected": -1229.528076171875,
      "loss": 0.6947,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.016985321417450905,
      "rewards/margins": -0.0023811336141079664,
      "rewards/rejected": -0.01460418663918972,
      "step": 2542
    },
    {
      "epoch": 0.6688914826809581,
      "grad_norm": 524.0,
      "learning_rate": 1.8391812865497075e-07,
      "logits/chosen": -0.6647757291793823,
      "logits/rejected": -0.664480984210968,
      "logps/chosen": -1270.6016845703125,
      "logps/rejected": -1112.2738037109375,
      "loss": 0.6957,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0005610473453998566,
      "rewards/margins": -0.00416984548792243,
      "rewards/rejected": 0.0047308942303061485,
      "step": 2543
    },
    {
      "epoch": 0.6691545151161452,
      "grad_norm": 572.0,
      "learning_rate": 1.837719298245614e-07,
      "logits/chosen": -0.6472330093383789,
      "logits/rejected": -0.6670860052108765,
      "logps/chosen": -1229.5355224609375,
      "logps/rejected": -1156.598876953125,
      "loss": 0.6909,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.008359813131392002,
      "rewards/margins": 0.005844116676598787,
      "rewards/rejected": -0.014203930273652077,
      "step": 2544
    },
    {
      "epoch": 0.6694175475513324,
      "grad_norm": 644.0,
      "learning_rate": 1.8362573099415204e-07,
      "logits/chosen": -0.6530901193618774,
      "logits/rejected": -0.6608686447143555,
      "logps/chosen": -1374.9228515625,
      "logps/rejected": -1026.76708984375,
      "loss": 0.6908,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0015720361843705177,
      "rewards/margins": 0.006518841255456209,
      "rewards/rejected": -0.008090877905488014,
      "step": 2545
    },
    {
      "epoch": 0.6696805799865196,
      "grad_norm": 612.0,
      "learning_rate": 1.834795321637427e-07,
      "logits/chosen": -0.6604514718055725,
      "logits/rejected": -0.6636355519294739,
      "logps/chosen": -1235.39501953125,
      "logps/rejected": -864.6480102539062,
      "loss": 0.693,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.001689148135483265,
      "rewards/margins": 0.0014474857598543167,
      "rewards/rejected": 0.00024166121147572994,
      "step": 2546
    },
    {
      "epoch": 0.6699436124217067,
      "grad_norm": 548.0,
      "learning_rate": 1.833333333333333e-07,
      "logits/chosen": -0.6714003682136536,
      "logits/rejected": -0.6627403497695923,
      "logps/chosen": -1322.63427734375,
      "logps/rejected": -1265.227783203125,
      "loss": 0.6962,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.011477090418338776,
      "rewards/margins": -0.005346393212676048,
      "rewards/rejected": -0.006130695343017578,
      "step": 2547
    },
    {
      "epoch": 0.670206644856894,
      "grad_norm": 486.0,
      "learning_rate": 1.8318713450292398e-07,
      "logits/chosen": -0.6309757232666016,
      "logits/rejected": -0.6313732862472534,
      "logps/chosen": -1310.013916015625,
      "logps/rejected": -1169.1405029296875,
      "loss": 0.6786,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.018428802490234375,
      "rewards/margins": 0.02977466583251953,
      "rewards/rejected": -0.011345863342285156,
      "step": 2548
    },
    {
      "epoch": 0.6704696772920811,
      "grad_norm": 442.0,
      "learning_rate": 1.830409356725146e-07,
      "logits/chosen": -0.6503892540931702,
      "logits/rejected": -0.65325927734375,
      "logps/chosen": -780.2728271484375,
      "logps/rejected": -583.5225219726562,
      "loss": 0.6909,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0060027106665074825,
      "rewards/margins": 0.005329038016498089,
      "rewards/rejected": -0.011331749148666859,
      "step": 2549
    },
    {
      "epoch": 0.6707327097272683,
      "grad_norm": 648.0,
      "learning_rate": 1.8289473684210527e-07,
      "logits/chosen": -0.6441823840141296,
      "logits/rejected": NaN,
      "logps/chosen": -1557.111083984375,
      "logps/rejected": -984.195068359375,
      "loss": 0.6826,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004688930697739124,
      "rewards/margins": 0.02195139043033123,
      "rewards/rejected": -0.01726245880126953,
      "step": 2550
    },
    {
      "epoch": 0.6709957421624554,
      "grad_norm": 482.0,
      "learning_rate": 1.827485380116959e-07,
      "logits/chosen": -0.6652400493621826,
      "logits/rejected": -0.6564779281616211,
      "logps/chosen": -870.3773193359375,
      "logps/rejected": -658.31689453125,
      "loss": 0.6913,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.01690692827105522,
      "rewards/margins": 0.005012225825339556,
      "rewards/rejected": 0.011894703842699528,
      "step": 2551
    },
    {
      "epoch": 0.6712587745976426,
      "grad_norm": 536.0,
      "learning_rate": 1.8260233918128654e-07,
      "logits/chosen": -0.6457812190055847,
      "logits/rejected": -0.6410222053527832,
      "logps/chosen": -1088.58447265625,
      "logps/rejected": -1010.3573608398438,
      "loss": 0.6892,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005645275115966797,
      "rewards/margins": 0.008510207757353783,
      "rewards/rejected": -0.0028649328742176294,
      "step": 2552
    },
    {
      "epoch": 0.6715218070328297,
      "grad_norm": 454.0,
      "learning_rate": 1.8245614035087719e-07,
      "logits/chosen": -0.6955446600914001,
      "logits/rejected": -0.7022078633308411,
      "logps/chosen": -1028.2540283203125,
      "logps/rejected": -735.611572265625,
      "loss": 0.6901,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.009657287038862705,
      "rewards/margins": 0.007060335949063301,
      "rewards/rejected": 0.0025969501584768295,
      "step": 2553
    },
    {
      "epoch": 0.6717848394680169,
      "grad_norm": 512.0,
      "learning_rate": 1.8230994152046783e-07,
      "logits/chosen": -0.6248042583465576,
      "logits/rejected": -0.634777307510376,
      "logps/chosen": -987.6810302734375,
      "logps/rejected": -832.318359375,
      "loss": 0.6988,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.010699653998017311,
      "rewards/margins": -0.010302925482392311,
      "rewards/rejected": -0.00039672874845564365,
      "step": 2554
    },
    {
      "epoch": 0.672047871903204,
      "grad_norm": 450.0,
      "learning_rate": 1.8216374269005845e-07,
      "logits/chosen": -0.6564825773239136,
      "logits/rejected": -0.663636326789856,
      "logps/chosen": -1137.5318603515625,
      "logps/rejected": -891.1506958007812,
      "loss": 0.6951,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.012122536078095436,
      "rewards/margins": -0.0034516346640884876,
      "rewards/rejected": -0.008670901879668236,
      "step": 2555
    },
    {
      "epoch": 0.6723109043383912,
      "grad_norm": 624.0,
      "learning_rate": 1.8201754385964912e-07,
      "logits/chosen": -0.6381446719169617,
      "logits/rejected": -0.6334151029586792,
      "logps/chosen": -1385.2125244140625,
      "logps/rejected": -1074.848876953125,
      "loss": 0.7068,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.016535568982362747,
      "rewards/margins": -0.026581192389130592,
      "rewards/rejected": 0.010045623406767845,
      "step": 2556
    },
    {
      "epoch": 0.6725739367735784,
      "grad_norm": 480.0,
      "learning_rate": 1.8187134502923977e-07,
      "logits/chosen": -0.6482254266738892,
      "logits/rejected": -0.6449135541915894,
      "logps/chosen": -1263.8397216796875,
      "logps/rejected": -871.5711059570312,
      "loss": 0.6848,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02496357075870037,
      "rewards/margins": 0.01779809035360813,
      "rewards/rejected": 0.00716547854244709,
      "step": 2557
    },
    {
      "epoch": 0.6728369692087656,
      "grad_norm": 480.0,
      "learning_rate": 1.817251461988304e-07,
      "logits/chosen": -0.6662725210189819,
      "logits/rejected": -0.6715227961540222,
      "logps/chosen": -767.4185791015625,
      "logps/rejected": -830.5485229492188,
      "loss": 0.6839,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005254841409623623,
      "rewards/margins": 0.019102908670902252,
      "rewards/rejected": -0.013848066329956055,
      "step": 2558
    },
    {
      "epoch": 0.6731000016439527,
      "grad_norm": 490.0,
      "learning_rate": 1.8157894736842106e-07,
      "logits/chosen": -0.6495141386985779,
      "logits/rejected": -0.6680509448051453,
      "logps/chosen": -1103.0555419921875,
      "logps/rejected": -965.1211547851562,
      "loss": 0.6868,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0035794260911643505,
      "rewards/margins": 0.01338410284370184,
      "rewards/rejected": -0.016963530331850052,
      "step": 2559
    },
    {
      "epoch": 0.6733630340791399,
      "grad_norm": 592.0,
      "learning_rate": 1.8143274853801168e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.649153470993042,
      "logps/chosen": -1520.98193359375,
      "logps/rejected": -1282.890380859375,
      "loss": 0.6909,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.006897544953972101,
      "rewards/margins": 0.005061244126409292,
      "rewards/rejected": -0.011958790011703968,
      "step": 2560
    },
    {
      "epoch": 0.673626066514327,
      "grad_norm": 488.0,
      "learning_rate": 1.8128654970760233e-07,
      "logits/chosen": -0.6629458069801331,
      "logits/rejected": -0.6644704341888428,
      "logps/chosen": -1194.1044921875,
      "logps/rejected": -885.4844970703125,
      "loss": 0.6928,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.008757877163589,
      "rewards/margins": 0.0013977530179545283,
      "rewards/rejected": -0.010155631229281425,
      "step": 2561
    },
    {
      "epoch": 0.6738890989495142,
      "grad_norm": 564.0,
      "learning_rate": 1.8114035087719298e-07,
      "logits/chosen": -0.6565994024276733,
      "logits/rejected": -0.6732109189033508,
      "logps/chosen": -1487.7093505859375,
      "logps/rejected": -1073.7818603515625,
      "loss": 0.7011,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.022992707788944244,
      "rewards/margins": -0.015269089490175247,
      "rewards/rejected": -0.007723617367446423,
      "step": 2562
    },
    {
      "epoch": 0.6741521313847014,
      "grad_norm": 616.0,
      "learning_rate": 1.8099415204678362e-07,
      "logits/chosen": -0.664306640625,
      "logits/rejected": -0.6644791960716248,
      "logps/chosen": -1132.6961669921875,
      "logps/rejected": -1021.7442626953125,
      "loss": 0.6861,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02313671074807644,
      "rewards/margins": 0.014836693182587624,
      "rewards/rejected": 0.008300017565488815,
      "step": 2563
    },
    {
      "epoch": 0.6744151638198885,
      "grad_norm": 604.0,
      "learning_rate": 1.8084795321637424e-07,
      "logits/chosen": -0.6496307849884033,
      "logits/rejected": NaN,
      "logps/chosen": -1091.61474609375,
      "logps/rejected": -907.8250732421875,
      "loss": 0.6968,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.023510266095399857,
      "rewards/margins": -0.006878567859530449,
      "rewards/rejected": 0.030388832092285156,
      "step": 2564
    },
    {
      "epoch": 0.6746781962550757,
      "grad_norm": 572.0,
      "learning_rate": 1.8070175438596492e-07,
      "logits/chosen": -0.6369785666465759,
      "logits/rejected": -0.636480987071991,
      "logps/chosen": -1117.4420166015625,
      "logps/rejected": -936.2134399414062,
      "loss": 0.688,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.02114086225628853,
      "rewards/margins": 0.010855148546397686,
      "rewards/rejected": 0.010285710915923119,
      "step": 2565
    },
    {
      "epoch": 0.6749412286902629,
      "grad_norm": 482.0,
      "learning_rate": 1.8055555555555554e-07,
      "logits/chosen": -0.6587521433830261,
      "logits/rejected": -0.6724913716316223,
      "logps/chosen": -1018.190185546875,
      "logps/rejected": -766.869384765625,
      "loss": 0.6888,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0026724333874881268,
      "rewards/margins": 0.009251071140170097,
      "rewards/rejected": -0.006578635890036821,
      "step": 2566
    },
    {
      "epoch": 0.6752042611254501,
      "grad_norm": 532.0,
      "learning_rate": 1.8040935672514618e-07,
      "logits/chosen": -0.6840521693229675,
      "logits/rejected": -0.6777867674827576,
      "logps/chosen": -1067.7977294921875,
      "logps/rejected": -784.219482421875,
      "loss": 0.69,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.011629867367446423,
      "rewards/margins": 0.007367516867816448,
      "rewards/rejected": 0.004262352362275124,
      "step": 2567
    },
    {
      "epoch": 0.6754672935606372,
      "grad_norm": 474.0,
      "learning_rate": 1.8026315789473683e-07,
      "logits/chosen": -0.6329525113105774,
      "logits/rejected": -0.6370811462402344,
      "logps/chosen": -1178.909423828125,
      "logps/rejected": -970.798828125,
      "loss": 0.6729,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.026966668665409088,
      "rewards/margins": 0.04223346710205078,
      "rewards/rejected": -0.015266800299286842,
      "step": 2568
    },
    {
      "epoch": 0.6757303259958244,
      "grad_norm": 604.0,
      "learning_rate": 1.8011695906432747e-07,
      "logits/chosen": -0.6426510810852051,
      "logits/rejected": -0.6349800825119019,
      "logps/chosen": -1121.060302734375,
      "logps/rejected": -970.4880981445312,
      "loss": 0.7047,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.02795424498617649,
      "rewards/margins": -0.022332336753606796,
      "rewards/rejected": -0.0056219096295535564,
      "step": 2569
    },
    {
      "epoch": 0.6759933584310115,
      "grad_norm": 480.0,
      "learning_rate": 1.7997076023391812e-07,
      "logits/chosen": -0.6502484083175659,
      "logits/rejected": NaN,
      "logps/chosen": -941.2312622070312,
      "logps/rejected": -730.8341064453125,
      "loss": 0.707,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.008924674242734909,
      "rewards/margins": -0.02652301825582981,
      "rewards/rejected": 0.017598342150449753,
      "step": 2570
    },
    {
      "epoch": 0.6762563908661987,
      "grad_norm": 502.0,
      "learning_rate": 1.7982456140350877e-07,
      "logits/chosen": -0.6400399804115295,
      "logits/rejected": -0.6546875238418579,
      "logps/chosen": -929.2113647460938,
      "logps/rejected": -785.7130737304688,
      "loss": 0.6982,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.018267346546053886,
      "rewards/margins": -0.009547518566250801,
      "rewards/rejected": -0.008719825185835361,
      "step": 2571
    },
    {
      "epoch": 0.6765194233013858,
      "grad_norm": 552.0,
      "learning_rate": 1.7967836257309941e-07,
      "logits/chosen": -0.6528176665306091,
      "logits/rejected": -0.6378264427185059,
      "logps/chosen": -1234.9769287109375,
      "logps/rejected": -1088.482177734375,
      "loss": 0.6905,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.001471137860789895,
      "rewards/margins": 0.005873773712664843,
      "rewards/rejected": -0.004402637016028166,
      "step": 2572
    },
    {
      "epoch": 0.676782455736573,
      "grad_norm": 580.0,
      "learning_rate": 1.7953216374269006e-07,
      "logits/chosen": -0.6729887127876282,
      "logits/rejected": -0.6901230216026306,
      "logps/chosen": -1539.912841796875,
      "logps/rejected": -963.122802734375,
      "loss": 0.68,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.023344233632087708,
      "rewards/margins": 0.028084568679332733,
      "rewards/rejected": -0.004740334115922451,
      "step": 2573
    },
    {
      "epoch": 0.6770454881717601,
      "grad_norm": 552.0,
      "learning_rate": 1.793859649122807e-07,
      "logits/chosen": -0.6363253593444824,
      "logits/rejected": -0.6468588709831238,
      "logps/chosen": -1055.608154296875,
      "logps/rejected": -667.7079467773438,
      "loss": 0.7081,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.022667264565825462,
      "rewards/margins": -0.028980014845728874,
      "rewards/rejected": 0.006312753073871136,
      "step": 2574
    },
    {
      "epoch": 0.6773085206069474,
      "grad_norm": 536.0,
      "learning_rate": 1.7923976608187133e-07,
      "logits/chosen": -0.6686968803405762,
      "logits/rejected": -0.6795003414154053,
      "logps/chosen": -1583.6865234375,
      "logps/rejected": -1353.993896484375,
      "loss": 0.6923,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.021053219214081764,
      "rewards/margins": 0.0024460782296955585,
      "rewards/rejected": 0.018607137724757195,
      "step": 2575
    },
    {
      "epoch": 0.6775715530421345,
      "grad_norm": 528.0,
      "learning_rate": 1.79093567251462e-07,
      "logits/chosen": -0.6671249866485596,
      "logits/rejected": -0.6567728519439697,
      "logps/chosen": -1123.5538330078125,
      "logps/rejected": -848.74365234375,
      "loss": 0.6926,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0007375718560069799,
      "rewards/margins": 0.001758100464940071,
      "rewards/rejected": -0.002495669759809971,
      "step": 2576
    },
    {
      "epoch": 0.6778345854773217,
      "grad_norm": 592.0,
      "learning_rate": 1.7894736842105262e-07,
      "logits/chosen": -0.6532186269760132,
      "logits/rejected": -0.653377115726471,
      "logps/chosen": -1218.5264892578125,
      "logps/rejected": -1013.6592407226562,
      "loss": 0.6893,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01623067818582058,
      "rewards/margins": 0.008310318924486637,
      "rewards/rejected": 0.007920358330011368,
      "step": 2577
    },
    {
      "epoch": 0.6780976179125089,
      "grad_norm": 440.0,
      "learning_rate": 1.7880116959064327e-07,
      "logits/chosen": -0.6550713181495667,
      "logits/rejected": -0.6622418165206909,
      "logps/chosen": -1037.245361328125,
      "logps/rejected": -861.7142944335938,
      "loss": 0.6972,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 5.9653655625879765e-05,
      "rewards/margins": -0.00733141927048564,
      "rewards/rejected": 0.007391071878373623,
      "step": 2578
    },
    {
      "epoch": 0.678360650347696,
      "grad_norm": 636.0,
      "learning_rate": 1.786549707602339e-07,
      "logits/chosen": -0.6426979303359985,
      "logits/rejected": -0.6617500185966492,
      "logps/chosen": -1032.4798583984375,
      "logps/rejected": -746.4656982421875,
      "loss": 0.696,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0005370136350393295,
      "rewards/margins": -0.005060482304543257,
      "rewards/rejected": 0.005597496870905161,
      "step": 2579
    },
    {
      "epoch": 0.6786236827828832,
      "grad_norm": 532.0,
      "learning_rate": 1.7850877192982456e-07,
      "logits/chosen": -0.6520968079566956,
      "logits/rejected": -0.64825838804245,
      "logps/chosen": -994.9453125,
      "logps/rejected": -876.505859375,
      "loss": 0.6965,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.002122592879459262,
      "rewards/margins": -0.006393909454345703,
      "rewards/rejected": 0.004271316342055798,
      "step": 2580
    },
    {
      "epoch": 0.6788867152180703,
      "grad_norm": 474.0,
      "learning_rate": 1.7836257309941518e-07,
      "logits/chosen": -0.6552681922912598,
      "logits/rejected": -0.6627440452575684,
      "logps/chosen": -929.5455322265625,
      "logps/rejected": -607.55419921875,
      "loss": 0.7017,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.016570281237363815,
      "rewards/margins": -0.015849687159061432,
      "rewards/rejected": -0.0007205968722701073,
      "step": 2581
    },
    {
      "epoch": 0.6791497476532575,
      "grad_norm": 520.0,
      "learning_rate": 1.7821637426900585e-07,
      "logits/chosen": -0.6702927947044373,
      "logits/rejected": -0.674415647983551,
      "logps/chosen": -820.5868530273438,
      "logps/rejected": -635.0540161132812,
      "loss": 0.686,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.010760211385786533,
      "rewards/margins": 0.014992667362093925,
      "rewards/rejected": -0.00423245457932353,
      "step": 2582
    },
    {
      "epoch": 0.6794127800884446,
      "grad_norm": 572.0,
      "learning_rate": 1.7807017543859647e-07,
      "logits/chosen": -0.6261100769042969,
      "logits/rejected": -0.6284269094467163,
      "logps/chosen": -1088.412109375,
      "logps/rejected": -1010.6591796875,
      "loss": 0.7052,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.010517550632357597,
      "rewards/margins": -0.02263178862631321,
      "rewards/rejected": 0.03314933925867081,
      "step": 2583
    },
    {
      "epoch": 0.6796758125236319,
      "grad_norm": 604.0,
      "learning_rate": 1.7792397660818712e-07,
      "logits/chosen": -0.6524789333343506,
      "logits/rejected": -0.6555595397949219,
      "logps/chosen": -1293.1597900390625,
      "logps/rejected": -1186.702880859375,
      "loss": 0.6937,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009717511013150215,
      "rewards/margins": -0.0004228106699883938,
      "rewards/rejected": -0.009294699877500534,
      "step": 2584
    },
    {
      "epoch": 0.679938844958819,
      "grad_norm": 680.0,
      "learning_rate": 1.7777777777777776e-07,
      "logits/chosen": -0.6584887504577637,
      "logits/rejected": -0.6722382307052612,
      "logps/chosen": -1322.0416259765625,
      "logps/rejected": -1226.22021484375,
      "loss": 0.7011,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.016272544860839844,
      "rewards/margins": -0.014113235287368298,
      "rewards/rejected": -0.0021593086421489716,
      "step": 2585
    },
    {
      "epoch": 0.6802018773940062,
      "grad_norm": 608.0,
      "learning_rate": 1.776315789473684e-07,
      "logits/chosen": -0.6467815637588501,
      "logits/rejected": -0.6533858776092529,
      "logps/chosen": -1177.4716796875,
      "logps/rejected": -1067.4847412109375,
      "loss": 0.692,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006802654825150967,
      "rewards/margins": 0.004854774102568626,
      "rewards/rejected": 0.0019478783942759037,
      "step": 2586
    },
    {
      "epoch": 0.6804649098291933,
      "grad_norm": 620.0,
      "learning_rate": 1.7748538011695906e-07,
      "logits/chosen": -0.6286081075668335,
      "logits/rejected": -0.6381439566612244,
      "logps/chosen": -1288.6446533203125,
      "logps/rejected": -1038.445068359375,
      "loss": 0.6932,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0035452847369015217,
      "rewards/margins": 0.0009936322458088398,
      "rewards/rejected": 0.0025516515597701073,
      "step": 2587
    },
    {
      "epoch": 0.6807279422643805,
      "grad_norm": 488.0,
      "learning_rate": 1.773391812865497e-07,
      "logits/chosen": -0.6607728004455566,
      "logits/rejected": -0.6662877798080444,
      "logps/chosen": -1012.5003662109375,
      "logps/rejected": -695.5357666015625,
      "loss": 0.6919,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.018166348338127136,
      "rewards/margins": 0.0030373577028512955,
      "rewards/rejected": 0.01512899436056614,
      "step": 2588
    },
    {
      "epoch": 0.6809909746995676,
      "grad_norm": 486.0,
      "learning_rate": 1.7719298245614035e-07,
      "logits/chosen": -0.6643304824829102,
      "logits/rejected": -0.6692365407943726,
      "logps/chosen": -915.9443969726562,
      "logps/rejected": -847.5911254882812,
      "loss": 0.6969,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0007575994823127985,
      "rewards/margins": -0.007110308855772018,
      "rewards/rejected": 0.0063527110032737255,
      "step": 2589
    },
    {
      "epoch": 0.6812540071347548,
      "grad_norm": 692.0,
      "learning_rate": 1.77046783625731e-07,
      "logits/chosen": -0.6621668934822083,
      "logits/rejected": -0.680962324142456,
      "logps/chosen": -1867.2537841796875,
      "logps/rejected": -1315.8443603515625,
      "loss": 0.6911,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.010718153789639473,
      "rewards/margins": 0.00439529400318861,
      "rewards/rejected": -0.015113448724150658,
      "step": 2590
    },
    {
      "epoch": 0.6815170395699419,
      "grad_norm": 520.0,
      "learning_rate": 1.7690058479532164e-07,
      "logits/chosen": -0.638093113899231,
      "logits/rejected": -0.6454176902770996,
      "logps/chosen": -1241.830810546875,
      "logps/rejected": -1002.1646118164062,
      "loss": 0.6918,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.02127647213637829,
      "rewards/margins": 0.0031396858394145966,
      "rewards/rejected": -0.024416161701083183,
      "step": 2591
    },
    {
      "epoch": 0.6817800720051291,
      "grad_norm": 532.0,
      "learning_rate": 1.7675438596491226e-07,
      "logits/chosen": -0.6338937878608704,
      "logits/rejected": -0.6462991833686829,
      "logps/chosen": -1096.8961181640625,
      "logps/rejected": -879.4561767578125,
      "loss": 0.6919,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0032754894345998764,
      "rewards/margins": 0.0034211152233183384,
      "rewards/rejected": -0.006696605123579502,
      "step": 2592
    },
    {
      "epoch": 0.6820431044403164,
      "grad_norm": 564.0,
      "learning_rate": 1.7660818713450293e-07,
      "logits/chosen": -0.656765878200531,
      "logits/rejected": -0.6551672220230103,
      "logps/chosen": -1213.9794921875,
      "logps/rejected": -793.4441528320312,
      "loss": 0.6951,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007199096959084272,
      "rewards/margins": -0.0034744255244731903,
      "rewards/rejected": 0.0106735248118639,
      "step": 2593
    },
    {
      "epoch": 0.6823061368755035,
      "grad_norm": 528.0,
      "learning_rate": 1.7646198830409355e-07,
      "logits/chosen": -0.6541402339935303,
      "logits/rejected": -0.6733601093292236,
      "logps/chosen": -1072.119384765625,
      "logps/rejected": -695.946044921875,
      "loss": 0.6997,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.022533465176820755,
      "rewards/margins": -0.01183333620429039,
      "rewards/rejected": -0.010700130835175514,
      "step": 2594
    },
    {
      "epoch": 0.6825691693106907,
      "grad_norm": 452.0,
      "learning_rate": 1.763157894736842e-07,
      "logits/chosen": -0.6334986686706543,
      "logits/rejected": -0.6469516754150391,
      "logps/chosen": -1354.6009521484375,
      "logps/rejected": -1070.8023681640625,
      "loss": 0.6913,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.020074937492609024,
      "rewards/margins": 0.004184341989457607,
      "rewards/rejected": 0.01589059829711914,
      "step": 2595
    },
    {
      "epoch": 0.6828322017458778,
      "grad_norm": 520.0,
      "learning_rate": 1.7616959064327485e-07,
      "logits/chosen": -0.6589939594268799,
      "logits/rejected": -0.6612581610679626,
      "logps/chosen": -845.99169921875,
      "logps/rejected": -949.689453125,
      "loss": 0.6948,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009970664978027344,
      "rewards/margins": -0.0023657800629734993,
      "rewards/rejected": -0.007604885846376419,
      "step": 2596
    },
    {
      "epoch": 0.683095234181065,
      "grad_norm": 552.0,
      "learning_rate": 1.760233918128655e-07,
      "logits/chosen": -0.6671252846717834,
      "logits/rejected": -0.6473062634468079,
      "logps/chosen": -1245.9404296875,
      "logps/rejected": -1168.8831787109375,
      "loss": 0.6975,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.01585274003446102,
      "rewards/margins": -0.007823850028216839,
      "rewards/rejected": -0.008028889074921608,
      "step": 2597
    },
    {
      "epoch": 0.6833582666162521,
      "grad_norm": 442.0,
      "learning_rate": 1.7587719298245611e-07,
      "logits/chosen": -0.6611535549163818,
      "logits/rejected": -0.6579986214637756,
      "logps/chosen": -812.8536376953125,
      "logps/rejected": -562.0006713867188,
      "loss": 0.6905,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008153343573212624,
      "rewards/margins": 0.00566072529181838,
      "rewards/rejected": 0.0024926187470555305,
      "step": 2598
    },
    {
      "epoch": 0.6836212990514393,
      "grad_norm": 498.0,
      "learning_rate": 1.7573099415204679e-07,
      "logits/chosen": -0.6569631099700928,
      "logits/rejected": -0.6657888293266296,
      "logps/chosen": -1180.02197265625,
      "logps/rejected": -912.8524169921875,
      "loss": 0.6874,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0014986996538937092,
      "rewards/margins": 0.012346028350293636,
      "rewards/rejected": -0.013844728469848633,
      "step": 2599
    },
    {
      "epoch": 0.6838843314866264,
      "grad_norm": 454.0,
      "learning_rate": 1.755847953216374e-07,
      "logits/chosen": -0.6711715459823608,
      "logits/rejected": -0.6717315316200256,
      "logps/chosen": -1143.81640625,
      "logps/rejected": -881.7138061523438,
      "loss": 0.6924,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.02014932595193386,
      "rewards/margins": 0.002119349082931876,
      "rewards/rejected": -0.02226867713034153,
      "step": 2600
    },
    {
      "epoch": 0.6841473639218136,
      "grad_norm": 512.0,
      "learning_rate": 1.7543859649122805e-07,
      "logits/chosen": -0.6491185426712036,
      "logits/rejected": -0.6688830852508545,
      "logps/chosen": -942.535400390625,
      "logps/rejected": -792.1473388671875,
      "loss": 0.7073,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.023563288152217865,
      "rewards/margins": -0.027194594964385033,
      "rewards/rejected": 0.003631305880844593,
      "step": 2601
    },
    {
      "epoch": 0.6844103963570007,
      "grad_norm": 592.0,
      "learning_rate": 1.7529239766081873e-07,
      "logits/chosen": -0.6581214666366577,
      "logits/rejected": -0.6657259464263916,
      "logps/chosen": -1348.2666015625,
      "logps/rejected": -1085.846435546875,
      "loss": 0.6979,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01932082325220108,
      "rewards/margins": -0.008046484552323818,
      "rewards/rejected": 0.027367305010557175,
      "step": 2602
    },
    {
      "epoch": 0.684673428792188,
      "grad_norm": 604.0,
      "learning_rate": 1.7514619883040935e-07,
      "logits/chosen": -0.6497529745101929,
      "logits/rejected": -0.651394248008728,
      "logps/chosen": -997.7003784179688,
      "logps/rejected": -824.9900512695312,
      "loss": 0.6987,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.005501365754753351,
      "rewards/margins": -0.01029958575963974,
      "rewards/rejected": 0.004798221867531538,
      "step": 2603
    },
    {
      "epoch": 0.6849364612273751,
      "grad_norm": 648.0,
      "learning_rate": 1.75e-07,
      "logits/chosen": -0.6375795602798462,
      "logits/rejected": -0.6412901282310486,
      "logps/chosen": -1310.5164794921875,
      "logps/rejected": -1070.9066162109375,
      "loss": 0.6812,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02043437771499157,
      "rewards/margins": 0.025006389245390892,
      "rewards/rejected": -0.0045720115303993225,
      "step": 2604
    },
    {
      "epoch": 0.6851994936625623,
      "grad_norm": 524.0,
      "learning_rate": 1.7485380116959064e-07,
      "logits/chosen": -0.6566619277000427,
      "logits/rejected": -0.653830885887146,
      "logps/chosen": -1053.042236328125,
      "logps/rejected": -766.3473510742188,
      "loss": 0.6975,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004414845257997513,
      "rewards/margins": -0.007921600714325905,
      "rewards/rejected": 0.012336445041000843,
      "step": 2605
    },
    {
      "epoch": 0.6854625260977494,
      "grad_norm": 532.0,
      "learning_rate": 1.7470760233918128e-07,
      "logits/chosen": -0.666443943977356,
      "logits/rejected": -0.6768444180488586,
      "logps/chosen": -1043.7327880859375,
      "logps/rejected": -904.2237548828125,
      "loss": 0.6985,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.007715273182839155,
      "rewards/margins": -0.00977177731692791,
      "rewards/rejected": 0.0020565036684274673,
      "step": 2606
    },
    {
      "epoch": 0.6857255585329366,
      "grad_norm": 564.0,
      "learning_rate": 1.745614035087719e-07,
      "logits/chosen": -0.6556258797645569,
      "logits/rejected": -0.6571499705314636,
      "logps/chosen": -888.1942138671875,
      "logps/rejected": -858.8865356445312,
      "loss": 0.694,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01382436789572239,
      "rewards/margins": -0.0008882516995072365,
      "rewards/rejected": 0.0147126205265522,
      "step": 2607
    },
    {
      "epoch": 0.6859885909681237,
      "grad_norm": 488.0,
      "learning_rate": 1.7441520467836258e-07,
      "logits/chosen": -0.6471810340881348,
      "logits/rejected": -0.6592945456504822,
      "logps/chosen": -986.79296875,
      "logps/rejected": -731.095947265625,
      "loss": 0.6871,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012972448952496052,
      "rewards/margins": 0.012730312533676624,
      "rewards/rejected": 0.0002421378158032894,
      "step": 2608
    },
    {
      "epoch": 0.6862516234033109,
      "grad_norm": 672.0,
      "learning_rate": 1.742690058479532e-07,
      "logits/chosen": -0.6607182025909424,
      "logits/rejected": -0.668997049331665,
      "logps/chosen": -1360.3642578125,
      "logps/rejected": -1259.141845703125,
      "loss": 0.6801,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.015361309051513672,
      "rewards/margins": 0.0276242233812809,
      "rewards/rejected": -0.012262916192412376,
      "step": 2609
    },
    {
      "epoch": 0.6865146558384981,
      "grad_norm": 540.0,
      "learning_rate": 1.7412280701754384e-07,
      "logits/chosen": -0.6642789244651794,
      "logits/rejected": -0.6614712476730347,
      "logps/chosen": -1273.1087646484375,
      "logps/rejected": -1277.4007568359375,
      "loss": 0.6858,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.010639953427016735,
      "rewards/margins": 0.01603693887591362,
      "rewards/rejected": -0.005396985448896885,
      "step": 2610
    },
    {
      "epoch": 0.6867776882736852,
      "grad_norm": 616.0,
      "learning_rate": 1.739766081871345e-07,
      "logits/chosen": -0.6855452060699463,
      "logits/rejected": -0.6824914813041687,
      "logps/chosen": -1195.0450439453125,
      "logps/rejected": -979.1802978515625,
      "loss": 0.6866,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.028938913717865944,
      "rewards/margins": 0.014757444150745869,
      "rewards/rejected": 0.01418147049844265,
      "step": 2611
    },
    {
      "epoch": 0.6870407207088725,
      "grad_norm": 552.0,
      "learning_rate": 1.7383040935672514e-07,
      "logits/chosen": -0.6569883823394775,
      "logits/rejected": -0.6702904105186462,
      "logps/chosen": -1144.8865966796875,
      "logps/rejected": -741.0120239257812,
      "loss": 0.7007,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0028257854282855988,
      "rewards/margins": -0.014072895050048828,
      "rewards/rejected": 0.016898680478334427,
      "step": 2612
    },
    {
      "epoch": 0.6873037531440596,
      "grad_norm": 458.0,
      "learning_rate": 1.7368421052631578e-07,
      "logits/chosen": -0.6391347646713257,
      "logits/rejected": -0.631706178188324,
      "logps/chosen": -917.8595581054688,
      "logps/rejected": -778.1911010742188,
      "loss": 0.682,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.02472362294793129,
      "rewards/margins": 0.024473384022712708,
      "rewards/rejected": 0.0002502439310774207,
      "step": 2613
    },
    {
      "epoch": 0.6875667855792468,
      "grad_norm": 516.0,
      "learning_rate": 1.7353801169590643e-07,
      "logits/chosen": -0.6492421627044678,
      "logits/rejected": -0.6556302905082703,
      "logps/chosen": -1012.7456665039062,
      "logps/rejected": -729.5426635742188,
      "loss": 0.6983,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.0013745310716331005,
      "rewards/margins": -0.00951995700597763,
      "rewards/rejected": 0.010894488543272018,
      "step": 2614
    },
    {
      "epoch": 0.6878298180144339,
      "grad_norm": 536.0,
      "learning_rate": 1.7339181286549705e-07,
      "logits/chosen": -0.6473304629325867,
      "logits/rejected": -0.6485757827758789,
      "logps/chosen": -1370.2977294921875,
      "logps/rejected": -907.4951782226562,
      "loss": 0.6924,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.011621476151049137,
      "rewards/margins": 0.0018857958493754268,
      "rewards/rejected": 0.009735679253935814,
      "step": 2615
    },
    {
      "epoch": 0.6880928504496211,
      "grad_norm": 478.0,
      "learning_rate": 1.7324561403508772e-07,
      "logits/chosen": -0.6560674905776978,
      "logits/rejected": -0.6573900580406189,
      "logps/chosen": -1089.6939697265625,
      "logps/rejected": -750.3167114257812,
      "loss": 0.6884,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.006820297800004482,
      "rewards/margins": 0.00986933708190918,
      "rewards/rejected": -0.016689633950591087,
      "step": 2616
    },
    {
      "epoch": 0.6883558828848082,
      "grad_norm": 506.0,
      "learning_rate": 1.7309941520467837e-07,
      "logits/chosen": -0.6521254777908325,
      "logits/rejected": -0.6571142673492432,
      "logps/chosen": -972.430908203125,
      "logps/rejected": -848.623291015625,
      "loss": 0.6898,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.017354819923639297,
      "rewards/margins": 0.007243204861879349,
      "rewards/rejected": 0.010111617855727673,
      "step": 2617
    },
    {
      "epoch": 0.6886189153199954,
      "grad_norm": 640.0,
      "learning_rate": 1.72953216374269e-07,
      "logits/chosen": -0.6487240791320801,
      "logits/rejected": -0.6543018817901611,
      "logps/chosen": -1359.571533203125,
      "logps/rejected": -1192.738525390625,
      "loss": 0.6876,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0115598663687706,
      "rewards/margins": 0.012851428240537643,
      "rewards/rejected": -0.0012915609404444695,
      "step": 2618
    },
    {
      "epoch": 0.6888819477551825,
      "grad_norm": 560.0,
      "learning_rate": 1.7280701754385966e-07,
      "logits/chosen": -0.6516550779342651,
      "logits/rejected": -0.6584345698356628,
      "logps/chosen": -1146.8065185546875,
      "logps/rejected": -726.9389038085938,
      "loss": 0.7017,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.010731029324233532,
      "rewards/margins": -0.016025066375732422,
      "rewards/rejected": 0.0052940379828214645,
      "step": 2619
    },
    {
      "epoch": 0.6891449801903697,
      "grad_norm": 556.0,
      "learning_rate": 1.7266081871345028e-07,
      "logits/chosen": -0.6649938225746155,
      "logits/rejected": -0.6773594617843628,
      "logps/chosen": -1151.30712890625,
      "logps/rejected": -731.2389526367188,
      "loss": 0.6981,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0008961199782788754,
      "rewards/margins": -0.00943074282258749,
      "rewards/rejected": 0.010326864197850227,
      "step": 2620
    },
    {
      "epoch": 0.6894080126255568,
      "grad_norm": 560.0,
      "learning_rate": 1.7251461988304093e-07,
      "logits/chosen": -0.634190559387207,
      "logits/rejected": -0.6447445154190063,
      "logps/chosen": -1147.6246337890625,
      "logps/rejected": -1034.0577392578125,
      "loss": 0.6876,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00256948359310627,
      "rewards/margins": 0.011927985586225986,
      "rewards/rejected": -0.009358502924442291,
      "step": 2621
    },
    {
      "epoch": 0.6896710450607441,
      "grad_norm": 556.0,
      "learning_rate": 1.7236842105263157e-07,
      "logits/chosen": -0.679141640663147,
      "logits/rejected": -0.6851596832275391,
      "logps/chosen": -1366.8135986328125,
      "logps/rejected": -1136.374267578125,
      "loss": 0.6968,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004591084085404873,
      "rewards/margins": -0.006284999195486307,
      "rewards/rejected": 0.010876083746552467,
      "step": 2622
    },
    {
      "epoch": 0.6899340774959312,
      "grad_norm": 640.0,
      "learning_rate": 1.7222222222222222e-07,
      "logits/chosen": -0.6755372285842896,
      "logits/rejected": -0.6980873346328735,
      "logps/chosen": -1291.140625,
      "logps/rejected": -1102.166259765625,
      "loss": 0.6893,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0016707414761185646,
      "rewards/margins": 0.008199691772460938,
      "rewards/rejected": -0.006528949365019798,
      "step": 2623
    },
    {
      "epoch": 0.6901971099311184,
      "grad_norm": 424.0,
      "learning_rate": 1.7207602339181284e-07,
      "logits/chosen": -0.6530894041061401,
      "logits/rejected": -0.656785249710083,
      "logps/chosen": -1156.5567626953125,
      "logps/rejected": -777.4067993164062,
      "loss": 0.7038,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.021533679217100143,
      "rewards/margins": -0.020232107490301132,
      "rewards/rejected": -0.0013015749864280224,
      "step": 2624
    },
    {
      "epoch": 0.6904601423663056,
      "grad_norm": 512.0,
      "learning_rate": 1.719298245614035e-07,
      "logits/chosen": -0.6485044360160828,
      "logits/rejected": -0.6631413102149963,
      "logps/chosen": -1016.0335693359375,
      "logps/rejected": -754.713134765625,
      "loss": 0.7006,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0064674378372728825,
      "rewards/margins": -0.014141322113573551,
      "rewards/rejected": 0.007673883810639381,
      "step": 2625
    },
    {
      "epoch": 0.6907231748014927,
      "grad_norm": 560.0,
      "learning_rate": 1.7178362573099413e-07,
      "logits/chosen": -0.6525874137878418,
      "logits/rejected": -0.6518059968948364,
      "logps/chosen": -1234.2755126953125,
      "logps/rejected": -998.8113403320312,
      "loss": 0.6888,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.005736255086958408,
      "rewards/margins": 0.009563636034727097,
      "rewards/rejected": -0.015299891121685505,
      "step": 2626
    },
    {
      "epoch": 0.6909862072366799,
      "grad_norm": 608.0,
      "learning_rate": 1.7163742690058478e-07,
      "logits/chosen": -0.6578863859176636,
      "logits/rejected": -0.6594116687774658,
      "logps/chosen": -1396.8519287109375,
      "logps/rejected": -963.9940185546875,
      "loss": 0.7005,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003274344839155674,
      "rewards/margins": -0.012779236771166325,
      "rewards/rejected": 0.016053583472967148,
      "step": 2627
    },
    {
      "epoch": 0.691249239671867,
      "grad_norm": 492.0,
      "learning_rate": 1.7149122807017543e-07,
      "logits/chosen": -0.6398900747299194,
      "logits/rejected": -0.6382143497467041,
      "logps/chosen": -1053.3560791015625,
      "logps/rejected": -1111.938720703125,
      "loss": 0.6769,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02573576010763645,
      "rewards/margins": 0.03374910354614258,
      "rewards/rejected": -0.008013344369828701,
      "step": 2628
    },
    {
      "epoch": 0.6915122721070542,
      "grad_norm": 528.0,
      "learning_rate": 1.7134502923976607e-07,
      "logits/chosen": -0.6887590289115906,
      "logits/rejected": NaN,
      "logps/chosen": -1102.387451171875,
      "logps/rejected": -775.1672973632812,
      "loss": 0.6966,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.009973240084946156,
      "rewards/margins": -0.006164549849927425,
      "rewards/rejected": 0.01613778993487358,
      "step": 2629
    },
    {
      "epoch": 0.6917753045422413,
      "grad_norm": 596.0,
      "learning_rate": 1.711988304093567e-07,
      "logits/chosen": -0.6425120830535889,
      "logits/rejected": -0.6426916718482971,
      "logps/chosen": -920.6181640625,
      "logps/rejected": -818.0421142578125,
      "loss": 0.6777,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0026871678419411182,
      "rewards/margins": 0.03239230811595917,
      "rewards/rejected": -0.029705144464969635,
      "step": 2630
    },
    {
      "epoch": 0.6920383369774286,
      "grad_norm": 536.0,
      "learning_rate": 1.7105263157894736e-07,
      "logits/chosen": -0.6418213844299316,
      "logits/rejected": -0.6405131816864014,
      "logps/chosen": -984.848388671875,
      "logps/rejected": -945.728759765625,
      "loss": 0.6911,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0057915206998586655,
      "rewards/margins": 0.005159855354577303,
      "rewards/rejected": 0.0006316660437732935,
      "step": 2631
    },
    {
      "epoch": 0.6923013694126157,
      "grad_norm": 476.0,
      "learning_rate": 1.70906432748538e-07,
      "logits/chosen": -0.6510913968086243,
      "logits/rejected": -0.6493534445762634,
      "logps/chosen": -954.4729614257812,
      "logps/rejected": -696.335693359375,
      "loss": 0.6866,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005953693296760321,
      "rewards/margins": 0.013714790344238281,
      "rewards/rejected": -0.007761097513139248,
      "step": 2632
    },
    {
      "epoch": 0.6925644018478029,
      "grad_norm": 520.0,
      "learning_rate": 1.7076023391812866e-07,
      "logits/chosen": -0.6675391793251038,
      "logits/rejected": NaN,
      "logps/chosen": -1135.31689453125,
      "logps/rejected": -756.6192626953125,
      "loss": 0.7007,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.00963516253978014,
      "rewards/margins": -0.014413261786103249,
      "rewards/rejected": 0.004778098780661821,
      "step": 2633
    },
    {
      "epoch": 0.69282743428299,
      "grad_norm": 482.0,
      "learning_rate": 1.706140350877193e-07,
      "logits/chosen": -0.6326684355735779,
      "logits/rejected": -0.639183521270752,
      "logps/chosen": -1084.10546875,
      "logps/rejected": -783.77294921875,
      "loss": 0.6836,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.027678396552801132,
      "rewards/margins": 0.01985483430325985,
      "rewards/rejected": 0.007823563180863857,
      "step": 2634
    },
    {
      "epoch": 0.6930904667181772,
      "grad_norm": 556.0,
      "learning_rate": 1.7046783625730992e-07,
      "logits/chosen": -0.629315972328186,
      "logits/rejected": -0.6326960325241089,
      "logps/chosen": -1152.416259765625,
      "logps/rejected": -1066.845703125,
      "loss": 0.6898,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.009186934679746628,
      "rewards/margins": 0.007410476915538311,
      "rewards/rejected": -0.016597412526607513,
      "step": 2635
    },
    {
      "epoch": 0.6933534991533643,
      "grad_norm": 528.0,
      "learning_rate": 1.703216374269006e-07,
      "logits/chosen": -0.6417547464370728,
      "logits/rejected": -0.6624243855476379,
      "logps/chosen": -1466.0054931640625,
      "logps/rejected": -1107.0380859375,
      "loss": 0.6993,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.008932016789913177,
      "rewards/margins": -0.010984991677105427,
      "rewards/rejected": 0.0020529753528535366,
      "step": 2636
    },
    {
      "epoch": 0.6936165315885515,
      "grad_norm": 568.0,
      "learning_rate": 1.7017543859649122e-07,
      "logits/chosen": -0.6578078866004944,
      "logits/rejected": -0.6591187119483948,
      "logps/chosen": -1345.9002685546875,
      "logps/rejected": -1132.1396484375,
      "loss": 0.6998,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.021920014172792435,
      "rewards/margins": -0.012496089562773705,
      "rewards/rejected": -0.009423923678696156,
      "step": 2637
    },
    {
      "epoch": 0.6938795640237386,
      "grad_norm": 564.0,
      "learning_rate": 1.7002923976608186e-07,
      "logits/chosen": -0.6644219756126404,
      "logits/rejected": -0.6516191959381104,
      "logps/chosen": -1190.7750244140625,
      "logps/rejected": -893.7635498046875,
      "loss": 0.6943,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0011075020302087069,
      "rewards/margins": -0.0012801163829863071,
      "rewards/rejected": 0.00017261505126953125,
      "step": 2638
    },
    {
      "epoch": 0.6941425964589258,
      "grad_norm": 392.0,
      "learning_rate": 1.698830409356725e-07,
      "logits/chosen": -0.6623871326446533,
      "logits/rejected": -0.6618313789367676,
      "logps/chosen": -1149.4935302734375,
      "logps/rejected": -1201.0040283203125,
      "loss": 0.691,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.005185032729059458,
      "rewards/margins": 0.005053998902440071,
      "rewards/rejected": -0.010239029303193092,
      "step": 2639
    },
    {
      "epoch": 0.6944056288941131,
      "grad_norm": 628.0,
      "learning_rate": 1.6973684210526316e-07,
      "logits/chosen": -0.6486073136329651,
      "logits/rejected": -0.6443687081336975,
      "logps/chosen": -1316.0218505859375,
      "logps/rejected": -1051.772705078125,
      "loss": 0.7012,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.019783399999141693,
      "rewards/margins": -0.0143897058442235,
      "rewards/rejected": -0.005393695086240768,
      "step": 2640
    },
    {
      "epoch": 0.6946686613293002,
      "grad_norm": 548.0,
      "learning_rate": 1.6959064327485378e-07,
      "logits/chosen": -0.6462827920913696,
      "logits/rejected": -0.6413983106613159,
      "logps/chosen": -1273.5682373046875,
      "logps/rejected": -1053.260009765625,
      "loss": 0.6898,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.023955773562192917,
      "rewards/margins": 0.007565545849502087,
      "rewards/rejected": 0.016390228644013405,
      "step": 2641
    },
    {
      "epoch": 0.6949316937644874,
      "grad_norm": 572.0,
      "learning_rate": 1.6944444444444445e-07,
      "logits/chosen": -0.6490366458892822,
      "logits/rejected": -0.6509654521942139,
      "logps/chosen": -1223.7474365234375,
      "logps/rejected": -1118.1783447265625,
      "loss": 0.694,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0057963356375694275,
      "rewards/margins": -0.00028529390692710876,
      "rewards/rejected": -0.005511045455932617,
      "step": 2642
    },
    {
      "epoch": 0.6951947261996745,
      "grad_norm": 504.0,
      "learning_rate": 1.6929824561403507e-07,
      "logits/chosen": -0.6231969594955444,
      "logits/rejected": -0.6352795362472534,
      "logps/chosen": -781.2806396484375,
      "logps/rejected": -816.68896484375,
      "loss": 0.6967,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.024308109655976295,
      "rewards/margins": -0.0062396046705543995,
      "rewards/rejected": -0.018068505451083183,
      "step": 2643
    },
    {
      "epoch": 0.6954577586348617,
      "grad_norm": 414.0,
      "learning_rate": 1.6915204678362571e-07,
      "logits/chosen": -0.6452412605285645,
      "logits/rejected": -0.6461801528930664,
      "logps/chosen": -904.6901245117188,
      "logps/rejected": -765.7967529296875,
      "loss": 0.687,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02420954592525959,
      "rewards/margins": 0.013371134176850319,
      "rewards/rejected": 0.010838412679731846,
      "step": 2644
    },
    {
      "epoch": 0.6957207910700488,
      "grad_norm": 700.0,
      "learning_rate": 1.6900584795321636e-07,
      "logits/chosen": -0.666991651058197,
      "logits/rejected": -0.6611008048057556,
      "logps/chosen": -1255.10986328125,
      "logps/rejected": -1081.7393798828125,
      "loss": 0.6885,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.009994886815547943,
      "rewards/margins": 0.009675453417003155,
      "rewards/rejected": 0.00031943339854478836,
      "step": 2645
    },
    {
      "epoch": 0.695983823505236,
      "grad_norm": 450.0,
      "learning_rate": 1.68859649122807e-07,
      "logits/chosen": -0.6512260437011719,
      "logits/rejected": -0.6552143096923828,
      "logps/chosen": -705.7593994140625,
      "logps/rejected": -670.24658203125,
      "loss": 0.6882,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0010035515297204256,
      "rewards/margins": 0.010328960604965687,
      "rewards/rejected": -0.00932540837675333,
      "step": 2646
    },
    {
      "epoch": 0.6962468559404231,
      "grad_norm": 506.0,
      "learning_rate": 1.6871345029239765e-07,
      "logits/chosen": -0.6654328107833862,
      "logits/rejected": -0.6737309694290161,
      "logps/chosen": -1166.521484375,
      "logps/rejected": -1090.93359375,
      "loss": 0.7009,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0112838726490736,
      "rewards/margins": -0.01476964820176363,
      "rewards/rejected": 0.0034857753198593855,
      "step": 2647
    },
    {
      "epoch": 0.6965098883756103,
      "grad_norm": 532.0,
      "learning_rate": 1.685672514619883e-07,
      "logits/chosen": -0.6428027153015137,
      "logits/rejected": -0.6436770558357239,
      "logps/chosen": -1392.0059814453125,
      "logps/rejected": -1098.3861083984375,
      "loss": 0.6828,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.005873584188520908,
      "rewards/margins": 0.021993255242705345,
      "rewards/rejected": -0.01611967198550701,
      "step": 2648
    },
    {
      "epoch": 0.6967729208107974,
      "grad_norm": 416.0,
      "learning_rate": 1.6842105263157895e-07,
      "logits/chosen": -0.6627613306045532,
      "logits/rejected": -0.6730532050132751,
      "logps/chosen": -734.2318115234375,
      "logps/rejected": -664.9530639648438,
      "loss": 0.7062,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.029033759608864784,
      "rewards/margins": -0.025425434112548828,
      "rewards/rejected": -0.0036083227023482323,
      "step": 2649
    },
    {
      "epoch": 0.6970359532459847,
      "grad_norm": 470.0,
      "learning_rate": 1.6827485380116957e-07,
      "logits/chosen": -0.6362329721450806,
      "logits/rejected": -0.6452879905700684,
      "logps/chosen": -1152.3963623046875,
      "logps/rejected": -1256.13720703125,
      "loss": 0.6908,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0020300867035984993,
      "rewards/margins": 0.004922770895063877,
      "rewards/rejected": -0.006952858995646238,
      "step": 2650
    },
    {
      "epoch": 0.6972989856811718,
      "grad_norm": 390.0,
      "learning_rate": 1.6812865497076024e-07,
      "logits/chosen": -0.6551613807678223,
      "logits/rejected": -0.6563472151756287,
      "logps/chosen": -966.6697998046875,
      "logps/rejected": -636.2593994140625,
      "loss": 0.6814,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.019137287512421608,
      "rewards/margins": 0.02410917542874813,
      "rewards/rejected": -0.004971885122358799,
      "step": 2651
    },
    {
      "epoch": 0.697562018116359,
      "grad_norm": 552.0,
      "learning_rate": 1.6798245614035086e-07,
      "logits/chosen": -0.6610347628593445,
      "logits/rejected": -0.6664692163467407,
      "logps/chosen": -1065.6912841796875,
      "logps/rejected": -620.437744140625,
      "loss": 0.699,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0057763089425861835,
      "rewards/margins": -0.010319804772734642,
      "rewards/rejected": 0.0045434958301484585,
      "step": 2652
    },
    {
      "epoch": 0.6978250505515461,
      "grad_norm": 652.0,
      "learning_rate": 1.6783625730994153e-07,
      "logits/chosen": -0.641615629196167,
      "logits/rejected": -0.6370218992233276,
      "logps/chosen": -1162.32275390625,
      "logps/rejected": -794.9998168945312,
      "loss": 0.7041,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.011382484808564186,
      "rewards/margins": -0.021277476102113724,
      "rewards/rejected": 0.009894991293549538,
      "step": 2653
    },
    {
      "epoch": 0.6980880829867333,
      "grad_norm": 552.0,
      "learning_rate": 1.6769005847953215e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6481488943099976,
      "logps/chosen": -1183.27783203125,
      "logps/rejected": -1269.9564208984375,
      "loss": 0.6962,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.004146385006606579,
      "rewards/margins": -0.004994394723325968,
      "rewards/rejected": 0.009140778332948685,
      "step": 2654
    },
    {
      "epoch": 0.6983511154219205,
      "grad_norm": 560.0,
      "learning_rate": 1.675438596491228e-07,
      "logits/chosen": -0.6327568888664246,
      "logits/rejected": -0.6308062076568604,
      "logps/chosen": -1266.38330078125,
      "logps/rejected": -893.136962890625,
      "loss": 0.7007,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.007756423205137253,
      "rewards/margins": -0.01388626080006361,
      "rewards/rejected": 0.021642684936523438,
      "step": 2655
    },
    {
      "epoch": 0.6986141478571076,
      "grad_norm": 524.0,
      "learning_rate": 1.6739766081871344e-07,
      "logits/chosen": -0.6596713066101074,
      "logits/rejected": -0.6671175956726074,
      "logps/chosen": -1277.9664306640625,
      "logps/rejected": -1056.014404296875,
      "loss": 0.6946,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.005148124881088734,
      "rewards/margins": -0.002405547769740224,
      "rewards/rejected": 0.007553672883659601,
      "step": 2656
    },
    {
      "epoch": 0.6988771802922948,
      "grad_norm": 624.0,
      "learning_rate": 1.672514619883041e-07,
      "logits/chosen": -0.6631814241409302,
      "logits/rejected": -0.6462252140045166,
      "logps/chosen": -1190.81396484375,
      "logps/rejected": -1027.4791259765625,
      "loss": 0.6744,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.03136644512414932,
      "rewards/margins": 0.03945665434002876,
      "rewards/rejected": -0.008090210147202015,
      "step": 2657
    },
    {
      "epoch": 0.699140212727482,
      "grad_norm": 454.0,
      "learning_rate": 1.671052631578947e-07,
      "logits/chosen": -0.6576123237609863,
      "logits/rejected": -0.6607055068016052,
      "logps/chosen": -1041.5205078125,
      "logps/rejected": -854.3233032226562,
      "loss": 0.6998,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.010272979736328125,
      "rewards/margins": -0.012583637610077858,
      "rewards/rejected": 0.022856615483760834,
      "step": 2658
    },
    {
      "epoch": 0.6994032451626692,
      "grad_norm": 456.0,
      "learning_rate": 1.6695906432748538e-07,
      "logits/chosen": -0.6212790012359619,
      "logits/rejected": -0.6379221081733704,
      "logps/chosen": -676.9898071289062,
      "logps/rejected": -867.9517822265625,
      "loss": 0.6924,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0019687640015035868,
      "rewards/margins": 0.001643658964894712,
      "rewards/rejected": -0.003612423548474908,
      "step": 2659
    },
    {
      "epoch": 0.6996662775978563,
      "grad_norm": 450.0,
      "learning_rate": 1.66812865497076e-07,
      "logits/chosen": -0.6409100294113159,
      "logits/rejected": -0.6501747965812683,
      "logps/chosen": -1190.67333984375,
      "logps/rejected": -1235.2711181640625,
      "loss": 0.6894,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.025917910039424896,
      "rewards/margins": 0.008552835322916508,
      "rewards/rejected": 0.017365075647830963,
      "step": 2660
    },
    {
      "epoch": 0.6999293100330435,
      "grad_norm": 532.0,
      "learning_rate": 1.6666666666666665e-07,
      "logits/chosen": -0.6448707580566406,
      "logits/rejected": NaN,
      "logps/chosen": -1181.0699462890625,
      "logps/rejected": -712.9246215820312,
      "loss": 0.6871,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0023056981153786182,
      "rewards/margins": 0.013115310110151768,
      "rewards/rejected": -0.010809611529111862,
      "step": 2661
    },
    {
      "epoch": 0.7001923424682306,
      "grad_norm": 572.0,
      "learning_rate": 1.6652046783625732e-07,
      "logits/chosen": -0.6742115020751953,
      "logits/rejected": -0.675292432308197,
      "logps/chosen": -1095.743408203125,
      "logps/rejected": -709.0694580078125,
      "loss": 0.6986,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.009147359058260918,
      "rewards/margins": -0.010348033159971237,
      "rewards/rejected": 0.019495392218232155,
      "step": 2662
    },
    {
      "epoch": 0.7004553749034178,
      "grad_norm": 456.0,
      "learning_rate": 1.6637426900584794e-07,
      "logits/chosen": -0.669702410697937,
      "logits/rejected": -0.6647616624832153,
      "logps/chosen": -817.632568359375,
      "logps/rejected": -911.402587890625,
      "loss": 0.7034,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.0042716022580862045,
      "rewards/margins": -0.019766615703701973,
      "rewards/rejected": 0.024038221687078476,
      "step": 2663
    },
    {
      "epoch": 0.7007184073386049,
      "grad_norm": 528.0,
      "learning_rate": 1.662280701754386e-07,
      "logits/chosen": -0.6483110189437866,
      "logits/rejected": -0.6597796082496643,
      "logps/chosen": -1409.9788818359375,
      "logps/rejected": -1149.246337890625,
      "loss": 0.6924,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009440041147172451,
      "rewards/margins": 0.002535153180360794,
      "rewards/rejected": 0.006904887966811657,
      "step": 2664
    },
    {
      "epoch": 0.7009814397737921,
      "grad_norm": 556.0,
      "learning_rate": 1.6608187134502924e-07,
      "logits/chosen": -0.6439387798309326,
      "logits/rejected": -0.6446430683135986,
      "logps/chosen": -1126.2572021484375,
      "logps/rejected": -717.02001953125,
      "loss": 0.6998,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0031457894947379827,
      "rewards/margins": -0.012506389990448952,
      "rewards/rejected": 0.015652179718017578,
      "step": 2665
    },
    {
      "epoch": 0.7012444722089792,
      "grad_norm": 450.0,
      "learning_rate": 1.6593567251461988e-07,
      "logits/chosen": -0.6552091240882874,
      "logits/rejected": -0.6539607048034668,
      "logps/chosen": -1158.3392333984375,
      "logps/rejected": -954.2371826171875,
      "loss": 0.6994,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.005330181680619717,
      "rewards/margins": -0.011842061765491962,
      "rewards/rejected": 0.006511879153549671,
      "step": 2666
    },
    {
      "epoch": 0.7015075046441664,
      "grad_norm": 374.0,
      "learning_rate": 1.657894736842105e-07,
      "logits/chosen": -0.6391000747680664,
      "logits/rejected": -0.6401264667510986,
      "logps/chosen": -860.1273193359375,
      "logps/rejected": -724.8858642578125,
      "loss": 0.6917,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0030979150906205177,
      "rewards/margins": 0.003214930882677436,
      "rewards/rejected": -0.00011701596667990088,
      "step": 2667
    },
    {
      "epoch": 0.7017705370793536,
      "grad_norm": 568.0,
      "learning_rate": 1.6564327485380117e-07,
      "logits/chosen": -0.6606258749961853,
      "logits/rejected": -0.6550887823104858,
      "logps/chosen": -1110.9769287109375,
      "logps/rejected": -883.1051635742188,
      "loss": 0.6912,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0018848427571356297,
      "rewards/margins": 0.0047866832464933395,
      "rewards/rejected": -0.0029018400236964226,
      "step": 2668
    },
    {
      "epoch": 0.7020335695145408,
      "grad_norm": 564.0,
      "learning_rate": 1.654970760233918e-07,
      "logits/chosen": -0.646327793598175,
      "logits/rejected": -0.6495417356491089,
      "logps/chosen": -1165.129638671875,
      "logps/rejected": -930.5452880859375,
      "loss": 0.7011,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.003556252224370837,
      "rewards/margins": -0.014341259375214577,
      "rewards/rejected": 0.010785006918013096,
      "step": 2669
    },
    {
      "epoch": 0.7022966019497279,
      "grad_norm": 540.0,
      "learning_rate": 1.6535087719298244e-07,
      "logits/chosen": -0.6462218761444092,
      "logits/rejected": -0.6508486270904541,
      "logps/chosen": -1006.42626953125,
      "logps/rejected": -936.112548828125,
      "loss": 0.7025,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.011709023267030716,
      "rewards/margins": -0.01790618896484375,
      "rewards/rejected": 0.006197167094796896,
      "step": 2670
    },
    {
      "epoch": 0.7025596343849151,
      "grad_norm": 652.0,
      "learning_rate": 1.652046783625731e-07,
      "logits/chosen": -0.6306328773498535,
      "logits/rejected": NaN,
      "logps/chosen": -1294.2774658203125,
      "logps/rejected": -1091.2652587890625,
      "loss": 0.6899,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.018639564514160156,
      "rewards/margins": 0.007356642745435238,
      "rewards/rejected": 0.011282920837402344,
      "step": 2671
    },
    {
      "epoch": 0.7028226668201023,
      "grad_norm": 600.0,
      "learning_rate": 1.6505847953216373e-07,
      "logits/chosen": -0.6306108832359314,
      "logits/rejected": -0.6256356239318848,
      "logps/chosen": -1210.754150390625,
      "logps/rejected": -944.432861328125,
      "loss": 0.6784,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0028851989191025496,
      "rewards/margins": 0.03074479103088379,
      "rewards/rejected": -0.03362999111413956,
      "step": 2672
    },
    {
      "epoch": 0.7030856992552894,
      "grad_norm": 564.0,
      "learning_rate": 1.6491228070175438e-07,
      "logits/chosen": -0.6561620831489563,
      "logits/rejected": -0.6516034603118896,
      "logps/chosen": -1097.377685546875,
      "logps/rejected": -975.1513061523438,
      "loss": 0.6939,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.009811115451157093,
      "rewards/margins": -0.0007667543832212687,
      "rewards/rejected": 0.010577870532870293,
      "step": 2673
    },
    {
      "epoch": 0.7033487316904766,
      "grad_norm": 502.0,
      "learning_rate": 1.6476608187134503e-07,
      "logits/chosen": -0.6412538886070251,
      "logits/rejected": -0.6427103281021118,
      "logps/chosen": -850.919189453125,
      "logps/rejected": -826.8286743164062,
      "loss": 0.6966,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012814807705581188,
      "rewards/margins": -0.006547260098159313,
      "rewards/rejected": 0.01936206966638565,
      "step": 2674
    },
    {
      "epoch": 0.7036117641256637,
      "grad_norm": 488.0,
      "learning_rate": 1.6461988304093565e-07,
      "logits/chosen": -0.6417672038078308,
      "logits/rejected": -0.6522983312606812,
      "logps/chosen": -867.3577270507812,
      "logps/rejected": -584.4844970703125,
      "loss": 0.6876,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.014188384637236595,
      "rewards/margins": 0.011504745110869408,
      "rewards/rejected": 0.0026836395263671875,
      "step": 2675
    },
    {
      "epoch": 0.703874796560851,
      "grad_norm": 588.0,
      "learning_rate": 1.6447368421052632e-07,
      "logits/chosen": -0.6659315228462219,
      "logits/rejected": -0.6653506755828857,
      "logps/chosen": -980.9749145507812,
      "logps/rejected": -911.2174682617188,
      "loss": 0.6985,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.006762028206139803,
      "rewards/margins": -0.010193253867328167,
      "rewards/rejected": 0.003431225661188364,
      "step": 2676
    },
    {
      "epoch": 0.704137828996038,
      "grad_norm": 564.0,
      "learning_rate": 1.6432748538011697e-07,
      "logits/chosen": -0.6448072195053101,
      "logits/rejected": -0.6608377695083618,
      "logps/chosen": -1106.797607421875,
      "logps/rejected": -1060.6666259765625,
      "loss": 0.6793,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.028527548536658287,
      "rewards/margins": 0.02860083431005478,
      "rewards/rejected": -7.328903302550316e-05,
      "step": 2677
    },
    {
      "epoch": 0.7044008614312253,
      "grad_norm": 564.0,
      "learning_rate": 1.6418128654970759e-07,
      "logits/chosen": -0.6530671119689941,
      "logits/rejected": -0.6502853631973267,
      "logps/chosen": -1489.5155029296875,
      "logps/rejected": -1124.1021728515625,
      "loss": 0.702,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.002964401850476861,
      "rewards/margins": -0.017182445153594017,
      "rewards/rejected": 0.020146846771240234,
      "step": 2678
    },
    {
      "epoch": 0.7046638938664124,
      "grad_norm": 416.0,
      "learning_rate": 1.6403508771929826e-07,
      "logits/chosen": -0.6392725706100464,
      "logits/rejected": -0.6455729007720947,
      "logps/chosen": -1036.0987548828125,
      "logps/rejected": -744.60888671875,
      "loss": 0.6879,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0036505695898085833,
      "rewards/margins": 0.01164632011204958,
      "rewards/rejected": -0.00799574889242649,
      "step": 2679
    },
    {
      "epoch": 0.7049269263015996,
      "grad_norm": 420.0,
      "learning_rate": 1.6388888888888888e-07,
      "logits/chosen": -0.6236318349838257,
      "logits/rejected": -0.6292478442192078,
      "logps/chosen": -809.4129638671875,
      "logps/rejected": -584.3892211914062,
      "loss": 0.7001,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.009584330022335052,
      "rewards/margins": -0.013178825378417969,
      "rewards/rejected": 0.02276316098868847,
      "step": 2680
    },
    {
      "epoch": 0.7051899587367867,
      "grad_norm": 632.0,
      "learning_rate": 1.6374269005847952e-07,
      "logits/chosen": -0.670492947101593,
      "logits/rejected": -0.6710132956504822,
      "logps/chosen": -1316.791259765625,
      "logps/rejected": -843.5927734375,
      "loss": 0.6909,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.015058135613799095,
      "rewards/margins": 0.0052860272116959095,
      "rewards/rejected": 0.009772108867764473,
      "step": 2681
    },
    {
      "epoch": 0.7054529911719739,
      "grad_norm": 516.0,
      "learning_rate": 1.6359649122807017e-07,
      "logits/chosen": -0.634947657585144,
      "logits/rejected": -0.6416313648223877,
      "logps/chosen": -1073.13232421875,
      "logps/rejected": -984.1331176757812,
      "loss": 0.7026,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": 0.0010829927632585168,
      "rewards/margins": -0.018733883276581764,
      "rewards/rejected": 0.019816875457763672,
      "step": 2682
    },
    {
      "epoch": 0.705716023607161,
      "grad_norm": 448.0,
      "learning_rate": 1.6345029239766082e-07,
      "logits/chosen": -0.6505751013755798,
      "logits/rejected": -0.6457257270812988,
      "logps/chosen": -981.4185791015625,
      "logps/rejected": -798.3640747070312,
      "loss": 0.7002,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00530509976670146,
      "rewards/margins": -0.013262750580906868,
      "rewards/rejected": 0.00795765034854412,
      "step": 2683
    },
    {
      "epoch": 0.7059790560423482,
      "grad_norm": 436.0,
      "learning_rate": 1.6330409356725144e-07,
      "logits/chosen": -0.6722528338432312,
      "logits/rejected": -0.6795468330383301,
      "logps/chosen": -714.3519287109375,
      "logps/rejected": -550.369384765625,
      "loss": 0.6967,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0017860900843515992,
      "rewards/margins": -0.006311607081443071,
      "rewards/rejected": 0.008097696118056774,
      "step": 2684
    },
    {
      "epoch": 0.7062420884775353,
      "grad_norm": 580.0,
      "learning_rate": 1.631578947368421e-07,
      "logits/chosen": -0.6488203406333923,
      "logits/rejected": -0.6444510817527771,
      "logps/chosen": -1256.9906005859375,
      "logps/rejected": -988.8255004882812,
      "loss": 0.6768,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.029222963377833366,
      "rewards/margins": 0.03421745449304581,
      "rewards/rejected": -0.004994487389922142,
      "step": 2685
    },
    {
      "epoch": 0.7065051209127226,
      "grad_norm": 452.0,
      "learning_rate": 1.6301169590643273e-07,
      "logits/chosen": -0.6563942432403564,
      "logits/rejected": -0.6705854535102844,
      "logps/chosen": -1200.602294921875,
      "logps/rejected": -899.6668090820312,
      "loss": 0.6882,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0010618207743391395,
      "rewards/margins": 0.010317279025912285,
      "rewards/rejected": -0.009255457669496536,
      "step": 2686
    },
    {
      "epoch": 0.7067681533479098,
      "grad_norm": 552.0,
      "learning_rate": 1.6286549707602338e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6647754311561584,
      "logps/chosen": -1294.9051513671875,
      "logps/rejected": -1069.765380859375,
      "loss": 0.6842,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02438507229089737,
      "rewards/margins": 0.018630696460604668,
      "rewards/rejected": 0.005754375830292702,
      "step": 2687
    },
    {
      "epoch": 0.7070311857830969,
      "grad_norm": 532.0,
      "learning_rate": 1.6271929824561402e-07,
      "logits/chosen": -0.654552698135376,
      "logits/rejected": -0.6617540121078491,
      "logps/chosen": -1129.989990234375,
      "logps/rejected": -971.543212890625,
      "loss": 0.6944,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.00796117726713419,
      "rewards/margins": -0.0012725833803415298,
      "rewards/rejected": 0.009233761578798294,
      "step": 2688
    },
    {
      "epoch": 0.7072942182182841,
      "grad_norm": 560.0,
      "learning_rate": 1.6257309941520467e-07,
      "logits/chosen": -0.6541368961334229,
      "logits/rejected": -0.6566901803016663,
      "logps/chosen": -1227.691650390625,
      "logps/rejected": -923.2622680664062,
      "loss": 0.6986,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01823425106704235,
      "rewards/margins": -0.01003341842442751,
      "rewards/rejected": 0.028267672285437584,
      "step": 2689
    },
    {
      "epoch": 0.7075572506534712,
      "grad_norm": 482.0,
      "learning_rate": 1.624269005847953e-07,
      "logits/chosen": -0.6425924897193909,
      "logits/rejected": -0.654448926448822,
      "logps/chosen": -911.0653686523438,
      "logps/rejected": -735.7422485351562,
      "loss": 0.6835,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009702729992568493,
      "rewards/margins": 0.019986486062407494,
      "rewards/rejected": -0.010283756069839,
      "step": 2690
    },
    {
      "epoch": 0.7078202830886584,
      "grad_norm": 608.0,
      "learning_rate": 1.6228070175438596e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6908234357833862,
      "logps/chosen": -1118.5272216796875,
      "logps/rejected": -1341.76904296875,
      "loss": 0.6887,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.002988338004797697,
      "rewards/margins": 0.009313869290053844,
      "rewards/rejected": -0.00632553081959486,
      "step": 2691
    },
    {
      "epoch": 0.7080833155238455,
      "grad_norm": 500.0,
      "learning_rate": 1.621345029239766e-07,
      "logits/chosen": -0.6545394659042358,
      "logits/rejected": -0.6624373197555542,
      "logps/chosen": -1346.44384765625,
      "logps/rejected": -1063.5234375,
      "loss": 0.6867,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0028686528094112873,
      "rewards/margins": 0.013293837197124958,
      "rewards/rejected": -0.010425185784697533,
      "step": 2692
    },
    {
      "epoch": 0.7083463479590327,
      "grad_norm": 532.0,
      "learning_rate": 1.6198830409356725e-07,
      "logits/chosen": -0.6415391564369202,
      "logits/rejected": -0.6525993347167969,
      "logps/chosen": -1116.37939453125,
      "logps/rejected": -969.4417114257812,
      "loss": 0.6988,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.019166473299264908,
      "rewards/margins": -0.010924053378403187,
      "rewards/rejected": -0.008242416195571423,
      "step": 2693
    },
    {
      "epoch": 0.7086093803942198,
      "grad_norm": 516.0,
      "learning_rate": 1.618421052631579e-07,
      "logits/chosen": -0.6285092830657959,
      "logits/rejected": -0.6323376893997192,
      "logps/chosen": -1200.6883544921875,
      "logps/rejected": -972.2559204101562,
      "loss": 0.7096,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.03212361037731171,
      "rewards/margins": -0.031488943845033646,
      "rewards/rejected": -0.0006346700247377157,
      "step": 2694
    },
    {
      "epoch": 0.708872412829407,
      "grad_norm": 484.0,
      "learning_rate": 1.6169590643274852e-07,
      "logits/chosen": -0.6674685478210449,
      "logits/rejected": -0.6754300594329834,
      "logps/chosen": -1375.52978515625,
      "logps/rejected": -818.7550048828125,
      "loss": 0.696,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.017188645899295807,
      "rewards/margins": -0.004709720611572266,
      "rewards/rejected": 0.021898364648222923,
      "step": 2695
    },
    {
      "epoch": 0.7091354452645942,
      "grad_norm": 466.0,
      "learning_rate": 1.615497076023392e-07,
      "logits/chosen": -0.6627993583679199,
      "logits/rejected": -0.6576562523841858,
      "logps/chosen": -822.2078857421875,
      "logps/rejected": -883.6824951171875,
      "loss": 0.6958,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0007634162902832031,
      "rewards/margins": -0.004583931528031826,
      "rewards/rejected": 0.005347347818315029,
      "step": 2696
    },
    {
      "epoch": 0.7093984776997814,
      "grad_norm": 504.0,
      "learning_rate": 1.6140350877192981e-07,
      "logits/chosen": -0.6378529071807861,
      "logits/rejected": -0.6464843153953552,
      "logps/chosen": -1039.084228515625,
      "logps/rejected": -984.201904296875,
      "loss": 0.6815,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0009558665915392339,
      "rewards/margins": 0.024115752428770065,
      "rewards/rejected": -0.02507162094116211,
      "step": 2697
    },
    {
      "epoch": 0.7096615101349685,
      "grad_norm": 476.0,
      "learning_rate": 1.6125730994152046e-07,
      "logits/chosen": -0.6577842831611633,
      "logits/rejected": -0.6649168133735657,
      "logps/chosen": -797.5346069335938,
      "logps/rejected": -722.9840087890625,
      "loss": 0.6996,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.006650161929428577,
      "rewards/margins": -0.011787127703428268,
      "rewards/rejected": 0.005136967170983553,
      "step": 2698
    },
    {
      "epoch": 0.7099245425701557,
      "grad_norm": 444.0,
      "learning_rate": 1.611111111111111e-07,
      "logits/chosen": -0.636089563369751,
      "logits/rejected": -0.6267828345298767,
      "logps/chosen": -1165.850341796875,
      "logps/rejected": -1109.1134033203125,
      "loss": 0.6801,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.00892267283052206,
      "rewards/margins": 0.026766492053866386,
      "rewards/rejected": -0.0178438201546669,
      "step": 2699
    },
    {
      "epoch": 0.7101875750053428,
      "grad_norm": 482.0,
      "learning_rate": 1.6096491228070175e-07,
      "logits/chosen": -0.642646849155426,
      "logits/rejected": -0.645525336265564,
      "logps/chosen": -1107.0284423828125,
      "logps/rejected": -723.5670776367188,
      "loss": 0.6937,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.015371989458799362,
      "rewards/margins": -0.0005181307205930352,
      "rewards/rejected": 0.015890121459960938,
      "step": 2700
    },
    {
      "epoch": 0.71045060744053,
      "grad_norm": 584.0,
      "learning_rate": 1.6081871345029237e-07,
      "logits/chosen": -0.6074498295783997,
      "logits/rejected": -0.6237501502037048,
      "logps/chosen": -897.3597412109375,
      "logps/rejected": -622.71337890625,
      "loss": 0.6912,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.00677490234375,
      "rewards/margins": 0.004507827572524548,
      "rewards/rejected": 0.002267075004056096,
      "step": 2701
    },
    {
      "epoch": 0.7107136398757172,
      "grad_norm": 676.0,
      "learning_rate": 1.6067251461988305e-07,
      "logits/chosen": -0.6659957766532898,
      "logits/rejected": -0.6702220439910889,
      "logps/chosen": -922.7147827148438,
      "logps/rejected": -761.72216796875,
      "loss": 0.6863,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.011242961511015892,
      "rewards/margins": 0.014635276049375534,
      "rewards/rejected": -0.003392315935343504,
      "step": 2702
    },
    {
      "epoch": 0.7109766723109043,
      "grad_norm": 600.0,
      "learning_rate": 1.6052631578947367e-07,
      "logits/chosen": -0.6620630025863647,
      "logits/rejected": -0.6651301980018616,
      "logps/chosen": -1550.9739990234375,
      "logps/rejected": -965.86376953125,
      "loss": 0.6931,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01421833224594593,
      "rewards/margins": 0.0008906382136046886,
      "rewards/rejected": 0.013327695429325104,
      "step": 2703
    },
    {
      "epoch": 0.7112397047460916,
      "grad_norm": 490.0,
      "learning_rate": 1.603801169590643e-07,
      "logits/chosen": -0.6213585138320923,
      "logits/rejected": -0.6288468837738037,
      "logps/chosen": -875.4088134765625,
      "logps/rejected": -812.4227294921875,
      "loss": 0.6756,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.027069948613643646,
      "rewards/margins": 0.0359707809984684,
      "rewards/rejected": -0.008900833316147327,
      "step": 2704
    },
    {
      "epoch": 0.7115027371812787,
      "grad_norm": 502.0,
      "learning_rate": 1.6023391812865496e-07,
      "logits/chosen": -0.6733487844467163,
      "logits/rejected": -0.6683581471443176,
      "logps/chosen": -1288.242431640625,
      "logps/rejected": -891.24853515625,
      "loss": 0.6923,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.013188362121582031,
      "rewards/margins": 0.0023270605597645044,
      "rewards/rejected": 0.01086130179464817,
      "step": 2705
    },
    {
      "epoch": 0.7117657696164659,
      "grad_norm": 502.0,
      "learning_rate": 1.600877192982456e-07,
      "logits/chosen": -0.6494274735450745,
      "logits/rejected": -0.6452908515930176,
      "logps/chosen": -1010.6585083007812,
      "logps/rejected": -889.49462890625,
      "loss": 0.6938,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004820442758500576,
      "rewards/margins": -0.00024013477377593517,
      "rewards/rejected": 0.005060576368123293,
      "step": 2706
    },
    {
      "epoch": 0.712028802051653,
      "grad_norm": 684.0,
      "learning_rate": 1.5994152046783625e-07,
      "logits/chosen": -0.6259518265724182,
      "logits/rejected": -0.6349918246269226,
      "logps/chosen": -1217.275634765625,
      "logps/rejected": -1097.386474609375,
      "loss": 0.6951,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0043853758834302425,
      "rewards/margins": -0.00276679964736104,
      "rewards/rejected": 0.007152176927775145,
      "step": 2707
    },
    {
      "epoch": 0.7122918344868402,
      "grad_norm": 540.0,
      "learning_rate": 1.597953216374269e-07,
      "logits/chosen": -0.6474976539611816,
      "logits/rejected": -0.6251518726348877,
      "logps/chosen": -1281.3028564453125,
      "logps/rejected": -1204.414306640625,
      "loss": 0.6892,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.022591400891542435,
      "rewards/margins": 0.008557558059692383,
      "rewards/rejected": 0.014033842831850052,
      "step": 2708
    },
    {
      "epoch": 0.7125548669220273,
      "grad_norm": 552.0,
      "learning_rate": 1.5964912280701754e-07,
      "logits/chosen": -0.6440001130104065,
      "logits/rejected": -0.6510630249977112,
      "logps/chosen": -1235.56005859375,
      "logps/rejected": -690.0528564453125,
      "loss": 0.6936,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.014408302493393421,
      "rewards/margins": -0.0006611819844692945,
      "rewards/rejected": 0.01506948471069336,
      "step": 2709
    },
    {
      "epoch": 0.7128178993572145,
      "grad_norm": 462.0,
      "learning_rate": 1.5950292397660816e-07,
      "logits/chosen": -0.6530559659004211,
      "logits/rejected": -0.6550235748291016,
      "logps/chosen": -628.7738037109375,
      "logps/rejected": -695.732666015625,
      "loss": 0.6949,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.015481854788959026,
      "rewards/margins": -0.003335857531055808,
      "rewards/rejected": -0.01214599609375,
      "step": 2710
    },
    {
      "epoch": 0.7130809317924016,
      "grad_norm": 560.0,
      "learning_rate": 1.5935672514619884e-07,
      "logits/chosen": -0.6535099148750305,
      "logits/rejected": -0.6595154404640198,
      "logps/chosen": -1172.290283203125,
      "logps/rejected": -925.5460815429688,
      "loss": 0.694,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009554149582982063,
      "rewards/margins": -0.0012381072156131268,
      "rewards/rejected": 0.010792254470288754,
      "step": 2711
    },
    {
      "epoch": 0.7133439642275888,
      "grad_norm": 502.0,
      "learning_rate": 1.5921052631578946e-07,
      "logits/chosen": -0.6486512422561646,
      "logits/rejected": -0.6509102582931519,
      "logps/chosen": -1045.9215087890625,
      "logps/rejected": -854.1004638671875,
      "loss": 0.6852,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.00893411599099636,
      "rewards/margins": 0.016849040985107422,
      "rewards/rejected": -0.007914924062788486,
      "step": 2712
    },
    {
      "epoch": 0.7136069966627759,
      "grad_norm": 512.0,
      "learning_rate": 1.5906432748538013e-07,
      "logits/chosen": -0.6609392762184143,
      "logits/rejected": -0.669374406337738,
      "logps/chosen": -1037.4403076171875,
      "logps/rejected": -792.8221435546875,
      "loss": 0.7001,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.014978456310927868,
      "rewards/margins": -0.013223409652709961,
      "rewards/rejected": -0.0017550471238791943,
      "step": 2713
    },
    {
      "epoch": 0.7138700290979632,
      "grad_norm": 506.0,
      "learning_rate": 1.5891812865497075e-07,
      "logits/chosen": -0.6568335294723511,
      "logits/rejected": -0.6708038449287415,
      "logps/chosen": -1051.4915771484375,
      "logps/rejected": -880.2788696289062,
      "loss": 0.7007,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.015018034726381302,
      "rewards/margins": -0.01433100737631321,
      "rewards/rejected": -0.0006870268261991441,
      "step": 2714
    },
    {
      "epoch": 0.7141330615331503,
      "grad_norm": 500.0,
      "learning_rate": 1.587719298245614e-07,
      "logits/chosen": -0.628830075263977,
      "logits/rejected": -0.6346434354782104,
      "logps/chosen": -1132.118408203125,
      "logps/rejected": -1052.036865234375,
      "loss": 0.6903,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.010329915210604668,
      "rewards/margins": 0.006185341160744429,
      "rewards/rejected": 0.004144572652876377,
      "step": 2715
    },
    {
      "epoch": 0.7143960939683375,
      "grad_norm": 512.0,
      "learning_rate": 1.5862573099415204e-07,
      "logits/chosen": -0.6431689262390137,
      "logits/rejected": -0.6481711268424988,
      "logps/chosen": -900.7753295898438,
      "logps/rejected": -825.8290405273438,
      "loss": 0.6999,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.006890392862260342,
      "rewards/margins": -0.013096237555146217,
      "rewards/rejected": 0.006205844227224588,
      "step": 2716
    },
    {
      "epoch": 0.7146591264035246,
      "grad_norm": 556.0,
      "learning_rate": 1.584795321637427e-07,
      "logits/chosen": -0.6389023065567017,
      "logits/rejected": -0.6567662358283997,
      "logps/chosen": -1106.195068359375,
      "logps/rejected": -558.3985595703125,
      "loss": 0.6796,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03481893613934517,
      "rewards/margins": 0.02792511135339737,
      "rewards/rejected": 0.006893825251609087,
      "step": 2717
    },
    {
      "epoch": 0.7149221588387118,
      "grad_norm": 536.0,
      "learning_rate": 1.583333333333333e-07,
      "logits/chosen": -0.6505718231201172,
      "logits/rejected": -0.6657577753067017,
      "logps/chosen": -1217.326171875,
      "logps/rejected": -768.9844970703125,
      "loss": 0.6873,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.021050073206424713,
      "rewards/margins": 0.01274251937866211,
      "rewards/rejected": 0.00830755289644003,
      "step": 2718
    },
    {
      "epoch": 0.715185191273899,
      "grad_norm": 532.0,
      "learning_rate": 1.5818713450292398e-07,
      "logits/chosen": -0.6486319303512573,
      "logits/rejected": -0.6517800092697144,
      "logps/chosen": -1496.4949951171875,
      "logps/rejected": -1044.15478515625,
      "loss": 0.7061,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01631012000143528,
      "rewards/margins": -0.02460050769150257,
      "rewards/rejected": 0.008290385827422142,
      "step": 2719
    },
    {
      "epoch": 0.7154482237090861,
      "grad_norm": 572.0,
      "learning_rate": 1.580409356725146e-07,
      "logits/chosen": -0.630958080291748,
      "logits/rejected": -0.6443774104118347,
      "logps/chosen": -1294.0869140625,
      "logps/rejected": -768.6648559570312,
      "loss": 0.6939,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01322326809167862,
      "rewards/margins": -0.0002419459051452577,
      "rewards/rejected": -0.012981319800019264,
      "step": 2720
    },
    {
      "epoch": 0.7157112561442733,
      "grad_norm": 568.0,
      "learning_rate": 1.5789473684210525e-07,
      "logits/chosen": -0.6455844640731812,
      "logits/rejected": -0.6581219434738159,
      "logps/chosen": -1237.3228759765625,
      "logps/rejected": -888.8882446289062,
      "loss": 0.6815,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0008742325007915497,
      "rewards/margins": 0.023879002779722214,
      "rewards/rejected": -0.024753237143158913,
      "step": 2721
    },
    {
      "epoch": 0.7159742885794604,
      "grad_norm": 544.0,
      "learning_rate": 1.5774853801169592e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6564012765884399,
      "logps/chosen": -763.56982421875,
      "logps/rejected": -908.5248413085938,
      "loss": 0.693,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.011854076758027077,
      "rewards/margins": 0.0008522990392521024,
      "rewards/rejected": -0.012706374749541283,
      "step": 2722
    },
    {
      "epoch": 0.7162373210146477,
      "grad_norm": 608.0,
      "learning_rate": 1.5760233918128654e-07,
      "logits/chosen": -0.6389914751052856,
      "logits/rejected": -0.6398055553436279,
      "logps/chosen": -1093.8619384765625,
      "logps/rejected": -1033.209228515625,
      "loss": 0.6877,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008353614248335361,
      "rewards/margins": 0.012324618175625801,
      "rewards/rejected": -0.003971004858613014,
      "step": 2723
    },
    {
      "epoch": 0.7165003534498348,
      "grad_norm": 488.0,
      "learning_rate": 1.574561403508772e-07,
      "logits/chosen": -0.6062157154083252,
      "logits/rejected": -0.6123801469802856,
      "logps/chosen": -1052.0562744140625,
      "logps/rejected": -762.4910278320312,
      "loss": 0.6886,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.003048800863325596,
      "rewards/margins": 0.009804533794522285,
      "rewards/rejected": -0.006755733862519264,
      "step": 2724
    },
    {
      "epoch": 0.716763385885022,
      "grad_norm": 490.0,
      "learning_rate": 1.5730994152046783e-07,
      "logits/chosen": -0.675511360168457,
      "logits/rejected": -0.6812565326690674,
      "logps/chosen": -1073.1978759765625,
      "logps/rejected": -877.884521484375,
      "loss": 0.7076,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.022420406341552734,
      "rewards/margins": -0.02811298333108425,
      "rewards/rejected": 0.005692576989531517,
      "step": 2725
    },
    {
      "epoch": 0.7170264183202091,
      "grad_norm": 580.0,
      "learning_rate": 1.5716374269005848e-07,
      "logits/chosen": -0.649242103099823,
      "logits/rejected": -0.6441524028778076,
      "logps/chosen": -1152.114013671875,
      "logps/rejected": -893.8735961914062,
      "loss": 0.6889,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0040464382618665695,
      "rewards/margins": 0.009293556213378906,
      "rewards/rejected": -0.01333999540656805,
      "step": 2726
    },
    {
      "epoch": 0.7172894507553963,
      "grad_norm": 424.0,
      "learning_rate": 1.570175438596491e-07,
      "logits/chosen": -0.6340929865837097,
      "logits/rejected": -0.6454668641090393,
      "logps/chosen": -903.4024047851562,
      "logps/rejected": -994.0285034179688,
      "loss": 0.7003,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.005445767194032669,
      "rewards/margins": -0.013460065238177776,
      "rewards/rejected": 0.008014298975467682,
      "step": 2727
    },
    {
      "epoch": 0.7175524831905834,
      "grad_norm": 444.0,
      "learning_rate": 1.5687134502923977e-07,
      "logits/chosen": -0.6749991774559021,
      "logits/rejected": -0.6711257100105286,
      "logps/chosen": -980.7447509765625,
      "logps/rejected": -1096.408935546875,
      "loss": 0.679,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.012854098342359066,
      "rewards/margins": 0.02897491678595543,
      "rewards/rejected": -0.01612081564962864,
      "step": 2728
    },
    {
      "epoch": 0.7178155156257706,
      "grad_norm": 512.0,
      "learning_rate": 1.567251461988304e-07,
      "logits/chosen": -0.6313380002975464,
      "logits/rejected": -0.6397923827171326,
      "logps/chosen": -813.2284545898438,
      "logps/rejected": -927.988525390625,
      "loss": 0.6933,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.025463249534368515,
      "rewards/margins": 0.00042481347918510437,
      "rewards/rejected": -0.02588806301355362,
      "step": 2729
    },
    {
      "epoch": 0.7180785480609577,
      "grad_norm": 540.0,
      "learning_rate": 1.5657894736842104e-07,
      "logits/chosen": -0.6559814214706421,
      "logits/rejected": -0.657745361328125,
      "logps/chosen": -1090.54541015625,
      "logps/rejected": -917.3023071289062,
      "loss": 0.6945,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.017522096633911133,
      "rewards/margins": -0.002182244323194027,
      "rewards/rejected": 0.019704341888427734,
      "step": 2730
    },
    {
      "epoch": 0.7183415804961449,
      "grad_norm": 544.0,
      "learning_rate": 1.5643274853801168e-07,
      "logits/chosen": -0.6540019512176514,
      "logits/rejected": -0.6684377193450928,
      "logps/chosen": -1233.160888671875,
      "logps/rejected": -861.023681640625,
      "loss": 0.6902,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.014184664934873581,
      "rewards/margins": 0.006993675604462624,
      "rewards/rejected": 0.007190990261733532,
      "step": 2731
    },
    {
      "epoch": 0.718604612931332,
      "grad_norm": 708.0,
      "learning_rate": 1.5628654970760233e-07,
      "logits/chosen": -0.622768223285675,
      "logits/rejected": -0.631099283695221,
      "logps/chosen": -957.18994140625,
      "logps/rejected": -781.185546875,
      "loss": 0.6847,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.016972731798887253,
      "rewards/margins": 0.018100546672940254,
      "rewards/rejected": -0.001127815805375576,
      "step": 2732
    },
    {
      "epoch": 0.7188676453665193,
      "grad_norm": 604.0,
      "learning_rate": 1.5614035087719298e-07,
      "logits/chosen": -0.654698371887207,
      "logits/rejected": -0.6569703817367554,
      "logps/chosen": -1494.1160888671875,
      "logps/rejected": -1255.265869140625,
      "loss": 0.6779,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.013101482763886452,
      "rewards/margins": 0.03194227069616318,
      "rewards/rejected": -0.018840789794921875,
      "step": 2733
    },
    {
      "epoch": 0.7191306778017065,
      "grad_norm": 448.0,
      "learning_rate": 1.5599415204678362e-07,
      "logits/chosen": -0.6122406721115112,
      "logits/rejected": -0.6102606058120728,
      "logps/chosen": -650.9703369140625,
      "logps/rejected": -715.779052734375,
      "loss": 0.6932,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.008546829223632812,
      "rewards/margins": 0.00038824090734124184,
      "rewards/rejected": -0.008935069665312767,
      "step": 2734
    },
    {
      "epoch": 0.7193937102368936,
      "grad_norm": 592.0,
      "learning_rate": 1.5584795321637424e-07,
      "logits/chosen": -0.6231708526611328,
      "logits/rejected": -0.634101927280426,
      "logps/chosen": -1042.1639404296875,
      "logps/rejected": -1065.9322509765625,
      "loss": 0.6985,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.011345671489834785,
      "rewards/margins": -0.009412765502929688,
      "rewards/rejected": -0.0019329071510583162,
      "step": 2735
    },
    {
      "epoch": 0.7196567426720808,
      "grad_norm": 632.0,
      "learning_rate": 1.5570175438596492e-07,
      "logits/chosen": -0.6511104106903076,
      "logits/rejected": -0.6514275074005127,
      "logps/chosen": -1476.2978515625,
      "logps/rejected": -914.260498046875,
      "loss": 0.6816,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010341263376176357,
      "rewards/margins": 0.024749755859375,
      "rewards/rejected": -0.014408493414521217,
      "step": 2736
    },
    {
      "epoch": 0.7199197751072679,
      "grad_norm": 480.0,
      "learning_rate": 1.5555555555555556e-07,
      "logits/chosen": -0.6544967293739319,
      "logits/rejected": -0.6482164859771729,
      "logps/chosen": -1345.3858642578125,
      "logps/rejected": -1164.4140625,
      "loss": 0.7001,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.012228012084960938,
      "rewards/margins": -0.013044738210737705,
      "rewards/rejected": 0.000816726591438055,
      "step": 2737
    },
    {
      "epoch": 0.7201828075424551,
      "grad_norm": 500.0,
      "learning_rate": 1.5540935672514618e-07,
      "logits/chosen": -0.6419723629951477,
      "logits/rejected": -0.6578111052513123,
      "logps/chosen": -1157.2208251953125,
      "logps/rejected": -829.3580322265625,
      "loss": 0.7025,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.008195686154067516,
      "rewards/margins": -0.018352270126342773,
      "rewards/rejected": 0.026547957211732864,
      "step": 2738
    },
    {
      "epoch": 0.7204458399776422,
      "grad_norm": 480.0,
      "learning_rate": 1.5526315789473686e-07,
      "logits/chosen": -0.6569488048553467,
      "logits/rejected": -0.6666551828384399,
      "logps/chosen": -1057.283935546875,
      "logps/rejected": -966.6237182617188,
      "loss": 0.6871,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.008949564769864082,
      "rewards/margins": 0.012466907501220703,
      "rewards/rejected": -0.003517341800034046,
      "step": 2739
    },
    {
      "epoch": 0.7207088724128294,
      "grad_norm": 532.0,
      "learning_rate": 1.5511695906432748e-07,
      "logits/chosen": -0.6479158997535706,
      "logits/rejected": -0.6542364358901978,
      "logps/chosen": -1230.549072265625,
      "logps/rejected": -869.9510498046875,
      "loss": 0.6853,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.034238435328006744,
      "rewards/margins": 0.01636791229248047,
      "rewards/rejected": 0.017870521172881126,
      "step": 2740
    },
    {
      "epoch": 0.7209719048480165,
      "grad_norm": 510.0,
      "learning_rate": 1.5497076023391812e-07,
      "logits/chosen": -0.658110499382019,
      "logits/rejected": -0.6510437726974487,
      "logps/chosen": -861.8399658203125,
      "logps/rejected": -753.2762451171875,
      "loss": 0.7009,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.005005456507205963,
      "rewards/margins": -0.015122700482606888,
      "rewards/rejected": 0.0201281551271677,
      "step": 2741
    },
    {
      "epoch": 0.7212349372832038,
      "grad_norm": 572.0,
      "learning_rate": 1.5482456140350877e-07,
      "logits/chosen": -0.6470133662223816,
      "logits/rejected": -0.6619980335235596,
      "logps/chosen": -1204.2366943359375,
      "logps/rejected": -765.1849365234375,
      "loss": 0.6893,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03161315992474556,
      "rewards/margins": 0.008366584777832031,
      "rewards/rejected": 0.02324657514691353,
      "step": 2742
    },
    {
      "epoch": 0.7214979697183909,
      "grad_norm": 440.0,
      "learning_rate": 1.5467836257309942e-07,
      "logits/chosen": -0.6638219952583313,
      "logits/rejected": -0.669202446937561,
      "logps/chosen": -1097.2021484375,
      "logps/rejected": -947.8193359375,
      "loss": 0.6879,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.023892022669315338,
      "rewards/margins": 0.012104128487408161,
      "rewards/rejected": 0.011787891387939453,
      "step": 2743
    },
    {
      "epoch": 0.7217610021535781,
      "grad_norm": 506.0,
      "learning_rate": 1.5453216374269003e-07,
      "logits/chosen": -0.6602814793586731,
      "logits/rejected": -0.6683177947998047,
      "logps/chosen": -1122.805419921875,
      "logps/rejected": -1018.3699340820312,
      "loss": 0.7104,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.01956653594970703,
      "rewards/margins": -0.03349623829126358,
      "rewards/rejected": 0.0139297004789114,
      "step": 2744
    },
    {
      "epoch": 0.7220240345887652,
      "grad_norm": 584.0,
      "learning_rate": 1.543859649122807e-07,
      "logits/chosen": -0.635463297367096,
      "logits/rejected": -0.6333527565002441,
      "logps/chosen": -1197.2188720703125,
      "logps/rejected": -1034.27099609375,
      "loss": 0.6947,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.008409119211137295,
      "rewards/margins": -0.0019698149990290403,
      "rewards/rejected": -0.006439303979277611,
      "step": 2745
    },
    {
      "epoch": 0.7222870670239524,
      "grad_norm": 616.0,
      "learning_rate": 1.5423976608187133e-07,
      "logits/chosen": -0.6520084142684937,
      "logits/rejected": -0.6601744294166565,
      "logps/chosen": -1153.082763671875,
      "logps/rejected": -1202.3763427734375,
      "loss": 0.694,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.010146332904696465,
      "rewards/margins": -0.0010373122058808804,
      "rewards/rejected": -0.009109020233154297,
      "step": 2746
    },
    {
      "epoch": 0.7225500994591395,
      "grad_norm": 544.0,
      "learning_rate": 1.5409356725146197e-07,
      "logits/chosen": -0.6625645756721497,
      "logits/rejected": -0.655506432056427,
      "logps/chosen": -1243.3741455078125,
      "logps/rejected": -656.4716186523438,
      "loss": 0.6918,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0020573618821799755,
      "rewards/margins": 0.0031544193625450134,
      "rewards/rejected": -0.005211781710386276,
      "step": 2747
    },
    {
      "epoch": 0.7228131318943267,
      "grad_norm": 564.0,
      "learning_rate": 1.5394736842105262e-07,
      "logits/chosen": -0.6388412117958069,
      "logits/rejected": -0.6416765451431274,
      "logps/chosen": -1128.8585205078125,
      "logps/rejected": -1023.964599609375,
      "loss": 0.698,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.015685655176639557,
      "rewards/margins": -0.008201980032026768,
      "rewards/rejected": 0.0238876361399889,
      "step": 2748
    },
    {
      "epoch": 0.7230761643295139,
      "grad_norm": 488.0,
      "learning_rate": 1.5380116959064327e-07,
      "logits/chosen": -0.6253708004951477,
      "logits/rejected": -0.633371114730835,
      "logps/chosen": -1206.8184814453125,
      "logps/rejected": -1133.7340087890625,
      "loss": 0.6829,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02012004889547825,
      "rewards/margins": 0.02115764655172825,
      "rewards/rejected": -0.001037596259266138,
      "step": 2749
    },
    {
      "epoch": 0.723339196764701,
      "grad_norm": 576.0,
      "learning_rate": 1.5365497076023389e-07,
      "logits/chosen": -0.6833817362785339,
      "logits/rejected": -0.6873800754547119,
      "logps/chosen": -934.58203125,
      "logps/rejected": -693.0155639648438,
      "loss": 0.6907,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00028142926748842,
      "rewards/margins": 0.005804204382002354,
      "rewards/rejected": -0.0060856337659060955,
      "step": 2750
    },
    {
      "epoch": 0.7236022291998883,
      "grad_norm": 744.0,
      "learning_rate": 1.5350877192982456e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6563517451286316,
      "logps/chosen": -1155.8292236328125,
      "logps/rejected": -1014.1583251953125,
      "loss": 0.6801,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0380464568734169,
      "rewards/margins": 0.028248783200979233,
      "rewards/rejected": 0.009797669015824795,
      "step": 2751
    },
    {
      "epoch": 0.7238652616350754,
      "grad_norm": 536.0,
      "learning_rate": 1.533625730994152e-07,
      "logits/chosen": -0.649585485458374,
      "logits/rejected": -0.6520538926124573,
      "logps/chosen": -1237.24609375,
      "logps/rejected": -1123.732421875,
      "loss": 0.7003,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0033138268627226353,
      "rewards/margins": -0.01367168314754963,
      "rewards/rejected": 0.010357856750488281,
      "step": 2752
    },
    {
      "epoch": 0.7241282940702626,
      "grad_norm": 508.0,
      "learning_rate": 1.5321637426900585e-07,
      "logits/chosen": -0.6276424527168274,
      "logits/rejected": -0.629257082939148,
      "logps/chosen": -1128.25,
      "logps/rejected": -558.0504150390625,
      "loss": 0.6968,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.00043067988008260727,
      "rewards/margins": -0.006336974911391735,
      "rewards/rejected": 0.005906295962631702,
      "step": 2753
    },
    {
      "epoch": 0.7243913265054497,
      "grad_norm": 604.0,
      "learning_rate": 1.530701754385965e-07,
      "logits/chosen": -0.6682845950126648,
      "logits/rejected": -0.6688908934593201,
      "logps/chosen": -1258.16064453125,
      "logps/rejected": -945.4196166992188,
      "loss": 0.6871,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0143311507999897,
      "rewards/margins": 0.012830162420868874,
      "rewards/rejected": 0.0015009879134595394,
      "step": 2754
    },
    {
      "epoch": 0.7246543589406369,
      "grad_norm": 516.0,
      "learning_rate": 1.5292397660818712e-07,
      "logits/chosen": -0.6639088988304138,
      "logits/rejected": -0.6819472908973694,
      "logps/chosen": -1232.83544921875,
      "logps/rejected": -982.1172485351562,
      "loss": 0.7012,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.001787281595170498,
      "rewards/margins": -0.015253734774887562,
      "rewards/rejected": 0.01704101637005806,
      "step": 2755
    },
    {
      "epoch": 0.724917391375824,
      "grad_norm": 420.0,
      "learning_rate": 1.527777777777778e-07,
      "logits/chosen": -0.6507056355476379,
      "logits/rejected": -0.6422981023788452,
      "logps/chosen": -797.692626953125,
      "logps/rejected": -522.092529296875,
      "loss": 0.6906,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.009758664295077324,
      "rewards/margins": 0.005357217043638229,
      "rewards/rejected": -0.015115882270038128,
      "step": 2756
    },
    {
      "epoch": 0.7251804238110112,
      "grad_norm": 502.0,
      "learning_rate": 1.526315789473684e-07,
      "logits/chosen": -0.6610029935836792,
      "logits/rejected": -0.6601552963256836,
      "logps/chosen": -1065.380126953125,
      "logps/rejected": -715.7852172851562,
      "loss": 0.6772,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02770671620965004,
      "rewards/margins": 0.03292426839470863,
      "rewards/rejected": -0.005217551253736019,
      "step": 2757
    },
    {
      "epoch": 0.7254434562461983,
      "grad_norm": 498.0,
      "learning_rate": 1.5248538011695906e-07,
      "logits/chosen": -0.6737204194068909,
      "logits/rejected": -0.6661854386329651,
      "logps/chosen": -1160.5789794921875,
      "logps/rejected": -1046.19677734375,
      "loss": 0.6832,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.005446529947221279,
      "rewards/margins": 0.020609762519598007,
      "rewards/rejected": -0.026056289672851562,
      "step": 2758
    },
    {
      "epoch": 0.7257064886813855,
      "grad_norm": 464.0,
      "learning_rate": 1.523391812865497e-07,
      "logits/chosen": -0.6588224172592163,
      "logits/rejected": -0.663213312625885,
      "logps/chosen": -1132.0419921875,
      "logps/rejected": -846.7618408203125,
      "loss": 0.6948,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00223884591832757,
      "rewards/margins": -0.003163957502692938,
      "rewards/rejected": 0.005402803421020508,
      "step": 2759
    },
    {
      "epoch": 0.7259695211165726,
      "grad_norm": 504.0,
      "learning_rate": 1.5219298245614035e-07,
      "logits/chosen": -0.6378465890884399,
      "logits/rejected": -0.6489494442939758,
      "logps/chosen": -1136.3592529296875,
      "logps/rejected": -848.7054443359375,
      "loss": 0.6891,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008271216414868832,
      "rewards/margins": 0.008583449758589268,
      "rewards/rejected": -0.0003122324123978615,
      "step": 2760
    },
    {
      "epoch": 0.7262325535517599,
      "grad_norm": 620.0,
      "learning_rate": 1.5204678362573097e-07,
      "logits/chosen": -0.6518926024436951,
      "logits/rejected": -0.6648382544517517,
      "logps/chosen": -1282.6732177734375,
      "logps/rejected": -959.8485717773438,
      "loss": 0.6967,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.022686291486024857,
      "rewards/margins": -0.0062202936969697475,
      "rewards/rejected": -0.01646599732339382,
      "step": 2761
    },
    {
      "epoch": 0.726495585986947,
      "grad_norm": 502.0,
      "learning_rate": 1.5190058479532164e-07,
      "logits/chosen": -0.6544584631919861,
      "logits/rejected": -0.663172721862793,
      "logps/chosen": -1140.1856689453125,
      "logps/rejected": -831.45751953125,
      "loss": 0.6892,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.012654447928071022,
      "rewards/margins": 0.00854411255568266,
      "rewards/rejected": 0.0041103363037109375,
      "step": 2762
    },
    {
      "epoch": 0.7267586184221342,
      "grad_norm": 464.0,
      "learning_rate": 1.5175438596491226e-07,
      "logits/chosen": -0.6551575660705566,
      "logits/rejected": -0.6703879237174988,
      "logps/chosen": -839.8267822265625,
      "logps/rejected": -603.3770751953125,
      "loss": 0.6887,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010711859911680222,
      "rewards/margins": 0.009296035394072533,
      "rewards/rejected": 0.0014158254489302635,
      "step": 2763
    },
    {
      "epoch": 0.7270216508573213,
      "grad_norm": 506.0,
      "learning_rate": 1.516081871345029e-07,
      "logits/chosen": -0.644404411315918,
      "logits/rejected": -0.6487775444984436,
      "logps/chosen": -1191.6248779296875,
      "logps/rejected": -1044.8585205078125,
      "loss": 0.6932,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.012634183280169964,
      "rewards/margins": 0.0007832520641386509,
      "rewards/rejected": -0.013417433947324753,
      "step": 2764
    },
    {
      "epoch": 0.7272846832925085,
      "grad_norm": 508.0,
      "learning_rate": 1.5146198830409356e-07,
      "logits/chosen": -0.6730031967163086,
      "logits/rejected": -0.6714160442352295,
      "logps/chosen": -1085.103271484375,
      "logps/rejected": -862.9012451171875,
      "loss": 0.687,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.013506937772035599,
      "rewards/margins": 0.013351868838071823,
      "rewards/rejected": 0.0001550675369799137,
      "step": 2765
    },
    {
      "epoch": 0.7275477157276957,
      "grad_norm": 612.0,
      "learning_rate": 1.513157894736842e-07,
      "logits/chosen": -0.6487469673156738,
      "logits/rejected": -0.6554509401321411,
      "logps/chosen": -1585.75341796875,
      "logps/rejected": -1170.3271484375,
      "loss": 0.6978,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008106422610580921,
      "rewards/margins": -0.008521556854248047,
      "rewards/rejected": 0.016627978533506393,
      "step": 2766
    },
    {
      "epoch": 0.7278107481628828,
      "grad_norm": 556.0,
      "learning_rate": 1.5116959064327485e-07,
      "logits/chosen": -0.6475014686584473,
      "logits/rejected": -0.6434681415557861,
      "logps/chosen": -1455.7254638671875,
      "logps/rejected": -1267.3087158203125,
      "loss": 0.6927,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01399068720638752,
      "rewards/margins": 0.0017479904927313328,
      "rewards/rejected": 0.012242699041962624,
      "step": 2767
    },
    {
      "epoch": 0.72807378059807,
      "grad_norm": 504.0,
      "learning_rate": 1.510233918128655e-07,
      "logits/chosen": -0.6398704648017883,
      "logits/rejected": -0.6516213417053223,
      "logps/chosen": -1082.03955078125,
      "logps/rejected": -669.7230834960938,
      "loss": 0.6868,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.021284960210323334,
      "rewards/margins": 0.01313853356987238,
      "rewards/rejected": 0.008146428503096104,
      "step": 2768
    },
    {
      "epoch": 0.7283368130332571,
      "grad_norm": 502.0,
      "learning_rate": 1.5087719298245614e-07,
      "logits/chosen": -0.6560777425765991,
      "logits/rejected": -0.6552812457084656,
      "logps/chosen": -720.3955688476562,
      "logps/rejected": -636.8732299804688,
      "loss": 0.6901,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00019903189968317747,
      "rewards/margins": 0.006278611719608307,
      "rewards/rejected": -0.006477643270045519,
      "step": 2769
    },
    {
      "epoch": 0.7285998454684444,
      "grad_norm": 412.0,
      "learning_rate": 1.5073099415204676e-07,
      "logits/chosen": -0.64947509765625,
      "logits/rejected": -0.6494320034980774,
      "logps/chosen": -604.5276489257812,
      "logps/rejected": -654.6356811523438,
      "loss": 0.6986,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01389980223029852,
      "rewards/margins": -0.010509110055863857,
      "rewards/rejected": -0.003390694037079811,
      "step": 2770
    },
    {
      "epoch": 0.7288628779036315,
      "grad_norm": 616.0,
      "learning_rate": 1.5058479532163743e-07,
      "logits/chosen": -0.6481850147247314,
      "logits/rejected": -0.6529717445373535,
      "logps/chosen": -1178.358154296875,
      "logps/rejected": -909.00732421875,
      "loss": 0.6844,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.013917256146669388,
      "rewards/margins": 0.01920156553387642,
      "rewards/rejected": -0.005284308921545744,
      "step": 2771
    },
    {
      "epoch": 0.7291259103388187,
      "grad_norm": 548.0,
      "learning_rate": 1.5043859649122805e-07,
      "logits/chosen": -0.6417382955551147,
      "logits/rejected": -0.649388313293457,
      "logps/chosen": -861.0947265625,
      "logps/rejected": -905.59375,
      "loss": 0.7026,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0004913331940770149,
      "rewards/margins": -0.017675209790468216,
      "rewards/rejected": 0.018166542053222656,
      "step": 2772
    },
    {
      "epoch": 0.7293889427740058,
      "grad_norm": 544.0,
      "learning_rate": 1.5029239766081873e-07,
      "logits/chosen": -0.6642417907714844,
      "logits/rejected": -0.6549023985862732,
      "logps/chosen": -1084.6859130859375,
      "logps/rejected": -954.3203125,
      "loss": 0.6864,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00447578402236104,
      "rewards/margins": 0.01469392515718937,
      "rewards/rejected": -0.01021814439445734,
      "step": 2773
    },
    {
      "epoch": 0.729651975209193,
      "grad_norm": 508.0,
      "learning_rate": 1.5014619883040935e-07,
      "logits/chosen": -0.6365706920623779,
      "logits/rejected": -0.6363348364830017,
      "logps/chosen": -1080.052490234375,
      "logps/rejected": -1080.06396484375,
      "loss": 0.6951,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0042285919189453125,
      "rewards/margins": -0.0029620167333632708,
      "rewards/rejected": -0.0012665753019973636,
      "step": 2774
    },
    {
      "epoch": 0.7299150076443801,
      "grad_norm": 584.0,
      "learning_rate": 1.5e-07,
      "logits/chosen": -0.638320803642273,
      "logits/rejected": -0.6492252349853516,
      "logps/chosen": -1358.17041015625,
      "logps/rejected": -642.9224853515625,
      "loss": 0.6815,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.007892798632383347,
      "rewards/margins": 0.024650290608406067,
      "rewards/rejected": -0.01675749011337757,
      "step": 2775
    },
    {
      "epoch": 0.7301780400795673,
      "grad_norm": 560.0,
      "learning_rate": 1.4985380116959064e-07,
      "logits/chosen": -0.6448950171470642,
      "logits/rejected": -0.644895076751709,
      "logps/chosen": -697.749267578125,
      "logps/rejected": -664.1777954101562,
      "loss": 0.7066,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.017244720831513405,
      "rewards/margins": -0.026216555386781693,
      "rewards/rejected": 0.008971835486590862,
      "step": 2776
    },
    {
      "epoch": 0.7304410725147544,
      "grad_norm": 456.0,
      "learning_rate": 1.4970760233918129e-07,
      "logits/chosen": -0.6642726063728333,
      "logits/rejected": -0.6597088575363159,
      "logps/chosen": -1290.7584228515625,
      "logps/rejected": -958.5341186523438,
      "loss": 0.6912,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.000565958209335804,
      "rewards/margins": 0.005385112948715687,
      "rewards/rejected": -0.0048191542737185955,
      "step": 2777
    },
    {
      "epoch": 0.7307041049499416,
      "grad_norm": 576.0,
      "learning_rate": 1.495614035087719e-07,
      "logits/chosen": -0.684855580329895,
      "logits/rejected": -0.6788724064826965,
      "logps/chosen": -1425.5928955078125,
      "logps/rejected": -1472.494873046875,
      "loss": 0.6856,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.003283977508544922,
      "rewards/margins": 0.016805125400424004,
      "rewards/rejected": -0.020089102908968925,
      "step": 2778
    },
    {
      "epoch": 0.7309671373851288,
      "grad_norm": 502.0,
      "learning_rate": 1.4941520467836258e-07,
      "logits/chosen": -0.6395469903945923,
      "logits/rejected": -0.6322569847106934,
      "logps/chosen": -1089.68701171875,
      "logps/rejected": -878.797119140625,
      "loss": 0.6985,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.021961594000458717,
      "rewards/margins": -0.009956741705536842,
      "rewards/rejected": -0.012004852294921875,
      "step": 2779
    },
    {
      "epoch": 0.731230169820316,
      "grad_norm": 382.0,
      "learning_rate": 1.492690058479532e-07,
      "logits/chosen": -0.6390182375907898,
      "logits/rejected": -0.6427680253982544,
      "logps/chosen": -669.160888671875,
      "logps/rejected": -681.1229248046875,
      "loss": 0.7006,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02816200628876686,
      "rewards/margins": -0.014276888221502304,
      "rewards/rejected": -0.013885116204619408,
      "step": 2780
    },
    {
      "epoch": 0.7314932022555032,
      "grad_norm": 660.0,
      "learning_rate": 1.4912280701754385e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6529526710510254,
      "logps/chosen": -1302.80029296875,
      "logps/rejected": -1033.6246337890625,
      "loss": 0.6835,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010104561224579811,
      "rewards/margins": 0.020321082323789597,
      "rewards/rejected": -0.01021652389317751,
      "step": 2781
    },
    {
      "epoch": 0.7317562346906903,
      "grad_norm": 588.0,
      "learning_rate": 1.4897660818713452e-07,
      "logits/chosen": -0.6553542613983154,
      "logits/rejected": -0.6592764854431152,
      "logps/chosen": -1595.626708984375,
      "logps/rejected": -875.6629638671875,
      "loss": 0.6905,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.009132671169936657,
      "rewards/margins": 0.00579757709056139,
      "rewards/rejected": 0.003335094079375267,
      "step": 2782
    },
    {
      "epoch": 0.7320192671258775,
      "grad_norm": 484.0,
      "learning_rate": 1.4883040935672514e-07,
      "logits/chosen": -0.65860915184021,
      "logits/rejected": -0.6623561382293701,
      "logps/chosen": -1314.421142578125,
      "logps/rejected": -1340.2698974609375,
      "loss": 0.6905,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008842659182846546,
      "rewards/margins": 0.006326578091830015,
      "rewards/rejected": 0.0025160801596939564,
      "step": 2783
    },
    {
      "epoch": 0.7322822995610646,
      "grad_norm": 520.0,
      "learning_rate": 1.4868421052631578e-07,
      "logits/chosen": -0.6556957960128784,
      "logits/rejected": -0.6495500206947327,
      "logps/chosen": -841.963623046875,
      "logps/rejected": -992.7110595703125,
      "loss": 0.6931,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0004406929947435856,
      "rewards/margins": 0.0008275029249489307,
      "rewards/rejected": -0.0012681963853538036,
      "step": 2784
    },
    {
      "epoch": 0.7325453319962518,
      "grad_norm": 792.0,
      "learning_rate": 1.4853801169590643e-07,
      "logits/chosen": -0.6396197080612183,
      "logits/rejected": -0.642005443572998,
      "logps/chosen": -1429.9423828125,
      "logps/rejected": -1044.175048828125,
      "loss": 0.6902,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02032937854528427,
      "rewards/margins": 0.006196498870849609,
      "rewards/rejected": 0.014132880605757236,
      "step": 2785
    },
    {
      "epoch": 0.7328083644314389,
      "grad_norm": 668.0,
      "learning_rate": 1.4839181286549708e-07,
      "logits/chosen": -0.6606702208518982,
      "logits/rejected": -0.6782314777374268,
      "logps/chosen": -1038.75341796875,
      "logps/rejected": -590.8568115234375,
      "loss": 0.7009,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0062747010961174965,
      "rewards/margins": -0.014689636416733265,
      "rewards/rejected": 0.020964335650205612,
      "step": 2786
    },
    {
      "epoch": 0.7330713968666261,
      "grad_norm": 440.0,
      "learning_rate": 1.482456140350877e-07,
      "logits/chosen": -0.6405964493751526,
      "logits/rejected": -0.6510019302368164,
      "logps/chosen": -843.543701171875,
      "logps/rejected": -736.0372924804688,
      "loss": 0.6898,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010057258419692516,
      "rewards/margins": 0.007235622499138117,
      "rewards/rejected": 0.0028216370847076178,
      "step": 2787
    },
    {
      "epoch": 0.7333344293018133,
      "grad_norm": 448.0,
      "learning_rate": 1.4809941520467837e-07,
      "logits/chosen": -0.6540394425392151,
      "logits/rejected": -0.6644504070281982,
      "logps/chosen": -1280.04150390625,
      "logps/rejected": -889.4193725585938,
      "loss": 0.6917,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.005512143485248089,
      "rewards/margins": 0.0036212923005223274,
      "rewards/rejected": -0.009133435785770416,
      "step": 2788
    },
    {
      "epoch": 0.7335974617370005,
      "grad_norm": 552.0,
      "learning_rate": 1.47953216374269e-07,
      "logits/chosen": -0.6684538125991821,
      "logits/rejected": NaN,
      "logps/chosen": -1246.61865234375,
      "logps/rejected": -1263.1849365234375,
      "loss": 0.7039,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.005100440699607134,
      "rewards/margins": -0.02057666704058647,
      "rewards/rejected": 0.015476226806640625,
      "step": 2789
    },
    {
      "epoch": 0.7338604941721876,
      "grad_norm": 492.0,
      "learning_rate": 1.4780701754385964e-07,
      "logits/chosen": -0.6741089820861816,
      "logits/rejected": -0.6855223178863525,
      "logps/chosen": -1243.986572265625,
      "logps/rejected": -904.1809692382812,
      "loss": 0.684,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.034960322082042694,
      "rewards/margins": 0.01877264678478241,
      "rewards/rejected": 0.01618766598403454,
      "step": 2790
    },
    {
      "epoch": 0.7341235266073748,
      "grad_norm": 560.0,
      "learning_rate": 1.4766081871345028e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6521430015563965,
      "logps/chosen": -857.2271728515625,
      "logps/rejected": -954.5675048828125,
      "loss": 0.6955,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.012257575988769531,
      "rewards/margins": -0.0034454353153705597,
      "rewards/rejected": -0.008812140673398972,
      "step": 2791
    },
    {
      "epoch": 0.7343865590425619,
      "grad_norm": 552.0,
      "learning_rate": 1.4751461988304093e-07,
      "logits/chosen": -0.6345746517181396,
      "logits/rejected": -0.6372842788696289,
      "logps/chosen": -847.2598876953125,
      "logps/rejected": -873.610107421875,
      "loss": 0.6812,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.01085815392434597,
      "rewards/margins": 0.02463054656982422,
      "rewards/rejected": -0.013772392645478249,
      "step": 2792
    },
    {
      "epoch": 0.7346495914777491,
      "grad_norm": 466.0,
      "learning_rate": 1.4736842105263155e-07,
      "logits/chosen": -0.6449092030525208,
      "logits/rejected": -0.6397304534912109,
      "logps/chosen": -1059.446044921875,
      "logps/rejected": -877.572265625,
      "loss": 0.6826,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.029506303369998932,
      "rewards/margins": 0.021953202784061432,
      "rewards/rejected": 0.007553101051598787,
      "step": 2793
    },
    {
      "epoch": 0.7349126239129362,
      "grad_norm": 524.0,
      "learning_rate": 1.4722222222222222e-07,
      "logits/chosen": -0.6614583134651184,
      "logits/rejected": -0.6710426211357117,
      "logps/chosen": -1210.6219482421875,
      "logps/rejected": -800.7382202148438,
      "loss": 0.6965,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.007378387730568647,
      "rewards/margins": -0.006205845158547163,
      "rewards/rejected": -0.001172542106360197,
      "step": 2794
    },
    {
      "epoch": 0.7351756563481234,
      "grad_norm": 500.0,
      "learning_rate": 1.4707602339181284e-07,
      "logits/chosen": -0.6450443267822266,
      "logits/rejected": -0.6656309962272644,
      "logps/chosen": -1049.324951171875,
      "logps/rejected": -926.43310546875,
      "loss": 0.675,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.027350615710020065,
      "rewards/margins": 0.03734974563121796,
      "rewards/rejected": -0.00999913178384304,
      "step": 2795
    },
    {
      "epoch": 0.7354386887833106,
      "grad_norm": 552.0,
      "learning_rate": 1.4692982456140351e-07,
      "logits/chosen": -0.6339246034622192,
      "logits/rejected": -0.6313683986663818,
      "logps/chosen": -1190.2432861328125,
      "logps/rejected": -1029.6240234375,
      "loss": 0.6835,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.008285045623779297,
      "rewards/margins": 0.019788170233368874,
      "rewards/rejected": -0.011503123678267002,
      "step": 2796
    },
    {
      "epoch": 0.7357017212184978,
      "grad_norm": 470.0,
      "learning_rate": 1.4678362573099416e-07,
      "logits/chosen": -0.6613317728042603,
      "logits/rejected": -0.6663296222686768,
      "logps/chosen": -1067.236572265625,
      "logps/rejected": -1021.6639404296875,
      "loss": 0.6938,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0019967069383710623,
      "rewards/margins": -0.0005569455679506063,
      "rewards/rejected": 0.002553652971982956,
      "step": 2797
    },
    {
      "epoch": 0.735964753653685,
      "grad_norm": 472.0,
      "learning_rate": 1.4663742690058478e-07,
      "logits/chosen": -0.6455118656158447,
      "logits/rejected": -0.6492133140563965,
      "logps/chosen": -1058.8494873046875,
      "logps/rejected": -1072.08740234375,
      "loss": 0.6956,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.010071851313114166,
      "rewards/margins": -0.0042793285101652145,
      "rewards/rejected": 0.014351178891956806,
      "step": 2798
    },
    {
      "epoch": 0.7362277860888721,
      "grad_norm": 632.0,
      "learning_rate": 1.4649122807017545e-07,
      "logits/chosen": -0.6528271436691284,
      "logits/rejected": -0.6594875454902649,
      "logps/chosen": -1256.1092529296875,
      "logps/rejected": -1160.0889892578125,
      "loss": 0.71,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.007068920414894819,
      "rewards/margins": -0.03224201127886772,
      "rewards/rejected": 0.025173094123601913,
      "step": 2799
    },
    {
      "epoch": 0.7364908185240593,
      "grad_norm": 600.0,
      "learning_rate": 1.4634502923976607e-07,
      "logits/chosen": -0.6674180626869202,
      "logits/rejected": -0.6689170598983765,
      "logps/chosen": -1274.165283203125,
      "logps/rejected": -728.4630126953125,
      "loss": 0.6927,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0038723002653568983,
      "rewards/margins": 0.0029321680776774883,
      "rewards/rejected": 0.0009401328861713409,
      "step": 2800
    },
    {
      "epoch": 0.7367538509592464,
      "grad_norm": 640.0,
      "learning_rate": 1.4619883040935672e-07,
      "logits/chosen": -0.6392984390258789,
      "logits/rejected": -0.6436573266983032,
      "logps/chosen": -1096.7337646484375,
      "logps/rejected": -890.942626953125,
      "loss": 0.6921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008210184052586555,
      "rewards/margins": 0.003107358468696475,
      "rewards/rejected": 0.005102824419736862,
      "step": 2801
    },
    {
      "epoch": 0.7370168833944336,
      "grad_norm": 556.0,
      "learning_rate": 1.4605263157894737e-07,
      "logits/chosen": -0.6567301750183105,
      "logits/rejected": NaN,
      "logps/chosen": -1091.66748046875,
      "logps/rejected": -846.6884765625,
      "loss": 0.6992,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.0012284276308491826,
      "rewards/margins": -0.011368371546268463,
      "rewards/rejected": 0.012596799060702324,
      "step": 2802
    },
    {
      "epoch": 0.7372799158296207,
      "grad_norm": 504.0,
      "learning_rate": 1.45906432748538e-07,
      "logits/chosen": -0.658532440662384,
      "logits/rejected": -0.6825674176216125,
      "logps/chosen": -994.8868408203125,
      "logps/rejected": -967.7178344726562,
      "loss": 0.691,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006802368443459272,
      "rewards/margins": 0.005285739898681641,
      "rewards/rejected": 0.0015166286611929536,
      "step": 2803
    },
    {
      "epoch": 0.7375429482648079,
      "grad_norm": 468.0,
      "learning_rate": 1.4576023391812863e-07,
      "logits/chosen": -0.6450074911117554,
      "logits/rejected": -0.6514925360679626,
      "logps/chosen": -1163.1832275390625,
      "logps/rejected": -1061.1287841796875,
      "loss": 0.6826,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.009712887927889824,
      "rewards/margins": 0.021603010594844818,
      "rewards/rejected": -0.03131590038537979,
      "step": 2804
    },
    {
      "epoch": 0.737805980699995,
      "grad_norm": 572.0,
      "learning_rate": 1.456140350877193e-07,
      "logits/chosen": -0.6672284603118896,
      "logits/rejected": -0.6609168648719788,
      "logps/chosen": -1256.2391357421875,
      "logps/rejected": -930.4191284179688,
      "loss": 0.6797,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0011235238052904606,
      "rewards/margins": 0.028020191937685013,
      "rewards/rejected": -0.029143715277314186,
      "step": 2805
    },
    {
      "epoch": 0.7380690131351823,
      "grad_norm": 496.0,
      "learning_rate": 1.4546783625730993e-07,
      "logits/chosen": -0.6575067639350891,
      "logits/rejected": -0.6615526676177979,
      "logps/chosen": -1307.5595703125,
      "logps/rejected": -1017.8839111328125,
      "loss": 0.6847,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0007848746608942747,
      "rewards/margins": 0.017710400745272636,
      "rewards/rejected": -0.016925524920225143,
      "step": 2806
    },
    {
      "epoch": 0.7383320455703694,
      "grad_norm": 528.0,
      "learning_rate": 1.4532163742690057e-07,
      "logits/chosen": -0.6497219204902649,
      "logits/rejected": -0.6579546928405762,
      "logps/chosen": -992.97021484375,
      "logps/rejected": -795.361328125,
      "loss": 0.6921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009019089862704277,
      "rewards/margins": 0.0028284071013331413,
      "rewards/rejected": -0.011847496032714844,
      "step": 2807
    },
    {
      "epoch": 0.7385950780055566,
      "grad_norm": 608.0,
      "learning_rate": 1.4517543859649122e-07,
      "logits/chosen": -0.6494870781898499,
      "logits/rejected": -0.6436678767204285,
      "logps/chosen": -1163.3497314453125,
      "logps/rejected": -764.9762573242188,
      "loss": 0.6929,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.004176139831542969,
      "rewards/margins": 0.0010676388628780842,
      "rewards/rejected": -0.00524377916008234,
      "step": 2808
    },
    {
      "epoch": 0.7388581104407437,
      "grad_norm": 466.0,
      "learning_rate": 1.4502923976608186e-07,
      "logits/chosen": -0.6297202110290527,
      "logits/rejected": -0.6333346962928772,
      "logps/chosen": -958.8175659179688,
      "logps/rejected": -1139.1712646484375,
      "loss": 0.6895,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.018270112574100494,
      "rewards/margins": 0.00770607078447938,
      "rewards/rejected": -0.025976182892918587,
      "step": 2809
    },
    {
      "epoch": 0.7391211428759309,
      "grad_norm": 536.0,
      "learning_rate": 1.4488304093567248e-07,
      "logits/chosen": -0.6449642181396484,
      "logits/rejected": -0.6506787538528442,
      "logps/chosen": -1289.74462890625,
      "logps/rejected": -1138.2108154296875,
      "loss": 0.6814,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.006085681263357401,
      "rewards/margins": 0.024576712399721146,
      "rewards/rejected": -0.018491029739379883,
      "step": 2810
    },
    {
      "epoch": 0.739384175311118,
      "grad_norm": 676.0,
      "learning_rate": 1.4473684210526316e-07,
      "logits/chosen": -0.6812987327575684,
      "logits/rejected": -0.6613837480545044,
      "logps/chosen": -846.7111206054688,
      "logps/rejected": -867.71923828125,
      "loss": 0.7025,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.005131388083100319,
      "rewards/margins": -0.018025731667876244,
      "rewards/rejected": 0.023157117888331413,
      "step": 2811
    },
    {
      "epoch": 0.7396472077463052,
      "grad_norm": 446.0,
      "learning_rate": 1.445906432748538e-07,
      "logits/chosen": -0.6537484526634216,
      "logits/rejected": -0.6510051488876343,
      "logps/chosen": -1081.9300537109375,
      "logps/rejected": -853.2740478515625,
      "loss": 0.6927,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.000981807941570878,
      "rewards/margins": 0.0016705500893294811,
      "rewards/rejected": -0.0026523589622229338,
      "step": 2812
    },
    {
      "epoch": 0.7399102401814924,
      "grad_norm": 592.0,
      "learning_rate": 1.4444444444444442e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6848729848861694,
      "logps/chosen": -1023.384765625,
      "logps/rejected": -930.9459228515625,
      "loss": 0.6989,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.013453102670609951,
      "rewards/margins": -0.010706139728426933,
      "rewards/rejected": -0.0027469638735055923,
      "step": 2813
    },
    {
      "epoch": 0.7401732726166795,
      "grad_norm": 572.0,
      "learning_rate": 1.442982456140351e-07,
      "logits/chosen": -0.6258058547973633,
      "logits/rejected": -0.6285696625709534,
      "logps/chosen": -1203.126708984375,
      "logps/rejected": -835.5938110351562,
      "loss": 0.6884,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0022707940079271793,
      "rewards/margins": 0.010632514953613281,
      "rewards/rejected": -0.00836172141134739,
      "step": 2814
    },
    {
      "epoch": 0.7404363050518667,
      "grad_norm": 556.0,
      "learning_rate": 1.4415204678362572e-07,
      "logits/chosen": -0.6577321290969849,
      "logits/rejected": -0.6618712544441223,
      "logps/chosen": -1564.5089111328125,
      "logps/rejected": -1063.820556640625,
      "loss": 0.6991,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.00460443552583456,
      "rewards/margins": -0.010864161886274815,
      "rewards/rejected": 0.015468599274754524,
      "step": 2815
    },
    {
      "epoch": 0.7406993374870539,
      "grad_norm": 524.0,
      "learning_rate": 1.440058479532164e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6411066651344299,
      "logps/chosen": -769.9176635742188,
      "logps/rejected": -886.3406982421875,
      "loss": 0.6964,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0021017075050622225,
      "rewards/margins": -0.005865667015314102,
      "rewards/rejected": 0.0037639630027115345,
      "step": 2816
    },
    {
      "epoch": 0.7409623699222411,
      "grad_norm": 552.0,
      "learning_rate": 1.43859649122807e-07,
      "logits/chosen": -0.6699517965316772,
      "logits/rejected": -0.668219804763794,
      "logps/chosen": -1197.087646484375,
      "logps/rejected": -1156.261962890625,
      "loss": 0.6925,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.013480758294463158,
      "rewards/margins": 0.00208129920065403,
      "rewards/rejected": 0.011399460025131702,
      "step": 2817
    },
    {
      "epoch": 0.7412254023574282,
      "grad_norm": 502.0,
      "learning_rate": 1.4371345029239766e-07,
      "logits/chosen": -0.6556330919265747,
      "logits/rejected": -0.664771318435669,
      "logps/chosen": -948.6073608398438,
      "logps/rejected": -753.1939697265625,
      "loss": 0.691,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0024900436401367188,
      "rewards/margins": 0.005753135308623314,
      "rewards/rejected": -0.003263092366978526,
      "step": 2818
    },
    {
      "epoch": 0.7414884347926154,
      "grad_norm": 524.0,
      "learning_rate": 1.435672514619883e-07,
      "logits/chosen": -0.6468884944915771,
      "logits/rejected": -0.6505743265151978,
      "logps/chosen": -1079.7938232421875,
      "logps/rejected": -721.0860595703125,
      "loss": 0.7018,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.004790211096405983,
      "rewards/margins": -0.016822433099150658,
      "rewards/rejected": 0.012032223865389824,
      "step": 2819
    },
    {
      "epoch": 0.7417514672278025,
      "grad_norm": 436.0,
      "learning_rate": 1.4342105263157895e-07,
      "logits/chosen": -0.6680721640586853,
      "logits/rejected": -0.6629724502563477,
      "logps/chosen": -796.9575805664062,
      "logps/rejected": -599.84765625,
      "loss": 0.6831,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006266498472541571,
      "rewards/margins": 0.02043447643518448,
      "rewards/rejected": -0.014167976565659046,
      "step": 2820
    },
    {
      "epoch": 0.7420144996629897,
      "grad_norm": 436.0,
      "learning_rate": 1.4327485380116957e-07,
      "logits/chosen": -0.6224533915519714,
      "logits/rejected": -0.6241660118103027,
      "logps/chosen": -809.344970703125,
      "logps/rejected": -650.1683349609375,
      "loss": 0.7057,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.015360069461166859,
      "rewards/margins": -0.02422313578426838,
      "rewards/rejected": 0.00886306818574667,
      "step": 2821
    },
    {
      "epoch": 0.7422775320981768,
      "grad_norm": 520.0,
      "learning_rate": 1.4312865497076024e-07,
      "logits/chosen": -0.6621918678283691,
      "logits/rejected": -0.6617098450660706,
      "logps/chosen": -1053.949462890625,
      "logps/rejected": -636.3399658203125,
      "loss": 0.6937,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0002683163620531559,
      "rewards/margins": 0.0012548938393592834,
      "rewards/rejected": -0.0009865760803222656,
      "step": 2822
    },
    {
      "epoch": 0.742540564533364,
      "grad_norm": 600.0,
      "learning_rate": 1.4298245614035086e-07,
      "logits/chosen": -0.6684285402297974,
      "logits/rejected": -0.6702723503112793,
      "logps/chosen": -1311.188720703125,
      "logps/rejected": -1207.6788330078125,
      "loss": 0.6981,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.008822061121463776,
      "rewards/margins": -0.008623600006103516,
      "rewards/rejected": -0.00019845901988446712,
      "step": 2823
    },
    {
      "epoch": 0.7428035969685511,
      "grad_norm": 548.0,
      "learning_rate": 1.428362573099415e-07,
      "logits/chosen": -0.6547247171401978,
      "logits/rejected": -0.6557334065437317,
      "logps/chosen": -1280.232666015625,
      "logps/rejected": -712.16845703125,
      "loss": 0.7022,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.010619448497891426,
      "rewards/margins": -0.017319774255156517,
      "rewards/rejected": 0.006700325291603804,
      "step": 2824
    },
    {
      "epoch": 0.7430666294037384,
      "grad_norm": 432.0,
      "learning_rate": 1.4269005847953215e-07,
      "logits/chosen": -0.6522669792175293,
      "logits/rejected": -0.6578132510185242,
      "logps/chosen": -876.403076171875,
      "logps/rejected": -802.1896362304688,
      "loss": 0.6972,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0030246733222156763,
      "rewards/margins": -0.007840252481400967,
      "rewards/rejected": 0.004815578926354647,
      "step": 2825
    },
    {
      "epoch": 0.7433296618389255,
      "grad_norm": 520.0,
      "learning_rate": 1.425438596491228e-07,
      "logits/chosen": -0.6631354093551636,
      "logits/rejected": -0.6756503582000732,
      "logps/chosen": -1145.0064697265625,
      "logps/rejected": -1060.922119140625,
      "loss": 0.6935,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00954589806497097,
      "rewards/margins": -5.532056093215942e-06,
      "rewards/rejected": 0.009551430121064186,
      "step": 2826
    },
    {
      "epoch": 0.7435926942741127,
      "grad_norm": 482.0,
      "learning_rate": 1.4239766081871345e-07,
      "logits/chosen": -0.6563794612884521,
      "logits/rejected": -0.6581883430480957,
      "logps/chosen": -1339.2774658203125,
      "logps/rejected": -1094.3255615234375,
      "loss": 0.6882,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.026412010192871094,
      "rewards/margins": 0.01081104390323162,
      "rewards/rejected": -0.037223052233457565,
      "step": 2827
    },
    {
      "epoch": 0.7438557267092999,
      "grad_norm": 652.0,
      "learning_rate": 1.422514619883041e-07,
      "logits/chosen": -0.6320923566818237,
      "logits/rejected": -0.6351295709609985,
      "logps/chosen": -1387.177001953125,
      "logps/rejected": -1014.970458984375,
      "loss": 0.6942,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011058807373046875,
      "rewards/margins": -0.0006893177051097155,
      "rewards/rejected": -0.010369490832090378,
      "step": 2828
    },
    {
      "epoch": 0.744118759144487,
      "grad_norm": 516.0,
      "learning_rate": 1.4210526315789474e-07,
      "logits/chosen": -0.6382217407226562,
      "logits/rejected": -0.6362735033035278,
      "logps/chosen": -916.4055786132812,
      "logps/rejected": -830.4010620117188,
      "loss": 0.7026,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.00042228668462485075,
      "rewards/margins": -0.018198110163211823,
      "rewards/rejected": 0.01862039603292942,
      "step": 2829
    },
    {
      "epoch": 0.7443817915796742,
      "grad_norm": 486.0,
      "learning_rate": 1.4195906432748536e-07,
      "logits/chosen": -0.6314092874526978,
      "logits/rejected": -0.6419201493263245,
      "logps/chosen": -1115.34326171875,
      "logps/rejected": -1157.85205078125,
      "loss": 0.6886,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.013622571714222431,
      "rewards/margins": 0.009508037939667702,
      "rewards/rejected": 0.00411453191190958,
      "step": 2830
    },
    {
      "epoch": 0.7446448240148613,
      "grad_norm": 528.0,
      "learning_rate": 1.4181286549707603e-07,
      "logits/chosen": -0.6492841243743896,
      "logits/rejected": -0.6513559222221375,
      "logps/chosen": -1121.8739013671875,
      "logps/rejected": -948.150390625,
      "loss": 0.6957,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.014403153210878372,
      "rewards/margins": -0.0037611965090036392,
      "rewards/rejected": -0.010641956701874733,
      "step": 2831
    },
    {
      "epoch": 0.7449078564500485,
      "grad_norm": 524.0,
      "learning_rate": 1.4166666666666665e-07,
      "logits/chosen": -0.6566678285598755,
      "logits/rejected": -0.6537079215049744,
      "logps/chosen": -1215.7396240234375,
      "logps/rejected": -1057.9776611328125,
      "loss": 0.6994,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.00466117775067687,
      "rewards/margins": -0.01185703370720148,
      "rewards/rejected": 0.007195853628218174,
      "step": 2832
    },
    {
      "epoch": 0.7451708888852356,
      "grad_norm": 584.0,
      "learning_rate": 1.415204678362573e-07,
      "logits/chosen": -0.6542900204658508,
      "logits/rejected": -0.6535565257072449,
      "logps/chosen": -1255.5308837890625,
      "logps/rejected": -1022.7744750976562,
      "loss": 0.6946,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.004945563618093729,
      "rewards/margins": -0.0017232894897460938,
      "rewards/rejected": -0.0032222741283476353,
      "step": 2833
    },
    {
      "epoch": 0.7454339213204229,
      "grad_norm": 572.0,
      "learning_rate": 1.4137426900584794e-07,
      "logits/chosen": -0.6665676832199097,
      "logits/rejected": -0.6709633469581604,
      "logps/chosen": -1695.3551025390625,
      "logps/rejected": -1115.334716796875,
      "loss": 0.7,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0011238094884902239,
      "rewards/margins": -0.01264038123190403,
      "rewards/rejected": 0.01151657197624445,
      "step": 2834
    },
    {
      "epoch": 0.74569695375561,
      "grad_norm": 548.0,
      "learning_rate": 1.412280701754386e-07,
      "logits/chosen": -0.6385918259620667,
      "logits/rejected": -0.6368504762649536,
      "logps/chosen": -1152.4102783203125,
      "logps/rejected": -1168.2762451171875,
      "loss": 0.6789,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.011355876922607422,
      "rewards/margins": 0.02982654795050621,
      "rewards/rejected": -0.01847066916525364,
      "step": 2835
    },
    {
      "epoch": 0.7459599861907972,
      "grad_norm": 486.0,
      "learning_rate": 1.4108187134502924e-07,
      "logits/chosen": -0.6437346339225769,
      "logits/rejected": -0.6555527448654175,
      "logps/chosen": -1144.1285400390625,
      "logps/rejected": -798.9088745117188,
      "loss": 0.7069,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.024910833686590195,
      "rewards/margins": -0.02638835832476616,
      "rewards/rejected": 0.001477526966482401,
      "step": 2836
    },
    {
      "epoch": 0.7462230186259843,
      "grad_norm": 532.0,
      "learning_rate": 1.4093567251461988e-07,
      "logits/chosen": -0.6722620725631714,
      "logits/rejected": -0.6670442819595337,
      "logps/chosen": -1175.8016357421875,
      "logps/rejected": -943.110595703125,
      "loss": 0.7021,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.02772665210068226,
      "rewards/margins": -0.017053985968232155,
      "rewards/rejected": -0.010672666132450104,
      "step": 2837
    },
    {
      "epoch": 0.7464860510611715,
      "grad_norm": 502.0,
      "learning_rate": 1.407894736842105e-07,
      "logits/chosen": -0.6668862104415894,
      "logits/rejected": -0.663573145866394,
      "logps/chosen": -874.0318603515625,
      "logps/rejected": -874.1575927734375,
      "loss": 0.6994,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.006462764926254749,
      "rewards/margins": -0.012115096673369408,
      "rewards/rejected": 0.005652332212775946,
      "step": 2838
    },
    {
      "epoch": 0.7467490834963586,
      "grad_norm": 532.0,
      "learning_rate": 1.4064327485380118e-07,
      "logits/chosen": -0.6565831899642944,
      "logits/rejected": NaN,
      "logps/chosen": -1304.1551513671875,
      "logps/rejected": -696.1148681640625,
      "loss": 0.6923,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008270359598100185,
      "rewards/margins": 0.002488899976015091,
      "rewards/rejected": 0.005781460087746382,
      "step": 2839
    },
    {
      "epoch": 0.7470121159315458,
      "grad_norm": 668.0,
      "learning_rate": 1.404970760233918e-07,
      "logits/chosen": -0.6459083557128906,
      "logits/rejected": -0.651572585105896,
      "logps/chosen": -1070.053466796875,
      "logps/rejected": -938.9510498046875,
      "loss": 0.6935,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0011061187833547592,
      "rewards/margins": -0.0004937648773193359,
      "rewards/rejected": -0.0006123539060354233,
      "step": 2840
    },
    {
      "epoch": 0.7472751483667329,
      "grad_norm": 556.0,
      "learning_rate": 1.4035087719298244e-07,
      "logits/chosen": -0.6113241314888,
      "logits/rejected": -0.6242326498031616,
      "logps/chosen": -1183.1630859375,
      "logps/rejected": -1001.0397338867188,
      "loss": 0.6964,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.006498337257653475,
      "rewards/margins": -0.005628681275993586,
      "rewards/rejected": -0.0008696555159986019,
      "step": 2841
    },
    {
      "epoch": 0.7475381808019201,
      "grad_norm": 468.0,
      "learning_rate": 1.4020467836257312e-07,
      "logits/chosen": -0.6435651183128357,
      "logits/rejected": -0.6448827385902405,
      "logps/chosen": -1151.6055908203125,
      "logps/rejected": -931.039306640625,
      "loss": 0.698,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.006904506590217352,
      "rewards/margins": -0.009045791812241077,
      "rewards/rejected": 0.01595029979944229,
      "step": 2842
    },
    {
      "epoch": 0.7478012132371074,
      "grad_norm": 524.0,
      "learning_rate": 1.4005847953216374e-07,
      "logits/chosen": -0.6558980941772461,
      "logits/rejected": -0.6531051397323608,
      "logps/chosen": -1232.780517578125,
      "logps/rejected": -748.8264770507812,
      "loss": 0.6929,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.007551956921815872,
      "rewards/margins": 0.0010718352859839797,
      "rewards/rejected": -0.008623791858553886,
      "step": 2843
    },
    {
      "epoch": 0.7480642456722945,
      "grad_norm": 616.0,
      "learning_rate": 1.3991228070175438e-07,
      "logits/chosen": -0.6454529166221619,
      "logits/rejected": -0.6391083598136902,
      "logps/chosen": -1181.07666015625,
      "logps/rejected": -775.9994506835938,
      "loss": 0.7093,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01691265217959881,
      "rewards/margins": -0.03053407371044159,
      "rewards/rejected": 0.013621426187455654,
      "step": 2844
    },
    {
      "epoch": 0.7483272781074817,
      "grad_norm": 510.0,
      "learning_rate": 1.3976608187134503e-07,
      "logits/chosen": -0.6589341163635254,
      "logits/rejected": -0.6558266878128052,
      "logps/chosen": -1191.86279296875,
      "logps/rejected": -1009.4698486328125,
      "loss": 0.6988,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.016065027564764023,
      "rewards/margins": -0.010699749924242496,
      "rewards/rejected": 0.026764774695038795,
      "step": 2845
    },
    {
      "epoch": 0.7485903105426688,
      "grad_norm": 584.0,
      "learning_rate": 1.3961988304093567e-07,
      "logits/chosen": -0.6524041891098022,
      "logits/rejected": -0.6520177721977234,
      "logps/chosen": -1604.1578369140625,
      "logps/rejected": -1507.3951416015625,
      "loss": 0.6965,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0029491428285837173,
      "rewards/margins": -0.005398941226303577,
      "rewards/rejected": 0.00834808312356472,
      "step": 2846
    },
    {
      "epoch": 0.748853342977856,
      "grad_norm": 588.0,
      "learning_rate": 1.394736842105263e-07,
      "logits/chosen": -0.6668943166732788,
      "logits/rejected": -0.6586811542510986,
      "logps/chosen": -1201.516357421875,
      "logps/rejected": -849.9126586914062,
      "loss": 0.6968,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.024584483355283737,
      "rewards/margins": -0.006687642075121403,
      "rewards/rejected": -0.01789684221148491,
      "step": 2847
    },
    {
      "epoch": 0.7491163754130431,
      "grad_norm": 636.0,
      "learning_rate": 1.3932748538011697e-07,
      "logits/chosen": -0.6555715799331665,
      "logits/rejected": -0.6498937010765076,
      "logps/chosen": -1457.2528076171875,
      "logps/rejected": -811.0347290039062,
      "loss": 0.7017,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.024450400844216347,
      "rewards/margins": -0.015402890741825104,
      "rewards/rejected": -0.009047510102391243,
      "step": 2848
    },
    {
      "epoch": 0.7493794078482303,
      "grad_norm": 520.0,
      "learning_rate": 1.391812865497076e-07,
      "logits/chosen": -0.615814208984375,
      "logits/rejected": -0.6055403351783752,
      "logps/chosen": -894.00390625,
      "logps/rejected": -737.65478515625,
      "loss": 0.7026,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.013521958142518997,
      "rewards/margins": -0.017908480018377304,
      "rewards/rejected": 0.004386520944535732,
      "step": 2849
    },
    {
      "epoch": 0.7496424402834174,
      "grad_norm": 496.0,
      "learning_rate": 1.3903508771929823e-07,
      "logits/chosen": -0.6576830744743347,
      "logits/rejected": -0.6563670039176941,
      "logps/chosen": -1048.1595458984375,
      "logps/rejected": -706.1083374023438,
      "loss": 0.6917,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.003975869156420231,
      "rewards/margins": 0.0034140595234930515,
      "rewards/rejected": 0.0005618096329271793,
      "step": 2850
    },
    {
      "epoch": 0.7499054727186046,
      "grad_norm": 616.0,
      "learning_rate": 1.3888888888888888e-07,
      "logits/chosen": -0.6514093279838562,
      "logits/rejected": NaN,
      "logps/chosen": -731.8203735351562,
      "logps/rejected": -602.3480834960938,
      "loss": 0.6891,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003906250465661287,
      "rewards/margins": 0.008883573114871979,
      "rewards/rejected": -0.004977322183549404,
      "step": 2851
    },
    {
      "epoch": 0.7501685051537917,
      "grad_norm": 584.0,
      "learning_rate": 1.3874269005847953e-07,
      "logits/chosen": -0.6510293483734131,
      "logits/rejected": -0.6572810411453247,
      "logps/chosen": -1081.2523193359375,
      "logps/rejected": -1106.086181640625,
      "loss": 0.6968,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.006578636355698109,
      "rewards/margins": -0.00632224278524518,
      "rewards/rejected": -0.0002563952002674341,
      "step": 2852
    },
    {
      "epoch": 0.750431537588979,
      "grad_norm": 560.0,
      "learning_rate": 1.3859649122807015e-07,
      "logits/chosen": -0.6797590255737305,
      "logits/rejected": -0.6627106666564941,
      "logps/chosen": -1102.0374755859375,
      "logps/rejected": -994.3728637695312,
      "loss": 0.6941,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.01851820945739746,
      "rewards/margins": -0.0015847677132114768,
      "rewards/rejected": -0.016933441162109375,
      "step": 2853
    },
    {
      "epoch": 0.7506945700241661,
      "grad_norm": 584.0,
      "learning_rate": 1.3845029239766082e-07,
      "logits/chosen": -0.6493445634841919,
      "logits/rejected": -0.6472252011299133,
      "logps/chosen": -903.3333740234375,
      "logps/rejected": -1005.7427978515625,
      "loss": 0.6977,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.014477730728685856,
      "rewards/margins": -0.00758280698210001,
      "rewards/rejected": -0.006894922815263271,
      "step": 2854
    },
    {
      "epoch": 0.7509576024593533,
      "grad_norm": 568.0,
      "learning_rate": 1.3830409356725144e-07,
      "logits/chosen": -0.64121413230896,
      "logits/rejected": -0.6436904072761536,
      "logps/chosen": -1085.3621826171875,
      "logps/rejected": -857.326416015625,
      "loss": 0.6951,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.005198096856474876,
      "rewards/margins": -0.003084946423768997,
      "rewards/rejected": 0.008283043280243874,
      "step": 2855
    },
    {
      "epoch": 0.7512206348945404,
      "grad_norm": 452.0,
      "learning_rate": 1.381578947368421e-07,
      "logits/chosen": -0.6534321308135986,
      "logits/rejected": -0.6517364978790283,
      "logps/chosen": -746.4666748046875,
      "logps/rejected": -607.0756225585938,
      "loss": 0.6921,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0032073974143713713,
      "rewards/margins": 0.0024813655763864517,
      "rewards/rejected": 0.0007260320708155632,
      "step": 2856
    },
    {
      "epoch": 0.7514836673297276,
      "grad_norm": 624.0,
      "learning_rate": 1.3801169590643276e-07,
      "logits/chosen": -0.6420336365699768,
      "logits/rejected": -0.6497561931610107,
      "logps/chosen": -1365.7913818359375,
      "logps/rejected": -796.6788330078125,
      "loss": 0.701,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.008725166320800781,
      "rewards/margins": -0.014573669992387295,
      "rewards/rejected": 0.005848502740263939,
      "step": 2857
    },
    {
      "epoch": 0.7517466997649148,
      "grad_norm": 908.0,
      "learning_rate": 1.3786549707602338e-07,
      "logits/chosen": -0.6063089370727539,
      "logits/rejected": -0.6247958540916443,
      "logps/chosen": -1275.62158203125,
      "logps/rejected": -958.4146118164062,
      "loss": 0.6939,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.017577460035681725,
      "rewards/margins": -0.0008793836459517479,
      "rewards/rejected": -0.016698075458407402,
      "step": 2858
    },
    {
      "epoch": 0.7520097322001019,
      "grad_norm": 560.0,
      "learning_rate": 1.3771929824561405e-07,
      "logits/chosen": -0.626107394695282,
      "logits/rejected": -0.6241676807403564,
      "logps/chosen": -1131.732177734375,
      "logps/rejected": -982.7147216796875,
      "loss": 0.6878,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.015516280196607113,
      "rewards/margins": 0.011284827254712582,
      "rewards/rejected": 0.0042314534075558186,
      "step": 2859
    },
    {
      "epoch": 0.7522727646352891,
      "grad_norm": 496.0,
      "learning_rate": 1.3757309941520467e-07,
      "logits/chosen": -0.6412286758422852,
      "logits/rejected": -0.6523388028144836,
      "logps/chosen": -858.5220947265625,
      "logps/rejected": -789.43115234375,
      "loss": 0.6954,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01885814592242241,
      "rewards/margins": -0.0038494099862873554,
      "rewards/rejected": -0.015008736401796341,
      "step": 2860
    },
    {
      "epoch": 0.7525357970704762,
      "grad_norm": 552.0,
      "learning_rate": 1.3742690058479532e-07,
      "logits/chosen": -0.6643031239509583,
      "logits/rejected": -0.6736161112785339,
      "logps/chosen": -1233.312744140625,
      "logps/rejected": -1025.0557861328125,
      "loss": 0.6983,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.024642374366521835,
      "rewards/margins": -0.010163117200136185,
      "rewards/rejected": -0.014479255303740501,
      "step": 2861
    },
    {
      "epoch": 0.7527988295056635,
      "grad_norm": 696.0,
      "learning_rate": 1.3728070175438596e-07,
      "logits/chosen": -0.6502087116241455,
      "logits/rejected": -0.6425583362579346,
      "logps/chosen": -1654.7476806640625,
      "logps/rejected": -1289.56884765625,
      "loss": 0.6994,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.012768364511430264,
      "rewards/margins": -0.01153793465346098,
      "rewards/rejected": -0.0012304307892918587,
      "step": 2862
    },
    {
      "epoch": 0.7530618619408506,
      "grad_norm": 382.0,
      "learning_rate": 1.371345029239766e-07,
      "logits/chosen": -0.6315250992774963,
      "logits/rejected": -0.6414729952812195,
      "logps/chosen": -782.4785766601562,
      "logps/rejected": -492.3353576660156,
      "loss": 0.6919,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.022052479907870293,
      "rewards/margins": 0.0030139917507767677,
      "rewards/rejected": -0.025066472589969635,
      "step": 2863
    },
    {
      "epoch": 0.7533248943760378,
      "grad_norm": 502.0,
      "learning_rate": 1.3698830409356723e-07,
      "logits/chosen": -0.6462290287017822,
      "logits/rejected": -0.6511814594268799,
      "logps/chosen": -1199.297607421875,
      "logps/rejected": -1203.2857666015625,
      "loss": 0.6942,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.031111670657992363,
      "rewards/margins": -0.0007664188742637634,
      "rewards/rejected": 0.031878091394901276,
      "step": 2864
    },
    {
      "epoch": 0.7535879268112249,
      "grad_norm": 484.0,
      "learning_rate": 1.368421052631579e-07,
      "logits/chosen": -0.6670660972595215,
      "logits/rejected": -0.6587507724761963,
      "logps/chosen": -1196.574951171875,
      "logps/rejected": -1105.2020263671875,
      "loss": 0.711,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.028478147462010384,
      "rewards/margins": -0.034903526306152344,
      "rewards/rejected": 0.006425380706787109,
      "step": 2865
    },
    {
      "epoch": 0.7538509592464121,
      "grad_norm": 448.0,
      "learning_rate": 1.3669590643274852e-07,
      "logits/chosen": -0.6645668745040894,
      "logits/rejected": -0.6686758399009705,
      "logps/chosen": -998.0924682617188,
      "logps/rejected": -757.4697265625,
      "loss": 0.6973,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0024175639264285564,
      "rewards/margins": -0.007639978546649218,
      "rewards/rejected": 0.005222415551543236,
      "step": 2866
    },
    {
      "epoch": 0.7541139916815992,
      "grad_norm": 592.0,
      "learning_rate": 1.3654970760233917e-07,
      "logits/chosen": -0.6685031652450562,
      "logits/rejected": -0.6696310639381409,
      "logps/chosen": -1057.813720703125,
      "logps/rejected": -956.0999755859375,
      "loss": 0.693,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.016450118273496628,
      "rewards/margins": 0.0012475019320845604,
      "rewards/rejected": 0.015202617272734642,
      "step": 2867
    },
    {
      "epoch": 0.7543770241167864,
      "grad_norm": 636.0,
      "learning_rate": 1.3640350877192982e-07,
      "logits/chosen": -0.6560739278793335,
      "logits/rejected": -0.6726723313331604,
      "logps/chosen": -1668.9345703125,
      "logps/rejected": -1479.9044189453125,
      "loss": 0.6898,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01499862689524889,
      "rewards/margins": 0.008434867486357689,
      "rewards/rejected": 0.006563759408891201,
      "step": 2868
    },
    {
      "epoch": 0.7546400565519735,
      "grad_norm": 428.0,
      "learning_rate": 1.3625730994152046e-07,
      "logits/chosen": -0.6597565412521362,
      "logits/rejected": -0.6596719026565552,
      "logps/chosen": -1069.579345703125,
      "logps/rejected": -1023.8731079101562,
      "loss": 0.6923,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.013259600847959518,
      "rewards/margins": 0.0021209707483649254,
      "rewards/rejected": -0.015380572527647018,
      "step": 2869
    },
    {
      "epoch": 0.7549030889871607,
      "grad_norm": 604.0,
      "learning_rate": 1.3611111111111108e-07,
      "logits/chosen": -0.6568987369537354,
      "logits/rejected": -0.653316855430603,
      "logps/chosen": -1313.018798828125,
      "logps/rejected": -1111.183837890625,
      "loss": 0.7072,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.014445113949477673,
      "rewards/margins": -0.025542832911014557,
      "rewards/rejected": 0.011097718961536884,
      "step": 2870
    },
    {
      "epoch": 0.7551661214223478,
      "grad_norm": 478.0,
      "learning_rate": 1.3596491228070175e-07,
      "logits/chosen": -0.6606465578079224,
      "logits/rejected": -0.6774815917015076,
      "logps/chosen": -1140.212890625,
      "logps/rejected": -793.3834838867188,
      "loss": 0.6894,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02038755640387535,
      "rewards/margins": 0.008987808600068092,
      "rewards/rejected": 0.011399745009839535,
      "step": 2871
    },
    {
      "epoch": 0.7554291538575351,
      "grad_norm": 632.0,
      "learning_rate": 1.358187134502924e-07,
      "logits/chosen": -0.648594856262207,
      "logits/rejected": NaN,
      "logps/chosen": -1499.9576416015625,
      "logps/rejected": -1067.6280517578125,
      "loss": 0.681,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.01822815090417862,
      "rewards/margins": 0.02487487904727459,
      "rewards/rejected": -0.00664672814309597,
      "step": 2872
    },
    {
      "epoch": 0.7556921862927222,
      "grad_norm": 584.0,
      "learning_rate": 1.3567251461988302e-07,
      "logits/chosen": -0.6360582709312439,
      "logits/rejected": -0.6287628412246704,
      "logps/chosen": -1240.3546142578125,
      "logps/rejected": -1017.6780395507812,
      "loss": 0.707,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.023273944854736328,
      "rewards/margins": -0.02676372602581978,
      "rewards/rejected": 0.003489780705422163,
      "step": 2873
    },
    {
      "epoch": 0.7559552187279094,
      "grad_norm": 580.0,
      "learning_rate": 1.355263157894737e-07,
      "logits/chosen": -0.65123450756073,
      "logits/rejected": -0.6508548259735107,
      "logps/chosen": -1247.5426025390625,
      "logps/rejected": -892.3460693359375,
      "loss": 0.6907,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.016141558066010475,
      "rewards/margins": 0.005907298065721989,
      "rewards/rejected": 0.010234260000288486,
      "step": 2874
    },
    {
      "epoch": 0.7562182511630966,
      "grad_norm": 596.0,
      "learning_rate": 1.3538011695906431e-07,
      "logits/chosen": -0.657943069934845,
      "logits/rejected": -0.6656385064125061,
      "logps/chosen": -1378.8345947265625,
      "logps/rejected": -1246.94189453125,
      "loss": 0.6844,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.021577931940555573,
      "rewards/margins": 0.018635842949151993,
      "rewards/rejected": 0.0029420857317745686,
      "step": 2875
    },
    {
      "epoch": 0.7564812835982837,
      "grad_norm": 580.0,
      "learning_rate": 1.3523391812865499e-07,
      "logits/chosen": -0.6579172611236572,
      "logits/rejected": -0.6600273847579956,
      "logps/chosen": -1419.59423828125,
      "logps/rejected": -653.5997924804688,
      "loss": 0.6847,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.015509702265262604,
      "rewards/margins": 0.01814851723611355,
      "rewards/rejected": -0.0026388168334960938,
      "step": 2876
    },
    {
      "epoch": 0.7567443160334709,
      "grad_norm": 564.0,
      "learning_rate": 1.350877192982456e-07,
      "logits/chosen": -0.6417227983474731,
      "logits/rejected": -0.6575944423675537,
      "logps/chosen": -1426.0306396484375,
      "logps/rejected": -1137.728271484375,
      "loss": 0.6933,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008169079199433327,
      "rewards/margins": 0.00024375878274440765,
      "rewards/rejected": 0.007925320416688919,
      "step": 2877
    },
    {
      "epoch": 0.757007348468658,
      "grad_norm": 506.0,
      "learning_rate": 1.3494152046783625e-07,
      "logits/chosen": -0.6460013389587402,
      "logits/rejected": -0.6373641490936279,
      "logps/chosen": -829.8611450195312,
      "logps/rejected": -956.5850219726562,
      "loss": 0.6979,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.002337264595553279,
      "rewards/margins": -0.007994554936885834,
      "rewards/rejected": 0.0056572905741631985,
      "step": 2878
    },
    {
      "epoch": 0.7572703809038452,
      "grad_norm": 512.0,
      "learning_rate": 1.347953216374269e-07,
      "logits/chosen": -0.6419814229011536,
      "logits/rejected": -0.6459513902664185,
      "logps/chosen": -1318.68359375,
      "logps/rejected": -1099.21435546875,
      "loss": 0.6896,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.012612342834472656,
      "rewards/margins": 0.007558059878647327,
      "rewards/rejected": 0.005054282955825329,
      "step": 2879
    },
    {
      "epoch": 0.7575334133390323,
      "grad_norm": 640.0,
      "learning_rate": 1.3464912280701755e-07,
      "logits/chosen": -0.6565784215927124,
      "logits/rejected": -0.6487941145896912,
      "logps/chosen": -1135.7200927734375,
      "logps/rejected": -975.0217895507812,
      "loss": 0.7004,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008233070373535156,
      "rewards/margins": -0.012933063320815563,
      "rewards/rejected": 0.004699993412941694,
      "step": 2880
    },
    {
      "epoch": 0.7577964457742196,
      "grad_norm": 668.0,
      "learning_rate": 1.3450292397660817e-07,
      "logits/chosen": -0.6568828821182251,
      "logits/rejected": -0.6384181976318359,
      "logps/chosen": -1276.1024169921875,
      "logps/rejected": -1200.542724609375,
      "loss": 0.6992,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.004854965955018997,
      "rewards/margins": -0.010898018255829811,
      "rewards/rejected": 0.006043051835149527,
      "step": 2881
    },
    {
      "epoch": 0.7580594782094067,
      "grad_norm": 548.0,
      "learning_rate": 1.3435672514619884e-07,
      "logits/chosen": -0.6527139544487,
      "logits/rejected": -0.66792893409729,
      "logps/chosen": -1052.3919677734375,
      "logps/rejected": -933.7657470703125,
      "loss": 0.6862,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.004604530520737171,
      "rewards/margins": 0.014379501342773438,
      "rewards/rejected": -0.018984030932188034,
      "step": 2882
    },
    {
      "epoch": 0.7583225106445939,
      "grad_norm": 652.0,
      "learning_rate": 1.3421052631578946e-07,
      "logits/chosen": -0.615423858165741,
      "logits/rejected": -0.6239583492279053,
      "logps/chosen": -911.5792236328125,
      "logps/rejected": -560.8551025390625,
      "loss": 0.7015,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.017015933990478516,
      "rewards/margins": -0.01611451990902424,
      "rewards/rejected": -0.0009014131501317024,
      "step": 2883
    },
    {
      "epoch": 0.758585543079781,
      "grad_norm": 568.0,
      "learning_rate": 1.340643274853801e-07,
      "logits/chosen": -0.6759713888168335,
      "logits/rejected": -0.6701563596725464,
      "logps/chosen": -1229.3150634765625,
      "logps/rejected": -1012.4299926757812,
      "loss": 0.686,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0015460022259503603,
      "rewards/margins": 0.015229130163788795,
      "rewards/rejected": -0.013683128170669079,
      "step": 2884
    },
    {
      "epoch": 0.7588485755149682,
      "grad_norm": 676.0,
      "learning_rate": 1.3391812865497075e-07,
      "logits/chosen": -0.6408696174621582,
      "logits/rejected": -0.6340575218200684,
      "logps/chosen": -1209.471435546875,
      "logps/rejected": -1161.840576171875,
      "loss": 0.7003,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.018430424854159355,
      "rewards/margins": -0.01199941523373127,
      "rewards/rejected": 0.030429840087890625,
      "step": 2885
    },
    {
      "epoch": 0.7591116079501553,
      "grad_norm": 508.0,
      "learning_rate": 1.337719298245614e-07,
      "logits/chosen": -0.6497725248336792,
      "logits/rejected": NaN,
      "logps/chosen": -905.3015747070312,
      "logps/rejected": -807.0968017578125,
      "loss": 0.6913,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005910682491958141,
      "rewards/margins": 0.00433502160012722,
      "rewards/rejected": -0.010245705023407936,
      "step": 2886
    },
    {
      "epoch": 0.7593746403853425,
      "grad_norm": 524.0,
      "learning_rate": 1.3362573099415204e-07,
      "logits/chosen": -0.656498372554779,
      "logits/rejected": -0.6623371243476868,
      "logps/chosen": -1021.5068969726562,
      "logps/rejected": -689.8670654296875,
      "loss": 0.711,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.02635040506720543,
      "rewards/margins": -0.034230850636959076,
      "rewards/rejected": 0.007880449295043945,
      "step": 2887
    },
    {
      "epoch": 0.7596376728205296,
      "grad_norm": 470.0,
      "learning_rate": 1.334795321637427e-07,
      "logits/chosen": -0.6598510146141052,
      "logits/rejected": -0.6625670194625854,
      "logps/chosen": -842.7522583007812,
      "logps/rejected": -680.3128662109375,
      "loss": 0.6925,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0016502384096384048,
      "rewards/margins": 0.001641798298805952,
      "rewards/rejected": -0.003292036708444357,
      "step": 2888
    },
    {
      "epoch": 0.7599007052557168,
      "grad_norm": 422.0,
      "learning_rate": 1.3333333333333334e-07,
      "logits/chosen": -0.6563752889633179,
      "logits/rejected": -0.6569174528121948,
      "logps/chosen": -1367.53076171875,
      "logps/rejected": -1102.89599609375,
      "loss": 0.6881,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.005733775440603495,
      "rewards/margins": 0.01030588336288929,
      "rewards/rejected": -0.004572105593979359,
      "step": 2889
    },
    {
      "epoch": 0.7601637376909041,
      "grad_norm": 552.0,
      "learning_rate": 1.3318713450292396e-07,
      "logits/chosen": -0.6494410634040833,
      "logits/rejected": -0.6751868724822998,
      "logps/chosen": -1378.60546875,
      "logps/rejected": -734.5305786132812,
      "loss": 0.6872,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006455325521528721,
      "rewards/margins": 0.012345502153038979,
      "rewards/rejected": -0.005890179425477982,
      "step": 2890
    },
    {
      "epoch": 0.7604267701260912,
      "grad_norm": 438.0,
      "learning_rate": 1.3304093567251463e-07,
      "logits/chosen": -0.6272516846656799,
      "logits/rejected": -0.6228997707366943,
      "logps/chosen": -901.1267700195312,
      "logps/rejected": -751.1129150390625,
      "loss": 0.6914,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010811137035489082,
      "rewards/margins": 0.004589080344885588,
      "rewards/rejected": 0.006222057621926069,
      "step": 2891
    },
    {
      "epoch": 0.7606898025612784,
      "grad_norm": 544.0,
      "learning_rate": 1.3289473684210525e-07,
      "logits/chosen": -0.6504191160202026,
      "logits/rejected": -0.6497429013252258,
      "logps/chosen": -1116.0902099609375,
      "logps/rejected": -729.9406127929688,
      "loss": 0.6804,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.007408713456243277,
      "rewards/margins": 0.026308059692382812,
      "rewards/rejected": -0.01889934577047825,
      "step": 2892
    },
    {
      "epoch": 0.7609528349964655,
      "grad_norm": 482.0,
      "learning_rate": 1.327485380116959e-07,
      "logits/chosen": -0.6489050388336182,
      "logits/rejected": -0.6519675254821777,
      "logps/chosen": -902.3904418945312,
      "logps/rejected": -709.076904296875,
      "loss": 0.6964,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0012046813499182463,
      "rewards/margins": -0.005560209043323994,
      "rewards/rejected": 0.006764889694750309,
      "step": 2893
    },
    {
      "epoch": 0.7612158674316527,
      "grad_norm": 464.0,
      "learning_rate": 1.3260233918128654e-07,
      "logits/chosen": -0.6589822769165039,
      "logits/rejected": -0.6559711694717407,
      "logps/chosen": -1050.6851806640625,
      "logps/rejected": -760.375244140625,
      "loss": 0.6834,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.008087255991995335,
      "rewards/margins": 0.020135022699832916,
      "rewards/rejected": -0.012047767639160156,
      "step": 2894
    },
    {
      "epoch": 0.7614788998668398,
      "grad_norm": 672.0,
      "learning_rate": 1.324561403508772e-07,
      "logits/chosen": -0.6586939096450806,
      "logits/rejected": -0.6735078692436218,
      "logps/chosen": -1309.2799072265625,
      "logps/rejected": -1226.6817626953125,
      "loss": 0.6999,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01174392644315958,
      "rewards/margins": -0.010475875809788704,
      "rewards/rejected": 0.02221979945898056,
      "step": 2895
    },
    {
      "epoch": 0.761741932302027,
      "grad_norm": 442.0,
      "learning_rate": 1.3230994152046783e-07,
      "logits/chosen": -0.6450798511505127,
      "logits/rejected": -0.6580714583396912,
      "logps/chosen": -1116.3935546875,
      "logps/rejected": -733.23828125,
      "loss": 0.6921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.014256381429731846,
      "rewards/margins": 0.0027151103131473064,
      "rewards/rejected": 0.011541271582245827,
      "step": 2896
    },
    {
      "epoch": 0.7620049647372141,
      "grad_norm": 564.0,
      "learning_rate": 1.3216374269005848e-07,
      "logits/chosen": -0.6383944153785706,
      "logits/rejected": -0.6390804052352905,
      "logps/chosen": -1045.447509765625,
      "logps/rejected": -1072.6485595703125,
      "loss": 0.6914,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0012148375390097499,
      "rewards/margins": 0.004234790336340666,
      "rewards/rejected": -0.0030199536122381687,
      "step": 2897
    },
    {
      "epoch": 0.7622679971724013,
      "grad_norm": 584.0,
      "learning_rate": 1.320175438596491e-07,
      "logits/chosen": -0.6651397347450256,
      "logits/rejected": -0.6658257842063904,
      "logps/chosen": -1267.8455810546875,
      "logps/rejected": -1041.513427734375,
      "loss": 0.6961,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.011106586083769798,
      "rewards/margins": -0.005466460715979338,
      "rewards/rejected": -0.005640125833451748,
      "step": 2898
    },
    {
      "epoch": 0.7625310296075885,
      "grad_norm": 592.0,
      "learning_rate": 1.3187134502923977e-07,
      "logits/chosen": -0.6402325630187988,
      "logits/rejected": -0.6442704200744629,
      "logps/chosen": -1052.02587890625,
      "logps/rejected": -996.8583984375,
      "loss": 0.6836,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.010133743286132812,
      "rewards/margins": 0.02065715752542019,
      "rewards/rejected": -0.03079090267419815,
      "step": 2899
    },
    {
      "epoch": 0.7627940620427757,
      "grad_norm": 568.0,
      "learning_rate": 1.317251461988304e-07,
      "logits/chosen": -0.6520156860351562,
      "logits/rejected": -0.6676664352416992,
      "logps/chosen": -1281.2047119140625,
      "logps/rejected": -972.5804443359375,
      "loss": 0.6884,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008137131109833717,
      "rewards/margins": 0.010580631904304028,
      "rewards/rejected": -0.002443504286929965,
      "step": 2900
    },
    {
      "epoch": 0.7630570944779628,
      "grad_norm": 660.0,
      "learning_rate": 1.3157894736842104e-07,
      "logits/chosen": -0.6500234603881836,
      "logits/rejected": -0.6496862173080444,
      "logps/chosen": -1177.5224609375,
      "logps/rejected": -790.7552490234375,
      "loss": 0.6953,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0021087643690407276,
      "rewards/margins": -0.003585577942430973,
      "rewards/rejected": 0.005694341845810413,
      "step": 2901
    },
    {
      "epoch": 0.76332012691315,
      "grad_norm": 560.0,
      "learning_rate": 1.314327485380117e-07,
      "logits/chosen": -0.6137101650238037,
      "logits/rejected": -0.626933217048645,
      "logps/chosen": -785.6611938476562,
      "logps/rejected": -466.38397216796875,
      "loss": 0.6925,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.00948476791381836,
      "rewards/margins": 0.0018846499733626842,
      "rewards/rejected": 0.007600117474794388,
      "step": 2902
    },
    {
      "epoch": 0.7635831593483371,
      "grad_norm": 502.0,
      "learning_rate": 1.3128654970760233e-07,
      "logits/chosen": -0.6462364196777344,
      "logits/rejected": -0.6572590470314026,
      "logps/chosen": -1035.8687744140625,
      "logps/rejected": -780.1309204101562,
      "loss": 0.6926,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.015423775650560856,
      "rewards/margins": 0.0024161324836313725,
      "rewards/rejected": 0.013007640838623047,
      "step": 2903
    },
    {
      "epoch": 0.7638461917835243,
      "grad_norm": 540.0,
      "learning_rate": 1.3114035087719298e-07,
      "logits/chosen": -0.6555508971214294,
      "logits/rejected": -0.6555748581886292,
      "logps/chosen": -1083.2069091796875,
      "logps/rejected": -928.3280029296875,
      "loss": 0.6955,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0005743028596043587,
      "rewards/margins": -0.0031950478442013264,
      "rewards/rejected": 0.0026207452174276114,
      "step": 2904
    },
    {
      "epoch": 0.7641092242187115,
      "grad_norm": 580.0,
      "learning_rate": 1.3099415204678363e-07,
      "logits/chosen": -0.6673796772956848,
      "logits/rejected": -0.6728176474571228,
      "logps/chosen": -1371.78125,
      "logps/rejected": -599.9879150390625,
      "loss": 0.6913,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0006235130131244659,
      "rewards/margins": 0.004463100340217352,
      "rewards/rejected": -0.0038395882584154606,
      "step": 2905
    },
    {
      "epoch": 0.7643722566538986,
      "grad_norm": 474.0,
      "learning_rate": 1.3084795321637427e-07,
      "logits/chosen": -0.6396604776382446,
      "logits/rejected": -0.6485790014266968,
      "logps/chosen": -942.5158081054688,
      "logps/rejected": -609.0107421875,
      "loss": 0.7001,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.012314606457948685,
      "rewards/margins": -0.012927485629916191,
      "rewards/rejected": 0.0006128789391368628,
      "step": 2906
    },
    {
      "epoch": 0.7646352890890858,
      "grad_norm": 572.0,
      "learning_rate": 1.307017543859649e-07,
      "logits/chosen": -0.6337326169013977,
      "logits/rejected": -0.6417036056518555,
      "logps/chosen": -1343.503662109375,
      "logps/rejected": -1170.1536865234375,
      "loss": 0.6999,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0007949825376272202,
      "rewards/margins": -0.01225071121007204,
      "rewards/rejected": 0.011455726809799671,
      "step": 2907
    },
    {
      "epoch": 0.764898321524273,
      "grad_norm": 632.0,
      "learning_rate": 1.3055555555555556e-07,
      "logits/chosen": -0.6613941788673401,
      "logits/rejected": -0.665387749671936,
      "logps/chosen": -1433.105224609375,
      "logps/rejected": -1151.6611328125,
      "loss": 0.7034,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.018126774579286575,
      "rewards/margins": -0.019764423370361328,
      "rewards/rejected": 0.0016376491403207183,
      "step": 2908
    },
    {
      "epoch": 0.7651613539594602,
      "grad_norm": 572.0,
      "learning_rate": 1.3040935672514618e-07,
      "logits/chosen": -0.6459305882453918,
      "logits/rejected": -0.6475269198417664,
      "logps/chosen": -1479.9404296875,
      "logps/rejected": -1129.5291748046875,
      "loss": 0.689,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.002532768528908491,
      "rewards/margins": 0.009254932403564453,
      "rewards/rejected": -0.006722164340317249,
      "step": 2909
    },
    {
      "epoch": 0.7654243863946473,
      "grad_norm": 528.0,
      "learning_rate": 1.3026315789473683e-07,
      "logits/chosen": -0.6659905910491943,
      "logits/rejected": -0.6571232080459595,
      "logps/chosen": -1122.71533203125,
      "logps/rejected": -836.293212890625,
      "loss": 0.688,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.011005211621522903,
      "rewards/margins": 0.0106925955042243,
      "rewards/rejected": -0.02169780619442463,
      "step": 2910
    },
    {
      "epoch": 0.7656874188298345,
      "grad_norm": 454.0,
      "learning_rate": 1.3011695906432748e-07,
      "logits/chosen": -0.6823515295982361,
      "logits/rejected": -0.6706895232200623,
      "logps/chosen": -692.4215087890625,
      "logps/rejected": -651.6350708007812,
      "loss": 0.69,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0027593132108449936,
      "rewards/margins": 0.007207442540675402,
      "rewards/rejected": -0.0044481270015239716,
      "step": 2911
    },
    {
      "epoch": 0.7659504512650216,
      "grad_norm": 668.0,
      "learning_rate": 1.2997076023391812e-07,
      "logits/chosen": -0.6608957648277283,
      "logits/rejected": -0.6674666404724121,
      "logps/chosen": -1564.1575927734375,
      "logps/rejected": -1175.7796630859375,
      "loss": 0.6985,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.011435793712735176,
      "rewards/margins": -0.010120201855897903,
      "rewards/rejected": -0.0013155927881598473,
      "step": 2912
    },
    {
      "epoch": 0.7662134837002088,
      "grad_norm": 552.0,
      "learning_rate": 1.2982456140350874e-07,
      "logits/chosen": -0.6804169416427612,
      "logits/rejected": -0.6677326560020447,
      "logps/chosen": -1150.7506103515625,
      "logps/rejected": -1079.69580078125,
      "loss": 0.6958,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.03599090501666069,
      "rewards/margins": -0.004821493290364742,
      "rewards/rejected": -0.031169412657618523,
      "step": 2913
    },
    {
      "epoch": 0.7664765161353959,
      "grad_norm": 544.0,
      "learning_rate": 1.2967836257309942e-07,
      "logits/chosen": -0.6611994504928589,
      "logits/rejected": -0.6754468083381653,
      "logps/chosen": -1216.161865234375,
      "logps/rejected": -816.0665283203125,
      "loss": 0.7009,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.025156976655125618,
      "rewards/margins": -0.01442422904074192,
      "rewards/rejected": -0.010732745751738548,
      "step": 2914
    },
    {
      "epoch": 0.7667395485705831,
      "grad_norm": 496.0,
      "learning_rate": 1.2953216374269004e-07,
      "logits/chosen": -0.6462204456329346,
      "logits/rejected": -0.6441465616226196,
      "logps/chosen": -1056.7042236328125,
      "logps/rejected": -887.6611328125,
      "loss": 0.6898,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.009610462933778763,
      "rewards/margins": 0.007340144831687212,
      "rewards/rejected": 0.00227031740359962,
      "step": 2915
    },
    {
      "epoch": 0.7670025810057702,
      "grad_norm": 592.0,
      "learning_rate": 1.293859649122807e-07,
      "logits/chosen": -0.6307873725891113,
      "logits/rejected": -0.628736138343811,
      "logps/chosen": -1119.88916015625,
      "logps/rejected": -840.6903076171875,
      "loss": 0.6977,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.02230510674417019,
      "rewards/margins": -0.008208656683564186,
      "rewards/rejected": -0.014096450991928577,
      "step": 2916
    },
    {
      "epoch": 0.7672656134409575,
      "grad_norm": 496.0,
      "learning_rate": 1.2923976608187136e-07,
      "logits/chosen": -0.6650938987731934,
      "logits/rejected": -0.6572220325469971,
      "logps/chosen": -1012.1614379882812,
      "logps/rejected": -926.8328857421875,
      "loss": 0.6955,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.001093864324502647,
      "rewards/margins": -0.00413556070998311,
      "rewards/rejected": 0.003041696734726429,
      "step": 2917
    },
    {
      "epoch": 0.7675286458761446,
      "grad_norm": 544.0,
      "learning_rate": 1.2909356725146198e-07,
      "logits/chosen": -0.6572489142417908,
      "logits/rejected": -0.6647710800170898,
      "logps/chosen": -1159.1392822265625,
      "logps/rejected": -1078.8543701171875,
      "loss": 0.6953,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0003619182389229536,
      "rewards/margins": -0.003551100380718708,
      "rewards/rejected": 0.003189182374626398,
      "step": 2918
    },
    {
      "epoch": 0.7677916783113318,
      "grad_norm": 736.0,
      "learning_rate": 1.2894736842105265e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6558340787887573,
      "logps/chosen": -1192.3780517578125,
      "logps/rejected": -990.3837280273438,
      "loss": 0.6862,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0017250055680051446,
      "rewards/margins": 0.015040013007819653,
      "rewards/rejected": -0.013315009884536266,
      "step": 2919
    },
    {
      "epoch": 0.7680547107465189,
      "grad_norm": 496.0,
      "learning_rate": 1.2880116959064327e-07,
      "logits/chosen": -0.6273506879806519,
      "logits/rejected": -0.6275460124015808,
      "logps/chosen": -959.8878784179688,
      "logps/rejected": -825.9881591796875,
      "loss": 0.69,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00854044035077095,
      "rewards/margins": 0.006805610377341509,
      "rewards/rejected": 0.0017348299734294415,
      "step": 2920
    },
    {
      "epoch": 0.7683177431817061,
      "grad_norm": 426.0,
      "learning_rate": 1.2865497076023391e-07,
      "logits/chosen": -0.6450777053833008,
      "logits/rejected": -0.6584668159484863,
      "logps/chosen": -992.3240356445312,
      "logps/rejected": -622.11376953125,
      "loss": 0.6995,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.02519364468753338,
      "rewards/margins": -0.012525893747806549,
      "rewards/rejected": -0.012667751871049404,
      "step": 2921
    },
    {
      "epoch": 0.7685807756168933,
      "grad_norm": 556.0,
      "learning_rate": 1.2850877192982456e-07,
      "logits/chosen": -0.656484842300415,
      "logits/rejected": -0.6586698889732361,
      "logps/chosen": -1523.2509765625,
      "logps/rejected": -1355.0369873046875,
      "loss": 0.7021,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.002774715656414628,
      "rewards/margins": -0.016996100544929504,
      "rewards/rejected": 0.014221381396055222,
      "step": 2922
    },
    {
      "epoch": 0.7688438080520804,
      "grad_norm": 572.0,
      "learning_rate": 1.283625730994152e-07,
      "logits/chosen": -0.6610444784164429,
      "logits/rejected": -0.6652792692184448,
      "logps/chosen": -1287.6591796875,
      "logps/rejected": -1067.7093505859375,
      "loss": 0.7023,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.007934189401566982,
      "rewards/margins": -0.016866160556674004,
      "rewards/rejected": 0.008931972086429596,
      "step": 2923
    },
    {
      "epoch": 0.7691068404872676,
      "grad_norm": 486.0,
      "learning_rate": 1.2821637426900583e-07,
      "logits/chosen": -0.6394516825675964,
      "logits/rejected": -0.6469146013259888,
      "logps/chosen": -939.1729736328125,
      "logps/rejected": -792.9782104492188,
      "loss": 0.6975,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.009140204638242722,
      "rewards/margins": -0.00828847847878933,
      "rewards/rejected": -0.0008517267415300012,
      "step": 2924
    },
    {
      "epoch": 0.7693698729224547,
      "grad_norm": 488.0,
      "learning_rate": 1.280701754385965e-07,
      "logits/chosen": -0.6761570572853088,
      "logits/rejected": -0.6730417609214783,
      "logps/chosen": -1147.21875,
      "logps/rejected": -962.4324951171875,
      "loss": 0.6909,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.006599329877644777,
      "rewards/margins": 0.006222439929842949,
      "rewards/rejected": -0.012821769341826439,
      "step": 2925
    },
    {
      "epoch": 0.769632905357642,
      "grad_norm": 540.0,
      "learning_rate": 1.2792397660818712e-07,
      "logits/chosen": -0.6356284022331238,
      "logits/rejected": -0.634972333908081,
      "logps/chosen": -1474.263916015625,
      "logps/rejected": -1082.116943359375,
      "loss": 0.6854,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.019648361951112747,
      "rewards/margins": 0.01651163026690483,
      "rewards/rejected": 0.0031367307528853416,
      "step": 2926
    },
    {
      "epoch": 0.769895937792829,
      "grad_norm": 494.0,
      "learning_rate": 1.2777777777777777e-07,
      "logits/chosen": -0.6287850141525269,
      "logits/rejected": -0.631969690322876,
      "logps/chosen": -866.1444702148438,
      "logps/rejected": -784.1875,
      "loss": 0.693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.013872146606445312,
      "rewards/margins": 0.0005476949736475945,
      "rewards/rejected": -0.014419840648770332,
      "step": 2927
    },
    {
      "epoch": 0.7701589702280163,
      "grad_norm": 508.0,
      "learning_rate": 1.276315789473684e-07,
      "logits/chosen": -0.6408559083938599,
      "logits/rejected": -0.63983553647995,
      "logps/chosen": -1053.3709716796875,
      "logps/rejected": -877.0570678710938,
      "loss": 0.6803,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.02390460856258869,
      "rewards/margins": 0.026200199499726295,
      "rewards/rejected": -0.002295589307323098,
      "step": 2928
    },
    {
      "epoch": 0.7704220026632034,
      "grad_norm": 510.0,
      "learning_rate": 1.2748538011695906e-07,
      "logits/chosen": -0.6328433752059937,
      "logits/rejected": -0.6395856142044067,
      "logps/chosen": -1029.4827880859375,
      "logps/rejected": -721.7337036132812,
      "loss": 0.6963,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0029979716055095196,
      "rewards/margins": -0.005710314959287643,
      "rewards/rejected": 0.008708285167813301,
      "step": 2929
    },
    {
      "epoch": 0.7706850350983906,
      "grad_norm": 482.0,
      "learning_rate": 1.2733918128654968e-07,
      "logits/chosen": -0.6621591448783875,
      "logits/rejected": -0.6649135947227478,
      "logps/chosen": -1109.80712890625,
      "logps/rejected": -878.7603759765625,
      "loss": 0.688,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.014557743445038795,
      "rewards/margins": 0.010902881622314453,
      "rewards/rejected": -0.02546062506735325,
      "step": 2930
    },
    {
      "epoch": 0.7709480675335777,
      "grad_norm": 456.0,
      "learning_rate": 1.2719298245614035e-07,
      "logits/chosen": -0.6402015089988708,
      "logits/rejected": -0.6470654606819153,
      "logps/chosen": -1144.590576171875,
      "logps/rejected": -784.2476806640625,
      "loss": 0.6908,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.026760101318359375,
      "rewards/margins": 0.00506133958697319,
      "rewards/rejected": 0.021698761731386185,
      "step": 2931
    },
    {
      "epoch": 0.7712110999687649,
      "grad_norm": 508.0,
      "learning_rate": 1.27046783625731e-07,
      "logits/chosen": -0.6359025239944458,
      "logits/rejected": -0.6351311802864075,
      "logps/chosen": -1044.363037109375,
      "logps/rejected": -993.1834716796875,
      "loss": 0.6789,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.02498188242316246,
      "rewards/margins": 0.029418280348181725,
      "rewards/rejected": -0.004436397925019264,
      "step": 2932
    },
    {
      "epoch": 0.771474132403952,
      "grad_norm": 474.0,
      "learning_rate": 1.2690058479532162e-07,
      "logits/chosen": -0.665466845035553,
      "logits/rejected": -0.6629998087882996,
      "logps/chosen": -846.192626953125,
      "logps/rejected": -1138.30224609375,
      "loss": 0.6973,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.02201557159423828,
      "rewards/margins": -0.007827567867934704,
      "rewards/rejected": -0.014188004657626152,
      "step": 2933
    },
    {
      "epoch": 0.7717371648391392,
      "grad_norm": 452.0,
      "learning_rate": 1.267543859649123e-07,
      "logits/chosen": -0.6654480695724487,
      "logits/rejected": -0.6754098534584045,
      "logps/chosen": -888.40380859375,
      "logps/rejected": -692.2198486328125,
      "loss": 0.6975,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.004708671942353249,
      "rewards/margins": -0.008095264434814453,
      "rewards/rejected": 0.003386593423783779,
      "step": 2934
    },
    {
      "epoch": 0.7720001972743263,
      "grad_norm": 520.0,
      "learning_rate": 1.266081871345029e-07,
      "logits/chosen": -0.6434311866760254,
      "logits/rejected": -0.646565854549408,
      "logps/chosen": -898.0802001953125,
      "logps/rejected": -749.1541137695312,
      "loss": 0.6899,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.018010331317782402,
      "rewards/margins": 0.007562732324004173,
      "rewards/rejected": 0.01044759713113308,
      "step": 2935
    },
    {
      "epoch": 0.7722632297095136,
      "grad_norm": 556.0,
      "learning_rate": 1.2646198830409358e-07,
      "logits/chosen": -0.6610864996910095,
      "logits/rejected": -0.6760924458503723,
      "logps/chosen": -1203.2781982421875,
      "logps/rejected": -799.912353515625,
      "loss": 0.6909,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.005577565170824528,
      "rewards/margins": 0.005298711359500885,
      "rewards/rejected": 0.00027885398594662547,
      "step": 2936
    },
    {
      "epoch": 0.7725262621447008,
      "grad_norm": 520.0,
      "learning_rate": 1.263157894736842e-07,
      "logits/chosen": -0.6354469656944275,
      "logits/rejected": NaN,
      "logps/chosen": -910.7615356445312,
      "logps/rejected": -575.707763671875,
      "loss": 0.6972,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.018035124987363815,
      "rewards/margins": -0.006946373265236616,
      "rewards/rejected": -0.011088753119111061,
      "step": 2937
    },
    {
      "epoch": 0.7727892945798879,
      "grad_norm": 536.0,
      "learning_rate": 1.2616959064327485e-07,
      "logits/chosen": -0.6279518604278564,
      "logits/rejected": -0.6269229054450989,
      "logps/chosen": -1336.597900390625,
      "logps/rejected": -942.8492431640625,
      "loss": 0.7016,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.006841850467026234,
      "rewards/margins": -0.016169358044862747,
      "rewards/rejected": 0.023011207580566406,
      "step": 2938
    },
    {
      "epoch": 0.7730523270150751,
      "grad_norm": 536.0,
      "learning_rate": 1.260233918128655e-07,
      "logits/chosen": -0.6676974892616272,
      "logits/rejected": -0.6623947620391846,
      "logps/chosen": -1085.122802734375,
      "logps/rejected": -897.5638427734375,
      "loss": 0.6985,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.0023145675659179688,
      "rewards/margins": -0.009930896572768688,
      "rewards/rejected": 0.012245464138686657,
      "step": 2939
    },
    {
      "epoch": 0.7733153594502622,
      "grad_norm": 462.0,
      "learning_rate": 1.2587719298245614e-07,
      "logits/chosen": -0.6665894985198975,
      "logits/rejected": -0.6598592400550842,
      "logps/chosen": -974.2919311523438,
      "logps/rejected": -686.5120849609375,
      "loss": 0.6848,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0001011854037642479,
      "rewards/margins": 0.017275428399443626,
      "rewards/rejected": -0.01737661473453045,
      "step": 2940
    },
    {
      "epoch": 0.7735783918854494,
      "grad_norm": 592.0,
      "learning_rate": 1.2573099415204676e-07,
      "logits/chosen": -0.6482647061347961,
      "logits/rejected": -0.6486324667930603,
      "logps/chosen": -1199.829833984375,
      "logps/rejected": -1049.510986328125,
      "loss": 0.6944,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003855801187455654,
      "rewards/margins": -0.0019878381863236427,
      "rewards/rejected": 0.005843639373779297,
      "step": 2941
    },
    {
      "epoch": 0.7738414243206365,
      "grad_norm": 640.0,
      "learning_rate": 1.2558479532163744e-07,
      "logits/chosen": -0.6264622211456299,
      "logits/rejected": -0.6251124739646912,
      "logps/chosen": -1208.4071044921875,
      "logps/rejected": -991.9962768554688,
      "loss": 0.6972,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.006384086795151234,
      "rewards/margins": -0.007271957118064165,
      "rewards/rejected": 0.0008878707885742188,
      "step": 2942
    },
    {
      "epoch": 0.7741044567558237,
      "grad_norm": 676.0,
      "learning_rate": 1.2543859649122806e-07,
      "logits/chosen": -0.6654595136642456,
      "logits/rejected": -0.6839585900306702,
      "logps/chosen": -1787.85009765625,
      "logps/rejected": -1781.094482421875,
      "loss": 0.676,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0014420514926314354,
      "rewards/margins": 0.03564129024744034,
      "rewards/rejected": -0.03419923782348633,
      "step": 2943
    },
    {
      "epoch": 0.7743674891910108,
      "grad_norm": 696.0,
      "learning_rate": 1.252923976608187e-07,
      "logits/chosen": -0.6426265239715576,
      "logits/rejected": NaN,
      "logps/chosen": -1491.895751953125,
      "logps/rejected": -1069.772216796875,
      "loss": 0.6991,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.010078240185976028,
      "rewards/margins": -0.011116217821836472,
      "rewards/rejected": 0.0211944617331028,
      "step": 2944
    },
    {
      "epoch": 0.774630521626198,
      "grad_norm": 568.0,
      "learning_rate": 1.2514619883040935e-07,
      "logits/chosen": -0.6500392556190491,
      "logits/rejected": -0.6550148725509644,
      "logps/chosen": -1371.5076904296875,
      "logps/rejected": -1044.63720703125,
      "loss": 0.6746,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0188172347843647,
      "rewards/margins": 0.03887739032506943,
      "rewards/rejected": -0.020060157403349876,
      "step": 2945
    },
    {
      "epoch": 0.7748935540613852,
      "grad_norm": 684.0,
      "learning_rate": 1.25e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6596629619598389,
      "logps/chosen": -1088.54296875,
      "logps/rejected": -810.112060546875,
      "loss": 0.699,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0022165297996252775,
      "rewards/margins": -0.010601140558719635,
      "rewards/rejected": 0.00838460959494114,
      "step": 2946
    },
    {
      "epoch": 0.7751565864965724,
      "grad_norm": 502.0,
      "learning_rate": 1.2485380116959064e-07,
      "logits/chosen": -0.6619423627853394,
      "logits/rejected": -0.6585667729377747,
      "logps/chosen": -911.1507568359375,
      "logps/rejected": -882.9927368164062,
      "loss": 0.6911,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.009537409991025925,
      "rewards/margins": 0.0045191762037575245,
      "rewards/rejected": -0.014056587591767311,
      "step": 2947
    },
    {
      "epoch": 0.7754196189317595,
      "grad_norm": 560.0,
      "learning_rate": 1.247076023391813e-07,
      "logits/chosen": -0.6539044380187988,
      "logits/rejected": -0.6490092277526855,
      "logps/chosen": -1110.29736328125,
      "logps/rejected": -973.7979736328125,
      "loss": 0.7004,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.013790321536362171,
      "rewards/margins": -0.013246728107333183,
      "rewards/rejected": -0.0005435948260128498,
      "step": 2948
    },
    {
      "epoch": 0.7756826513669467,
      "grad_norm": 502.0,
      "learning_rate": 1.2456140350877193e-07,
      "logits/chosen": -0.6628484129905701,
      "logits/rejected": -0.668097734451294,
      "logps/chosen": -1607.3897705078125,
      "logps/rejected": -1310.2005615234375,
      "loss": 0.6808,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0403594970703125,
      "rewards/margins": 0.025326969102025032,
      "rewards/rejected": 0.015032529830932617,
      "step": 2949
    },
    {
      "epoch": 0.7759456838021338,
      "grad_norm": 502.0,
      "learning_rate": 1.2441520467836258e-07,
      "logits/chosen": -0.6463396549224854,
      "logits/rejected": -0.6576664447784424,
      "logps/chosen": -1220.5120849609375,
      "logps/rejected": -1104.130615234375,
      "loss": 0.6935,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008241461589932442,
      "rewards/margins": -0.0001161585096269846,
      "rewards/rejected": -0.00812530517578125,
      "step": 2950
    },
    {
      "epoch": 0.776208716237321,
      "grad_norm": 408.0,
      "learning_rate": 1.242690058479532e-07,
      "logits/chosen": -0.6649579405784607,
      "logits/rejected": -0.6680021286010742,
      "logps/chosen": -824.1510009765625,
      "logps/rejected": -598.02490234375,
      "loss": 0.7002,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.015504550188779831,
      "rewards/margins": -0.013508033938705921,
      "rewards/rejected": -0.001996516715735197,
      "step": 2951
    },
    {
      "epoch": 0.7764717486725082,
      "grad_norm": 528.0,
      "learning_rate": 1.2412280701754385e-07,
      "logits/chosen": -0.636077344417572,
      "logits/rejected": -0.6385592818260193,
      "logps/chosen": -1026.989990234375,
      "logps/rejected": -960.28125,
      "loss": 0.6883,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.009910392574965954,
      "rewards/margins": 0.011008741334080696,
      "rewards/rejected": -0.0010983466636389494,
      "step": 2952
    },
    {
      "epoch": 0.7767347811076953,
      "grad_norm": 628.0,
      "learning_rate": 1.239766081871345e-07,
      "logits/chosen": -0.6899619102478027,
      "logits/rejected": NaN,
      "logps/chosen": -1110.75927734375,
      "logps/rejected": -1076.0870361328125,
      "loss": 0.6816,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.010098936036229134,
      "rewards/margins": 0.02526245079934597,
      "rewards/rejected": -0.01516351755708456,
      "step": 2953
    },
    {
      "epoch": 0.7769978135428826,
      "grad_norm": 596.0,
      "learning_rate": 1.2383040935672514e-07,
      "logits/chosen": -0.6250823736190796,
      "logits/rejected": -0.636611819267273,
      "logps/chosen": -1384.5849609375,
      "logps/rejected": -779.0554809570312,
      "loss": 0.7,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.014867497608065605,
      "rewards/margins": -0.012824440374970436,
      "rewards/rejected": -0.0020430567674338818,
      "step": 2954
    },
    {
      "epoch": 0.7772608459780697,
      "grad_norm": 616.0,
      "learning_rate": 1.2368421052631579e-07,
      "logits/chosen": -0.6419767141342163,
      "logits/rejected": -0.6478297710418701,
      "logps/chosen": -1247.78662109375,
      "logps/rejected": -924.8546752929688,
      "loss": 0.7005,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.009142779745161533,
      "rewards/margins": -0.013456248678267002,
      "rewards/rejected": 0.004313468933105469,
      "step": 2955
    },
    {
      "epoch": 0.7775238784132569,
      "grad_norm": 506.0,
      "learning_rate": 1.2353801169590643e-07,
      "logits/chosen": -0.6545580625534058,
      "logits/rejected": -0.6584768295288086,
      "logps/chosen": -1610.775146484375,
      "logps/rejected": -1383.1614990234375,
      "loss": 0.6983,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011879349127411842,
      "rewards/margins": -0.00985574722290039,
      "rewards/rejected": -0.0020236014388501644,
      "step": 2956
    },
    {
      "epoch": 0.777786910848444,
      "grad_norm": 480.0,
      "learning_rate": 1.2339181286549708e-07,
      "logits/chosen": -0.6418601274490356,
      "logits/rejected": -0.6388294696807861,
      "logps/chosen": -828.7135620117188,
      "logps/rejected": -717.914306640625,
      "loss": 0.6866,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.006746768485754728,
      "rewards/margins": 0.014011573977768421,
      "rewards/rejected": -0.020758341997861862,
      "step": 2957
    },
    {
      "epoch": 0.7780499432836312,
      "grad_norm": 516.0,
      "learning_rate": 1.232456140350877e-07,
      "logits/chosen": -0.6275331377983093,
      "logits/rejected": -0.6416249871253967,
      "logps/chosen": -1270.9754638671875,
      "logps/rejected": -1193.0316162109375,
      "loss": 0.6899,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.01137762051075697,
      "rewards/margins": 0.0070015909150242805,
      "rewards/rejected": -0.01837921142578125,
      "step": 2958
    },
    {
      "epoch": 0.7783129757188183,
      "grad_norm": 556.0,
      "learning_rate": 1.2309941520467834e-07,
      "logits/chosen": -0.6689621210098267,
      "logits/rejected": -0.6711395978927612,
      "logps/chosen": -1143.3878173828125,
      "logps/rejected": -1220.2705078125,
      "loss": 0.6986,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.010601901449263096,
      "rewards/margins": -0.009872723370790482,
      "rewards/rejected": -0.0007291794754564762,
      "step": 2959
    },
    {
      "epoch": 0.7785760081540055,
      "grad_norm": 524.0,
      "learning_rate": 1.22953216374269e-07,
      "logits/chosen": -0.6318433880805969,
      "logits/rejected": -0.6490228772163391,
      "logps/chosen": -918.5328369140625,
      "logps/rejected": -803.7342529296875,
      "loss": 0.7005,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.0012321476824581623,
      "rewards/margins": -0.013923167251050472,
      "rewards/rejected": 0.015155315399169922,
      "step": 2960
    },
    {
      "epoch": 0.7788390405891926,
      "grad_norm": 502.0,
      "learning_rate": 1.2280701754385964e-07,
      "logits/chosen": -0.6463727951049805,
      "logits/rejected": -0.6391818523406982,
      "logps/chosen": -1115.2716064453125,
      "logps/rejected": -1140.662841796875,
      "loss": 0.6931,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.007838916964828968,
      "rewards/margins": 0.0007185940630733967,
      "rewards/rejected": -0.008557509630918503,
      "step": 2961
    },
    {
      "epoch": 0.7791020730243798,
      "grad_norm": 498.0,
      "learning_rate": 1.2266081871345028e-07,
      "logits/chosen": -0.6577270030975342,
      "logits/rejected": -0.6496078372001648,
      "logps/chosen": -983.7638549804688,
      "logps/rejected": -797.537841796875,
      "loss": 0.6938,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.002861880697309971,
      "rewards/margins": -0.00047025829553604126,
      "rewards/rejected": -0.0023916237987577915,
      "step": 2962
    },
    {
      "epoch": 0.7793651054595669,
      "grad_norm": 1680.0,
      "learning_rate": 1.2251461988304093e-07,
      "logits/chosen": -0.6459437608718872,
      "logits/rejected": NaN,
      "logps/chosen": -1320.662353515625,
      "logps/rejected": -1112.065185546875,
      "loss": 0.6791,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.029017452150583267,
      "rewards/margins": 0.029838280752301216,
      "rewards/rejected": -0.000820827903226018,
      "step": 2963
    },
    {
      "epoch": 0.7796281378947542,
      "grad_norm": 528.0,
      "learning_rate": 1.2236842105263158e-07,
      "logits/chosen": -0.6643575429916382,
      "logits/rejected": -0.6608487367630005,
      "logps/chosen": -890.793212890625,
      "logps/rejected": -818.0955810546875,
      "loss": 0.6921,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.012698078528046608,
      "rewards/margins": 0.0027978909201920033,
      "rewards/rejected": -0.015495968982577324,
      "step": 2964
    },
    {
      "epoch": 0.7798911703299413,
      "grad_norm": 616.0,
      "learning_rate": 1.2222222222222222e-07,
      "logits/chosen": -0.6410666704177856,
      "logits/rejected": -0.6539512872695923,
      "logps/chosen": -1298.90625,
      "logps/rejected": -1178.9246826171875,
      "loss": 0.6865,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02105269394814968,
      "rewards/margins": 0.014084387570619583,
      "rewards/rejected": 0.006968307308852673,
      "step": 2965
    },
    {
      "epoch": 0.7801542027651285,
      "grad_norm": 444.0,
      "learning_rate": 1.2207602339181287e-07,
      "logits/chosen": -0.6565585136413574,
      "logits/rejected": -0.6589246988296509,
      "logps/chosen": -843.2117309570312,
      "logps/rejected": -635.9147338867188,
      "loss": 0.6924,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.008041095919907093,
      "rewards/margins": 0.00230417400598526,
      "rewards/rejected": -0.010345268994569778,
      "step": 2966
    },
    {
      "epoch": 0.7804172352003156,
      "grad_norm": 540.0,
      "learning_rate": 1.2192982456140352e-07,
      "logits/chosen": -0.6582983732223511,
      "logits/rejected": -0.6747217178344727,
      "logps/chosen": -1178.722412109375,
      "logps/rejected": -1093.31884765625,
      "loss": 0.6936,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.005085754673928022,
      "rewards/margins": -0.0004273415543138981,
      "rewards/rejected": 0.005513096693903208,
      "step": 2967
    },
    {
      "epoch": 0.7806802676355028,
      "grad_norm": 450.0,
      "learning_rate": 1.2178362573099414e-07,
      "logits/chosen": -0.6458456516265869,
      "logits/rejected": -0.6478209495544434,
      "logps/chosen": -977.4246826171875,
      "logps/rejected": -814.8508911132812,
      "loss": 0.6958,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.014417266473174095,
      "rewards/margins": -0.0042395577766001225,
      "rewards/rejected": 0.01865682750940323,
      "step": 2968
    },
    {
      "epoch": 0.78094330007069,
      "grad_norm": 548.0,
      "learning_rate": 1.2163742690058478e-07,
      "logits/chosen": -0.65446537733078,
      "logits/rejected": -0.6566927433013916,
      "logps/chosen": -922.3807983398438,
      "logps/rejected": -850.9197998046875,
      "loss": 0.6952,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0017298695165663958,
      "rewards/margins": -0.003592777531594038,
      "rewards/rejected": 0.005322647280991077,
      "step": 2969
    },
    {
      "epoch": 0.7812063325058771,
      "grad_norm": 612.0,
      "learning_rate": 1.2149122807017543e-07,
      "logits/chosen": -0.6541953086853027,
      "logits/rejected": -0.6655829548835754,
      "logps/chosen": -1420.06884765625,
      "logps/rejected": -1275.7353515625,
      "loss": 0.6876,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.013441753573715687,
      "rewards/margins": 0.011710261926054955,
      "rewards/rejected": 0.0017314916476607323,
      "step": 2970
    },
    {
      "epoch": 0.7814693649410643,
      "grad_norm": 524.0,
      "learning_rate": 1.2134502923976607e-07,
      "logits/chosen": -0.6693195700645447,
      "logits/rejected": -0.6743403673171997,
      "logps/chosen": -1129.7998046875,
      "logps/rejected": -870.783203125,
      "loss": 0.6849,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.021114347502589226,
      "rewards/margins": 0.017060471698641777,
      "rewards/rejected": 0.004053880460560322,
      "step": 2971
    },
    {
      "epoch": 0.7817323973762514,
      "grad_norm": 524.0,
      "learning_rate": 1.2119883040935672e-07,
      "logits/chosen": -0.6517824530601501,
      "logits/rejected": -0.6549857258796692,
      "logps/chosen": -1142.203125,
      "logps/rejected": -912.7759399414062,
      "loss": 0.6921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00024623831268399954,
      "rewards/margins": 0.003198862075805664,
      "rewards/rejected": -0.0029526236467063427,
      "step": 2972
    },
    {
      "epoch": 0.7819954298114387,
      "grad_norm": 430.0,
      "learning_rate": 1.2105263157894737e-07,
      "logits/chosen": -0.637471616268158,
      "logits/rejected": -0.6401655673980713,
      "logps/chosen": -769.7769165039062,
      "logps/rejected": -757.48291015625,
      "loss": 0.6947,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.003619813360273838,
      "rewards/margins": -0.0023198602721095085,
      "rewards/rejected": -0.0012999530881643295,
      "step": 2973
    },
    {
      "epoch": 0.7822584622466258,
      "grad_norm": 576.0,
      "learning_rate": 1.2090643274853801e-07,
      "logits/chosen": -0.6818697452545166,
      "logits/rejected": -0.6896127462387085,
      "logps/chosen": -1110.111572265625,
      "logps/rejected": -863.3563842773438,
      "loss": 0.6894,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.0001715649850666523,
      "rewards/margins": 0.008635615929961205,
      "rewards/rejected": -0.00846404954791069,
      "step": 2974
    },
    {
      "epoch": 0.782521494681813,
      "grad_norm": 540.0,
      "learning_rate": 1.2076023391812863e-07,
      "logits/chosen": -0.6698651313781738,
      "logits/rejected": -0.6733620762825012,
      "logps/chosen": -1095.715576171875,
      "logps/rejected": -936.7593994140625,
      "loss": 0.693,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.014054108411073685,
      "rewards/margins": 0.0007480615749955177,
      "rewards/rejected": -0.014802169986069202,
      "step": 2975
    },
    {
      "epoch": 0.7827845271170001,
      "grad_norm": 438.0,
      "learning_rate": 1.206140350877193e-07,
      "logits/chosen": -0.6489357948303223,
      "logits/rejected": -0.6532344818115234,
      "logps/chosen": -995.4595947265625,
      "logps/rejected": -776.760009765625,
      "loss": 0.6826,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.02309408225119114,
      "rewards/margins": 0.021759701892733574,
      "rewards/rejected": 0.0013343817554414272,
      "step": 2976
    },
    {
      "epoch": 0.7830475595521873,
      "grad_norm": 552.0,
      "learning_rate": 1.2046783625730995e-07,
      "logits/chosen": -0.6443350315093994,
      "logits/rejected": -0.6473351716995239,
      "logps/chosen": -1207.53759765625,
      "logps/rejected": -801.4724731445312,
      "loss": 0.6898,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.004357719328254461,
      "rewards/margins": 0.007266044616699219,
      "rewards/rejected": -0.011623764410614967,
      "step": 2977
    },
    {
      "epoch": 0.7833105919873744,
      "grad_norm": 532.0,
      "learning_rate": 1.2032163742690057e-07,
      "logits/chosen": -0.6401515007019043,
      "logits/rejected": -0.6405608057975769,
      "logps/chosen": -1304.346435546875,
      "logps/rejected": -909.208984375,
      "loss": 0.6919,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.004727839957922697,
      "rewards/margins": 0.0034880642779171467,
      "rewards/rejected": -0.008215904235839844,
      "step": 2978
    },
    {
      "epoch": 0.7835736244225616,
      "grad_norm": 528.0,
      "learning_rate": 1.2017543859649122e-07,
      "logits/chosen": -0.6644949316978455,
      "logits/rejected": -0.6751905083656311,
      "logps/chosen": -1074.65478515625,
      "logps/rejected": -996.1395874023438,
      "loss": 0.6926,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008553791791200638,
      "rewards/margins": 0.0014013283653184772,
      "rewards/rejected": -0.009955119341611862,
      "step": 2979
    },
    {
      "epoch": 0.7838366568577487,
      "grad_norm": 432.0,
      "learning_rate": 1.2002923976608187e-07,
      "logits/chosen": -0.6669502854347229,
      "logits/rejected": -0.6687959432601929,
      "logps/chosen": -836.7439575195312,
      "logps/rejected": -574.269287109375,
      "loss": 0.6947,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0020879749208688736,
      "rewards/margins": -0.0027673724107444286,
      "rewards/rejected": 0.004855346400290728,
      "step": 2980
    },
    {
      "epoch": 0.7840996892929359,
      "grad_norm": 612.0,
      "learning_rate": 1.198830409356725e-07,
      "logits/chosen": -0.6377488374710083,
      "logits/rejected": -0.6408671140670776,
      "logps/chosen": -965.1942138671875,
      "logps/rejected": -874.8770751953125,
      "loss": 0.7095,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004439639858901501,
      "rewards/margins": -0.030509568750858307,
      "rewards/rejected": 0.034949205815792084,
      "step": 2981
    },
    {
      "epoch": 0.784362721728123,
      "grad_norm": 584.0,
      "learning_rate": 1.1973684210526316e-07,
      "logits/chosen": -0.6408869624137878,
      "logits/rejected": -0.645166277885437,
      "logps/chosen": -1323.8109130859375,
      "logps/rejected": -921.1005859375,
      "loss": 0.698,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.00028724665753543377,
      "rewards/margins": -0.009242056868970394,
      "rewards/rejected": 0.008954810909926891,
      "step": 2982
    },
    {
      "epoch": 0.7846257541633103,
      "grad_norm": 520.0,
      "learning_rate": 1.195906432748538e-07,
      "logits/chosen": -0.6500129103660583,
      "logits/rejected": -0.6553506255149841,
      "logps/chosen": -1130.749267578125,
      "logps/rejected": -931.5020751953125,
      "loss": 0.6966,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.013669872656464577,
      "rewards/margins": -0.006415367592126131,
      "rewards/rejected": 0.02008523978292942,
      "step": 2983
    },
    {
      "epoch": 0.7848887865984975,
      "grad_norm": 480.0,
      "learning_rate": 1.1944444444444445e-07,
      "logits/chosen": -0.6748690009117126,
      "logits/rejected": -0.6806661486625671,
      "logps/chosen": -875.830810546875,
      "logps/rejected": -829.5699462890625,
      "loss": 0.6962,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.013169383630156517,
      "rewards/margins": -0.005436323583126068,
      "rewards/rejected": -0.007733059115707874,
      "step": 2984
    },
    {
      "epoch": 0.7851518190336846,
      "grad_norm": 552.0,
      "learning_rate": 1.1929824561403507e-07,
      "logits/chosen": -0.6095074415206909,
      "logits/rejected": -0.6333892345428467,
      "logps/chosen": -1279.6907958984375,
      "logps/rejected": -825.252197265625,
      "loss": 0.6881,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0009944923222064972,
      "rewards/margins": 0.01051626168191433,
      "rewards/rejected": -0.011510753072798252,
      "step": 2985
    },
    {
      "epoch": 0.7854148514688718,
      "grad_norm": 492.0,
      "learning_rate": 1.1915204678362573e-07,
      "logits/chosen": -0.6359192728996277,
      "logits/rejected": -0.6324865818023682,
      "logps/chosen": -1252.8883056640625,
      "logps/rejected": -907.541015625,
      "loss": 0.7166,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.04534473642706871,
      "rewards/margins": -0.04533624276518822,
      "rewards/rejected": -8.488539606332779e-06,
      "step": 2986
    },
    {
      "epoch": 0.7856778839040589,
      "grad_norm": 508.0,
      "learning_rate": 1.1900584795321638e-07,
      "logits/chosen": -0.6206987500190735,
      "logits/rejected": -0.6313142776489258,
      "logps/chosen": -1071.276123046875,
      "logps/rejected": -751.1524658203125,
      "loss": 0.6893,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0005922317504882812,
      "rewards/margins": 0.009205438196659088,
      "rewards/rejected": -0.008613205514848232,
      "step": 2987
    },
    {
      "epoch": 0.7859409163392461,
      "grad_norm": 520.0,
      "learning_rate": 1.1885964912280701e-07,
      "logits/chosen": -0.6616555452346802,
      "logits/rejected": -0.6537785530090332,
      "logps/chosen": -1200.8907470703125,
      "logps/rejected": -939.209716796875,
      "loss": 0.6954,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00012626685202121735,
      "rewards/margins": -0.0037302011623978615,
      "rewards/rejected": 0.003856468014419079,
      "step": 2988
    },
    {
      "epoch": 0.7862039487744332,
      "grad_norm": 632.0,
      "learning_rate": 1.1871345029239766e-07,
      "logits/chosen": -0.6441537141799927,
      "logits/rejected": -0.6403115391731262,
      "logps/chosen": -1367.7607421875,
      "logps/rejected": -1128.91015625,
      "loss": 0.7038,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0015002242289483547,
      "rewards/margins": -0.020143698900938034,
      "rewards/rejected": 0.021643925458192825,
      "step": 2989
    },
    {
      "epoch": 0.7864669812096204,
      "grad_norm": 516.0,
      "learning_rate": 1.185672514619883e-07,
      "logits/chosen": -0.6199233531951904,
      "logits/rejected": -0.6236492395401001,
      "logps/chosen": -1078.658447265625,
      "logps/rejected": -791.238525390625,
      "loss": 0.6973,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0032736780121922493,
      "rewards/margins": -0.007944869808852673,
      "rewards/rejected": 0.004671192262321711,
      "step": 2990
    },
    {
      "epoch": 0.7867300136448075,
      "grad_norm": 474.0,
      "learning_rate": 1.1842105263157894e-07,
      "logits/chosen": -0.6867377758026123,
      "logits/rejected": -0.6659277677536011,
      "logps/chosen": -1143.5777587890625,
      "logps/rejected": -1024.14453125,
      "loss": 0.6886,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00656080199405551,
      "rewards/margins": 0.009954358451068401,
      "rewards/rejected": -0.016515161842107773,
      "step": 2991
    },
    {
      "epoch": 0.7869930460799948,
      "grad_norm": 520.0,
      "learning_rate": 1.1827485380116958e-07,
      "logits/chosen": -0.6627026200294495,
      "logits/rejected": -0.6800730228424072,
      "logps/chosen": -1075.56103515625,
      "logps/rejected": -838.4968872070312,
      "loss": 0.7016,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0057468414306640625,
      "rewards/margins": -0.016032695770263672,
      "rewards/rejected": 0.010285856202244759,
      "step": 2992
    },
    {
      "epoch": 0.7872560785151819,
      "grad_norm": 572.0,
      "learning_rate": 1.1812865497076023e-07,
      "logits/chosen": -0.643214762210846,
      "logits/rejected": NaN,
      "logps/chosen": -1134.8876953125,
      "logps/rejected": -736.866455078125,
      "loss": 0.6877,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0020992280915379524,
      "rewards/margins": 0.01158428005874157,
      "rewards/rejected": -0.009485052898526192,
      "step": 2993
    },
    {
      "epoch": 0.7875191109503691,
      "grad_norm": 516.0,
      "learning_rate": 1.1798245614035087e-07,
      "logits/chosen": -0.6436773538589478,
      "logits/rejected": -0.6339983940124512,
      "logps/chosen": -1289.822021484375,
      "logps/rejected": -953.6966552734375,
      "loss": 0.6965,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.012637043371796608,
      "rewards/margins": -0.005869103595614433,
      "rewards/rejected": -0.006767940707504749,
      "step": 2994
    },
    {
      "epoch": 0.7877821433855562,
      "grad_norm": 498.0,
      "learning_rate": 1.1783625730994151e-07,
      "logits/chosen": -0.6525716781616211,
      "logits/rejected": -0.6557015180587769,
      "logps/chosen": -1224.758056640625,
      "logps/rejected": -1313.339599609375,
      "loss": 0.6892,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0041050901636481285,
      "rewards/margins": 0.008972263894975185,
      "rewards/rejected": -0.004867171868681908,
      "step": 2995
    },
    {
      "epoch": 0.7880451758207434,
      "grad_norm": 478.0,
      "learning_rate": 1.1769005847953215e-07,
      "logits/chosen": -0.6255052089691162,
      "logits/rejected": -0.6298424601554871,
      "logps/chosen": -882.44140625,
      "logps/rejected": -735.43359375,
      "loss": 0.6775,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005583858583122492,
      "rewards/margins": 0.032147981226444244,
      "rewards/rejected": -0.02656412310898304,
      "step": 2996
    },
    {
      "epoch": 0.7883082082559305,
      "grad_norm": 576.0,
      "learning_rate": 1.175438596491228e-07,
      "logits/chosen": -0.6456974744796753,
      "logits/rejected": -0.6595993638038635,
      "logps/chosen": -1489.0283203125,
      "logps/rejected": -962.1127319335938,
      "loss": 0.6815,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.008089733310043812,
      "rewards/margins": 0.02411508560180664,
      "rewards/rejected": -0.016025351360440254,
      "step": 2997
    },
    {
      "epoch": 0.7885712406911177,
      "grad_norm": 528.0,
      "learning_rate": 1.1739766081871343e-07,
      "logits/chosen": -0.6535341143608093,
      "logits/rejected": -0.6504753828048706,
      "logps/chosen": -1012.0231323242188,
      "logps/rejected": -829.2048950195312,
      "loss": 0.6797,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03823985531926155,
      "rewards/margins": 0.027817727997899055,
      "rewards/rejected": 0.010422133840620518,
      "step": 2998
    },
    {
      "epoch": 0.7888342731263049,
      "grad_norm": 424.0,
      "learning_rate": 1.172514619883041e-07,
      "logits/chosen": -0.6441114544868469,
      "logits/rejected": -0.6493874192237854,
      "logps/chosen": -823.6314697265625,
      "logps/rejected": -601.8837890625,
      "loss": 0.6945,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.014962196350097656,
      "rewards/margins": -0.002319050021469593,
      "rewards/rejected": -0.012643147259950638,
      "step": 2999
    },
    {
      "epoch": 0.789097305561492,
      "grad_norm": 776.0,
      "learning_rate": 1.1710526315789474e-07,
      "logits/chosen": -0.6550151109695435,
      "logits/rejected": -0.6506397128105164,
      "logps/chosen": -1190.815673828125,
      "logps/rejected": -1224.5966796875,
      "loss": 0.6984,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0007740980945527554,
      "rewards/margins": -0.010095883160829544,
      "rewards/rejected": 0.010869979858398438,
      "step": 3000
    },
    {
      "epoch": 0.7893603379966793,
      "grad_norm": 532.0,
      "learning_rate": 1.1695906432748537e-07,
      "logits/chosen": -0.6601715087890625,
      "logits/rejected": -0.6509326696395874,
      "logps/chosen": -1133.55810546875,
      "logps/rejected": -1027.7188720703125,
      "loss": 0.6747,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.008268642239272594,
      "rewards/margins": 0.03764772415161133,
      "rewards/rejected": -0.029379084706306458,
      "step": 3001
    },
    {
      "epoch": 0.7896233704318664,
      "grad_norm": 636.0,
      "learning_rate": 1.1681286549707602e-07,
      "logits/chosen": -0.6370751261711121,
      "logits/rejected": -0.6399537920951843,
      "logps/chosen": -1168.429443359375,
      "logps/rejected": -892.5645141601562,
      "loss": 0.6999,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.016179850324988365,
      "rewards/margins": -0.012905404902994633,
      "rewards/rejected": -0.0032744407653808594,
      "step": 3002
    },
    {
      "epoch": 0.7898864028670536,
      "grad_norm": 552.0,
      "learning_rate": 1.1666666666666667e-07,
      "logits/chosen": -0.6054538488388062,
      "logits/rejected": -0.6072606444358826,
      "logps/chosen": -1213.3839111328125,
      "logps/rejected": -771.4307861328125,
      "loss": 0.7018,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.03248348459601402,
      "rewards/margins": -0.016810227185487747,
      "rewards/rejected": -0.015673257410526276,
      "step": 3003
    },
    {
      "epoch": 0.7901494353022407,
      "grad_norm": 524.0,
      "learning_rate": 1.1652046783625731e-07,
      "logits/chosen": -0.6543042063713074,
      "logits/rejected": -0.6530195474624634,
      "logps/chosen": -978.1156005859375,
      "logps/rejected": -620.66552734375,
      "loss": 0.6922,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.021552085876464844,
      "rewards/margins": 0.0024414542131125927,
      "rewards/rejected": 0.019110634922981262,
      "step": 3004
    },
    {
      "epoch": 0.7904124677374279,
      "grad_norm": 450.0,
      "learning_rate": 1.1637426900584795e-07,
      "logits/chosen": -0.6406712532043457,
      "logits/rejected": -0.6382167339324951,
      "logps/chosen": -753.9407348632812,
      "logps/rejected": -771.4132690429688,
      "loss": 0.686,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004065322689712048,
      "rewards/margins": 0.014707565307617188,
      "rewards/rejected": -0.01064224261790514,
      "step": 3005
    },
    {
      "epoch": 0.790675500172615,
      "grad_norm": 524.0,
      "learning_rate": 1.1622807017543859e-07,
      "logits/chosen": -0.6414549350738525,
      "logits/rejected": -0.6443113088607788,
      "logps/chosen": -1210.22119140625,
      "logps/rejected": -794.2150268554688,
      "loss": 0.6816,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004156970418989658,
      "rewards/margins": 0.023850250989198685,
      "rewards/rejected": -0.019693277776241302,
      "step": 3006
    },
    {
      "epoch": 0.7909385326078022,
      "grad_norm": 552.0,
      "learning_rate": 1.1608187134502924e-07,
      "logits/chosen": -0.640629768371582,
      "logits/rejected": -0.6356412172317505,
      "logps/chosen": -1210.53564453125,
      "logps/rejected": -896.6212768554688,
      "loss": 0.6868,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0034040447790175676,
      "rewards/margins": 0.013825512491166592,
      "rewards/rejected": -0.017229558899998665,
      "step": 3007
    },
    {
      "epoch": 0.7912015650429893,
      "grad_norm": 632.0,
      "learning_rate": 1.1593567251461987e-07,
      "logits/chosen": -0.6468772292137146,
      "logits/rejected": -0.6453491449356079,
      "logps/chosen": -1295.93603515625,
      "logps/rejected": -913.569580078125,
      "loss": 0.6892,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.004311179742217064,
      "rewards/margins": 0.00868086889386177,
      "rewards/rejected": -0.012992048636078835,
      "step": 3008
    },
    {
      "epoch": 0.7914645974781765,
      "grad_norm": 520.0,
      "learning_rate": 1.1578947368421052e-07,
      "logits/chosen": -0.634351372718811,
      "logits/rejected": -0.6334606409072876,
      "logps/chosen": -1313.59912109375,
      "logps/rejected": -1057.03662109375,
      "loss": 0.6842,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0040305135771632195,
      "rewards/margins": 0.01928587071597576,
      "rewards/rejected": -0.023316383361816406,
      "step": 3009
    },
    {
      "epoch": 0.7917276299133637,
      "grad_norm": 440.0,
      "learning_rate": 1.1564327485380116e-07,
      "logits/chosen": -0.6561477184295654,
      "logits/rejected": -0.656568169593811,
      "logps/chosen": -888.0858154296875,
      "logps/rejected": -782.4454345703125,
      "loss": 0.7018,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.021453620865941048,
      "rewards/margins": -0.01690812036395073,
      "rewards/rejected": -0.004545498173683882,
      "step": 3010
    },
    {
      "epoch": 0.7919906623485509,
      "grad_norm": 592.0,
      "learning_rate": 1.154970760233918e-07,
      "logits/chosen": -0.6574982404708862,
      "logits/rejected": NaN,
      "logps/chosen": -1499.75146484375,
      "logps/rejected": -1168.212646484375,
      "loss": 0.6971,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0035184849984943867,
      "rewards/margins": -0.007394981570541859,
      "rewards/rejected": 0.010913467966020107,
      "step": 3011
    },
    {
      "epoch": 0.792253694783738,
      "grad_norm": 466.0,
      "learning_rate": 1.1535087719298244e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6303816437721252,
      "logps/chosen": -839.3978271484375,
      "logps/rejected": -943.063232421875,
      "loss": 0.6914,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0007652752101421356,
      "rewards/margins": 0.00405163737013936,
      "rewards/rejected": -0.0032863616943359375,
      "step": 3012
    },
    {
      "epoch": 0.7925167272189252,
      "grad_norm": 532.0,
      "learning_rate": 1.1520467836257309e-07,
      "logits/chosen": -0.6640670299530029,
      "logits/rejected": -0.662115216255188,
      "logps/chosen": -848.4741821289062,
      "logps/rejected": -990.502685546875,
      "loss": 0.6902,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0039088246412575245,
      "rewards/margins": 0.006870649755001068,
      "rewards/rejected": -0.01077947672456503,
      "step": 3013
    },
    {
      "epoch": 0.7927797596541123,
      "grad_norm": 520.0,
      "learning_rate": 1.1505847953216375e-07,
      "logits/chosen": -0.6650838255882263,
      "logits/rejected": -0.6838716268539429,
      "logps/chosen": -946.326416015625,
      "logps/rejected": -580.1788940429688,
      "loss": 0.6943,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.007036018650978804,
      "rewards/margins": -0.0013347618514671922,
      "rewards/rejected": -0.005701256450265646,
      "step": 3014
    },
    {
      "epoch": 0.7930427920892995,
      "grad_norm": 596.0,
      "learning_rate": 1.1491228070175438e-07,
      "logits/chosen": -0.6470340490341187,
      "logits/rejected": -0.651513397693634,
      "logps/chosen": -1480.5048828125,
      "logps/rejected": -1048.8822021484375,
      "loss": 0.6967,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0046834927052259445,
      "rewards/margins": -0.005214786157011986,
      "rewards/rejected": 0.009898282587528229,
      "step": 3015
    },
    {
      "epoch": 0.7933058245244867,
      "grad_norm": 512.0,
      "learning_rate": 1.1476608187134503e-07,
      "logits/chosen": -0.6622188091278076,
      "logits/rejected": -0.6670612692832947,
      "logps/chosen": -903.0376586914062,
      "logps/rejected": -887.0097045898438,
      "loss": 0.6835,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.016764450818300247,
      "rewards/margins": 0.01958007737994194,
      "rewards/rejected": -0.0028156270273029804,
      "step": 3016
    },
    {
      "epoch": 0.7935688569596738,
      "grad_norm": 564.0,
      "learning_rate": 1.1461988304093568e-07,
      "logits/chosen": -0.6583251357078552,
      "logits/rejected": -0.6573376655578613,
      "logps/chosen": -1437.6151123046875,
      "logps/rejected": -1312.6976318359375,
      "loss": 0.7008,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01178445853292942,
      "rewards/margins": -0.013778403401374817,
      "rewards/rejected": 0.0019939434714615345,
      "step": 3017
    },
    {
      "epoch": 0.793831889394861,
      "grad_norm": 422.0,
      "learning_rate": 1.1447368421052631e-07,
      "logits/chosen": -0.6457491517066956,
      "logits/rejected": -0.6595008969306946,
      "logps/chosen": -736.9329833984375,
      "logps/rejected": -468.47552490234375,
      "loss": 0.6981,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0027799615636467934,
      "rewards/margins": -0.009323119185864925,
      "rewards/rejected": 0.006543159484863281,
      "step": 3018
    },
    {
      "epoch": 0.7940949218300482,
      "grad_norm": 402.0,
      "learning_rate": 1.1432748538011695e-07,
      "logits/chosen": -0.6476060748100281,
      "logits/rejected": -0.6493068933486938,
      "logps/chosen": -710.2827758789062,
      "logps/rejected": -713.6487426757812,
      "loss": 0.6918,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.01728658564388752,
      "rewards/margins": 0.0032399189658463,
      "rewards/rejected": -0.020526506006717682,
      "step": 3019
    },
    {
      "epoch": 0.7943579542652354,
      "grad_norm": 528.0,
      "learning_rate": 1.141812865497076e-07,
      "logits/chosen": -0.6374313235282898,
      "logits/rejected": -0.6408869028091431,
      "logps/chosen": -1285.1844482421875,
      "logps/rejected": -860.7015380859375,
      "loss": 0.683,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.011444282718002796,
      "rewards/margins": 0.021709349006414413,
      "rewards/rejected": -0.010265066288411617,
      "step": 3020
    },
    {
      "epoch": 0.7946209867004225,
      "grad_norm": 540.0,
      "learning_rate": 1.1403508771929823e-07,
      "logits/chosen": -0.6510322093963623,
      "logits/rejected": -0.6530540585517883,
      "logps/chosen": -1186.562744140625,
      "logps/rejected": -1111.37548828125,
      "loss": 0.6845,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.015636825934052467,
      "rewards/margins": 0.01793689653277397,
      "rewards/rejected": -0.002300071530044079,
      "step": 3021
    },
    {
      "epoch": 0.7948840191356097,
      "grad_norm": 452.0,
      "learning_rate": 1.1388888888888888e-07,
      "logits/chosen": -0.6635140776634216,
      "logits/rejected": -0.6615000367164612,
      "logps/chosen": -845.0914306640625,
      "logps/rejected": -985.9503173828125,
      "loss": 0.6987,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.03229489177465439,
      "rewards/margins": -0.0108321662992239,
      "rewards/rejected": -0.021462729200720787,
      "step": 3022
    },
    {
      "epoch": 0.7951470515707968,
      "grad_norm": 512.0,
      "learning_rate": 1.1374269005847953e-07,
      "logits/chosen": -0.663800835609436,
      "logits/rejected": -0.6739013195037842,
      "logps/chosen": -1221.6329345703125,
      "logps/rejected": -677.0823974609375,
      "loss": 0.6778,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.030390357598662376,
      "rewards/margins": 0.031946372240781784,
      "rewards/rejected": -0.0015560152241960168,
      "step": 3023
    },
    {
      "epoch": 0.795410084005984,
      "grad_norm": 656.0,
      "learning_rate": 1.1359649122807017e-07,
      "logits/chosen": -0.6648057699203491,
      "logits/rejected": -0.6556268930435181,
      "logps/chosen": -1362.3616943359375,
      "logps/rejected": -1116.41650390625,
      "loss": 0.6978,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.013890029862523079,
      "rewards/margins": -0.008702708408236504,
      "rewards/rejected": -0.005187320057302713,
      "step": 3024
    },
    {
      "epoch": 0.7956731164411711,
      "grad_norm": 470.0,
      "learning_rate": 1.1345029239766081e-07,
      "logits/chosen": -0.6597840189933777,
      "logits/rejected": -0.6569154858589172,
      "logps/chosen": -1105.4136962890625,
      "logps/rejected": -796.0147705078125,
      "loss": 0.7007,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01542759034782648,
      "rewards/margins": -0.014232159592211246,
      "rewards/rejected": -0.001195430988445878,
      "step": 3025
    },
    {
      "epoch": 0.7959361488763583,
      "grad_norm": 520.0,
      "learning_rate": 1.1330409356725145e-07,
      "logits/chosen": -0.6531025171279907,
      "logits/rejected": -0.6603391170501709,
      "logps/chosen": -1231.66650390625,
      "logps/rejected": -994.1181030273438,
      "loss": 0.6836,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.006329535972326994,
      "rewards/margins": 0.02028779871761799,
      "rewards/rejected": -0.013958265073597431,
      "step": 3026
    },
    {
      "epoch": 0.7961991813115454,
      "grad_norm": 760.0,
      "learning_rate": 1.131578947368421e-07,
      "logits/chosen": -0.6550372242927551,
      "logits/rejected": -0.6618703007698059,
      "logps/chosen": -1235.327880859375,
      "logps/rejected": -1175.181396484375,
      "loss": 0.7035,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0026673790998756886,
      "rewards/margins": -0.01915755495429039,
      "rewards/rejected": 0.021824931725859642,
      "step": 3027
    },
    {
      "epoch": 0.7964622137467327,
      "grad_norm": 596.0,
      "learning_rate": 1.1301169590643273e-07,
      "logits/chosen": -0.6479083895683289,
      "logits/rejected": -0.6511667370796204,
      "logps/chosen": -1066.5572509765625,
      "logps/rejected": -870.9901123046875,
      "loss": 0.6994,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.019387532025575638,
      "rewards/margins": -0.012061595916748047,
      "rewards/rejected": -0.007325935177505016,
      "step": 3028
    },
    {
      "epoch": 0.7967252461819198,
      "grad_norm": 516.0,
      "learning_rate": 1.1286549707602339e-07,
      "logits/chosen": -0.632980465888977,
      "logits/rejected": -0.6436626315116882,
      "logps/chosen": -1120.7454833984375,
      "logps/rejected": -636.7688598632812,
      "loss": 0.6872,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009200572036206722,
      "rewards/margins": 0.01254663523286581,
      "rewards/rejected": -0.00334606203250587,
      "step": 3029
    },
    {
      "epoch": 0.796988278617107,
      "grad_norm": 528.0,
      "learning_rate": 1.1271929824561404e-07,
      "logits/chosen": -0.643271803855896,
      "logits/rejected": -0.6400721073150635,
      "logps/chosen": -1171.3984375,
      "logps/rejected": -1007.7478637695312,
      "loss": 0.6854,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007837390527129173,
      "rewards/margins": 0.016339970752596855,
      "rewards/rejected": -0.008502578362822533,
      "step": 3030
    },
    {
      "epoch": 0.7972513110522942,
      "grad_norm": 576.0,
      "learning_rate": 1.1257309941520467e-07,
      "logits/chosen": -0.6369674205780029,
      "logits/rejected": -0.6506530046463013,
      "logps/chosen": -1412.905517578125,
      "logps/rejected": -1117.77294921875,
      "loss": 0.7056,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.018762877210974693,
      "rewards/margins": -0.023592570796608925,
      "rewards/rejected": 0.004829693119972944,
      "step": 3031
    },
    {
      "epoch": 0.7975143434874813,
      "grad_norm": 556.0,
      "learning_rate": 1.1242690058479532e-07,
      "logits/chosen": -0.643574595451355,
      "logits/rejected": -0.6482370495796204,
      "logps/chosen": -1203.458740234375,
      "logps/rejected": -948.5445556640625,
      "loss": 0.7048,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.026997946202754974,
      "rewards/margins": -0.022320939227938652,
      "rewards/rejected": -0.004677009303122759,
      "step": 3032
    },
    {
      "epoch": 0.7977773759226685,
      "grad_norm": 552.0,
      "learning_rate": 1.1228070175438596e-07,
      "logits/chosen": -0.6516302824020386,
      "logits/rejected": -0.6605326533317566,
      "logps/chosen": -1304.91552734375,
      "logps/rejected": -1052.080810546875,
      "loss": 0.6881,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.016172124072909355,
      "rewards/margins": 0.011062337085604668,
      "rewards/rejected": 0.005109787452965975,
      "step": 3033
    },
    {
      "epoch": 0.7980404083578556,
      "grad_norm": 532.0,
      "learning_rate": 1.1213450292397661e-07,
      "logits/chosen": -0.6417713165283203,
      "logits/rejected": -0.6446352005004883,
      "logps/chosen": -1429.031982421875,
      "logps/rejected": -1090.77197265625,
      "loss": 0.6943,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00012760190293192863,
      "rewards/margins": -0.001428700052201748,
      "rewards/rejected": 0.0015563014894723892,
      "step": 3034
    },
    {
      "epoch": 0.7983034407930428,
      "grad_norm": 612.0,
      "learning_rate": 1.1198830409356724e-07,
      "logits/chosen": -0.6658928394317627,
      "logits/rejected": -0.6599076390266418,
      "logps/chosen": -1113.9049072265625,
      "logps/rejected": -887.826171875,
      "loss": 0.6928,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.003667067736387253,
      "rewards/margins": 0.002001000102609396,
      "rewards/rejected": -0.005668067838996649,
      "step": 3035
    },
    {
      "epoch": 0.7985664732282299,
      "grad_norm": 498.0,
      "learning_rate": 1.1184210526315789e-07,
      "logits/chosen": -0.6639343500137329,
      "logits/rejected": -0.652082622051239,
      "logps/chosen": -964.0457153320312,
      "logps/rejected": -721.9544677734375,
      "loss": 0.711,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.02741832844913006,
      "rewards/margins": -0.03450784832239151,
      "rewards/rejected": 0.007089519407600164,
      "step": 3036
    },
    {
      "epoch": 0.7988295056634171,
      "grad_norm": 572.0,
      "learning_rate": 1.1169590643274854e-07,
      "logits/chosen": -0.6565208435058594,
      "logits/rejected": NaN,
      "logps/chosen": -1263.2037353515625,
      "logps/rejected": -864.40673828125,
      "loss": 0.7007,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0039564138278365135,
      "rewards/margins": -0.014253045432269573,
      "rewards/rejected": 0.010296630673110485,
      "step": 3037
    },
    {
      "epoch": 0.7990925380986043,
      "grad_norm": 556.0,
      "learning_rate": 1.1154970760233917e-07,
      "logits/chosen": -0.6752192378044128,
      "logits/rejected": -0.6704786419868469,
      "logps/chosen": -1182.018310546875,
      "logps/rejected": -894.1630249023438,
      "loss": 0.6828,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02893648110330105,
      "rewards/margins": 0.021404078230261803,
      "rewards/rejected": 0.007532406132668257,
      "step": 3038
    },
    {
      "epoch": 0.7993555705337915,
      "grad_norm": 552.0,
      "learning_rate": 1.1140350877192982e-07,
      "logits/chosen": -0.6568704843521118,
      "logits/rejected": -0.6766375303268433,
      "logps/chosen": -1143.466064453125,
      "logps/rejected": -857.5857543945312,
      "loss": 0.6971,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0018135067075490952,
      "rewards/margins": -0.006958008278161287,
      "rewards/rejected": 0.0051445006392896175,
      "step": 3039
    },
    {
      "epoch": 0.7996186029689786,
      "grad_norm": 552.0,
      "learning_rate": 1.1125730994152046e-07,
      "logits/chosen": -0.6545029878616333,
      "logits/rejected": -0.6570905447006226,
      "logps/chosen": -1042.9149169921875,
      "logps/rejected": -707.614013671875,
      "loss": 0.6879,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00539398193359375,
      "rewards/margins": 0.011483478359878063,
      "rewards/rejected": -0.00608949875459075,
      "step": 3040
    },
    {
      "epoch": 0.7998816354041658,
      "grad_norm": 512.0,
      "learning_rate": 1.111111111111111e-07,
      "logits/chosen": -0.6644318103790283,
      "logits/rejected": -0.6625558137893677,
      "logps/chosen": -802.4232177734375,
      "logps/rejected": -970.1728515625,
      "loss": 0.693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.006552790757268667,
      "rewards/margins": 0.0006854063831269741,
      "rewards/rejected": -0.007238198071718216,
      "step": 3041
    },
    {
      "epoch": 0.8001446678393529,
      "grad_norm": 552.0,
      "learning_rate": 1.1096491228070174e-07,
      "logits/chosen": -0.6545116901397705,
      "logits/rejected": -0.6552636623382568,
      "logps/chosen": -1109.9718017578125,
      "logps/rejected": -993.0795288085938,
      "loss": 0.6974,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.011519718915224075,
      "rewards/margins": -0.00803442019969225,
      "rewards/rejected": 0.01955413818359375,
      "step": 3042
    },
    {
      "epoch": 0.8004077002745401,
      "grad_norm": 442.0,
      "learning_rate": 1.1081871345029239e-07,
      "logits/chosen": -0.6127803325653076,
      "logits/rejected": -0.6270270943641663,
      "logps/chosen": -770.3411865234375,
      "logps/rejected": -663.7061767578125,
      "loss": 0.6848,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02792034111917019,
      "rewards/margins": 0.017203092575073242,
      "rewards/rejected": 0.010717248544096947,
      "step": 3043
    },
    {
      "epoch": 0.8006707327097272,
      "grad_norm": 492.0,
      "learning_rate": 1.1067251461988305e-07,
      "logits/chosen": -0.6414623260498047,
      "logits/rejected": -0.6421915888786316,
      "logps/chosen": -1276.674560546875,
      "logps/rejected": -942.494384765625,
      "loss": 0.6837,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0008152010850608349,
      "rewards/margins": 0.020038845017552376,
      "rewards/rejected": -0.02085404470562935,
      "step": 3044
    },
    {
      "epoch": 0.8009337651449144,
      "grad_norm": 540.0,
      "learning_rate": 1.1052631578947368e-07,
      "logits/chosen": -0.6563635468482971,
      "logits/rejected": -0.6495784521102905,
      "logps/chosen": -1175.5428466796875,
      "logps/rejected": -1049.1988525390625,
      "loss": 0.6934,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.002253961283713579,
      "rewards/margins": 8.46857437863946e-05,
      "rewards/rejected": -0.0023386478424072266,
      "step": 3045
    },
    {
      "epoch": 0.8011967975801016,
      "grad_norm": 564.0,
      "learning_rate": 1.1038011695906433e-07,
      "logits/chosen": -0.6229547262191772,
      "logits/rejected": -0.6256071329116821,
      "logps/chosen": -1411.3563232421875,
      "logps/rejected": -1116.4710693359375,
      "loss": 0.6888,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.036362551152706146,
      "rewards/margins": 0.009528064168989658,
      "rewards/rejected": 0.026834487915039062,
      "step": 3046
    },
    {
      "epoch": 0.8014598300152888,
      "grad_norm": 508.0,
      "learning_rate": 1.1023391812865497e-07,
      "logits/chosen": -0.6543561816215515,
      "logits/rejected": -0.6631639003753662,
      "logps/chosen": -1033.917236328125,
      "logps/rejected": -928.097900390625,
      "loss": 0.7018,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.009203911758959293,
      "rewards/margins": -0.016410065814852715,
      "rewards/rejected": 0.007206153590232134,
      "step": 3047
    },
    {
      "epoch": 0.801722862450476,
      "grad_norm": 620.0,
      "learning_rate": 1.1008771929824561e-07,
      "logits/chosen": -0.6364015340805054,
      "logits/rejected": NaN,
      "logps/chosen": -1442.955810546875,
      "logps/rejected": -1133.1644287109375,
      "loss": 0.7018,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.00027008005417883396,
      "rewards/margins": -0.01628255844116211,
      "rewards/rejected": 0.01601247861981392,
      "step": 3048
    },
    {
      "epoch": 0.8019858948856631,
      "grad_norm": 544.0,
      "learning_rate": 1.0994152046783625e-07,
      "logits/chosen": -0.6647913455963135,
      "logits/rejected": -0.6622175574302673,
      "logps/chosen": -1250.5572509765625,
      "logps/rejected": -938.2456665039062,
      "loss": 0.7084,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0036627769004553556,
      "rewards/margins": -0.02971210516989231,
      "rewards/rejected": 0.026049330830574036,
      "step": 3049
    },
    {
      "epoch": 0.8022489273208503,
      "grad_norm": 588.0,
      "learning_rate": 1.097953216374269e-07,
      "logits/chosen": -0.6154829859733582,
      "logits/rejected": -0.6199524998664856,
      "logps/chosen": -1421.6358642578125,
      "logps/rejected": -1264.804931640625,
      "loss": 0.6978,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.007608603686094284,
      "rewards/margins": -0.007738208398222923,
      "rewards/rejected": 0.0001296042464673519,
      "step": 3050
    },
    {
      "epoch": 0.8025119597560374,
      "grad_norm": 450.0,
      "learning_rate": 1.0964912280701753e-07,
      "logits/chosen": -0.6731074452400208,
      "logits/rejected": -0.6640039682388306,
      "logps/chosen": -916.9373168945312,
      "logps/rejected": -647.952880859375,
      "loss": 0.6897,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.017802810296416283,
      "rewards/margins": 0.007424114737659693,
      "rewards/rejected": -0.025226926431059837,
      "step": 3051
    },
    {
      "epoch": 0.8027749921912246,
      "grad_norm": 560.0,
      "learning_rate": 1.0950292397660818e-07,
      "logits/chosen": -0.6619291305541992,
      "logits/rejected": -0.6644441485404968,
      "logps/chosen": -972.77685546875,
      "logps/rejected": -1104.9482421875,
      "loss": 0.7018,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0064827920868992805,
      "rewards/margins": -0.016192248091101646,
      "rewards/rejected": 0.022675035521388054,
      "step": 3052
    },
    {
      "epoch": 0.8030380246264117,
      "grad_norm": 510.0,
      "learning_rate": 1.0935672514619883e-07,
      "logits/chosen": -0.6561866998672485,
      "logits/rejected": -0.6568306684494019,
      "logps/chosen": -1226.2158203125,
      "logps/rejected": -805.2485961914062,
      "loss": 0.6876,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.011938285082578659,
      "rewards/margins": 0.011437321081757545,
      "rewards/rejected": 0.0005009653978049755,
      "step": 3053
    },
    {
      "epoch": 0.8033010570615989,
      "grad_norm": 524.0,
      "learning_rate": 1.0921052631578947e-07,
      "logits/chosen": -0.652617335319519,
      "logits/rejected": -0.6372511386871338,
      "logps/chosen": -1071.89697265625,
      "logps/rejected": -912.9149780273438,
      "loss": 0.6799,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01906108856201172,
      "rewards/margins": 0.027562426403164864,
      "rewards/rejected": -0.008501339703798294,
      "step": 3054
    },
    {
      "epoch": 0.803564089496786,
      "grad_norm": 504.0,
      "learning_rate": 1.090643274853801e-07,
      "logits/chosen": -0.6588916778564453,
      "logits/rejected": -0.6586756110191345,
      "logps/chosen": -1285.550537109375,
      "logps/rejected": -619.8344116210938,
      "loss": 0.6863,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.00572996074333787,
      "rewards/margins": 0.014781427569687366,
      "rewards/rejected": -0.020511390641331673,
      "step": 3055
    },
    {
      "epoch": 0.8038271219319733,
      "grad_norm": 548.0,
      "learning_rate": 1.0891812865497075e-07,
      "logits/chosen": -0.653578519821167,
      "logits/rejected": -0.6569923758506775,
      "logps/chosen": -1358.573974609375,
      "logps/rejected": -1088.056396484375,
      "loss": 0.686,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01870746724307537,
      "rewards/margins": 0.015101337805390358,
      "rewards/rejected": 0.003606128040701151,
      "step": 3056
    },
    {
      "epoch": 0.8040901543671604,
      "grad_norm": 516.0,
      "learning_rate": 1.087719298245614e-07,
      "logits/chosen": -0.6376374959945679,
      "logits/rejected": -0.648896336555481,
      "logps/chosen": -1110.16064453125,
      "logps/rejected": -1096.094482421875,
      "loss": 0.692,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.023789023980498314,
      "rewards/margins": 0.0026929841842502356,
      "rewards/rejected": 0.021096037700772285,
      "step": 3057
    },
    {
      "epoch": 0.8043531868023476,
      "grad_norm": 390.0,
      "learning_rate": 1.0862573099415203e-07,
      "logits/chosen": -0.6423291563987732,
      "logits/rejected": -0.6595551371574402,
      "logps/chosen": -745.7821044921875,
      "logps/rejected": -329.25848388671875,
      "loss": 0.6837,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.013064241036772728,
      "rewards/margins": 0.019118165597319603,
      "rewards/rejected": -0.00605392549186945,
      "step": 3058
    },
    {
      "epoch": 0.8046162192375347,
      "grad_norm": 660.0,
      "learning_rate": 1.0847953216374269e-07,
      "logits/chosen": -0.644808292388916,
      "logits/rejected": -0.6500125527381897,
      "logps/chosen": -1140.348876953125,
      "logps/rejected": -1166.8741455078125,
      "loss": 0.68,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.00958790723234415,
      "rewards/margins": 0.02697915956377983,
      "rewards/rejected": -0.017391253262758255,
      "step": 3059
    },
    {
      "epoch": 0.8048792516727219,
      "grad_norm": 540.0,
      "learning_rate": 1.0833333333333334e-07,
      "logits/chosen": -0.6743789315223694,
      "logits/rejected": -0.6820530891418457,
      "logps/chosen": -1183.2587890625,
      "logps/rejected": -914.913330078125,
      "loss": 0.7042,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.00468177767470479,
      "rewards/margins": -0.020998671650886536,
      "rewards/rejected": 0.02568044513463974,
      "step": 3060
    },
    {
      "epoch": 0.8051422841079091,
      "grad_norm": 544.0,
      "learning_rate": 1.0818713450292397e-07,
      "logits/chosen": -0.6411153078079224,
      "logits/rejected": -0.6427088379859924,
      "logps/chosen": -869.2591552734375,
      "logps/rejected": -843.7501220703125,
      "loss": 0.7012,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0001863487996160984,
      "rewards/margins": -0.01510615460574627,
      "rewards/rejected": 0.014919806271791458,
      "step": 3061
    },
    {
      "epoch": 0.8054053165430962,
      "grad_norm": 458.0,
      "learning_rate": 1.0804093567251462e-07,
      "logits/chosen": -0.6378189921379089,
      "logits/rejected": -0.6466752290725708,
      "logps/chosen": -868.1760864257812,
      "logps/rejected": -596.857177734375,
      "loss": 0.6856,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.016784191131591797,
      "rewards/margins": 0.01566591113805771,
      "rewards/rejected": 0.0011182805756106973,
      "step": 3062
    },
    {
      "epoch": 0.8056683489782834,
      "grad_norm": 482.0,
      "learning_rate": 1.0789473684210526e-07,
      "logits/chosen": -0.6607366800308228,
      "logits/rejected": -0.6651526689529419,
      "logps/chosen": -1093.24365234375,
      "logps/rejected": -906.6337280273438,
      "loss": 0.6872,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003091144375503063,
      "rewards/margins": 0.012700080871582031,
      "rewards/rejected": -0.009608935564756393,
      "step": 3063
    },
    {
      "epoch": 0.8059313814134705,
      "grad_norm": 640.0,
      "learning_rate": 1.0774853801169591e-07,
      "logits/chosen": -0.6716864109039307,
      "logits/rejected": -0.6687491536140442,
      "logps/chosen": -1634.3458251953125,
      "logps/rejected": -1263.014404296875,
      "loss": 0.6995,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0055776601657271385,
      "rewards/margins": -0.010945416986942291,
      "rewards/rejected": 0.005367754492908716,
      "step": 3064
    },
    {
      "epoch": 0.8061944138486578,
      "grad_norm": 636.0,
      "learning_rate": 1.0760233918128654e-07,
      "logits/chosen": -0.6400673389434814,
      "logits/rejected": -0.6535574197769165,
      "logps/chosen": -1415.262939453125,
      "logps/rejected": -1280.681396484375,
      "loss": 0.7074,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.014692495577037334,
      "rewards/margins": -0.027874372899532318,
      "rewards/rejected": 0.042566873133182526,
      "step": 3065
    },
    {
      "epoch": 0.8064574462838449,
      "grad_norm": 612.0,
      "learning_rate": 1.0745614035087719e-07,
      "logits/chosen": -0.6395871639251709,
      "logits/rejected": -0.6493513584136963,
      "logps/chosen": -1321.439697265625,
      "logps/rejected": -1244.7000732421875,
      "loss": 0.6995,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0005495066288858652,
      "rewards/margins": -0.01164941769093275,
      "rewards/rejected": 0.01109991129487753,
      "step": 3066
    },
    {
      "epoch": 0.8067204787190321,
      "grad_norm": 552.0,
      "learning_rate": 1.0730994152046784e-07,
      "logits/chosen": -0.6577805876731873,
      "logits/rejected": NaN,
      "logps/chosen": -1321.002197265625,
      "logps/rejected": -812.8565063476562,
      "loss": 0.6914,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01164865680038929,
      "rewards/margins": 0.004408073611557484,
      "rewards/rejected": 0.007240582723170519,
      "step": 3067
    },
    {
      "epoch": 0.8069835111542192,
      "grad_norm": 552.0,
      "learning_rate": 1.0716374269005847e-07,
      "logits/chosen": -0.6633928418159485,
      "logits/rejected": -0.6653744578361511,
      "logps/chosen": -1016.696533203125,
      "logps/rejected": -774.4203491210938,
      "loss": 0.6942,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.002265692688524723,
      "rewards/margins": -0.001458026934415102,
      "rewards/rejected": 0.003723717527464032,
      "step": 3068
    },
    {
      "epoch": 0.8072465435894064,
      "grad_norm": 608.0,
      "learning_rate": 1.0701754385964911e-07,
      "logits/chosen": -0.6660125255584717,
      "logits/rejected": -0.6777380704879761,
      "logps/chosen": -1432.06201171875,
      "logps/rejected": -1061.070556640625,
      "loss": 0.694,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.011365222744643688,
      "rewards/margins": -0.0002731312997639179,
      "rewards/rejected": 0.011638354510068893,
      "step": 3069
    },
    {
      "epoch": 0.8075095760245935,
      "grad_norm": 468.0,
      "learning_rate": 1.0687134502923976e-07,
      "logits/chosen": -0.641345202922821,
      "logits/rejected": -0.6445460319519043,
      "logps/chosen": -804.6190795898438,
      "logps/rejected": -798.1475830078125,
      "loss": 0.6964,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.005612373352050781,
      "rewards/margins": -0.005601024255156517,
      "rewards/rejected": -1.1348631232976913e-05,
      "step": 3070
    },
    {
      "epoch": 0.8077726084597807,
      "grad_norm": 418.0,
      "learning_rate": 1.067251461988304e-07,
      "logits/chosen": -0.6628224849700928,
      "logits/rejected": -0.6672185659408569,
      "logps/chosen": -927.4708862304688,
      "logps/rejected": -581.675048828125,
      "loss": 0.6902,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0021494869142770767,
      "rewards/margins": 0.006222724914550781,
      "rewards/rejected": -0.0040732380002737045,
      "step": 3071
    },
    {
      "epoch": 0.8080356408949678,
      "grad_norm": 620.0,
      "learning_rate": 1.0657894736842104e-07,
      "logits/chosen": -0.6577390432357788,
      "logits/rejected": -0.6640547513961792,
      "logps/chosen": -967.6265258789062,
      "logps/rejected": -805.6134033203125,
      "loss": 0.6945,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.008587837219238281,
      "rewards/margins": -0.002398585667833686,
      "rewards/rejected": 0.01098642311990261,
      "step": 3072
    },
    {
      "epoch": 0.808298673330155,
      "grad_norm": 544.0,
      "learning_rate": 1.0643274853801169e-07,
      "logits/chosen": -0.6535549163818359,
      "logits/rejected": -0.6665686368942261,
      "logps/chosen": -1019.926513671875,
      "logps/rejected": -905.2774658203125,
      "loss": 0.6986,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.001447963761165738,
      "rewards/margins": -0.009745406918227673,
      "rewards/rejected": 0.008297443389892578,
      "step": 3073
    },
    {
      "epoch": 0.8085617057653421,
      "grad_norm": 640.0,
      "learning_rate": 1.0628654970760235e-07,
      "logits/chosen": -0.6696242690086365,
      "logits/rejected": -0.6812995672225952,
      "logps/chosen": -1487.888671875,
      "logps/rejected": -1336.039306640625,
      "loss": 0.7021,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.022159770131111145,
      "rewards/margins": -0.017346860840916634,
      "rewards/rejected": -0.004812909290194511,
      "step": 3074
    },
    {
      "epoch": 0.8088247382005294,
      "grad_norm": 580.0,
      "learning_rate": 1.0614035087719298e-07,
      "logits/chosen": -0.6648626923561096,
      "logits/rejected": -0.6636992692947388,
      "logps/chosen": -1016.841552734375,
      "logps/rejected": -951.31298828125,
      "loss": 0.6889,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.018021199852228165,
      "rewards/margins": 0.00987777579575777,
      "rewards/rejected": 0.008143424987792969,
      "step": 3075
    },
    {
      "epoch": 0.8090877706357165,
      "grad_norm": 424.0,
      "learning_rate": 1.0599415204678363e-07,
      "logits/chosen": -0.6522674560546875,
      "logits/rejected": -0.6455101370811462,
      "logps/chosen": -1004.9118041992188,
      "logps/rejected": -884.4464111328125,
      "loss": 0.6863,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0002829539589583874,
      "rewards/margins": 0.014721011742949486,
      "rewards/rejected": -0.014438057318329811,
      "step": 3076
    },
    {
      "epoch": 0.8093508030709037,
      "grad_norm": 486.0,
      "learning_rate": 1.0584795321637427e-07,
      "logits/chosen": -0.6274969577789307,
      "logits/rejected": NaN,
      "logps/chosen": -1170.8131103515625,
      "logps/rejected": -553.8674926757812,
      "loss": 0.6832,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.008209038525819778,
      "rewards/margins": 0.020884178578853607,
      "rewards/rejected": -0.012675141915678978,
      "step": 3077
    },
    {
      "epoch": 0.8096138355060909,
      "grad_norm": 438.0,
      "learning_rate": 1.057017543859649e-07,
      "logits/chosen": -0.6605338454246521,
      "logits/rejected": -0.6606274843215942,
      "logps/chosen": -1059.3985595703125,
      "logps/rejected": -697.3804321289062,
      "loss": 0.6866,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0043718344531953335,
      "rewards/margins": 0.013470076955854893,
      "rewards/rejected": -0.009098242968320847,
      "step": 3078
    },
    {
      "epoch": 0.809876867941278,
      "grad_norm": 584.0,
      "learning_rate": 1.0555555555555555e-07,
      "logits/chosen": -0.6732715964317322,
      "logits/rejected": -0.673062801361084,
      "logps/chosen": -1124.668701171875,
      "logps/rejected": -893.3306884765625,
      "loss": 0.6913,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008391667157411575,
      "rewards/margins": 0.004388522822409868,
      "rewards/rejected": 0.0040031434036791325,
      "step": 3079
    },
    {
      "epoch": 0.8101399003764652,
      "grad_norm": 488.0,
      "learning_rate": 1.054093567251462e-07,
      "logits/chosen": -0.6709380149841309,
      "logits/rejected": -0.6612571477890015,
      "logps/chosen": -1060.4534912109375,
      "logps/rejected": -665.4569091796875,
      "loss": 0.6923,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0008249282836914062,
      "rewards/margins": 0.002062702551484108,
      "rewards/rejected": -0.0028876299038529396,
      "step": 3080
    },
    {
      "epoch": 0.8104029328116523,
      "grad_norm": 500.0,
      "learning_rate": 1.0526315789473683e-07,
      "logits/chosen": -0.6493566036224365,
      "logits/rejected": -0.6658902168273926,
      "logps/chosen": -999.0272216796875,
      "logps/rejected": -926.74755859375,
      "loss": 0.6849,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.018936442211270332,
      "rewards/margins": 0.017075154930353165,
      "rewards/rejected": 0.0018612861167639494,
      "step": 3081
    },
    {
      "epoch": 0.8106659652468395,
      "grad_norm": 708.0,
      "learning_rate": 1.0511695906432748e-07,
      "logits/chosen": -0.6475787162780762,
      "logits/rejected": -0.6542171835899353,
      "logps/chosen": -1812.344482421875,
      "logps/rejected": -970.99462890625,
      "loss": 0.6727,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.026108168065547943,
      "rewards/margins": 0.04203081130981445,
      "rewards/rejected": -0.01592264324426651,
      "step": 3082
    },
    {
      "epoch": 0.8109289976820266,
      "grad_norm": 448.0,
      "learning_rate": 1.0497076023391812e-07,
      "logits/chosen": -0.6435776948928833,
      "logits/rejected": -0.6446930170059204,
      "logps/chosen": -1088.455078125,
      "logps/rejected": -919.3524169921875,
      "loss": 0.6927,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.001961517846211791,
      "rewards/margins": 0.0014039033558219671,
      "rewards/rejected": -0.003365420736372471,
      "step": 3083
    },
    {
      "epoch": 0.8111920301172139,
      "grad_norm": 704.0,
      "learning_rate": 1.0482456140350877e-07,
      "logits/chosen": -0.6763936281204224,
      "logits/rejected": -0.6807339787483215,
      "logps/chosen": -1659.3477783203125,
      "logps/rejected": -1277.9971923828125,
      "loss": 0.6983,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.002040672115981579,
      "rewards/margins": -0.008502577431499958,
      "rewards/rejected": 0.010543251410126686,
      "step": 3084
    },
    {
      "epoch": 0.811455062552401,
      "grad_norm": 490.0,
      "learning_rate": 1.046783625730994e-07,
      "logits/chosen": -0.6815317869186401,
      "logits/rejected": -0.6712371110916138,
      "logps/chosen": -1081.906494140625,
      "logps/rejected": -975.8335571289062,
      "loss": 0.6934,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.011946868151426315,
      "rewards/margins": 0.0002398500218987465,
      "rewards/rejected": -0.012186718173325062,
      "step": 3085
    },
    {
      "epoch": 0.8117180949875882,
      "grad_norm": 482.0,
      "learning_rate": 1.0453216374269005e-07,
      "logits/chosen": -0.6633809208869934,
      "logits/rejected": -0.6697562336921692,
      "logps/chosen": -960.0272827148438,
      "logps/rejected": -939.7998657226562,
      "loss": 0.7121,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.025159263983368874,
      "rewards/margins": -0.036658238619565964,
      "rewards/rejected": 0.011498975567519665,
      "step": 3086
    },
    {
      "epoch": 0.8119811274227753,
      "grad_norm": 580.0,
      "learning_rate": 1.043859649122807e-07,
      "logits/chosen": -0.6566758155822754,
      "logits/rejected": -0.6530481576919556,
      "logps/chosen": -1484.966064453125,
      "logps/rejected": -1395.7322998046875,
      "loss": 0.6938,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0062320721335709095,
      "rewards/margins": 0.00021314737387001514,
      "rewards/rejected": -0.0064452169463038445,
      "step": 3087
    },
    {
      "epoch": 0.8122441598579625,
      "grad_norm": 636.0,
      "learning_rate": 1.0423976608187133e-07,
      "logits/chosen": -0.6724252104759216,
      "logits/rejected": -0.6744530200958252,
      "logps/chosen": -1315.07763671875,
      "logps/rejected": -936.1355590820312,
      "loss": 0.6843,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.021331405267119408,
      "rewards/margins": 0.019062520936131477,
      "rewards/rejected": 0.002268886659294367,
      "step": 3088
    },
    {
      "epoch": 0.8125071922931496,
      "grad_norm": 484.0,
      "learning_rate": 1.0409356725146199e-07,
      "logits/chosen": -0.6395668387413025,
      "logits/rejected": -0.6525348424911499,
      "logps/chosen": -1151.028564453125,
      "logps/rejected": -1001.6638793945312,
      "loss": 0.688,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.022893045097589493,
      "rewards/margins": 0.011174965649843216,
      "rewards/rejected": 0.011718082241714,
      "step": 3089
    },
    {
      "epoch": 0.8127702247283368,
      "grad_norm": 560.0,
      "learning_rate": 1.0394736842105264e-07,
      "logits/chosen": -0.6696428656578064,
      "logits/rejected": -0.6685363054275513,
      "logps/chosen": -1434.43994140625,
      "logps/rejected": -1303.4688720703125,
      "loss": 0.6965,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0035721776075661182,
      "rewards/margins": -0.006235981360077858,
      "rewards/rejected": 0.002663802122697234,
      "step": 3090
    },
    {
      "epoch": 0.8130332571635239,
      "grad_norm": 482.0,
      "learning_rate": 1.0380116959064327e-07,
      "logits/chosen": -0.6416225433349609,
      "logits/rejected": -0.6516193151473999,
      "logps/chosen": -1114.9072265625,
      "logps/rejected": -790.7998046875,
      "loss": 0.6862,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.012632180005311966,
      "rewards/margins": 0.014163685031235218,
      "rewards/rejected": -0.02679586410522461,
      "step": 3091
    },
    {
      "epoch": 0.8132962895987111,
      "grad_norm": 492.0,
      "learning_rate": 1.0365497076023392e-07,
      "logits/chosen": -0.6480438709259033,
      "logits/rejected": -0.6474385857582092,
      "logps/chosen": -969.995361328125,
      "logps/rejected": -845.527587890625,
      "loss": 0.7004,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02351093292236328,
      "rewards/margins": -0.013577653095126152,
      "rewards/rejected": -0.009933280758559704,
      "step": 3092
    },
    {
      "epoch": 0.8135593220338984,
      "grad_norm": 592.0,
      "learning_rate": 1.0350877192982456e-07,
      "logits/chosen": -0.6303064227104187,
      "logits/rejected": -0.6356742978096008,
      "logps/chosen": -1630.501953125,
      "logps/rejected": -1305.9931640625,
      "loss": 0.6854,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.003818607423454523,
      "rewards/margins": 0.017307281494140625,
      "rewards/rejected": -0.02112589031457901,
      "step": 3093
    },
    {
      "epoch": 0.8138223544690855,
      "grad_norm": 482.0,
      "learning_rate": 1.0336257309941521e-07,
      "logits/chosen": -0.6509390473365784,
      "logits/rejected": -0.6557955741882324,
      "logps/chosen": -786.3133544921875,
      "logps/rejected": -743.58203125,
      "loss": 0.6944,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.015150021761655807,
      "rewards/margins": -0.002030991716310382,
      "rewards/rejected": 0.017181016504764557,
      "step": 3094
    },
    {
      "epoch": 0.8140853869042727,
      "grad_norm": 488.0,
      "learning_rate": 1.0321637426900584e-07,
      "logits/chosen": -0.6435707807540894,
      "logits/rejected": -0.6577169895172119,
      "logps/chosen": -1020.3583984375,
      "logps/rejected": -632.1079711914062,
      "loss": 0.6792,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.002051735296845436,
      "rewards/margins": 0.028857994824647903,
      "rewards/rejected": -0.03090972825884819,
      "step": 3095
    },
    {
      "epoch": 0.8143484193394598,
      "grad_norm": 656.0,
      "learning_rate": 1.0307017543859649e-07,
      "logits/chosen": -0.6711965799331665,
      "logits/rejected": -0.6681015491485596,
      "logps/chosen": -1180.1568603515625,
      "logps/rejected": -1239.140380859375,
      "loss": 0.6889,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.006234646309167147,
      "rewards/margins": 0.00937032699584961,
      "rewards/rejected": -0.015604972839355469,
      "step": 3096
    },
    {
      "epoch": 0.814611451774647,
      "grad_norm": 576.0,
      "learning_rate": 1.0292397660818713e-07,
      "logits/chosen": -0.6640009880065918,
      "logits/rejected": -0.6776437759399414,
      "logps/chosen": -1448.489990234375,
      "logps/rejected": -1091.086181640625,
      "loss": 0.7023,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005104159936308861,
      "rewards/margins": -0.016802120953798294,
      "rewards/rejected": 0.011697960086166859,
      "step": 3097
    },
    {
      "epoch": 0.8148744842098341,
      "grad_norm": 458.0,
      "learning_rate": 1.0277777777777777e-07,
      "logits/chosen": -0.6394426822662354,
      "logits/rejected": -0.6405981779098511,
      "logps/chosen": -949.38671875,
      "logps/rejected": -822.097900390625,
      "loss": 0.6933,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0008785251993685961,
      "rewards/margins": 0.00016441429033875465,
      "rewards/rejected": 0.0007141106761991978,
      "step": 3098
    },
    {
      "epoch": 0.8151375166450213,
      "grad_norm": 680.0,
      "learning_rate": 1.0263157894736841e-07,
      "logits/chosen": -0.6588632464408875,
      "logits/rejected": -0.6620913147926331,
      "logps/chosen": -1442.6644287109375,
      "logps/rejected": -1308.075927734375,
      "loss": 0.7045,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.003102874383330345,
      "rewards/margins": -0.021140195429325104,
      "rewards/rejected": 0.01803731545805931,
      "step": 3099
    },
    {
      "epoch": 0.8154005490802084,
      "grad_norm": 612.0,
      "learning_rate": 1.0248538011695906e-07,
      "logits/chosen": -0.6497437357902527,
      "logits/rejected": -0.6469516158103943,
      "logps/chosen": -1256.705078125,
      "logps/rejected": -1051.3133544921875,
      "loss": 0.7148,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.012089919298887253,
      "rewards/margins": -0.042327880859375,
      "rewards/rejected": 0.030237963423132896,
      "step": 3100
    },
    {
      "epoch": 0.8156635815153956,
      "grad_norm": 616.0,
      "learning_rate": 1.0233918128654969e-07,
      "logits/chosen": -0.6584959626197815,
      "logits/rejected": -0.6577004194259644,
      "logps/chosen": -1264.1619873046875,
      "logps/rejected": -1023.99462890625,
      "loss": 0.6915,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.002954005729407072,
      "rewards/margins": 0.0038615227676928043,
      "rewards/rejected": -0.0009075164562091231,
      "step": 3101
    },
    {
      "epoch": 0.8159266139505827,
      "grad_norm": 536.0,
      "learning_rate": 1.0219298245614034e-07,
      "logits/chosen": -0.5953653454780579,
      "logits/rejected": -0.6156496405601501,
      "logps/chosen": -1125.7569580078125,
      "logps/rejected": -1061.8128662109375,
      "loss": 0.7021,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.00330200232565403,
      "rewards/margins": -0.017438601702451706,
      "rewards/rejected": 0.020740604028105736,
      "step": 3102
    },
    {
      "epoch": 0.81618964638577,
      "grad_norm": 872.0,
      "learning_rate": 1.0204678362573099e-07,
      "logits/chosen": -0.6560984253883362,
      "logits/rejected": -0.6598957777023315,
      "logps/chosen": -1309.7275390625,
      "logps/rejected": -655.1785278320312,
      "loss": 0.6854,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0031002038158476353,
      "rewards/margins": 0.01590728759765625,
      "rewards/rejected": -0.012807084247469902,
      "step": 3103
    },
    {
      "epoch": 0.8164526788209571,
      "grad_norm": 524.0,
      "learning_rate": 1.0190058479532165e-07,
      "logits/chosen": -0.6603973507881165,
      "logits/rejected": -0.6575002670288086,
      "logps/chosen": -1058.5836181640625,
      "logps/rejected": -1164.6552734375,
      "loss": 0.6943,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.008421801961958408,
      "rewards/margins": -0.0015010836068540812,
      "rewards/rejected": -0.0069207195192575455,
      "step": 3104
    },
    {
      "epoch": 0.8167157112561443,
      "grad_norm": 462.0,
      "learning_rate": 1.0175438596491228e-07,
      "logits/chosen": -0.6586344242095947,
      "logits/rejected": -0.6540685892105103,
      "logps/chosen": -885.923095703125,
      "logps/rejected": -560.654052734375,
      "loss": 0.6836,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.011759663000702858,
      "rewards/margins": 0.020105555653572083,
      "rewards/rejected": -0.0083458898589015,
      "step": 3105
    },
    {
      "epoch": 0.8169787436913314,
      "grad_norm": 500.0,
      "learning_rate": 1.0160818713450293e-07,
      "logits/chosen": -0.638836681842804,
      "logits/rejected": -0.6391725540161133,
      "logps/chosen": -1140.58642578125,
      "logps/rejected": -877.6340942382812,
      "loss": 0.6947,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.017110155895352364,
      "rewards/margins": -0.0025281913112848997,
      "rewards/rejected": 0.019638346508145332,
      "step": 3106
    },
    {
      "epoch": 0.8172417761265186,
      "grad_norm": 488.0,
      "learning_rate": 1.0146198830409357e-07,
      "logits/chosen": -0.6563538908958435,
      "logits/rejected": -0.6548364758491516,
      "logps/chosen": -1010.258056640625,
      "logps/rejected": -825.1688232421875,
      "loss": 0.686,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.00677642785012722,
      "rewards/margins": 0.015717696398496628,
      "rewards/rejected": -0.008941269479691982,
      "step": 3107
    },
    {
      "epoch": 0.8175048085617058,
      "grad_norm": 438.0,
      "learning_rate": 1.013157894736842e-07,
      "logits/chosen": -0.6282313466072083,
      "logits/rejected": -0.6372925639152527,
      "logps/chosen": -1056.2137451171875,
      "logps/rejected": -808.550537109375,
      "loss": 0.6921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.014647865667939186,
      "rewards/margins": 0.002556801540777087,
      "rewards/rejected": 0.01209106482565403,
      "step": 3108
    },
    {
      "epoch": 0.8177678409968929,
      "grad_norm": 462.0,
      "learning_rate": 1.0116959064327485e-07,
      "logits/chosen": -0.6587303876876831,
      "logits/rejected": -0.6659932136535645,
      "logps/chosen": -700.372314453125,
      "logps/rejected": -710.6408081054688,
      "loss": 0.7051,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0007645124569535255,
      "rewards/margins": -0.022833680734038353,
      "rewards/rejected": 0.022069167345762253,
      "step": 3109
    },
    {
      "epoch": 0.8180308734320801,
      "grad_norm": 416.0,
      "learning_rate": 1.010233918128655e-07,
      "logits/chosen": -0.6664876937866211,
      "logits/rejected": -0.6751996278762817,
      "logps/chosen": -840.6024169921875,
      "logps/rejected": -711.9725341796875,
      "loss": 0.6921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.013484382070600986,
      "rewards/margins": 0.0025016781874001026,
      "rewards/rejected": -0.015986062586307526,
      "step": 3110
    },
    {
      "epoch": 0.8182939058672672,
      "grad_norm": 424.0,
      "learning_rate": 1.0087719298245613e-07,
      "logits/chosen": -0.6521726250648499,
      "logits/rejected": -0.6511208415031433,
      "logps/chosen": -885.8353881835938,
      "logps/rejected": -830.36572265625,
      "loss": 0.6908,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005616665352135897,
      "rewards/margins": 0.005079841241240501,
      "rewards/rejected": 0.0005368231795728207,
      "step": 3111
    },
    {
      "epoch": 0.8185569383024545,
      "grad_norm": 2288.0,
      "learning_rate": 1.0073099415204678e-07,
      "logits/chosen": NaN,
      "logits/rejected": -0.6451514959335327,
      "logps/chosen": -1077.0606689453125,
      "logps/rejected": -826.3995361328125,
      "loss": 0.7013,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.014278887771070004,
      "rewards/margins": -0.015486717224121094,
      "rewards/rejected": 0.001207828288897872,
      "step": 3112
    },
    {
      "epoch": 0.8188199707376416,
      "grad_norm": 502.0,
      "learning_rate": 1.0058479532163742e-07,
      "logits/chosen": -0.6312406659126282,
      "logits/rejected": -0.63173508644104,
      "logps/chosen": -1027.4495849609375,
      "logps/rejected": -1012.4934692382812,
      "loss": 0.6981,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0021219244226813316,
      "rewards/margins": -0.00956878624856472,
      "rewards/rejected": 0.011690711602568626,
      "step": 3113
    },
    {
      "epoch": 0.8190830031728288,
      "grad_norm": 454.0,
      "learning_rate": 1.0043859649122807e-07,
      "logits/chosen": -0.643940269947052,
      "logits/rejected": -0.6451991200447083,
      "logps/chosen": -922.69482421875,
      "logps/rejected": -667.579345703125,
      "loss": 0.6928,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.012196684256196022,
      "rewards/margins": 0.001380204688757658,
      "rewards/rejected": -0.013576889410614967,
      "step": 3114
    },
    {
      "epoch": 0.8193460356080159,
      "grad_norm": 498.0,
      "learning_rate": 1.002923976608187e-07,
      "logits/chosen": -0.6496515274047852,
      "logits/rejected": -0.6620759963989258,
      "logps/chosen": -1173.6070556640625,
      "logps/rejected": -916.7084350585938,
      "loss": 0.6755,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.03537778928875923,
      "rewards/margins": 0.03690795972943306,
      "rewards/rejected": -0.0015301704406738281,
      "step": 3115
    },
    {
      "epoch": 0.8196090680432031,
      "grad_norm": 524.0,
      "learning_rate": 1.0014619883040935e-07,
      "logits/chosen": -0.6469399333000183,
      "logits/rejected": -0.6576545238494873,
      "logps/chosen": -1005.9658813476562,
      "logps/rejected": -666.291015625,
      "loss": 0.6997,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.00240683532319963,
      "rewards/margins": -0.012662364169955254,
      "rewards/rejected": 0.010255527682602406,
      "step": 3116
    },
    {
      "epoch": 0.8198721004783902,
      "grad_norm": 506.0,
      "learning_rate": 1e-07,
      "logits/chosen": -0.6555800437927246,
      "logits/rejected": -0.6564617156982422,
      "logps/chosen": -725.892333984375,
      "logps/rejected": -735.7594604492188,
      "loss": 0.7045,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.013011741451919079,
      "rewards/margins": -0.022017860785126686,
      "rewards/rejected": 0.009006118401885033,
      "step": 3117
    },
    {
      "epoch": 0.8201351329135774,
      "grad_norm": 532.0,
      "learning_rate": 9.985380116959063e-08,
      "logits/chosen": -0.6607158184051514,
      "logits/rejected": -0.6670182943344116,
      "logps/chosen": -1245.3804931640625,
      "logps/rejected": -856.943603515625,
      "loss": 0.6856,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0016169548034667969,
      "rewards/margins": 0.01625690422952175,
      "rewards/rejected": -0.014639951288700104,
      "step": 3118
    },
    {
      "epoch": 0.8203981653487645,
      "grad_norm": 440.0,
      "learning_rate": 9.970760233918129e-08,
      "logits/chosen": -0.6673503518104553,
      "logits/rejected": -0.6802698373794556,
      "logps/chosen": -1146.6173095703125,
      "logps/rejected": -835.4993896484375,
      "loss": 0.6934,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.015252208337187767,
      "rewards/margins": 6.980914622545242e-05,
      "rewards/rejected": 0.01518240012228489,
      "step": 3119
    },
    {
      "epoch": 0.8206611977839517,
      "grad_norm": 510.0,
      "learning_rate": 9.956140350877193e-08,
      "logits/chosen": -0.6627019047737122,
      "logits/rejected": -0.6625579595565796,
      "logps/chosen": -1197.836669921875,
      "logps/rejected": -953.51806640625,
      "loss": 0.6877,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.010551834478974342,
      "rewards/margins": 0.011415768414735794,
      "rewards/rejected": -0.0008639339357614517,
      "step": 3120
    },
    {
      "epoch": 0.8209242302191389,
      "grad_norm": 490.0,
      "learning_rate": 9.941520467836257e-08,
      "logits/chosen": -0.666553795337677,
      "logits/rejected": NaN,
      "logps/chosen": -921.5494384765625,
      "logps/rejected": -787.8171997070312,
      "loss": 0.6972,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.014895439147949219,
      "rewards/margins": -0.007588195614516735,
      "rewards/rejected": -0.007307243533432484,
      "step": 3121
    },
    {
      "epoch": 0.8211872626543261,
      "grad_norm": 536.0,
      "learning_rate": 9.926900584795321e-08,
      "logits/chosen": -0.6516963839530945,
      "logits/rejected": -0.6572454571723938,
      "logps/chosen": -1058.36962890625,
      "logps/rejected": -842.5775756835938,
      "loss": 0.7036,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.022330379113554955,
      "rewards/margins": -0.01847038045525551,
      "rewards/rejected": -0.003859997494146228,
      "step": 3122
    },
    {
      "epoch": 0.8214502950895132,
      "grad_norm": 644.0,
      "learning_rate": 9.912280701754386e-08,
      "logits/chosen": -0.6732162237167358,
      "logits/rejected": -0.6689034104347229,
      "logps/chosen": -1150.4803466796875,
      "logps/rejected": -871.0369262695312,
      "loss": 0.7065,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.030858803540468216,
      "rewards/margins": -0.025983717292547226,
      "rewards/rejected": -0.004875087179243565,
      "step": 3123
    },
    {
      "epoch": 0.8217133275247004,
      "grad_norm": 716.0,
      "learning_rate": 9.89766081871345e-08,
      "logits/chosen": -0.6548664569854736,
      "logits/rejected": -0.6621005535125732,
      "logps/chosen": -1103.85888671875,
      "logps/rejected": -910.7575073242188,
      "loss": 0.6986,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.015536021441221237,
      "rewards/margins": -0.010148812085390091,
      "rewards/rejected": -0.0053872112184762955,
      "step": 3124
    },
    {
      "epoch": 0.8219763599598876,
      "grad_norm": 560.0,
      "learning_rate": 9.883040935672514e-08,
      "logits/chosen": -0.6457083821296692,
      "logits/rejected": -0.6486974358558655,
      "logps/chosen": -1151.2391357421875,
      "logps/rejected": -807.1131591796875,
      "loss": 0.6904,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0011754034785553813,
      "rewards/margins": 0.00670041935518384,
      "rewards/rejected": -0.005525016225874424,
      "step": 3125
    },
    {
      "epoch": 0.8222393923950747,
      "grad_norm": 632.0,
      "learning_rate": 9.868421052631579e-08,
      "logits/chosen": -0.66362065076828,
      "logits/rejected": -0.668292760848999,
      "logps/chosen": -1338.03466796875,
      "logps/rejected": -1034.155029296875,
      "loss": 0.6845,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01386880874633789,
      "rewards/margins": 0.018381688743829727,
      "rewards/rejected": -0.00451288279145956,
      "step": 3126
    },
    {
      "epoch": 0.8225024248302619,
      "grad_norm": 428.0,
      "learning_rate": 9.853801169590643e-08,
      "logits/chosen": -0.646625280380249,
      "logits/rejected": -0.6526135206222534,
      "logps/chosen": -1047.16650390625,
      "logps/rejected": -767.7439575195312,
      "loss": 0.6876,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.00937642902135849,
      "rewards/margins": 0.011468123644590378,
      "rewards/rejected": -0.002091693924739957,
      "step": 3127
    },
    {
      "epoch": 0.822765457265449,
      "grad_norm": 544.0,
      "learning_rate": 9.839181286549707e-08,
      "logits/chosen": -0.6182600855827332,
      "logits/rejected": -0.6200603246688843,
      "logps/chosen": -1417.6444091796875,
      "logps/rejected": -877.1878662109375,
      "loss": 0.6837,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02163219451904297,
      "rewards/margins": 0.019789408892393112,
      "rewards/rejected": 0.0018427850445732474,
      "step": 3128
    },
    {
      "epoch": 0.8230284897006362,
      "grad_norm": 624.0,
      "learning_rate": 9.824561403508771e-08,
      "logits/chosen": -0.6204847097396851,
      "logits/rejected": -0.6167067289352417,
      "logps/chosen": -1503.366455078125,
      "logps/rejected": -904.1260986328125,
      "loss": 0.7081,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.011684991419315338,
      "rewards/margins": -0.028606224805116653,
      "rewards/rejected": 0.016921237111091614,
      "step": 3129
    },
    {
      "epoch": 0.8232915221358234,
      "grad_norm": 580.0,
      "learning_rate": 9.809941520467836e-08,
      "logits/chosen": -0.6641979217529297,
      "logits/rejected": -0.6627258062362671,
      "logps/chosen": -1091.533935546875,
      "logps/rejected": -814.0621337890625,
      "loss": 0.6961,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.002291583688929677,
      "rewards/margins": -0.005443429574370384,
      "rewards/rejected": 0.007735013030469418,
      "step": 3130
    },
    {
      "epoch": 0.8235545545710106,
      "grad_norm": 548.0,
      "learning_rate": 9.795321637426899e-08,
      "logits/chosen": -0.6676608324050903,
      "logits/rejected": -0.6648872494697571,
      "logps/chosen": -869.0403442382812,
      "logps/rejected": -1055.052490234375,
      "loss": 0.6846,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007253838703036308,
      "rewards/margins": 0.01769561693072319,
      "rewards/rejected": -0.010441780090332031,
      "step": 3131
    },
    {
      "epoch": 0.8238175870061977,
      "grad_norm": 528.0,
      "learning_rate": 9.780701754385964e-08,
      "logits/chosen": -0.6466774940490723,
      "logits/rejected": NaN,
      "logps/chosen": -1052.957275390625,
      "logps/rejected": -786.0465698242188,
      "loss": 0.6891,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.005566501058638096,
      "rewards/margins": 0.009007454849779606,
      "rewards/rejected": -0.0034409521613270044,
      "step": 3132
    },
    {
      "epoch": 0.8240806194413849,
      "grad_norm": 474.0,
      "learning_rate": 9.766081871345028e-08,
      "logits/chosen": -0.6458091735839844,
      "logits/rejected": -0.6400072574615479,
      "logps/chosen": -864.3069458007812,
      "logps/rejected": -692.6690673828125,
      "loss": 0.6928,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0036135674454271793,
      "rewards/margins": 0.0015161512419581413,
      "rewards/rejected": -0.005129719153046608,
      "step": 3133
    },
    {
      "epoch": 0.824343651876572,
      "grad_norm": 588.0,
      "learning_rate": 9.751461988304093e-08,
      "logits/chosen": -0.6609140038490295,
      "logits/rejected": -0.6555876135826111,
      "logps/chosen": -1462.552001953125,
      "logps/rejected": -1106.7646484375,
      "loss": 0.6937,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007085991092026234,
      "rewards/margins": 0.0003578178584575653,
      "rewards/rejected": 0.006728171370923519,
      "step": 3134
    },
    {
      "epoch": 0.8246066843117592,
      "grad_norm": 648.0,
      "learning_rate": 9.736842105263158e-08,
      "logits/chosen": -0.6717917323112488,
      "logits/rejected": -0.6800412535667419,
      "logps/chosen": -1262.407958984375,
      "logps/rejected": -1302.0419921875,
      "loss": 0.6982,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.009768581949174404,
      "rewards/margins": -0.009445189498364925,
      "rewards/rejected": 0.01921377331018448,
      "step": 3135
    },
    {
      "epoch": 0.8248697167469463,
      "grad_norm": 510.0,
      "learning_rate": 9.722222222222222e-08,
      "logits/chosen": -0.6410205364227295,
      "logits/rejected": -0.6680540442466736,
      "logps/chosen": -1062.4332275390625,
      "logps/rejected": -787.5418701171875,
      "loss": 0.6922,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00811815820634365,
      "rewards/margins": 0.0021861116401851177,
      "rewards/rejected": 0.005932046100497246,
      "step": 3136
    },
    {
      "epoch": 0.8251327491821335,
      "grad_norm": 508.0,
      "learning_rate": 9.707602339181287e-08,
      "logits/chosen": -0.6623189449310303,
      "logits/rejected": -0.6616138815879822,
      "logps/chosen": -1108.5155029296875,
      "logps/rejected": -836.4717407226562,
      "loss": 0.69,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.007639360148459673,
      "rewards/margins": 0.008105708286166191,
      "rewards/rejected": -0.015745066106319427,
      "step": 3137
    },
    {
      "epoch": 0.8253957816173206,
      "grad_norm": 584.0,
      "learning_rate": 9.69298245614035e-08,
      "logits/chosen": -0.635015070438385,
      "logits/rejected": -0.6384009718894958,
      "logps/chosen": -1259.1876220703125,
      "logps/rejected": -1079.8153076171875,
      "loss": 0.6926,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0012762066908180714,
      "rewards/margins": 0.002050112234428525,
      "rewards/rejected": -0.003326320555061102,
      "step": 3138
    },
    {
      "epoch": 0.8256588140525079,
      "grad_norm": 476.0,
      "learning_rate": 9.678362573099415e-08,
      "logits/chosen": -0.6627237796783447,
      "logits/rejected": -0.6589187979698181,
      "logps/chosen": -799.9613647460938,
      "logps/rejected": -720.7649536132812,
      "loss": 0.6971,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.000775766558945179,
      "rewards/margins": -0.007356786169111729,
      "rewards/rejected": 0.006581020541489124,
      "step": 3139
    },
    {
      "epoch": 0.8259218464876951,
      "grad_norm": 454.0,
      "learning_rate": 9.66374269005848e-08,
      "logits/chosen": -0.6247923374176025,
      "logits/rejected": -0.6416362524032593,
      "logps/chosen": -837.634521484375,
      "logps/rejected": -790.06201171875,
      "loss": 0.7023,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0011581408325582743,
      "rewards/margins": -0.017464734613895416,
      "rewards/rejected": 0.016306592151522636,
      "step": 3140
    },
    {
      "epoch": 0.8261848789228822,
      "grad_norm": 660.0,
      "learning_rate": 9.649122807017543e-08,
      "logits/chosen": -0.6490626335144043,
      "logits/rejected": -0.6599718332290649,
      "logps/chosen": -1724.2476806640625,
      "logps/rejected": -1468.7010498046875,
      "loss": 0.7005,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.015984345227479935,
      "rewards/margins": -0.013444519601762295,
      "rewards/rejected": -0.0025398251600563526,
      "step": 3141
    },
    {
      "epoch": 0.8264479113580694,
      "grad_norm": 406.0,
      "learning_rate": 9.634502923976608e-08,
      "logits/chosen": -0.6625759601593018,
      "logits/rejected": -0.6693873405456543,
      "logps/chosen": -820.9176635742188,
      "logps/rejected": -594.2835083007812,
      "loss": 0.6946,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.004601955413818359,
      "rewards/margins": -0.0022669790778309107,
      "rewards/rejected": 0.006868934258818626,
      "step": 3142
    },
    {
      "epoch": 0.8267109437932565,
      "grad_norm": 564.0,
      "learning_rate": 9.619883040935672e-08,
      "logits/chosen": -0.665353000164032,
      "logits/rejected": -0.6686097979545593,
      "logps/chosen": -1341.153564453125,
      "logps/rejected": -1289.473876953125,
      "loss": 0.6936,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0030926712788641453,
      "rewards/margins": -0.0002047545276582241,
      "rewards/rejected": 0.0032974244095385075,
      "step": 3143
    },
    {
      "epoch": 0.8269739762284437,
      "grad_norm": 660.0,
      "learning_rate": 9.605263157894736e-08,
      "logits/chosen": -0.6551513671875,
      "logits/rejected": -0.656772792339325,
      "logps/chosen": -1586.9840087890625,
      "logps/rejected": -961.141357421875,
      "loss": 0.6903,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.009640121832489967,
      "rewards/margins": 0.006658269092440605,
      "rewards/rejected": 0.002981853671371937,
      "step": 3144
    },
    {
      "epoch": 0.8272370086636308,
      "grad_norm": 660.0,
      "learning_rate": 9.5906432748538e-08,
      "logits/chosen": -0.6636427640914917,
      "logits/rejected": NaN,
      "logps/chosen": -1192.9033203125,
      "logps/rejected": -580.1927490234375,
      "loss": 0.6881,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012355709448456764,
      "rewards/margins": 0.011560153216123581,
      "rewards/rejected": 0.0007955559995025396,
      "step": 3145
    },
    {
      "epoch": 0.827500041098818,
      "grad_norm": 452.0,
      "learning_rate": 9.576023391812865e-08,
      "logits/chosen": -0.670131504535675,
      "logits/rejected": -0.6766309142112732,
      "logps/chosen": -820.1306762695312,
      "logps/rejected": -652.0450439453125,
      "loss": 0.6883,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.006164454855024815,
      "rewards/margins": 0.010244369506835938,
      "rewards/rejected": -0.00407991511747241,
      "step": 3146
    },
    {
      "epoch": 0.8277630735340051,
      "grad_norm": 416.0,
      "learning_rate": 9.56140350877193e-08,
      "logits/chosen": -0.6729354858398438,
      "logits/rejected": -0.6799106001853943,
      "logps/chosen": -736.7586669921875,
      "logps/rejected": -583.8125610351562,
      "loss": 0.6992,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01173095591366291,
      "rewards/margins": -0.011246967129409313,
      "rewards/rejected": -0.0004839897155761719,
      "step": 3147
    },
    {
      "epoch": 0.8280261059691923,
      "grad_norm": 548.0,
      "learning_rate": 9.546783625730993e-08,
      "logits/chosen": -0.6538181900978088,
      "logits/rejected": NaN,
      "logps/chosen": -1058.92236328125,
      "logps/rejected": -724.246826171875,
      "loss": 0.6756,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.013523770496249199,
      "rewards/margins": 0.036868806928396225,
      "rewards/rejected": -0.023345040157437325,
      "step": 3148
    },
    {
      "epoch": 0.8282891384043795,
      "grad_norm": 486.0,
      "learning_rate": 9.532163742690059e-08,
      "logits/chosen": -0.6297296285629272,
      "logits/rejected": -0.6364357471466064,
      "logps/chosen": -1111.584228515625,
      "logps/rejected": -668.5236206054688,
      "loss": 0.6976,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0005801203660666943,
      "rewards/margins": -0.008349132724106312,
      "rewards/rejected": 0.0077690123580396175,
      "step": 3149
    },
    {
      "epoch": 0.8285521708395667,
      "grad_norm": 620.0,
      "learning_rate": 9.517543859649123e-08,
      "logits/chosen": -0.6677242517471313,
      "logits/rejected": -0.6645706295967102,
      "logps/chosen": -1383.9644775390625,
      "logps/rejected": -1344.65283203125,
      "loss": 0.6919,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0022916793823242188,
      "rewards/margins": 0.0033433912321925163,
      "rewards/rejected": -0.005635070614516735,
      "step": 3150
    },
    {
      "epoch": 0.8288152032747538,
      "grad_norm": 600.0,
      "learning_rate": 9.502923976608187e-08,
      "logits/chosen": -0.6601651310920715,
      "logits/rejected": -0.6668791770935059,
      "logps/chosen": -1434.416015625,
      "logps/rejected": -1120.1591796875,
      "loss": 0.6859,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.020583057776093483,
      "rewards/margins": 0.01567869447171688,
      "rewards/rejected": 0.004904366098344326,
      "step": 3151
    },
    {
      "epoch": 0.829078235709941,
      "grad_norm": 474.0,
      "learning_rate": 9.488304093567251e-08,
      "logits/chosen": -0.676851212978363,
      "logits/rejected": NaN,
      "logps/chosen": -844.5677490234375,
      "logps/rejected": -471.6210021972656,
      "loss": 0.678,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.020040608942508698,
      "rewards/margins": 0.031192302703857422,
      "rewards/rejected": -0.011151695623993874,
      "step": 3152
    },
    {
      "epoch": 0.8293412681451281,
      "grad_norm": 584.0,
      "learning_rate": 9.473684210526316e-08,
      "logits/chosen": -0.6455064415931702,
      "logits/rejected": -0.6406381130218506,
      "logps/chosen": -1146.66455078125,
      "logps/rejected": -881.472412109375,
      "loss": 0.689,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.019095612689852715,
      "rewards/margins": 0.010205366648733616,
      "rewards/rejected": -0.029300976544618607,
      "step": 3153
    },
    {
      "epoch": 0.8296043005803153,
      "grad_norm": 596.0,
      "learning_rate": 9.459064327485379e-08,
      "logits/chosen": -0.6541174650192261,
      "logits/rejected": -0.6475069522857666,
      "logps/chosen": -1260.94140625,
      "logps/rejected": -987.63525390625,
      "loss": 0.6833,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.018733026459813118,
      "rewards/margins": 0.02042550966143608,
      "rewards/rejected": -0.0016924855299293995,
      "step": 3154
    },
    {
      "epoch": 0.8298673330155025,
      "grad_norm": 552.0,
      "learning_rate": 9.444444444444444e-08,
      "logits/chosen": -0.667989194393158,
      "logits/rejected": -0.6729382276535034,
      "logps/chosen": -1394.792236328125,
      "logps/rejected": -872.547119140625,
      "loss": 0.692,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.00717926025390625,
      "rewards/margins": 0.003242588136345148,
      "rewards/rejected": 0.003936672583222389,
      "step": 3155
    },
    {
      "epoch": 0.8301303654506896,
      "grad_norm": 560.0,
      "learning_rate": 9.429824561403509e-08,
      "logits/chosen": -0.6323013305664062,
      "logits/rejected": -0.6291437149047852,
      "logps/chosen": -1088.5877685546875,
      "logps/rejected": -740.6365356445312,
      "loss": 0.6878,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.016421223059296608,
      "rewards/margins": 0.01151418685913086,
      "rewards/rejected": 0.004907035734504461,
      "step": 3156
    },
    {
      "epoch": 0.8303933978858768,
      "grad_norm": 382.0,
      "learning_rate": 9.415204678362573e-08,
      "logits/chosen": -0.6410701274871826,
      "logits/rejected": -0.6447040438652039,
      "logps/chosen": -947.462646484375,
      "logps/rejected": -929.7703857421875,
      "loss": 0.6927,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.009973715990781784,
      "rewards/margins": 0.001269721775315702,
      "rewards/rejected": 0.008703993633389473,
      "step": 3157
    },
    {
      "epoch": 0.830656430321064,
      "grad_norm": 478.0,
      "learning_rate": 9.400584795321636e-08,
      "logits/chosen": -0.6464026570320129,
      "logits/rejected": -0.6409633159637451,
      "logps/chosen": -878.4969482421875,
      "logps/rejected": -803.2787475585938,
      "loss": 0.6806,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008281802758574486,
      "rewards/margins": 0.025943946093320847,
      "rewards/rejected": -0.01766214519739151,
      "step": 3158
    },
    {
      "epoch": 0.8309194627562512,
      "grad_norm": 544.0,
      "learning_rate": 9.385964912280701e-08,
      "logits/chosen": -0.6511998176574707,
      "logits/rejected": -0.6592254042625427,
      "logps/chosen": -1242.8250732421875,
      "logps/rejected": -919.7384033203125,
      "loss": 0.6875,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.001997376326471567,
      "rewards/margins": 0.012355757877230644,
      "rewards/rejected": -0.010358382947742939,
      "step": 3159
    },
    {
      "epoch": 0.8311824951914383,
      "grad_norm": 548.0,
      "learning_rate": 9.371345029239766e-08,
      "logits/chosen": -0.6401716470718384,
      "logits/rejected": -0.6428238749504089,
      "logps/chosen": -1072.1068115234375,
      "logps/rejected": -801.153564453125,
      "loss": 0.7008,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.004693127237260342,
      "rewards/margins": -0.014149334281682968,
      "rewards/rejected": 0.009456205181777477,
      "step": 3160
    },
    {
      "epoch": 0.8314455276266255,
      "grad_norm": 648.0,
      "learning_rate": 9.356725146198829e-08,
      "logits/chosen": -0.6558099985122681,
      "logits/rejected": -0.6560882925987244,
      "logps/chosen": -1005.0870971679688,
      "logps/rejected": -936.8909301757812,
      "loss": 0.7108,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0070542339235544205,
      "rewards/margins": -0.03383960947394371,
      "rewards/rejected": 0.02678537555038929,
      "step": 3161
    },
    {
      "epoch": 0.8317085600618126,
      "grad_norm": 478.0,
      "learning_rate": 9.342105263157894e-08,
      "logits/chosen": -0.6737765669822693,
      "logits/rejected": NaN,
      "logps/chosen": -983.5452880859375,
      "logps/rejected": -659.5822143554688,
      "loss": 0.6868,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.00014381436631083488,
      "rewards/margins": 0.013880632817745209,
      "rewards/rejected": -0.01373682077974081,
      "step": 3162
    },
    {
      "epoch": 0.8319715924969998,
      "grad_norm": 576.0,
      "learning_rate": 9.327485380116958e-08,
      "logits/chosen": -0.6426894068717957,
      "logits/rejected": -0.6382356286048889,
      "logps/chosen": -1017.703369140625,
      "logps/rejected": -970.1842041015625,
      "loss": 0.692,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.008561897091567516,
      "rewards/margins": 0.0029208180494606495,
      "rewards/rejected": -0.011482715606689453,
      "step": 3163
    },
    {
      "epoch": 0.8322346249321869,
      "grad_norm": 466.0,
      "learning_rate": 9.312865497076023e-08,
      "logits/chosen": -0.6733658909797668,
      "logits/rejected": NaN,
      "logps/chosen": -1169.7137451171875,
      "logps/rejected": -939.7636108398438,
      "loss": 0.6906,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.00279998779296875,
      "rewards/margins": 0.005722903646528721,
      "rewards/rejected": -0.008522892370820045,
      "step": 3164
    },
    {
      "epoch": 0.8324976573673741,
      "grad_norm": 676.0,
      "learning_rate": 9.298245614035088e-08,
      "logits/chosen": -0.6514444351196289,
      "logits/rejected": -0.6544346213340759,
      "logps/chosen": -1373.0291748046875,
      "logps/rejected": -979.21044921875,
      "loss": 0.6945,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.003233910072594881,
      "rewards/margins": -0.001854324946179986,
      "rewards/rejected": 0.0050882333889603615,
      "step": 3165
    },
    {
      "epoch": 0.8327606898025612,
      "grad_norm": 540.0,
      "learning_rate": 9.283625730994152e-08,
      "logits/chosen": -0.6475692391395569,
      "logits/rejected": -0.6505753993988037,
      "logps/chosen": -1161.5445556640625,
      "logps/rejected": -972.479736328125,
      "loss": 0.6905,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.009520484134554863,
      "rewards/margins": 0.0060898312367498875,
      "rewards/rejected": -0.015610315836966038,
      "step": 3166
    },
    {
      "epoch": 0.8330237222377485,
      "grad_norm": 398.0,
      "learning_rate": 9.269005847953217e-08,
      "logits/chosen": -0.6401280760765076,
      "logits/rejected": -0.6553716063499451,
      "logps/chosen": -723.6717529296875,
      "logps/rejected": -555.5863037109375,
      "loss": 0.705,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.009460162371397018,
      "rewards/margins": -0.02227029949426651,
      "rewards/rejected": 0.012810136191546917,
      "step": 3167
    },
    {
      "epoch": 0.8332867546729356,
      "grad_norm": 438.0,
      "learning_rate": 9.25438596491228e-08,
      "logits/chosen": -0.6538376212120056,
      "logits/rejected": -0.6436510682106018,
      "logps/chosen": -745.5510864257812,
      "logps/rejected": -748.15966796875,
      "loss": 0.6896,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.016487503424286842,
      "rewards/margins": 0.008050345815718174,
      "rewards/rejected": -0.02453785017132759,
      "step": 3168
    },
    {
      "epoch": 0.8335497871081228,
      "grad_norm": 556.0,
      "learning_rate": 9.239766081871345e-08,
      "logits/chosen": -0.6695256233215332,
      "logits/rejected": -0.6628408432006836,
      "logps/chosen": -1130.2337646484375,
      "logps/rejected": -703.3238525390625,
      "loss": 0.6971,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.019150113686919212,
      "rewards/margins": -0.00783448200672865,
      "rewards/rejected": -0.011315632611513138,
      "step": 3169
    },
    {
      "epoch": 0.8338128195433099,
      "grad_norm": 548.0,
      "learning_rate": 9.22514619883041e-08,
      "logits/chosen": -0.6537450551986694,
      "logits/rejected": -0.6632758975028992,
      "logps/chosen": -1110.9859619140625,
      "logps/rejected": -784.7333374023438,
      "loss": 0.7042,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0024360655806958675,
      "rewards/margins": -0.02059640735387802,
      "rewards/rejected": 0.018160343170166016,
      "step": 3170
    },
    {
      "epoch": 0.8340758519784971,
      "grad_norm": 498.0,
      "learning_rate": 9.210526315789473e-08,
      "logits/chosen": -0.6336286664009094,
      "logits/rejected": -0.6329505443572998,
      "logps/chosen": -940.4557495117188,
      "logps/rejected": -834.5531005859375,
      "loss": 0.6824,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.017410852015018463,
      "rewards/margins": 0.02208547666668892,
      "rewards/rejected": -0.0046746255829930305,
      "step": 3171
    },
    {
      "epoch": 0.8343388844136843,
      "grad_norm": 544.0,
      "learning_rate": 9.195906432748537e-08,
      "logits/chosen": -0.6437140703201294,
      "logits/rejected": -0.6412384510040283,
      "logps/chosen": -1251.2357177734375,
      "logps/rejected": -1036.9949951171875,
      "loss": 0.6874,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.010869026184082031,
      "rewards/margins": 0.01189127005636692,
      "rewards/rejected": -0.022760294377803802,
      "step": 3172
    },
    {
      "epoch": 0.8346019168488714,
      "grad_norm": 482.0,
      "learning_rate": 9.181286549707602e-08,
      "logits/chosen": -0.6569189429283142,
      "logits/rejected": -0.6566504836082458,
      "logps/chosen": -1148.4879150390625,
      "logps/rejected": -899.4322509765625,
      "loss": 0.6888,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.01072235032916069,
      "rewards/margins": 0.009427260607481003,
      "rewards/rejected": -0.020149612799286842,
      "step": 3173
    },
    {
      "epoch": 0.8348649492840586,
      "grad_norm": 588.0,
      "learning_rate": 9.166666666666665e-08,
      "logits/chosen": -0.6518215537071228,
      "logits/rejected": -0.6388339996337891,
      "logps/chosen": -1299.568115234375,
      "logps/rejected": -1125.84033203125,
      "loss": 0.6851,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.00800781324505806,
      "rewards/margins": 0.016851425170898438,
      "rewards/rejected": -0.008843613788485527,
      "step": 3174
    },
    {
      "epoch": 0.8351279817192457,
      "grad_norm": 660.0,
      "learning_rate": 9.15204678362573e-08,
      "logits/chosen": -0.6446437835693359,
      "logits/rejected": -0.6509314775466919,
      "logps/chosen": -1469.1658935546875,
      "logps/rejected": -1149.28857421875,
      "loss": 0.6936,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02923126332461834,
      "rewards/margins": 3.623543307185173e-06,
      "rewards/rejected": -0.029234886169433594,
      "step": 3175
    },
    {
      "epoch": 0.835391014154433,
      "grad_norm": 648.0,
      "learning_rate": 9.137426900584795e-08,
      "logits/chosen": -0.6145474910736084,
      "logits/rejected": -0.6245757937431335,
      "logps/chosen": -1394.7349853515625,
      "logps/rejected": -1323.9378662109375,
      "loss": 0.6919,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.009173584170639515,
      "rewards/margins": 0.003583813551813364,
      "rewards/rejected": 0.005589771084487438,
      "step": 3176
    },
    {
      "epoch": 0.8356540465896201,
      "grad_norm": 532.0,
      "learning_rate": 9.122807017543859e-08,
      "logits/chosen": -0.6636046171188354,
      "logits/rejected": -0.6663906574249268,
      "logps/chosen": -1353.697265625,
      "logps/rejected": -984.5608520507812,
      "loss": 0.6897,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.013770961202681065,
      "rewards/margins": 0.007429886609315872,
      "rewards/rejected": 0.006341077852994204,
      "step": 3177
    },
    {
      "epoch": 0.8359170790248073,
      "grad_norm": 480.0,
      "learning_rate": 9.108187134502923e-08,
      "logits/chosen": -0.654447615146637,
      "logits/rejected": -0.6614193916320801,
      "logps/chosen": -914.51806640625,
      "logps/rejected": -648.1600341796875,
      "loss": 0.6914,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.016954470425844193,
      "rewards/margins": 0.0039154523983597755,
      "rewards/rejected": 0.013039018958806992,
      "step": 3178
    },
    {
      "epoch": 0.8361801114599944,
      "grad_norm": 480.0,
      "learning_rate": 9.093567251461989e-08,
      "logits/chosen": -0.6548720002174377,
      "logits/rejected": -0.668830156326294,
      "logps/chosen": -1006.463134765625,
      "logps/rejected": -902.2659912109375,
      "loss": 0.6921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.021423721686005592,
      "rewards/margins": 0.0027763349935412407,
      "rewards/rejected": 0.018647383898496628,
      "step": 3179
    },
    {
      "epoch": 0.8364431438951816,
      "grad_norm": 568.0,
      "learning_rate": 9.078947368421053e-08,
      "logits/chosen": -0.6690472364425659,
      "logits/rejected": -0.6752098798751831,
      "logps/chosen": -1233.4691162109375,
      "logps/rejected": -1126.8233642578125,
      "loss": 0.6912,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.006287765223532915,
      "rewards/margins": 0.00459823664277792,
      "rewards/rejected": -0.010886002331972122,
      "step": 3180
    },
    {
      "epoch": 0.8367061763303687,
      "grad_norm": 454.0,
      "learning_rate": 9.064327485380117e-08,
      "logits/chosen": -0.6336712837219238,
      "logits/rejected": -0.636134147644043,
      "logps/chosen": -1104.55859375,
      "logps/rejected": -886.3878173828125,
      "loss": 0.6914,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0013401983305811882,
      "rewards/margins": 0.004308795556426048,
      "rewards/rejected": -0.005648996215313673,
      "step": 3181
    },
    {
      "epoch": 0.8369692087655559,
      "grad_norm": 398.0,
      "learning_rate": 9.049707602339181e-08,
      "logits/chosen": -0.6675209999084473,
      "logits/rejected": -0.6725955605506897,
      "logps/chosen": -940.54833984375,
      "logps/rejected": -907.073974609375,
      "loss": 0.6939,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0008510590996593237,
      "rewards/margins": -0.00025343825109303,
      "rewards/rejected": 0.001104497816413641,
      "step": 3182
    },
    {
      "epoch": 0.837232241200743,
      "grad_norm": 576.0,
      "learning_rate": 9.035087719298246e-08,
      "logits/chosen": -0.6577436327934265,
      "logits/rejected": -0.6604604721069336,
      "logps/chosen": -1101.60498046875,
      "logps/rejected": -881.2540893554688,
      "loss": 0.6901,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0009000776335597038,
      "rewards/margins": 0.007030487060546875,
      "rewards/rejected": -0.006130409426987171,
      "step": 3183
    },
    {
      "epoch": 0.8374952736359302,
      "grad_norm": 426.0,
      "learning_rate": 9.020467836257309e-08,
      "logits/chosen": -0.6597630381584167,
      "logits/rejected": -0.6514955759048462,
      "logps/chosen": -931.0550537109375,
      "logps/rejected": -660.2808227539062,
      "loss": 0.6955,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.02312927134335041,
      "rewards/margins": -0.004062365740537643,
      "rewards/rejected": 0.027191638946533203,
      "step": 3184
    },
    {
      "epoch": 0.8377583060711173,
      "grad_norm": 488.0,
      "learning_rate": 9.005847953216374e-08,
      "logits/chosen": -0.6349915266036987,
      "logits/rejected": -0.6396504640579224,
      "logps/chosen": -1168.4136962890625,
      "logps/rejected": -800.0033569335938,
      "loss": 0.6859,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.026759911328554153,
      "rewards/margins": 0.01482248306274414,
      "rewards/rejected": 0.011937427334487438,
      "step": 3185
    },
    {
      "epoch": 0.8380213385063046,
      "grad_norm": 592.0,
      "learning_rate": 8.991228070175438e-08,
      "logits/chosen": -0.6663191318511963,
      "logits/rejected": -0.646336019039154,
      "logps/chosen": -825.8386840820312,
      "logps/rejected": -782.5701293945312,
      "loss": 0.6889,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008855628781020641,
      "rewards/margins": 0.009570885449647903,
      "rewards/rejected": -0.0007152566686272621,
      "step": 3186
    },
    {
      "epoch": 0.8382843709414918,
      "grad_norm": 394.0,
      "learning_rate": 8.976608187134503e-08,
      "logits/chosen": -0.6590477824211121,
      "logits/rejected": -0.6613969206809998,
      "logps/chosen": -1191.87939453125,
      "logps/rejected": -1201.9720458984375,
      "loss": 0.6843,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.041945647448301315,
      "rewards/margins": 0.018125439062714577,
      "rewards/rejected": 0.023820210248231888,
      "step": 3187
    },
    {
      "epoch": 0.8385474033766789,
      "grad_norm": 552.0,
      "learning_rate": 8.961988304093566e-08,
      "logits/chosen": -0.6571621894836426,
      "logits/rejected": -0.656110405921936,
      "logps/chosen": -1301.4981689453125,
      "logps/rejected": -1043.1175537109375,
      "loss": 0.6902,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02105255052447319,
      "rewards/margins": 0.006622314918786287,
      "rewards/rejected": 0.01443023607134819,
      "step": 3188
    },
    {
      "epoch": 0.8388104358118661,
      "grad_norm": 544.0,
      "learning_rate": 8.947368421052631e-08,
      "logits/chosen": -0.6472318768501282,
      "logits/rejected": -0.641131579875946,
      "logps/chosen": -1226.401123046875,
      "logps/rejected": -996.0534057617188,
      "loss": 0.6944,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01609344407916069,
      "rewards/margins": -0.00203018169850111,
      "rewards/rejected": -0.014063265174627304,
      "step": 3189
    },
    {
      "epoch": 0.8390734682470532,
      "grad_norm": 624.0,
      "learning_rate": 8.932748538011696e-08,
      "logits/chosen": -0.6689949035644531,
      "logits/rejected": -0.6784027218818665,
      "logps/chosen": -1363.607666015625,
      "logps/rejected": -1377.705322265625,
      "loss": 0.6891,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00027999933809041977,
      "rewards/margins": 0.008402442559599876,
      "rewards/rejected": -0.008682440966367722,
      "step": 3190
    },
    {
      "epoch": 0.8393365006822404,
      "grad_norm": 592.0,
      "learning_rate": 8.918128654970759e-08,
      "logits/chosen": -0.6511867046356201,
      "logits/rejected": -0.6459702849388123,
      "logps/chosen": -1359.9490966796875,
      "logps/rejected": -1239.22802734375,
      "loss": 0.687,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.025782395154237747,
      "rewards/margins": 0.013148976489901543,
      "rewards/rejected": 0.012633420526981354,
      "step": 3191
    },
    {
      "epoch": 0.8395995331174275,
      "grad_norm": 426.0,
      "learning_rate": 8.903508771929824e-08,
      "logits/chosen": -0.6500661969184875,
      "logits/rejected": -0.6402787566184998,
      "logps/chosen": -981.3475341796875,
      "logps/rejected": -568.975830078125,
      "loss": 0.6964,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.004650306887924671,
      "rewards/margins": -0.00591969583183527,
      "rewards/rejected": 0.010570002719759941,
      "step": 3192
    },
    {
      "epoch": 0.8398625655526147,
      "grad_norm": 432.0,
      "learning_rate": 8.888888888888888e-08,
      "logits/chosen": -0.650058925151825,
      "logits/rejected": -0.6485845446586609,
      "logps/chosen": -924.0294799804688,
      "logps/rejected": -748.59423828125,
      "loss": 0.704,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.018601512536406517,
      "rewards/margins": -0.021023893728852272,
      "rewards/rejected": 0.0024223807267844677,
      "step": 3193
    },
    {
      "epoch": 0.8401255979878018,
      "grad_norm": 612.0,
      "learning_rate": 8.874269005847953e-08,
      "logits/chosen": NaN,
      "logits/rejected": -0.66829514503479,
      "logps/chosen": -1219.520263671875,
      "logps/rejected": -826.725341796875,
      "loss": 0.7,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 2.3365486413240433e-05,
      "rewards/margins": -0.012750721536576748,
      "rewards/rejected": 0.012774085626006126,
      "step": 3194
    },
    {
      "epoch": 0.8403886304229891,
      "grad_norm": 616.0,
      "learning_rate": 8.859649122807017e-08,
      "logits/chosen": -0.6664535403251648,
      "logits/rejected": -0.6519583463668823,
      "logps/chosen": -1315.7506103515625,
      "logps/rejected": -840.9656372070312,
      "loss": 0.7026,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.015039253048598766,
      "rewards/margins": -0.017634868621826172,
      "rewards/rejected": 0.002595615340396762,
      "step": 3195
    },
    {
      "epoch": 0.8406516628581762,
      "grad_norm": 464.0,
      "learning_rate": 8.845029239766082e-08,
      "logits/chosen": -0.6521514654159546,
      "logits/rejected": -0.6634317636489868,
      "logps/chosen": -1014.70751953125,
      "logps/rejected": -675.4851684570312,
      "loss": 0.6993,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.007772158831357956,
      "rewards/margins": -0.011765193194150925,
      "rewards/rejected": 0.01953735388815403,
      "step": 3196
    },
    {
      "epoch": 0.8409146952933634,
      "grad_norm": 464.0,
      "learning_rate": 8.830409356725147e-08,
      "logits/chosen": -0.6516308784484863,
      "logits/rejected": -0.6539063453674316,
      "logps/chosen": -1148.0609130859375,
      "logps/rejected": -861.16015625,
      "loss": 0.6936,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0019753456581383944,
      "rewards/margins": -0.00011992454528808594,
      "rewards/rejected": 0.002095270436257124,
      "step": 3197
    },
    {
      "epoch": 0.8411777277285505,
      "grad_norm": 486.0,
      "learning_rate": 8.81578947368421e-08,
      "logits/chosen": -0.658532977104187,
      "logits/rejected": NaN,
      "logps/chosen": -919.2666625976562,
      "logps/rejected": -482.7397766113281,
      "loss": 0.6941,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0011703497730195522,
      "rewards/margins": -0.0012095941929146647,
      "rewards/rejected": 0.0023799424525350332,
      "step": 3198
    },
    {
      "epoch": 0.8414407601637377,
      "grad_norm": 924.0,
      "learning_rate": 8.801169590643275e-08,
      "logits/chosen": -0.6575806140899658,
      "logits/rejected": NaN,
      "logps/chosen": -1440.3992919921875,
      "logps/rejected": -895.9146728515625,
      "loss": 0.685,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008664989843964577,
      "rewards/margins": 0.017586614936590195,
      "rewards/rejected": -0.008921623229980469,
      "step": 3199
    },
    {
      "epoch": 0.8417037925989248,
      "grad_norm": 478.0,
      "learning_rate": 8.786549707602339e-08,
      "logits/chosen": -0.637775182723999,
      "logits/rejected": -0.6442499756813049,
      "logps/chosen": -1063.008544921875,
      "logps/rejected": -975.8729858398438,
      "loss": 0.696,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.007962511852383614,
      "rewards/margins": -0.004131700843572617,
      "rewards/rejected": -0.003830814966931939,
      "step": 3200
    },
    {
      "epoch": 0.841966825034112,
      "grad_norm": 636.0,
      "learning_rate": 8.771929824561403e-08,
      "logits/chosen": -0.6327064037322998,
      "logits/rejected": -0.6409910321235657,
      "logps/chosen": -1082.6265869140625,
      "logps/rejected": -1009.8580932617188,
      "loss": 0.7079,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.021933365613222122,
      "rewards/margins": -0.028352737426757812,
      "rewards/rejected": 0.006419372744858265,
      "step": 3201
    },
    {
      "epoch": 0.8422298574692992,
      "grad_norm": 556.0,
      "learning_rate": 8.757309941520467e-08,
      "logits/chosen": -0.6311368346214294,
      "logits/rejected": -0.6399248242378235,
      "logps/chosen": -1029.9154052734375,
      "logps/rejected": -937.9061889648438,
      "loss": 0.6896,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0195509921759367,
      "rewards/margins": 0.008302593603730202,
      "rewards/rejected": 0.011248398572206497,
      "step": 3202
    },
    {
      "epoch": 0.8424928899044863,
      "grad_norm": 668.0,
      "learning_rate": 8.742690058479532e-08,
      "logits/chosen": -0.6516834497451782,
      "logits/rejected": -0.6660537123680115,
      "logps/chosen": -1605.946044921875,
      "logps/rejected": -1078.1463623046875,
      "loss": 0.6905,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.026884794235229492,
      "rewards/margins": 0.005911683663725853,
      "rewards/rejected": 0.02097310870885849,
      "step": 3203
    },
    {
      "epoch": 0.8427559223396736,
      "grad_norm": 490.0,
      "learning_rate": 8.728070175438595e-08,
      "logits/chosen": -0.6361050009727478,
      "logits/rejected": -0.6304967403411865,
      "logps/chosen": -1110.484130859375,
      "logps/rejected": -505.97314453125,
      "loss": 0.6807,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.017351532354950905,
      "rewards/margins": 0.025928115472197533,
      "rewards/rejected": -0.008576584979891777,
      "step": 3204
    },
    {
      "epoch": 0.8430189547748607,
      "grad_norm": 424.0,
      "learning_rate": 8.71345029239766e-08,
      "logits/chosen": -0.633505642414093,
      "logits/rejected": -0.6435647010803223,
      "logps/chosen": -943.6782836914062,
      "logps/rejected": -938.8497924804688,
      "loss": 0.691,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007497215643525124,
      "rewards/margins": 0.004706002771854401,
      "rewards/rejected": 0.0027912138029932976,
      "step": 3205
    },
    {
      "epoch": 0.8432819872100479,
      "grad_norm": 628.0,
      "learning_rate": 8.698830409356725e-08,
      "logits/chosen": -0.6570183634757996,
      "logits/rejected": -0.6679726839065552,
      "logps/chosen": -1311.8533935546875,
      "logps/rejected": -1020.91259765625,
      "loss": 0.6899,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008501434698700905,
      "rewards/margins": 0.007734869606792927,
      "rewards/rejected": 0.0007665640441700816,
      "step": 3206
    },
    {
      "epoch": 0.843545019645235,
      "grad_norm": 520.0,
      "learning_rate": 8.684210526315789e-08,
      "logits/chosen": -0.6281495094299316,
      "logits/rejected": -0.636099636554718,
      "logps/chosen": -1170.428466796875,
      "logps/rejected": -1022.51611328125,
      "loss": 0.6821,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.011423682793974876,
      "rewards/margins": 0.02317972294986248,
      "rewards/rejected": -0.011756038293242455,
      "step": 3207
    },
    {
      "epoch": 0.8438080520804222,
      "grad_norm": 438.0,
      "learning_rate": 8.669590643274852e-08,
      "logits/chosen": -0.6416327357292175,
      "logits/rejected": -0.6417866945266724,
      "logps/chosen": -1096.88427734375,
      "logps/rejected": -916.5576171875,
      "loss": 0.6822,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.007086658384650946,
      "rewards/margins": 0.02214984968304634,
      "rewards/rejected": -0.015063190832734108,
      "step": 3208
    },
    {
      "epoch": 0.8440710845156093,
      "grad_norm": 524.0,
      "learning_rate": 8.654970760233918e-08,
      "logits/chosen": -0.6535831093788147,
      "logits/rejected": -0.6589617729187012,
      "logps/chosen": -1236.376953125,
      "logps/rejected": -1055.758544921875,
      "loss": 0.6906,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.011138916015625,
      "rewards/margins": 0.005595589056611061,
      "rewards/rejected": -0.01673450693488121,
      "step": 3209
    },
    {
      "epoch": 0.8443341169507965,
      "grad_norm": 664.0,
      "learning_rate": 8.640350877192983e-08,
      "logits/chosen": -0.663545548915863,
      "logits/rejected": -0.6761699914932251,
      "logps/chosen": -1323.433837890625,
      "logps/rejected": -1026.335205078125,
      "loss": 0.6858,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01525425910949707,
      "rewards/margins": 0.015047024935483932,
      "rewards/rejected": 0.00020723347552120686,
      "step": 3210
    },
    {
      "epoch": 0.8445971493859836,
      "grad_norm": 432.0,
      "learning_rate": 8.625730994152046e-08,
      "logits/chosen": -0.6588646769523621,
      "logits/rejected": -0.6669729351997375,
      "logps/chosen": -1213.5106201171875,
      "logps/rejected": -950.8565063476562,
      "loss": 0.7016,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.023194694891572,
      "rewards/margins": -0.015997599810361862,
      "rewards/rejected": -0.007197094149887562,
      "step": 3211
    },
    {
      "epoch": 0.8448601818211708,
      "grad_norm": 492.0,
      "learning_rate": 8.611111111111111e-08,
      "logits/chosen": -0.6520324945449829,
      "logits/rejected": -0.6632599830627441,
      "logps/chosen": -1031.7276611328125,
      "logps/rejected": -855.9935913085938,
      "loss": 0.6776,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01523294486105442,
      "rewards/margins": 0.032732680439949036,
      "rewards/rejected": -0.017499733716249466,
      "step": 3212
    },
    {
      "epoch": 0.845123214256358,
      "grad_norm": 588.0,
      "learning_rate": 8.596491228070176e-08,
      "logits/chosen": -0.6545071601867676,
      "logits/rejected": -0.6495238542556763,
      "logps/chosen": -1479.2919921875,
      "logps/rejected": -916.7642822265625,
      "loss": 0.6854,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.013067342340946198,
      "rewards/margins": 0.01596040651202202,
      "rewards/rejected": -0.00289306603372097,
      "step": 3213
    },
    {
      "epoch": 0.8453862466915452,
      "grad_norm": 524.0,
      "learning_rate": 8.581871345029239e-08,
      "logits/chosen": -0.6560004949569702,
      "logits/rejected": -0.6577983498573303,
      "logps/chosen": -887.2601928710938,
      "logps/rejected": -862.5444946289062,
      "loss": 0.7011,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01542816124856472,
      "rewards/margins": -0.014720774255692959,
      "rewards/rejected": -0.0007073874585330486,
      "step": 3214
    },
    {
      "epoch": 0.8456492791267323,
      "grad_norm": 600.0,
      "learning_rate": 8.567251461988304e-08,
      "logits/chosen": -0.6668698191642761,
      "logits/rejected": -0.6728308796882629,
      "logps/chosen": -999.9003295898438,
      "logps/rejected": -922.9695434570312,
      "loss": 0.6982,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.008017873391509056,
      "rewards/margins": -0.009196043014526367,
      "rewards/rejected": 0.0011781689245253801,
      "step": 3215
    },
    {
      "epoch": 0.8459123115619195,
      "grad_norm": 680.0,
      "learning_rate": 8.552631578947368e-08,
      "logits/chosen": -0.6582079529762268,
      "logits/rejected": NaN,
      "logps/chosen": -1304.0015869140625,
      "logps/rejected": -819.5643920898438,
      "loss": 0.7008,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011647416278719902,
      "rewards/margins": -0.01417617779225111,
      "rewards/rejected": 0.0025287640746682882,
      "step": 3216
    },
    {
      "epoch": 0.8461753439971066,
      "grad_norm": 540.0,
      "learning_rate": 8.538011695906433e-08,
      "logits/chosen": -0.6470508575439453,
      "logits/rejected": -0.6514258980751038,
      "logps/chosen": -1276.38671875,
      "logps/rejected": -1124.4039306640625,
      "loss": 0.699,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.010028745047748089,
      "rewards/margins": -0.010513783432543278,
      "rewards/rejected": 0.0004850379191339016,
      "step": 3217
    },
    {
      "epoch": 0.8464383764322938,
      "grad_norm": 472.0,
      "learning_rate": 8.523391812865496e-08,
      "logits/chosen": -0.6541500091552734,
      "logits/rejected": -0.6505104899406433,
      "logps/chosen": -1045.529541015625,
      "logps/rejected": -746.3432006835938,
      "loss": 0.6924,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004521941300481558,
      "rewards/margins": 0.0024454109370708466,
      "rewards/rejected": 0.002076530596241355,
      "step": 3218
    },
    {
      "epoch": 0.846701408867481,
      "grad_norm": 520.0,
      "learning_rate": 8.508771929824561e-08,
      "logits/chosen": -0.6737242937088013,
      "logits/rejected": -0.6921232342720032,
      "logps/chosen": -1273.2159423828125,
      "logps/rejected": -830.71484375,
      "loss": 0.6875,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.010721301659941673,
      "rewards/margins": 0.012498663738369942,
      "rewards/rejected": -0.0017773611471056938,
      "step": 3219
    },
    {
      "epoch": 0.8469644413026681,
      "grad_norm": 472.0,
      "learning_rate": 8.494152046783625e-08,
      "logits/chosen": -0.6672663688659668,
      "logits/rejected": -0.6778590679168701,
      "logps/chosen": -951.527099609375,
      "logps/rejected": -837.19775390625,
      "loss": 0.6846,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.015975473448634148,
      "rewards/margins": 0.017393827438354492,
      "rewards/rejected": -0.001418351661413908,
      "step": 3220
    },
    {
      "epoch": 0.8472274737378553,
      "grad_norm": 536.0,
      "learning_rate": 8.479532163742689e-08,
      "logits/chosen": -0.6483941078186035,
      "logits/rejected": -0.6622878313064575,
      "logps/chosen": -1374.068115234375,
      "logps/rejected": -1000.5634765625,
      "loss": 0.6948,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0034072871785610914,
      "rewards/margins": -0.0023124704603105783,
      "rewards/rejected": 0.00571975763887167,
      "step": 3221
    },
    {
      "epoch": 0.8474905061730424,
      "grad_norm": 458.0,
      "learning_rate": 8.464912280701753e-08,
      "logits/chosen": -0.6438848972320557,
      "logits/rejected": -0.6427682638168335,
      "logps/chosen": -1102.671142578125,
      "logps/rejected": -859.4422607421875,
      "loss": 0.6825,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.0067747123539447784,
      "rewards/margins": 0.021633148193359375,
      "rewards/rejected": -0.014858437702059746,
      "step": 3222
    },
    {
      "epoch": 0.8477535386082297,
      "grad_norm": 588.0,
      "learning_rate": 8.450292397660818e-08,
      "logits/chosen": -0.6472891569137573,
      "logits/rejected": -0.6477611660957336,
      "logps/chosen": -1355.5291748046875,
      "logps/rejected": -850.711181640625,
      "loss": 0.6701,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.025197410956025124,
      "rewards/margins": 0.04785308986902237,
      "rewards/rejected": -0.022655677050352097,
      "step": 3223
    },
    {
      "epoch": 0.8480165710434168,
      "grad_norm": 604.0,
      "learning_rate": 8.435672514619883e-08,
      "logits/chosen": -0.6507835984230042,
      "logits/rejected": -0.6510043740272522,
      "logps/chosen": -1752.1607666015625,
      "logps/rejected": -1438.943359375,
      "loss": 0.6864,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.012084389105439186,
      "rewards/margins": 0.014587212353944778,
      "rewards/rejected": -0.026671599596738815,
      "step": 3224
    },
    {
      "epoch": 0.848279603478604,
      "grad_norm": 524.0,
      "learning_rate": 8.421052631578947e-08,
      "logits/chosen": -0.6474308967590332,
      "logits/rejected": -0.6459529399871826,
      "logps/chosen": -1453.3641357421875,
      "logps/rejected": -1020.2943115234375,
      "loss": 0.6929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.009649467654526234,
      "rewards/margins": 0.0013620364479720592,
      "rewards/rejected": -0.01101150456815958,
      "step": 3225
    },
    {
      "epoch": 0.8485426359137911,
      "grad_norm": 422.0,
      "learning_rate": 8.406432748538012e-08,
      "logits/chosen": -0.6370900869369507,
      "logits/rejected": -0.6408418416976929,
      "logps/chosen": -896.0045166015625,
      "logps/rejected": -585.59619140625,
      "loss": 0.7042,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0053119659423828125,
      "rewards/margins": -0.021613599732518196,
      "rewards/rejected": 0.016301631927490234,
      "step": 3226
    },
    {
      "epoch": 0.8488056683489783,
      "grad_norm": 452.0,
      "learning_rate": 8.391812865497077e-08,
      "logits/chosen": -0.6568695902824402,
      "logits/rejected": -0.655859112739563,
      "logps/chosen": -948.7345581054688,
      "logps/rejected": -808.1008911132812,
      "loss": 0.687,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.009806010872125626,
      "rewards/margins": 0.012727167457342148,
      "rewards/rejected": -0.002921151928603649,
      "step": 3227
    },
    {
      "epoch": 0.8490687007841654,
      "grad_norm": 528.0,
      "learning_rate": 8.37719298245614e-08,
      "logits/chosen": -0.6751624941825867,
      "logits/rejected": -0.6677406430244446,
      "logps/chosen": -1351.8148193359375,
      "logps/rejected": -1258.112548828125,
      "loss": 0.68,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.014886094257235527,
      "rewards/margins": 0.027344513684511185,
      "rewards/rejected": -0.012458419427275658,
      "step": 3228
    },
    {
      "epoch": 0.8493317332193526,
      "grad_norm": 572.0,
      "learning_rate": 8.362573099415205e-08,
      "logits/chosen": -0.632387638092041,
      "logits/rejected": -0.6353925466537476,
      "logps/chosen": -837.486572265625,
      "logps/rejected": -1254.4454345703125,
      "loss": 0.7001,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0013339517172425985,
      "rewards/margins": -0.012645101174712181,
      "rewards/rejected": 0.011311151087284088,
      "step": 3229
    },
    {
      "epoch": 0.8495947656545397,
      "grad_norm": 494.0,
      "learning_rate": 8.347953216374269e-08,
      "logits/chosen": -0.6607198119163513,
      "logits/rejected": -0.6490933895111084,
      "logps/chosen": -1127.4923095703125,
      "logps/rejected": -787.2693481445312,
      "loss": 0.7019,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.008535194210708141,
      "rewards/margins": -0.016623258590698242,
      "rewards/rejected": 0.0080880643799901,
      "step": 3230
    },
    {
      "epoch": 0.849857798089727,
      "grad_norm": 512.0,
      "learning_rate": 8.333333333333333e-08,
      "logits/chosen": -0.6517124772071838,
      "logits/rejected": -0.652377724647522,
      "logps/chosen": -939.16552734375,
      "logps/rejected": -724.71044921875,
      "loss": 0.6929,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01817931979894638,
      "rewards/margins": 0.0010308274067938328,
      "rewards/rejected": 0.017148494720458984,
      "step": 3231
    },
    {
      "epoch": 0.850120830524914,
      "grad_norm": 656.0,
      "learning_rate": 8.318713450292397e-08,
      "logits/chosen": -0.6529958248138428,
      "logits/rejected": -0.6430515050888062,
      "logps/chosen": -1455.50927734375,
      "logps/rejected": -1032.9454345703125,
      "loss": 0.6875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.009445859119296074,
      "rewards/margins": 0.011718655936419964,
      "rewards/rejected": -0.021164512261748314,
      "step": 3232
    },
    {
      "epoch": 0.8503838629601013,
      "grad_norm": 624.0,
      "learning_rate": 8.304093567251462e-08,
      "logits/chosen": -0.6423118114471436,
      "logits/rejected": -0.6507422924041748,
      "logps/chosen": -1099.23876953125,
      "logps/rejected": -922.616455078125,
      "loss": 0.7072,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.006547641009092331,
      "rewards/margins": -0.027285005897283554,
      "rewards/rejected": 0.020737361162900925,
      "step": 3233
    },
    {
      "epoch": 0.8506468953952885,
      "grad_norm": 486.0,
      "learning_rate": 8.289473684210525e-08,
      "logits/chosen": -0.6431856155395508,
      "logits/rejected": -0.6391401886940002,
      "logps/chosen": -1006.3553466796875,
      "logps/rejected": -991.5045776367188,
      "loss": 0.6847,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.020748522132635117,
      "rewards/margins": 0.018328476697206497,
      "rewards/rejected": 0.0024200433399528265,
      "step": 3234
    },
    {
      "epoch": 0.8509099278304756,
      "grad_norm": 456.0,
      "learning_rate": 8.27485380116959e-08,
      "logits/chosen": -0.6351351141929626,
      "logits/rejected": -0.640040934085846,
      "logps/chosen": -881.1217041015625,
      "logps/rejected": -841.6795043945312,
      "loss": 0.6945,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0019451153930276632,
      "rewards/margins": -0.0019219398964196444,
      "rewards/rejected": 0.003867052961140871,
      "step": 3235
    },
    {
      "epoch": 0.8511729602656628,
      "grad_norm": 492.0,
      "learning_rate": 8.260233918128654e-08,
      "logits/chosen": -0.6244111657142639,
      "logits/rejected": -0.6335752010345459,
      "logps/chosen": -1088.9945068359375,
      "logps/rejected": -895.7163696289062,
      "loss": 0.6951,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.002993774600327015,
      "rewards/margins": -0.0035832407884299755,
      "rewards/rejected": 0.00657701538875699,
      "step": 3236
    },
    {
      "epoch": 0.8514359927008499,
      "grad_norm": 416.0,
      "learning_rate": 8.245614035087719e-08,
      "logits/chosen": -0.6503568291664124,
      "logits/rejected": -0.6535883545875549,
      "logps/chosen": -948.9725341796875,
      "logps/rejected": -626.2445678710938,
      "loss": 0.6904,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008340359665453434,
      "rewards/margins": 0.005896759685128927,
      "rewards/rejected": 0.0024435999803245068,
      "step": 3237
    },
    {
      "epoch": 0.8516990251360371,
      "grad_norm": 556.0,
      "learning_rate": 8.230994152046782e-08,
      "logits/chosen": -0.6506377458572388,
      "logits/rejected": -0.6434728503227234,
      "logps/chosen": -1139.031494140625,
      "logps/rejected": -760.432373046875,
      "loss": 0.7018,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.014787101186811924,
      "rewards/margins": -0.016878511756658554,
      "rewards/rejected": 0.0020914082415401936,
      "step": 3238
    },
    {
      "epoch": 0.8519620575712242,
      "grad_norm": 620.0,
      "learning_rate": 8.216374269005848e-08,
      "logits/chosen": -0.6497676968574524,
      "logits/rejected": -0.6461203694343567,
      "logps/chosen": -1765.1063232421875,
      "logps/rejected": -1235.5003662109375,
      "loss": 0.704,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.008115958422422409,
      "rewards/margins": -0.020458603277802467,
      "rewards/rejected": 0.012342646718025208,
      "step": 3239
    },
    {
      "epoch": 0.8522250900064114,
      "grad_norm": 486.0,
      "learning_rate": 8.201754385964913e-08,
      "logits/chosen": -0.6662532687187195,
      "logits/rejected": -0.6759926080703735,
      "logps/chosen": -1128.159423828125,
      "logps/rejected": -847.908203125,
      "loss": 0.68,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.012285996228456497,
      "rewards/margins": 0.028205586597323418,
      "rewards/rejected": -0.01591958850622177,
      "step": 3240
    },
    {
      "epoch": 0.8524881224415986,
      "grad_norm": 500.0,
      "learning_rate": 8.187134502923976e-08,
      "logits/chosen": -0.645044207572937,
      "logits/rejected": -0.6494666934013367,
      "logps/chosen": -711.4000854492188,
      "logps/rejected": -697.2318725585938,
      "loss": 0.6976,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.003260707948356867,
      "rewards/margins": -0.008532144129276276,
      "rewards/rejected": 0.01179285068064928,
      "step": 3241
    },
    {
      "epoch": 0.8527511548767858,
      "grad_norm": 636.0,
      "learning_rate": 8.172514619883041e-08,
      "logits/chosen": -0.6664349436759949,
      "logits/rejected": -0.6684752702713013,
      "logps/chosen": -1121.2828369140625,
      "logps/rejected": -1015.7236328125,
      "loss": 0.6919,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.002587985247373581,
      "rewards/margins": 0.0034381861332803965,
      "rewards/rejected": -0.0008502008276991546,
      "step": 3242
    },
    {
      "epoch": 0.8530141873119729,
      "grad_norm": 478.0,
      "learning_rate": 8.157894736842106e-08,
      "logits/chosen": -0.6562628149986267,
      "logits/rejected": -0.6565245389938354,
      "logps/chosen": -1043.6824951171875,
      "logps/rejected": -923.9534301757812,
      "loss": 0.694,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006365442182868719,
      "rewards/margins": -0.0010212890338152647,
      "rewards/rejected": 0.007386732846498489,
      "step": 3243
    },
    {
      "epoch": 0.8532772197471601,
      "grad_norm": 492.0,
      "learning_rate": 8.143274853801169e-08,
      "logits/chosen": -0.6577645540237427,
      "logits/rejected": -0.6583184003829956,
      "logps/chosen": -1073.1875,
      "logps/rejected": -1089.4429931640625,
      "loss": 0.6961,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.003784084226936102,
      "rewards/margins": -0.005475235171616077,
      "rewards/rejected": 0.009259319864213467,
      "step": 3244
    },
    {
      "epoch": 0.8535402521823472,
      "grad_norm": 548.0,
      "learning_rate": 8.128654970760233e-08,
      "logits/chosen": -0.6519213914871216,
      "logits/rejected": -0.6463601589202881,
      "logps/chosen": -1144.9791259765625,
      "logps/rejected": -1030.94140625,
      "loss": 0.7023,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.021016694605350494,
      "rewards/margins": -0.017288779839873314,
      "rewards/rejected": -0.0037279133684933186,
      "step": 3245
    },
    {
      "epoch": 0.8538032846175344,
      "grad_norm": 442.0,
      "learning_rate": 8.114035087719298e-08,
      "logits/chosen": -0.6519219875335693,
      "logits/rejected": -0.6444314122200012,
      "logps/chosen": -733.347412109375,
      "logps/rejected": -619.8319702148438,
      "loss": 0.6927,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010591506958007812,
      "rewards/margins": 0.001203441061079502,
      "rewards/rejected": 0.009388066828250885,
      "step": 3246
    },
    {
      "epoch": 0.8540663170527215,
      "grad_norm": 560.0,
      "learning_rate": 8.099415204678363e-08,
      "logits/chosen": -0.6413103342056274,
      "logits/rejected": -0.6580100655555725,
      "logps/chosen": -1300.26171875,
      "logps/rejected": -1158.32470703125,
      "loss": 0.6876,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0026614186353981495,
      "rewards/margins": 0.012272548861801624,
      "rewards/rejected": -0.009611129760742188,
      "step": 3247
    },
    {
      "epoch": 0.8543293494879087,
      "grad_norm": 472.0,
      "learning_rate": 8.084795321637426e-08,
      "logits/chosen": -0.6193199753761292,
      "logits/rejected": -0.6234081983566284,
      "logps/chosen": -890.6010131835938,
      "logps/rejected": -706.2280883789062,
      "loss": 0.6925,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00754356337711215,
      "rewards/margins": 0.0022826199419796467,
      "rewards/rejected": 0.005260943900793791,
      "step": 3248
    },
    {
      "epoch": 0.854592381923096,
      "grad_norm": 612.0,
      "learning_rate": 8.070175438596491e-08,
      "logits/chosen": -0.6526350975036621,
      "logits/rejected": NaN,
      "logps/chosen": -1291.0909423828125,
      "logps/rejected": -676.9744262695312,
      "loss": 0.6896,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004611586220562458,
      "rewards/margins": 0.008078286424279213,
      "rewards/rejected": -0.003466702066361904,
      "step": 3249
    },
    {
      "epoch": 0.854855414358283,
      "grad_norm": 652.0,
      "learning_rate": 8.055555555555555e-08,
      "logits/chosen": -0.633461594581604,
      "logits/rejected": -0.6413441300392151,
      "logps/chosen": -1527.9862060546875,
      "logps/rejected": -1455.353515625,
      "loss": 0.6932,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.020053576678037643,
      "rewards/margins": 0.001220703125,
      "rewards/rejected": 0.018832873553037643,
      "step": 3250
    },
    {
      "epoch": 0.8551184467934703,
      "grad_norm": 556.0,
      "learning_rate": 8.040935672514619e-08,
      "logits/chosen": -0.6674156785011292,
      "logits/rejected": -0.6480216979980469,
      "logps/chosen": -1029.763427734375,
      "logps/rejected": -670.03173828125,
      "loss": 0.6872,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0018802635604515672,
      "rewards/margins": 0.012790536507964134,
      "rewards/rejected": -0.010910272598266602,
      "step": 3251
    },
    {
      "epoch": 0.8553814792286574,
      "grad_norm": 520.0,
      "learning_rate": 8.026315789473683e-08,
      "logits/chosen": -0.6696175336837769,
      "logits/rejected": -0.6761606335639954,
      "logps/chosen": -1127.3173828125,
      "logps/rejected": -848.0814208984375,
      "loss": 0.708,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.008928965777158737,
      "rewards/margins": -0.02921333536505699,
      "rewards/rejected": 0.020284369587898254,
      "step": 3252
    },
    {
      "epoch": 0.8556445116638446,
      "grad_norm": 506.0,
      "learning_rate": 8.011695906432748e-08,
      "logits/chosen": -0.6516194939613342,
      "logits/rejected": -0.6596845984458923,
      "logps/chosen": -797.725830078125,
      "logps/rejected": -667.3380126953125,
      "loss": 0.7088,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.019742585718631744,
      "rewards/margins": -0.030303671956062317,
      "rewards/rejected": 0.010561084374785423,
      "step": 3253
    },
    {
      "epoch": 0.8559075440990317,
      "grad_norm": 520.0,
      "learning_rate": 7.997076023391813e-08,
      "logits/chosen": -0.6320622563362122,
      "logits/rejected": -0.6362646818161011,
      "logps/chosen": -1212.3973388671875,
      "logps/rejected": -978.1367797851562,
      "loss": 0.6927,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.008259488269686699,
      "rewards/margins": 0.0019453051500022411,
      "rewards/rejected": 0.006314182188361883,
      "step": 3254
    },
    {
      "epoch": 0.8561705765342189,
      "grad_norm": 620.0,
      "learning_rate": 7.982456140350877e-08,
      "logits/chosen": -0.6721535921096802,
      "logits/rejected": -0.6784196496009827,
      "logps/chosen": -1391.67724609375,
      "logps/rejected": -1005.5279541015625,
      "loss": 0.6986,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.004137610550969839,
      "rewards/margins": -0.010103225708007812,
      "rewards/rejected": 0.014240836724638939,
      "step": 3255
    },
    {
      "epoch": 0.856433608969406,
      "grad_norm": 624.0,
      "learning_rate": 7.967836257309942e-08,
      "logits/chosen": -0.6513473987579346,
      "logits/rejected": -0.6501880884170532,
      "logps/chosen": -1186.188232421875,
      "logps/rejected": -919.5859985351562,
      "loss": 0.6834,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.019839096814393997,
      "rewards/margins": 0.020133066922426224,
      "rewards/rejected": -0.00029396999161690474,
      "step": 3256
    },
    {
      "epoch": 0.8566966414045932,
      "grad_norm": 516.0,
      "learning_rate": 7.953216374269006e-08,
      "logits/chosen": -0.6417821049690247,
      "logits/rejected": -0.6428112387657166,
      "logps/chosen": -1223.85498046875,
      "logps/rejected": -816.617431640625,
      "loss": 0.6846,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006094741635024548,
      "rewards/margins": 0.01774120330810547,
      "rewards/rejected": -0.011646460741758347,
      "step": 3257
    },
    {
      "epoch": 0.8569596738397803,
      "grad_norm": 596.0,
      "learning_rate": 7.93859649122807e-08,
      "logits/chosen": -0.6616490483283997,
      "logits/rejected": -0.6587249636650085,
      "logps/chosen": -1104.277099609375,
      "logps/rejected": -881.3031616210938,
      "loss": 0.6997,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.006755925249308348,
      "rewards/margins": -0.012341692112386227,
      "rewards/rejected": 0.005585765466094017,
      "step": 3258
    },
    {
      "epoch": 0.8572227062749675,
      "grad_norm": 520.0,
      "learning_rate": 7.923976608187134e-08,
      "logits/chosen": -0.6590576171875,
      "logits/rejected": -0.657992959022522,
      "logps/chosen": -1106.75732421875,
      "logps/rejected": -800.8434448242188,
      "loss": 0.6834,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.018776515498757362,
      "rewards/margins": 0.0203036330640316,
      "rewards/rejected": -0.0015271184965968132,
      "step": 3259
    },
    {
      "epoch": 0.8574857387101547,
      "grad_norm": 616.0,
      "learning_rate": 7.909356725146199e-08,
      "logits/chosen": -0.6351436376571655,
      "logits/rejected": -0.6243358850479126,
      "logps/chosen": -944.7576904296875,
      "logps/rejected": -792.4100952148438,
      "loss": 0.6923,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.005015850067138672,
      "rewards/margins": 0.003031731117516756,
      "rewards/rejected": -0.00804758071899414,
      "step": 3260
    },
    {
      "epoch": 0.8577487711453419,
      "grad_norm": 872.0,
      "learning_rate": 7.894736842105262e-08,
      "logits/chosen": -0.647138237953186,
      "logits/rejected": -0.65133136510849,
      "logps/chosen": -1562.202392578125,
      "logps/rejected": -1291.87841796875,
      "loss": 0.7046,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.00376052875071764,
      "rewards/margins": -0.022075653076171875,
      "rewards/rejected": 0.01831512525677681,
      "step": 3261
    },
    {
      "epoch": 0.858011803580529,
      "grad_norm": 592.0,
      "learning_rate": 7.880116959064327e-08,
      "logits/chosen": -0.6532014608383179,
      "logits/rejected": -0.6656672954559326,
      "logps/chosen": -1374.210205078125,
      "logps/rejected": -1022.9791259765625,
      "loss": 0.6947,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0006122584454715252,
      "rewards/margins": -0.0025421143509447575,
      "rewards/rejected": 0.0019298559054732323,
      "step": 3262
    },
    {
      "epoch": 0.8582748360157162,
      "grad_norm": 644.0,
      "learning_rate": 7.865497076023392e-08,
      "logits/chosen": -0.6254233121871948,
      "logits/rejected": -0.6331100463867188,
      "logps/chosen": -1131.0963134765625,
      "logps/rejected": -938.9735107421875,
      "loss": 0.7015,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.001954269129782915,
      "rewards/margins": -0.0155105609446764,
      "rewards/rejected": 0.013556289486587048,
      "step": 3263
    },
    {
      "epoch": 0.8585378684509034,
      "grad_norm": 616.0,
      "learning_rate": 7.850877192982455e-08,
      "logits/chosen": -0.6652982234954834,
      "logits/rejected": -0.6729580163955688,
      "logps/chosen": -1274.7144775390625,
      "logps/rejected": -832.6801147460938,
      "loss": 0.6952,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.009119939990341663,
      "rewards/margins": -0.003138113534078002,
      "rewards/rejected": 0.012258053757250309,
      "step": 3264
    },
    {
      "epoch": 0.8588009008860905,
      "grad_norm": 476.0,
      "learning_rate": 7.83625730994152e-08,
      "logits/chosen": -0.6461670994758606,
      "logits/rejected": -0.6414079070091248,
      "logps/chosen": -900.649169921875,
      "logps/rejected": -729.321533203125,
      "loss": 0.6915,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.007098723202943802,
      "rewards/margins": 0.003995037637650967,
      "rewards/rejected": -0.011093759909272194,
      "step": 3265
    },
    {
      "epoch": 0.8590639333212777,
      "grad_norm": 564.0,
      "learning_rate": 7.821637426900584e-08,
      "logits/chosen": -0.6477855443954468,
      "logits/rejected": -0.6563706398010254,
      "logps/chosen": -1204.449951171875,
      "logps/rejected": -1040.5155029296875,
      "loss": 0.699,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0006578443571925163,
      "rewards/margins": -0.010553838685154915,
      "rewards/rejected": 0.00989599246531725,
      "step": 3266
    },
    {
      "epoch": 0.8593269657564648,
      "grad_norm": 482.0,
      "learning_rate": 7.807017543859649e-08,
      "logits/chosen": -0.638007640838623,
      "logits/rejected": -0.6437931060791016,
      "logps/chosen": -1144.1357421875,
      "logps/rejected": -844.1317749023438,
      "loss": 0.6845,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0007126815617084503,
      "rewards/margins": 0.01763749308884144,
      "rewards/rejected": -0.01835017278790474,
      "step": 3267
    },
    {
      "epoch": 0.859589998191652,
      "grad_norm": 604.0,
      "learning_rate": 7.792397660818712e-08,
      "logits/chosen": -0.6382611393928528,
      "logits/rejected": NaN,
      "logps/chosen": -1185.1488037109375,
      "logps/rejected": -907.9736938476562,
      "loss": 0.6995,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008332634344696999,
      "rewards/margins": -0.011759091168642044,
      "rewards/rejected": 0.003426456358283758,
      "step": 3268
    },
    {
      "epoch": 0.8598530306268392,
      "grad_norm": 580.0,
      "learning_rate": 7.777777777777778e-08,
      "logits/chosen": -0.6584889888763428,
      "logits/rejected": -0.654693603515625,
      "logps/chosen": -875.2708129882812,
      "logps/rejected": -869.2724609375,
      "loss": 0.6994,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.004052257630974054,
      "rewards/margins": -0.01208591554313898,
      "rewards/rejected": 0.016138173639774323,
      "step": 3269
    },
    {
      "epoch": 0.8601160630620264,
      "grad_norm": 720.0,
      "learning_rate": 7.763157894736843e-08,
      "logits/chosen": -0.64955735206604,
      "logits/rejected": -0.6260396838188171,
      "logps/chosen": -995.9995727539062,
      "logps/rejected": -793.5276489257812,
      "loss": 0.7046,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.024645425379276276,
      "rewards/margins": -0.0216948501765728,
      "rewards/rejected": -0.0029505728743970394,
      "step": 3270
    },
    {
      "epoch": 0.8603790954972135,
      "grad_norm": 472.0,
      "learning_rate": 7.748538011695906e-08,
      "logits/chosen": -0.6627300977706909,
      "logits/rejected": -0.669899046421051,
      "logps/chosen": -1100.4827880859375,
      "logps/rejected": -847.4136962890625,
      "loss": 0.6991,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.006078529171645641,
      "rewards/margins": -0.009894752874970436,
      "rewards/rejected": 0.00381622277200222,
      "step": 3271
    },
    {
      "epoch": 0.8606421279324007,
      "grad_norm": 524.0,
      "learning_rate": 7.733918128654971e-08,
      "logits/chosen": -0.6378411054611206,
      "logits/rejected": -0.6365580558776855,
      "logps/chosen": -1269.43701171875,
      "logps/rejected": -1257.8502197265625,
      "loss": 0.6983,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0074903471395373344,
      "rewards/margins": -0.009580993093550205,
      "rewards/rejected": 0.00209064525552094,
      "step": 3272
    },
    {
      "epoch": 0.8609051603675878,
      "grad_norm": 476.0,
      "learning_rate": 7.719298245614035e-08,
      "logits/chosen": -0.6598923206329346,
      "logits/rejected": -0.671076238155365,
      "logps/chosen": -769.6491088867188,
      "logps/rejected": -836.1409912109375,
      "loss": 0.6797,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.016742896288633347,
      "rewards/margins": 0.027881624177098274,
      "rewards/rejected": -0.011138725094497204,
      "step": 3273
    },
    {
      "epoch": 0.861168192802775,
      "grad_norm": 478.0,
      "learning_rate": 7.704678362573099e-08,
      "logits/chosen": -0.658406674861908,
      "logits/rejected": -0.667233407497406,
      "logps/chosen": -1156.212158203125,
      "logps/rejected": -786.0572509765625,
      "loss": 0.6964,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.02029419131577015,
      "rewards/margins": -0.0061096190474927425,
      "rewards/rejected": -0.01418457180261612,
      "step": 3274
    },
    {
      "epoch": 0.8614312252379621,
      "grad_norm": 532.0,
      "learning_rate": 7.690058479532163e-08,
      "logits/chosen": -0.6453378796577454,
      "logits/rejected": -0.6482320427894592,
      "logps/chosen": -1125.201904296875,
      "logps/rejected": -919.5277099609375,
      "loss": 0.6911,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.011185646057128906,
      "rewards/margins": 0.004293347243219614,
      "rewards/rejected": 0.0068922992795705795,
      "step": 3275
    },
    {
      "epoch": 0.8616942576731493,
      "grad_norm": 446.0,
      "learning_rate": 7.675438596491228e-08,
      "logits/chosen": -0.6599135398864746,
      "logits/rejected": -0.6489640474319458,
      "logps/chosen": -1188.2568359375,
      "logps/rejected": -913.2368774414062,
      "loss": 0.6801,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0128326416015625,
      "rewards/margins": 0.02707653120160103,
      "rewards/rejected": -0.014243889600038528,
      "step": 3276
    },
    {
      "epoch": 0.8619572901083364,
      "grad_norm": 600.0,
      "learning_rate": 7.660818713450293e-08,
      "logits/chosen": -0.6269041299819946,
      "logits/rejected": -0.652098536491394,
      "logps/chosen": -1127.419677734375,
      "logps/rejected": -872.3948974609375,
      "loss": 0.6971,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.007996559143066406,
      "rewards/margins": -0.007401943672448397,
      "rewards/rejected": -0.0005946164019405842,
      "step": 3277
    },
    {
      "epoch": 0.8622203225435237,
      "grad_norm": 398.0,
      "learning_rate": 7.646198830409356e-08,
      "logits/chosen": -0.6370671987533569,
      "logits/rejected": -0.6406158804893494,
      "logps/chosen": -751.1280517578125,
      "logps/rejected": -526.74169921875,
      "loss": 0.6811,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.017989348620176315,
      "rewards/margins": 0.02468118816614151,
      "rewards/rejected": -0.0066918376833200455,
      "step": 3278
    },
    {
      "epoch": 0.8624833549787108,
      "grad_norm": 592.0,
      "learning_rate": 7.63157894736842e-08,
      "logits/chosen": -0.659761369228363,
      "logits/rejected": -0.6586990356445312,
      "logps/chosen": -1069.2847900390625,
      "logps/rejected": -652.7595825195312,
      "loss": 0.6955,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.004741573706269264,
      "rewards/margins": -0.0042244913056492805,
      "rewards/rejected": -0.0005170824006199837,
      "step": 3279
    },
    {
      "epoch": 0.862746387413898,
      "grad_norm": 576.0,
      "learning_rate": 7.616959064327485e-08,
      "logits/chosen": -0.6585174798965454,
      "logits/rejected": -0.6666780114173889,
      "logps/chosen": -735.5028076171875,
      "logps/rejected": -846.6329345703125,
      "loss": 0.6928,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.005530738737434149,
      "rewards/margins": 0.0011217121500521898,
      "rewards/rejected": -0.006652450188994408,
      "step": 3280
    },
    {
      "epoch": 0.8630094198490852,
      "grad_norm": 544.0,
      "learning_rate": 7.602339181286549e-08,
      "logits/chosen": -0.6450285911560059,
      "logits/rejected": -0.6513838768005371,
      "logps/chosen": -1082.934326171875,
      "logps/rejected": -1043.83642578125,
      "loss": 0.6925,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008806515485048294,
      "rewards/margins": 0.001932429731823504,
      "rewards/rejected": 0.006874084006994963,
      "step": 3281
    },
    {
      "epoch": 0.8632724522842723,
      "grad_norm": 500.0,
      "learning_rate": 7.587719298245613e-08,
      "logits/chosen": -0.6284255385398865,
      "logits/rejected": -0.6298770308494568,
      "logps/chosen": -989.906494140625,
      "logps/rejected": -472.905517578125,
      "loss": 0.6932,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0028237346559762955,
      "rewards/margins": 0.00041227368637919426,
      "rewards/rejected": -0.0032360078766942024,
      "step": 3282
    },
    {
      "epoch": 0.8635354847194595,
      "grad_norm": 314.0,
      "learning_rate": 7.573099415204678e-08,
      "logits/chosen": -0.6420348286628723,
      "logits/rejected": -0.6443520784378052,
      "logps/chosen": -590.810302734375,
      "logps/rejected": -538.568359375,
      "loss": 0.6948,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.01972350850701332,
      "rewards/margins": -0.0030495640821754932,
      "rewards/rejected": 0.022773077711462975,
      "step": 3283
    },
    {
      "epoch": 0.8637985171546466,
      "grad_norm": 532.0,
      "learning_rate": 7.558479532163742e-08,
      "logits/chosen": -0.6497179269790649,
      "logits/rejected": -0.6508959531784058,
      "logps/chosen": -1076.2578125,
      "logps/rejected": -912.958740234375,
      "loss": 0.6938,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.013932084664702415,
      "rewards/margins": -0.00010762270539999008,
      "rewards/rejected": 0.014039706438779831,
      "step": 3284
    },
    {
      "epoch": 0.8640615495898338,
      "grad_norm": 506.0,
      "learning_rate": 7.543859649122807e-08,
      "logits/chosen": -0.6543749570846558,
      "logits/rejected": -0.660913348197937,
      "logps/chosen": -1213.541259765625,
      "logps/rejected": -1030.0521240234375,
      "loss": 0.6929,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.005094052758067846,
      "rewards/margins": 0.0007834434509277344,
      "rewards/rejected": 0.004310608841478825,
      "step": 3285
    },
    {
      "epoch": 0.8643245820250209,
      "grad_norm": 716.0,
      "learning_rate": 7.529239766081872e-08,
      "logits/chosen": -0.6344072818756104,
      "logits/rejected": -0.6408343315124512,
      "logps/chosen": -1268.7685546875,
      "logps/rejected": -1021.3092041015625,
      "loss": 0.7135,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.013287162408232689,
      "rewards/margins": -0.039223577827215195,
      "rewards/rejected": 0.025936413556337357,
      "step": 3286
    },
    {
      "epoch": 0.8645876144602082,
      "grad_norm": 548.0,
      "learning_rate": 7.514619883040936e-08,
      "logits/chosen": -0.6340353488922119,
      "logits/rejected": -0.6384716033935547,
      "logps/chosen": -1072.9052734375,
      "logps/rejected": -811.98046875,
      "loss": 0.6927,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.008032036013901234,
      "rewards/margins": 0.001496695913374424,
      "rewards/rejected": -0.009528732858598232,
      "step": 3287
    },
    {
      "epoch": 0.8648506468953953,
      "grad_norm": 506.0,
      "learning_rate": 7.5e-08,
      "logits/chosen": -0.6605416536331177,
      "logits/rejected": -0.6586287021636963,
      "logps/chosen": -1052.60986328125,
      "logps/rejected": -774.4370727539062,
      "loss": 0.7076,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.009255267679691315,
      "rewards/margins": -0.027408838272094727,
      "rewards/rejected": 0.01815357245504856,
      "step": 3288
    },
    {
      "epoch": 0.8651136793305825,
      "grad_norm": 624.0,
      "learning_rate": 7.485380116959064e-08,
      "logits/chosen": -0.6762297749519348,
      "logits/rejected": NaN,
      "logps/chosen": -1299.098876953125,
      "logps/rejected": -868.644287109375,
      "loss": 0.6995,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.00936050433665514,
      "rewards/margins": -0.0123443603515625,
      "rewards/rejected": 0.021704865619540215,
      "step": 3289
    },
    {
      "epoch": 0.8653767117657696,
      "grad_norm": 544.0,
      "learning_rate": 7.470760233918129e-08,
      "logits/chosen": -0.6361531615257263,
      "logits/rejected": -0.643949031829834,
      "logps/chosen": -1188.8294677734375,
      "logps/rejected": -819.0469970703125,
      "loss": 0.7038,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.010104657150804996,
      "rewards/margins": -0.020136738196015358,
      "rewards/rejected": 0.010032081045210361,
      "step": 3290
    },
    {
      "epoch": 0.8656397442009568,
      "grad_norm": 572.0,
      "learning_rate": 7.456140350877192e-08,
      "logits/chosen": -0.6475133299827576,
      "logits/rejected": -0.6587579250335693,
      "logps/chosen": -1297.3360595703125,
      "logps/rejected": -923.0369873046875,
      "loss": 0.6866,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0022972102742642164,
      "rewards/margins": 0.014293957501649857,
      "rewards/rejected": -0.011996746994554996,
      "step": 3291
    },
    {
      "epoch": 0.8659027766361439,
      "grad_norm": 576.0,
      "learning_rate": 7.441520467836257e-08,
      "logits/chosen": -0.6786836981773376,
      "logits/rejected": -0.672932505607605,
      "logps/chosen": -1102.0106201171875,
      "logps/rejected": -968.7608032226562,
      "loss": 0.6933,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00945138931274414,
      "rewards/margins": 0.00047264061868190765,
      "rewards/rejected": -0.009924029931426048,
      "step": 3292
    },
    {
      "epoch": 0.8661658090713311,
      "grad_norm": 536.0,
      "learning_rate": 7.426900584795322e-08,
      "logits/chosen": -0.654382586479187,
      "logits/rejected": -0.641045331954956,
      "logps/chosen": -876.082275390625,
      "logps/rejected": -764.7080688476562,
      "loss": 0.6934,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.005729581229388714,
      "rewards/margins": 1.859758049249649e-05,
      "rewards/rejected": 0.005710983648896217,
      "step": 3293
    },
    {
      "epoch": 0.8664288415065182,
      "grad_norm": 480.0,
      "learning_rate": 7.412280701754385e-08,
      "logits/chosen": -0.6267431378364563,
      "logits/rejected": -0.6191626787185669,
      "logps/chosen": -663.6785278320312,
      "logps/rejected": -694.9309692382812,
      "loss": 0.6899,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00787277240306139,
      "rewards/margins": 0.007191753946244717,
      "rewards/rejected": -0.015064526349306107,
      "step": 3294
    },
    {
      "epoch": 0.8666918739417054,
      "grad_norm": 580.0,
      "learning_rate": 7.39766081871345e-08,
      "logits/chosen": -0.6639124751091003,
      "logits/rejected": -0.6605195999145508,
      "logps/chosen": -1001.0604858398438,
      "logps/rejected": -812.028076171875,
      "loss": 0.6868,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.003547096624970436,
      "rewards/margins": 0.013690091669559479,
      "rewards/rejected": -0.010142994113266468,
      "step": 3295
    },
    {
      "epoch": 0.8669549063768927,
      "grad_norm": 532.0,
      "learning_rate": 7.383040935672514e-08,
      "logits/chosen": -0.6565116047859192,
      "logits/rejected": -0.6563141345977783,
      "logps/chosen": -1120.2906494140625,
      "logps/rejected": -733.1324462890625,
      "loss": 0.6941,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010544445365667343,
      "rewards/margins": -0.0014967446913942695,
      "rewards/rejected": 0.012041186913847923,
      "step": 3296
    },
    {
      "epoch": 0.8672179388120798,
      "grad_norm": 596.0,
      "learning_rate": 7.368421052631577e-08,
      "logits/chosen": -0.6624851822853088,
      "logits/rejected": -0.6539595723152161,
      "logps/chosen": -1361.165283203125,
      "logps/rejected": -1288.0941162109375,
      "loss": 0.6972,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0009618748445063829,
      "rewards/margins": -0.0059892646968364716,
      "rewards/rejected": 0.005027389153838158,
      "step": 3297
    },
    {
      "epoch": 0.867480971247267,
      "grad_norm": 544.0,
      "learning_rate": 7.353801169590642e-08,
      "logits/chosen": -0.6556301116943359,
      "logits/rejected": -0.6600915193557739,
      "logps/chosen": -1333.813720703125,
      "logps/rejected": -933.5856323242188,
      "loss": 0.6933,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00595540925860405,
      "rewards/margins": 0.0005091670900583267,
      "rewards/rejected": 0.005446243565529585,
      "step": 3298
    },
    {
      "epoch": 0.8677440036824541,
      "grad_norm": 696.0,
      "learning_rate": 7.339181286549708e-08,
      "logits/chosen": -0.6582173705101013,
      "logits/rejected": -0.6646440625190735,
      "logps/chosen": -1406.7999267578125,
      "logps/rejected": -1232.02099609375,
      "loss": 0.6984,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005128670018166304,
      "rewards/margins": -0.009297657757997513,
      "rewards/rejected": 0.01442632731050253,
      "step": 3299
    },
    {
      "epoch": 0.8680070361176413,
      "grad_norm": 636.0,
      "learning_rate": 7.324561403508773e-08,
      "logits/chosen": -0.6479684114456177,
      "logits/rejected": -0.6450048685073853,
      "logps/chosen": -922.08642578125,
      "logps/rejected": -760.0916748046875,
      "loss": 0.6937,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0012054448015987873,
      "rewards/margins": -0.0005072602070868015,
      "rewards/rejected": -0.0006981850019656122,
      "step": 3300
    },
    {
      "epoch": 0.8682700685528284,
      "grad_norm": 506.0,
      "learning_rate": 7.309941520467836e-08,
      "logits/chosen": -0.6394906044006348,
      "logits/rejected": -0.6542608141899109,
      "logps/chosen": -991.427734375,
      "logps/rejected": -750.2321166992188,
      "loss": 0.6928,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.003981113899499178,
      "rewards/margins": 0.001583194825798273,
      "rewards/rejected": -0.005564308259636164,
      "step": 3301
    },
    {
      "epoch": 0.8685331009880156,
      "grad_norm": 544.0,
      "learning_rate": 7.2953216374269e-08,
      "logits/chosen": -0.660277783870697,
      "logits/rejected": -0.6689329147338867,
      "logps/chosen": -1275.43603515625,
      "logps/rejected": -1186.8045654296875,
      "loss": 0.6831,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02330799028277397,
      "rewards/margins": 0.020740699023008347,
      "rewards/rejected": 0.002567291259765625,
      "step": 3302
    },
    {
      "epoch": 0.8687961334232027,
      "grad_norm": 668.0,
      "learning_rate": 7.280701754385965e-08,
      "logits/chosen": -0.6245625019073486,
      "logits/rejected": -0.6430758237838745,
      "logps/chosen": -1019.095947265625,
      "logps/rejected": -730.6171264648438,
      "loss": 0.711,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.005514050368219614,
      "rewards/margins": -0.03410053253173828,
      "rewards/rejected": 0.028586482629179955,
      "step": 3303
    },
    {
      "epoch": 0.8690591658583899,
      "grad_norm": 612.0,
      "learning_rate": 7.266081871345029e-08,
      "logits/chosen": -0.6467731595039368,
      "logits/rejected": -0.6456674933433533,
      "logps/chosen": -1186.7843017578125,
      "logps/rejected": -1004.819580078125,
      "loss": 0.6987,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.012545301578938961,
      "rewards/margins": -0.01016778964549303,
      "rewards/rejected": -0.0023775105364620686,
      "step": 3304
    },
    {
      "epoch": 0.869322198293577,
      "grad_norm": 572.0,
      "learning_rate": 7.251461988304093e-08,
      "logits/chosen": -0.6582485437393188,
      "logits/rejected": -0.6476956009864807,
      "logps/chosen": -1241.5284423828125,
      "logps/rejected": -818.299072265625,
      "loss": 0.6783,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.020972348749637604,
      "rewards/margins": 0.03017578274011612,
      "rewards/rejected": -0.009203433059155941,
      "step": 3305
    },
    {
      "epoch": 0.8695852307287643,
      "grad_norm": 552.0,
      "learning_rate": 7.236842105263158e-08,
      "logits/chosen": -0.6773831248283386,
      "logits/rejected": -0.6791634559631348,
      "logps/chosen": -1004.8665771484375,
      "logps/rejected": -1061.919921875,
      "loss": 0.6919,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008483124896883965,
      "rewards/margins": 0.003317928407341242,
      "rewards/rejected": 0.005165196489542723,
      "step": 3306
    },
    {
      "epoch": 0.8698482631639514,
      "grad_norm": 504.0,
      "learning_rate": 7.222222222222221e-08,
      "logits/chosen": -0.6559242606163025,
      "logits/rejected": -0.6549919247627258,
      "logps/chosen": -1377.62744140625,
      "logps/rejected": -1057.73974609375,
      "loss": 0.6962,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003231524955481291,
      "rewards/margins": -0.00443272665143013,
      "rewards/rejected": 0.007664251606911421,
      "step": 3307
    },
    {
      "epoch": 0.8701112955991386,
      "grad_norm": 516.0,
      "learning_rate": 7.207602339181286e-08,
      "logits/chosen": -0.6725159287452698,
      "logits/rejected": -0.6699955463409424,
      "logps/chosen": -1315.7645263671875,
      "logps/rejected": -967.8173217773438,
      "loss": 0.6837,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0024604795034974813,
      "rewards/margins": 0.02020101621747017,
      "rewards/rejected": -0.017740538343787193,
      "step": 3308
    },
    {
      "epoch": 0.8703743280343257,
      "grad_norm": 612.0,
      "learning_rate": 7.19298245614035e-08,
      "logits/chosen": -0.6469012498855591,
      "logits/rejected": -0.6592949032783508,
      "logps/chosen": -1407.5859375,
      "logps/rejected": -879.0189819335938,
      "loss": 0.6933,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.008965492248535156,
      "rewards/margins": 0.0004056922625750303,
      "rewards/rejected": -0.009371185675263405,
      "step": 3309
    },
    {
      "epoch": 0.8706373604695129,
      "grad_norm": 506.0,
      "learning_rate": 7.178362573099415e-08,
      "logits/chosen": -0.6555987000465393,
      "logits/rejected": -0.6481788158416748,
      "logps/chosen": -1442.661376953125,
      "logps/rejected": -1193.031005859375,
      "loss": 0.6908,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.004664612002670765,
      "rewards/margins": 0.005277441814541817,
      "rewards/rejected": -0.0006128313252702355,
      "step": 3310
    },
    {
      "epoch": 0.8709003929047001,
      "grad_norm": 572.0,
      "learning_rate": 7.163742690058478e-08,
      "logits/chosen": -0.6808643341064453,
      "logits/rejected": -0.6783626675605774,
      "logps/chosen": -1103.548583984375,
      "logps/rejected": -928.8287353515625,
      "loss": 0.6912,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01259922981262207,
      "rewards/margins": 0.004801703151315451,
      "rewards/rejected": -0.01740093342959881,
      "step": 3311
    },
    {
      "epoch": 0.8711634253398872,
      "grad_norm": 608.0,
      "learning_rate": 7.149122807017543e-08,
      "logits/chosen": -0.6449525356292725,
      "logits/rejected": NaN,
      "logps/chosen": -1386.9615478515625,
      "logps/rejected": -1110.485107421875,
      "loss": 0.6963,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0017875682096928358,
      "rewards/margins": -0.0056434632278978825,
      "rewards/rejected": 0.007431030739098787,
      "step": 3312
    },
    {
      "epoch": 0.8714264577750744,
      "grad_norm": 468.0,
      "learning_rate": 7.134502923976608e-08,
      "logits/chosen": -0.6447348594665527,
      "logits/rejected": -0.6519539952278137,
      "logps/chosen": -1086.9970703125,
      "logps/rejected": -732.6560668945312,
      "loss": 0.6908,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0017361636273562908,
      "rewards/margins": 0.005409908015280962,
      "rewards/rejected": -0.00714607210829854,
      "step": 3313
    },
    {
      "epoch": 0.8716894902102615,
      "grad_norm": 510.0,
      "learning_rate": 7.119883040935672e-08,
      "logits/chosen": -0.6231545805931091,
      "logits/rejected": -0.6319749355316162,
      "logps/chosen": -979.8604125976562,
      "logps/rejected": -731.2313842773438,
      "loss": 0.7023,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.003822517581284046,
      "rewards/margins": -0.01683826372027397,
      "rewards/rejected": 0.02066078409552574,
      "step": 3314
    },
    {
      "epoch": 0.8719525226454488,
      "grad_norm": 418.0,
      "learning_rate": 7.105263157894737e-08,
      "logits/chosen": -0.6396182179450989,
      "logits/rejected": -0.6424137353897095,
      "logps/chosen": -873.4619140625,
      "logps/rejected": -752.3609619140625,
      "loss": 0.6916,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.001724052825011313,
      "rewards/margins": 0.0038292882964015007,
      "rewards/rejected": -0.0055533419363200665,
      "step": 3315
    },
    {
      "epoch": 0.8722155550806359,
      "grad_norm": 512.0,
      "learning_rate": 7.090643274853802e-08,
      "logits/chosen": -0.6706371903419495,
      "logits/rejected": -0.6814874410629272,
      "logps/chosen": -864.208251953125,
      "logps/rejected": -840.4650268554688,
      "loss": 0.7001,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.02318720705807209,
      "rewards/margins": -0.013213586062192917,
      "rewards/rejected": 0.036400794982910156,
      "step": 3316
    },
    {
      "epoch": 0.8724785875158231,
      "grad_norm": 484.0,
      "learning_rate": 7.076023391812865e-08,
      "logits/chosen": -0.6520350575447083,
      "logits/rejected": -0.6496422290802002,
      "logps/chosen": -865.1934814453125,
      "logps/rejected": -619.2722778320312,
      "loss": 0.688,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01616806909441948,
      "rewards/margins": 0.011342192068696022,
      "rewards/rejected": 0.004825877957046032,
      "step": 3317
    },
    {
      "epoch": 0.8727416199510102,
      "grad_norm": 468.0,
      "learning_rate": 7.06140350877193e-08,
      "logits/chosen": -0.650132417678833,
      "logits/rejected": -0.6651622653007507,
      "logps/chosen": -829.9290161132812,
      "logps/rejected": -698.1661376953125,
      "loss": 0.6915,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0003759388346225023,
      "rewards/margins": 0.004034757148474455,
      "rewards/rejected": -0.0036588204093277454,
      "step": 3318
    },
    {
      "epoch": 0.8730046523861974,
      "grad_norm": 436.0,
      "learning_rate": 7.046783625730994e-08,
      "logits/chosen": -0.6581771373748779,
      "logits/rejected": -0.6524503231048584,
      "logps/chosen": -956.50048828125,
      "logps/rejected": -894.1967163085938,
      "loss": 0.6943,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.00044011976569890976,
      "rewards/margins": -0.00169286725576967,
      "rewards/rejected": 0.0012527465587481856,
      "step": 3319
    },
    {
      "epoch": 0.8732676848213845,
      "grad_norm": 900.0,
      "learning_rate": 7.032163742690059e-08,
      "logits/chosen": -0.6486842632293701,
      "logits/rejected": -0.64107346534729,
      "logps/chosen": -1157.8626708984375,
      "logps/rejected": -1053.545654296875,
      "loss": 0.7102,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.01925487630069256,
      "rewards/margins": -0.03334865719079971,
      "rewards/rejected": 0.014093780890107155,
      "step": 3320
    },
    {
      "epoch": 0.8735307172565717,
      "grad_norm": 552.0,
      "learning_rate": 7.017543859649122e-08,
      "logits/chosen": -0.6630244255065918,
      "logits/rejected": -0.6432112455368042,
      "logps/chosen": -1273.2044677734375,
      "logps/rejected": -889.2352905273438,
      "loss": 0.6902,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0005239476449787617,
      "rewards/margins": 0.006487464532256126,
      "rewards/rejected": -0.007011414505541325,
      "step": 3321
    },
    {
      "epoch": 0.8737937496917588,
      "grad_norm": 564.0,
      "learning_rate": 7.002923976608187e-08,
      "logits/chosen": -0.6349449157714844,
      "logits/rejected": NaN,
      "logps/chosen": -1097.4622802734375,
      "logps/rejected": -1028.759033203125,
      "loss": 0.679,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.028394319117069244,
      "rewards/margins": 0.029569054022431374,
      "rewards/rejected": -0.0011747351381927729,
      "step": 3322
    },
    {
      "epoch": 0.874056782126946,
      "grad_norm": 552.0,
      "learning_rate": 6.988304093567251e-08,
      "logits/chosen": -0.6551691293716431,
      "logits/rejected": -0.6623196601867676,
      "logps/chosen": -1389.1956787109375,
      "logps/rejected": -1133.3194580078125,
      "loss": 0.7028,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.01500844955444336,
      "rewards/margins": -0.018818285316228867,
      "rewards/rejected": 0.003809833200648427,
      "step": 3323
    },
    {
      "epoch": 0.8743198145621331,
      "grad_norm": 640.0,
      "learning_rate": 6.973684210526315e-08,
      "logits/chosen": -0.6510536670684814,
      "logits/rejected": -0.6575330495834351,
      "logps/chosen": -1280.75537109375,
      "logps/rejected": -859.8114013671875,
      "loss": 0.6951,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0018513684626668692,
      "rewards/margins": -0.0030152322724461555,
      "rewards/rejected": 0.0011638645082712173,
      "step": 3324
    },
    {
      "epoch": 0.8745828469973204,
      "grad_norm": 556.0,
      "learning_rate": 6.95906432748538e-08,
      "logits/chosen": -0.6426869630813599,
      "logits/rejected": NaN,
      "logps/chosen": -1405.4752197265625,
      "logps/rejected": -650.5057373046875,
      "loss": 0.6946,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0050802226178348064,
      "rewards/margins": -0.0018208511173725128,
      "rewards/rejected": 0.006901074200868607,
      "step": 3325
    },
    {
      "epoch": 0.8748458794325075,
      "grad_norm": 560.0,
      "learning_rate": 6.944444444444444e-08,
      "logits/chosen": -0.6579841375350952,
      "logits/rejected": -0.6602885127067566,
      "logps/chosen": -1177.374267578125,
      "logps/rejected": -1009.621826171875,
      "loss": 0.6794,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.029500866308808327,
      "rewards/margins": 0.029352858662605286,
      "rewards/rejected": 0.00014800997450947762,
      "step": 3326
    },
    {
      "epoch": 0.8751089118676947,
      "grad_norm": 470.0,
      "learning_rate": 6.929824561403507e-08,
      "logits/chosen": -0.6357713341712952,
      "logits/rejected": -0.6387973427772522,
      "logps/chosen": -1040.64013671875,
      "logps/rejected": -706.4808959960938,
      "loss": 0.6883,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0022647855803370476,
      "rewards/margins": 0.010563088580965996,
      "rewards/rejected": -0.012827873229980469,
      "step": 3327
    },
    {
      "epoch": 0.8753719443028819,
      "grad_norm": 528.0,
      "learning_rate": 6.915204678362572e-08,
      "logits/chosen": -0.6425808072090149,
      "logits/rejected": -0.6465873718261719,
      "logps/chosen": -1096.2059326171875,
      "logps/rejected": -1051.3919677734375,
      "loss": 0.689,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.019069766625761986,
      "rewards/margins": 0.009790897369384766,
      "rewards/rejected": 0.00927886925637722,
      "step": 3328
    },
    {
      "epoch": 0.875634976738069,
      "grad_norm": 374.0,
      "learning_rate": 6.900584795321638e-08,
      "logits/chosen": -0.6586532592773438,
      "logits/rejected": -0.6638520359992981,
      "logps/chosen": -931.0596313476562,
      "logps/rejected": -746.3638916015625,
      "loss": 0.6936,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.009158611297607422,
      "rewards/margins": -0.0009346962906420231,
      "rewards/rejected": -0.008223914541304111,
      "step": 3329
    },
    {
      "epoch": 0.8758980091732562,
      "grad_norm": 496.0,
      "learning_rate": 6.885964912280703e-08,
      "logits/chosen": -0.644572377204895,
      "logits/rejected": -0.6507918834686279,
      "logps/chosen": -854.765380859375,
      "logps/rejected": -723.91357421875,
      "loss": 0.6812,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.031231215223670006,
      "rewards/margins": 0.025348855182528496,
      "rewards/rejected": 0.005882358178496361,
      "step": 3330
    },
    {
      "epoch": 0.8761610416084433,
      "grad_norm": 478.0,
      "learning_rate": 6.871345029239766e-08,
      "logits/chosen": -0.6536635160446167,
      "logits/rejected": -0.6646372079849243,
      "logps/chosen": -1221.7745361328125,
      "logps/rejected": -1039.1689453125,
      "loss": 0.685,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010815620422363281,
      "rewards/margins": 0.016930200159549713,
      "rewards/rejected": -0.006114579271525145,
      "step": 3331
    },
    {
      "epoch": 0.8764240740436305,
      "grad_norm": 470.0,
      "learning_rate": 6.85672514619883e-08,
      "logits/chosen": -0.6884351968765259,
      "logits/rejected": -0.6993507742881775,
      "logps/chosen": -967.7879028320312,
      "logps/rejected": -933.1483154296875,
      "loss": 0.698,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.005880833603441715,
      "rewards/margins": -0.009079314768314362,
      "rewards/rejected": 0.0031984804663807154,
      "step": 3332
    },
    {
      "epoch": 0.8766871064788176,
      "grad_norm": 620.0,
      "learning_rate": 6.842105263157895e-08,
      "logits/chosen": -0.6499440670013428,
      "logits/rejected": -0.6491253972053528,
      "logps/chosen": -1064.53173828125,
      "logps/rejected": -717.68701171875,
      "loss": 0.6834,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.012990093789994717,
      "rewards/margins": 0.020641708746552467,
      "rewards/rejected": -0.007651615422219038,
      "step": 3333
    },
    {
      "epoch": 0.8769501389140049,
      "grad_norm": 624.0,
      "learning_rate": 6.827485380116958e-08,
      "logits/chosen": -0.6549053192138672,
      "logits/rejected": -0.6571095585823059,
      "logps/chosen": -1390.1328125,
      "logps/rejected": -818.8182373046875,
      "loss": 0.7112,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0029862397350370884,
      "rewards/margins": -0.03470306470990181,
      "rewards/rejected": 0.03171682357788086,
      "step": 3334
    },
    {
      "epoch": 0.877213171349192,
      "grad_norm": 632.0,
      "learning_rate": 6.812865497076023e-08,
      "logits/chosen": -0.6469749808311462,
      "logits/rejected": -0.6471234560012817,
      "logps/chosen": -1303.443359375,
      "logps/rejected": -1072.3309326171875,
      "loss": 0.6964,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.011981011368334293,
      "rewards/margins": -0.005528259556740522,
      "rewards/rejected": 0.01750926859676838,
      "step": 3335
    },
    {
      "epoch": 0.8774762037843792,
      "grad_norm": 486.0,
      "learning_rate": 6.798245614035088e-08,
      "logits/chosen": -0.6621634364128113,
      "logits/rejected": -0.6766715049743652,
      "logps/chosen": -1025.89404296875,
      "logps/rejected": -788.89599609375,
      "loss": 0.6989,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.011002302169799805,
      "rewards/margins": -0.010875081643462181,
      "rewards/rejected": -0.0001272200606763363,
      "step": 3336
    },
    {
      "epoch": 0.8777392362195663,
      "grad_norm": 1072.0,
      "learning_rate": 6.783625730994151e-08,
      "logits/chosen": -0.6601682901382446,
      "logits/rejected": -0.6665721535682678,
      "logps/chosen": -1419.782470703125,
      "logps/rejected": -1280.409912109375,
      "loss": 0.7037,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.009871672838926315,
      "rewards/margins": -0.020154759287834167,
      "rewards/rejected": 0.010283088311553001,
      "step": 3337
    },
    {
      "epoch": 0.8780022686547535,
      "grad_norm": 470.0,
      "learning_rate": 6.769005847953216e-08,
      "logits/chosen": -0.6837950348854065,
      "logits/rejected": NaN,
      "logps/chosen": -1299.1602783203125,
      "logps/rejected": -860.30419921875,
      "loss": 0.6869,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.012578008696436882,
      "rewards/margins": 0.01291818730533123,
      "rewards/rejected": -0.00034017558209598064,
      "step": 3338
    },
    {
      "epoch": 0.8782653010899406,
      "grad_norm": 486.0,
      "learning_rate": 6.75438596491228e-08,
      "logits/chosen": -0.6568670272827148,
      "logits/rejected": -0.6488601565361023,
      "logps/chosen": -912.5657958984375,
      "logps/rejected": -841.9625854492188,
      "loss": 0.6901,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.011412430554628372,
      "rewards/margins": 0.007235717494040728,
      "rewards/rejected": 0.004176713526248932,
      "step": 3339
    },
    {
      "epoch": 0.8785283335251278,
      "grad_norm": 880.0,
      "learning_rate": 6.739766081871345e-08,
      "logits/chosen": -0.6423778533935547,
      "logits/rejected": NaN,
      "logps/chosen": -988.7031860351562,
      "logps/rejected": -808.810302734375,
      "loss": 0.6925,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012179471552371979,
      "rewards/margins": 0.0016954420134425163,
      "rewards/rejected": 0.010484028607606888,
      "step": 3340
    },
    {
      "epoch": 0.8787913659603149,
      "grad_norm": 540.0,
      "learning_rate": 6.725146198830408e-08,
      "logits/chosen": -0.6471083164215088,
      "logits/rejected": -0.649207353591919,
      "logps/chosen": -1125.2486572265625,
      "logps/rejected": -730.363037109375,
      "loss": 0.6958,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.013697433285415173,
      "rewards/margins": -0.003898859489709139,
      "rewards/rejected": -0.009798574261367321,
      "step": 3341
    },
    {
      "epoch": 0.8790543983955021,
      "grad_norm": 576.0,
      "learning_rate": 6.710526315789473e-08,
      "logits/chosen": -0.6306007504463196,
      "logits/rejected": -0.6378833055496216,
      "logps/chosen": -1213.6202392578125,
      "logps/rejected": -724.6646728515625,
      "loss": 0.6814,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.009320499375462532,
      "rewards/margins": 0.025113727897405624,
      "rewards/rejected": -0.015793228521943092,
      "step": 3342
    },
    {
      "epoch": 0.8793174308306894,
      "grad_norm": 460.0,
      "learning_rate": 6.695906432748538e-08,
      "logits/chosen": -0.6439581513404846,
      "logits/rejected": -0.6372848153114319,
      "logps/chosen": -878.7255249023438,
      "logps/rejected": -964.8104248046875,
      "loss": 0.6999,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.02171630784869194,
      "rewards/margins": -0.013082265853881836,
      "rewards/rejected": -0.008634041994810104,
      "step": 3343
    },
    {
      "epoch": 0.8795804632658765,
      "grad_norm": 596.0,
      "learning_rate": 6.681286549707602e-08,
      "logits/chosen": -0.6558066010475159,
      "logits/rejected": -0.652820348739624,
      "logps/chosen": -1235.007080078125,
      "logps/rejected": -1101.0306396484375,
      "loss": 0.6824,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0039215087890625,
      "rewards/margins": 0.021963072940707207,
      "rewards/rejected": -0.025884581729769707,
      "step": 3344
    },
    {
      "epoch": 0.8798434957010637,
      "grad_norm": 498.0,
      "learning_rate": 6.666666666666667e-08,
      "logits/chosen": -0.643157958984375,
      "logits/rejected": -0.657748818397522,
      "logps/chosen": -1023.181396484375,
      "logps/rejected": -661.638427734375,
      "loss": 0.6966,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.007722283247858286,
      "rewards/margins": -0.006667709443718195,
      "rewards/rejected": -0.0010545728728175163,
      "step": 3345
    },
    {
      "epoch": 0.8801065281362508,
      "grad_norm": 580.0,
      "learning_rate": 6.652046783625731e-08,
      "logits/chosen": -0.6430746912956238,
      "logits/rejected": -0.6501944065093994,
      "logps/chosen": -1486.3641357421875,
      "logps/rejected": -946.2194213867188,
      "loss": 0.7006,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.012693215161561966,
      "rewards/margins": -0.014148903079330921,
      "rewards/rejected": 0.026842117309570312,
      "step": 3346
    },
    {
      "epoch": 0.880369560571438,
      "grad_norm": 494.0,
      "learning_rate": 6.637426900584795e-08,
      "logits/chosen": -0.6783816814422607,
      "logits/rejected": NaN,
      "logps/chosen": -1048.434814453125,
      "logps/rejected": -779.072998046875,
      "loss": 0.6758,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.011593629606068134,
      "rewards/margins": 0.035547539591789246,
      "rewards/rejected": -0.023953914642333984,
      "step": 3347
    },
    {
      "epoch": 0.8806325930066251,
      "grad_norm": 540.0,
      "learning_rate": 6.62280701754386e-08,
      "logits/chosen": -0.6236498951911926,
      "logits/rejected": -0.6228928565979004,
      "logps/chosen": -1037.330810546875,
      "logps/rejected": -1086.7513427734375,
      "loss": 0.7056,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.029458332806825638,
      "rewards/margins": -0.02293558418750763,
      "rewards/rejected": -0.006522750947624445,
      "step": 3348
    },
    {
      "epoch": 0.8808956254418123,
      "grad_norm": 474.0,
      "learning_rate": 6.608187134502924e-08,
      "logits/chosen": -0.663070797920227,
      "logits/rejected": -0.6629195213317871,
      "logps/chosen": -992.6729736328125,
      "logps/rejected": -780.7151489257812,
      "loss": 0.6989,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.012313080951571465,
      "rewards/margins": -0.010716536082327366,
      "rewards/rejected": 0.023029614239931107,
      "step": 3349
    },
    {
      "epoch": 0.8811586578769994,
      "grad_norm": 472.0,
      "learning_rate": 6.593567251461989e-08,
      "logits/chosen": -0.6576346755027771,
      "logits/rejected": -0.6748489737510681,
      "logps/chosen": -884.522216796875,
      "logps/rejected": -526.9737548828125,
      "loss": 0.6773,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.03525300323963165,
      "rewards/margins": 0.03289799392223358,
      "rewards/rejected": 0.0023550037294626236,
      "step": 3350
    },
    {
      "epoch": 0.8814216903121866,
      "grad_norm": 604.0,
      "learning_rate": 6.578947368421052e-08,
      "logits/chosen": -0.6649136543273926,
      "logits/rejected": NaN,
      "logps/chosen": -1247.912841796875,
      "logps/rejected": -878.8477783203125,
      "loss": 0.6987,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.018221285194158554,
      "rewards/margins": -0.010152148082852364,
      "rewards/rejected": -0.008069133386015892,
      "step": 3351
    },
    {
      "epoch": 0.8816847227473738,
      "grad_norm": 596.0,
      "learning_rate": 6.564327485380117e-08,
      "logits/chosen": -0.654082715511322,
      "logits/rejected": -0.6601244211196899,
      "logps/chosen": -1347.3203125,
      "logps/rejected": -1100.0784912109375,
      "loss": 0.697,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008539581671357155,
      "rewards/margins": -0.005406570620834827,
      "rewards/rejected": 0.013946153223514557,
      "step": 3352
    },
    {
      "epoch": 0.881947755182561,
      "grad_norm": 588.0,
      "learning_rate": 6.549707602339181e-08,
      "logits/chosen": -0.6602250933647156,
      "logits/rejected": NaN,
      "logps/chosen": -1683.856689453125,
      "logps/rejected": -1089.9549560546875,
      "loss": 0.6859,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.005688285920768976,
      "rewards/margins": 0.015462303534150124,
      "rewards/rejected": -0.009774018079042435,
      "step": 3353
    },
    {
      "epoch": 0.8822107876177481,
      "grad_norm": 552.0,
      "learning_rate": 6.535087719298245e-08,
      "logits/chosen": -0.6753349304199219,
      "logits/rejected": -0.6705410480499268,
      "logps/chosen": -1374.6905517578125,
      "logps/rejected": -1096.6427001953125,
      "loss": 0.6885,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.01842470094561577,
      "rewards/margins": 0.010466478765010834,
      "rewards/rejected": 0.00795822124928236,
      "step": 3354
    },
    {
      "epoch": 0.8824738200529353,
      "grad_norm": 472.0,
      "learning_rate": 6.520467836257309e-08,
      "logits/chosen": -0.6527849435806274,
      "logits/rejected": -0.658403217792511,
      "logps/chosen": -1305.375,
      "logps/rejected": -1169.7872314453125,
      "loss": 0.6925,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.015990352258086205,
      "rewards/margins": 0.00167942070402205,
      "rewards/rejected": 0.014310930855572224,
      "step": 3355
    },
    {
      "epoch": 0.8827368524881224,
      "grad_norm": 640.0,
      "learning_rate": 6.505847953216374e-08,
      "logits/chosen": -0.6627431511878967,
      "logits/rejected": -0.6711357831954956,
      "logps/chosen": -1623.3756103515625,
      "logps/rejected": -1357.5909423828125,
      "loss": 0.7035,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.001583099365234375,
      "rewards/margins": -0.019278809428215027,
      "rewards/rejected": 0.01769571378827095,
      "step": 3356
    },
    {
      "epoch": 0.8829998849233096,
      "grad_norm": 584.0,
      "learning_rate": 6.491228070175437e-08,
      "logits/chosen": -0.6753395199775696,
      "logits/rejected": NaN,
      "logps/chosen": -1138.2169189453125,
      "logps/rejected": -705.9979248046875,
      "loss": 0.6977,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0007600788958370686,
      "rewards/margins": -0.008328677155077457,
      "rewards/rejected": 0.00908875372260809,
      "step": 3357
    },
    {
      "epoch": 0.8832629173584968,
      "grad_norm": 604.0,
      "learning_rate": 6.476608187134502e-08,
      "logits/chosen": -0.6511144042015076,
      "logits/rejected": -0.6563586592674255,
      "logps/chosen": -1322.368896484375,
      "logps/rejected": -841.9169311523438,
      "loss": 0.6989,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.03205757588148117,
      "rewards/margins": -0.009763050824403763,
      "rewards/rejected": -0.02229452133178711,
      "step": 3358
    },
    {
      "epoch": 0.8835259497936839,
      "grad_norm": 480.0,
      "learning_rate": 6.461988304093568e-08,
      "logits/chosen": -0.6134464740753174,
      "logits/rejected": -0.6335148215293884,
      "logps/chosen": -1090.837646484375,
      "logps/rejected": -917.7800903320312,
      "loss": 0.7028,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01629638671875,
      "rewards/margins": -0.01874361000955105,
      "rewards/rejected": 0.0024472237564623356,
      "step": 3359
    },
    {
      "epoch": 0.8837889822288711,
      "grad_norm": 648.0,
      "learning_rate": 6.447368421052632e-08,
      "logits/chosen": -0.644108772277832,
      "logits/rejected": -0.6470839381217957,
      "logps/chosen": -1122.1767578125,
      "logps/rejected": -1185.55419921875,
      "loss": 0.6935,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.017400886863470078,
      "rewards/margins": 0.0005037779919803143,
      "rewards/rejected": -0.01790466345846653,
      "step": 3360
    },
    {
      "epoch": 0.8840520146640582,
      "grad_norm": 628.0,
      "learning_rate": 6.432748538011696e-08,
      "logits/chosen": NaN,
      "logits/rejected": -0.6639124751091003,
      "logps/chosen": -1295.824462890625,
      "logps/rejected": -1261.575927734375,
      "loss": 0.697,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0024531360249966383,
      "rewards/margins": -0.006440544966608286,
      "rewards/rejected": 0.003987406846135855,
      "step": 3361
    },
    {
      "epoch": 0.8843150470992455,
      "grad_norm": 620.0,
      "learning_rate": 6.41812865497076e-08,
      "logits/chosen": -0.6498810052871704,
      "logits/rejected": NaN,
      "logps/chosen": -945.8009643554688,
      "logps/rejected": -703.9736938476562,
      "loss": 0.7061,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.005751180928200483,
      "rewards/margins": -0.025162458419799805,
      "rewards/rejected": 0.019411277025938034,
      "step": 3362
    },
    {
      "epoch": 0.8845780795344326,
      "grad_norm": 552.0,
      "learning_rate": 6.403508771929825e-08,
      "logits/chosen": -0.6527444124221802,
      "logits/rejected": NaN,
      "logps/chosen": -1512.112060546875,
      "logps/rejected": -957.2814331054688,
      "loss": 0.6982,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.0012028696946799755,
      "rewards/margins": -0.009521913714706898,
      "rewards/rejected": 0.010724782943725586,
      "step": 3363
    },
    {
      "epoch": 0.8848411119696198,
      "grad_norm": 580.0,
      "learning_rate": 6.388888888888888e-08,
      "logits/chosen": -0.6586375832557678,
      "logits/rejected": -0.6647230386734009,
      "logps/chosen": -1390.7923583984375,
      "logps/rejected": -1011.0933837890625,
      "loss": 0.6876,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 3.108987584710121e-05,
      "rewards/margins": 0.012259338982403278,
      "rewards/rejected": -0.012228250503540039,
      "step": 3364
    },
    {
      "epoch": 0.8851041444048069,
      "grad_norm": 408.0,
      "learning_rate": 6.374269005847953e-08,
      "logits/chosen": -0.6550005674362183,
      "logits/rejected": -0.6513956785202026,
      "logps/chosen": -805.738037109375,
      "logps/rejected": -704.6289672851562,
      "loss": 0.6853,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005261707119643688,
      "rewards/margins": 0.01627025566995144,
      "rewards/rejected": -0.01100854855030775,
      "step": 3365
    },
    {
      "epoch": 0.8853671768399941,
      "grad_norm": 478.0,
      "learning_rate": 6.359649122807018e-08,
      "logits/chosen": -0.6336343288421631,
      "logits/rejected": -0.6372069716453552,
      "logps/chosen": -947.7276611328125,
      "logps/rejected": -753.9300537109375,
      "loss": 0.6918,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01288623921573162,
      "rewards/margins": 0.003699684515595436,
      "rewards/rejected": 0.00918655563145876,
      "step": 3366
    },
    {
      "epoch": 0.8856302092751812,
      "grad_norm": 494.0,
      "learning_rate": 6.345029239766081e-08,
      "logits/chosen": -0.6299291253089905,
      "logits/rejected": -0.6357690691947937,
      "logps/chosen": -1173.0281982421875,
      "logps/rejected": -911.9971313476562,
      "loss": 0.6892,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.003915023989975452,
      "rewards/margins": 0.008411504328250885,
      "rewards/rejected": -0.012326526455581188,
      "step": 3367
    },
    {
      "epoch": 0.8858932417103684,
      "grad_norm": 536.0,
      "learning_rate": 6.330409356725146e-08,
      "logits/chosen": -0.6573790907859802,
      "logits/rejected": -0.6570665836334229,
      "logps/chosen": -1029.539306640625,
      "logps/rejected": -844.3427734375,
      "loss": 0.6866,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.021638011559844017,
      "rewards/margins": 0.014554694294929504,
      "rewards/rejected": 0.007083320524543524,
      "step": 3368
    },
    {
      "epoch": 0.8861562741455555,
      "grad_norm": 588.0,
      "learning_rate": 6.31578947368421e-08,
      "logits/chosen": -0.6413217186927795,
      "logits/rejected": -0.6328826546669006,
      "logps/chosen": -1272.714111328125,
      "logps/rejected": -1301.6715087890625,
      "loss": 0.7063,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.015728378668427467,
      "rewards/margins": -0.02517671510577202,
      "rewards/rejected": 0.009448337368667126,
      "step": 3369
    },
    {
      "epoch": 0.8864193065807427,
      "grad_norm": 404.0,
      "learning_rate": 6.301169590643275e-08,
      "logits/chosen": -0.6624298691749573,
      "logits/rejected": -0.6601360440254211,
      "logps/chosen": -986.1869506835938,
      "logps/rejected": -774.7682495117188,
      "loss": 0.6907,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.015934467315673828,
      "rewards/margins": 0.0056559559889137745,
      "rewards/rejected": 0.010278512723743916,
      "step": 3370
    },
    {
      "epoch": 0.8866823390159299,
      "grad_norm": 368.0,
      "learning_rate": 6.286549707602338e-08,
      "logits/chosen": -0.6613453030586243,
      "logits/rejected": -0.6669487953186035,
      "logps/chosen": -604.0625,
      "logps/rejected": -572.9750366210938,
      "loss": 0.7031,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.0027565963100641966,
      "rewards/margins": -0.019178103655576706,
      "rewards/rejected": 0.02193470112979412,
      "step": 3371
    },
    {
      "epoch": 0.8869453714511171,
      "grad_norm": 556.0,
      "learning_rate": 6.271929824561403e-08,
      "logits/chosen": -0.6682577133178711,
      "logits/rejected": -0.6797242164611816,
      "logps/chosen": -1143.114013671875,
      "logps/rejected": -1111.3909912109375,
      "loss": 0.6913,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.00539359962567687,
      "rewards/margins": 0.005430986173450947,
      "rewards/rejected": -0.01082458533346653,
      "step": 3372
    },
    {
      "epoch": 0.8872084038863042,
      "grad_norm": 552.0,
      "learning_rate": 6.257309941520467e-08,
      "logits/chosen": -0.6597737073898315,
      "logits/rejected": -0.6629911661148071,
      "logps/chosen": -1093.72705078125,
      "logps/rejected": -889.7152099609375,
      "loss": 0.682,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.024031737819314003,
      "rewards/margins": 0.023343849927186966,
      "rewards/rejected": 0.0006878853309899569,
      "step": 3373
    },
    {
      "epoch": 0.8874714363214914,
      "grad_norm": 488.0,
      "learning_rate": 6.242690058479532e-08,
      "logits/chosen": -0.6636601090431213,
      "logits/rejected": -0.6624003648757935,
      "logps/chosen": -1064.66015625,
      "logps/rejected": -909.6530151367188,
      "loss": 0.7052,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.030515098944306374,
      "rewards/margins": -0.023048589006066322,
      "rewards/rejected": -0.007466507144272327,
      "step": 3374
    },
    {
      "epoch": 0.8877344687566786,
      "grad_norm": 364.0,
      "learning_rate": 6.228070175438597e-08,
      "logits/chosen": -0.6397337913513184,
      "logits/rejected": -0.6387092471122742,
      "logps/chosen": -719.9305419921875,
      "logps/rejected": -736.3624267578125,
      "loss": 0.6927,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00926980935037136,
      "rewards/margins": 0.0010398866143077612,
      "rewards/rejected": 0.008229922503232956,
      "step": 3375
    },
    {
      "epoch": 0.8879975011918657,
      "grad_norm": 576.0,
      "learning_rate": 6.21345029239766e-08,
      "logits/chosen": -0.6659041047096252,
      "logits/rejected": -0.672226071357727,
      "logps/chosen": -1312.965087890625,
      "logps/rejected": -883.725830078125,
      "loss": 0.6905,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0241070743650198,
      "rewards/margins": 0.006130027584731579,
      "rewards/rejected": -0.030237101018428802,
      "step": 3376
    },
    {
      "epoch": 0.8882605336270529,
      "grad_norm": 540.0,
      "learning_rate": 6.198830409356725e-08,
      "logits/chosen": -0.663260281085968,
      "logits/rejected": -0.6643805503845215,
      "logps/chosen": -896.876220703125,
      "logps/rejected": -759.8771362304688,
      "loss": 0.6805,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.020761772990226746,
      "rewards/margins": 0.025801274925470352,
      "rewards/rejected": -0.005039501003921032,
      "step": 3377
    },
    {
      "epoch": 0.88852356606224,
      "grad_norm": 592.0,
      "learning_rate": 6.184210526315789e-08,
      "logits/chosen": -0.632621705532074,
      "logits/rejected": -0.6394215822219849,
      "logps/chosen": -1039.656005859375,
      "logps/rejected": -763.8220825195312,
      "loss": 0.7057,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.021007250994443893,
      "rewards/margins": -0.024343494325876236,
      "rewards/rejected": 0.003336238209158182,
      "step": 3378
    },
    {
      "epoch": 0.8887865984974272,
      "grad_norm": 418.0,
      "learning_rate": 6.169590643274854e-08,
      "logits/chosen": -0.6275845766067505,
      "logits/rejected": -0.6193620562553406,
      "logps/chosen": -735.7340087890625,
      "logps/rejected": -716.8938598632812,
      "loss": 0.6896,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0017191892256960273,
      "rewards/margins": 0.007543946150690317,
      "rewards/rejected": -0.005824756808578968,
      "step": 3379
    },
    {
      "epoch": 0.8890496309326144,
      "grad_norm": 500.0,
      "learning_rate": 6.154970760233917e-08,
      "logits/chosen": -0.6453436017036438,
      "logits/rejected": -0.651308536529541,
      "logps/chosen": -1170.764404296875,
      "logps/rejected": -953.3676147460938,
      "loss": 0.7046,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.02083568461239338,
      "rewards/margins": -0.022501660510897636,
      "rewards/rejected": 0.0016659735701978207,
      "step": 3380
    },
    {
      "epoch": 0.8893126633678016,
      "grad_norm": 624.0,
      "learning_rate": 6.140350877192982e-08,
      "logits/chosen": -0.6522870063781738,
      "logits/rejected": NaN,
      "logps/chosen": -1447.687744140625,
      "logps/rejected": -1202.3782958984375,
      "loss": 0.68,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03291058540344238,
      "rewards/margins": 0.027699235826730728,
      "rewards/rejected": 0.005211354233324528,
      "step": 3381
    },
    {
      "epoch": 0.8895756958029887,
      "grad_norm": 4192.0,
      "learning_rate": 6.125730994152046e-08,
      "logits/chosen": -0.6694400310516357,
      "logits/rejected": -0.6774771809577942,
      "logps/chosen": -1372.1021728515625,
      "logps/rejected": -1307.3756103515625,
      "loss": 0.6913,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.023311711847782135,
      "rewards/margins": 0.005886507220566273,
      "rewards/rejected": 0.017425205558538437,
      "step": 3382
    },
    {
      "epoch": 0.8898387282381759,
      "grad_norm": 616.0,
      "learning_rate": 6.111111111111111e-08,
      "logits/chosen": -0.6473677158355713,
      "logits/rejected": -0.6418097615242004,
      "logps/chosen": -1303.4200439453125,
      "logps/rejected": -1193.798095703125,
      "loss": 0.6954,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.007534313481301069,
      "rewards/margins": -0.0033480648417025805,
      "rewards/rejected": -0.004186248406767845,
      "step": 3383
    },
    {
      "epoch": 0.890101760673363,
      "grad_norm": 548.0,
      "learning_rate": 6.096491228070176e-08,
      "logits/chosen": -0.6490757465362549,
      "logits/rejected": -0.6547098755836487,
      "logps/chosen": -1134.2489013671875,
      "logps/rejected": -911.7296142578125,
      "loss": 0.6823,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02545461803674698,
      "rewards/margins": 0.023097706958651543,
      "rewards/rejected": 0.002356909681111574,
      "step": 3384
    },
    {
      "epoch": 0.8903647931085502,
      "grad_norm": 482.0,
      "learning_rate": 6.081871345029239e-08,
      "logits/chosen": -0.6482840776443481,
      "logits/rejected": -0.6349965333938599,
      "logps/chosen": -974.89306640625,
      "logps/rejected": -731.7068481445312,
      "loss": 0.6858,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01974296383559704,
      "rewards/margins": 0.015197992324829102,
      "rewards/rejected": 0.004544973373413086,
      "step": 3385
    },
    {
      "epoch": 0.8906278255437373,
      "grad_norm": 552.0,
      "learning_rate": 6.067251461988304e-08,
      "logits/chosen": -0.6652541160583496,
      "logits/rejected": -0.6616442203521729,
      "logps/chosen": -1551.87158203125,
      "logps/rejected": -1119.5189208984375,
      "loss": 0.6729,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.02581758424639702,
      "rewards/margins": 0.04216260835528374,
      "rewards/rejected": -0.01634502410888672,
      "step": 3386
    },
    {
      "epoch": 0.8908908579789245,
      "grad_norm": 620.0,
      "learning_rate": 6.052631578947368e-08,
      "logits/chosen": -0.6759100556373596,
      "logits/rejected": -0.672700822353363,
      "logps/chosen": -1215.93603515625,
      "logps/rejected": -833.8815307617188,
      "loss": 0.6841,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.017633534967899323,
      "rewards/margins": 0.019564343616366386,
      "rewards/rejected": -0.0019308084156364202,
      "step": 3387
    },
    {
      "epoch": 0.8911538904141116,
      "grad_norm": 740.0,
      "learning_rate": 6.038011695906432e-08,
      "logits/chosen": -0.6460457444190979,
      "logits/rejected": -0.6473469734191895,
      "logps/chosen": -1670.867431640625,
      "logps/rejected": -983.6109008789062,
      "loss": 0.6937,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.007305050268769264,
      "rewards/margins": 0.0006979936733841896,
      "rewards/rejected": -0.008003043942153454,
      "step": 3388
    },
    {
      "epoch": 0.8914169228492989,
      "grad_norm": 560.0,
      "learning_rate": 6.023391812865498e-08,
      "logits/chosen": -0.6327630877494812,
      "logits/rejected": -0.6326889991760254,
      "logps/chosen": -1159.455810546875,
      "logps/rejected": -842.8922729492188,
      "loss": 0.6958,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0002408034633845091,
      "rewards/margins": -0.005064154043793678,
      "rewards/rejected": 0.005304957274347544,
      "step": 3389
    },
    {
      "epoch": 0.8916799552844861,
      "grad_norm": 512.0,
      "learning_rate": 6.008771929824561e-08,
      "logits/chosen": -0.6607513427734375,
      "logits/rejected": -0.668337345123291,
      "logps/chosen": -907.6136474609375,
      "logps/rejected": -952.306396484375,
      "loss": 0.7077,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.016440868377685547,
      "rewards/margins": -0.027883337810635567,
      "rewards/rejected": 0.011442470364272594,
      "step": 3390
    },
    {
      "epoch": 0.8919429877196732,
      "grad_norm": 456.0,
      "learning_rate": 5.994152046783626e-08,
      "logits/chosen": -0.6541591882705688,
      "logits/rejected": NaN,
      "logps/chosen": -852.3823852539062,
      "logps/rejected": -484.8687744140625,
      "loss": 0.7021,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.022922802716493607,
      "rewards/margins": -0.017580032348632812,
      "rewards/rejected": -0.005342768505215645,
      "step": 3391
    },
    {
      "epoch": 0.8922060201548604,
      "grad_norm": 648.0,
      "learning_rate": 5.97953216374269e-08,
      "logits/chosen": -0.6742181181907654,
      "logits/rejected": -0.6681491136550903,
      "logps/chosen": -1266.8427734375,
      "logps/rejected": -841.0453491210938,
      "loss": 0.7107,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.004016399383544922,
      "rewards/margins": -0.033189110457897186,
      "rewards/rejected": 0.029172707349061966,
      "step": 3392
    },
    {
      "epoch": 0.8924690525900475,
      "grad_norm": 620.0,
      "learning_rate": 5.964912280701754e-08,
      "logits/chosen": -0.6537283062934875,
      "logits/rejected": -0.6452088356018066,
      "logps/chosen": -1364.5386962890625,
      "logps/rejected": -1333.1087646484375,
      "loss": 0.6936,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.009355640038847923,
      "rewards/margins": 0.000982954166829586,
      "rewards/rejected": 0.008372689597308636,
      "step": 3393
    },
    {
      "epoch": 0.8927320850252347,
      "grad_norm": 616.0,
      "learning_rate": 5.950292397660819e-08,
      "logits/chosen": -0.6728650331497192,
      "logits/rejected": -0.6575299501419067,
      "logps/chosen": -950.358642578125,
      "logps/rejected": -854.3661499023438,
      "loss": 0.685,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0006067268550395966,
      "rewards/margins": 0.017618561163544655,
      "rewards/rejected": -0.0182252898812294,
      "step": 3394
    },
    {
      "epoch": 0.8929951174604218,
      "grad_norm": 544.0,
      "learning_rate": 5.935672514619883e-08,
      "logits/chosen": -0.6770226955413818,
      "logits/rejected": -0.6750378012657166,
      "logps/chosen": -1152.269775390625,
      "logps/rejected": -962.0238037109375,
      "loss": 0.7051,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01006174087524414,
      "rewards/margins": -0.02340707555413246,
      "rewards/rejected": 0.013345337472856045,
      "step": 3395
    },
    {
      "epoch": 0.893258149895609,
      "grad_norm": 462.0,
      "learning_rate": 5.921052631578947e-08,
      "logits/chosen": -0.6578441858291626,
      "logits/rejected": -0.6554123759269714,
      "logps/chosen": -993.5431518554688,
      "logps/rejected": -754.868408203125,
      "loss": 0.6922,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005994701758027077,
      "rewards/margins": 0.0026514045894145966,
      "rewards/rejected": 0.003343296702951193,
      "step": 3396
    },
    {
      "epoch": 0.8935211823307961,
      "grad_norm": 572.0,
      "learning_rate": 5.9064327485380114e-08,
      "logits/chosen": -0.6435279846191406,
      "logits/rejected": -0.656071662902832,
      "logps/chosen": -1063.452392578125,
      "logps/rejected": -980.2382202148438,
      "loss": 0.6862,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.009694194421172142,
      "rewards/margins": 0.01418008841574192,
      "rewards/rejected": -0.004485893528908491,
      "step": 3397
    },
    {
      "epoch": 0.8937842147659834,
      "grad_norm": 552.0,
      "learning_rate": 5.8918128654970754e-08,
      "logits/chosen": -0.6513954997062683,
      "logits/rejected": -0.6528202891349792,
      "logps/chosen": -1326.1912841796875,
      "logps/rejected": -879.0120239257812,
      "loss": 0.6865,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0160432830452919,
      "rewards/margins": 0.014684963971376419,
      "rewards/rejected": 0.0013583190739154816,
      "step": 3398
    },
    {
      "epoch": 0.8940472472011705,
      "grad_norm": 500.0,
      "learning_rate": 5.87719298245614e-08,
      "logits/chosen": -0.632662832736969,
      "logits/rejected": -0.6350637674331665,
      "logps/chosen": -1094.225830078125,
      "logps/rejected": -916.166015625,
      "loss": 0.6894,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.013127708807587624,
      "rewards/margins": 0.007695198990404606,
      "rewards/rejected": 0.005432510282844305,
      "step": 3399
    },
    {
      "epoch": 0.8943102796363577,
      "grad_norm": 540.0,
      "learning_rate": 5.862573099415205e-08,
      "logits/chosen": -0.6310463547706604,
      "logits/rejected": -0.6283692717552185,
      "logps/chosen": -1092.2415771484375,
      "logps/rejected": -814.9226684570312,
      "loss": 0.6976,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.01245031412690878,
      "rewards/margins": -0.007743120193481445,
      "rewards/rejected": 0.02019343338906765,
      "step": 3400
    },
    {
      "epoch": 0.8945733120715448,
      "grad_norm": 520.0,
      "learning_rate": 5.8479532163742687e-08,
      "logits/chosen": -0.6873680353164673,
      "logits/rejected": -0.6746306419372559,
      "logps/chosen": -1427.287109375,
      "logps/rejected": -1075.5740966796875,
      "loss": 0.6942,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006187534425407648,
      "rewards/margins": -0.0010510431602597237,
      "rewards/rejected": 0.007238578516989946,
      "step": 3401
    },
    {
      "epoch": 0.894836344506732,
      "grad_norm": 620.0,
      "learning_rate": 5.833333333333333e-08,
      "logits/chosen": -0.6663298010826111,
      "logits/rejected": -0.6697719693183899,
      "logps/chosen": -922.9501953125,
      "logps/rejected": -1017.1685180664062,
      "loss": 0.7112,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": 0.012495137751102448,
      "rewards/margins": -0.03524837642908096,
      "rewards/rejected": 0.047743506729602814,
      "step": 3402
    },
    {
      "epoch": 0.8950993769419191,
      "grad_norm": 600.0,
      "learning_rate": 5.818713450292397e-08,
      "logits/chosen": -0.6226766705513,
      "logits/rejected": -0.6291599273681641,
      "logps/chosen": -1305.6937255859375,
      "logps/rejected": -858.45849609375,
      "loss": 0.6988,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.015216542407870293,
      "rewards/margins": -0.01060466654598713,
      "rewards/rejected": -0.0046118744648993015,
      "step": 3403
    },
    {
      "epoch": 0.8953624093771063,
      "grad_norm": 478.0,
      "learning_rate": 5.804093567251462e-08,
      "logits/chosen": -0.6739323735237122,
      "logits/rejected": -0.6757798790931702,
      "logps/chosen": -892.9237670898438,
      "logps/rejected": -636.7422485351562,
      "loss": 0.6952,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.009115887805819511,
      "rewards/margins": -0.003608226776123047,
      "rewards/rejected": 0.012724113650619984,
      "step": 3404
    },
    {
      "epoch": 0.8956254418122935,
      "grad_norm": 568.0,
      "learning_rate": 5.789473684210526e-08,
      "logits/chosen": -0.6614107489585876,
      "logits/rejected": -0.6642999649047852,
      "logps/chosen": -1383.5657958984375,
      "logps/rejected": -1403.82666015625,
      "loss": 0.6867,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.016951370984315872,
      "rewards/margins": 0.01386871188879013,
      "rewards/rejected": -0.030820084735751152,
      "step": 3405
    },
    {
      "epoch": 0.8958884742474806,
      "grad_norm": 484.0,
      "learning_rate": 5.77485380116959e-08,
      "logits/chosen": -0.6527329683303833,
      "logits/rejected": -0.6554860472679138,
      "logps/chosen": -1050.8492431640625,
      "logps/rejected": -774.726806640625,
      "loss": 0.6922,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0011498446110635996,
      "rewards/margins": 0.002330589108169079,
      "rewards/rejected": -0.0011807437986135483,
      "step": 3406
    },
    {
      "epoch": 0.8961515066826679,
      "grad_norm": 532.0,
      "learning_rate": 5.7602339181286545e-08,
      "logits/chosen": -0.6384435892105103,
      "logits/rejected": -0.6464004516601562,
      "logps/chosen": -1211.672607421875,
      "logps/rejected": -894.7320556640625,
      "loss": 0.6985,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0034747119061648846,
      "rewards/margins": -0.009758186526596546,
      "rewards/rejected": 0.0062834732234478,
      "step": 3407
    },
    {
      "epoch": 0.896414539117855,
      "grad_norm": 482.0,
      "learning_rate": 5.745614035087719e-08,
      "logits/chosen": -0.6521759033203125,
      "logits/rejected": -0.6569759249687195,
      "logps/chosen": -766.4893798828125,
      "logps/rejected": -934.1320190429688,
      "loss": 0.6919,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.02144317701458931,
      "rewards/margins": 0.003413391299545765,
      "rewards/rejected": -0.02485656924545765,
      "step": 3408
    },
    {
      "epoch": 0.8966775715530422,
      "grad_norm": 528.0,
      "learning_rate": 5.730994152046784e-08,
      "logits/chosen": -0.6325199007987976,
      "logits/rejected": -0.6252445578575134,
      "logps/chosen": -963.607177734375,
      "logps/rejected": -713.6912231445312,
      "loss": 0.6957,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0013335223775357008,
      "rewards/margins": -0.004232216160744429,
      "rewards/rejected": 0.002898693783208728,
      "step": 3409
    },
    {
      "epoch": 0.8969406039882293,
      "grad_norm": 488.0,
      "learning_rate": 5.716374269005848e-08,
      "logits/chosen": -0.6374890804290771,
      "logits/rejected": -0.6455501317977905,
      "logps/chosen": -1127.2003173828125,
      "logps/rejected": -629.1759643554688,
      "loss": 0.7011,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.002790069440379739,
      "rewards/margins": -0.015293026342988014,
      "rewards/rejected": 0.012502956204116344,
      "step": 3410
    },
    {
      "epoch": 0.8972036364234165,
      "grad_norm": 604.0,
      "learning_rate": 5.701754385964912e-08,
      "logits/chosen": -0.6378210783004761,
      "logits/rejected": -0.6381642818450928,
      "logps/chosen": -1235.2694091796875,
      "logps/rejected": -1047.202392578125,
      "loss": 0.7019,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.008102798834443092,
      "rewards/margins": -0.01671438105404377,
      "rewards/rejected": 0.008611584082245827,
      "step": 3411
    },
    {
      "epoch": 0.8974666688586036,
      "grad_norm": 484.0,
      "learning_rate": 5.6871345029239764e-08,
      "logits/chosen": -0.644895076751709,
      "logits/rejected": -0.6506465673446655,
      "logps/chosen": -787.8405151367188,
      "logps/rejected": -814.62109375,
      "loss": 0.6907,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007804393768310547,
      "rewards/margins": 0.005407811142504215,
      "rewards/rejected": 0.0023965835571289062,
      "step": 3412
    },
    {
      "epoch": 0.8977297012937908,
      "grad_norm": 478.0,
      "learning_rate": 5.6725146198830403e-08,
      "logits/chosen": -0.6630858182907104,
      "logits/rejected": -0.6672934889793396,
      "logps/chosen": -892.0255126953125,
      "logps/rejected": -849.4006958007812,
      "loss": 0.6808,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01628732681274414,
      "rewards/margins": 0.026238633319735527,
      "rewards/rejected": -0.009951305575668812,
      "step": 3413
    },
    {
      "epoch": 0.8979927337289779,
      "grad_norm": 406.0,
      "learning_rate": 5.657894736842105e-08,
      "logits/chosen": -0.6645647287368774,
      "logits/rejected": -0.6554023027420044,
      "logps/chosen": -591.4409790039062,
      "logps/rejected": -647.976318359375,
      "loss": 0.6778,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.006723023019731045,
      "rewards/margins": 0.0315672867000103,
      "rewards/rejected": -0.024844268336892128,
      "step": 3414
    },
    {
      "epoch": 0.8982557661641651,
      "grad_norm": 472.0,
      "learning_rate": 5.6432748538011696e-08,
      "logits/chosen": -0.6496878862380981,
      "logits/rejected": -0.6371416449546814,
      "logps/chosen": -1056.2457275390625,
      "logps/rejected": -842.629150390625,
      "loss": 0.6895,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.004804420284926891,
      "rewards/margins": 0.007496929727494717,
      "rewards/rejected": -0.012301350943744183,
      "step": 3415
    },
    {
      "epoch": 0.8985187985993522,
      "grad_norm": 612.0,
      "learning_rate": 5.6286549707602336e-08,
      "logits/chosen": -0.6388391852378845,
      "logits/rejected": -0.6386699080467224,
      "logps/chosen": -1602.12939453125,
      "logps/rejected": -1113.4599609375,
      "loss": 0.6663,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.042005348950624466,
      "rewards/margins": 0.05661468580365181,
      "rewards/rejected": -0.014609336853027344,
      "step": 3416
    },
    {
      "epoch": 0.8987818310345395,
      "grad_norm": 9088.0,
      "learning_rate": 5.614035087719298e-08,
      "logits/chosen": -0.6697995662689209,
      "logits/rejected": -0.6726348400115967,
      "logps/chosen": -1184.9442138671875,
      "logps/rejected": -792.05615234375,
      "loss": 0.6861,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.027091408148407936,
      "rewards/margins": 0.015437174588441849,
      "rewards/rejected": 0.011654233559966087,
      "step": 3417
    },
    {
      "epoch": 0.8990448634697266,
      "grad_norm": 552.0,
      "learning_rate": 5.599415204678362e-08,
      "logits/chosen": -0.637041449546814,
      "logits/rejected": -0.6378015875816345,
      "logps/chosen": -1182.2735595703125,
      "logps/rejected": -755.146728515625,
      "loss": 0.6977,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.011063860729336739,
      "rewards/margins": -0.00858326070010662,
      "rewards/rejected": 0.01964712142944336,
      "step": 3418
    },
    {
      "epoch": 0.8993078959049138,
      "grad_norm": 422.0,
      "learning_rate": 5.584795321637427e-08,
      "logits/chosen": -0.6448235511779785,
      "logits/rejected": -0.6533509492874146,
      "logps/chosen": -1190.739013671875,
      "logps/rejected": -848.7125854492188,
      "loss": 0.6874,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.020593833178281784,
      "rewards/margins": 0.012339972890913486,
      "rewards/rejected": 0.008253861218690872,
      "step": 3419
    },
    {
      "epoch": 0.8995709283401009,
      "grad_norm": 430.0,
      "learning_rate": 5.570175438596491e-08,
      "logits/chosen": -0.6475235819816589,
      "logits/rejected": -0.6620596647262573,
      "logps/chosen": -734.5870361328125,
      "logps/rejected": -553.9248657226562,
      "loss": 0.699,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.002601147396489978,
      "rewards/margins": -0.01130685769021511,
      "rewards/rejected": 0.013908005319535732,
      "step": 3420
    },
    {
      "epoch": 0.8998339607752881,
      "grad_norm": 544.0,
      "learning_rate": 5.555555555555555e-08,
      "logits/chosen": -0.6301195025444031,
      "logits/rejected": -0.6468613743782043,
      "logps/chosen": -1052.1685791015625,
      "logps/rejected": -954.4635620117188,
      "loss": 0.6957,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.01940794289112091,
      "rewards/margins": -0.004431437700986862,
      "rewards/rejected": -0.014976502396166325,
      "step": 3421
    },
    {
      "epoch": 0.9000969932104753,
      "grad_norm": 536.0,
      "learning_rate": 5.5409356725146194e-08,
      "logits/chosen": -0.6477853655815125,
      "logits/rejected": -0.6508433818817139,
      "logps/chosen": -1278.554931640625,
      "logps/rejected": -1076.295654296875,
      "loss": 0.692,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.012912178412079811,
      "rewards/margins": 0.0028555868193507195,
      "rewards/rejected": -0.015767766162753105,
      "step": 3422
    },
    {
      "epoch": 0.9003600256456624,
      "grad_norm": 660.0,
      "learning_rate": 5.526315789473684e-08,
      "logits/chosen": -0.6686428189277649,
      "logits/rejected": -0.6713559627532959,
      "logps/chosen": -1719.636474609375,
      "logps/rejected": -1178.7723388671875,
      "loss": 0.6939,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0059337611310184,
      "rewards/margins": -0.00029211118817329407,
      "rewards/rejected": 0.00622587138786912,
      "step": 3423
    },
    {
      "epoch": 0.9006230580808496,
      "grad_norm": 492.0,
      "learning_rate": 5.511695906432749e-08,
      "logits/chosen": -0.6670421361923218,
      "logits/rejected": -0.6817688345909119,
      "logps/chosen": -1027.8050537109375,
      "logps/rejected": -701.3116455078125,
      "loss": 0.6938,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011316300369799137,
      "rewards/margins": -0.0004912849981337786,
      "rewards/rejected": -0.010825015604496002,
      "step": 3424
    },
    {
      "epoch": 0.9008860905160367,
      "grad_norm": 588.0,
      "learning_rate": 5.497076023391813e-08,
      "logits/chosen": -0.6291927099227905,
      "logits/rejected": -0.6279299855232239,
      "logps/chosen": -1149.6859130859375,
      "logps/rejected": -972.236083984375,
      "loss": 0.69,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.018508436158299446,
      "rewards/margins": 0.0075966836884617805,
      "rewards/rejected": 0.010911749675869942,
      "step": 3425
    },
    {
      "epoch": 0.901149122951224,
      "grad_norm": 680.0,
      "learning_rate": 5.4824561403508767e-08,
      "logits/chosen": -0.6499423384666443,
      "logits/rejected": -0.6585263013839722,
      "logps/chosen": -1418.63916015625,
      "logps/rejected": -1001.45556640625,
      "loss": 0.706,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.013634299859404564,
      "rewards/margins": -0.024980926886200905,
      "rewards/rejected": 0.011346627026796341,
      "step": 3426
    },
    {
      "epoch": 0.9014121553864111,
      "grad_norm": 540.0,
      "learning_rate": 5.467836257309941e-08,
      "logits/chosen": -0.6525763273239136,
      "logits/rejected": -0.6554872393608093,
      "logps/chosen": -1699.5235595703125,
      "logps/rejected": -1321.2479248046875,
      "loss": 0.6947,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0007810587994754314,
      "rewards/margins": -0.0027418131940066814,
      "rewards/rejected": 0.00196075439453125,
      "step": 3427
    },
    {
      "epoch": 0.9016751878215983,
      "grad_norm": 548.0,
      "learning_rate": 5.453216374269005e-08,
      "logits/chosen": -0.6505122184753418,
      "logits/rejected": -0.6408189535140991,
      "logps/chosen": -1097.529052734375,
      "logps/rejected": -1197.163818359375,
      "loss": 0.6892,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.009613323956727982,
      "rewards/margins": 0.008530521765351295,
      "rewards/rejected": 0.001082803588360548,
      "step": 3428
    },
    {
      "epoch": 0.9019382202567854,
      "grad_norm": 648.0,
      "learning_rate": 5.43859649122807e-08,
      "logits/chosen": -0.6562507152557373,
      "logits/rejected": -0.6554645895957947,
      "logps/chosen": -1197.1529541015625,
      "logps/rejected": -1053.7506103515625,
      "loss": 0.692,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.017024852335453033,
      "rewards/margins": 0.0029623520094901323,
      "rewards/rejected": 0.01406249962747097,
      "step": 3429
    },
    {
      "epoch": 0.9022012526919726,
      "grad_norm": 592.0,
      "learning_rate": 5.4239766081871345e-08,
      "logits/chosen": -0.6266906261444092,
      "logits/rejected": -0.6353515982627869,
      "logps/chosen": -1167.2684326171875,
      "logps/rejected": -929.553466796875,
      "loss": 0.6995,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.008692549541592598,
      "rewards/margins": -0.012221241369843483,
      "rewards/rejected": 0.003528691129758954,
      "step": 3430
    },
    {
      "epoch": 0.9024642851271597,
      "grad_norm": 486.0,
      "learning_rate": 5.4093567251461985e-08,
      "logits/chosen": -0.6371377110481262,
      "logits/rejected": NaN,
      "logps/chosen": -1296.65869140625,
      "logps/rejected": -1089.9078369140625,
      "loss": 0.6857,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.018557025119662285,
      "rewards/margins": 0.015602636151015759,
      "rewards/rejected": -0.03415966033935547,
      "step": 3431
    },
    {
      "epoch": 0.9027273175623469,
      "grad_norm": 568.0,
      "learning_rate": 5.394736842105263e-08,
      "logits/chosen": -0.6644036769866943,
      "logits/rejected": -0.6576153635978699,
      "logps/chosen": -1090.9801025390625,
      "logps/rejected": -834.6051635742188,
      "loss": 0.6847,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01654214784502983,
      "rewards/margins": 0.018221713602542877,
      "rewards/rejected": -0.0016795629635453224,
      "step": 3432
    },
    {
      "epoch": 0.902990349997534,
      "grad_norm": 466.0,
      "learning_rate": 5.380116959064327e-08,
      "logits/chosen": -0.6460679173469543,
      "logits/rejected": -0.6513463258743286,
      "logps/chosen": -869.1341552734375,
      "logps/rejected": -785.5119018554688,
      "loss": 0.6948,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0011790287680923939,
      "rewards/margins": -0.0030733102466911077,
      "rewards/rejected": 0.0018942835740745068,
      "step": 3433
    },
    {
      "epoch": 0.9032533824327212,
      "grad_norm": 496.0,
      "learning_rate": 5.365497076023392e-08,
      "logits/chosen": -0.6556447744369507,
      "logits/rejected": -0.6604781150817871,
      "logps/chosen": -776.6028442382812,
      "logps/rejected": -803.989501953125,
      "loss": 0.6975,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0024640089832246304,
      "rewards/margins": -0.008358382619917393,
      "rewards/rejected": 0.010822391137480736,
      "step": 3434
    },
    {
      "epoch": 0.9035164148679083,
      "grad_norm": 624.0,
      "learning_rate": 5.350877192982456e-08,
      "logits/chosen": -0.6753517389297485,
      "logits/rejected": -0.6730762720108032,
      "logps/chosen": -1428.744140625,
      "logps/rejected": -1399.3153076171875,
      "loss": 0.6933,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02102513425052166,
      "rewards/margins": 0.0006379117257893085,
      "rewards/rejected": 0.02038721926510334,
      "step": 3435
    },
    {
      "epoch": 0.9037794473030956,
      "grad_norm": 676.0,
      "learning_rate": 5.33625730994152e-08,
      "logits/chosen": -0.6776134371757507,
      "logits/rejected": -0.6760495901107788,
      "logps/chosen": -1685.88525390625,
      "logps/rejected": -1526.18017578125,
      "loss": 0.7007,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011550522409379482,
      "rewards/margins": -0.014225959777832031,
      "rewards/rejected": 0.002675439231097698,
      "step": 3436
    },
    {
      "epoch": 0.9040424797382828,
      "grad_norm": 448.0,
      "learning_rate": 5.3216374269005844e-08,
      "logits/chosen": -0.6628233790397644,
      "logits/rejected": NaN,
      "logps/chosen": -972.29345703125,
      "logps/rejected": -746.4439697265625,
      "loss": 0.6916,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01920023001730442,
      "rewards/margins": 0.0040032388642430305,
      "rewards/rejected": 0.015196992084383965,
      "step": 3437
    },
    {
      "epoch": 0.9043055121734699,
      "grad_norm": 416.0,
      "learning_rate": 5.307017543859649e-08,
      "logits/chosen": -0.6449562907218933,
      "logits/rejected": -0.6508886814117432,
      "logps/chosen": -676.195068359375,
      "logps/rejected": -553.112548828125,
      "loss": 0.6976,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.008757304400205612,
      "rewards/margins": -0.008411312475800514,
      "rewards/rejected": -0.0003459928557276726,
      "step": 3438
    },
    {
      "epoch": 0.9045685446086571,
      "grad_norm": 508.0,
      "learning_rate": 5.2923976608187136e-08,
      "logits/chosen": -0.6369644403457642,
      "logits/rejected": -0.6280168294906616,
      "logps/chosen": -996.4756469726562,
      "logps/rejected": -927.0615234375,
      "loss": 0.6846,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.011884784325957298,
      "rewards/margins": 0.01789550855755806,
      "rewards/rejected": -0.006010723300278187,
      "step": 3439
    },
    {
      "epoch": 0.9048315770438442,
      "grad_norm": 486.0,
      "learning_rate": 5.2777777777777776e-08,
      "logits/chosen": -0.6663307547569275,
      "logits/rejected": -0.6609893441200256,
      "logps/chosen": -1024.9346923828125,
      "logps/rejected": -878.5617065429688,
      "loss": 0.688,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0053632743656635284,
      "rewards/margins": 0.011234667152166367,
      "rewards/rejected": -0.0058713918551802635,
      "step": 3440
    },
    {
      "epoch": 0.9050946094790314,
      "grad_norm": 444.0,
      "learning_rate": 5.2631578947368416e-08,
      "logits/chosen": -0.6514651775360107,
      "logits/rejected": -0.6538288593292236,
      "logps/chosen": -1093.1959228515625,
      "logps/rejected": -980.659423828125,
      "loss": 0.6877,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.005612373352050781,
      "rewards/margins": 0.011513233184814453,
      "rewards/rejected": -0.005900860298424959,
      "step": 3441
    },
    {
      "epoch": 0.9053576419142185,
      "grad_norm": 500.0,
      "learning_rate": 5.248538011695906e-08,
      "logits/chosen": -0.6528193354606628,
      "logits/rejected": -0.6685958504676819,
      "logps/chosen": -1349.942138671875,
      "logps/rejected": -1029.767578125,
      "loss": 0.6863,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02571888267993927,
      "rewards/margins": 0.014455603435635567,
      "rewards/rejected": 0.01126327458769083,
      "step": 3442
    },
    {
      "epoch": 0.9056206743494057,
      "grad_norm": 840.0,
      "learning_rate": 5.23391812865497e-08,
      "logits/chosen": NaN,
      "logits/rejected": -0.6478825807571411,
      "logps/chosen": -1415.7254638671875,
      "logps/rejected": -1068.4273681640625,
      "loss": 0.6934,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01161336898803711,
      "rewards/margins": 0.000809765886515379,
      "rewards/rejected": 0.010803602635860443,
      "step": 3443
    },
    {
      "epoch": 0.9058837067845928,
      "grad_norm": 454.0,
      "learning_rate": 5.219298245614035e-08,
      "logits/chosen": -0.6608107089996338,
      "logits/rejected": -0.660006582736969,
      "logps/chosen": -815.9180908203125,
      "logps/rejected": -774.3805541992188,
      "loss": 0.7023,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.005546950735151768,
      "rewards/margins": -0.017275236546993256,
      "rewards/rejected": 0.011728286743164062,
      "step": 3444
    },
    {
      "epoch": 0.9061467392197801,
      "grad_norm": 596.0,
      "learning_rate": 5.2046783625730995e-08,
      "logits/chosen": -0.6768559217453003,
      "logits/rejected": -0.6863192319869995,
      "logps/chosen": -1038.893310546875,
      "logps/rejected": -703.5724487304688,
      "loss": 0.6888,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0055190082639455795,
      "rewards/margins": 0.009128903038799763,
      "rewards/rejected": -0.003609895706176758,
      "step": 3445
    },
    {
      "epoch": 0.9064097716549672,
      "grad_norm": 524.0,
      "learning_rate": 5.1900584795321635e-08,
      "logits/chosen": -0.643725574016571,
      "logits/rejected": -0.6486415863037109,
      "logps/chosen": -1378.977783203125,
      "logps/rejected": -1061.8663330078125,
      "loss": 0.6931,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.002414131071418524,
      "rewards/margins": 0.0010513311717659235,
      "rewards/rejected": 0.001362800830975175,
      "step": 3446
    },
    {
      "epoch": 0.9066728040901544,
      "grad_norm": 536.0,
      "learning_rate": 5.175438596491228e-08,
      "logits/chosen": -0.6685237884521484,
      "logits/rejected": -0.6807296276092529,
      "logps/chosen": -1322.74560546875,
      "logps/rejected": -1173.16259765625,
      "loss": 0.6936,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.013927459716796875,
      "rewards/margins": -0.0006410597125068307,
      "rewards/rejected": 0.014568520709872246,
      "step": 3447
    },
    {
      "epoch": 0.9069358365253415,
      "grad_norm": 506.0,
      "learning_rate": 5.160818713450292e-08,
      "logits/chosen": -0.6571566462516785,
      "logits/rejected": -0.6571989059448242,
      "logps/chosen": -1126.1241455078125,
      "logps/rejected": -1082.0394287109375,
      "loss": 0.6881,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.005326652433723211,
      "rewards/margins": 0.010472582653164864,
      "rewards/rejected": -0.015799235552549362,
      "step": 3448
    },
    {
      "epoch": 0.9071988689605287,
      "grad_norm": 592.0,
      "learning_rate": 5.146198830409357e-08,
      "logits/chosen": -0.6397683620452881,
      "logits/rejected": -0.642620325088501,
      "logps/chosen": -1268.222900390625,
      "logps/rejected": -813.8260498046875,
      "loss": 0.7047,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.04324379190802574,
      "rewards/margins": -0.022009659558534622,
      "rewards/rejected": -0.02123413048684597,
      "step": 3449
    },
    {
      "epoch": 0.9074619013957158,
      "grad_norm": 584.0,
      "learning_rate": 5.131578947368421e-08,
      "logits/chosen": -0.6567603349685669,
      "logits/rejected": -0.6543141603469849,
      "logps/chosen": -1380.2174072265625,
      "logps/rejected": -1285.2691650390625,
      "loss": 0.69,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.005167102441191673,
      "rewards/margins": 0.007188320159912109,
      "rewards/rejected": -0.002021217253059149,
      "step": 3450
    },
    {
      "epoch": 0.907724933830903,
      "grad_norm": 506.0,
      "learning_rate": 5.1169590643274847e-08,
      "logits/chosen": -0.6523239612579346,
      "logits/rejected": -0.670351505279541,
      "logps/chosen": -1123.7816162109375,
      "logps/rejected": -659.546875,
      "loss": 0.6927,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0023952480405569077,
      "rewards/margins": 0.0015718452632427216,
      "rewards/rejected": -0.003967094700783491,
      "step": 3451
    },
    {
      "epoch": 0.9079879662660902,
      "grad_norm": 520.0,
      "learning_rate": 5.102339181286549e-08,
      "logits/chosen": -0.6414280533790588,
      "logits/rejected": -0.6433843970298767,
      "logps/chosen": -1045.7203369140625,
      "logps/rejected": -832.2088623046875,
      "loss": 0.7006,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.012178707867860794,
      "rewards/margins": -0.013887597247958183,
      "rewards/rejected": 0.0017088884487748146,
      "step": 3452
    },
    {
      "epoch": 0.9082509987012773,
      "grad_norm": 450.0,
      "learning_rate": 5.087719298245614e-08,
      "logits/chosen": -0.6417171955108643,
      "logits/rejected": -0.6521481275558472,
      "logps/chosen": -1075.802001953125,
      "logps/rejected": -916.159423828125,
      "loss": 0.695,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.004784583114087582,
      "rewards/margins": -0.0023160926066339016,
      "rewards/rejected": -0.0024684914387762547,
      "step": 3453
    },
    {
      "epoch": 0.9085140311364646,
      "grad_norm": 612.0,
      "learning_rate": 5.0730994152046786e-08,
      "logits/chosen": -0.6674172878265381,
      "logits/rejected": -0.6607555150985718,
      "logps/chosen": -1057.678466796875,
      "logps/rejected": -1106.3739013671875,
      "loss": 0.7021,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0020051011815667152,
      "rewards/margins": -0.017179012298583984,
      "rewards/rejected": 0.015173912048339844,
      "step": 3454
    },
    {
      "epoch": 0.9087770635716517,
      "grad_norm": 410.0,
      "learning_rate": 5.0584795321637425e-08,
      "logits/chosen": -0.6671954393386841,
      "logits/rejected": -0.6767348051071167,
      "logps/chosen": -1190.6700439453125,
      "logps/rejected": -1073.923828125,
      "loss": 0.6939,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0029704098124057055,
      "rewards/margins": -0.0012570377439260483,
      "rewards/rejected": -0.0017133720684796572,
      "step": 3455
    },
    {
      "epoch": 0.9090400960068389,
      "grad_norm": 620.0,
      "learning_rate": 5.0438596491228065e-08,
      "logits/chosen": -0.6540005207061768,
      "logits/rejected": -0.6689081192016602,
      "logps/chosen": -1366.28466796875,
      "logps/rejected": -1268.6640625,
      "loss": 0.6933,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.019598770886659622,
      "rewards/margins": 0.0008966458262875676,
      "rewards/rejected": 0.018702127039432526,
      "step": 3456
    },
    {
      "epoch": 0.909303128442026,
      "grad_norm": 436.0,
      "learning_rate": 5.029239766081871e-08,
      "logits/chosen": -0.6529109477996826,
      "logits/rejected": -0.6526429057121277,
      "logps/chosen": -799.3390502929688,
      "logps/rejected": -586.6447143554688,
      "loss": 0.6843,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.020569467917084694,
      "rewards/margins": 0.01880965381860733,
      "rewards/rejected": 0.0017598150297999382,
      "step": 3457
    },
    {
      "epoch": 0.9095661608772132,
      "grad_norm": 560.0,
      "learning_rate": 5.014619883040935e-08,
      "logits/chosen": -0.6520256996154785,
      "logits/rejected": -0.6729811429977417,
      "logps/chosen": -996.2218017578125,
      "logps/rejected": -651.7677612304688,
      "loss": 0.6785,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.019822167232632637,
      "rewards/margins": 0.03023548051714897,
      "rewards/rejected": -0.010413313284516335,
      "step": 3458
    },
    {
      "epoch": 0.9098291933124003,
      "grad_norm": 672.0,
      "learning_rate": 5e-08,
      "logits/chosen": -0.6521204710006714,
      "logits/rejected": -0.6548497080802917,
      "logps/chosen": -957.2633056640625,
      "logps/rejected": -1215.5819091796875,
      "loss": 0.6985,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0075971586629748344,
      "rewards/margins": -0.009964179247617722,
      "rewards/rejected": 0.002367019886150956,
      "step": 3459
    },
    {
      "epoch": 0.9100922257475875,
      "grad_norm": 544.0,
      "learning_rate": 4.9853801169590644e-08,
      "logits/chosen": -0.6828151345252991,
      "logits/rejected": -0.6797333359718323,
      "logps/chosen": -1161.3131103515625,
      "logps/rejected": -1022.3682861328125,
      "loss": 0.6863,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012512207962572575,
      "rewards/margins": 0.014718628488481045,
      "rewards/rejected": -0.0022064209915697575,
      "step": 3460
    },
    {
      "epoch": 0.9103552581827746,
      "grad_norm": 498.0,
      "learning_rate": 4.9707602339181284e-08,
      "logits/chosen": -0.6584212779998779,
      "logits/rejected": -0.6566469669342041,
      "logps/chosen": -929.5654296875,
      "logps/rejected": -904.3038330078125,
      "loss": 0.6819,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.020790956914424896,
      "rewards/margins": 0.02318134345114231,
      "rewards/rejected": -0.0023903853725641966,
      "step": 3461
    },
    {
      "epoch": 0.9106182906179618,
      "grad_norm": 616.0,
      "learning_rate": 4.956140350877193e-08,
      "logits/chosen": -0.655335545539856,
      "logits/rejected": -0.6544368267059326,
      "logps/chosen": -1358.1845703125,
      "logps/rejected": -978.5809326171875,
      "loss": 0.6981,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004593657795339823,
      "rewards/margins": -0.008575917221605778,
      "rewards/rejected": 0.013169574551284313,
      "step": 3462
    },
    {
      "epoch": 0.910881323053149,
      "grad_norm": 470.0,
      "learning_rate": 4.941520467836257e-08,
      "logits/chosen": -0.6421040892601013,
      "logits/rejected": -0.6424903869628906,
      "logps/chosen": -1226.176513671875,
      "logps/rejected": -835.6484375,
      "loss": 0.6963,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.008551504462957382,
      "rewards/margins": -0.005569459404796362,
      "rewards/rejected": -0.0029820427298545837,
      "step": 3463
    },
    {
      "epoch": 0.9111443554883362,
      "grad_norm": 556.0,
      "learning_rate": 4.9269005847953216e-08,
      "logits/chosen": -0.6311202645301819,
      "logits/rejected": -0.6340357065200806,
      "logps/chosen": -1138.71923828125,
      "logps/rejected": -689.7738647460938,
      "loss": 0.7026,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.013472937978804111,
      "rewards/margins": -0.018445394933223724,
      "rewards/rejected": 0.0049724578857421875,
      "step": 3464
    },
    {
      "epoch": 0.9114073879235233,
      "grad_norm": 652.0,
      "learning_rate": 4.9122807017543856e-08,
      "logits/chosen": -0.6457534432411194,
      "logits/rejected": -0.6614425778388977,
      "logps/chosen": -1234.7725830078125,
      "logps/rejected": -1080.375244140625,
      "loss": 0.7035,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.001014328096061945,
      "rewards/margins": -0.020062729716300964,
      "rewards/rejected": 0.019048402085900307,
      "step": 3465
    },
    {
      "epoch": 0.9116704203587105,
      "grad_norm": 516.0,
      "learning_rate": 4.8976608187134496e-08,
      "logits/chosen": -0.6401560306549072,
      "logits/rejected": -0.6358810067176819,
      "logps/chosen": -1217.781005859375,
      "logps/rejected": -1025.416015625,
      "loss": 0.693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.009161853231489658,
      "rewards/margins": 0.0007566453423351049,
      "rewards/rejected": 0.008405208587646484,
      "step": 3466
    },
    {
      "epoch": 0.9119334527938976,
      "grad_norm": 576.0,
      "learning_rate": 4.883040935672514e-08,
      "logits/chosen": -0.6344088912010193,
      "logits/rejected": -0.645176351070404,
      "logps/chosen": -1053.2552490234375,
      "logps/rejected": -908.4793701171875,
      "loss": 0.6966,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.012206554412841797,
      "rewards/margins": -0.00643739802762866,
      "rewards/rejected": -0.005769157316535711,
      "step": 3467
    },
    {
      "epoch": 0.9121964852290848,
      "grad_norm": 632.0,
      "learning_rate": 4.868421052631579e-08,
      "logits/chosen": -0.6517688632011414,
      "logits/rejected": -0.6629326939582825,
      "logps/chosen": -1406.5916748046875,
      "logps/rejected": -1117.5633544921875,
      "loss": 0.6869,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.018786240369081497,
      "rewards/margins": 0.013495014980435371,
      "rewards/rejected": 0.0052912249229848385,
      "step": 3468
    },
    {
      "epoch": 0.912459517664272,
      "grad_norm": 532.0,
      "learning_rate": 4.8538011695906435e-08,
      "logits/chosen": -0.6446672677993774,
      "logits/rejected": -0.649809718132019,
      "logps/chosen": -1419.323486328125,
      "logps/rejected": -1068.834716796875,
      "loss": 0.7032,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.017447758466005325,
      "rewards/margins": -0.019562434405088425,
      "rewards/rejected": 0.0021146778017282486,
      "step": 3469
    },
    {
      "epoch": 0.9127225500994591,
      "grad_norm": 680.0,
      "learning_rate": 4.8391812865497075e-08,
      "logits/chosen": -0.6585796475410461,
      "logits/rejected": -0.674196720123291,
      "logps/chosen": -1116.6915283203125,
      "logps/rejected": -1081.46484375,
      "loss": 0.687,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.020654870197176933,
      "rewards/margins": 0.013931989669799805,
      "rewards/rejected": 0.006722880527377129,
      "step": 3470
    },
    {
      "epoch": 0.9129855825346463,
      "grad_norm": 438.0,
      "learning_rate": 4.8245614035087715e-08,
      "logits/chosen": -0.6922568678855896,
      "logits/rejected": -0.6781750917434692,
      "logps/chosen": -1046.777587890625,
      "logps/rejected": -1213.0789794921875,
      "loss": 0.6845,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007888222113251686,
      "rewards/margins": 0.01913147233426571,
      "rewards/rejected": -0.011243248358368874,
      "step": 3471
    },
    {
      "epoch": 0.9132486149698334,
      "grad_norm": 15104.0,
      "learning_rate": 4.809941520467836e-08,
      "logits/chosen": -0.6438547372817993,
      "logits/rejected": NaN,
      "logps/chosen": -1171.1201171875,
      "logps/rejected": -726.397216796875,
      "loss": 0.8541,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.19108371436595917,
      "rewards/margins": -0.19389134645462036,
      "rewards/rejected": 0.0028076169546693563,
      "step": 3472
    },
    {
      "epoch": 0.9135116474050207,
      "grad_norm": 552.0,
      "learning_rate": 4.7953216374269e-08,
      "logits/chosen": -0.639188289642334,
      "logits/rejected": -0.6485818028450012,
      "logps/chosen": -1215.817138671875,
      "logps/rejected": -716.9418334960938,
      "loss": 0.6959,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.003009605221450329,
      "rewards/margins": -0.004809855483472347,
      "rewards/rejected": 0.0018002502620220184,
      "step": 3473
    },
    {
      "epoch": 0.9137746798402078,
      "grad_norm": 528.0,
      "learning_rate": 4.780701754385965e-08,
      "logits/chosen": -0.6531172394752502,
      "logits/rejected": -0.6534672975540161,
      "logps/chosen": -1143.852294921875,
      "logps/rejected": -911.259521484375,
      "loss": 0.7045,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.012006185948848724,
      "rewards/margins": -0.021910954266786575,
      "rewards/rejected": 0.009904766455292702,
      "step": 3474
    },
    {
      "epoch": 0.914037712275395,
      "grad_norm": 476.0,
      "learning_rate": 4.7660818713450293e-08,
      "logits/chosen": -0.6626449227333069,
      "logits/rejected": -0.6619241237640381,
      "logps/chosen": -883.5435791015625,
      "logps/rejected": -709.8071899414062,
      "loss": 0.6901,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0027026182506233454,
      "rewards/margins": 0.006360531318932772,
      "rewards/rejected": -0.00365791330114007,
      "step": 3475
    },
    {
      "epoch": 0.9143007447105821,
      "grad_norm": 552.0,
      "learning_rate": 4.751461988304093e-08,
      "logits/chosen": -0.6615011096000671,
      "logits/rejected": -0.6560609936714172,
      "logps/chosen": -1153.239990234375,
      "logps/rejected": -930.7307739257812,
      "loss": 0.7058,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.024569131433963776,
      "rewards/margins": -0.0246519073843956,
      "rewards/rejected": 8.277897723019123e-05,
      "step": 3476
    },
    {
      "epoch": 0.9145637771457693,
      "grad_norm": 532.0,
      "learning_rate": 4.736842105263158e-08,
      "logits/chosen": -0.6412147283554077,
      "logits/rejected": -0.6397028565406799,
      "logps/chosen": -1393.525390625,
      "logps/rejected": -1038.104736328125,
      "loss": 0.6881,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0006565088406205177,
      "rewards/margins": 0.010950660333037376,
      "rewards/rejected": -0.011607170104980469,
      "step": 3477
    },
    {
      "epoch": 0.9148268095809564,
      "grad_norm": 612.0,
      "learning_rate": 4.722222222222222e-08,
      "logits/chosen": -0.6383865475654602,
      "logits/rejected": -0.6422767639160156,
      "logps/chosen": -1325.8173828125,
      "logps/rejected": -872.7830810546875,
      "loss": 0.6838,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01688995398581028,
      "rewards/margins": 0.019739342853426933,
      "rewards/rejected": -0.0028493893332779408,
      "step": 3478
    },
    {
      "epoch": 0.9150898420161436,
      "grad_norm": 556.0,
      "learning_rate": 4.7076023391812866e-08,
      "logits/chosen": -0.6516299843788147,
      "logits/rejected": -0.6581024527549744,
      "logps/chosen": -1560.967529296875,
      "logps/rejected": -951.1961059570312,
      "loss": 0.6864,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0028625489212572575,
      "rewards/margins": 0.014503859914839268,
      "rewards/rejected": -0.011641312390565872,
      "step": 3479
    },
    {
      "epoch": 0.9153528744513307,
      "grad_norm": 488.0,
      "learning_rate": 4.6929824561403505e-08,
      "logits/chosen": -0.6541720628738403,
      "logits/rejected": -0.6650847792625427,
      "logps/chosen": -1434.1514892578125,
      "logps/rejected": -1000.2512817382812,
      "loss": 0.6906,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004056167788803577,
      "rewards/margins": 0.007034873589873314,
      "rewards/rejected": -0.002978706732392311,
      "step": 3480
    },
    {
      "epoch": 0.915615906886518,
      "grad_norm": 548.0,
      "learning_rate": 4.6783625730994145e-08,
      "logits/chosen": -0.6691577434539795,
      "logits/rejected": -0.6737751960754395,
      "logps/chosen": -1097.6309814453125,
      "logps/rejected": -795.2798461914062,
      "loss": 0.6916,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01251449529081583,
      "rewards/margins": 0.003739737905561924,
      "rewards/rejected": 0.00877475831657648,
      "step": 3481
    },
    {
      "epoch": 0.915878939321705,
      "grad_norm": 480.0,
      "learning_rate": 4.663742690058479e-08,
      "logits/chosen": -0.6667059063911438,
      "logits/rejected": -0.6652113199234009,
      "logps/chosen": -1233.4931640625,
      "logps/rejected": -834.3798828125,
      "loss": 0.6905,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007549955043941736,
      "rewards/margins": 0.006123875267803669,
      "rewards/rejected": 0.0014260767493396997,
      "step": 3482
    },
    {
      "epoch": 0.9161419717568923,
      "grad_norm": 600.0,
      "learning_rate": 4.649122807017544e-08,
      "logits/chosen": -0.6599374413490295,
      "logits/rejected": -0.6613249778747559,
      "logps/chosen": -1317.9136962890625,
      "logps/rejected": -917.3057861328125,
      "loss": 0.6955,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0013488763943314552,
      "rewards/margins": -0.003975868225097656,
      "rewards/rejected": 0.005324746016412973,
      "step": 3483
    },
    {
      "epoch": 0.9164050041920795,
      "grad_norm": 568.0,
      "learning_rate": 4.6345029239766084e-08,
      "logits/chosen": -0.638106644153595,
      "logits/rejected": NaN,
      "logps/chosen": -1210.369873046875,
      "logps/rejected": -665.1976318359375,
      "loss": 0.694,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0013077734038233757,
      "rewards/margins": -0.0013774872059002519,
      "rewards/rejected": 0.0026852607261389494,
      "step": 3484
    },
    {
      "epoch": 0.9166680366272666,
      "grad_norm": 592.0,
      "learning_rate": 4.6198830409356724e-08,
      "logits/chosen": -0.6539662480354309,
      "logits/rejected": NaN,
      "logps/chosen": -1288.4024658203125,
      "logps/rejected": -698.6084594726562,
      "loss": 0.6975,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008906221017241478,
      "rewards/margins": -0.008066605776548386,
      "rewards/rejected": -0.0008396140765398741,
      "step": 3485
    },
    {
      "epoch": 0.9169310690624538,
      "grad_norm": 708.0,
      "learning_rate": 4.6052631578947364e-08,
      "logits/chosen": -0.6451776623725891,
      "logits/rejected": NaN,
      "logps/chosen": -1539.2811279296875,
      "logps/rejected": -1386.387939453125,
      "loss": 0.6881,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007727050222456455,
      "rewards/margins": 0.012620735913515091,
      "rewards/rejected": -0.0048936838284134865,
      "step": 3486
    },
    {
      "epoch": 0.9171941014976409,
      "grad_norm": 498.0,
      "learning_rate": 4.590643274853801e-08,
      "logits/chosen": -0.6247484087944031,
      "logits/rejected": -0.6314512491226196,
      "logps/chosen": -1069.7861328125,
      "logps/rejected": -810.0946655273438,
      "loss": 0.6898,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.009874725714325905,
      "rewards/margins": 0.007336996495723724,
      "rewards/rejected": -0.01721172221004963,
      "step": 3487
    },
    {
      "epoch": 0.9174571339328281,
      "grad_norm": 700.0,
      "learning_rate": 4.576023391812865e-08,
      "logits/chosen": -0.630527138710022,
      "logits/rejected": -0.6459975242614746,
      "logps/chosen": -1243.4053955078125,
      "logps/rejected": -822.0938720703125,
      "loss": 0.6896,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.008922290056943893,
      "rewards/margins": 0.007996272295713425,
      "rewards/rejected": -0.016918564215302467,
      "step": 3488
    },
    {
      "epoch": 0.9177201663680152,
      "grad_norm": 592.0,
      "learning_rate": 4.5614035087719296e-08,
      "logits/chosen": -0.6452915668487549,
      "logits/rejected": -0.6414297223091125,
      "logps/chosen": -1246.51708984375,
      "logps/rejected": -1186.557373046875,
      "loss": 0.683,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.029920199885964394,
      "rewards/margins": 0.021027851849794388,
      "rewards/rejected": 0.008892346173524857,
      "step": 3489
    },
    {
      "epoch": 0.9179831988032024,
      "grad_norm": 652.0,
      "learning_rate": 4.546783625730994e-08,
      "logits/chosen": -0.6207660436630249,
      "logits/rejected": -0.6124374270439148,
      "logps/chosen": -1152.50244140625,
      "logps/rejected": -1030.7515869140625,
      "loss": 0.6868,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.007884025573730469,
      "rewards/margins": 0.013301515020430088,
      "rewards/rejected": -0.005417490378022194,
      "step": 3490
    },
    {
      "epoch": 0.9182462312383896,
      "grad_norm": 588.0,
      "learning_rate": 4.532163742690058e-08,
      "logits/chosen": -0.6606483459472656,
      "logits/rejected": -0.6560928821563721,
      "logps/chosen": -1404.06201171875,
      "logps/rejected": -1033.0640869140625,
      "loss": 0.675,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.01965484768152237,
      "rewards/margins": 0.037389468401670456,
      "rewards/rejected": -0.017734622582793236,
      "step": 3491
    },
    {
      "epoch": 0.9185092636735768,
      "grad_norm": 744.0,
      "learning_rate": 4.517543859649123e-08,
      "logits/chosen": -0.6501147150993347,
      "logits/rejected": -0.6606960296630859,
      "logps/chosen": -1561.843994140625,
      "logps/rejected": -1452.101318359375,
      "loss": 0.6931,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.01206207275390625,
      "rewards/margins": 0.0006262781098484993,
      "rewards/rejected": -0.012688351795077324,
      "step": 3492
    },
    {
      "epoch": 0.9187722961087639,
      "grad_norm": 552.0,
      "learning_rate": 4.502923976608187e-08,
      "logits/chosen": -0.6436847448348999,
      "logits/rejected": -0.6493408679962158,
      "logps/chosen": -1246.8746337890625,
      "logps/rejected": -1205.275634765625,
      "loss": 0.6999,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.011503029614686966,
      "rewards/margins": -0.012002754956483841,
      "rewards/rejected": 0.0004997255746275187,
      "step": 3493
    },
    {
      "epoch": 0.9190353285439511,
      "grad_norm": 648.0,
      "learning_rate": 4.4883040935672515e-08,
      "logits/chosen": -0.6560561060905457,
      "logits/rejected": -0.6554790139198303,
      "logps/chosen": -1314.805419921875,
      "logps/rejected": -1007.9675903320312,
      "loss": 0.6987,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.005167674273252487,
      "rewards/margins": -0.009677409194409847,
      "rewards/rejected": 0.004509735386818647,
      "step": 3494
    },
    {
      "epoch": 0.9192983609791382,
      "grad_norm": 478.0,
      "learning_rate": 4.4736842105263155e-08,
      "logits/chosen": -0.649517297744751,
      "logits/rejected": -0.6543390154838562,
      "logps/chosen": -1068.587646484375,
      "logps/rejected": -712.34228515625,
      "loss": 0.7044,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.010061835870146751,
      "rewards/margins": -0.020748615264892578,
      "rewards/rejected": 0.010686779394745827,
      "step": 3495
    },
    {
      "epoch": 0.9195613934143254,
      "grad_norm": 480.0,
      "learning_rate": 4.4590643274853795e-08,
      "logits/chosen": -0.6270099878311157,
      "logits/rejected": NaN,
      "logps/chosen": -909.9727783203125,
      "logps/rejected": -789.0153198242188,
      "loss": 0.697,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.010934448800981045,
      "rewards/margins": -0.007406425662338734,
      "rewards/rejected": 0.01834087260067463,
      "step": 3496
    },
    {
      "epoch": 0.9198244258495125,
      "grad_norm": 536.0,
      "learning_rate": 4.444444444444444e-08,
      "logits/chosen": -0.6300860643386841,
      "logits/rejected": -0.6246103048324585,
      "logps/chosen": -932.5971069335938,
      "logps/rejected": -1000.9440307617188,
      "loss": 0.7139,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.033109430223703384,
      "rewards/margins": -0.039957188069820404,
      "rewards/rejected": 0.006847763434052467,
      "step": 3497
    },
    {
      "epoch": 0.9200874582846997,
      "grad_norm": 414.0,
      "learning_rate": 4.429824561403509e-08,
      "logits/chosen": -0.6406261920928955,
      "logits/rejected": -0.6505134105682373,
      "logps/chosen": -757.1322021484375,
      "logps/rejected": -696.1787109375,
      "loss": 0.6947,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.003867196384817362,
      "rewards/margins": -0.0028921132907271385,
      "rewards/rejected": 0.0067593096755445,
      "step": 3498
    },
    {
      "epoch": 0.920350490719887,
      "grad_norm": 406.0,
      "learning_rate": 4.4152046783625734e-08,
      "logits/chosen": -0.6325331926345825,
      "logits/rejected": NaN,
      "logps/chosen": -564.6093139648438,
      "logps/rejected": -559.5171508789062,
      "loss": 0.6955,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0018476481782272458,
      "rewards/margins": -0.0043090349063277245,
      "rewards/rejected": 0.0024613861460238695,
      "step": 3499
    },
    {
      "epoch": 0.920613523155074,
      "grad_norm": 468.0,
      "learning_rate": 4.4005847953216373e-08,
      "logits/chosen": -0.6569591760635376,
      "logits/rejected": -0.6620717644691467,
      "logps/chosen": -1298.4190673828125,
      "logps/rejected": -986.3834838867188,
      "loss": 0.6885,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.006694411858916283,
      "rewards/margins": 0.009550858289003372,
      "rewards/rejected": -0.016245268285274506,
      "step": 3500
    },
    {
      "epoch": 0.9208765555902613,
      "grad_norm": 592.0,
      "learning_rate": 4.385964912280701e-08,
      "logits/chosen": -0.6403354406356812,
      "logits/rejected": -0.6561753749847412,
      "logps/chosen": -1260.90869140625,
      "logps/rejected": -999.398681640625,
      "loss": 0.6936,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.01139908004552126,
      "rewards/margins": 0.00016641570255160332,
      "rewards/rejected": 0.01123266201466322,
      "step": 3501
    },
    {
      "epoch": 0.9211395880254484,
      "grad_norm": 490.0,
      "learning_rate": 4.371345029239766e-08,
      "logits/chosen": -0.6491910219192505,
      "logits/rejected": -0.6466682553291321,
      "logps/chosen": -1232.2960205078125,
      "logps/rejected": -941.1094970703125,
      "loss": 0.6863,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.010585213080048561,
      "rewards/margins": 0.014504528604447842,
      "rewards/rejected": -0.025089740753173828,
      "step": 3502
    },
    {
      "epoch": 0.9214026204606356,
      "grad_norm": 494.0,
      "learning_rate": 4.35672514619883e-08,
      "logits/chosen": -0.648618221282959,
      "logits/rejected": -0.6474182605743408,
      "logps/chosen": -947.31298828125,
      "logps/rejected": -792.998291015625,
      "loss": 0.6962,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.006182956974953413,
      "rewards/margins": -0.005713080987334251,
      "rewards/rejected": -0.00046987493988126516,
      "step": 3503
    },
    {
      "epoch": 0.9216656528958227,
      "grad_norm": 540.0,
      "learning_rate": 4.3421052631578946e-08,
      "logits/chosen": -0.6551938652992249,
      "logits/rejected": -0.6471119523048401,
      "logps/chosen": -1261.748779296875,
      "logps/rejected": -1027.73193359375,
      "loss": 0.6794,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.022289276123046875,
      "rewards/margins": 0.028057096526026726,
      "rewards/rejected": -0.005767822265625,
      "step": 3504
    },
    {
      "epoch": 0.9219286853310099,
      "grad_norm": 588.0,
      "learning_rate": 4.327485380116959e-08,
      "logits/chosen": -0.6384057998657227,
      "logits/rejected": -0.637725830078125,
      "logps/chosen": -1223.154541015625,
      "logps/rejected": -883.8074951171875,
      "loss": 0.6967,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.004573059268295765,
      "rewards/margins": -0.006567574106156826,
      "rewards/rejected": 0.001994514837861061,
      "step": 3505
    },
    {
      "epoch": 0.922191717766197,
      "grad_norm": 502.0,
      "learning_rate": 4.312865497076023e-08,
      "logits/chosen": -0.6410529613494873,
      "logits/rejected": -0.6410629153251648,
      "logps/chosen": -1505.803955078125,
      "logps/rejected": -1173.229248046875,
      "loss": 0.6942,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0013446805533021688,
      "rewards/margins": -0.0014416697667911649,
      "rewards/rejected": 0.002786350669339299,
      "step": 3506
    },
    {
      "epoch": 0.9224547502013842,
      "grad_norm": 512.0,
      "learning_rate": 4.298245614035088e-08,
      "logits/chosen": -0.6482957601547241,
      "logits/rejected": -0.6770411133766174,
      "logps/chosen": -962.3346557617188,
      "logps/rejected": -609.4839477539062,
      "loss": 0.6967,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.017155267298221588,
      "rewards/margins": -0.006523419171571732,
      "rewards/rejected": -0.010631848126649857,
      "step": 3507
    },
    {
      "epoch": 0.9227177826365713,
      "grad_norm": 564.0,
      "learning_rate": 4.283625730994152e-08,
      "logits/chosen": -0.6592738628387451,
      "logits/rejected": -0.6674197912216187,
      "logps/chosen": -1096.51123046875,
      "logps/rejected": -975.037841796875,
      "loss": 0.7005,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.007945297285914421,
      "rewards/margins": -0.01229934673756361,
      "rewards/rejected": 0.004354048054665327,
      "step": 3508
    },
    {
      "epoch": 0.9229808150717586,
      "grad_norm": 556.0,
      "learning_rate": 4.2690058479532164e-08,
      "logits/chosen": -0.655340313911438,
      "logits/rejected": -0.6569402813911438,
      "logps/chosen": -1651.551025390625,
      "logps/rejected": -1332.9346923828125,
      "loss": 0.6894,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.006914997007697821,
      "rewards/margins": 0.008394097909331322,
      "rewards/rejected": -0.01530909538269043,
      "step": 3509
    },
    {
      "epoch": 0.9232438475069457,
      "grad_norm": 488.0,
      "learning_rate": 4.2543859649122804e-08,
      "logits/chosen": -0.6784421801567078,
      "logits/rejected": -0.6846999526023865,
      "logps/chosen": -823.0639038085938,
      "logps/rejected": -723.0499267578125,
      "loss": 0.6942,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.011634922586381435,
      "rewards/margins": -0.0014497768133878708,
      "rewards/rejected": 0.013084697537124157,
      "step": 3510
    },
    {
      "epoch": 0.9235068799421329,
      "grad_norm": 568.0,
      "learning_rate": 4.2397660818713444e-08,
      "logits/chosen": -0.6208233833312988,
      "logits/rejected": -0.6213954091072083,
      "logps/chosen": -1329.9189453125,
      "logps/rejected": -1033.283935546875,
      "loss": 0.6941,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.014218807220458984,
      "rewards/margins": -0.0006804461590945721,
      "rewards/rejected": -0.013538360595703125,
      "step": 3511
    },
    {
      "epoch": 0.92376991237732,
      "grad_norm": 728.0,
      "learning_rate": 4.225146198830409e-08,
      "logits/chosen": -0.6623427867889404,
      "logits/rejected": NaN,
      "logps/chosen": -1535.1749267578125,
      "logps/rejected": -773.6091918945312,
      "loss": 0.694,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.007249260321259499,
      "rewards/margins": -0.0012639043852686882,
      "rewards/rejected": -0.005985355470329523,
      "step": 3512
    },
    {
      "epoch": 0.9240329448125072,
      "grad_norm": 404.0,
      "learning_rate": 4.2105263157894737e-08,
      "logits/chosen": -0.6432838439941406,
      "logits/rejected": -0.6432971358299255,
      "logps/chosen": -871.55908203125,
      "logps/rejected": -670.91943359375,
      "loss": 0.6923,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.019702911376953125,
      "rewards/margins": 0.0019920356571674347,
      "rewards/rejected": 0.01771087758243084,
      "step": 3513
    },
    {
      "epoch": 0.9242959772476944,
      "grad_norm": 572.0,
      "learning_rate": 4.195906432748538e-08,
      "logits/chosen": -0.6460492610931396,
      "logits/rejected": -0.6490424871444702,
      "logps/chosen": -1372.8924560546875,
      "logps/rejected": -1103.8626708984375,
      "loss": 0.6935,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.01817188411951065,
      "rewards/margins": 0.00021400442346930504,
      "rewards/rejected": -0.018385887145996094,
      "step": 3514
    },
    {
      "epoch": 0.9245590096828815,
      "grad_norm": 500.0,
      "learning_rate": 4.181286549707602e-08,
      "logits/chosen": -0.6400216817855835,
      "logits/rejected": -0.6460124850273132,
      "logps/chosen": -1251.483642578125,
      "logps/rejected": -1132.32470703125,
      "loss": 0.7024,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.009454249404370785,
      "rewards/margins": -0.01793527603149414,
      "rewards/rejected": 0.008481025695800781,
      "step": 3515
    },
    {
      "epoch": 0.9248220421180687,
      "grad_norm": 640.0,
      "learning_rate": 4.166666666666666e-08,
      "logits/chosen": -0.6547168493270874,
      "logits/rejected": -0.6586366295814514,
      "logps/chosen": -1527.546875,
      "logps/rejected": -1101.77001953125,
      "loss": 0.7119,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.019457247108221054,
      "rewards/margins": -0.03608589246869087,
      "rewards/rejected": 0.016628647223114967,
      "step": 3516
    },
    {
      "epoch": 0.9250850745532558,
      "grad_norm": 7840.0,
      "learning_rate": 4.152046783625731e-08,
      "logits/chosen": NaN,
      "logits/rejected": -0.6531450748443604,
      "logps/chosen": -1138.218994140625,
      "logps/rejected": -1245.826416015625,
      "loss": 0.6889,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0015470520593225956,
      "rewards/margins": 0.009694862179458141,
      "rewards/rejected": -0.008147811517119408,
      "step": 3517
    },
    {
      "epoch": 0.925348106988443,
      "grad_norm": 648.0,
      "learning_rate": 4.137426900584795e-08,
      "logits/chosen": -0.616529643535614,
      "logits/rejected": -0.617911159992218,
      "logps/chosen": -1323.9425048828125,
      "logps/rejected": -1228.69140625,
      "loss": 0.697,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0031419750303030014,
      "rewards/margins": -0.006504630669951439,
      "rewards/rejected": 0.0033626556396484375,
      "step": 3518
    },
    {
      "epoch": 0.9256111394236302,
      "grad_norm": 560.0,
      "learning_rate": 4.1228070175438595e-08,
      "logits/chosen": -0.6388666033744812,
      "logits/rejected": -0.6503298878669739,
      "logps/chosen": -1232.069091796875,
      "logps/rejected": -926.751953125,
      "loss": 0.6847,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.022710515186190605,
      "rewards/margins": 0.019210627302527428,
      "rewards/rejected": 0.003499888814985752,
      "step": 3519
    },
    {
      "epoch": 0.9258741718588174,
      "grad_norm": 488.0,
      "learning_rate": 4.108187134502924e-08,
      "logits/chosen": -0.6375887393951416,
      "logits/rejected": -0.6366243958473206,
      "logps/chosen": -996.9891967773438,
      "logps/rejected": -1264.709228515625,
      "loss": 0.6897,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01937694475054741,
      "rewards/margins": 0.007460687309503555,
      "rewards/rejected": 0.01191625650972128,
      "step": 3520
    },
    {
      "epoch": 0.9261372042940045,
      "grad_norm": 540.0,
      "learning_rate": 4.093567251461988e-08,
      "logits/chosen": -0.6523405909538269,
      "logits/rejected": -0.6475331783294678,
      "logps/chosen": -896.8716430664062,
      "logps/rejected": -716.4595947265625,
      "loss": 0.6942,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.016576290130615234,
      "rewards/margins": -0.0019158360082656145,
      "rewards/rejected": 0.01849212683737278,
      "step": 3521
    },
    {
      "epoch": 0.9264002367291917,
      "grad_norm": 470.0,
      "learning_rate": 4.078947368421053e-08,
      "logits/chosen": -0.6507998704910278,
      "logits/rejected": -0.6514163017272949,
      "logps/chosen": -867.1123046875,
      "logps/rejected": -737.2111206054688,
      "loss": 0.6838,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0022676470689475536,
      "rewards/margins": 0.01926708035171032,
      "rewards/rejected": -0.016999436542391777,
      "step": 3522
    },
    {
      "epoch": 0.9266632691643788,
      "grad_norm": 516.0,
      "learning_rate": 4.064327485380117e-08,
      "logits/chosen": -0.667880117893219,
      "logits/rejected": -0.6650105714797974,
      "logps/chosen": -1146.814697265625,
      "logps/rejected": -827.4246215820312,
      "loss": 0.6938,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.004170132335275412,
      "rewards/margins": -0.0005574217066168785,
      "rewards/rejected": 0.004727554973214865,
      "step": 3523
    },
    {
      "epoch": 0.926926301599566,
      "grad_norm": 600.0,
      "learning_rate": 4.0497076023391814e-08,
      "logits/chosen": NaN,
      "logits/rejected": -0.6190497875213623,
      "logps/chosen": -1210.453369140625,
      "logps/rejected": -864.3591918945312,
      "loss": 0.6938,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.013087845407426357,
      "rewards/margins": 0.0003435146063566208,
      "rewards/rejected": -0.013431360013782978,
      "step": 3524
    },
    {
      "epoch": 0.9271893340347531,
      "grad_norm": 560.0,
      "learning_rate": 4.0350877192982454e-08,
      "logits/chosen": -0.649553656578064,
      "logits/rejected": -0.6438825726509094,
      "logps/chosen": -1131.7911376953125,
      "logps/rejected": -925.890380859375,
      "loss": 0.698,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0002707485109567642,
      "rewards/margins": -0.008755113929510117,
      "rewards/rejected": 0.009025860577821732,
      "step": 3525
    },
    {
      "epoch": 0.9274523664699403,
      "grad_norm": 516.0,
      "learning_rate": 4.020467836257309e-08,
      "logits/chosen": -0.6700380444526672,
      "logits/rejected": -0.6629108786582947,
      "logps/chosen": -1117.8148193359375,
      "logps/rejected": -705.48583984375,
      "loss": 0.6942,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.010302161797881126,
      "rewards/margins": -0.001426363131031394,
      "rewards/rejected": -0.008875800296664238,
      "step": 3526
    },
    {
      "epoch": 0.9277153989051274,
      "grad_norm": 580.0,
      "learning_rate": 4.005847953216374e-08,
      "logits/chosen": -0.6571382880210876,
      "logits/rejected": -0.6539669632911682,
      "logps/chosen": -1201.16943359375,
      "logps/rejected": -1085.5633544921875,
      "loss": 0.7045,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.01096658781170845,
      "rewards/margins": -0.02183542400598526,
      "rewards/rejected": 0.01086883619427681,
      "step": 3527
    },
    {
      "epoch": 0.9279784313403147,
      "grad_norm": 580.0,
      "learning_rate": 3.9912280701754386e-08,
      "logits/chosen": -0.6322513818740845,
      "logits/rejected": -0.6321525573730469,
      "logps/chosen": -1412.4515380859375,
      "logps/rejected": -1038.7745361328125,
      "loss": 0.6983,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0020519252866506577,
      "rewards/margins": -0.009350204840302467,
      "rewards/rejected": 0.007298279087990522,
      "step": 3528
    },
    {
      "epoch": 0.9282414637755018,
      "grad_norm": 556.0,
      "learning_rate": 3.976608187134503e-08,
      "logits/chosen": -0.6596602201461792,
      "logits/rejected": -0.6693388819694519,
      "logps/chosen": -856.8679809570312,
      "logps/rejected": -924.306884765625,
      "loss": 0.6966,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0019154548645019531,
      "rewards/margins": -0.005755901336669922,
      "rewards/rejected": 0.007671356201171875,
      "step": 3529
    },
    {
      "epoch": 0.928504496210689,
      "grad_norm": 548.0,
      "learning_rate": 3.961988304093567e-08,
      "logits/chosen": -0.6571451425552368,
      "logits/rejected": -0.6718400120735168,
      "logps/chosen": -963.9208374023438,
      "logps/rejected": -801.6516723632812,
      "loss": 0.6862,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.015983104705810547,
      "rewards/margins": 0.015075779519975185,
      "rewards/rejected": 0.0009073261171579361,
      "step": 3530
    },
    {
      "epoch": 0.9287675286458762,
      "grad_norm": 506.0,
      "learning_rate": 3.947368421052631e-08,
      "logits/chosen": -0.6483870148658752,
      "logits/rejected": -0.6425324082374573,
      "logps/chosen": -1229.28125,
      "logps/rejected": -862.3259887695312,
      "loss": 0.6892,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.023008158430457115,
      "rewards/margins": 0.008882141672074795,
      "rewards/rejected": 0.014126014895737171,
      "step": 3531
    },
    {
      "epoch": 0.9290305610810633,
      "grad_norm": 528.0,
      "learning_rate": 3.932748538011696e-08,
      "logits/chosen": NaN,
      "logits/rejected": -0.6730257272720337,
      "logps/chosen": -914.26953125,
      "logps/rejected": -1128.4637451171875,
      "loss": 0.6878,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006924630142748356,
      "rewards/margins": 0.011463547125458717,
      "rewards/rejected": -0.004538917914032936,
      "step": 3532
    },
    {
      "epoch": 0.9292935935162505,
      "grad_norm": 502.0,
      "learning_rate": 3.91812865497076e-08,
      "logits/chosen": -0.6448849439620972,
      "logits/rejected": -0.6372498869895935,
      "logps/chosen": -1157.9705810546875,
      "logps/rejected": -639.1945190429688,
      "loss": 0.6919,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0006355298683047295,
      "rewards/margins": 0.0029779444448649883,
      "rewards/rejected": -0.0036134719848632812,
      "step": 3533
    },
    {
      "epoch": 0.9295566259514376,
      "grad_norm": 516.0,
      "learning_rate": 3.9035087719298244e-08,
      "logits/chosen": -0.6317614316940308,
      "logits/rejected": -0.6358281970024109,
      "logps/chosen": -1287.6170654296875,
      "logps/rejected": -1163.573486328125,
      "loss": 0.6836,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01593952067196369,
      "rewards/margins": 0.020308686420321465,
      "rewards/rejected": -0.004369164817035198,
      "step": 3534
    },
    {
      "epoch": 0.9298196583866248,
      "grad_norm": 552.0,
      "learning_rate": 3.888888888888889e-08,
      "logits/chosen": -0.6624641418457031,
      "logits/rejected": -0.659186601638794,
      "logps/chosen": -1030.9385986328125,
      "logps/rejected": -902.978515625,
      "loss": 0.6947,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.010949993506073952,
      "rewards/margins": -0.0025606146082282066,
      "rewards/rejected": -0.008389377035200596,
      "step": 3535
    },
    {
      "epoch": 0.9300826908218119,
      "grad_norm": 556.0,
      "learning_rate": 3.874269005847953e-08,
      "logits/chosen": -0.6662799715995789,
      "logits/rejected": -0.6753017902374268,
      "logps/chosen": -1180.2977294921875,
      "logps/rejected": -1054.760986328125,
      "loss": 0.6907,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.003436849918216467,
      "rewards/margins": 0.0058650970458984375,
      "rewards/rejected": -0.0024282459635287523,
      "step": 3536
    },
    {
      "epoch": 0.9303457232569992,
      "grad_norm": 528.0,
      "learning_rate": 3.859649122807018e-08,
      "logits/chosen": -0.6458056569099426,
      "logits/rejected": -0.6454949378967285,
      "logps/chosen": -1122.4482421875,
      "logps/rejected": -1022.568603515625,
      "loss": 0.6911,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004679012577980757,
      "rewards/margins": 0.004787635989487171,
      "rewards/rejected": -0.0001086237607523799,
      "step": 3537
    },
    {
      "epoch": 0.9306087556921863,
      "grad_norm": 532.0,
      "learning_rate": 3.845029239766082e-08,
      "logits/chosen": -0.6297983527183533,
      "logits/rejected": -0.637830376625061,
      "logps/chosen": -1117.141357421875,
      "logps/rejected": -1105.255859375,
      "loss": 0.6832,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.022834017872810364,
      "rewards/margins": 0.020437147468328476,
      "rewards/rejected": 0.0023968685418367386,
      "step": 3538
    },
    {
      "epoch": 0.9308717881273735,
      "grad_norm": 520.0,
      "learning_rate": 3.830409356725146e-08,
      "logits/chosen": -0.6550691723823547,
      "logits/rejected": -0.6617773175239563,
      "logps/chosen": -1097.5804443359375,
      "logps/rejected": -842.7100830078125,
      "loss": 0.6981,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.00013198889791965485,
      "rewards/margins": -0.008992004208266735,
      "rewards/rejected": 0.00886001531034708,
      "step": 3539
    },
    {
      "epoch": 0.9311348205625606,
      "grad_norm": 528.0,
      "learning_rate": 3.81578947368421e-08,
      "logits/chosen": -0.6630775928497314,
      "logits/rejected": -0.6583985090255737,
      "logps/chosen": -1200.019775390625,
      "logps/rejected": -727.4110717773438,
      "loss": 0.6871,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0025232313200831413,
      "rewards/margins": 0.013276956975460052,
      "rewards/rejected": -0.010753725655376911,
      "step": 3540
    },
    {
      "epoch": 0.9313978529977478,
      "grad_norm": 536.0,
      "learning_rate": 3.801169590643274e-08,
      "logits/chosen": -0.6732129454612732,
      "logits/rejected": -0.6807814836502075,
      "logps/chosen": -1140.8360595703125,
      "logps/rejected": -998.4892578125,
      "loss": 0.7149,
      "rewards/accuracies": 0.0625,
      "rewards/chosen": -0.0485108345746994,
      "rewards/margins": -0.042411044239997864,
      "rewards/rejected": -0.006099795922636986,
      "step": 3541
    },
    {
      "epoch": 0.9316608854329349,
      "grad_norm": 510.0,
      "learning_rate": 3.786549707602339e-08,
      "logits/chosen": -0.6501483917236328,
      "logits/rejected": -0.6583142280578613,
      "logps/chosen": -1166.38232421875,
      "logps/rejected": -959.5213623046875,
      "loss": 0.6948,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.00879516638815403,
      "rewards/margins": -0.002902508247643709,
      "rewards/rejected": -0.005892658606171608,
      "step": 3542
    },
    {
      "epoch": 0.9319239178681221,
      "grad_norm": 414.0,
      "learning_rate": 3.7719298245614035e-08,
      "logits/chosen": -0.639336347579956,
      "logits/rejected": -0.6476901769638062,
      "logps/chosen": -1063.9888916015625,
      "logps/rejected": -766.8826293945312,
      "loss": 0.6849,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.019159890711307526,
      "rewards/margins": 0.017269421368837357,
      "rewards/rejected": 0.0018904677126556635,
      "step": 3543
    },
    {
      "epoch": 0.9321869503033092,
      "grad_norm": 652.0,
      "learning_rate": 3.757309941520468e-08,
      "logits/chosen": -0.6614826917648315,
      "logits/rejected": -0.6625419855117798,
      "logps/chosen": -1366.14111328125,
      "logps/rejected": -1149.3846435546875,
      "loss": 0.6987,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.008894920349121094,
      "rewards/margins": -0.0097992904484272,
      "rewards/rejected": 0.0009043694008141756,
      "step": 3544
    },
    {
      "epoch": 0.9324499827384964,
      "grad_norm": 442.0,
      "learning_rate": 3.742690058479532e-08,
      "logits/chosen": -0.6182335019111633,
      "logits/rejected": -0.6231035590171814,
      "logps/chosen": -825.02978515625,
      "logps/rejected": -682.9219360351562,
      "loss": 0.6953,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.008209705352783203,
      "rewards/margins": -0.003881740150973201,
      "rewards/rejected": 0.012091446667909622,
      "step": 3545
    },
    {
      "epoch": 0.9327130151736837,
      "grad_norm": 524.0,
      "learning_rate": 3.728070175438596e-08,
      "logits/chosen": -0.6432593464851379,
      "logits/rejected": -0.6472660899162292,
      "logps/chosen": -1357.2293701171875,
      "logps/rejected": -942.3654174804688,
      "loss": 0.7082,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0029697425197809935,
      "rewards/margins": -0.02879820019006729,
      "rewards/rejected": 0.02582845650613308,
      "step": 3546
    },
    {
      "epoch": 0.9329760476088708,
      "grad_norm": 628.0,
      "learning_rate": 3.713450292397661e-08,
      "logits/chosen": -0.6367866396903992,
      "logits/rejected": -0.6449475288391113,
      "logps/chosen": -1640.251220703125,
      "logps/rejected": -1325.520751953125,
      "loss": 0.684,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.023447230458259583,
      "rewards/margins": 0.019488241523504257,
      "rewards/rejected": 0.003958990331739187,
      "step": 3547
    },
    {
      "epoch": 0.933239080044058,
      "grad_norm": 716.0,
      "learning_rate": 3.698830409356725e-08,
      "logits/chosen": -0.655692458152771,
      "logits/rejected": -0.6644883155822754,
      "logps/chosen": -768.7650756835938,
      "logps/rejected": -726.3744506835938,
      "loss": 0.6996,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.008035469800233841,
      "rewards/margins": -0.012351131066679955,
      "rewards/rejected": 0.004315662197768688,
      "step": 3548
    },
    {
      "epoch": 0.9335021124792451,
      "grad_norm": 600.0,
      "learning_rate": 3.684210526315789e-08,
      "logits/chosen": -0.6362980008125305,
      "logits/rejected": NaN,
      "logps/chosen": -1517.2261962890625,
      "logps/rejected": -1100.27392578125,
      "loss": 0.7053,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.00011939927935600281,
      "rewards/margins": -0.022550297901034355,
      "rewards/rejected": 0.022430894896388054,
      "step": 3549
    },
    {
      "epoch": 0.9337651449144323,
      "grad_norm": 552.0,
      "learning_rate": 3.669590643274854e-08,
      "logits/chosen": -0.6514893770217896,
      "logits/rejected": -0.646170973777771,
      "logps/chosen": -771.5986938476562,
      "logps/rejected": -1042.4249267578125,
      "loss": 0.6898,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0005727771203964949,
      "rewards/margins": 0.006816101260483265,
      "rewards/rejected": -0.006243324838578701,
      "step": 3550
    },
    {
      "epoch": 0.9340281773496194,
      "grad_norm": 500.0,
      "learning_rate": 3.654970760233918e-08,
      "logits/chosen": -0.6577693223953247,
      "logits/rejected": -0.6673212051391602,
      "logps/chosen": -1144.27978515625,
      "logps/rejected": -721.2797241210938,
      "loss": 0.6886,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.005749702453613281,
      "rewards/margins": 0.009717082604765892,
      "rewards/rejected": -0.003967380616813898,
      "step": 3551
    },
    {
      "epoch": 0.9342912097848066,
      "grad_norm": 532.0,
      "learning_rate": 3.6403508771929826e-08,
      "logits/chosen": -0.6514194011688232,
      "logits/rejected": -0.6563311219215393,
      "logps/chosen": -1087.517822265625,
      "logps/rejected": -789.0596923828125,
      "loss": 0.703,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.02344217337667942,
      "rewards/margins": -0.019128896296024323,
      "rewards/rejected": -0.004313278943300247,
      "step": 3552
    },
    {
      "epoch": 0.9345542422199937,
      "grad_norm": 648.0,
      "learning_rate": 3.6257309941520466e-08,
      "logits/chosen": -0.6478517651557922,
      "logits/rejected": -0.6549041271209717,
      "logps/chosen": -1282.811279296875,
      "logps/rejected": -1264.9505615234375,
      "loss": 0.6818,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.01090707816183567,
      "rewards/margins": 0.02414255030453205,
      "rewards/rejected": -0.013235473074018955,
      "step": 3553
    },
    {
      "epoch": 0.9348172746551809,
      "grad_norm": 498.0,
      "learning_rate": 3.6111111111111106e-08,
      "logits/chosen": -0.6328191757202148,
      "logits/rejected": -0.6356954574584961,
      "logps/chosen": -917.7122192382812,
      "logps/rejected": -675.0130615234375,
      "loss": 0.6853,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.017266273498535156,
      "rewards/margins": 0.01628885045647621,
      "rewards/rejected": 0.0009774214122444391,
      "step": 3554
    },
    {
      "epoch": 0.935080307090368,
      "grad_norm": 506.0,
      "learning_rate": 3.596491228070175e-08,
      "logits/chosen": -0.651828944683075,
      "logits/rejected": -0.6690890192985535,
      "logps/chosen": -1097.2064208984375,
      "logps/rejected": -1060.30908203125,
      "loss": 0.6926,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.021245956420898438,
      "rewards/margins": 0.001665162038989365,
      "rewards/rejected": -0.02291111648082733,
      "step": 3555
    },
    {
      "epoch": 0.9353433395255553,
      "grad_norm": 492.0,
      "learning_rate": 3.581871345029239e-08,
      "logits/chosen": -0.6492640972137451,
      "logits/rejected": -0.6508204340934753,
      "logps/chosen": -1081.705078125,
      "logps/rejected": -863.026123046875,
      "loss": 0.6897,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00386619521304965,
      "rewards/margins": 0.007111692801117897,
      "rewards/rejected": -0.01097788941115141,
      "step": 3556
    },
    {
      "epoch": 0.9356063719607424,
      "grad_norm": 604.0,
      "learning_rate": 3.567251461988304e-08,
      "logits/chosen": -0.6678160429000854,
      "logits/rejected": -0.6571998000144958,
      "logps/chosen": -1407.4111328125,
      "logps/rejected": -1002.6168212890625,
      "loss": 0.6947,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.020519161596894264,
      "rewards/margins": -0.0018210415728390217,
      "rewards/rejected": 0.022340202704072,
      "step": 3557
    },
    {
      "epoch": 0.9358694043959296,
      "grad_norm": 648.0,
      "learning_rate": 3.5526315789473685e-08,
      "logits/chosen": -0.6824180483818054,
      "logits/rejected": -0.6770067811012268,
      "logps/chosen": -1420.859375,
      "logps/rejected": -1157.64501953125,
      "loss": 0.6881,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.02114696428179741,
      "rewards/margins": 0.010999679565429688,
      "rewards/rejected": 0.010147285647690296,
      "step": 3558
    },
    {
      "epoch": 0.9361324368311167,
      "grad_norm": 506.0,
      "learning_rate": 3.5380116959064324e-08,
      "logits/chosen": -0.6305007934570312,
      "logits/rejected": -0.625495970249176,
      "logps/chosen": -1040.58251953125,
      "logps/rejected": -731.08642578125,
      "loss": 0.6873,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.020233821123838425,
      "rewards/margins": 0.012640764005482197,
      "rewards/rejected": 0.007593059912323952,
      "step": 3559
    },
    {
      "epoch": 0.9363954692663039,
      "grad_norm": 508.0,
      "learning_rate": 3.523391812865497e-08,
      "logits/chosen": -0.6227651834487915,
      "logits/rejected": -0.6416245698928833,
      "logps/chosen": -1515.291015625,
      "logps/rejected": -1105.07470703125,
      "loss": 0.704,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.04853105545043945,
      "rewards/margins": -0.02098817750811577,
      "rewards/rejected": -0.027542876079678535,
      "step": 3560
    },
    {
      "epoch": 0.9366585017014911,
      "grad_norm": 380.0,
      "learning_rate": 3.508771929824561e-08,
      "logits/chosen": -0.6680868268013,
      "logits/rejected": -0.6690022945404053,
      "logps/chosen": -927.7781982421875,
      "logps/rejected": -676.7214965820312,
      "loss": 0.7005,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.009972667321562767,
      "rewards/margins": -0.013851545751094818,
      "rewards/rejected": 0.0038788795936852694,
      "step": 3561
    },
    {
      "epoch": 0.9369215341366782,
      "grad_norm": 616.0,
      "learning_rate": 3.494152046783626e-08,
      "logits/chosen": -0.6630189418792725,
      "logits/rejected": -0.6542893648147583,
      "logps/chosen": -1315.984375,
      "logps/rejected": -883.3235473632812,
      "loss": 0.7072,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.020028498023748398,
      "rewards/margins": -0.027154304087162018,
      "rewards/rejected": 0.00712580606341362,
      "step": 3562
    },
    {
      "epoch": 0.9371845665718654,
      "grad_norm": 652.0,
      "learning_rate": 3.47953216374269e-08,
      "logits/chosen": -0.6527445316314697,
      "logits/rejected": -0.6491957306861877,
      "logps/chosen": -1490.396484375,
      "logps/rejected": -1143.4420166015625,
      "loss": 0.683,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.009959603659808636,
      "rewards/margins": 0.021378040313720703,
      "rewards/rejected": -0.011418438516557217,
      "step": 3563
    },
    {
      "epoch": 0.9374475990070525,
      "grad_norm": 432.0,
      "learning_rate": 3.4649122807017536e-08,
      "logits/chosen": -0.6583700180053711,
      "logits/rejected": -0.6645134687423706,
      "logps/chosen": -803.051025390625,
      "logps/rejected": -707.102783203125,
      "loss": 0.6881,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01270828116685152,
      "rewards/margins": 0.010301018133759499,
      "rewards/rejected": 0.002407264895737171,
      "step": 3564
    },
    {
      "epoch": 0.9377106314422398,
      "grad_norm": 620.0,
      "learning_rate": 3.450292397660819e-08,
      "logits/chosen": -0.6512751579284668,
      "logits/rejected": -0.65968257188797,
      "logps/chosen": -1224.8861083984375,
      "logps/rejected": -837.40625,
      "loss": 0.6856,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.020142747089266777,
      "rewards/margins": 0.01570596545934677,
      "rewards/rejected": 0.004436779301613569,
      "step": 3565
    },
    {
      "epoch": 0.9379736638774269,
      "grad_norm": 512.0,
      "learning_rate": 3.435672514619883e-08,
      "logits/chosen": -0.6473593711853027,
      "logits/rejected": -0.6436243653297424,
      "logps/chosen": -903.5310668945312,
      "logps/rejected": -732.2176513671875,
      "loss": 0.6941,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0025683408603072166,
      "rewards/margins": -0.0014636993873864412,
      "rewards/rejected": -0.0011046414729207754,
      "step": 3566
    },
    {
      "epoch": 0.9382366963126141,
      "grad_norm": 500.0,
      "learning_rate": 3.4210526315789476e-08,
      "logits/chosen": -0.6571664810180664,
      "logits/rejected": -0.6696875691413879,
      "logps/chosen": -1350.871826171875,
      "logps/rejected": -946.6503295898438,
      "loss": 0.6915,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.002713393419981003,
      "rewards/margins": 0.003910731058567762,
      "rewards/rejected": -0.0011973390355706215,
      "step": 3567
    },
    {
      "epoch": 0.9384997287478012,
      "grad_norm": 596.0,
      "learning_rate": 3.4064327485380115e-08,
      "logits/chosen": -0.6447141170501709,
      "logits/rejected": -0.647674024105072,
      "logps/chosen": -1533.7645263671875,
      "logps/rejected": -1102.9727783203125,
      "loss": 0.695,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01894512213766575,
      "rewards/margins": -0.002906227018684149,
      "rewards/rejected": -0.016038894653320312,
      "step": 3568
    },
    {
      "epoch": 0.9387627611829884,
      "grad_norm": 624.0,
      "learning_rate": 3.3918128654970755e-08,
      "logits/chosen": -0.6492319703102112,
      "logits/rejected": -0.6458885669708252,
      "logps/chosen": -1112.7828369140625,
      "logps/rejected": -849.0638427734375,
      "loss": 0.6972,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.007483960594981909,
      "rewards/margins": -0.006812571547925472,
      "rewards/rejected": -0.00067138671875,
      "step": 3569
    },
    {
      "epoch": 0.9390257936181755,
      "grad_norm": 528.0,
      "learning_rate": 3.37719298245614e-08,
      "logits/chosen": -0.6637151837348938,
      "logits/rejected": -0.6588206887245178,
      "logps/chosen": -1273.94873046875,
      "logps/rejected": -1225.5152587890625,
      "loss": 0.6959,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.009100244380533695,
      "rewards/margins": -0.0046441080048680305,
      "rewards/rejected": -0.004456139635294676,
      "step": 3570
    },
    {
      "epoch": 0.9392888260533627,
      "grad_norm": 580.0,
      "learning_rate": 3.362573099415204e-08,
      "logits/chosen": -0.6699294447898865,
      "logits/rejected": -0.6700941920280457,
      "logps/chosen": -1310.160888671875,
      "logps/rejected": -952.6423950195312,
      "loss": 0.7062,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.00016651139594614506,
      "rewards/margins": -0.025232600048184395,
      "rewards/rejected": 0.025399111211299896,
      "step": 3571
    },
    {
      "epoch": 0.9395518584885498,
      "grad_norm": 420.0,
      "learning_rate": 3.347953216374269e-08,
      "logits/chosen": -0.6432759165763855,
      "logits/rejected": -0.648097574710846,
      "logps/chosen": -1000.5987548828125,
      "logps/rejected": -784.6714477539062,
      "loss": 0.6953,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.006503010168671608,
      "rewards/margins": -0.003597545437514782,
      "rewards/rejected": -0.002905464032664895,
      "step": 3572
    },
    {
      "epoch": 0.939814890923737,
      "grad_norm": 588.0,
      "learning_rate": 3.3333333333333334e-08,
      "logits/chosen": -0.6475728750228882,
      "logits/rejected": -0.6554844379425049,
      "logps/chosen": -1261.299072265625,
      "logps/rejected": -1220.870361328125,
      "loss": 0.6892,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0030107495840638876,
      "rewards/margins": 0.008141137659549713,
      "rewards/rejected": -0.005130387376993895,
      "step": 3573
    },
    {
      "epoch": 0.9400779233589242,
      "grad_norm": 600.0,
      "learning_rate": 3.3187134502923974e-08,
      "logits/chosen": -0.6406254768371582,
      "logits/rejected": -0.6461086273193359,
      "logps/chosen": -1251.0020751953125,
      "logps/rejected": -1043.0394287109375,
      "loss": 0.6975,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00953454989939928,
      "rewards/margins": -0.007778074126690626,
      "rewards/rejected": -0.001756476005539298,
      "step": 3574
    },
    {
      "epoch": 0.9403409557941114,
      "grad_norm": 1360.0,
      "learning_rate": 3.304093567251462e-08,
      "logits/chosen": -0.6284709572792053,
      "logits/rejected": NaN,
      "logps/chosen": -900.2763061523438,
      "logps/rejected": -809.6604614257812,
      "loss": 0.6904,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003771686926484108,
      "rewards/margins": 0.0058441162109375,
      "rewards/rejected": -0.0020724297501146793,
      "step": 3575
    },
    {
      "epoch": 0.9406039882292985,
      "grad_norm": 484.0,
      "learning_rate": 3.289473684210526e-08,
      "logits/chosen": -0.6687441468238831,
      "logits/rejected": -0.6690748929977417,
      "logps/chosen": -1257.3536376953125,
      "logps/rejected": -949.1014404296875,
      "loss": 0.6903,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.008952331729233265,
      "rewards/margins": 0.006027603521943092,
      "rewards/rejected": -0.014979934319853783,
      "step": 3576
    },
    {
      "epoch": 0.9408670206644857,
      "grad_norm": 396.0,
      "learning_rate": 3.2748538011695906e-08,
      "logits/chosen": -0.6224753260612488,
      "logits/rejected": -0.6287843585014343,
      "logps/chosen": -740.6133422851562,
      "logps/rejected": -485.0835876464844,
      "loss": 0.6912,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.007225608918815851,
      "rewards/margins": 0.0045236581936478615,
      "rewards/rejected": 0.002701951190829277,
      "step": 3577
    },
    {
      "epoch": 0.9411300530996729,
      "grad_norm": 484.0,
      "learning_rate": 3.2602339181286546e-08,
      "logits/chosen": -0.6494814157485962,
      "logits/rejected": -0.6572360992431641,
      "logps/chosen": -1051.8995361328125,
      "logps/rejected": -1123.8692626953125,
      "loss": 0.7037,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.005039215087890625,
      "rewards/margins": -0.02049112319946289,
      "rewards/rejected": 0.025530336424708366,
      "step": 3578
    },
    {
      "epoch": 0.94139308553486,
      "grad_norm": 504.0,
      "learning_rate": 3.2456140350877186e-08,
      "logits/chosen": -0.6531258225440979,
      "logits/rejected": -0.6390085816383362,
      "logps/chosen": -1416.4261474609375,
      "logps/rejected": -1031.471923828125,
      "loss": 0.6977,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.007444287184625864,
      "rewards/margins": -0.00847472995519638,
      "rewards/rejected": 0.0010304450988769531,
      "step": 3579
    },
    {
      "epoch": 0.9416561179700472,
      "grad_norm": 588.0,
      "learning_rate": 3.230994152046784e-08,
      "logits/chosen": -0.641931414604187,
      "logits/rejected": -0.6604354977607727,
      "logps/chosen": -1414.833984375,
      "logps/rejected": -979.5964965820312,
      "loss": 0.6955,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.011310767382383347,
      "rewards/margins": -0.0034116737078875303,
      "rewards/rejected": 0.014722442254424095,
      "step": 3580
    },
    {
      "epoch": 0.9419191504052343,
      "grad_norm": 632.0,
      "learning_rate": 3.216374269005848e-08,
      "logits/chosen": -0.6634135842323303,
      "logits/rejected": -0.6495901346206665,
      "logps/chosen": -1256.598388671875,
      "logps/rejected": -1034.655029296875,
      "loss": 0.6836,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02187786065042019,
      "rewards/margins": 0.02002887614071369,
      "rewards/rejected": 0.001848984626121819,
      "step": 3581
    },
    {
      "epoch": 0.9421821828404215,
      "grad_norm": 502.0,
      "learning_rate": 3.2017543859649125e-08,
      "logits/chosen": -0.6534737944602966,
      "logits/rejected": -0.6673213839530945,
      "logps/chosen": -1185.333740234375,
      "logps/rejected": -905.5267333984375,
      "loss": 0.6943,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0011466015130281448,
      "rewards/margins": -0.0017725944053381681,
      "rewards/rejected": 0.000625991728156805,
      "step": 3582
    },
    {
      "epoch": 0.9424452152756086,
      "grad_norm": 556.0,
      "learning_rate": 3.1871345029239765e-08,
      "logits/chosen": -0.6184487342834473,
      "logits/rejected": -0.6247429251670837,
      "logps/chosen": -1060.9593505859375,
      "logps/rejected": -617.1870727539062,
      "loss": 0.6836,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.015069389715790749,
      "rewards/margins": 0.020033597946166992,
      "rewards/rejected": -0.004964209161698818,
      "step": 3583
    },
    {
      "epoch": 0.9427082477107959,
      "grad_norm": 608.0,
      "learning_rate": 3.1725146198830404e-08,
      "logits/chosen": -0.6373208165168762,
      "logits/rejected": -0.6471738815307617,
      "logps/chosen": -1280.1644287109375,
      "logps/rejected": -879.8443603515625,
      "loss": 0.7191,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.03991737589240074,
      "rewards/margins": -0.04983111098408699,
      "rewards/rejected": 0.00991373136639595,
      "step": 3584
    },
    {
      "epoch": 0.942971280145983,
      "grad_norm": 468.0,
      "learning_rate": 3.157894736842105e-08,
      "logits/chosen": -0.6625751852989197,
      "logits/rejected": -0.6605185270309448,
      "logps/chosen": -994.1069946289062,
      "logps/rejected": -861.438232421875,
      "loss": 0.6934,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.014943505637347698,
      "rewards/margins": -3.1519681215286255e-05,
      "rewards/rejected": 0.01497502438724041,
      "step": 3585
    },
    {
      "epoch": 0.9432343125811702,
      "grad_norm": 508.0,
      "learning_rate": 3.143274853801169e-08,
      "logits/chosen": -0.6465138792991638,
      "logits/rejected": -0.6466808319091797,
      "logps/chosen": -1259.85498046875,
      "logps/rejected": -1241.07666015625,
      "loss": 0.6862,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.013419819064438343,
      "rewards/margins": 0.014727113768458366,
      "rewards/rejected": -0.0013072974979877472,
      "step": 3586
    },
    {
      "epoch": 0.9434973450163573,
      "grad_norm": 620.0,
      "learning_rate": 3.128654970760234e-08,
      "logits/chosen": -0.6382322907447815,
      "logits/rejected": -0.6628673076629639,
      "logps/chosen": -1714.1473388671875,
      "logps/rejected": -1370.6046142578125,
      "loss": 0.6928,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.019348716363310814,
      "rewards/margins": 0.0037038822192698717,
      "rewards/rejected": 0.01564483530819416,
      "step": 3587
    },
    {
      "epoch": 0.9437603774515445,
      "grad_norm": 466.0,
      "learning_rate": 3.1140350877192983e-08,
      "logits/chosen": -0.6450942754745483,
      "logits/rejected": -0.6432896852493286,
      "logps/chosen": -1313.6087646484375,
      "logps/rejected": -994.651123046875,
      "loss": 0.698,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.033516980707645416,
      "rewards/margins": -0.009447766467928886,
      "rewards/rejected": -0.02406921423971653,
      "step": 3588
    },
    {
      "epoch": 0.9440234098867316,
      "grad_norm": 502.0,
      "learning_rate": 3.099415204678362e-08,
      "logits/chosen": -0.6492279171943665,
      "logits/rejected": -0.6524342894554138,
      "logps/chosen": -1163.5433349609375,
      "logps/rejected": -968.0276489257812,
      "loss": 0.7021,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03024750016629696,
      "rewards/margins": -0.017342377454042435,
      "rewards/rejected": -0.012905120849609375,
      "step": 3589
    },
    {
      "epoch": 0.9442864423219188,
      "grad_norm": 476.0,
      "learning_rate": 3.084795321637427e-08,
      "logits/chosen": -0.6431664228439331,
      "logits/rejected": -0.6608923673629761,
      "logps/chosen": -1221.02978515625,
      "logps/rejected": -911.6732788085938,
      "loss": 0.6947,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01191578060388565,
      "rewards/margins": -0.002476023742929101,
      "rewards/rejected": 0.014391804113984108,
      "step": 3590
    },
    {
      "epoch": 0.9445494747571059,
      "grad_norm": 510.0,
      "learning_rate": 3.070175438596491e-08,
      "logits/chosen": -0.655609130859375,
      "logits/rejected": -0.6582940816879272,
      "logps/chosen": -803.2047729492188,
      "logps/rejected": -905.3411865234375,
      "loss": 0.6944,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0049772243946790695,
      "rewards/margins": -0.0016668315511196852,
      "rewards/rejected": -0.0033103935420513153,
      "step": 3591
    },
    {
      "epoch": 0.9448125071922931,
      "grad_norm": 494.0,
      "learning_rate": 3.0555555555555556e-08,
      "logits/chosen": -0.6423758268356323,
      "logits/rejected": -0.6408917307853699,
      "logps/chosen": -758.80810546875,
      "logps/rejected": -807.6687622070312,
      "loss": 0.6983,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.004472065716981888,
      "rewards/margins": -0.009341049008071423,
      "rewards/rejected": 0.004868984222412109,
      "step": 3592
    },
    {
      "epoch": 0.9450755396274804,
      "grad_norm": 708.0,
      "learning_rate": 3.0409356725146195e-08,
      "logits/chosen": -0.6730947494506836,
      "logits/rejected": NaN,
      "logps/chosen": -1203.162841796875,
      "logps/rejected": -862.4017333984375,
      "loss": 0.7003,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0030773174948990345,
      "rewards/margins": -0.0131378173828125,
      "rewards/rejected": 0.016215132549405098,
      "step": 3593
    },
    {
      "epoch": 0.9453385720626675,
      "grad_norm": 488.0,
      "learning_rate": 3.026315789473684e-08,
      "logits/chosen": -0.6728273630142212,
      "logits/rejected": -0.6554258465766907,
      "logps/chosen": -922.2473754882812,
      "logps/rejected": -742.1185913085938,
      "loss": 0.6884,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.020341014489531517,
      "rewards/margins": 0.01053762435913086,
      "rewards/rejected": 0.009803390130400658,
      "step": 3594
    },
    {
      "epoch": 0.9456016044978547,
      "grad_norm": 532.0,
      "learning_rate": 3.011695906432749e-08,
      "logits/chosen": -0.6630479097366333,
      "logits/rejected": -0.668860673904419,
      "logps/chosen": -1101.30517578125,
      "logps/rejected": -1007.612548828125,
      "loss": 0.6914,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0020483010448515415,
      "rewards/margins": 0.003876495873555541,
      "rewards/rejected": -0.005924796685576439,
      "step": 3595
    },
    {
      "epoch": 0.9458646369330418,
      "grad_norm": 490.0,
      "learning_rate": 2.997076023391813e-08,
      "logits/chosen": -0.7065263390541077,
      "logits/rejected": -0.6854152679443359,
      "logps/chosen": -861.343505859375,
      "logps/rejected": -721.2241821289062,
      "loss": 0.6912,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.01797466352581978,
      "rewards/margins": 0.004987239837646484,
      "rewards/rejected": -0.022961905226111412,
      "step": 3596
    },
    {
      "epoch": 0.946127669368229,
      "grad_norm": 576.0,
      "learning_rate": 2.982456140350877e-08,
      "logits/chosen": -0.659433126449585,
      "logits/rejected": -0.6627206802368164,
      "logps/chosen": -1361.6497802734375,
      "logps/rejected": -889.1085815429688,
      "loss": 0.6804,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02814321592450142,
      "rewards/margins": 0.026728058233857155,
      "rewards/rejected": 0.0014151576906442642,
      "step": 3597
    },
    {
      "epoch": 0.9463907018034161,
      "grad_norm": 504.0,
      "learning_rate": 2.9678362573099414e-08,
      "logits/chosen": -0.6295371055603027,
      "logits/rejected": -0.6280317902565002,
      "logps/chosen": -964.3347778320312,
      "logps/rejected": -945.5094604492188,
      "loss": 0.6832,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02421407587826252,
      "rewards/margins": 0.02056121826171875,
      "rewards/rejected": 0.0036528590135276318,
      "step": 3598
    },
    {
      "epoch": 0.9466537342386033,
      "grad_norm": 668.0,
      "learning_rate": 2.9532163742690057e-08,
      "logits/chosen": -0.6672441959381104,
      "logits/rejected": -0.6705939173698425,
      "logps/chosen": -1665.0693359375,
      "logps/rejected": -1083.7763671875,
      "loss": 0.6907,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007052422035485506,
      "rewards/margins": 0.007002257741987705,
      "rewards/rejected": 5.016382783651352e-05,
      "step": 3599
    },
    {
      "epoch": 0.9469167666737904,
      "grad_norm": 456.0,
      "learning_rate": 2.93859649122807e-08,
      "logits/chosen": -0.6460282802581787,
      "logits/rejected": -0.6463575959205627,
      "logps/chosen": -820.4546508789062,
      "logps/rejected": -722.01220703125,
      "loss": 0.6824,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.021806716918945312,
      "rewards/margins": 0.02233266830444336,
      "rewards/rejected": -0.0005259509198367596,
      "step": 3600
    },
    {
      "epoch": 0.9471797991089776,
      "grad_norm": 524.0,
      "learning_rate": 2.9239766081871343e-08,
      "logits/chosen": -0.6440892815589905,
      "logits/rejected": -0.6638578176498413,
      "logps/chosen": -1260.8133544921875,
      "logps/rejected": -1016.7652587890625,
      "loss": 0.6939,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.011509561911225319,
      "rewards/margins": -0.0010283479932695627,
      "rewards/rejected": -0.0104812141507864,
      "step": 3601
    },
    {
      "epoch": 0.9474428315441648,
      "grad_norm": 628.0,
      "learning_rate": 2.9093567251461986e-08,
      "logits/chosen": -0.6428508162498474,
      "logits/rejected": -0.662308931350708,
      "logps/chosen": -1148.562744140625,
      "logps/rejected": -1491.349853515625,
      "loss": 0.6887,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.021394826471805573,
      "rewards/margins": 0.01121749822050333,
      "rewards/rejected": 0.010177325457334518,
      "step": 3602
    },
    {
      "epoch": 0.947705863979352,
      "grad_norm": 556.0,
      "learning_rate": 2.894736842105263e-08,
      "logits/chosen": -0.6369608640670776,
      "logits/rejected": -0.6420468688011169,
      "logps/chosen": -1264.135986328125,
      "logps/rejected": -949.990478515625,
      "loss": 0.6903,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.01947040669620037,
      "rewards/margins": 0.0059782033786177635,
      "rewards/rejected": -0.02544860728085041,
      "step": 3603
    },
    {
      "epoch": 0.9479688964145391,
      "grad_norm": 500.0,
      "learning_rate": 2.8801169590643272e-08,
      "logits/chosen": -0.6648734211921692,
      "logits/rejected": -0.6533732414245605,
      "logps/chosen": -931.2484130859375,
      "logps/rejected": -1074.7972412109375,
      "loss": 0.6858,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.011796379461884499,
      "rewards/margins": 0.015302849933505058,
      "rewards/rejected": -0.0035064700059592724,
      "step": 3604
    },
    {
      "epoch": 0.9482319288497263,
      "grad_norm": 580.0,
      "learning_rate": 2.865497076023392e-08,
      "logits/chosen": -0.6481079459190369,
      "logits/rejected": -0.6566269993782043,
      "logps/chosen": -1269.5050048828125,
      "logps/rejected": -998.8388061523438,
      "loss": 0.6872,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0017040250822901726,
      "rewards/margins": 0.012398718856275082,
      "rewards/rejected": -0.014102745801210403,
      "step": 3605
    },
    {
      "epoch": 0.9484949612849134,
      "grad_norm": 544.0,
      "learning_rate": 2.850877192982456e-08,
      "logits/chosen": -0.651861310005188,
      "logits/rejected": -0.6493015885353088,
      "logps/chosen": -1167.63623046875,
      "logps/rejected": -1060.033203125,
      "loss": 0.6951,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006037329789251089,
      "rewards/margins": -0.003402900882065296,
      "rewards/rejected": 0.009440232068300247,
      "step": 3606
    },
    {
      "epoch": 0.9487579937201006,
      "grad_norm": 494.0,
      "learning_rate": 2.8362573099415202e-08,
      "logits/chosen": -0.656513512134552,
      "logits/rejected": -0.6556522846221924,
      "logps/chosen": -931.1025390625,
      "logps/rejected": -808.9110107421875,
      "loss": 0.6812,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.013919735327363014,
      "rewards/margins": 0.024449825286865234,
      "rewards/rejected": -0.01053008995950222,
      "step": 3607
    },
    {
      "epoch": 0.9490210261552878,
      "grad_norm": 552.0,
      "learning_rate": 2.8216374269005848e-08,
      "logits/chosen": -0.651675820350647,
      "logits/rejected": -0.6573686599731445,
      "logps/chosen": -1345.7579345703125,
      "logps/rejected": -1138.5169677734375,
      "loss": 0.6958,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00033855531364679337,
      "rewards/margins": -0.004078294616192579,
      "rewards/rejected": 0.004416847601532936,
      "step": 3608
    },
    {
      "epoch": 0.9492840585904749,
      "grad_norm": 564.0,
      "learning_rate": 2.807017543859649e-08,
      "logits/chosen": -0.6684085726737976,
      "logits/rejected": -0.6698814630508423,
      "logps/chosen": -1113.0838623046875,
      "logps/rejected": -876.3302001953125,
      "loss": 0.7066,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.014683818444609642,
      "rewards/margins": -0.02524747885763645,
      "rewards/rejected": 0.01056366041302681,
      "step": 3609
    },
    {
      "epoch": 0.9495470910256621,
      "grad_norm": 460.0,
      "learning_rate": 2.7923976608187134e-08,
      "logits/chosen": -0.6666925549507141,
      "logits/rejected": -0.6560845375061035,
      "logps/chosen": -933.080078125,
      "logps/rejected": -553.9661254882812,
      "loss": 0.6952,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.010544681921601295,
      "rewards/margins": -0.0037284852005541325,
      "rewards/rejected": -0.006816195789724588,
      "step": 3610
    },
    {
      "epoch": 0.9498101234608493,
      "grad_norm": 516.0,
      "learning_rate": 2.7777777777777774e-08,
      "logits/chosen": -0.6436052322387695,
      "logits/rejected": NaN,
      "logps/chosen": -1082.5107421875,
      "logps/rejected": -441.01617431640625,
      "loss": 0.6914,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0037012097891420126,
      "rewards/margins": 0.0040779123082757,
      "rewards/rejected": -0.00037670182064175606,
      "step": 3611
    },
    {
      "epoch": 0.9500731558960365,
      "grad_norm": 544.0,
      "learning_rate": 2.763157894736842e-08,
      "logits/chosen": -0.665773868560791,
      "logits/rejected": -0.6736968755722046,
      "logps/chosen": -1236.85205078125,
      "logps/rejected": -1252.2099609375,
      "loss": 0.6856,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005944157484918833,
      "rewards/margins": 0.01581707037985325,
      "rewards/rejected": -0.009872913360595703,
      "step": 3612
    },
    {
      "epoch": 0.9503361883312236,
      "grad_norm": 532.0,
      "learning_rate": 2.7485380116959063e-08,
      "logits/chosen": -0.6497246026992798,
      "logits/rejected": -0.6614676713943481,
      "logps/chosen": -1384.31298828125,
      "logps/rejected": -1113.7230224609375,
      "loss": 0.6803,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0030145649798214436,
      "rewards/margins": 0.02660837210714817,
      "rewards/rejected": -0.029622936621308327,
      "step": 3613
    },
    {
      "epoch": 0.9505992207664108,
      "grad_norm": 720.0,
      "learning_rate": 2.7339181286549706e-08,
      "logits/chosen": -0.6414322257041931,
      "logits/rejected": -0.6370628476142883,
      "logps/chosen": -1497.4639892578125,
      "logps/rejected": -1141.8515625,
      "loss": 0.6902,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.010650158859789371,
      "rewards/margins": 0.006595802493393421,
      "rewards/rejected": -0.017245961353182793,
      "step": 3614
    },
    {
      "epoch": 0.9508622532015979,
      "grad_norm": 552.0,
      "learning_rate": 2.719298245614035e-08,
      "logits/chosen": NaN,
      "logits/rejected": -0.6392214894294739,
      "logps/chosen": -940.8375244140625,
      "logps/rejected": -873.05517578125,
      "loss": 0.695,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.006729793734848499,
      "rewards/margins": -0.003079700283706188,
      "rewards/rejected": 0.009809494018554688,
      "step": 3615
    },
    {
      "epoch": 0.9511252856367851,
      "grad_norm": 572.0,
      "learning_rate": 2.7046783625730993e-08,
      "logits/chosen": -0.6604854464530945,
      "logits/rejected": -0.6665306091308594,
      "logps/chosen": -1291.8638916015625,
      "logps/rejected": -757.6700439453125,
      "loss": 0.6883,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02107858657836914,
      "rewards/margins": 0.011361788958311081,
      "rewards/rejected": 0.00971679762005806,
      "step": 3616
    },
    {
      "epoch": 0.9513883180719722,
      "grad_norm": 660.0,
      "learning_rate": 2.6900584795321636e-08,
      "logits/chosen": -0.6486155390739441,
      "logits/rejected": -0.6498717069625854,
      "logps/chosen": -1563.950439453125,
      "logps/rejected": -970.767578125,
      "loss": 0.6917,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0037790299393236637,
      "rewards/margins": 0.0040178303606808186,
      "rewards/rejected": -0.007796861231327057,
      "step": 3617
    },
    {
      "epoch": 0.9516513505071594,
      "grad_norm": 556.0,
      "learning_rate": 2.675438596491228e-08,
      "logits/chosen": -0.6467406749725342,
      "logits/rejected": -0.6536191701889038,
      "logps/chosen": -1089.73974609375,
      "logps/rejected": -968.9771728515625,
      "loss": 0.6992,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.009707736782729626,
      "rewards/margins": -0.011583996005356312,
      "rewards/rejected": 0.0018762589897960424,
      "step": 3618
    },
    {
      "epoch": 0.9519143829423465,
      "grad_norm": 560.0,
      "learning_rate": 2.6608187134502922e-08,
      "logits/chosen": -0.6461952924728394,
      "logits/rejected": -0.6502201557159424,
      "logps/chosen": -1349.7618408203125,
      "logps/rejected": -998.0076904296875,
      "loss": 0.6996,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.017459774389863014,
      "rewards/margins": -0.011908814311027527,
      "rewards/rejected": -0.005550956819206476,
      "step": 3619
    },
    {
      "epoch": 0.9521774153775338,
      "grad_norm": 504.0,
      "learning_rate": 2.6461988304093568e-08,
      "logits/chosen": -0.6574000120162964,
      "logits/rejected": -0.6575974225997925,
      "logps/chosen": -956.6575927734375,
      "logps/rejected": -830.480224609375,
      "loss": 0.6932,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.017816923558712006,
      "rewards/margins": 0.0006860727444291115,
      "rewards/rejected": -0.01850299909710884,
      "step": 3620
    },
    {
      "epoch": 0.9524404478127209,
      "grad_norm": 434.0,
      "learning_rate": 2.6315789473684208e-08,
      "logits/chosen": -0.6284709572792053,
      "logits/rejected": -0.6391719579696655,
      "logps/chosen": -720.021240234375,
      "logps/rejected": -657.0219116210938,
      "loss": 0.6974,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0029026977717876434,
      "rewards/margins": -0.008263301104307175,
      "rewards/rejected": 0.005360603332519531,
      "step": 3621
    },
    {
      "epoch": 0.9527034802479081,
      "grad_norm": 442.0,
      "learning_rate": 2.616959064327485e-08,
      "logits/chosen": -0.6513793468475342,
      "logits/rejected": -0.6500895619392395,
      "logps/chosen": -619.08349609375,
      "logps/rejected": -651.81689453125,
      "loss": 0.6962,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0035867691040039062,
      "rewards/margins": -0.005576134659349918,
      "rewards/rejected": 0.00916290283203125,
      "step": 3622
    },
    {
      "epoch": 0.9529665126830952,
      "grad_norm": 560.0,
      "learning_rate": 2.6023391812865497e-08,
      "logits/chosen": -0.6729080080986023,
      "logits/rejected": NaN,
      "logps/chosen": -926.1563720703125,
      "logps/rejected": -628.694091796875,
      "loss": 0.6899,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.011134816333651543,
      "rewards/margins": 0.007848070934414864,
      "rewards/rejected": 0.0032867430709302425,
      "step": 3623
    },
    {
      "epoch": 0.9532295451182824,
      "grad_norm": 532.0,
      "learning_rate": 2.587719298245614e-08,
      "logits/chosen": -0.6445181369781494,
      "logits/rejected": -0.6491214036941528,
      "logps/chosen": -1149.5860595703125,
      "logps/rejected": -749.5194702148438,
      "loss": 0.6923,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0055201053619384766,
      "rewards/margins": 0.00226016016677022,
      "rewards/rejected": -0.007780265063047409,
      "step": 3624
    },
    {
      "epoch": 0.9534925775534696,
      "grad_norm": 492.0,
      "learning_rate": 2.5730994152046784e-08,
      "logits/chosen": -0.6472630500793457,
      "logits/rejected": -0.6483676433563232,
      "logps/chosen": -915.6268310546875,
      "logps/rejected": -624.5567626953125,
      "loss": 0.6952,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -8.592591620981693e-05,
      "rewards/margins": -0.0037296898663043976,
      "rewards/rejected": 0.003643763717263937,
      "step": 3625
    },
    {
      "epoch": 0.9537556099886567,
      "grad_norm": 404.0,
      "learning_rate": 2.5584795321637423e-08,
      "logits/chosen": -0.6552817225456238,
      "logits/rejected": -0.6618032455444336,
      "logps/chosen": -704.391845703125,
      "logps/rejected": -494.8797912597656,
      "loss": 0.6973,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.026362231001257896,
      "rewards/margins": -0.00789861660450697,
      "rewards/rejected": -0.018463611602783203,
      "step": 3626
    },
    {
      "epoch": 0.9540186424238439,
      "grad_norm": 452.0,
      "learning_rate": 2.543859649122807e-08,
      "logits/chosen": -0.6336671710014343,
      "logits/rejected": -0.6328937411308289,
      "logps/chosen": -747.5546875,
      "logps/rejected": -770.4784545898438,
      "loss": 0.6902,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0012612335849553347,
      "rewards/margins": 0.006466960534453392,
      "rewards/rejected": -0.00772819435223937,
      "step": 3627
    },
    {
      "epoch": 0.954281674859031,
      "grad_norm": 584.0,
      "learning_rate": 2.5292397660818713e-08,
      "logits/chosen": -0.672512412071228,
      "logits/rejected": -0.6832613348960876,
      "logps/chosen": -1199.0421142578125,
      "logps/rejected": -1242.0361328125,
      "loss": 0.6693,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.037879083305597305,
      "rewards/margins": 0.049646761268377304,
      "rewards/rejected": -0.011767672374844551,
      "step": 3628
    },
    {
      "epoch": 0.9545447072942183,
      "grad_norm": 454.0,
      "learning_rate": 2.5146198830409356e-08,
      "logits/chosen": -0.6571257710456848,
      "logits/rejected": -0.6469557285308838,
      "logps/chosen": -910.6907958984375,
      "logps/rejected": -676.3983154296875,
      "loss": 0.6873,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004782295785844326,
      "rewards/margins": 0.012538623064756393,
      "rewards/rejected": -0.007756328210234642,
      "step": 3629
    },
    {
      "epoch": 0.9548077397294054,
      "grad_norm": 556.0,
      "learning_rate": 2.5e-08,
      "logits/chosen": -0.6806649565696716,
      "logits/rejected": NaN,
      "logps/chosen": -1126.025390625,
      "logps/rejected": -826.11083984375,
      "loss": 0.7021,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0018329622689634562,
      "rewards/margins": -0.01675577089190483,
      "rewards/rejected": 0.018588734790682793,
      "step": 3630
    },
    {
      "epoch": 0.9550707721645926,
      "grad_norm": 442.0,
      "learning_rate": 2.4853801169590642e-08,
      "logits/chosen": -0.6379818916320801,
      "logits/rejected": -0.6431123614311218,
      "logps/chosen": -1058.74951171875,
      "logps/rejected": -883.1752319335938,
      "loss": 0.6895,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.020937921479344368,
      "rewards/margins": 0.007612133398652077,
      "rewards/rejected": 0.013325787149369717,
      "step": 3631
    },
    {
      "epoch": 0.9553338045997797,
      "grad_norm": 510.0,
      "learning_rate": 2.4707602339181285e-08,
      "logits/chosen": -0.6629955768585205,
      "logits/rejected": -0.6753171682357788,
      "logps/chosen": -1305.4678955078125,
      "logps/rejected": -1047.3179931640625,
      "loss": 0.69,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03380336984992027,
      "rewards/margins": 0.007076169364154339,
      "rewards/rejected": 0.02672720141708851,
      "step": 3632
    },
    {
      "epoch": 0.9555968370349669,
      "grad_norm": 500.0,
      "learning_rate": 2.4561403508771928e-08,
      "logits/chosen": -0.6443710327148438,
      "logits/rejected": -0.6478410959243774,
      "logps/chosen": -1122.141845703125,
      "logps/rejected": -827.3502807617188,
      "loss": 0.7071,
      "rewards/accuracies": 0.0625,
      "rewards/chosen": 0.014067649841308594,
      "rewards/margins": -0.026349827647209167,
      "rewards/rejected": 0.04041747748851776,
      "step": 3633
    },
    {
      "epoch": 0.955859869470154,
      "grad_norm": 576.0,
      "learning_rate": 2.441520467836257e-08,
      "logits/chosen": -0.6338946223258972,
      "logits/rejected": -0.6389201879501343,
      "logps/chosen": -1173.59326171875,
      "logps/rejected": -1001.7758178710938,
      "loss": 0.6854,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.008956432342529297,
      "rewards/margins": 0.016294002532958984,
      "rewards/rejected": -0.007337570656090975,
      "step": 3634
    },
    {
      "epoch": 0.9561229019053412,
      "grad_norm": 608.0,
      "learning_rate": 2.4269005847953218e-08,
      "logits/chosen": -0.6323227882385254,
      "logits/rejected": -0.6487372517585754,
      "logps/chosen": -1014.027099609375,
      "logps/rejected": -728.8760986328125,
      "loss": 0.7017,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.009749507531523705,
      "rewards/margins": -0.014868591912090778,
      "rewards/rejected": 0.0051190853118896484,
      "step": 3635
    },
    {
      "epoch": 0.9563859343405283,
      "grad_norm": 458.0,
      "learning_rate": 2.4122807017543857e-08,
      "logits/chosen": -0.6518612504005432,
      "logits/rejected": -0.6656972765922546,
      "logps/chosen": -1271.0194091796875,
      "logps/rejected": -895.2845458984375,
      "loss": 0.6996,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00015926314517855644,
      "rewards/margins": -0.012378881685435772,
      "rewards/rejected": 0.012219619937241077,
      "step": 3636
    },
    {
      "epoch": 0.9566489667757155,
      "grad_norm": 506.0,
      "learning_rate": 2.39766081871345e-08,
      "logits/chosen": -0.6426987648010254,
      "logits/rejected": -0.6411332488059998,
      "logps/chosen": -1088.9129638671875,
      "logps/rejected": -899.3333740234375,
      "loss": 0.6865,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.005819607060402632,
      "rewards/margins": 0.014115525409579277,
      "rewards/rejected": -0.008295917883515358,
      "step": 3637
    },
    {
      "epoch": 0.9569119992109026,
      "grad_norm": 480.0,
      "learning_rate": 2.3830409356725147e-08,
      "logits/chosen": -0.6775981187820435,
      "logits/rejected": -0.6664124727249146,
      "logps/chosen": -1157.9090576171875,
      "logps/rejected": -1049.5927734375,
      "loss": 0.6834,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.015898942947387695,
      "rewards/margins": 0.020146897062659264,
      "rewards/rejected": -0.004247952252626419,
      "step": 3638
    },
    {
      "epoch": 0.9571750316460899,
      "grad_norm": 568.0,
      "learning_rate": 2.368421052631579e-08,
      "logits/chosen": -0.6431465148925781,
      "logits/rejected": -0.6541192531585693,
      "logps/chosen": -1251.1510009765625,
      "logps/rejected": -845.4124145507812,
      "loss": 0.685,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.002496432512998581,
      "rewards/margins": 0.016750048846006393,
      "rewards/rejected": -0.014253616333007812,
      "step": 3639
    },
    {
      "epoch": 0.9574380640812771,
      "grad_norm": 624.0,
      "learning_rate": 2.3538011695906433e-08,
      "logits/chosen": -0.6660674214363098,
      "logits/rejected": -0.6803959608078003,
      "logps/chosen": -1772.509521484375,
      "logps/rejected": -1073.638916015625,
      "loss": 0.6893,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.011581803672015667,
      "rewards/margins": 0.009045028127729893,
      "rewards/rejected": 0.002536773681640625,
      "step": 3640
    },
    {
      "epoch": 0.9577010965164642,
      "grad_norm": 572.0,
      "learning_rate": 2.3391812865497073e-08,
      "logits/chosen": -0.663933277130127,
      "logits/rejected": -0.6553525924682617,
      "logps/chosen": -1086.835205078125,
      "logps/rejected": -1019.71875,
      "loss": 0.6854,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.002435017377138138,
      "rewards/margins": 0.016219234094023705,
      "rewards/rejected": -0.013784218579530716,
      "step": 3641
    },
    {
      "epoch": 0.9579641289516514,
      "grad_norm": 532.0,
      "learning_rate": 2.324561403508772e-08,
      "logits/chosen": -0.6343416571617126,
      "logits/rejected": -0.6525821685791016,
      "logps/chosen": -1142.9576416015625,
      "logps/rejected": -782.0111083984375,
      "loss": 0.6868,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.05132942274212837,
      "rewards/margins": 0.014263438060879707,
      "rewards/rejected": 0.037065982818603516,
      "step": 3642
    },
    {
      "epoch": 0.9582271613868385,
      "grad_norm": 608.0,
      "learning_rate": 2.3099415204678362e-08,
      "logits/chosen": -0.6477875709533691,
      "logits/rejected": NaN,
      "logps/chosen": -909.705322265625,
      "logps/rejected": -885.5577392578125,
      "loss": 0.6938,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01352482009679079,
      "rewards/margins": -0.0005224226042628288,
      "rewards/rejected": 0.014047241769731045,
      "step": 3643
    },
    {
      "epoch": 0.9584901938220257,
      "grad_norm": 576.0,
      "learning_rate": 2.2953216374269005e-08,
      "logits/chosen": -0.6532465815544128,
      "logits/rejected": -0.6587045192718506,
      "logps/chosen": -1349.93896484375,
      "logps/rejected": -1065.2723388671875,
      "loss": 0.6986,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0003062246832996607,
      "rewards/margins": -0.010663080960512161,
      "rewards/rejected": 0.010969305410981178,
      "step": 3644
    },
    {
      "epoch": 0.9587532262572128,
      "grad_norm": 512.0,
      "learning_rate": 2.2807017543859648e-08,
      "logits/chosen": -0.6441444754600525,
      "logits/rejected": -0.6480869054794312,
      "logps/chosen": -1132.8485107421875,
      "logps/rejected": -888.7779541015625,
      "loss": 0.6882,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.017002200707793236,
      "rewards/margins": 0.011238384991884232,
      "rewards/rejected": -0.028240583837032318,
      "step": 3645
    },
    {
      "epoch": 0.9590162586924,
      "grad_norm": 620.0,
      "learning_rate": 2.266081871345029e-08,
      "logits/chosen": -0.6665751338005066,
      "logits/rejected": -0.6607148051261902,
      "logps/chosen": -1286.779296875,
      "logps/rejected": -1251.15673828125,
      "loss": 0.6915,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01914358139038086,
      "rewards/margins": 0.004236697684973478,
      "rewards/rejected": 0.014906883239746094,
      "step": 3646
    },
    {
      "epoch": 0.9592792911275871,
      "grad_norm": 506.0,
      "learning_rate": 2.2514619883040934e-08,
      "logits/chosen": -0.6481976509094238,
      "logits/rejected": -0.6525642275810242,
      "logps/chosen": -1053.4600830078125,
      "logps/rejected": -840.26806640625,
      "loss": 0.705,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.028914643451571465,
      "rewards/margins": -0.022565554827451706,
      "rewards/rejected": -0.006349086761474609,
      "step": 3647
    },
    {
      "epoch": 0.9595423235627744,
      "grad_norm": 712.0,
      "learning_rate": 2.2368421052631577e-08,
      "logits/chosen": -0.6531538963317871,
      "logits/rejected": NaN,
      "logps/chosen": -1088.83349609375,
      "logps/rejected": -1286.517333984375,
      "loss": 0.6874,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.009633541107177734,
      "rewards/margins": 0.012560558505356312,
      "rewards/rejected": -0.02219410054385662,
      "step": 3648
    },
    {
      "epoch": 0.9598053559979615,
      "grad_norm": 536.0,
      "learning_rate": 2.222222222222222e-08,
      "logits/chosen": -0.6556522846221924,
      "logits/rejected": -0.652103066444397,
      "logps/chosen": -1141.4168701171875,
      "logps/rejected": -701.5020751953125,
      "loss": 0.7096,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.020349692553281784,
      "rewards/margins": -0.03148674964904785,
      "rewards/rejected": 0.011137057095766068,
      "step": 3649
    },
    {
      "epoch": 0.9600683884331487,
      "grad_norm": 588.0,
      "learning_rate": 2.2076023391812867e-08,
      "logits/chosen": -0.6542815566062927,
      "logits/rejected": -0.6592427492141724,
      "logps/chosen": -1340.90234375,
      "logps/rejected": -1085.566650390625,
      "loss": 0.6922,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.005374336615204811,
      "rewards/margins": 0.002898501232266426,
      "rewards/rejected": 0.0024758335202932358,
      "step": 3650
    },
    {
      "epoch": 0.9603314208683358,
      "grad_norm": 552.0,
      "learning_rate": 2.1929824561403507e-08,
      "logits/chosen": -0.6488063931465149,
      "logits/rejected": -0.6589822173118591,
      "logps/chosen": -996.5255126953125,
      "logps/rejected": -1041.6904296875,
      "loss": 0.7012,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.002808189485222101,
      "rewards/margins": -0.014820480719208717,
      "rewards/rejected": 0.012012290768325329,
      "step": 3651
    },
    {
      "epoch": 0.960594453303523,
      "grad_norm": 532.0,
      "learning_rate": 2.178362573099415e-08,
      "logits/chosen": -0.641523540019989,
      "logits/rejected": -0.6578624844551086,
      "logps/chosen": -1192.7550048828125,
      "logps/rejected": -867.2699584960938,
      "loss": 0.6909,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00031576119363307953,
      "rewards/margins": 0.004909277893602848,
      "rewards/rejected": -0.004593516234308481,
      "step": 3652
    },
    {
      "epoch": 0.9608574857387101,
      "grad_norm": 536.0,
      "learning_rate": 2.1637426900584796e-08,
      "logits/chosen": -0.6601629257202148,
      "logits/rejected": -0.647522509098053,
      "logps/chosen": -1350.883544921875,
      "logps/rejected": -1118.4698486328125,
      "loss": 0.6991,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.015277337282896042,
      "rewards/margins": -0.011269617825746536,
      "rewards/rejected": 0.026546955108642578,
      "step": 3653
    },
    {
      "epoch": 0.9611205181738973,
      "grad_norm": 524.0,
      "learning_rate": 2.149122807017544e-08,
      "logits/chosen": -0.6365407705307007,
      "logits/rejected": -0.6475268602371216,
      "logps/chosen": -1404.4110107421875,
      "logps/rejected": -790.3076782226562,
      "loss": 0.7031,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.004180907271802425,
      "rewards/margins": -0.01843414269387722,
      "rewards/rejected": 0.014253235422074795,
      "step": 3654
    },
    {
      "epoch": 0.9613835506090845,
      "grad_norm": 472.0,
      "learning_rate": 2.1345029239766082e-08,
      "logits/chosen": -0.6552515029907227,
      "logits/rejected": -0.6572684645652771,
      "logps/chosen": -905.87744140625,
      "logps/rejected": -675.8429565429688,
      "loss": 0.6862,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0020763406064361334,
      "rewards/margins": 0.014746188186109066,
      "rewards/rejected": -0.016822529956698418,
      "step": 3655
    },
    {
      "epoch": 0.9616465830442716,
      "grad_norm": 620.0,
      "learning_rate": 2.1198830409356722e-08,
      "logits/chosen": -0.6550913453102112,
      "logits/rejected": -0.660200297832489,
      "logps/chosen": -1377.236328125,
      "logps/rejected": -1121.402587890625,
      "loss": 0.6875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008860589005053043,
      "rewards/margins": 0.011902429163455963,
      "rewards/rejected": -0.0030418401584029198,
      "step": 3656
    },
    {
      "epoch": 0.9619096154794589,
      "grad_norm": 544.0,
      "learning_rate": 2.1052631578947368e-08,
      "logits/chosen": -0.6519362926483154,
      "logits/rejected": -0.6584856510162354,
      "logps/chosen": -1018.410888671875,
      "logps/rejected": -786.8294677734375,
      "loss": 0.6989,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.013908100314438343,
      "rewards/margins": -0.010413648560643196,
      "rewards/rejected": -0.003494453150779009,
      "step": 3657
    },
    {
      "epoch": 0.962172647914646,
      "grad_norm": 600.0,
      "learning_rate": 2.090643274853801e-08,
      "logits/chosen": -0.6565706729888916,
      "logits/rejected": -0.6642155647277832,
      "logps/chosen": -1182.1290283203125,
      "logps/rejected": -1149.3717041015625,
      "loss": 0.6808,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.031192876398563385,
      "rewards/margins": 0.02576765976846218,
      "rewards/rejected": 0.005425214301794767,
      "step": 3658
    },
    {
      "epoch": 0.9624356803498332,
      "grad_norm": 460.0,
      "learning_rate": 2.0760233918128654e-08,
      "logits/chosen": -0.6681056022644043,
      "logits/rejected": -0.6875267624855042,
      "logps/chosen": -846.471435546875,
      "logps/rejected": -570.1781005859375,
      "loss": 0.7012,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.00537033099681139,
      "rewards/margins": -0.015184975229203701,
      "rewards/rejected": 0.009814643301069736,
      "step": 3659
    },
    {
      "epoch": 0.9626987127850203,
      "grad_norm": 494.0,
      "learning_rate": 2.0614035087719298e-08,
      "logits/chosen": -0.6490835547447205,
      "logits/rejected": -0.6538490056991577,
      "logps/chosen": -907.9824829101562,
      "logps/rejected": -596.983642578125,
      "loss": 0.6986,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.014292907901108265,
      "rewards/margins": -0.010376167483627796,
      "rewards/rejected": -0.003916740883141756,
      "step": 3660
    },
    {
      "epoch": 0.9629617452202075,
      "grad_norm": 482.0,
      "learning_rate": 2.046783625730994e-08,
      "logits/chosen": -0.648103654384613,
      "logits/rejected": -0.6517444849014282,
      "logps/chosen": -881.1998291015625,
      "logps/rejected": -705.36474609375,
      "loss": 0.6842,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.017116164788603783,
      "rewards/margins": 0.01810011826455593,
      "rewards/rejected": -0.035216283053159714,
      "step": 3661
    },
    {
      "epoch": 0.9632247776553946,
      "grad_norm": 688.0,
      "learning_rate": 2.0321637426900584e-08,
      "logits/chosen": -0.644417405128479,
      "logits/rejected": -0.6591110825538635,
      "logps/chosen": -1369.9598388671875,
      "logps/rejected": -1087.3853759765625,
      "loss": 0.7037,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.012934112921357155,
      "rewards/margins": -0.019334126263856888,
      "rewards/rejected": 0.03226824104785919,
      "step": 3662
    },
    {
      "epoch": 0.9634878100905818,
      "grad_norm": 460.0,
      "learning_rate": 2.0175438596491227e-08,
      "logits/chosen": -0.6425065994262695,
      "logits/rejected": -0.641772449016571,
      "logps/chosen": -958.5916748046875,
      "logps/rejected": -742.94140625,
      "loss": 0.6898,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.003305817022919655,
      "rewards/margins": 0.007421398069709539,
      "rewards/rejected": -0.010727215558290482,
      "step": 3663
    },
    {
      "epoch": 0.9637508425257689,
      "grad_norm": 508.0,
      "learning_rate": 2.002923976608187e-08,
      "logits/chosen": -0.6512281894683838,
      "logits/rejected": -0.6591291427612305,
      "logps/chosen": -1018.6607666015625,
      "logps/rejected": -739.7859497070312,
      "loss": 0.6941,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.006422902457416058,
      "rewards/margins": -0.0015522008761763573,
      "rewards/rejected": -0.004870700649917126,
      "step": 3664
    },
    {
      "epoch": 0.9640138749609561,
      "grad_norm": 644.0,
      "learning_rate": 1.9883040935672516e-08,
      "logits/chosen": -0.6648540496826172,
      "logits/rejected": -0.659632682800293,
      "logps/chosen": -1499.003173828125,
      "logps/rejected": -1011.794677734375,
      "loss": 0.6987,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.005850506480783224,
      "rewards/margins": -0.010399341583251953,
      "rewards/rejected": 0.016249846667051315,
      "step": 3665
    },
    {
      "epoch": 0.9642769073961432,
      "grad_norm": 496.0,
      "learning_rate": 1.9736842105263156e-08,
      "logits/chosen": -0.6377124190330505,
      "logits/rejected": -0.6400785446166992,
      "logps/chosen": -756.8292846679688,
      "logps/rejected": -806.5327758789062,
      "loss": 0.6834,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008797550573945045,
      "rewards/margins": 0.0202223788946867,
      "rewards/rejected": -0.011424828320741653,
      "step": 3666
    },
    {
      "epoch": 0.9645399398313305,
      "grad_norm": 496.0,
      "learning_rate": 1.95906432748538e-08,
      "logits/chosen": -0.6613236665725708,
      "logits/rejected": -0.663862407207489,
      "logps/chosen": -982.08056640625,
      "logps/rejected": -689.18212890625,
      "loss": 0.7011,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.022103026509284973,
      "rewards/margins": -0.014940355904400349,
      "rewards/rejected": -0.007162665948271751,
      "step": 3667
    },
    {
      "epoch": 0.9648029722665176,
      "grad_norm": 544.0,
      "learning_rate": 1.9444444444444445e-08,
      "logits/chosen": -0.646918773651123,
      "logits/rejected": -0.6232154965400696,
      "logps/chosen": -1174.7608642578125,
      "logps/rejected": -919.6180419921875,
      "loss": 0.6989,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.016654204577207565,
      "rewards/margins": -0.009960363619029522,
      "rewards/rejected": -0.006693840958178043,
      "step": 3668
    },
    {
      "epoch": 0.9650660047017048,
      "grad_norm": 500.0,
      "learning_rate": 1.929824561403509e-08,
      "logits/chosen": -0.640959620475769,
      "logits/rejected": -0.6444451808929443,
      "logps/chosen": -1322.068603515625,
      "logps/rejected": -1068.0916748046875,
      "loss": 0.6929,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005021381191909313,
      "rewards/margins": 0.0014434810727834702,
      "rewards/rejected": -0.006464863661676645,
      "step": 3669
    },
    {
      "epoch": 0.9653290371368919,
      "grad_norm": 564.0,
      "learning_rate": 1.915204678362573e-08,
      "logits/chosen": -0.6421504020690918,
      "logits/rejected": -0.6481562852859497,
      "logps/chosen": -1512.775390625,
      "logps/rejected": -1199.556396484375,
      "loss": 0.6854,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.010776519775390625,
      "rewards/margins": 0.015953540802001953,
      "rewards/rejected": -0.0051770200952887535,
      "step": 3670
    },
    {
      "epoch": 0.9655920695720791,
      "grad_norm": 388.0,
      "learning_rate": 1.900584795321637e-08,
      "logits/chosen": -0.638964056968689,
      "logits/rejected": -0.6531967520713806,
      "logps/chosen": -1062.5321044921875,
      "logps/rejected": -795.0450439453125,
      "loss": 0.6931,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.015092181973159313,
      "rewards/margins": 0.00033693318255245686,
      "rewards/rejected": -0.015429114922881126,
      "step": 3671
    },
    {
      "epoch": 0.9658551020072663,
      "grad_norm": 516.0,
      "learning_rate": 1.8859649122807018e-08,
      "logits/chosen": -0.645652711391449,
      "logits/rejected": -0.6480517983436584,
      "logps/chosen": -1466.681640625,
      "logps/rejected": -1156.564697265625,
      "loss": 0.6994,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.016120148822665215,
      "rewards/margins": -0.012030219659209251,
      "rewards/rejected": -0.0040899282321333885,
      "step": 3672
    },
    {
      "epoch": 0.9661181344424534,
      "grad_norm": 656.0,
      "learning_rate": 1.871345029239766e-08,
      "logits/chosen": -0.68013596534729,
      "logits/rejected": -0.6645231246948242,
      "logps/chosen": -1036.59375,
      "logps/rejected": -960.6732177734375,
      "loss": 0.6953,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01021642703562975,
      "rewards/margins": -0.0028958325274288654,
      "rewards/rejected": 0.013112258166074753,
      "step": 3673
    },
    {
      "epoch": 0.9663811668776406,
      "grad_norm": 648.0,
      "learning_rate": 1.8567251461988304e-08,
      "logits/chosen": -0.6675335764884949,
      "logits/rejected": -0.6631524562835693,
      "logps/chosen": -1091.5321044921875,
      "logps/rejected": -902.5523681640625,
      "loss": 0.6881,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.018523024395108223,
      "rewards/margins": 0.011061476543545723,
      "rewards/rejected": 0.0074615478515625,
      "step": 3674
    },
    {
      "epoch": 0.9666441993128277,
      "grad_norm": 596.0,
      "learning_rate": 1.8421052631578944e-08,
      "logits/chosen": -0.6508714556694031,
      "logits/rejected": -0.6557470560073853,
      "logps/chosen": -1411.4727783203125,
      "logps/rejected": -1130.649658203125,
      "loss": 0.6846,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005494594573974609,
      "rewards/margins": 0.017859935760498047,
      "rewards/rejected": -0.012365339323878288,
      "step": 3675
    },
    {
      "epoch": 0.966907231748015,
      "grad_norm": 884.0,
      "learning_rate": 1.827485380116959e-08,
      "logits/chosen": -0.6513664722442627,
      "logits/rejected": -0.6501315236091614,
      "logps/chosen": -1781.145263671875,
      "logps/rejected": -1283.607666015625,
      "loss": 0.6898,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.01948556676506996,
      "rewards/margins": 0.00853357370942831,
      "rewards/rejected": -0.028019143268465996,
      "step": 3676
    },
    {
      "epoch": 0.9671702641832021,
      "grad_norm": 604.0,
      "learning_rate": 1.8128654970760233e-08,
      "logits/chosen": -0.6631962060928345,
      "logits/rejected": -0.660104513168335,
      "logps/chosen": -1139.2314453125,
      "logps/rejected": -1027.3939208984375,
      "loss": 0.6812,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.03261380270123482,
      "rewards/margins": 0.02466578409075737,
      "rewards/rejected": 0.007948017679154873,
      "step": 3677
    },
    {
      "epoch": 0.9674332966183893,
      "grad_norm": 572.0,
      "learning_rate": 1.7982456140350876e-08,
      "logits/chosen": -0.654557466506958,
      "logits/rejected": -0.6568525433540344,
      "logps/chosen": -1123.876708984375,
      "logps/rejected": -807.8616333007812,
      "loss": 0.6915,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.0014432899188250303,
      "rewards/margins": 0.0043439860455691814,
      "rewards/rejected": -0.002900694962590933,
      "step": 3678
    },
    {
      "epoch": 0.9676963290535764,
      "grad_norm": 584.0,
      "learning_rate": 1.783625730994152e-08,
      "logits/chosen": -0.6639436483383179,
      "logits/rejected": -0.6742111444473267,
      "logps/chosen": -1324.3272705078125,
      "logps/rejected": -1091.17919921875,
      "loss": 0.698,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0214067455381155,
      "rewards/margins": -0.008998297154903412,
      "rewards/rejected": -0.01240844838321209,
      "step": 3679
    },
    {
      "epoch": 0.9679593614887636,
      "grad_norm": 502.0,
      "learning_rate": 1.7690058479532162e-08,
      "logits/chosen": -0.653083860874176,
      "logits/rejected": -0.6591416597366333,
      "logps/chosen": -920.3426513671875,
      "logps/rejected": -701.6717529296875,
      "loss": 0.6922,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.011736059561371803,
      "rewards/margins": 0.002914284821599722,
      "rewards/rejected": 0.008821774274110794,
      "step": 3680
    },
    {
      "epoch": 0.9682223939239507,
      "grad_norm": 576.0,
      "learning_rate": 1.7543859649122805e-08,
      "logits/chosen": -0.6625092029571533,
      "logits/rejected": -0.6830449104309082,
      "logps/chosen": -914.2405395507812,
      "logps/rejected": -884.3141479492188,
      "loss": 0.6994,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.004845857620239258,
      "rewards/margins": -0.011678364127874374,
      "rewards/rejected": 0.006832504644989967,
      "step": 3681
    },
    {
      "epoch": 0.9684854263591379,
      "grad_norm": 580.0,
      "learning_rate": 1.739766081871345e-08,
      "logits/chosen": -0.6644129157066345,
      "logits/rejected": -0.6599428653717041,
      "logps/chosen": -1376.3408203125,
      "logps/rejected": -855.6598510742188,
      "loss": 0.7029,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0107511505484581,
      "rewards/margins": -0.017627714201807976,
      "rewards/rejected": 0.006876563653349876,
      "step": 3682
    },
    {
      "epoch": 0.968748458794325,
      "grad_norm": 664.0,
      "learning_rate": 1.7251461988304095e-08,
      "logits/chosen": -0.6622790098190308,
      "logits/rejected": -0.6395334601402283,
      "logps/chosen": -1458.9903564453125,
      "logps/rejected": -1325.8614501953125,
      "loss": 0.6883,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007112599443644285,
      "rewards/margins": 0.011223889887332916,
      "rewards/rejected": -0.004111289046704769,
      "step": 3683
    },
    {
      "epoch": 0.9690114912295122,
      "grad_norm": 720.0,
      "learning_rate": 1.7105263157894738e-08,
      "logits/chosen": -0.6487477421760559,
      "logits/rejected": -0.6348318457603455,
      "logps/chosen": -1173.419677734375,
      "logps/rejected": -954.32275390625,
      "loss": 0.6951,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.014823340810835361,
      "rewards/margins": -0.0031803131569176912,
      "rewards/rejected": 0.01800365187227726,
      "step": 3684
    },
    {
      "epoch": 0.9692745236646994,
      "grad_norm": 460.0,
      "learning_rate": 1.6959064327485378e-08,
      "logits/chosen": -0.6482204794883728,
      "logits/rejected": -0.6513190269470215,
      "logps/chosen": -802.73291015625,
      "logps/rejected": -725.8057250976562,
      "loss": 0.6888,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004182624630630016,
      "rewards/margins": 0.009215925820171833,
      "rewards/rejected": -0.005033302120864391,
      "step": 3685
    },
    {
      "epoch": 0.9695375560998866,
      "grad_norm": 576.0,
      "learning_rate": 1.681286549707602e-08,
      "logits/chosen": -0.6726961135864258,
      "logits/rejected": -0.6820112466812134,
      "logps/chosen": -1053.5478515625,
      "logps/rejected": -947.9384155273438,
      "loss": 0.6787,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.025196075439453125,
      "rewards/margins": 0.030178263783454895,
      "rewards/rejected": -0.004982185550034046,
      "step": 3686
    },
    {
      "epoch": 0.9698005885350738,
      "grad_norm": 596.0,
      "learning_rate": 1.6666666666666667e-08,
      "logits/chosen": -0.6472095251083374,
      "logits/rejected": -0.6653664112091064,
      "logps/chosen": -1189.117431640625,
      "logps/rejected": -875.1129150390625,
      "loss": 0.6925,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.014418791979551315,
      "rewards/margins": 0.001958083361387253,
      "rewards/rejected": 0.012460710480809212,
      "step": 3687
    },
    {
      "epoch": 0.9700636209702609,
      "grad_norm": 402.0,
      "learning_rate": 1.652046783625731e-08,
      "logits/chosen": -0.6500493288040161,
      "logits/rejected": -0.6604065299034119,
      "logps/chosen": -726.4261474609375,
      "logps/rejected": -700.9683837890625,
      "loss": 0.6972,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.006486988626420498,
      "rewards/margins": -0.007366466801613569,
      "rewards/rejected": 0.0008794767782092094,
      "step": 3688
    },
    {
      "epoch": 0.9703266534054481,
      "grad_norm": 492.0,
      "learning_rate": 1.6374269005847953e-08,
      "logits/chosen": -0.6507377028465271,
      "logits/rejected": -0.6541205644607544,
      "logps/chosen": -1295.04443359375,
      "logps/rejected": -1023.0576782226562,
      "loss": 0.7005,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.01721353456377983,
      "rewards/margins": -0.013861943036317825,
      "rewards/rejected": 0.031075479462742805,
      "step": 3689
    },
    {
      "epoch": 0.9705896858406352,
      "grad_norm": 576.0,
      "learning_rate": 1.6228070175438593e-08,
      "logits/chosen": -0.6470456123352051,
      "logits/rejected": -0.6412432193756104,
      "logps/chosen": -1160.8179931640625,
      "logps/rejected": -921.3507080078125,
      "loss": 0.6992,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0032998076640069485,
      "rewards/margins": -0.010431957431137562,
      "rewards/rejected": 0.013731765560805798,
      "step": 3690
    },
    {
      "epoch": 0.9708527182758224,
      "grad_norm": 428.0,
      "learning_rate": 1.608187134502924e-08,
      "logits/chosen": -0.6632466912269592,
      "logits/rejected": -0.6765902638435364,
      "logps/chosen": -975.3798217773438,
      "logps/rejected": -847.5050659179688,
      "loss": 0.7028,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.009968089871108532,
      "rewards/margins": -0.018659211695194244,
      "rewards/rejected": 0.02862730249762535,
      "step": 3691
    },
    {
      "epoch": 0.9711157507110095,
      "grad_norm": 464.0,
      "learning_rate": 1.5935672514619882e-08,
      "logits/chosen": -0.6829199194908142,
      "logits/rejected": NaN,
      "logps/chosen": -766.581298828125,
      "logps/rejected": -679.057861328125,
      "loss": 0.6887,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.017476890236139297,
      "rewards/margins": 0.010102987289428711,
      "rewards/rejected": 0.007373905275017023,
      "step": 3692
    },
    {
      "epoch": 0.9713787831461967,
      "grad_norm": 510.0,
      "learning_rate": 1.5789473684210525e-08,
      "logits/chosen": -0.6375089287757874,
      "logits/rejected": -0.640126645565033,
      "logps/chosen": -1253.1624755859375,
      "logps/rejected": -1211.80224609375,
      "loss": 0.7057,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.005923175252974033,
      "rewards/margins": -0.024140167981386185,
      "rewards/rejected": 0.030063344165682793,
      "step": 3693
    },
    {
      "epoch": 0.9716418155813838,
      "grad_norm": 374.0,
      "learning_rate": 1.564327485380117e-08,
      "logits/chosen": -0.6340315341949463,
      "logits/rejected": -0.6380725502967834,
      "logps/chosen": -546.78857421875,
      "logps/rejected": -443.0596923828125,
      "loss": 0.6915,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0007369994418695569,
      "rewards/margins": 0.003634834662079811,
      "rewards/rejected": -0.0028978351037949324,
      "step": 3694
    },
    {
      "epoch": 0.9719048480165711,
      "grad_norm": 516.0,
      "learning_rate": 1.549707602339181e-08,
      "logits/chosen": -0.6265148520469666,
      "logits/rejected": -0.635733425617218,
      "logps/chosen": -923.4192504882812,
      "logps/rejected": -831.54833984375,
      "loss": 0.6872,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.010329723358154297,
      "rewards/margins": 0.012828541919589043,
      "rewards/rejected": -0.02315826527774334,
      "step": 3695
    },
    {
      "epoch": 0.9721678804517582,
      "grad_norm": 588.0,
      "learning_rate": 1.5350877192982455e-08,
      "logits/chosen": -0.6262720227241516,
      "logits/rejected": -0.6418746709823608,
      "logps/chosen": -1513.03271484375,
      "logps/rejected": -894.5862426757812,
      "loss": 0.6773,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03811988979578018,
      "rewards/margins": 0.033348750323057175,
      "rewards/rejected": 0.004771136678755283,
      "step": 3696
    },
    {
      "epoch": 0.9724309128869454,
      "grad_norm": 456.0,
      "learning_rate": 1.5204678362573098e-08,
      "logits/chosen": -0.6452531218528748,
      "logits/rejected": -0.6545727252960205,
      "logps/chosen": -1038.6717529296875,
      "logps/rejected": -792.2540283203125,
      "loss": 0.695,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0009577758610248566,
      "rewards/margins": -0.003190041519701481,
      "rewards/rejected": 0.0022322656586766243,
      "step": 3697
    },
    {
      "epoch": 0.9726939453221325,
      "grad_norm": 552.0,
      "learning_rate": 1.5058479532163744e-08,
      "logits/chosen": -0.6753573417663574,
      "logits/rejected": NaN,
      "logps/chosen": -1000.0925903320312,
      "logps/rejected": -816.60791015625,
      "loss": 0.6978,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01710968092083931,
      "rewards/margins": -0.008576011285185814,
      "rewards/rejected": -0.008533667773008347,
      "step": 3698
    },
    {
      "epoch": 0.9729569777573197,
      "grad_norm": 490.0,
      "learning_rate": 1.4912280701754384e-08,
      "logits/chosen": -0.6460564732551575,
      "logits/rejected": -0.6513080596923828,
      "logps/chosen": -985.5770874023438,
      "logps/rejected": -965.4955444335938,
      "loss": 0.6896,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02041010931134224,
      "rewards/margins": 0.008540010079741478,
      "rewards/rejected": 0.011870098300278187,
      "step": 3699
    },
    {
      "epoch": 0.9732200101925068,
      "grad_norm": 780.0,
      "learning_rate": 1.4766081871345029e-08,
      "logits/chosen": -0.6287484169006348,
      "logits/rejected": -0.627127468585968,
      "logps/chosen": -1020.7540283203125,
      "logps/rejected": -876.254638671875,
      "loss": 0.6963,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0068340301513671875,
      "rewards/margins": -0.0051725395023822784,
      "rewards/rejected": 0.012006567791104317,
      "step": 3700
    },
    {
      "epoch": 0.973483042627694,
      "grad_norm": 468.0,
      "learning_rate": 1.4619883040935672e-08,
      "logits/chosen": -0.6463375091552734,
      "logits/rejected": -0.6599588394165039,
      "logps/chosen": -1065.112060546875,
      "logps/rejected": -623.8089599609375,
      "loss": 0.6843,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02598886750638485,
      "rewards/margins": 0.018581770360469818,
      "rewards/rejected": 0.007407092489302158,
      "step": 3701
    },
    {
      "epoch": 0.9737460750628812,
      "grad_norm": 580.0,
      "learning_rate": 1.4473684210526315e-08,
      "logits/chosen": -0.6466684937477112,
      "logits/rejected": -0.6553696990013123,
      "logps/chosen": -1504.856689453125,
      "logps/rejected": -1350.4296875,
      "loss": 0.6941,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.013580035418272018,
      "rewards/margins": -0.0007648947648704052,
      "rewards/rejected": 0.014344930648803711,
      "step": 3702
    },
    {
      "epoch": 0.9740091074980683,
      "grad_norm": 524.0,
      "learning_rate": 1.432748538011696e-08,
      "logits/chosen": -0.6493467092514038,
      "logits/rejected": -0.6481364965438843,
      "logps/chosen": -928.8380737304688,
      "logps/rejected": -885.253662109375,
      "loss": 0.6924,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.006965828128159046,
      "rewards/margins": 0.0022797114215791225,
      "rewards/rejected": -0.009245539084076881,
      "step": 3703
    },
    {
      "epoch": 0.9742721399332556,
      "grad_norm": 600.0,
      "learning_rate": 1.4181286549707601e-08,
      "logits/chosen": -0.6712628602981567,
      "logits/rejected": -0.6698029637336731,
      "logps/chosen": -1215.6834716796875,
      "logps/rejected": -1138.079345703125,
      "loss": 0.6989,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.007605553604662418,
      "rewards/margins": -0.010472585447132587,
      "rewards/rejected": 0.0028670313768088818,
      "step": 3704
    },
    {
      "epoch": 0.9745351723684427,
      "grad_norm": 640.0,
      "learning_rate": 1.4035087719298246e-08,
      "logits/chosen": -0.6849843263626099,
      "logits/rejected": -0.6710667610168457,
      "logps/chosen": -1184.8463134765625,
      "logps/rejected": -931.357421875,
      "loss": 0.6987,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.001877402886748314,
      "rewards/margins": -0.010575483553111553,
      "rewards/rejected": 0.008698081597685814,
      "step": 3705
    },
    {
      "epoch": 0.9747982048036299,
      "grad_norm": 436.0,
      "learning_rate": 1.3888888888888887e-08,
      "logits/chosen": -0.6256542801856995,
      "logits/rejected": -0.6259094476699829,
      "logps/chosen": -736.1021118164062,
      "logps/rejected": -698.524169921875,
      "loss": 0.6913,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007246113382279873,
      "rewards/margins": 0.004502297379076481,
      "rewards/rejected": 0.0027438162360340357,
      "step": 3706
    },
    {
      "epoch": 0.975061237238817,
      "grad_norm": 524.0,
      "learning_rate": 1.3742690058479532e-08,
      "logits/chosen": -0.6301689147949219,
      "logits/rejected": -0.6335569620132446,
      "logps/chosen": -975.0182495117188,
      "logps/rejected": -646.4639892578125,
      "loss": 0.6922,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.016841985285282135,
      "rewards/margins": 0.0021893491502851248,
      "rewards/rejected": 0.014652634039521217,
      "step": 3707
    },
    {
      "epoch": 0.9753242696740042,
      "grad_norm": 704.0,
      "learning_rate": 1.3596491228070175e-08,
      "logits/chosen": -0.659173309803009,
      "logits/rejected": -0.6704741716384888,
      "logps/chosen": -1165.1707763671875,
      "logps/rejected": -800.9901123046875,
      "loss": 0.6976,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02153024822473526,
      "rewards/margins": -0.007834243588149548,
      "rewards/rejected": 0.029364492744207382,
      "step": 3708
    },
    {
      "epoch": 0.9755873021091913,
      "grad_norm": 564.0,
      "learning_rate": 1.3450292397660818e-08,
      "logits/chosen": -0.6494600176811218,
      "logits/rejected": -0.6669238805770874,
      "logps/chosen": -1202.9683837890625,
      "logps/rejected": -853.453857421875,
      "loss": 0.696,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.004216481931507587,
      "rewards/margins": -0.00447120750322938,
      "rewards/rejected": 0.0002547267358750105,
      "step": 3709
    },
    {
      "epoch": 0.9758503345443785,
      "grad_norm": 700.0,
      "learning_rate": 1.3304093567251461e-08,
      "logits/chosen": -0.6405373215675354,
      "logits/rejected": -0.6410971879959106,
      "logps/chosen": -1187.788818359375,
      "logps/rejected": -865.0184326171875,
      "loss": 0.6822,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.004827023483812809,
      "rewards/margins": 0.022553253918886185,
      "rewards/rejected": -0.01772623136639595,
      "step": 3710
    },
    {
      "epoch": 0.9761133669795656,
      "grad_norm": 600.0,
      "learning_rate": 1.3157894736842104e-08,
      "logits/chosen": -0.63765949010849,
      "logits/rejected": -0.6450225114822388,
      "logps/chosen": -1085.07470703125,
      "logps/rejected": -1105.581787109375,
      "loss": 0.694,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.002611733041703701,
      "rewards/margins": -0.0013572215102612972,
      "rewards/rejected": 0.003968954086303711,
      "step": 3711
    },
    {
      "epoch": 0.9763763994147528,
      "grad_norm": 820.0,
      "learning_rate": 1.3011695906432749e-08,
      "logits/chosen": -0.654403805732727,
      "logits/rejected": -0.6591957807540894,
      "logps/chosen": -1409.8404541015625,
      "logps/rejected": -1013.2894897460938,
      "loss": 0.7036,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03565540537238121,
      "rewards/margins": -0.019356584176421165,
      "rewards/rejected": -0.016298819333314896,
      "step": 3712
    },
    {
      "epoch": 0.97663943184994,
      "grad_norm": 418.0,
      "learning_rate": 1.2865497076023392e-08,
      "logits/chosen": -0.6776318550109863,
      "logits/rejected": -0.6725268959999084,
      "logps/chosen": -1033.4217529296875,
      "logps/rejected": -816.39013671875,
      "loss": 0.6907,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0006400110432878137,
      "rewards/margins": 0.005424500443041325,
      "rewards/rejected": -0.004784488584846258,
      "step": 3713
    },
    {
      "epoch": 0.9769024642851272,
      "grad_norm": 480.0,
      "learning_rate": 1.2719298245614035e-08,
      "logits/chosen": -0.6471365690231323,
      "logits/rejected": -0.6491914987564087,
      "logps/chosen": -993.0978393554688,
      "logps/rejected": -628.25390625,
      "loss": 0.6882,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.011140394024550915,
      "rewards/margins": 0.010766363702714443,
      "rewards/rejected": 0.000374031369574368,
      "step": 3714
    },
    {
      "epoch": 0.9771654967203143,
      "grad_norm": 496.0,
      "learning_rate": 1.2573099415204678e-08,
      "logits/chosen": -0.6367762088775635,
      "logits/rejected": -0.6433476805686951,
      "logps/chosen": -1274.02880859375,
      "logps/rejected": -1001.7074584960938,
      "loss": 0.6922,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0055217738263309,
      "rewards/margins": 0.002485274337232113,
      "rewards/rejected": 0.0030364985577762127,
      "step": 3715
    },
    {
      "epoch": 0.9774285291555015,
      "grad_norm": 580.0,
      "learning_rate": 1.2426900584795321e-08,
      "logits/chosen": -0.6788293719291687,
      "logits/rejected": -0.677128255367279,
      "logps/chosen": -1527.2264404296875,
      "logps/rejected": -1146.2120361328125,
      "loss": 0.6873,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007149314507842064,
      "rewards/margins": 0.012767935171723366,
      "rewards/rejected": -0.005618619732558727,
      "step": 3716
    },
    {
      "epoch": 0.9776915615906887,
      "grad_norm": 382.0,
      "learning_rate": 1.2280701754385964e-08,
      "logits/chosen": -0.6473948955535889,
      "logits/rejected": NaN,
      "logps/chosen": -1034.8109130859375,
      "logps/rejected": -696.8333740234375,
      "loss": 0.691,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.005596733186393976,
      "rewards/margins": 0.004642009269446135,
      "rewards/rejected": 0.0009547239169478416,
      "step": 3717
    },
    {
      "epoch": 0.9779545940258758,
      "grad_norm": 508.0,
      "learning_rate": 1.2134502923976609e-08,
      "logits/chosen": -0.6669297218322754,
      "logits/rejected": -0.6748805642127991,
      "logps/chosen": -1353.7491455078125,
      "logps/rejected": -1133.416259765625,
      "loss": 0.7016,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.023381996899843216,
      "rewards/margins": -0.016271067783236504,
      "rewards/rejected": -0.007110930513590574,
      "step": 3718
    },
    {
      "epoch": 0.978217626461063,
      "grad_norm": 544.0,
      "learning_rate": 1.198830409356725e-08,
      "logits/chosen": -0.640444278717041,
      "logits/rejected": -0.6291384696960449,
      "logps/chosen": -1167.611572265625,
      "logps/rejected": -896.3470458984375,
      "loss": 0.6889,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01710977591574192,
      "rewards/margins": 0.009132861159741879,
      "rewards/rejected": 0.007976913824677467,
      "step": 3719
    },
    {
      "epoch": 0.9784806588962501,
      "grad_norm": 438.0,
      "learning_rate": 1.1842105263157895e-08,
      "logits/chosen": -0.6456193327903748,
      "logits/rejected": -0.6395400762557983,
      "logps/chosen": -830.7965087890625,
      "logps/rejected": -815.5188598632812,
      "loss": 0.6987,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0006581309717148542,
      "rewards/margins": -0.010662270709872246,
      "rewards/rejected": 0.01132039912045002,
      "step": 3720
    },
    {
      "epoch": 0.9787436913314373,
      "grad_norm": 462.0,
      "learning_rate": 1.1695906432748536e-08,
      "logits/chosen": -0.6474482417106628,
      "logits/rejected": -0.6631388664245605,
      "logps/chosen": -904.5767822265625,
      "logps/rejected": -778.8714599609375,
      "loss": 0.7052,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.012630175799131393,
      "rewards/margins": -0.02341938018798828,
      "rewards/rejected": 0.010789204388856888,
      "step": 3721
    },
    {
      "epoch": 0.9790067237666245,
      "grad_norm": 660.0,
      "learning_rate": 1.1549707602339181e-08,
      "logits/chosen": -0.662979245185852,
      "logits/rejected": -0.6588938236236572,
      "logps/chosen": -1400.893310546875,
      "logps/rejected": -861.5513305664062,
      "loss": 0.7125,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.03405628353357315,
      "rewards/margins": -0.037047866731882095,
      "rewards/rejected": 0.0029915813356637955,
      "step": 3722
    },
    {
      "epoch": 0.9792697562018117,
      "grad_norm": 604.0,
      "learning_rate": 1.1403508771929824e-08,
      "logits/chosen": -0.6554350852966309,
      "logits/rejected": -0.6562175154685974,
      "logps/chosen": -1407.1595458984375,
      "logps/rejected": -1083.9688720703125,
      "loss": 0.6846,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0362059623003006,
      "rewards/margins": 0.01912546157836914,
      "rewards/rejected": 0.01708049885928631,
      "step": 3723
    },
    {
      "epoch": 0.9795327886369988,
      "grad_norm": 532.0,
      "learning_rate": 1.1257309941520467e-08,
      "logits/chosen": -0.6303397417068481,
      "logits/rejected": -0.6389692425727844,
      "logps/chosen": -1158.2059326171875,
      "logps/rejected": -1061.8621826171875,
      "loss": 0.6946,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0035329817328602076,
      "rewards/margins": -0.001903916010633111,
      "rewards/rejected": 0.005436896812170744,
      "step": 3724
    },
    {
      "epoch": 0.979795821072186,
      "grad_norm": 370.0,
      "learning_rate": 1.111111111111111e-08,
      "logits/chosen": -0.6497615575790405,
      "logits/rejected": -0.6505938768386841,
      "logps/chosen": -865.3934326171875,
      "logps/rejected": -643.4693603515625,
      "loss": 0.6975,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.013963795267045498,
      "rewards/margins": -0.008039474487304688,
      "rewards/rejected": -0.00592432077974081,
      "step": 3725
    },
    {
      "epoch": 0.9800588535073731,
      "grad_norm": 536.0,
      "learning_rate": 1.0964912280701753e-08,
      "logits/chosen": -0.6605362892150879,
      "logits/rejected": -0.6683392524719238,
      "logps/chosen": -1329.923583984375,
      "logps/rejected": -1086.1884765625,
      "loss": 0.6867,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0035534861963242292,
      "rewards/margins": 0.014456797391176224,
      "rewards/rejected": -0.01090331096202135,
      "step": 3726
    },
    {
      "epoch": 0.9803218859425603,
      "grad_norm": 616.0,
      "learning_rate": 1.0818713450292398e-08,
      "logits/chosen": -0.651559054851532,
      "logits/rejected": NaN,
      "logps/chosen": -1382.0987548828125,
      "logps/rejected": -973.28076171875,
      "loss": 0.6951,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.011132145300507545,
      "rewards/margins": -0.0023568167816847563,
      "rewards/rejected": 0.01348896138370037,
      "step": 3727
    },
    {
      "epoch": 0.9805849183777474,
      "grad_norm": 568.0,
      "learning_rate": 1.0672514619883041e-08,
      "logits/chosen": -0.6355002522468567,
      "logits/rejected": -0.6320976614952087,
      "logps/chosen": -1340.6995849609375,
      "logps/rejected": -1000.8758544921875,
      "loss": 0.6924,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.012518595904111862,
      "rewards/margins": 0.0021122931502759457,
      "rewards/rejected": 0.010406304150819778,
      "step": 3728
    },
    {
      "epoch": 0.9808479508129346,
      "grad_norm": 564.0,
      "learning_rate": 1.0526315789473684e-08,
      "logits/chosen": -0.6390841007232666,
      "logits/rejected": NaN,
      "logps/chosen": -1230.958984375,
      "logps/rejected": -888.6361083984375,
      "loss": 0.6915,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.005717849358916283,
      "rewards/margins": 0.004592893645167351,
      "rewards/rejected": -0.010310743935406208,
      "step": 3729
    },
    {
      "epoch": 0.9811109832481217,
      "grad_norm": 504.0,
      "learning_rate": 1.0380116959064327e-08,
      "logits/chosen": -0.628646194934845,
      "logits/rejected": -0.6360273957252502,
      "logps/chosen": -1006.9730224609375,
      "logps/rejected": -840.7105102539062,
      "loss": 0.6851,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.019911767914891243,
      "rewards/margins": 0.016774559393525124,
      "rewards/rejected": 0.0031372078228741884,
      "step": 3730
    },
    {
      "epoch": 0.981374015683309,
      "grad_norm": 708.0,
      "learning_rate": 1.023391812865497e-08,
      "logits/chosen": -0.6495046615600586,
      "logits/rejected": -0.6715840101242065,
      "logps/chosen": -1549.3594970703125,
      "logps/rejected": -862.3717041015625,
      "loss": 0.6934,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0022634502965956926,
      "rewards/margins": 0.0005841243546456099,
      "rewards/rejected": 0.0016793252434581518,
      "step": 3731
    },
    {
      "epoch": 0.9816370481184961,
      "grad_norm": 548.0,
      "learning_rate": 1.0087719298245613e-08,
      "logits/chosen": -0.632577121257782,
      "logits/rejected": -0.631367564201355,
      "logps/chosen": -1163.47705078125,
      "logps/rejected": -840.99560546875,
      "loss": 0.6839,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.016759108752012253,
      "rewards/margins": 0.019844816997647285,
      "rewards/rejected": -0.0030857082456350327,
      "step": 3732
    },
    {
      "epoch": 0.9819000805536833,
      "grad_norm": 576.0,
      "learning_rate": 9.941520467836258e-09,
      "logits/chosen": -0.6502166986465454,
      "logits/rejected": -0.6454451680183411,
      "logps/chosen": -1041.870361328125,
      "logps/rejected": -800.4879150390625,
      "loss": 0.6842,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.007137775886803865,
      "rewards/margins": 0.01873145066201687,
      "rewards/rejected": -0.01159367524087429,
      "step": 3733
    },
    {
      "epoch": 0.9821631129888705,
      "grad_norm": 458.0,
      "learning_rate": 9.7953216374269e-09,
      "logits/chosen": -0.6547713875770569,
      "logits/rejected": -0.6478408575057983,
      "logps/chosen": -1123.39208984375,
      "logps/rejected": -976.2310791015625,
      "loss": 0.6882,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 3.833754453808069e-05,
      "rewards/margins": 0.010568905621767044,
      "rewards/rejected": -0.010530567727982998,
      "step": 3734
    },
    {
      "epoch": 0.9824261454240576,
      "grad_norm": 564.0,
      "learning_rate": 9.649122807017544e-09,
      "logits/chosen": -0.6490260362625122,
      "logits/rejected": -0.6583230495452881,
      "logps/chosen": -1032.861083984375,
      "logps/rejected": -777.8314208984375,
      "loss": 0.6944,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.011350203305482864,
      "rewards/margins": -0.0016387469368055463,
      "rewards/rejected": -0.00971145648509264,
      "step": 3735
    },
    {
      "epoch": 0.9826891778592448,
      "grad_norm": 556.0,
      "learning_rate": 9.502923976608186e-09,
      "logits/chosen": -0.6609063744544983,
      "logits/rejected": -0.6668353080749512,
      "logps/chosen": -1389.4974365234375,
      "logps/rejected": -844.7257080078125,
      "loss": 0.6921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012459563091397285,
      "rewards/margins": 0.003049277700483799,
      "rewards/rejected": 0.009410286322236061,
      "step": 3736
    },
    {
      "epoch": 0.9829522102944319,
      "grad_norm": 616.0,
      "learning_rate": 9.35672514619883e-09,
      "logits/chosen": -0.6495007872581482,
      "logits/rejected": -0.651715874671936,
      "logps/chosen": -1436.33740234375,
      "logps/rejected": -918.18798828125,
      "loss": 0.6794,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.007639121264219284,
      "rewards/margins": 0.028923990204930305,
      "rewards/rejected": -0.021284867078065872,
      "step": 3737
    },
    {
      "epoch": 0.9832152427296191,
      "grad_norm": 500.0,
      "learning_rate": 9.210526315789472e-09,
      "logits/chosen": -0.6384575366973877,
      "logits/rejected": -0.6493431329727173,
      "logps/chosen": -1107.9427490234375,
      "logps/rejected": -808.049072265625,
      "loss": 0.6873,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0033023832365870476,
      "rewards/margins": 0.012289619073271751,
      "rewards/rejected": -0.008987235836684704,
      "step": 3738
    },
    {
      "epoch": 0.9834782751648062,
      "grad_norm": 480.0,
      "learning_rate": 9.064327485380117e-09,
      "logits/chosen": -0.6282951235771179,
      "logits/rejected": -0.6399606466293335,
      "logps/chosen": -1039.2109375,
      "logps/rejected": -837.0123291015625,
      "loss": 0.6953,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.002427483908832073,
      "rewards/margins": -0.003026484977453947,
      "rewards/rejected": 0.005453967954963446,
      "step": 3739
    },
    {
      "epoch": 0.9837413075999935,
      "grad_norm": 532.0,
      "learning_rate": 8.91812865497076e-09,
      "logits/chosen": -0.6644331812858582,
      "logits/rejected": -0.6592651605606079,
      "logps/chosen": -1195.408447265625,
      "logps/rejected": -1013.2694091796875,
      "loss": 0.6849,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.023195507004857063,
      "rewards/margins": 0.01680455170571804,
      "rewards/rejected": 0.006390952505171299,
      "step": 3740
    },
    {
      "epoch": 0.9840043400351806,
      "grad_norm": 624.0,
      "learning_rate": 8.771929824561403e-09,
      "logits/chosen": -0.6426019072532654,
      "logits/rejected": -0.6414057612419128,
      "logps/chosen": -1182.0291748046875,
      "logps/rejected": -915.6509399414062,
      "loss": 0.6785,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.014438441954553127,
      "rewards/margins": 0.03076324239373207,
      "rewards/rejected": -0.016324806958436966,
      "step": 3741
    },
    {
      "epoch": 0.9842673724703678,
      "grad_norm": 470.0,
      "learning_rate": 8.625730994152047e-09,
      "logits/chosen": -0.6447765231132507,
      "logits/rejected": -0.6507908701896667,
      "logps/chosen": -789.3982543945312,
      "logps/rejected": -671.189697265625,
      "loss": 0.6924,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0021194457076489925,
      "rewards/margins": 0.004041576758027077,
      "rewards/rejected": -0.0019221312832087278,
      "step": 3742
    },
    {
      "epoch": 0.9845304049055549,
      "grad_norm": 548.0,
      "learning_rate": 8.479532163742689e-09,
      "logits/chosen": -0.6791090965270996,
      "logits/rejected": -0.678197979927063,
      "logps/chosen": -1126.83203125,
      "logps/rejected": -1026.6334228515625,
      "loss": 0.6943,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.009707260876893997,
      "rewards/margins": -0.000994301401078701,
      "rewards/rejected": -0.008712958544492722,
      "step": 3743
    },
    {
      "epoch": 0.9847934373407421,
      "grad_norm": 616.0,
      "learning_rate": 8.333333333333334e-09,
      "logits/chosen": -0.6650434136390686,
      "logits/rejected": -0.6578271985054016,
      "logps/chosen": -1467.374755859375,
      "logps/rejected": -1246.935791015625,
      "loss": 0.692,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0058931345120072365,
      "rewards/margins": 0.003793048206716776,
      "rewards/rejected": -0.009686185047030449,
      "step": 3744
    },
    {
      "epoch": 0.9850564697759292,
      "grad_norm": 502.0,
      "learning_rate": 8.187134502923977e-09,
      "logits/chosen": -0.6600420475006104,
      "logits/rejected": -0.6465473175048828,
      "logps/chosen": -1078.023193359375,
      "logps/rejected": -1047.29833984375,
      "loss": 0.689,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.015838338062167168,
      "rewards/margins": 0.009472941979765892,
      "rewards/rejected": 0.006365396548062563,
      "step": 3745
    },
    {
      "epoch": 0.9853195022111164,
      "grad_norm": 648.0,
      "learning_rate": 8.04093567251462e-09,
      "logits/chosen": -0.6494150161743164,
      "logits/rejected": -0.6489682793617249,
      "logps/chosen": -1633.5213623046875,
      "logps/rejected": -1259.350341796875,
      "loss": 0.7036,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.01102361734956503,
      "rewards/margins": -0.019559288397431374,
      "rewards/rejected": 0.008535671979188919,
      "step": 3746
    },
    {
      "epoch": 0.9855825346463035,
      "grad_norm": 564.0,
      "learning_rate": 7.894736842105263e-09,
      "logits/chosen": -0.6484724879264832,
      "logits/rejected": -0.6397593021392822,
      "logps/chosen": -1205.81591796875,
      "logps/rejected": -867.8787841796875,
      "loss": 0.692,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.016136743128299713,
      "rewards/margins": 0.003930855542421341,
      "rewards/rejected": -0.020067596808075905,
      "step": 3747
    },
    {
      "epoch": 0.9858455670814907,
      "grad_norm": 476.0,
      "learning_rate": 7.748538011695906e-09,
      "logits/chosen": -0.6648894548416138,
      "logits/rejected": -0.664158821105957,
      "logps/chosen": -1172.14453125,
      "logps/rejected": -1027.90478515625,
      "loss": 0.6985,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.013193322345614433,
      "rewards/margins": -0.010458040982484818,
      "rewards/rejected": 0.0236513614654541,
      "step": 3748
    },
    {
      "epoch": 0.986108599516678,
      "grad_norm": 636.0,
      "learning_rate": 7.602339181286549e-09,
      "logits/chosen": -0.6202434301376343,
      "logits/rejected": -0.6233750581741333,
      "logps/chosen": -1065.7252197265625,
      "logps/rejected": -829.677490234375,
      "loss": 0.6962,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0019824975170195103,
      "rewards/margins": -0.0053830137476325035,
      "rewards/rejected": 0.003400516463443637,
      "step": 3749
    },
    {
      "epoch": 0.9863716319518651,
      "grad_norm": 468.0,
      "learning_rate": 7.456140350877192e-09,
      "logits/chosen": -0.6378900408744812,
      "logits/rejected": -0.6491154432296753,
      "logps/chosen": -1129.457275390625,
      "logps/rejected": -911.16064453125,
      "loss": 0.6856,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.019836140796542168,
      "rewards/margins": 0.015793897211551666,
      "rewards/rejected": 0.004042244050651789,
      "step": 3750
    },
    {
      "epoch": 0.9866346643870523,
      "grad_norm": 450.0,
      "learning_rate": 7.309941520467836e-09,
      "logits/chosen": -0.6269328594207764,
      "logits/rejected": -0.6253483891487122,
      "logps/chosen": -967.0362548828125,
      "logps/rejected": -583.60302734375,
      "loss": 0.6893,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.002660084515810013,
      "rewards/margins": 0.007916642352938652,
      "rewards/rejected": -0.005256557837128639,
      "step": 3751
    },
    {
      "epoch": 0.9868976968222394,
      "grad_norm": 520.0,
      "learning_rate": 7.16374269005848e-09,
      "logits/chosen": -0.6498017907142639,
      "logits/rejected": -0.6563748717308044,
      "logps/chosen": -976.703125,
      "logps/rejected": -819.5745849609375,
      "loss": 0.6942,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0007302288431674242,
      "rewards/margins": -0.0007504462264478207,
      "rewards/rejected": 2.0218401914462447e-05,
      "step": 3752
    },
    {
      "epoch": 0.9871607292574266,
      "grad_norm": 400.0,
      "learning_rate": 7.017543859649123e-09,
      "logits/chosen": -0.646841824054718,
      "logits/rejected": -0.6454628109931946,
      "logps/chosen": -801.099609375,
      "logps/rejected": -462.5408020019531,
      "loss": 0.7059,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.019234562292695045,
      "rewards/margins": -0.024610137566924095,
      "rewards/rejected": 0.005375576205551624,
      "step": 3753
    },
    {
      "epoch": 0.9874237616926137,
      "grad_norm": 572.0,
      "learning_rate": 6.871345029239766e-09,
      "logits/chosen": -0.650267481803894,
      "logits/rejected": -0.6601675152778625,
      "logps/chosen": -1183.302490234375,
      "logps/rejected": -878.3145751953125,
      "loss": 0.7073,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.008990001864731312,
      "rewards/margins": -0.026949025690555573,
      "rewards/rejected": 0.017959021031856537,
      "step": 3754
    },
    {
      "epoch": 0.9876867941278009,
      "grad_norm": 488.0,
      "learning_rate": 6.725146198830409e-09,
      "logits/chosen": -0.667273759841919,
      "logits/rejected": -0.6717514991760254,
      "logps/chosen": -1323.5242919921875,
      "logps/rejected": -870.9434204101562,
      "loss": 0.7116,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.013956641778349876,
      "rewards/margins": -0.035733889788389206,
      "rewards/rejected": 0.02177724987268448,
      "step": 3755
    },
    {
      "epoch": 0.987949826562988,
      "grad_norm": 458.0,
      "learning_rate": 6.578947368421052e-09,
      "logits/chosen": -0.6535683274269104,
      "logits/rejected": -0.651261031627655,
      "logps/chosen": -856.8758544921875,
      "logps/rejected": -801.8547973632812,
      "loss": 0.6992,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.017865657806396484,
      "rewards/margins": -0.01073541585355997,
      "rewards/rejected": -0.007130242418497801,
      "step": 3756
    },
    {
      "epoch": 0.9882128589981752,
      "grad_norm": 592.0,
      "learning_rate": 6.432748538011696e-09,
      "logits/chosen": -0.6394504308700562,
      "logits/rejected": -0.656962513923645,
      "logps/chosen": -1086.1605224609375,
      "logps/rejected": -828.7553100585938,
      "loss": 0.7068,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.017497632652521133,
      "rewards/margins": -0.0248552318662405,
      "rewards/rejected": 0.007357597816735506,
      "step": 3757
    },
    {
      "epoch": 0.9884758914333623,
      "grad_norm": 584.0,
      "learning_rate": 6.286549707602339e-09,
      "logits/chosen": -0.6642867922782898,
      "logits/rejected": NaN,
      "logps/chosen": -1374.673583984375,
      "logps/rejected": -987.1478271484375,
      "loss": 0.6855,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0018058775458484888,
      "rewards/margins": 0.016452694311738014,
      "rewards/rejected": -0.01825857162475586,
      "step": 3758
    },
    {
      "epoch": 0.9887389238685496,
      "grad_norm": 572.0,
      "learning_rate": 6.140350877192982e-09,
      "logits/chosen": -0.6446136236190796,
      "logits/rejected": -0.6354387402534485,
      "logps/chosen": -1062.3887939453125,
      "logps/rejected": -648.6693115234375,
      "loss": 0.6991,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.015560436993837357,
      "rewards/margins": -0.011321830563247204,
      "rewards/rejected": -0.004238605499267578,
      "step": 3759
    },
    {
      "epoch": 0.9890019563037367,
      "grad_norm": 548.0,
      "learning_rate": 5.994152046783625e-09,
      "logits/chosen": -0.640252947807312,
      "logits/rejected": -0.6400513052940369,
      "logps/chosen": -1203.4771728515625,
      "logps/rejected": -988.9998168945312,
      "loss": 0.6805,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02666444703936577,
      "rewards/margins": 0.026675604283809662,
      "rewards/rejected": -1.1157477274537086e-05,
      "step": 3760
    },
    {
      "epoch": 0.9892649887389239,
      "grad_norm": 452.0,
      "learning_rate": 5.847953216374268e-09,
      "logits/chosen": -0.658409059047699,
      "logits/rejected": -0.6700496673583984,
      "logps/chosen": -968.1329956054688,
      "logps/rejected": -1009.0603637695312,
      "loss": 0.7059,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.012692069634795189,
      "rewards/margins": -0.02507495880126953,
      "rewards/rejected": 0.012382889166474342,
      "step": 3761
    },
    {
      "epoch": 0.989528021174111,
      "grad_norm": 560.0,
      "learning_rate": 5.701754385964912e-09,
      "logits/chosen": -0.6348711848258972,
      "logits/rejected": -0.6607900857925415,
      "logps/chosen": -1336.710205078125,
      "logps/rejected": -884.5047607421875,
      "loss": 0.6889,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.002095985459163785,
      "rewards/margins": 0.00944375991821289,
      "rewards/rejected": -0.007347775157541037,
      "step": 3762
    },
    {
      "epoch": 0.9897910536092982,
      "grad_norm": 624.0,
      "learning_rate": 5.555555555555555e-09,
      "logits/chosen": -0.6338907480239868,
      "logits/rejected": -0.6314162611961365,
      "logps/chosen": -1494.412841796875,
      "logps/rejected": -1049.143798828125,
      "loss": 0.6908,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004014587961137295,
      "rewards/margins": 0.005159283056855202,
      "rewards/rejected": -0.001144695095717907,
      "step": 3763
    },
    {
      "epoch": 0.9900540860444854,
      "grad_norm": 568.0,
      "learning_rate": 5.409356725146199e-09,
      "logits/chosen": -0.6539363265037537,
      "logits/rejected": -0.6541299819946289,
      "logps/chosen": -1344.4947509765625,
      "logps/rejected": -1201.3541259765625,
      "loss": 0.6821,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.004969214089214802,
      "rewards/margins": 0.0227724090218544,
      "rewards/rejected": -0.027741622179746628,
      "step": 3764
    },
    {
      "epoch": 0.9903171184796725,
      "grad_norm": 456.0,
      "learning_rate": 5.263157894736842e-09,
      "logits/chosen": -0.6553119421005249,
      "logits/rejected": -0.6595979928970337,
      "logps/chosen": -881.0692749023438,
      "logps/rejected": -657.4501342773438,
      "loss": 0.6987,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.012490654364228249,
      "rewards/margins": -0.010412121191620827,
      "rewards/rejected": 0.022902775555849075,
      "step": 3765
    },
    {
      "epoch": 0.9905801509148597,
      "grad_norm": 560.0,
      "learning_rate": 5.116959064327485e-09,
      "logits/chosen": -0.6531348824501038,
      "logits/rejected": -0.6491808295249939,
      "logps/chosen": -1260.7191162109375,
      "logps/rejected": -861.709228515625,
      "loss": 0.688,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.009097672998905182,
      "rewards/margins": 0.010770797729492188,
      "rewards/rejected": -0.01986847072839737,
      "step": 3766
    },
    {
      "epoch": 0.9908431833500468,
      "grad_norm": 536.0,
      "learning_rate": 4.970760233918129e-09,
      "logits/chosen": -0.6468877792358398,
      "logits/rejected": -0.6551709771156311,
      "logps/chosen": -1035.5579833984375,
      "logps/rejected": -790.6180419921875,
      "loss": 0.6952,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.009172534570097923,
      "rewards/margins": -0.003061770461499691,
      "rewards/rejected": -0.00611076457425952,
      "step": 3767
    },
    {
      "epoch": 0.9911062157852341,
      "grad_norm": 588.0,
      "learning_rate": 4.824561403508772e-09,
      "logits/chosen": -0.6689450144767761,
      "logits/rejected": -0.6722847819328308,
      "logps/chosen": -1424.2724609375,
      "logps/rejected": -992.9471435546875,
      "loss": 0.6932,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.004378700628876686,
      "rewards/margins": 0.0001526834676042199,
      "rewards/rejected": -0.0045313844457268715,
      "step": 3768
    },
    {
      "epoch": 0.9913692482204212,
      "grad_norm": 556.0,
      "learning_rate": 4.678362573099415e-09,
      "logits/chosen": -0.6404889822006226,
      "logits/rejected": -0.6412849426269531,
      "logps/chosen": -1120.8363037109375,
      "logps/rejected": -1090.930419921875,
      "loss": 0.6918,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.014888191595673561,
      "rewards/margins": 0.003806876949965954,
      "rewards/rejected": 0.011081313714385033,
      "step": 3769
    },
    {
      "epoch": 0.9916322806556084,
      "grad_norm": 510.0,
      "learning_rate": 4.532163742690058e-09,
      "logits/chosen": -0.6454300880432129,
      "logits/rejected": -0.6470043659210205,
      "logps/chosen": -958.79833984375,
      "logps/rejected": -669.1065673828125,
      "loss": 0.7024,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0027042394503951073,
      "rewards/margins": -0.017972756177186966,
      "rewards/rejected": 0.0206769946962595,
      "step": 3770
    },
    {
      "epoch": 0.9918953130907955,
      "grad_norm": 536.0,
      "learning_rate": 4.385964912280701e-09,
      "logits/chosen": -0.6349174976348877,
      "logits/rejected": -0.6380563974380493,
      "logps/chosen": -967.9985961914062,
      "logps/rejected": -941.6422119140625,
      "loss": 0.6867,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.009718704968690872,
      "rewards/margins": 0.014170074835419655,
      "rewards/rejected": -0.00445137033239007,
      "step": 3771
    },
    {
      "epoch": 0.9921583455259827,
      "grad_norm": 560.0,
      "learning_rate": 4.239766081871344e-09,
      "logits/chosen": -0.6618072986602783,
      "logits/rejected": -0.6630210280418396,
      "logps/chosen": -816.4884033203125,
      "logps/rejected": -787.4816284179688,
      "loss": 0.6952,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.007110023871064186,
      "rewards/margins": -0.0033659455366432667,
      "rewards/rejected": -0.0037440776359289885,
      "step": 3772
    },
    {
      "epoch": 0.9924213779611698,
      "grad_norm": 496.0,
      "learning_rate": 4.093567251461988e-09,
      "logits/chosen": -0.6619963645935059,
      "logits/rejected": -0.6714959740638733,
      "logps/chosen": -1114.78662109375,
      "logps/rejected": -862.1914672851562,
      "loss": 0.6994,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02005481719970703,
      "rewards/margins": -0.011555385775864124,
      "rewards/rejected": -0.008499432355165482,
      "step": 3773
    },
    {
      "epoch": 0.992684410396357,
      "grad_norm": 632.0,
      "learning_rate": 3.947368421052631e-09,
      "logits/chosen": -0.6477305889129639,
      "logits/rejected": -0.646041750907898,
      "logps/chosen": -1294.515380859375,
      "logps/rejected": -877.783203125,
      "loss": 0.6966,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.000842951936647296,
      "rewards/margins": -0.005682754795998335,
      "rewards/rejected": 0.0065257069654762745,
      "step": 3774
    },
    {
      "epoch": 0.9929474428315441,
      "grad_norm": 504.0,
      "learning_rate": 3.8011695906432744e-09,
      "logits/chosen": -0.644190788269043,
      "logits/rejected": -0.655518114566803,
      "logps/chosen": -1149.142822265625,
      "logps/rejected": -727.2357788085938,
      "loss": 0.691,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.009537409991025925,
      "rewards/margins": 0.0047629838809370995,
      "rewards/rejected": 0.004774427507072687,
      "step": 3775
    },
    {
      "epoch": 0.9932104752667313,
      "grad_norm": 688.0,
      "learning_rate": 3.654970760233918e-09,
      "logits/chosen": -0.6559203863143921,
      "logits/rejected": -0.665469229221344,
      "logps/chosen": -1631.85986328125,
      "logps/rejected": -1067.382568359375,
      "loss": 0.7015,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004141045268625021,
      "rewards/margins": -0.01558017823845148,
      "rewards/rejected": 0.019721223041415215,
      "step": 3776
    },
    {
      "epoch": 0.9934735077019184,
      "grad_norm": 458.0,
      "learning_rate": 3.5087719298245614e-09,
      "logits/chosen": -0.6351502537727356,
      "logits/rejected": -0.6449070572853088,
      "logps/chosen": -940.953857421875,
      "logps/rejected": -717.211669921875,
      "loss": 0.6961,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.008180619217455387,
      "rewards/margins": -0.004487133584916592,
      "rewards/rejected": -0.003693485166877508,
      "step": 3777
    },
    {
      "epoch": 0.9937365401371057,
      "grad_norm": 440.0,
      "learning_rate": 3.3625730994152045e-09,
      "logits/chosen": -0.6403084397315979,
      "logits/rejected": -0.643638014793396,
      "logps/chosen": -1237.0736083984375,
      "logps/rejected": -1022.31884765625,
      "loss": 0.7005,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.016495466232299805,
      "rewards/margins": -0.01415238343179226,
      "rewards/rejected": -0.002343081869184971,
      "step": 3778
    },
    {
      "epoch": 0.9939995725722928,
      "grad_norm": 560.0,
      "learning_rate": 3.216374269005848e-09,
      "logits/chosen": -0.6803306341171265,
      "logits/rejected": -0.6803148984909058,
      "logps/chosen": -890.49462890625,
      "logps/rejected": -978.0850830078125,
      "loss": 0.6929,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.015111733227968216,
      "rewards/margins": 0.0016632070764899254,
      "rewards/rejected": -0.016774939373135567,
      "step": 3779
    },
    {
      "epoch": 0.99426260500748,
      "grad_norm": 494.0,
      "learning_rate": 3.070175438596491e-09,
      "logits/chosen": -0.6483170390129089,
      "logits/rejected": -0.6500400900840759,
      "logps/chosen": -1376.981201171875,
      "logps/rejected": -887.4374389648438,
      "loss": 0.6983,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.002719402778893709,
      "rewards/margins": -0.009472372010350227,
      "rewards/rejected": 0.006752968765795231,
      "step": 3780
    },
    {
      "epoch": 0.9945256374426672,
      "grad_norm": 494.0,
      "learning_rate": 2.923976608187134e-09,
      "logits/chosen": -0.6648719906806946,
      "logits/rejected": -0.6775849461555481,
      "logps/chosen": -1317.3704833984375,
      "logps/rejected": -939.9285888671875,
      "loss": 0.6961,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.012105559930205345,
      "rewards/margins": -0.005390262696892023,
      "rewards/rejected": 0.01749582216143608,
      "step": 3781
    },
    {
      "epoch": 0.9947886698778543,
      "grad_norm": 424.0,
      "learning_rate": 2.7777777777777776e-09,
      "logits/chosen": -0.6580301523208618,
      "logits/rejected": -0.6600703001022339,
      "logps/chosen": -1135.207275390625,
      "logps/rejected": -869.824462890625,
      "loss": 0.6996,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.006536912638694048,
      "rewards/margins": -0.01238551177084446,
      "rewards/rejected": 0.00584859773516655,
      "step": 3782
    },
    {
      "epoch": 0.9950517023130415,
      "grad_norm": 498.0,
      "learning_rate": 2.631578947368421e-09,
      "logits/chosen": -0.6575119495391846,
      "logits/rejected": -0.6740645170211792,
      "logps/chosen": -1241.6357421875,
      "logps/rejected": -902.6969604492188,
      "loss": 0.6942,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.005534839816391468,
      "rewards/margins": -0.0015354156494140625,
      "rewards/rejected": -0.003999424632638693,
      "step": 3783
    },
    {
      "epoch": 0.9953147347482286,
      "grad_norm": 494.0,
      "learning_rate": 2.4853801169590645e-09,
      "logits/chosen": -0.6449957489967346,
      "logits/rejected": -0.6556384563446045,
      "logps/chosen": -1092.9769287109375,
      "logps/rejected": -666.14892578125,
      "loss": 0.6849,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02437434159219265,
      "rewards/margins": 0.017385249957442284,
      "rewards/rejected": 0.006989091634750366,
      "step": 3784
    },
    {
      "epoch": 0.9955777671834158,
      "grad_norm": 536.0,
      "learning_rate": 2.3391812865497076e-09,
      "logits/chosen": -0.660904586315155,
      "logits/rejected": -0.6717630624771118,
      "logps/chosen": -1092.798583984375,
      "logps/rejected": -931.2366333007812,
      "loss": 0.6921,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.005208588670939207,
      "rewards/margins": 0.0026412950828671455,
      "rewards/rejected": -0.007849883288145065,
      "step": 3785
    },
    {
      "epoch": 0.995840799618603,
      "grad_norm": 580.0,
      "learning_rate": 2.1929824561403507e-09,
      "logits/chosen": -0.6498472094535828,
      "logits/rejected": -0.6398216485977173,
      "logps/chosen": -1161.79541015625,
      "logps/rejected": -964.65673828125,
      "loss": 0.6885,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.014545916579663754,
      "rewards/margins": 0.010391712188720703,
      "rewards/rejected": 0.004154205322265625,
      "step": 3786
    },
    {
      "epoch": 0.9961038320537902,
      "grad_norm": 488.0,
      "learning_rate": 2.046783625730994e-09,
      "logits/chosen": -0.6546145081520081,
      "logits/rejected": -0.6540607213973999,
      "logps/chosen": -1099.6534423828125,
      "logps/rejected": -960.9871215820312,
      "loss": 0.6895,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.019106483086943626,
      "rewards/margins": 0.008243083953857422,
      "rewards/rejected": 0.01086340006440878,
      "step": 3787
    },
    {
      "epoch": 0.9963668644889773,
      "grad_norm": 438.0,
      "learning_rate": 1.9005847953216372e-09,
      "logits/chosen": -0.6728687286376953,
      "logits/rejected": -0.6767741441726685,
      "logps/chosen": -998.0999755859375,
      "logps/rejected": -765.9454956054688,
      "loss": 0.6883,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004333686549216509,
      "rewards/margins": 0.010333538055419922,
      "rewards/rejected": -0.005999851040542126,
      "step": 3788
    },
    {
      "epoch": 0.9966298969241645,
      "grad_norm": 494.0,
      "learning_rate": 1.7543859649122807e-09,
      "logits/chosen": -0.6808867454528809,
      "logits/rejected": -0.6669116616249084,
      "logps/chosen": -1341.251220703125,
      "logps/rejected": -977.2194213867188,
      "loss": 0.6959,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.009070015512406826,
      "rewards/margins": -0.005059526301920414,
      "rewards/rejected": 0.01412954367697239,
      "step": 3789
    },
    {
      "epoch": 0.9968929293593516,
      "grad_norm": 580.0,
      "learning_rate": 1.608187134502924e-09,
      "logits/chosen": -0.6509830355644226,
      "logits/rejected": -0.6764453053474426,
      "logps/chosen": -1413.78076171875,
      "logps/rejected": -1029.342041015625,
      "loss": 0.6945,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.006590843200683594,
      "rewards/margins": -0.0014112461358308792,
      "rewards/rejected": 0.008002091199159622,
      "step": 3790
    },
    {
      "epoch": 0.9971559617945388,
      "grad_norm": 472.0,
      "learning_rate": 1.461988304093567e-09,
      "logits/chosen": -0.6613339185714722,
      "logits/rejected": -0.6641010046005249,
      "logps/chosen": -939.6774291992188,
      "logps/rejected": -706.87548828125,
      "loss": 0.6969,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.018255043774843216,
      "rewards/margins": -0.007141447160393,
      "rewards/rejected": -0.011113596148788929,
      "step": 3791
    },
    {
      "epoch": 0.9974189942297259,
      "grad_norm": 676.0,
      "learning_rate": 1.3157894736842105e-09,
      "logits/chosen": -0.6441837549209595,
      "logits/rejected": -0.6553003787994385,
      "logps/chosen": -1173.4339599609375,
      "logps/rejected": -1023.666259765625,
      "loss": 0.6927,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010010911151766777,
      "rewards/margins": 0.0019073518924415112,
      "rewards/rejected": 0.008103560656309128,
      "step": 3792
    },
    {
      "epoch": 0.9976820266649131,
      "grad_norm": 528.0,
      "learning_rate": 1.1695906432748538e-09,
      "logits/chosen": -0.6560029983520508,
      "logits/rejected": -0.6577152609825134,
      "logps/chosen": -1023.19970703125,
      "logps/rejected": -906.7197265625,
      "loss": 0.6983,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.015798427164554596,
      "rewards/margins": -0.009850358590483665,
      "rewards/rejected": -0.0059480671770870686,
      "step": 3793
    },
    {
      "epoch": 0.9979450591001002,
      "grad_norm": 520.0,
      "learning_rate": 1.023391812865497e-09,
      "logits/chosen": -0.6614007949829102,
      "logits/rejected": NaN,
      "logps/chosen": -1057.061767578125,
      "logps/rejected": -617.8920288085938,
      "loss": 0.7073,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": 0.004230117425322533,
      "rewards/margins": -0.02747058868408203,
      "rewards/rejected": 0.03170070797204971,
      "step": 3794
    },
    {
      "epoch": 0.9982080915352874,
      "grad_norm": 628.0,
      "learning_rate": 8.771929824561403e-10,
      "logits/chosen": -0.6574038863182068,
      "logits/rejected": -0.6570407748222351,
      "logps/chosen": -862.9151611328125,
      "logps/rejected": -786.498779296875,
      "loss": 0.6921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006813432089984417,
      "rewards/margins": 0.002883720677345991,
      "rewards/rejected": 0.0039297109469771385,
      "step": 3795
    },
    {
      "epoch": 0.9984711239704747,
      "grad_norm": 396.0,
      "learning_rate": 7.309941520467835e-10,
      "logits/chosen": -0.6685222387313843,
      "logits/rejected": -0.6767773032188416,
      "logps/chosen": -842.984619140625,
      "logps/rejected": -557.7953491210938,
      "loss": 0.6932,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0006474017864093184,
      "rewards/margins": 5.9652142226696014e-05,
      "rewards/rejected": -0.0007070540450513363,
      "step": 3796
    },
    {
      "epoch": 0.9987341564056618,
      "grad_norm": 524.0,
      "learning_rate": 5.847953216374269e-10,
      "logits/chosen": -0.6471827626228333,
      "logits/rejected": -0.6500213146209717,
      "logps/chosen": -895.4757080078125,
      "logps/rejected": -849.8888549804688,
      "loss": 0.6938,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.002966976026073098,
      "rewards/margins": -0.0006381040439009666,
      "rewards/rejected": 0.0036050803028047085,
      "step": 3797
    },
    {
      "epoch": 0.998997188840849,
      "grad_norm": 552.0,
      "learning_rate": 4.3859649122807017e-10,
      "logits/chosen": -0.6246211528778076,
      "logits/rejected": -0.6424671411514282,
      "logps/chosen": -1540.052734375,
      "logps/rejected": -1135.69091796875,
      "loss": 0.6885,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01598825491964817,
      "rewards/margins": 0.010256098583340645,
      "rewards/rejected": 0.005732155404984951,
      "step": 3798
    },
    {
      "epoch": 0.9992602212760361,
      "grad_norm": 624.0,
      "learning_rate": 2.9239766081871345e-10,
      "logits/chosen": -0.6695502996444702,
      "logits/rejected": -0.6722966432571411,
      "logps/chosen": -1362.272705078125,
      "logps/rejected": -1200.7576904296875,
      "loss": 0.7018,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.009025096893310547,
      "rewards/margins": -0.016296865418553352,
      "rewards/rejected": 0.02532196044921875,
      "step": 3799
    },
    {
      "epoch": 0.9995232537112233,
      "grad_norm": 604.0,
      "learning_rate": 1.4619883040935672e-10,
      "logits/chosen": -0.6634636521339417,
      "logits/rejected": NaN,
      "logps/chosen": -1318.8785400390625,
      "logps/rejected": -815.6595458984375,
      "loss": 0.691,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004341126419603825,
      "rewards/margins": 0.0048355115577578545,
      "rewards/rejected": -0.0004943850799463689,
      "step": 3800
    },
    {
      "epoch": 0.9997862861464104,
      "grad_norm": 560.0,
      "learning_rate": 0.0,
      "logits/chosen": -0.6446106433868408,
      "logits/rejected": -0.6577019691467285,
      "logps/chosen": -1536.625732421875,
      "logps/rejected": -1028.8563232421875,
      "loss": 0.6804,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01098184660077095,
      "rewards/margins": 0.027542687952518463,
      "rewards/rejected": -0.016560841351747513,
      "step": 3801
    }
  ],
  "logging_steps": 1,
  "max_steps": 3801,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}