{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 363,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.04155844155844156,
      "grad_norm": 3.118415117263794,
      "learning_rate": 1.8181818181818183e-06,
      "logits/chosen": -0.8127259016036987,
      "logits/rejected": -0.7634922862052917,
      "logps/chosen": -228.6646728515625,
      "logps/rejected": -271.3758239746094,
      "loss": 0.6923,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": -0.0006901646265760064,
      "rewards/margins": 0.001700992463156581,
      "rewards/rejected": -0.002391157206147909,
      "step": 5
    },
    {
      "epoch": 0.08311688311688312,
      "grad_norm": 3.1249163150787354,
      "learning_rate": 4.0909090909090915e-06,
      "logits/chosen": -0.8295533061027527,
      "logits/rejected": -0.7937226891517639,
      "logps/chosen": -219.41513061523438,
      "logps/rejected": -279.0150146484375,
      "loss": 0.6819,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0035048150457441807,
      "rewards/margins": 0.022944308817386627,
      "rewards/rejected": -0.02644912339746952,
      "step": 10
    },
    {
      "epoch": 0.12467532467532468,
      "grad_norm": 3.300832748413086,
      "learning_rate": 4.999103931944114e-06,
      "logits/chosen": -0.9027387499809265,
      "logits/rejected": -0.7889591455459595,
      "logps/chosen": -222.32571411132812,
      "logps/rejected": -293.6187744140625,
      "loss": 0.652,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0009602688951417804,
      "rewards/margins": 0.0870375782251358,
      "rewards/rejected": -0.08799784630537033,
      "step": 15
    },
    {
      "epoch": 0.16623376623376623,
      "grad_norm": 2.7533063888549805,
      "learning_rate": 4.993630286525634e-06,
      "logits/chosen": -0.8960437774658203,
      "logits/rejected": -0.824598491191864,
      "logps/chosen": -232.5225830078125,
      "logps/rejected": -296.52178955078125,
      "loss": 0.6123,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.017542826011776924,
      "rewards/margins": 0.18044860661029816,
      "rewards/rejected": -0.19799141585826874,
      "step": 20
    },
    {
      "epoch": 0.2077922077922078,
      "grad_norm": 2.7143568992614746,
      "learning_rate": 4.983191696363525e-06,
      "logits/chosen": -0.923931896686554,
      "logits/rejected": -0.8416165113449097,
      "logps/chosen": -220.6460418701172,
      "logps/rejected": -273.8211364746094,
      "loss": 0.5717,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.03508078306913376,
      "rewards/margins": 0.2878609597682953,
      "rewards/rejected": -0.32294172048568726,
      "step": 25
    },
    {
      "epoch": 0.24935064935064935,
      "grad_norm": 2.480844259262085,
      "learning_rate": 4.9678089452217785e-06,
      "logits/chosen": -0.9296607971191406,
      "logits/rejected": -0.7941066026687622,
      "logps/chosen": -239.7672882080078,
      "logps/rejected": -307.1377258300781,
      "loss": 0.5489,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.08770466595888138,
      "rewards/margins": 0.3718782067298889,
      "rewards/rejected": -0.4595828652381897,
      "step": 30
    },
    {
      "epoch": 0.2909090909090909,
      "grad_norm": 2.883957862854004,
      "learning_rate": 4.947512660940388e-06,
      "logits/chosen": -0.9467341303825378,
      "logits/rejected": -0.8958338499069214,
      "logps/chosen": -236.58203125,
      "logps/rejected": -299.7516174316406,
      "loss": 0.5295,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.06386536359786987,
      "rewards/margins": 0.455936998128891,
      "rewards/rejected": -0.519802451133728,
      "step": 35
    },
    {
      "epoch": 0.33246753246753247,
      "grad_norm": 2.9537580013275146,
      "learning_rate": 4.922343254453769e-06,
      "logits/chosen": -0.9613215327262878,
      "logits/rejected": -0.8938771486282349,
      "logps/chosen": -209.87484741210938,
      "logps/rejected": -261.8564453125,
      "loss": 0.5066,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.04000057652592659,
      "rewards/margins": 0.5617119669914246,
      "rewards/rejected": -0.6017125844955444,
      "step": 40
    },
    {
      "epoch": 0.37402597402597404,
      "grad_norm": 3.0379507541656494,
      "learning_rate": 4.8923508393305224e-06,
      "logits/chosen": -0.9870550036430359,
      "logits/rejected": -0.8364689946174622,
      "logps/chosen": -224.9286651611328,
      "logps/rejected": -275.58392333984375,
      "loss": 0.4765,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.03620234876871109,
      "rewards/margins": 0.6848794221878052,
      "rewards/rejected": -0.7210818529129028,
      "step": 45
    },
    {
      "epoch": 0.4155844155844156,
      "grad_norm": 2.38826847076416,
      "learning_rate": 4.857595131994781e-06,
      "logits/chosen": -1.0010199546813965,
      "logits/rejected": -0.9080400466918945,
      "logps/chosen": -250.68527221679688,
      "logps/rejected": -344.46478271484375,
      "loss": 0.4251,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.09372053295373917,
      "rewards/margins": 0.9252897500991821,
      "rewards/rejected": -1.019010305404663,
      "step": 50
    },
    {
      "epoch": 0.45714285714285713,
      "grad_norm": 2.718043804168701,
      "learning_rate": 4.818145332827762e-06,
      "logits/chosen": -1.1310384273529053,
      "logits/rejected": -0.9854855537414551,
      "logps/chosen": -231.0263214111328,
      "logps/rejected": -298.1824645996094,
      "loss": 0.4418,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.12084142863750458,
      "rewards/margins": 0.9410508275032043,
      "rewards/rejected": -1.0618922710418701,
      "step": 55
    },
    {
      "epoch": 0.4987012987012987,
      "grad_norm": 2.1554527282714844,
      "learning_rate": 4.7740799883862966e-06,
      "logits/chosen": -0.9824150800704956,
      "logits/rejected": -0.9336239099502563,
      "logps/chosen": -211.98684692382812,
      "logps/rejected": -292.4842224121094,
      "loss": 0.4413,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.06550006568431854,
      "rewards/margins": 0.9869768023490906,
      "rewards/rejected": -1.0524768829345703,
      "step": 60
    },
    {
      "epoch": 0.5402597402597402,
      "grad_norm": 2.788320302963257,
      "learning_rate": 4.725486835012632e-06,
      "logits/chosen": -1.1025915145874023,
      "logits/rejected": -1.0172704458236694,
      "logps/chosen": -228.7750244140625,
      "logps/rejected": -302.1152648925781,
      "loss": 0.4106,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.13185444474220276,
      "rewards/margins": 1.0933724641799927,
      "rewards/rejected": -1.225226879119873,
      "step": 65
    },
    {
      "epoch": 0.5818181818181818,
      "grad_norm": 2.9373393058776855,
      "learning_rate": 4.672462624146918e-06,
      "logits/chosen": -1.133795976638794,
      "logits/rejected": -0.9968055486679077,
      "logps/chosen": -224.1367950439453,
      "logps/rejected": -320.6668701171875,
      "loss": 0.3655,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.16425569355487823,
      "rewards/margins": 1.38435959815979,
      "rewards/rejected": -1.5486153364181519,
      "step": 70
    },
    {
      "epoch": 0.6233766233766234,
      "grad_norm": 2.555952548980713,
      "learning_rate": 4.615112929690163e-06,
      "logits/chosen": -1.009514570236206,
      "logits/rejected": -0.9566370248794556,
      "logps/chosen": -252.9624481201172,
      "logps/rejected": -294.30816650390625,
      "loss": 0.403,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.15342703461647034,
      "rewards/margins": 1.1806553602218628,
      "rewards/rejected": -1.3340823650360107,
      "step": 75
    },
    {
      "epoch": 0.6649350649350649,
      "grad_norm": 2.2290637493133545,
      "learning_rate": 4.553551937801229e-06,
      "logits/chosen": -1.0907081365585327,
      "logits/rejected": -1.0329780578613281,
      "logps/chosen": -233.9783172607422,
      "logps/rejected": -298.1121826171875,
      "loss": 0.3585,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.1846320629119873,
      "rewards/margins": 1.4989664554595947,
      "rewards/rejected": -1.6835987567901611,
      "step": 80
    },
    {
      "epoch": 0.7064935064935065,
      "grad_norm": 2.9799797534942627,
      "learning_rate": 4.487902219546383e-06,
      "logits/chosen": -1.2056300640106201,
      "logits/rejected": -1.0564621686935425,
      "logps/chosen": -225.80825805664062,
      "logps/rejected": -286.21380615234375,
      "loss": 0.3455,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.1425703465938568,
      "rewards/margins": 1.559043526649475,
      "rewards/rejected": -1.7016140222549438,
      "step": 85
    },
    {
      "epoch": 0.7480519480519481,
      "grad_norm": 4.232228755950928,
      "learning_rate": 4.418294486854064e-06,
      "logits/chosen": -1.1301639080047607,
      "logits/rejected": -0.9990237951278687,
      "logps/chosen": -235.9337158203125,
      "logps/rejected": -288.6039733886719,
      "loss": 0.3599,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.130110502243042,
      "rewards/margins": 1.5865774154663086,
      "rewards/rejected": -1.7166879177093506,
      "step": 90
    },
    {
      "epoch": 0.7896103896103897,
      "grad_norm": 3.81459641456604,
      "learning_rate": 4.3448673322607915e-06,
      "logits/chosen": -1.1437747478485107,
      "logits/rejected": -1.0269885063171387,
      "logps/chosen": -230.104248046875,
      "logps/rejected": -305.1618347167969,
      "loss": 0.3399,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.08539257198572159,
      "rewards/margins": 1.8511079549789429,
      "rewards/rejected": -1.9365005493164062,
      "step": 95
    },
    {
      "epoch": 0.8311688311688312,
      "grad_norm": 2.949822425842285,
      "learning_rate": 4.267766952966369e-06,
      "logits/chosen": -1.1605863571166992,
      "logits/rejected": -1.0450435876846313,
      "logps/chosen": -222.7265625,
      "logps/rejected": -324.3167419433594,
      "loss": 0.3465,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.04091508686542511,
      "rewards/margins": 1.9012877941131592,
      "rewards/rejected": -1.9422029256820679,
      "step": 100
    },
    {
      "epoch": 0.8727272727272727,
      "grad_norm": 2.7308614253997803,
      "learning_rate": 4.187146859747826e-06,
      "logits/chosen": -1.104167103767395,
      "logits/rejected": -1.0075145959854126,
      "logps/chosen": -228.25424194335938,
      "logps/rejected": -315.0980529785156,
      "loss": 0.3543,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.004563205409795046,
      "rewards/margins": 1.9307657480239868,
      "rewards/rejected": -1.9353290796279907,
      "step": 105
    },
    {
      "epoch": 0.9142857142857143,
      "grad_norm": 2.497864007949829,
      "learning_rate": 4.103167571311652e-06,
      "logits/chosen": -1.2123258113861084,
      "logits/rejected": -1.0642549991607666,
      "logps/chosen": -220.7777862548828,
      "logps/rejected": -307.74310302734375,
      "loss": 0.31,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.08179270476102829,
      "rewards/margins": 1.9646203517913818,
      "rewards/rejected": -2.046412944793701,
      "step": 110
    },
    {
      "epoch": 0.9558441558441558,
      "grad_norm": 2.9862191677093506,
      "learning_rate": 4.015996294692889e-06,
      "logits/chosen": -1.1412394046783447,
      "logits/rejected": -1.1184909343719482,
      "logps/chosen": -234.40109252929688,
      "logps/rejected": -285.2551574707031,
      "loss": 0.3687,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.0041265422478318214,
      "rewards/margins": 1.6963088512420654,
      "rewards/rejected": -1.6921823024749756,
      "step": 115
    },
    {
      "epoch": 0.9974025974025974,
      "grad_norm": 3.2241032123565674,
      "learning_rate": 3.9258065923374104e-06,
      "logits/chosen": -1.2083184719085693,
      "logits/rejected": -1.087278962135315,
      "logps/chosen": -208.6204376220703,
      "logps/rejected": -345.3356018066406,
      "loss": 0.3047,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.12136807292699814,
      "rewards/margins": 2.2191343307495117,
      "rewards/rejected": -2.097766160964966,
      "step": 120
    },
    {
      "epoch": 1.0332467532467533,
      "grad_norm": 2.3973424434661865,
      "learning_rate": 3.832778036530276e-06,
      "logits/chosen": -1.0659514665603638,
      "logits/rejected": -0.989536464214325,
      "logps/chosen": -229.2854766845703,
      "logps/rejected": -303.9345703125,
      "loss": 0.2628,
      "rewards/accuracies": 0.8913043737411499,
      "rewards/chosen": 0.2346079796552658,
      "rewards/margins": 2.2249648571014404,
      "rewards/rejected": -1.9903569221496582,
      "step": 125
    },
    {
      "epoch": 1.0748051948051949,
      "grad_norm": 2.8788726329803467,
      "learning_rate": 3.7370958518581694e-06,
      "logits/chosen": -1.17106294631958,
      "logits/rejected": -1.0628446340560913,
      "logps/chosen": -228.1200714111328,
      "logps/rejected": -287.6686096191406,
      "loss": 0.3305,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.18449993431568146,
      "rewards/margins": 1.797311782836914,
      "rewards/rejected": -1.612811803817749,
      "step": 130
    },
    {
      "epoch": 1.1163636363636364,
      "grad_norm": 1.94839346408844,
      "learning_rate": 3.638950546417824e-06,
      "logits/chosen": -1.198904275894165,
      "logits/rejected": -1.0711965560913086,
      "logps/chosen": -229.5878448486328,
      "logps/rejected": -309.1488952636719,
      "loss": 0.2589,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2590901851654053,
      "rewards/margins": 2.1628074645996094,
      "rewards/rejected": -1.9037176370620728,
      "step": 135
    },
    {
      "epoch": 1.157922077922078,
      "grad_norm": 2.6632540225982666,
      "learning_rate": 3.5385375325047167e-06,
      "logits/chosen": -1.2561719417572021,
      "logits/rejected": -1.0732624530792236,
      "logps/chosen": -213.3624267578125,
      "logps/rejected": -296.18035888671875,
      "loss": 0.2866,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.18848955631256104,
      "rewards/margins": 2.0759098529815674,
      "rewards/rejected": -1.887420415878296,
      "step": 140
    },
    {
      "epoch": 1.1994805194805194,
      "grad_norm": 2.6034018993377686,
      "learning_rate": 3.4360567375372426e-06,
      "logits/chosen": -1.2088276147842407,
      "logits/rejected": -1.0747075080871582,
      "logps/chosen": -224.8641815185547,
      "logps/rejected": -305.76824951171875,
      "loss": 0.2851,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.09721609950065613,
      "rewards/margins": 2.325167179107666,
      "rewards/rejected": -2.2279508113861084,
      "step": 145
    },
    {
      "epoch": 1.241038961038961,
      "grad_norm": 2.1052863597869873,
      "learning_rate": 3.3317122059910607e-06,
      "logits/chosen": -1.2572283744812012,
      "logits/rejected": -1.1288259029388428,
      "logps/chosen": -223.6044158935547,
      "logps/rejected": -327.5605163574219,
      "loss": 0.2647,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 0.09010759741067886,
      "rewards/margins": 2.6917688846588135,
      "rewards/rejected": -2.601661205291748,
      "step": 150
    },
    {
      "epoch": 1.2825974025974025,
      "grad_norm": 2.5198380947113037,
      "learning_rate": 3.225711693136156e-06,
      "logits/chosen": -1.2977545261383057,
      "logits/rejected": -1.166739821434021,
      "logps/chosen": -236.06753540039062,
      "logps/rejected": -299.90655517578125,
      "loss": 0.2904,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.016246700659394264,
      "rewards/margins": 2.2283177375793457,
      "rewards/rejected": -2.212070941925049,
      "step": 155
    },
    {
      "epoch": 1.324155844155844,
      "grad_norm": 2.1668765544891357,
      "learning_rate": 3.118266251385539e-06,
      "logits/chosen": -1.1280694007873535,
      "logits/rejected": -1.0987536907196045,
      "logps/chosen": -250.9461669921875,
      "logps/rejected": -305.9465026855469,
      "loss": 0.29,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.01789233088493347,
      "rewards/margins": 2.355379104614258,
      "rewards/rejected": -2.337486743927002,
      "step": 160
    },
    {
      "epoch": 1.3657142857142857,
      "grad_norm": 2.4839677810668945,
      "learning_rate": 3.0095898100791472e-06,
      "logits/chosen": -1.2380702495574951,
      "logits/rejected": -1.0927748680114746,
      "logps/chosen": -226.11709594726562,
      "logps/rejected": -324.3223876953125,
      "loss": 0.2639,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.16840746998786926,
      "rewards/margins": 2.606884479522705,
      "rewards/rejected": -2.775291919708252,
      "step": 165
    },
    {
      "epoch": 1.4072727272727272,
      "grad_norm": 2.544842481613159,
      "learning_rate": 2.8998987495396506e-06,
      "logits/chosen": -1.3116706609725952,
      "logits/rejected": -1.1664602756500244,
      "logps/chosen": -223.4804229736328,
      "logps/rejected": -307.60455322265625,
      "loss": 0.2416,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.16324956715106964,
      "rewards/margins": 2.7645745277404785,
      "rewards/rejected": -2.9278244972229004,
      "step": 170
    },
    {
      "epoch": 1.4488311688311688,
      "grad_norm": 2.2766852378845215,
      "learning_rate": 2.7894114702482254e-06,
      "logits/chosen": -1.2931270599365234,
      "logits/rejected": -1.1453607082366943,
      "logps/chosen": -225.3311004638672,
      "logps/rejected": -329.5317687988281,
      "loss": 0.275,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.1200091615319252,
      "rewards/margins": 2.7051124572753906,
      "rewards/rejected": -2.8251214027404785,
      "step": 175
    },
    {
      "epoch": 1.4903896103896104,
      "grad_norm": 2.1356124877929688,
      "learning_rate": 2.678347957998081e-06,
      "logits/chosen": -1.301461935043335,
      "logits/rejected": -1.1489789485931396,
      "logps/chosen": -223.24655151367188,
      "logps/rejected": -314.0998229980469,
      "loss": 0.2134,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.17566713690757751,
      "rewards/margins": 2.733579635620117,
      "rewards/rejected": -2.9092462062835693,
      "step": 180
    },
    {
      "epoch": 1.531948051948052,
      "grad_norm": 2.0670270919799805,
      "learning_rate": 2.566929345891562e-06,
      "logits/chosen": -1.1717848777770996,
      "logits/rejected": -1.0892704725265503,
      "logps/chosen": -230.27536010742188,
      "logps/rejected": -311.3392333984375,
      "loss": 0.2352,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.049989253282547,
      "rewards/margins": 2.9402599334716797,
      "rewards/rejected": -2.890270709991455,
      "step": 185
    },
    {
      "epoch": 1.5735064935064935,
      "grad_norm": 2.3355753421783447,
      "learning_rate": 2.455377474052885e-06,
      "logits/chosen": -1.1918342113494873,
      "logits/rejected": -1.0741770267486572,
      "logps/chosen": -239.7108154296875,
      "logps/rejected": -299.80035400390625,
      "loss": 0.2795,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.24867768585681915,
      "rewards/margins": 2.535564661026001,
      "rewards/rejected": -2.2868869304656982,
      "step": 190
    },
    {
      "epoch": 1.615064935064935,
      "grad_norm": 2.0783815383911133,
      "learning_rate": 2.3439144479331776e-06,
      "logits/chosen": -1.3459298610687256,
      "logits/rejected": -1.2816716432571411,
      "logps/chosen": -219.70138549804688,
      "logps/rejected": -328.4333801269531,
      "loss": 0.2118,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.07029136270284653,
      "rewards/margins": 3.2027976512908936,
      "rewards/rejected": -3.1325061321258545,
      "step": 195
    },
    {
      "epoch": 1.6566233766233767,
      "grad_norm": 1.9809454679489136,
      "learning_rate": 2.2327621960872187e-06,
      "logits/chosen": -1.2197625637054443,
      "logits/rejected": -1.0895978212356567,
      "logps/chosen": -229.3469696044922,
      "logps/rejected": -329.76580810546875,
      "loss": 0.253,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.10655846446752548,
      "rewards/margins": 2.7795863151550293,
      "rewards/rejected": -2.673027515411377,
      "step": 200
    },
    {
      "epoch": 1.6981818181818182,
      "grad_norm": 2.6429646015167236,
      "learning_rate": 2.1221420283023984e-06,
      "logits/chosen": -1.27398681640625,
      "logits/rejected": -1.0956755876541138,
      "logps/chosen": -229.72119140625,
      "logps/rejected": -311.6751403808594,
      "loss": 0.2748,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.03542603179812431,
      "rewards/margins": 2.605504035949707,
      "rewards/rejected": -2.570078134536743,
      "step": 205
    },
    {
      "epoch": 1.7397402597402598,
      "grad_norm": 2.050103187561035,
      "learning_rate": 2.01227419495968e-06,
      "logits/chosen": -1.2580993175506592,
      "logits/rejected": -1.2016228437423706,
      "logps/chosen": -225.03189086914062,
      "logps/rejected": -315.2294616699219,
      "loss": 0.2545,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.12700282037258148,
      "rewards/margins": 2.712761402130127,
      "rewards/rejected": -2.585758686065674,
      "step": 210
    },
    {
      "epoch": 1.7812987012987014,
      "grad_norm": 2.6419899463653564,
      "learning_rate": 1.9033774485038794e-06,
      "logits/chosen": -1.2604008913040161,
      "logits/rejected": -1.1434435844421387,
      "logps/chosen": -227.2825164794922,
      "logps/rejected": -290.4469299316406,
      "loss": 0.2702,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.12941348552703857,
      "rewards/margins": 2.5677075386047363,
      "rewards/rejected": -2.438293933868408,
      "step": 215
    },
    {
      "epoch": 1.822857142857143,
      "grad_norm": 2.732487440109253,
      "learning_rate": 1.7956686078964257e-06,
      "logits/chosen": -1.289029836654663,
      "logits/rejected": -1.2009754180908203,
      "logps/chosen": -225.47366333007812,
      "logps/rejected": -300.55572509765625,
      "loss": 0.2567,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.09105754643678665,
      "rewards/margins": 2.691190719604492,
      "rewards/rejected": -2.600132942199707,
      "step": 220
    },
    {
      "epoch": 1.8644155844155845,
      "grad_norm": 2.813246726989746,
      "learning_rate": 1.6893621269177821e-06,
      "logits/chosen": -1.2733620405197144,
      "logits/rejected": -1.145566463470459,
      "logps/chosen": -238.2940673828125,
      "logps/rejected": -295.91827392578125,
      "loss": 0.2587,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.13102269172668457,
      "rewards/margins": 2.568708896636963,
      "rewards/rejected": -2.4376862049102783,
      "step": 225
    },
    {
      "epoch": 1.905974025974026,
      "grad_norm": 2.8184409141540527,
      "learning_rate": 1.5846696671790524e-06,
      "logits/chosen": -1.233093023300171,
      "logits/rejected": -1.1369177103042603,
      "logps/chosen": -209.8083953857422,
      "logps/rejected": -361.1885986328125,
      "loss": 0.2688,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.050070393830537796,
      "rewards/margins": 2.9840142726898193,
      "rewards/rejected": -2.9339439868927,
      "step": 230
    },
    {
      "epoch": 1.9475324675324677,
      "grad_norm": 2.254124641418457,
      "learning_rate": 1.4817996766929532e-06,
      "logits/chosen": -1.1859557628631592,
      "logits/rejected": -1.1998345851898193,
      "logps/chosen": -212.0352783203125,
      "logps/rejected": -320.67767333984375,
      "loss": 0.2405,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.10705818235874176,
      "rewards/margins": 3.07060170173645,
      "rewards/rejected": -2.96354341506958,
      "step": 235
    },
    {
      "epoch": 1.9890909090909092,
      "grad_norm": 2.4340288639068604,
      "learning_rate": 1.380956974843219e-06,
      "logits/chosen": -1.3080205917358398,
      "logits/rejected": -1.1855772733688354,
      "logps/chosen": -234.13052368164062,
      "logps/rejected": -322.0392150878906,
      "loss": 0.2689,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.0906623974442482,
      "rewards/margins": 2.882385730743408,
      "rewards/rejected": -2.7917237281799316,
      "step": 240
    },
    {
      "epoch": 2.024935064935065,
      "grad_norm": 2.381237030029297,
      "learning_rate": 1.282342344578782e-06,
      "logits/chosen": -1.2758800983428955,
      "logits/rejected": -1.1762343645095825,
      "logps/chosen": -210.26197814941406,
      "logps/rejected": -304.9644775390625,
      "loss": 0.2013,
      "rewards/accuracies": 0.9347826242446899,
      "rewards/chosen": 0.18103013932704926,
      "rewards/margins": 3.0971362590789795,
      "rewards/rejected": -2.9161057472229004,
      "step": 245
    },
    {
      "epoch": 2.0664935064935066,
      "grad_norm": 1.9230319261550903,
      "learning_rate": 1.186152132644718e-06,
      "logits/chosen": -1.2861894369125366,
      "logits/rejected": -1.1287521123886108,
      "logps/chosen": -218.40707397460938,
      "logps/rejected": -319.4270935058594,
      "loss": 0.224,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.07273466885089874,
      "rewards/margins": 3.143413543701172,
      "rewards/rejected": -3.070678949356079,
      "step": 250
    },
    {
      "epoch": 2.108051948051948,
      "grad_norm": 2.3148598670959473,
      "learning_rate": 1.0925778586458813e-06,
      "logits/chosen": -1.3198808431625366,
      "logits/rejected": -1.221094012260437,
      "logps/chosen": -214.1598663330078,
      "logps/rejected": -299.68084716796875,
      "loss": 0.2052,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.11255063116550446,
      "rewards/margins": 3.0849621295928955,
      "rewards/rejected": -2.972411632537842,
      "step": 255
    },
    {
      "epoch": 2.1496103896103897,
      "grad_norm": 2.569136142730713,
      "learning_rate": 1.0018058337216327e-06,
      "logits/chosen": -1.2940465211868286,
      "logits/rejected": -1.2342337369918823,
      "logps/chosen": -226.5349578857422,
      "logps/rejected": -316.8858337402344,
      "loss": 0.2304,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.11650697886943817,
      "rewards/margins": 2.7404751777648926,
      "rewards/rejected": -2.6239686012268066,
      "step": 260
    },
    {
      "epoch": 2.1911688311688313,
      "grad_norm": 1.9575142860412598,
      "learning_rate": 9.140167895908867e-07,
      "logits/chosen": -1.186423659324646,
      "logits/rejected": -1.1101840734481812,
      "logps/chosen": -224.22244262695312,
      "logps/rejected": -320.3080139160156,
      "loss": 0.2132,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.07010473310947418,
      "rewards/margins": 2.980109691619873,
      "rewards/rejected": -2.9100048542022705,
      "step": 265
    },
    {
      "epoch": 2.232727272727273,
      "grad_norm": 1.8908374309539795,
      "learning_rate": 8.293855187060601e-07,
      "logits/chosen": -1.2614704370498657,
      "logits/rejected": -1.1322941780090332,
      "logps/chosen": -245.6070098876953,
      "logps/rejected": -335.3045654296875,
      "loss": 0.2018,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.18193891644477844,
      "rewards/margins": 3.2319583892822266,
      "rewards/rejected": -3.0500195026397705,
      "step": 270
    },
    {
      "epoch": 2.2742857142857145,
      "grad_norm": 1.9720425605773926,
      "learning_rate": 7.480805262324112e-07,
      "logits/chosen": -1.2367236614227295,
      "logits/rejected": -1.1424002647399902,
      "logps/chosen": -225.2272491455078,
      "logps/rejected": -308.2543640136719,
      "loss": 0.2476,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.06707721948623657,
      "rewards/margins": 2.86738920211792,
      "rewards/rejected": -2.8003122806549072,
      "step": 275
    },
    {
      "epoch": 2.315844155844156,
      "grad_norm": 2.199530601501465,
      "learning_rate": 6.70263694545687e-07,
      "logits/chosen": -1.2416431903839111,
      "logits/rejected": -1.1705714464187622,
      "logps/chosen": -216.54519653320312,
      "logps/rejected": -313.75299072265625,
      "loss": 0.2173,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.14036111533641815,
      "rewards/margins": 3.161411762237549,
      "rewards/rejected": -3.0210506916046143,
      "step": 280
    },
    {
      "epoch": 2.357402597402597,
      "grad_norm": 2.594045639038086,
      "learning_rate": 5.960899609160725e-07,
      "logits/chosen": -1.3122602701187134,
      "logits/rejected": -1.1919419765472412,
      "logps/chosen": -229.4156494140625,
      "logps/rejected": -325.54656982421875,
      "loss": 0.2024,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.0318605899810791,
      "rewards/margins": 3.3778507709503174,
      "rewards/rejected": -3.3459904193878174,
      "step": 285
    },
    {
      "epoch": 2.3989610389610387,
      "grad_norm": 2.318593740463257,
      "learning_rate": 5.257070090202101e-07,
      "logits/chosen": -1.296639084815979,
      "logits/rejected": -1.1731235980987549,
      "logps/chosen": -223.69839477539062,
      "logps/rejected": -317.9754333496094,
      "loss": 0.2178,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.005478203296661377,
      "rewards/margins": 3.3782591819763184,
      "rewards/rejected": -3.383737564086914,
      "step": 290
    },
    {
      "epoch": 2.4405194805194803,
      "grad_norm": 2.6067514419555664,
      "learning_rate": 4.592549748954858e-07,
      "logits/chosen": -1.2447936534881592,
      "logits/rejected": -1.1643249988555908,
      "logps/chosen": -237.61068725585938,
      "logps/rejected": -288.54351806640625,
      "loss": 0.2758,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03469625487923622,
      "rewards/margins": 2.545546054840088,
      "rewards/rejected": -2.510849952697754,
      "step": 295
    },
    {
      "epoch": 2.482077922077922,
      "grad_norm": 1.8700066804885864,
      "learning_rate": 3.9686616792204677e-07,
      "logits/chosen": -1.2930142879486084,
      "logits/rejected": -1.1466633081436157,
      "logps/chosen": -240.37625122070312,
      "logps/rejected": -333.6015625,
      "loss": 0.1731,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.03592072054743767,
      "rewards/margins": 3.346038341522217,
      "rewards/rejected": -3.3819591999053955,
      "step": 300
    },
    {
      "epoch": 2.5236363636363635,
      "grad_norm": 2.5204155445098877,
      "learning_rate": 3.386648073881019e-07,
      "logits/chosen": -1.2621270418167114,
      "logits/rejected": -1.1422834396362305,
      "logps/chosen": -232.3268585205078,
      "logps/rejected": -293.70245361328125,
      "loss": 0.2533,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.11021828651428223,
      "rewards/margins": 2.717705249786377,
      "rewards/rejected": -2.607487201690674,
      "step": 305
    },
    {
      "epoch": 2.565194805194805,
      "grad_norm": 3.1326005458831787,
      "learning_rate": 2.84766775162994e-07,
      "logits/chosen": -1.2745239734649658,
      "logits/rejected": -1.189447045326233,
      "logps/chosen": -248.35812377929688,
      "logps/rejected": -322.8234558105469,
      "loss": 0.2217,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.07117398083209991,
      "rewards/margins": 3.0546634197235107,
      "rewards/rejected": -2.9834892749786377,
      "step": 310
    },
    {
      "epoch": 2.6067532467532466,
      "grad_norm": 1.8207591772079468,
      "learning_rate": 2.3527938497050678e-07,
      "logits/chosen": -1.3199981451034546,
      "logits/rejected": -1.2175192832946777,
      "logps/chosen": -229.90371704101562,
      "logps/rejected": -376.37445068359375,
      "loss": 0.1869,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.0209323950111866,
      "rewards/margins": 3.4481894969940186,
      "rewards/rejected": -3.4272570610046387,
      "step": 315
    },
    {
      "epoch": 2.648311688311688,
      "grad_norm": 2.0498719215393066,
      "learning_rate": 1.9030116872178317e-07,
      "logits/chosen": -1.2227413654327393,
      "logits/rejected": -1.1524617671966553,
      "logps/chosen": -226.3324737548828,
      "logps/rejected": -299.68780517578125,
      "loss": 0.258,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.059439968317747116,
      "rewards/margins": 2.7543835639953613,
      "rewards/rejected": -2.8138232231140137,
      "step": 320
    },
    {
      "epoch": 2.6898701298701297,
      "grad_norm": 2.4288649559020996,
      "learning_rate": 1.4992168033327482e-07,
      "logits/chosen": -1.3343864679336548,
      "logits/rejected": -1.2201334238052368,
      "logps/chosen": -213.62020874023438,
      "logps/rejected": -305.35467529296875,
      "loss": 0.197,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.12054959684610367,
      "rewards/margins": 2.9617972373962402,
      "rewards/rejected": -2.841248035430908,
      "step": 325
    },
    {
      "epoch": 2.7314285714285713,
      "grad_norm": 1.6564867496490479,
      "learning_rate": 1.1422131742034048e-07,
      "logits/chosen": -1.328583002090454,
      "logits/rejected": -1.1812621355056763,
      "logps/chosen": -216.4242706298828,
      "logps/rejected": -310.3609313964844,
      "loss": 0.2156,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.05895359441637993,
      "rewards/margins": 3.110816478729248,
      "rewards/rejected": -3.1697702407836914,
      "step": 330
    },
    {
      "epoch": 2.772987012987013,
      "grad_norm": 2.33535099029541,
      "learning_rate": 8.327116122150647e-08,
      "logits/chosen": -1.2217092514038086,
      "logits/rejected": -1.1084274053573608,
      "logps/chosen": -221.68539428710938,
      "logps/rejected": -336.6221618652344,
      "loss": 0.1963,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 0.07560758292675018,
      "rewards/margins": 3.6819679737091064,
      "rewards/rejected": -3.606360673904419,
      "step": 335
    },
    {
      "epoch": 2.8145454545454545,
      "grad_norm": 1.6087279319763184,
      "learning_rate": 5.7132835072101486e-08,
      "logits/chosen": -1.3235546350479126,
      "logits/rejected": -1.2055761814117432,
      "logps/chosen": -224.8578338623047,
      "logps/rejected": -323.5769958496094,
      "loss": 0.2213,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.050518132746219635,
      "rewards/margins": 3.143819808959961,
      "rewards/rejected": -3.093301773071289,
      "step": 340
    },
    {
      "epoch": 2.856103896103896,
      "grad_norm": 2.025810718536377,
      "learning_rate": 3.585838170906353e-08,
      "logits/chosen": -1.3240934610366821,
      "logits/rejected": -1.1658360958099365,
      "logps/chosen": -233.7198028564453,
      "logps/rejected": -306.67138671875,
      "loss": 0.2142,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.00679791858419776,
      "rewards/margins": 3.107149839401245,
      "rewards/rejected": -3.1003518104553223,
      "step": 345
    },
    {
      "epoch": 2.8976623376623376,
      "grad_norm": 2.809882640838623,
      "learning_rate": 1.9490159651203988e-08,
      "logits/chosen": -1.3499139547348022,
      "logits/rejected": -1.290265679359436,
      "logps/chosen": -239.82080078125,
      "logps/rejected": -331.41510009765625,
      "loss": 0.212,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.10003846883773804,
      "rewards/margins": 3.5371921062469482,
      "rewards/rejected": -3.4371535778045654,
      "step": 350
    },
    {
      "epoch": 2.939220779220779,
      "grad_norm": 2.6004693508148193,
      "learning_rate": 8.060758861239414e-09,
      "logits/chosen": -1.3200178146362305,
      "logits/rejected": -1.2445639371871948,
      "logps/chosen": -216.0928192138672,
      "logps/rejected": -329.24566650390625,
      "loss": 0.2011,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.07652398198843002,
      "rewards/margins": 3.232393264770508,
      "rewards/rejected": -3.155869483947754,
      "step": 355
    },
    {
      "epoch": 2.9807792207792208,
      "grad_norm": 2.5794732570648193,
      "learning_rate": 1.5929358575206277e-09,
      "logits/chosen": -1.3764307498931885,
      "logits/rejected": -1.218400478363037,
      "logps/chosen": -225.61376953125,
      "logps/rejected": -306.9019470214844,
      "loss": 0.2348,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.024400614202022552,
      "rewards/margins": 3.0199968814849854,
      "rewards/rejected": -3.0443975925445557,
      "step": 360
    }
  ],
  "logging_steps": 5,
  "max_steps": 363,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}