{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 1407,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010666666666666666,
      "grad_norm": 3.0879628658294678,
      "learning_rate": 4.651162790697675e-07,
      "logits/chosen": -1.4615767002105713,
      "logits/rejected": -1.345485806465149,
      "logps/chosen": -254.444580078125,
      "logps/rejected": -294.7290954589844,
      "loss": 0.693,
      "rewards/accuracies": 0.33125001192092896,
      "rewards/chosen": -0.000815639563370496,
      "rewards/margins": 0.0002851820026990026,
      "rewards/rejected": -0.0011008214205503464,
      "step": 5
    },
    {
      "epoch": 0.021333333333333333,
      "grad_norm": 3.0283706188201904,
      "learning_rate": 1.0465116279069768e-06,
      "logits/chosen": -1.4140113592147827,
      "logits/rejected": -1.314051866531372,
      "logps/chosen": -235.2375030517578,
      "logps/rejected": -272.03533935546875,
      "loss": 0.6916,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.0029421807266771793,
      "rewards/margins": 0.0031576156616210938,
      "rewards/rejected": -0.00609979685395956,
      "step": 10
    },
    {
      "epoch": 0.032,
      "grad_norm": 3.4339029788970947,
      "learning_rate": 1.6279069767441862e-06,
      "logits/chosen": -1.3767629861831665,
      "logits/rejected": -1.2262457609176636,
      "logps/chosen": -244.1561279296875,
      "logps/rejected": -282.74468994140625,
      "loss": 0.6856,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.0028314352966845036,
      "rewards/margins": 0.015238702297210693,
      "rewards/rejected": -0.01807013712823391,
      "step": 15
    },
    {
      "epoch": 0.042666666666666665,
      "grad_norm": 2.816466808319092,
      "learning_rate": 2.2093023255813954e-06,
      "logits/chosen": -1.368130087852478,
      "logits/rejected": -1.2854630947113037,
      "logps/chosen": -240.41079711914062,
      "logps/rejected": -290.0007629394531,
      "loss": 0.6771,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.005986953154206276,
      "rewards/margins": 0.03268471360206604,
      "rewards/rejected": -0.03867166489362717,
      "step": 20
    },
    {
      "epoch": 0.05333333333333334,
      "grad_norm": 3.2854762077331543,
      "learning_rate": 2.790697674418605e-06,
      "logits/chosen": -1.3610224723815918,
      "logits/rejected": -1.2579267024993896,
      "logps/chosen": -228.66049194335938,
      "logps/rejected": -317.1540832519531,
      "loss": 0.6559,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.01977255381643772,
      "rewards/margins": 0.07735180109739304,
      "rewards/rejected": -0.09712435305118561,
      "step": 25
    },
    {
      "epoch": 0.064,
      "grad_norm": 3.386852264404297,
      "learning_rate": 3.372093023255814e-06,
      "logits/chosen": -1.5247161388397217,
      "logits/rejected": -1.3776860237121582,
      "logps/chosen": -228.644287109375,
      "logps/rejected": -294.73724365234375,
      "loss": 0.6424,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.03603464737534523,
      "rewards/margins": 0.10792265087366104,
      "rewards/rejected": -0.14395730197429657,
      "step": 30
    },
    {
      "epoch": 0.07466666666666667,
      "grad_norm": 3.72318434715271,
      "learning_rate": 3.953488372093024e-06,
      "logits/chosen": -1.5082950592041016,
      "logits/rejected": -1.4147998094558716,
      "logps/chosen": -243.84384155273438,
      "logps/rejected": -301.76507568359375,
      "loss": 0.6181,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.06065545603632927,
      "rewards/margins": 0.16406485438346863,
      "rewards/rejected": -0.2247203290462494,
      "step": 35
    },
    {
      "epoch": 0.08533333333333333,
      "grad_norm": 2.8949575424194336,
      "learning_rate": 4.5348837209302326e-06,
      "logits/chosen": -1.4104528427124023,
      "logits/rejected": -1.2926199436187744,
      "logps/chosen": -242.8151397705078,
      "logps/rejected": -290.7886047363281,
      "loss": 0.6012,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.08680661767721176,
      "rewards/margins": 0.20925834774971008,
      "rewards/rejected": -0.29606494307518005,
      "step": 40
    },
    {
      "epoch": 0.096,
      "grad_norm": 3.3057022094726562,
      "learning_rate": 4.99999336897255e-06,
      "logits/chosen": -1.4417957067489624,
      "logits/rejected": -1.2875239849090576,
      "logps/chosen": -255.4944610595703,
      "logps/rejected": -300.8448486328125,
      "loss": 0.5696,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.15032631158828735,
      "rewards/margins": 0.29992568492889404,
      "rewards/rejected": -0.4502519965171814,
      "step": 45
    },
    {
      "epoch": 0.10666666666666667,
      "grad_norm": 3.007528781890869,
      "learning_rate": 4.999761286705285e-06,
      "logits/chosen": -1.4402955770492554,
      "logits/rejected": -1.3425570726394653,
      "logps/chosen": -252.9389190673828,
      "logps/rejected": -297.7711486816406,
      "loss": 0.5204,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.1917630285024643,
      "rewards/margins": 0.45220041275024414,
      "rewards/rejected": -0.6439634561538696,
      "step": 50
    },
    {
      "epoch": 0.11733333333333333,
      "grad_norm": 4.166290760040283,
      "learning_rate": 4.999197688241076e-06,
      "logits/chosen": -1.4534099102020264,
      "logits/rejected": -1.3092278242111206,
      "logps/chosen": -246.3505859375,
      "logps/rejected": -299.24774169921875,
      "loss": 0.4838,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2579441964626312,
      "rewards/margins": 0.5496667623519897,
      "rewards/rejected": -0.8076111078262329,
      "step": 55
    },
    {
      "epoch": 0.128,
      "grad_norm": 3.144472599029541,
      "learning_rate": 4.9983026483238685e-06,
      "logits/chosen": -1.5001187324523926,
      "logits/rejected": -1.3482980728149414,
      "logps/chosen": -234.2502899169922,
      "logps/rejected": -308.27545166015625,
      "loss": 0.4739,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.2955595850944519,
      "rewards/margins": 0.6387100219726562,
      "rewards/rejected": -0.9342697262763977,
      "step": 60
    },
    {
      "epoch": 0.13866666666666666,
      "grad_norm": 3.791512966156006,
      "learning_rate": 4.997076285653087e-06,
      "logits/chosen": -1.572967290878296,
      "logits/rejected": -1.4168083667755127,
      "logps/chosen": -252.80081176757812,
      "logps/rejected": -302.7495422363281,
      "loss": 0.4457,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.3314843773841858,
      "rewards/margins": 0.7851157188415527,
      "rewards/rejected": -1.1166000366210938,
      "step": 65
    },
    {
      "epoch": 0.14933333333333335,
      "grad_norm": 2.9347736835479736,
      "learning_rate": 4.995518762867897e-06,
      "logits/chosen": -1.5696239471435547,
      "logits/rejected": -1.4655719995498657,
      "logps/chosen": -224.55050659179688,
      "logps/rejected": -306.6766052246094,
      "loss": 0.4296,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.4790423512458801,
      "rewards/margins": 0.930472195148468,
      "rewards/rejected": -1.4095146656036377,
      "step": 70
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.7714455127716064,
      "learning_rate": 4.993630286525634e-06,
      "logits/chosen": -1.5546661615371704,
      "logits/rejected": -1.4705660343170166,
      "logps/chosen": -223.55191040039062,
      "logps/rejected": -324.2339172363281,
      "loss": 0.408,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.4930631220340729,
      "rewards/margins": 1.1179817914962769,
      "rewards/rejected": -1.6110451221466064,
      "step": 75
    },
    {
      "epoch": 0.17066666666666666,
      "grad_norm": 2.880370616912842,
      "learning_rate": 4.99141110707441e-06,
      "logits/chosen": -1.5705974102020264,
      "logits/rejected": -1.4546583890914917,
      "logps/chosen": -251.497314453125,
      "logps/rejected": -295.24920654296875,
      "loss": 0.4206,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.48983249068260193,
      "rewards/margins": 1.0172946453094482,
      "rewards/rejected": -1.507127046585083,
      "step": 80
    },
    {
      "epoch": 0.18133333333333335,
      "grad_norm": 2.310981273651123,
      "learning_rate": 4.9888615188199e-06,
      "logits/chosen": -1.649092435836792,
      "logits/rejected": -1.4132144451141357,
      "logps/chosen": -237.09988403320312,
      "logps/rejected": -305.7721252441406,
      "loss": 0.3328,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.4604329466819763,
      "rewards/margins": 1.4129736423492432,
      "rewards/rejected": -1.8734064102172852,
      "step": 85
    },
    {
      "epoch": 0.192,
      "grad_norm": 3.936634063720703,
      "learning_rate": 4.9859818598863104e-06,
      "logits/chosen": -1.5428638458251953,
      "logits/rejected": -1.3714148998260498,
      "logps/chosen": -253.7353973388672,
      "logps/rejected": -307.34625244140625,
      "loss": 0.4253,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.6021460294723511,
      "rewards/margins": 1.2255902290344238,
      "rewards/rejected": -1.827736496925354,
      "step": 90
    },
    {
      "epoch": 0.20266666666666666,
      "grad_norm": 2.6783416271209717,
      "learning_rate": 4.982772512171538e-06,
      "logits/chosen": -1.6300023794174194,
      "logits/rejected": -1.4778879880905151,
      "logps/chosen": -233.5361328125,
      "logps/rejected": -318.7696838378906,
      "loss": 0.3059,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.39014920592308044,
      "rewards/margins": 1.6632559299468994,
      "rewards/rejected": -2.053405284881592,
      "step": 95
    },
    {
      "epoch": 0.21333333333333335,
      "grad_norm": 2.8961541652679443,
      "learning_rate": 4.979233901296523e-06,
      "logits/chosen": -1.707256555557251,
      "logits/rejected": -1.5723159313201904,
      "logps/chosen": -225.2086181640625,
      "logps/rejected": -306.49444580078125,
      "loss": 0.297,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.455028235912323,
      "rewards/margins": 1.6697375774383545,
      "rewards/rejected": -2.124765634536743,
      "step": 100
    },
    {
      "epoch": 0.224,
      "grad_norm": 3.754955530166626,
      "learning_rate": 4.975366496548804e-06,
      "logits/chosen": -1.6757465600967407,
      "logits/rejected": -1.4587806463241577,
      "logps/chosen": -262.8829345703125,
      "logps/rejected": -323.92864990234375,
      "loss": 0.37,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.7448396682739258,
      "rewards/margins": 1.6061832904815674,
      "rewards/rejected": -2.351022720336914,
      "step": 105
    },
    {
      "epoch": 0.23466666666666666,
      "grad_norm": 2.480962038040161,
      "learning_rate": 4.971170810820279e-06,
      "logits/chosen": -1.6342111825942993,
      "logits/rejected": -1.4541571140289307,
      "logps/chosen": -237.62551879882812,
      "logps/rejected": -295.17742919921875,
      "loss": 0.2926,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5155616998672485,
      "rewards/margins": 1.8526878356933594,
      "rewards/rejected": -2.3682494163513184,
      "step": 110
    },
    {
      "epoch": 0.24533333333333332,
      "grad_norm": 3.0674545764923096,
      "learning_rate": 4.96664740053919e-06,
      "logits/chosen": -1.6263113021850586,
      "logits/rejected": -1.4940664768218994,
      "logps/chosen": -238.77255249023438,
      "logps/rejected": -313.794189453125,
      "loss": 0.2864,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.5373059511184692,
      "rewards/margins": 1.9535423517227173,
      "rewards/rejected": -2.4908483028411865,
      "step": 115
    },
    {
      "epoch": 0.256,
      "grad_norm": 2.636179208755493,
      "learning_rate": 4.961796865596328e-06,
      "logits/chosen": -1.698141098022461,
      "logits/rejected": -1.5799622535705566,
      "logps/chosen": -229.77719116210938,
      "logps/rejected": -291.76953125,
      "loss": 0.3145,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.5624663233757019,
      "rewards/margins": 1.8811826705932617,
      "rewards/rejected": -2.4436488151550293,
      "step": 120
    },
    {
      "epoch": 0.26666666666666666,
      "grad_norm": 2.3341727256774902,
      "learning_rate": 4.956619849265473e-06,
      "logits/chosen": -1.6567869186401367,
      "logits/rejected": -1.4683877229690552,
      "logps/chosen": -228.9341583251953,
      "logps/rejected": -299.1568603515625,
      "loss": 0.2756,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.6057113409042358,
      "rewards/margins": 2.1040005683898926,
      "rewards/rejected": -2.709711790084839,
      "step": 125
    },
    {
      "epoch": 0.2773333333333333,
      "grad_norm": 2.6476287841796875,
      "learning_rate": 4.951117038118091e-06,
      "logits/chosen": -1.7967493534088135,
      "logits/rejected": -1.561000108718872,
      "logps/chosen": -254.7824249267578,
      "logps/rejected": -323.19622802734375,
      "loss": 0.2694,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.6340410709381104,
      "rewards/margins": 2.2163166999816895,
      "rewards/rejected": -2.8503577709198,
      "step": 130
    },
    {
      "epoch": 0.288,
      "grad_norm": 2.2833471298217773,
      "learning_rate": 4.945289161932273e-06,
      "logits/chosen": -1.771174669265747,
      "logits/rejected": -1.6656172275543213,
      "logps/chosen": -248.03750610351562,
      "logps/rejected": -318.04083251953125,
      "loss": 0.2534,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.5317661762237549,
      "rewards/margins": 2.3246371746063232,
      "rewards/rejected": -2.856403350830078,
      "step": 135
    },
    {
      "epoch": 0.2986666666666667,
      "grad_norm": 3.619351625442505,
      "learning_rate": 4.93913699359596e-06,
      "logits/chosen": -1.758262038230896,
      "logits/rejected": -1.5464653968811035,
      "logps/chosen": -248.1946563720703,
      "logps/rejected": -311.05731201171875,
      "loss": 0.2719,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.5504440069198608,
      "rewards/margins": 2.0690348148345947,
      "rewards/rejected": -2.619478702545166,
      "step": 140
    },
    {
      "epoch": 0.30933333333333335,
      "grad_norm": 2.925778865814209,
      "learning_rate": 4.9326613490044364e-06,
      "logits/chosen": -1.6672502756118774,
      "logits/rejected": -1.4521372318267822,
      "logps/chosen": -261.5872802734375,
      "logps/rejected": -328.64715576171875,
      "loss": 0.2461,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.511759877204895,
      "rewards/margins": 2.3731701374053955,
      "rewards/rejected": -2.88493013381958,
      "step": 145
    },
    {
      "epoch": 0.32,
      "grad_norm": 2.139474391937256,
      "learning_rate": 4.925863086952132e-06,
      "logits/chosen": -1.646867036819458,
      "logits/rejected": -1.548393964767456,
      "logps/chosen": -252.48684692382812,
      "logps/rejected": -318.29766845703125,
      "loss": 0.2392,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.4442919194698334,
      "rewards/margins": 2.4507598876953125,
      "rewards/rejected": -2.895052194595337,
      "step": 150
    },
    {
      "epoch": 0.33066666666666666,
      "grad_norm": 2.3797218799591064,
      "learning_rate": 4.918743109018727e-06,
      "logits/chosen": -1.7487760782241821,
      "logits/rejected": -1.5734509229660034,
      "logps/chosen": -227.19985961914062,
      "logps/rejected": -301.46002197265625,
      "loss": 0.2587,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.5879919528961182,
      "rewards/margins": 2.3543381690979004,
      "rewards/rejected": -2.9423301219940186,
      "step": 155
    },
    {
      "epoch": 0.3413333333333333,
      "grad_norm": 2.023538112640381,
      "learning_rate": 4.911302359449585e-06,
      "logits/chosen": -1.854344129562378,
      "logits/rejected": -1.6243727207183838,
      "logps/chosen": -243.7124481201172,
      "logps/rejected": -324.5124206542969,
      "loss": 0.2205,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.5555624961853027,
      "rewards/margins": 2.51129412651062,
      "rewards/rejected": -3.066856861114502,
      "step": 160
    },
    {
      "epoch": 0.352,
      "grad_norm": 3.3351263999938965,
      "learning_rate": 4.9035418250305314e-06,
      "logits/chosen": -1.700575828552246,
      "logits/rejected": -1.545649766921997,
      "logps/chosen": -241.32308959960938,
      "logps/rejected": -319.8808898925781,
      "loss": 0.2507,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.5143498182296753,
      "rewards/margins": 2.7291011810302734,
      "rewards/rejected": -3.243450880050659,
      "step": 165
    },
    {
      "epoch": 0.3626666666666667,
      "grad_norm": 4.075646877288818,
      "learning_rate": 4.895462534956981e-06,
      "logits/chosen": -1.7546123266220093,
      "logits/rejected": -1.6150308847427368,
      "logps/chosen": -246.62918090820312,
      "logps/rejected": -305.169677734375,
      "loss": 0.2609,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.4843987822532654,
      "rewards/margins": 2.729264736175537,
      "rewards/rejected": -3.2136638164520264,
      "step": 170
    },
    {
      "epoch": 0.37333333333333335,
      "grad_norm": 2.9243505001068115,
      "learning_rate": 4.887065560697449e-06,
      "logits/chosen": -1.6826279163360596,
      "logits/rejected": -1.5745372772216797,
      "logps/chosen": -251.0845489501953,
      "logps/rejected": -311.9941711425781,
      "loss": 0.2163,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.40687593817710876,
      "rewards/margins": 2.7896533012390137,
      "rewards/rejected": -3.1965291500091553,
      "step": 175
    },
    {
      "epoch": 0.384,
      "grad_norm": 2.442293643951416,
      "learning_rate": 4.878352015851459e-06,
      "logits/chosen": -1.8285224437713623,
      "logits/rejected": -1.5991096496582031,
      "logps/chosen": -239.60757446289062,
      "logps/rejected": -300.1702880859375,
      "loss": 0.2069,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.2699597477912903,
      "rewards/margins": 2.888909101486206,
      "rewards/rejected": -3.1588692665100098,
      "step": 180
    },
    {
      "epoch": 0.39466666666666667,
      "grad_norm": 2.2139148712158203,
      "learning_rate": 4.869323056001852e-06,
      "logits/chosen": -1.6878550052642822,
      "logits/rejected": -1.532397985458374,
      "logps/chosen": -234.5260467529297,
      "logps/rejected": -294.7267150878906,
      "loss": 0.2396,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.4796174466609955,
      "rewards/margins": 2.7517666816711426,
      "rewards/rejected": -3.231383800506592,
      "step": 185
    },
    {
      "epoch": 0.4053333333333333,
      "grad_norm": 3.6296610832214355,
      "learning_rate": 4.859979878561536e-06,
      "logits/chosen": -1.7104793787002563,
      "logits/rejected": -1.550100564956665,
      "logps/chosen": -233.6837921142578,
      "logps/rejected": -302.03216552734375,
      "loss": 0.2463,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.5261295437812805,
      "rewards/margins": 2.788874387741089,
      "rewards/rejected": -3.3150038719177246,
      "step": 190
    },
    {
      "epoch": 0.416,
      "grad_norm": 4.420226573944092,
      "learning_rate": 4.850323722614685e-06,
      "logits/chosen": -1.785644769668579,
      "logits/rejected": -1.643869161605835,
      "logps/chosen": -284.42401123046875,
      "logps/rejected": -325.4669494628906,
      "loss": 0.2562,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.466386616230011,
      "rewards/margins": 3.1091275215148926,
      "rewards/rejected": -3.575514316558838,
      "step": 195
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 2.722252607345581,
      "learning_rate": 4.840355868752416e-06,
      "logits/chosen": -1.6250245571136475,
      "logits/rejected": -1.4949519634246826,
      "logps/chosen": -302.90606689453125,
      "logps/rejected": -318.97576904296875,
      "loss": 0.276,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.5031349062919617,
      "rewards/margins": 2.656146764755249,
      "rewards/rejected": -3.1592819690704346,
      "step": 200
    },
    {
      "epoch": 0.43733333333333335,
      "grad_norm": 2.4931721687316895,
      "learning_rate": 4.830077638902955e-06,
      "logits/chosen": -1.869330644607544,
      "logits/rejected": -1.6783950328826904,
      "logps/chosen": -216.135986328125,
      "logps/rejected": -323.79736328125,
      "loss": 0.1944,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.6569713950157166,
      "rewards/margins": 3.1940982341766357,
      "rewards/rejected": -3.851069927215576,
      "step": 205
    },
    {
      "epoch": 0.448,
      "grad_norm": 2.207162857055664,
      "learning_rate": 4.8194903961563255e-06,
      "logits/chosen": -1.837034821510315,
      "logits/rejected": -1.7211320400238037,
      "logps/chosen": -256.60430908203125,
      "logps/rejected": -348.6403503417969,
      "loss": 0.1971,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.7174243927001953,
      "rewards/margins": 3.4919838905334473,
      "rewards/rejected": -4.209408760070801,
      "step": 210
    },
    {
      "epoch": 0.45866666666666667,
      "grad_norm": 2.2889108657836914,
      "learning_rate": 4.808595544583578e-06,
      "logits/chosen": -1.724928617477417,
      "logits/rejected": -1.6084390878677368,
      "logps/chosen": -240.2550048828125,
      "logps/rejected": -350.73138427734375,
      "loss": 0.1624,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.8236926198005676,
      "rewards/margins": 3.7942657470703125,
      "rewards/rejected": -4.617957592010498,
      "step": 215
    },
    {
      "epoch": 0.4693333333333333,
      "grad_norm": 5.622860908508301,
      "learning_rate": 4.797394529050577e-06,
      "logits/chosen": -1.7057397365570068,
      "logits/rejected": -1.578086256980896,
      "logps/chosen": -269.3270263671875,
      "logps/rejected": -328.9361572265625,
      "loss": 0.2316,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.7075448632240295,
      "rewards/margins": 3.286930799484253,
      "rewards/rejected": -3.9944756031036377,
      "step": 220
    },
    {
      "epoch": 0.48,
      "grad_norm": 1.7874726057052612,
      "learning_rate": 4.785888835026393e-06,
      "logits/chosen": -1.8103277683258057,
      "logits/rejected": -1.6355822086334229,
      "logps/chosen": -248.0997314453125,
      "logps/rejected": -305.6097106933594,
      "loss": 0.1906,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.5874157547950745,
      "rewards/margins": 3.4188735485076904,
      "rewards/rejected": -4.006289482116699,
      "step": 225
    },
    {
      "epoch": 0.49066666666666664,
      "grad_norm": 4.244070053100586,
      "learning_rate": 4.7740799883862966e-06,
      "logits/chosen": -1.7845863103866577,
      "logits/rejected": -1.6956688165664673,
      "logps/chosen": -234.7997589111328,
      "logps/rejected": -296.48455810546875,
      "loss": 0.2286,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.7968846559524536,
      "rewards/margins": 3.4404284954071045,
      "rewards/rejected": -4.237313270568848,
      "step": 230
    },
    {
      "epoch": 0.5013333333333333,
      "grad_norm": 1.5574147701263428,
      "learning_rate": 4.761969555209394e-06,
      "logits/chosen": -1.7761728763580322,
      "logits/rejected": -1.6755540370941162,
      "logps/chosen": -257.5173034667969,
      "logps/rejected": -344.9666442871094,
      "loss": 0.2145,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.023348093032837,
      "rewards/margins": 3.6855106353759766,
      "rewards/rejected": -4.708858489990234,
      "step": 235
    },
    {
      "epoch": 0.512,
      "grad_norm": 3.5682854652404785,
      "learning_rate": 4.7495591415709435e-06,
      "logits/chosen": -1.7692630290985107,
      "logits/rejected": -1.662489891052246,
      "logps/chosen": -250.2325897216797,
      "logps/rejected": -339.6884765625,
      "loss": 0.1866,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.8209010362625122,
      "rewards/margins": 3.811253070831299,
      "rewards/rejected": -4.63215446472168,
      "step": 240
    },
    {
      "epoch": 0.5226666666666666,
      "grad_norm": 2.5317459106445312,
      "learning_rate": 4.736850393329353e-06,
      "logits/chosen": -1.8111603260040283,
      "logits/rejected": -1.6281133890151978,
      "logps/chosen": -264.32891845703125,
      "logps/rejected": -339.49945068359375,
      "loss": 0.1849,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8485740423202515,
      "rewards/margins": 3.6573715209960938,
      "rewards/rejected": -4.505945205688477,
      "step": 245
    },
    {
      "epoch": 0.5333333333333333,
      "grad_norm": 4.821799278259277,
      "learning_rate": 4.7238449959079095e-06,
      "logits/chosen": -1.8489344120025635,
      "logits/rejected": -1.737826943397522,
      "logps/chosen": -252.2685546875,
      "logps/rejected": -352.93438720703125,
      "loss": 0.1943,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.8746472597122192,
      "rewards/margins": 3.947354793548584,
      "rewards/rejected": -4.822002410888672,
      "step": 250
    },
    {
      "epoch": 0.544,
      "grad_norm": 1.9329137802124023,
      "learning_rate": 4.710544674071259e-06,
      "logits/chosen": -1.7850000858306885,
      "logits/rejected": -1.655548334121704,
      "logps/chosen": -264.97052001953125,
      "logps/rejected": -344.24969482421875,
      "loss": 0.2013,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.9141705632209778,
      "rewards/margins": 4.228996753692627,
      "rewards/rejected": -5.143166542053223,
      "step": 255
    },
    {
      "epoch": 0.5546666666666666,
      "grad_norm": 2.2834794521331787,
      "learning_rate": 4.696951191696673e-06,
      "logits/chosen": -1.8504130840301514,
      "logits/rejected": -1.6865558624267578,
      "logps/chosen": -246.1381378173828,
      "logps/rejected": -348.99932861328125,
      "loss": 0.1407,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.8693944811820984,
      "rewards/margins": 4.502323150634766,
      "rewards/rejected": -5.371718406677246,
      "step": 260
    },
    {
      "epoch": 0.5653333333333334,
      "grad_norm": 4.157337665557861,
      "learning_rate": 4.6830663515401196e-06,
      "logits/chosen": -1.8646320104599,
      "logits/rejected": -1.66921865940094,
      "logps/chosen": -252.67324829101562,
      "logps/rejected": -351.05133056640625,
      "loss": 0.1778,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.7936631441116333,
      "rewards/margins": 4.153098106384277,
      "rewards/rejected": -4.946761608123779,
      "step": 265
    },
    {
      "epoch": 0.576,
      "grad_norm": 1.9427365064620972,
      "learning_rate": 4.6688919949971855e-06,
      "logits/chosen": -1.761182188987732,
      "logits/rejected": -1.6403383016586304,
      "logps/chosen": -245.98727416992188,
      "logps/rejected": -356.08294677734375,
      "loss": 0.1804,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.0275193452835083,
      "rewards/margins": 3.9113502502441406,
      "rewards/rejected": -4.938869476318359,
      "step": 270
    },
    {
      "epoch": 0.5866666666666667,
      "grad_norm": 1.9152891635894775,
      "learning_rate": 4.654430001858874e-06,
      "logits/chosen": -1.7100435495376587,
      "logits/rejected": -1.6215158700942993,
      "logps/chosen": -258.38714599609375,
      "logps/rejected": -336.82720947265625,
      "loss": 0.2111,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.7515047788619995,
      "rewards/margins": 3.80024790763855,
      "rewards/rejected": -4.551753044128418,
      "step": 275
    },
    {
      "epoch": 0.5973333333333334,
      "grad_norm": 2.2597246170043945,
      "learning_rate": 4.6396822900623075e-06,
      "logits/chosen": -1.8039283752441406,
      "logits/rejected": -1.66970956325531,
      "logps/chosen": -264.4094543457031,
      "logps/rejected": -344.8823547363281,
      "loss": 0.1296,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.8846942782402039,
      "rewards/margins": 4.342747688293457,
      "rewards/rejected": -5.227442741394043,
      "step": 280
    },
    {
      "epoch": 0.608,
      "grad_norm": 4.062746047973633,
      "learning_rate": 4.624650815436368e-06,
      "logits/chosen": -1.8472976684570312,
      "logits/rejected": -1.7211005687713623,
      "logps/chosen": -253.5005645751953,
      "logps/rejected": -343.5950622558594,
      "loss": 0.1942,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.9936168789863586,
      "rewards/margins": 4.138308048248291,
      "rewards/rejected": -5.131924629211426,
      "step": 285
    },
    {
      "epoch": 0.6186666666666667,
      "grad_norm": 2.604783535003662,
      "learning_rate": 4.609337571442324e-06,
      "logits/chosen": -1.8356084823608398,
      "logits/rejected": -1.7241547107696533,
      "logps/chosen": -243.8323974609375,
      "logps/rejected": -360.04669189453125,
      "loss": 0.1449,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.9927956461906433,
      "rewards/margins": 4.568629264831543,
      "rewards/rejected": -5.561424732208252,
      "step": 290
    },
    {
      "epoch": 0.6293333333333333,
      "grad_norm": 1.896136999130249,
      "learning_rate": 4.5937445889094535e-06,
      "logits/chosen": -1.8107223510742188,
      "logits/rejected": -1.6749213933944702,
      "logps/chosen": -250.9614715576172,
      "logps/rejected": -343.70001220703125,
      "loss": 0.1625,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9373570680618286,
      "rewards/margins": 4.263827323913574,
      "rewards/rejected": -5.201184272766113,
      "step": 295
    },
    {
      "epoch": 0.64,
      "grad_norm": 4.003625392913818,
      "learning_rate": 4.577873935765722e-06,
      "logits/chosen": -1.9519007205963135,
      "logits/rejected": -1.7255446910858154,
      "logps/chosen": -245.7286376953125,
      "logps/rejected": -343.4656066894531,
      "loss": 0.1946,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.0532045364379883,
      "rewards/margins": 4.6587090492248535,
      "rewards/rejected": -5.711912631988525,
      "step": 300
    },
    {
      "epoch": 0.6506666666666666,
      "grad_norm": 2.2551584243774414,
      "learning_rate": 4.561727716763532e-06,
      "logits/chosen": -1.8950430154800415,
      "logits/rejected": -1.746446967124939,
      "logps/chosen": -257.13311767578125,
      "logps/rejected": -330.2293701171875,
      "loss": 0.2094,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.9331668615341187,
      "rewards/margins": 3.9878017902374268,
      "rewards/rejected": -4.920968532562256,
      "step": 305
    },
    {
      "epoch": 0.6613333333333333,
      "grad_norm": 2.4485409259796143,
      "learning_rate": 4.545308073200591e-06,
      "logits/chosen": -1.805546522140503,
      "logits/rejected": -1.6681687831878662,
      "logps/chosen": -247.18472290039062,
      "logps/rejected": -348.66400146484375,
      "loss": 0.2251,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.8696174621582031,
      "rewards/margins": 4.181191921234131,
      "rewards/rejected": -5.050809860229492,
      "step": 310
    },
    {
      "epoch": 0.672,
      "grad_norm": 4.527316570281982,
      "learning_rate": 4.528617182635943e-06,
      "logits/chosen": -1.8156181573867798,
      "logits/rejected": -1.715304970741272,
      "logps/chosen": -260.9232177734375,
      "logps/rejected": -349.6746520996094,
      "loss": 0.1582,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2807674407958984,
      "rewards/margins": 4.808705806732178,
      "rewards/rejected": -6.089472770690918,
      "step": 315
    },
    {
      "epoch": 0.6826666666666666,
      "grad_norm": 2.6639559268951416,
      "learning_rate": 4.511657258601171e-06,
      "logits/chosen": -1.814720869064331,
      "logits/rejected": -1.5974810123443604,
      "logps/chosen": -273.97039794921875,
      "logps/rejected": -347.71075439453125,
      "loss": 0.1918,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.1705496311187744,
      "rewards/margins": 4.306663990020752,
      "rewards/rejected": -5.4772138595581055,
      "step": 320
    },
    {
      "epoch": 0.6933333333333334,
      "grad_norm": 3.535733699798584,
      "learning_rate": 4.494430550306848e-06,
      "logits/chosen": -1.6838403940200806,
      "logits/rejected": -1.6364444494247437,
      "logps/chosen": -251.2472686767578,
      "logps/rejected": -342.08489990234375,
      "loss": 0.2131,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.7756344079971313,
      "rewards/margins": 4.671334266662598,
      "rewards/rejected": -5.446969032287598,
      "step": 325
    },
    {
      "epoch": 0.704,
      "grad_norm": 4.793890476226807,
      "learning_rate": 4.476939342344246e-06,
      "logits/chosen": -1.9221420288085938,
      "logits/rejected": -1.7676732540130615,
      "logps/chosen": -247.0741424560547,
      "logps/rejected": -332.93292236328125,
      "loss": 0.1601,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.9986921548843384,
      "rewards/margins": 4.544274806976318,
      "rewards/rejected": -5.542966842651367,
      "step": 330
    },
    {
      "epoch": 0.7146666666666667,
      "grad_norm": 6.12879753112793,
      "learning_rate": 4.45918595438236e-06,
      "logits/chosen": -1.7161098718643188,
      "logits/rejected": -1.6740608215332031,
      "logps/chosen": -261.7930908203125,
      "logps/rejected": -353.0909118652344,
      "loss": 0.1974,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.0057028532028198,
      "rewards/margins": 4.220341205596924,
      "rewards/rejected": -5.226044178009033,
      "step": 335
    },
    {
      "epoch": 0.7253333333333334,
      "grad_norm": 3.4764912128448486,
      "learning_rate": 4.441172740860267e-06,
      "logits/chosen": -1.812603235244751,
      "logits/rejected": -1.6671600341796875,
      "logps/chosen": -256.25775146484375,
      "logps/rejected": -343.37750244140625,
      "loss": 0.1811,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.1001125574111938,
      "rewards/margins": 4.227419853210449,
      "rewards/rejected": -5.327532768249512,
      "step": 340
    },
    {
      "epoch": 0.736,
      "grad_norm": 3.187913656234741,
      "learning_rate": 4.422902090674886e-06,
      "logits/chosen": -1.6854442358016968,
      "logits/rejected": -1.6130621433258057,
      "logps/chosen": -248.1710662841797,
      "logps/rejected": -364.08953857421875,
      "loss": 0.1584,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.8657025098800659,
      "rewards/margins": 4.714477062225342,
      "rewards/rejected": -5.580178737640381,
      "step": 345
    },
    {
      "epoch": 0.7466666666666667,
      "grad_norm": 2.092165470123291,
      "learning_rate": 4.4043764268641705e-06,
      "logits/chosen": -1.7814304828643799,
      "logits/rejected": -1.6742616891860962,
      "logps/chosen": -243.86672973632812,
      "logps/rejected": -343.56060791015625,
      "loss": 0.183,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.8583084940910339,
      "rewards/margins": 4.422949314117432,
      "rewards/rejected": -5.2812581062316895,
      "step": 350
    },
    {
      "epoch": 0.7573333333333333,
      "grad_norm": 2.406649351119995,
      "learning_rate": 4.385598206285758e-06,
      "logits/chosen": -1.8445268869400024,
      "logits/rejected": -1.7397022247314453,
      "logps/chosen": -274.53399658203125,
      "logps/rejected": -365.04168701171875,
      "loss": 0.1659,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0924501419067383,
      "rewards/margins": 4.701749324798584,
      "rewards/rejected": -5.794199466705322,
      "step": 355
    },
    {
      "epoch": 0.768,
      "grad_norm": 3.1149144172668457,
      "learning_rate": 4.3665699192911495e-06,
      "logits/chosen": -1.784285545349121,
      "logits/rejected": -1.664687156677246,
      "logps/chosen": -251.3987579345703,
      "logps/rejected": -350.62103271484375,
      "loss": 0.1909,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1020582914352417,
      "rewards/margins": 4.7440361976623535,
      "rewards/rejected": -5.846094131469727,
      "step": 360
    },
    {
      "epoch": 0.7786666666666666,
      "grad_norm": 3.1475017070770264,
      "learning_rate": 4.347294089395438e-06,
      "logits/chosen": -1.75801682472229,
      "logits/rejected": -1.629412293434143,
      "logps/chosen": -267.57562255859375,
      "logps/rejected": -339.25567626953125,
      "loss": 0.1784,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.2324341535568237,
      "rewards/margins": 4.7525458335876465,
      "rewards/rejected": -5.984980583190918,
      "step": 365
    },
    {
      "epoch": 0.7893333333333333,
      "grad_norm": 5.428934097290039,
      "learning_rate": 4.327773272942647e-06,
      "logits/chosen": -1.7988414764404297,
      "logits/rejected": -1.610979437828064,
      "logps/chosen": -254.54312133789062,
      "logps/rejected": -333.9504699707031,
      "loss": 0.1686,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1293680667877197,
      "rewards/margins": 5.0215744972229,
      "rewards/rejected": -6.150942802429199,
      "step": 370
    },
    {
      "epoch": 0.8,
      "grad_norm": 3.3105201721191406,
      "learning_rate": 4.308010058766703e-06,
      "logits/chosen": -1.7741140127182007,
      "logits/rejected": -1.7098277807235718,
      "logps/chosen": -270.8016357421875,
      "logps/rejected": -379.1322326660156,
      "loss": 0.1397,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.3129689693450928,
      "rewards/margins": 4.9763569831848145,
      "rewards/rejected": -6.289326190948486,
      "step": 375
    },
    {
      "epoch": 0.8106666666666666,
      "grad_norm": 2.7516579627990723,
      "learning_rate": 4.2880070678481135e-06,
      "logits/chosen": -1.8347715139389038,
      "logits/rejected": -1.6875629425048828,
      "logps/chosen": -234.46694946289062,
      "logps/rejected": -346.2992248535156,
      "loss": 0.1281,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.0527235269546509,
      "rewards/margins": 5.5215559005737305,
      "rewards/rejected": -6.57427978515625,
      "step": 380
    },
    {
      "epoch": 0.8213333333333334,
      "grad_norm": 3.6830787658691406,
      "learning_rate": 4.267766952966369e-06,
      "logits/chosen": -1.76150643825531,
      "logits/rejected": -1.695257544517517,
      "logps/chosen": -247.07595825195312,
      "logps/rejected": -348.04656982421875,
      "loss": 0.1845,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.2883862257003784,
      "rewards/margins": 4.600207328796387,
      "rewards/rejected": -5.8885931968688965,
      "step": 385
    },
    {
      "epoch": 0.832,
      "grad_norm": 4.31989860534668,
      "learning_rate": 4.24729239834814e-06,
      "logits/chosen": -1.7455536127090454,
      "logits/rejected": -1.6559183597564697,
      "logps/chosen": -253.7515869140625,
      "logps/rejected": -348.94537353515625,
      "loss": 0.21,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.2032108306884766,
      "rewards/margins": 4.9352593421936035,
      "rewards/rejected": -6.138470649719238,
      "step": 390
    },
    {
      "epoch": 0.8426666666666667,
      "grad_norm": 4.060668468475342,
      "learning_rate": 4.226586119311291e-06,
      "logits/chosen": -1.775027871131897,
      "logits/rejected": -1.7871853113174438,
      "logps/chosen": -264.8758239746094,
      "logps/rejected": -360.35614013671875,
      "loss": 0.1818,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.096254825592041,
      "rewards/margins": 5.317919731140137,
      "rewards/rejected": -6.414175510406494,
      "step": 395
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 3.2177236080169678,
      "learning_rate": 4.205650861904782e-06,
      "logits/chosen": -1.8364406824111938,
      "logits/rejected": -1.6941852569580078,
      "logps/chosen": -226.8248291015625,
      "logps/rejected": -345.60479736328125,
      "loss": 0.1841,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.129191517829895,
      "rewards/margins": 4.975957870483398,
      "rewards/rejected": -6.105149269104004,
      "step": 400
    },
    {
      "epoch": 0.864,
      "grad_norm": 3.1651129722595215,
      "learning_rate": 4.184489402544486e-06,
      "logits/chosen": -1.8180274963378906,
      "logits/rejected": -1.723665475845337,
      "logps/chosen": -229.318115234375,
      "logps/rejected": -314.98675537109375,
      "loss": 0.2303,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.1190117597579956,
      "rewards/margins": 4.453101634979248,
      "rewards/rejected": -5.572113037109375,
      "step": 405
    },
    {
      "epoch": 0.8746666666666667,
      "grad_norm": 3.7597198486328125,
      "learning_rate": 4.1631045476449865e-06,
      "logits/chosen": -1.9217109680175781,
      "logits/rejected": -1.765979528427124,
      "logps/chosen": -237.1017303466797,
      "logps/rejected": -344.9607849121094,
      "loss": 0.1314,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.8680995106697083,
      "rewards/margins": 5.295489311218262,
      "rewards/rejected": -6.163588523864746,
      "step": 410
    },
    {
      "epoch": 0.8853333333333333,
      "grad_norm": 1.5503333806991577,
      "learning_rate": 4.141499133247391e-06,
      "logits/chosen": -1.674448013305664,
      "logits/rejected": -1.6276872158050537,
      "logps/chosen": -268.17816162109375,
      "logps/rejected": -357.49810791015625,
      "loss": 0.1353,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.7698339223861694,
      "rewards/margins": 5.1189494132995605,
      "rewards/rejected": -5.8887834548950195,
      "step": 415
    },
    {
      "epoch": 0.896,
      "grad_norm": 3.54842472076416,
      "learning_rate": 4.119676024643217e-06,
      "logits/chosen": -1.6974074840545654,
      "logits/rejected": -1.6472240686416626,
      "logps/chosen": -257.77655029296875,
      "logps/rejected": -349.54205322265625,
      "loss": 0.1515,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.0973252058029175,
      "rewards/margins": 5.101239204406738,
      "rewards/rejected": -6.198564529418945,
      "step": 420
    },
    {
      "epoch": 0.9066666666666666,
      "grad_norm": 5.144434452056885,
      "learning_rate": 4.0976381159944055e-06,
      "logits/chosen": -1.803943395614624,
      "logits/rejected": -1.6611639261245728,
      "logps/chosen": -271.3909912109375,
      "logps/rejected": -372.34234619140625,
      "loss": 0.1513,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.0220460891723633,
      "rewards/margins": 5.32880163192749,
      "rewards/rejected": -6.350848197937012,
      "step": 425
    },
    {
      "epoch": 0.9173333333333333,
      "grad_norm": 2.2385873794555664,
      "learning_rate": 4.07538832994949e-06,
      "logits/chosen": -1.7902886867523193,
      "logits/rejected": -1.5918269157409668,
      "logps/chosen": -251.48599243164062,
      "logps/rejected": -345.81292724609375,
      "loss": 0.1848,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.0862038135528564,
      "rewards/margins": 5.2021284103393555,
      "rewards/rejected": -6.288332462310791,
      "step": 430
    },
    {
      "epoch": 0.928,
      "grad_norm": 4.186896324157715,
      "learning_rate": 4.0529296172560055e-06,
      "logits/chosen": -1.8904718160629272,
      "logits/rejected": -1.6817710399627686,
      "logps/chosen": -238.9318389892578,
      "logps/rejected": -353.1915588378906,
      "loss": 0.1555,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.286082148551941,
      "rewards/margins": 5.383737087249756,
      "rewards/rejected": -6.6698198318481445,
      "step": 435
    },
    {
      "epoch": 0.9386666666666666,
      "grad_norm": 2.397501230239868,
      "learning_rate": 4.030264956369158e-06,
      "logits/chosen": -1.8631141185760498,
      "logits/rejected": -1.6818256378173828,
      "logps/chosen": -233.68563842773438,
      "logps/rejected": -355.0934143066406,
      "loss": 0.1211,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.9764868021011353,
      "rewards/margins": 5.496334075927734,
      "rewards/rejected": -6.472821235656738,
      "step": 440
    },
    {
      "epoch": 0.9493333333333334,
      "grad_norm": 6.384281635284424,
      "learning_rate": 4.0073973530568274e-06,
      "logits/chosen": -1.823751449584961,
      "logits/rejected": -1.6468112468719482,
      "logps/chosen": -261.18084716796875,
      "logps/rejected": -354.4629211425781,
      "loss": 0.2524,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.3924033641815186,
      "rewards/margins": 5.070466995239258,
      "rewards/rejected": -6.462870121002197,
      "step": 445
    },
    {
      "epoch": 0.96,
      "grad_norm": 3.978393077850342,
      "learning_rate": 3.9843298400009435e-06,
      "logits/chosen": -1.8724792003631592,
      "logits/rejected": -1.7450891733169556,
      "logps/chosen": -235.0048065185547,
      "logps/rejected": -355.290771484375,
      "loss": 0.1176,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.1844699382781982,
      "rewards/margins": 5.504419326782227,
      "rewards/rejected": -6.688889503479004,
      "step": 450
    },
    {
      "epoch": 0.9706666666666667,
      "grad_norm": 2.8582115173339844,
      "learning_rate": 3.961065476395296e-06,
      "logits/chosen": -1.8186849355697632,
      "logits/rejected": -1.6841886043548584,
      "logps/chosen": -266.6614990234375,
      "logps/rejected": -351.87646484375,
      "loss": 0.1637,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.1914622783660889,
      "rewards/margins": 5.31570291519165,
      "rewards/rejected": -6.507164001464844,
      "step": 455
    },
    {
      "epoch": 0.9813333333333333,
      "grad_norm": 2.4496090412139893,
      "learning_rate": 3.937607347539823e-06,
      "logits/chosen": -1.772597074508667,
      "logits/rejected": -1.6899995803833008,
      "logps/chosen": -258.0954895019531,
      "logps/rejected": -382.6227722167969,
      "loss": 0.1429,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5825755596160889,
      "rewards/margins": 5.6097493171691895,
      "rewards/rejected": -7.192324638366699,
      "step": 460
    },
    {
      "epoch": 0.992,
      "grad_norm": 1.6265991926193237,
      "learning_rate": 3.913958564431447e-06,
      "logits/chosen": -1.9242236614227295,
      "logits/rejected": -1.760114312171936,
      "logps/chosen": -239.49014282226562,
      "logps/rejected": -336.96502685546875,
      "loss": 0.1317,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.038116693496704,
      "rewards/margins": 5.815273761749268,
      "rewards/rejected": -6.853390693664551,
      "step": 465
    },
    {
      "epoch": 1.0021333333333333,
      "grad_norm": 2.376556396484375,
      "learning_rate": 3.89012226335149e-06,
      "logits/chosen": -1.7828489542007446,
      "logits/rejected": -1.676961064338684,
      "logps/chosen": -239.7880859375,
      "logps/rejected": -330.6512145996094,
      "loss": 0.2273,
      "rewards/accuracies": 0.9013158082962036,
      "rewards/chosen": -1.1525920629501343,
      "rewards/margins": 4.860694408416748,
      "rewards/rejected": -6.013286113739014,
      "step": 470
    },
    {
      "epoch": 1.0128,
      "grad_norm": 3.2269089221954346,
      "learning_rate": 3.86610160544975e-06,
      "logits/chosen": -1.8129554986953735,
      "logits/rejected": -1.6979223489761353,
      "logps/chosen": -257.1238098144531,
      "logps/rejected": -371.0331726074219,
      "loss": 0.1038,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.2004741430282593,
      "rewards/margins": 5.8469557762146,
      "rewards/rejected": -7.047430515289307,
      "step": 475
    },
    {
      "epoch": 1.0234666666666667,
      "grad_norm": 3.285102605819702,
      "learning_rate": 3.841899776325267e-06,
      "logits/chosen": -1.6993519067764282,
      "logits/rejected": -1.5625213384628296,
      "logps/chosen": -264.1551818847656,
      "logps/rejected": -342.95697021484375,
      "loss": 0.143,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.0834065675735474,
      "rewards/margins": 5.366703987121582,
      "rewards/rejected": -6.450111389160156,
      "step": 480
    },
    {
      "epoch": 1.0341333333333333,
      "grad_norm": 2.0090343952178955,
      "learning_rate": 3.817519985603855e-06,
      "logits/chosen": -1.8892101049423218,
      "logits/rejected": -1.7172794342041016,
      "logps/chosen": -235.93984985351562,
      "logps/rejected": -370.1104736328125,
      "loss": 0.0987,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.0467970371246338,
      "rewards/margins": 5.996671199798584,
      "rewards/rejected": -7.0434675216674805,
      "step": 485
    },
    {
      "epoch": 1.0448,
      "grad_norm": 1.3430184125900269,
      "learning_rate": 3.792965466512437e-06,
      "logits/chosen": -1.7806406021118164,
      "logits/rejected": -1.664324402809143,
      "logps/chosen": -261.94598388671875,
      "logps/rejected": -347.97021484375,
      "loss": 0.1651,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.9251692891120911,
      "rewards/margins": 5.451348304748535,
      "rewards/rejected": -6.376518249511719,
      "step": 490
    },
    {
      "epoch": 1.0554666666666668,
      "grad_norm": 3.2143454551696777,
      "learning_rate": 3.7682394754502687e-06,
      "logits/chosen": -1.9018503427505493,
      "logits/rejected": -1.776320457458496,
      "logps/chosen": -250.692138671875,
      "logps/rejected": -362.97637939453125,
      "loss": 0.1558,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.189073920249939,
      "rewards/margins": 5.650210380554199,
      "rewards/rejected": -6.839284420013428,
      "step": 495
    },
    {
      "epoch": 1.0661333333333334,
      "grad_norm": 2.513195276260376,
      "learning_rate": 3.7433452915570685e-06,
      "logits/chosen": -1.8409998416900635,
      "logits/rejected": -1.7353975772857666,
      "logps/chosen": -232.98306274414062,
      "logps/rejected": -346.6318664550781,
      "loss": 0.1149,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9560970067977905,
      "rewards/margins": 5.840074062347412,
      "rewards/rejected": -6.796170711517334,
      "step": 500
    },
    {
      "epoch": 1.0768,
      "grad_norm": 2.5323407649993896,
      "learning_rate": 3.718286216278143e-06,
      "logits/chosen": -1.850425124168396,
      "logits/rejected": -1.6678615808486938,
      "logps/chosen": -254.27713012695312,
      "logps/rejected": -357.68731689453125,
      "loss": 0.1227,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.3727056980133057,
      "rewards/margins": 5.668168067932129,
      "rewards/rejected": -7.0408735275268555,
      "step": 505
    },
    {
      "epoch": 1.0874666666666666,
      "grad_norm": 4.070751667022705,
      "learning_rate": 3.6930655729265557e-06,
      "logits/chosen": -1.8335998058319092,
      "logits/rejected": -1.613699197769165,
      "logps/chosen": -284.2073974609375,
      "logps/rejected": -358.90069580078125,
      "loss": 0.122,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.4791011810302734,
      "rewards/margins": 5.614500522613525,
      "rewards/rejected": -7.093601226806641,
      "step": 510
    },
    {
      "epoch": 1.0981333333333334,
      "grad_norm": 2.953993558883667,
      "learning_rate": 3.667686706242386e-06,
      "logits/chosen": -1.7201465368270874,
      "logits/rejected": -1.6716951131820679,
      "logps/chosen": -245.04855346679688,
      "logps/rejected": -360.8467102050781,
      "loss": 0.1177,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.494508981704712,
      "rewards/margins": 5.6371870040893555,
      "rewards/rejected": -7.131695747375488,
      "step": 515
    },
    {
      "epoch": 1.1088,
      "grad_norm": 4.227024078369141,
      "learning_rate": 3.6421529819491576e-06,
      "logits/chosen": -1.8493587970733643,
      "logits/rejected": -1.7108128070831299,
      "logps/chosen": -256.4563903808594,
      "logps/rejected": -361.6827087402344,
      "loss": 0.1113,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.465549111366272,
      "rewards/margins": 6.157153129577637,
      "rewards/rejected": -7.622702121734619,
      "step": 520
    },
    {
      "epoch": 1.1194666666666666,
      "grad_norm": 4.163751125335693,
      "learning_rate": 3.6164677863074783e-06,
      "logits/chosen": -1.834673523902893,
      "logits/rejected": -1.723134994506836,
      "logps/chosen": -245.7028350830078,
      "logps/rejected": -347.9207763671875,
      "loss": 0.1446,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.819270372390747,
      "rewards/margins": 5.923323154449463,
      "rewards/rejected": -7.742593288421631,
      "step": 525
    },
    {
      "epoch": 1.1301333333333332,
      "grad_norm": 2.243851661682129,
      "learning_rate": 3.590634525665955e-06,
      "logits/chosen": -1.9202182292938232,
      "logits/rejected": -1.7526201009750366,
      "logps/chosen": -237.74996948242188,
      "logps/rejected": -365.3797302246094,
      "loss": 0.1187,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.5946757793426514,
      "rewards/margins": 6.295979022979736,
      "rewards/rejected": -7.890654563903809,
      "step": 530
    },
    {
      "epoch": 1.1408,
      "grad_norm": 1.7002406120300293,
      "learning_rate": 3.564656626009446e-06,
      "logits/chosen": -1.9501392841339111,
      "logits/rejected": -1.775882363319397,
      "logps/chosen": -249.8088836669922,
      "logps/rejected": -361.65216064453125,
      "loss": 0.1579,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.8825603723526,
      "rewards/margins": 6.139547824859619,
      "rewards/rejected": -8.02210807800293,
      "step": 535
    },
    {
      "epoch": 1.1514666666666666,
      "grad_norm": 2.868656873703003,
      "learning_rate": 3.5385375325047167e-06,
      "logits/chosen": -1.900363564491272,
      "logits/rejected": -1.7623260021209717,
      "logps/chosen": -258.2088317871094,
      "logps/rejected": -369.24456787109375,
      "loss": 0.1002,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.8428981304168701,
      "rewards/margins": 6.456591606140137,
      "rewards/rejected": -8.299489974975586,
      "step": 540
    },
    {
      "epoch": 1.1621333333333332,
      "grad_norm": 4.660250186920166,
      "learning_rate": 3.512280709043534e-06,
      "logits/chosen": -1.8281036615371704,
      "logits/rejected": -1.7140634059906006,
      "logps/chosen": -260.32257080078125,
      "logps/rejected": -350.28045654296875,
      "loss": 0.161,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2407617568969727,
      "rewards/margins": 5.985723972320557,
      "rewards/rejected": -7.226485252380371,
      "step": 545
    },
    {
      "epoch": 1.1728,
      "grad_norm": 2.281893014907837,
      "learning_rate": 3.4858896377832966e-06,
      "logits/chosen": -1.729958176612854,
      "logits/rejected": -1.7020686864852905,
      "logps/chosen": -252.7555389404297,
      "logps/rejected": -387.8126525878906,
      "loss": 0.098,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.817659616470337,
      "rewards/margins": 6.727684020996094,
      "rewards/rejected": -8.545343399047852,
      "step": 550
    },
    {
      "epoch": 1.1834666666666667,
      "grad_norm": 2.4751644134521484,
      "learning_rate": 3.459367818685231e-06,
      "logits/chosen": -1.7661396265029907,
      "logits/rejected": -1.7389978170394897,
      "logps/chosen": -279.11309814453125,
      "logps/rejected": -386.31610107421875,
      "loss": 0.0959,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.7105395793914795,
      "rewards/margins": 6.346243381500244,
      "rewards/rejected": -8.056783676147461,
      "step": 555
    },
    {
      "epoch": 1.1941333333333333,
      "grad_norm": 1.502102255821228,
      "learning_rate": 3.432718769050228e-06,
      "logits/chosen": -1.709663987159729,
      "logits/rejected": -1.6350157260894775,
      "logps/chosen": -248.32553100585938,
      "logps/rejected": -371.6622009277344,
      "loss": 0.0819,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.5119163990020752,
      "rewards/margins": 6.394644737243652,
      "rewards/rejected": -7.906561374664307,
      "step": 560
    },
    {
      "epoch": 1.2048,
      "grad_norm": 2.423750162124634,
      "learning_rate": 3.405946023052386e-06,
      "logits/chosen": -1.772951364517212,
      "logits/rejected": -1.7209113836288452,
      "logps/chosen": -244.55960083007812,
      "logps/rejected": -361.45416259765625,
      "loss": 0.1398,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.6022535562515259,
      "rewards/margins": 6.2257208824157715,
      "rewards/rejected": -7.82797384262085,
      "step": 565
    },
    {
      "epoch": 1.2154666666666667,
      "grad_norm": 5.750537395477295,
      "learning_rate": 3.3790531312703055e-06,
      "logits/chosen": -1.8179404735565186,
      "logits/rejected": -1.689639687538147,
      "logps/chosen": -268.6795959472656,
      "logps/rejected": -380.632080078125,
      "loss": 0.1195,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.895990014076233,
      "rewards/margins": 6.5666608810424805,
      "rewards/rejected": -8.462652206420898,
      "step": 570
    },
    {
      "epoch": 1.2261333333333333,
      "grad_norm": 2.973874092102051,
      "learning_rate": 3.35204366021622e-06,
      "logits/chosen": -1.7112888097763062,
      "logits/rejected": -1.6027559041976929,
      "logps/chosen": -248.86685180664062,
      "logps/rejected": -377.16497802734375,
      "loss": 0.1024,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.2404453754425049,
      "rewards/margins": 6.802675724029541,
      "rewards/rejected": -8.043119430541992,
      "step": 575
    },
    {
      "epoch": 1.2368000000000001,
      "grad_norm": 3.057502508163452,
      "learning_rate": 3.3249211918630055e-06,
      "logits/chosen": -1.7631298303604126,
      "logits/rejected": -1.6719868183135986,
      "logps/chosen": -265.61187744140625,
      "logps/rejected": -348.72979736328125,
      "loss": 0.1675,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.6381809711456299,
      "rewards/margins": 5.736741065979004,
      "rewards/rejected": -7.374920845031738,
      "step": 580
    },
    {
      "epoch": 1.2474666666666667,
      "grad_norm": 3.961693525314331,
      "learning_rate": 3.2976893231691424e-06,
      "logits/chosen": -1.7621526718139648,
      "logits/rejected": -1.6848115921020508,
      "logps/chosen": -258.19683837890625,
      "logps/rejected": -355.4867858886719,
      "loss": 0.1452,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.344029188156128,
      "rewards/margins": 5.742987632751465,
      "rewards/rejected": -7.087018013000488,
      "step": 585
    },
    {
      "epoch": 1.2581333333333333,
      "grad_norm": 4.668637752532959,
      "learning_rate": 3.270351665601691e-06,
      "logits/chosen": -1.7807848453521729,
      "logits/rejected": -1.6941245794296265,
      "logps/chosen": -263.23193359375,
      "logps/rejected": -364.79876708984375,
      "loss": 0.1789,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.6055269241333008,
      "rewards/margins": 5.881946086883545,
      "rewards/rejected": -7.4874725341796875,
      "step": 590
    },
    {
      "epoch": 1.2688,
      "grad_norm": 1.626665472984314,
      "learning_rate": 3.2429118446573417e-06,
      "logits/chosen": -1.7861591577529907,
      "logits/rejected": -1.6627706289291382,
      "logps/chosen": -242.6871795654297,
      "logps/rejected": -369.5167541503906,
      "loss": 0.124,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.4150798320770264,
      "rewards/margins": 6.047471523284912,
      "rewards/rejected": -7.462552070617676,
      "step": 595
    },
    {
      "epoch": 1.2794666666666665,
      "grad_norm": 1.1785833835601807,
      "learning_rate": 3.215373499381602e-06,
      "logits/chosen": -1.7399671077728271,
      "logits/rejected": -1.6407349109649658,
      "logps/chosen": -253.98306274414062,
      "logps/rejected": -362.9200439453125,
      "loss": 0.1039,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.304102897644043,
      "rewards/margins": 6.121998310089111,
      "rewards/rejected": -7.426100730895996,
      "step": 600
    },
    {
      "epoch": 1.2901333333333334,
      "grad_norm": 2.841127395629883,
      "learning_rate": 3.1877402818861954e-06,
      "logits/chosen": -1.7944116592407227,
      "logits/rejected": -1.6870648860931396,
      "logps/chosen": -275.91058349609375,
      "logps/rejected": -376.3899230957031,
      "loss": 0.0709,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.7364799976348877,
      "rewards/margins": 6.606210231781006,
      "rewards/rejected": -8.342691421508789,
      "step": 605
    },
    {
      "epoch": 1.3008,
      "grad_norm": 4.171568393707275,
      "learning_rate": 3.1600158568647145e-06,
      "logits/chosen": -1.7378740310668945,
      "logits/rejected": -1.6512172222137451,
      "logps/chosen": -256.87786865234375,
      "logps/rejected": -380.0071105957031,
      "loss": 0.0842,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.5280927419662476,
      "rewards/margins": 6.7154693603515625,
      "rewards/rejected": -8.243562698364258,
      "step": 610
    },
    {
      "epoch": 1.3114666666666666,
      "grad_norm": 1.7711259126663208,
      "learning_rate": 3.1322039011066172e-06,
      "logits/chosen": -1.7013614177703857,
      "logits/rejected": -1.6217601299285889,
      "logps/chosen": -290.1616516113281,
      "logps/rejected": -374.9538269042969,
      "loss": 0.1086,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.6339313983917236,
      "rewards/margins": 7.052574157714844,
      "rewards/rejected": -8.686506271362305,
      "step": 615
    },
    {
      "epoch": 1.3221333333333334,
      "grad_norm": 1.6588112115859985,
      "learning_rate": 3.1043081030096125e-06,
      "logits/chosen": -1.6777381896972656,
      "logits/rejected": -1.5866796970367432,
      "logps/chosen": -251.49923706054688,
      "logps/rejected": -379.4061584472656,
      "loss": 0.0912,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.4971685409545898,
      "rewards/margins": 7.028067111968994,
      "rewards/rejected": -8.525236129760742,
      "step": 620
    },
    {
      "epoch": 1.3328,
      "grad_norm": 1.202467441558838,
      "learning_rate": 3.0763321620905072e-06,
      "logits/chosen": -1.757503867149353,
      "logits/rejected": -1.6371698379516602,
      "logps/chosen": -273.4452209472656,
      "logps/rejected": -375.41668701171875,
      "loss": 0.108,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.6967731714248657,
      "rewards/margins": 6.615039825439453,
      "rewards/rejected": -8.311813354492188,
      "step": 625
    },
    {
      "epoch": 1.3434666666666666,
      "grad_norm": 2.2688076496124268,
      "learning_rate": 3.048279788494583e-06,
      "logits/chosen": -1.785783052444458,
      "logits/rejected": -1.7002527713775635,
      "logps/chosen": -257.37322998046875,
      "logps/rejected": -377.8409423828125,
      "loss": 0.1454,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.9974949359893799,
      "rewards/margins": 6.213410377502441,
      "rewards/rejected": -8.210905075073242,
      "step": 630
    },
    {
      "epoch": 1.3541333333333334,
      "grad_norm": 2.4637012481689453,
      "learning_rate": 3.0201547025035553e-06,
      "logits/chosen": -1.751521110534668,
      "logits/rejected": -1.6897504329681396,
      "logps/chosen": -254.302978515625,
      "logps/rejected": -378.50323486328125,
      "loss": 0.1022,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.6483898162841797,
      "rewards/margins": 6.3102569580078125,
      "rewards/rejected": -7.958646297454834,
      "step": 635
    },
    {
      "epoch": 1.3648,
      "grad_norm": 3.8461568355560303,
      "learning_rate": 2.9919606340421963e-06,
      "logits/chosen": -1.7511999607086182,
      "logits/rejected": -1.624850869178772,
      "logps/chosen": -235.1289825439453,
      "logps/rejected": -380.01055908203125,
      "loss": 0.0942,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.9077924489974976,
      "rewards/margins": 6.974759101867676,
      "rewards/rejected": -8.882550239562988,
      "step": 640
    },
    {
      "epoch": 1.3754666666666666,
      "grad_norm": 3.0227408409118652,
      "learning_rate": 2.9637013221836723e-06,
      "logits/chosen": -1.8264532089233398,
      "logits/rejected": -1.678780198097229,
      "logps/chosen": -258.0480651855469,
      "logps/rejected": -377.66595458984375,
      "loss": 0.129,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.6189113855361938,
      "rewards/margins": 6.719698429107666,
      "rewards/rejected": -8.33860969543457,
      "step": 645
    },
    {
      "epoch": 1.3861333333333334,
      "grad_norm": 2.1811294555664062,
      "learning_rate": 2.9353805146536762e-06,
      "logits/chosen": -1.7129695415496826,
      "logits/rejected": -1.649914026260376,
      "logps/chosen": -264.8773498535156,
      "logps/rejected": -379.33892822265625,
      "loss": 0.0935,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.8796179294586182,
      "rewards/margins": 7.090023994445801,
      "rewards/rejected": -8.96964168548584,
      "step": 650
    },
    {
      "epoch": 1.3968,
      "grad_norm": 4.21340799331665,
      "learning_rate": 2.907001967333401e-06,
      "logits/chosen": -1.7497447729110718,
      "logits/rejected": -1.6952203512191772,
      "logps/chosen": -251.13497924804688,
      "logps/rejected": -375.689697265625,
      "loss": 0.1294,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.122582197189331,
      "rewards/margins": 6.757372856140137,
      "rewards/rejected": -8.879955291748047,
      "step": 655
    },
    {
      "epoch": 1.4074666666666666,
      "grad_norm": 1.3289350271224976,
      "learning_rate": 2.878569443761442e-06,
      "logits/chosen": -1.6650772094726562,
      "logits/rejected": -1.5900347232818604,
      "logps/chosen": -247.1836395263672,
      "logps/rejected": -369.2092590332031,
      "loss": 0.108,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.7309074401855469,
      "rewards/margins": 6.929853916168213,
      "rewards/rejected": -8.660760879516602,
      "step": 660
    },
    {
      "epoch": 1.4181333333333335,
      "grad_norm": 3.958869218826294,
      "learning_rate": 2.85008671463468e-06,
      "logits/chosen": -1.7087314128875732,
      "logits/rejected": -1.6473360061645508,
      "logps/chosen": -262.6588134765625,
      "logps/rejected": -387.8702087402344,
      "loss": 0.1213,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.9649025201797485,
      "rewards/margins": 6.696756839752197,
      "rewards/rejected": -8.66165828704834,
      "step": 665
    },
    {
      "epoch": 1.4288,
      "grad_norm": 3.1254711151123047,
      "learning_rate": 2.8215575573082127e-06,
      "logits/chosen": -1.730808973312378,
      "logits/rejected": -1.6865466833114624,
      "logps/chosen": -242.3497314453125,
      "logps/rejected": -411.3504943847656,
      "loss": 0.1223,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.7539913654327393,
      "rewards/margins": 7.253933906555176,
      "rewards/rejected": -9.007925033569336,
      "step": 670
    },
    {
      "epoch": 1.4394666666666667,
      "grad_norm": 0.7078645825386047,
      "learning_rate": 2.792985755294404e-06,
      "logits/chosen": -1.8119432926177979,
      "logits/rejected": -1.7146638631820679,
      "logps/chosen": -256.441162109375,
      "logps/rejected": -366.22210693359375,
      "loss": 0.1173,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.8375685214996338,
      "rewards/margins": 6.874407768249512,
      "rewards/rejected": -8.711977005004883,
      "step": 675
    },
    {
      "epoch": 1.4501333333333333,
      "grad_norm": 3.9787957668304443,
      "learning_rate": 2.764375097761122e-06,
      "logits/chosen": -1.6281518936157227,
      "logits/rejected": -1.5047622919082642,
      "logps/chosen": -261.5027770996094,
      "logps/rejected": -338.6407775878906,
      "loss": 0.1576,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.5838897228240967,
      "rewards/margins": 6.113183498382568,
      "rewards/rejected": -7.697073459625244,
      "step": 680
    },
    {
      "epoch": 1.4607999999999999,
      "grad_norm": 3.9224820137023926,
      "learning_rate": 2.73572937902922e-06,
      "logits/chosen": -1.756115198135376,
      "logits/rejected": -1.6281507015228271,
      "logps/chosen": -256.65557861328125,
      "logps/rejected": -374.08941650390625,
      "loss": 0.1303,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.3831443786621094,
      "rewards/margins": 6.403082847595215,
      "rewards/rejected": -7.786227226257324,
      "step": 685
    },
    {
      "epoch": 1.4714666666666667,
      "grad_norm": 1.6743059158325195,
      "learning_rate": 2.7070523980693362e-06,
      "logits/chosen": -1.5727207660675049,
      "logits/rejected": -1.5360314846038818,
      "logps/chosen": -290.92431640625,
      "logps/rejected": -380.76177978515625,
      "loss": 0.1178,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.7124334573745728,
      "rewards/margins": 6.1151628494262695,
      "rewards/rejected": -7.827596187591553,
      "step": 690
    },
    {
      "epoch": 1.4821333333333333,
      "grad_norm": 5.704246520996094,
      "learning_rate": 2.678347957998081e-06,
      "logits/chosen": -1.8342517614364624,
      "logits/rejected": -1.7212531566619873,
      "logps/chosen": -252.77523803710938,
      "logps/rejected": -354.2868957519531,
      "loss": 0.1209,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.7579208612442017,
      "rewards/margins": 6.413499355316162,
      "rewards/rejected": -8.171420097351074,
      "step": 695
    },
    {
      "epoch": 1.4928,
      "grad_norm": 1.1863569021224976,
      "learning_rate": 2.649619865573666e-06,
      "logits/chosen": -1.764653205871582,
      "logits/rejected": -1.6264612674713135,
      "logps/chosen": -278.6958312988281,
      "logps/rejected": -401.44366455078125,
      "loss": 0.0682,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.005868434906006,
      "rewards/margins": 7.329827785491943,
      "rewards/rejected": -9.33569622039795,
      "step": 700
    },
    {
      "epoch": 1.5034666666666667,
      "grad_norm": 5.131344795227051,
      "learning_rate": 2.620871930691058e-06,
      "logits/chosen": -1.8136281967163086,
      "logits/rejected": -1.7170677185058594,
      "logps/chosen": -265.4339294433594,
      "logps/rejected": -364.80029296875,
      "loss": 0.1557,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.9572274684906006,
      "rewards/margins": 6.483831882476807,
      "rewards/rejected": -8.441060066223145,
      "step": 705
    },
    {
      "epoch": 1.5141333333333333,
      "grad_norm": 4.181100845336914,
      "learning_rate": 2.5921079658767152e-06,
      "logits/chosen": -1.7611749172210693,
      "logits/rejected": -1.6329116821289062,
      "logps/chosen": -267.61041259765625,
      "logps/rejected": -355.01861572265625,
      "loss": 0.1373,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.8553298711776733,
      "rewards/margins": 6.1329426765441895,
      "rewards/rejected": -7.988273620605469,
      "step": 710
    },
    {
      "epoch": 1.5248,
      "grad_norm": 2.085317373275757,
      "learning_rate": 2.56333178578297e-06,
      "logits/chosen": -1.8043543100357056,
      "logits/rejected": -1.6410627365112305,
      "logps/chosen": -242.35958862304688,
      "logps/rejected": -369.40277099609375,
      "loss": 0.0793,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.4961168766021729,
      "rewards/margins": 7.451325416564941,
      "rewards/rejected": -8.947443008422852,
      "step": 715
    },
    {
      "epoch": 1.5354666666666668,
      "grad_norm": 1.7874598503112793,
      "learning_rate": 2.534547206682136e-06,
      "logits/chosen": -1.819313645362854,
      "logits/rejected": -1.7426176071166992,
      "logps/chosen": -257.6264343261719,
      "logps/rejected": -392.6166076660156,
      "loss": 0.0841,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.016232967376709,
      "rewards/margins": 7.167724609375,
      "rewards/rejected": -9.183959007263184,
      "step": 720
    },
    {
      "epoch": 1.5461333333333334,
      "grad_norm": 2.4844038486480713,
      "learning_rate": 2.5057580459603947e-06,
      "logits/chosen": -1.6962602138519287,
      "logits/rejected": -1.6825191974639893,
      "logps/chosen": -258.6492004394531,
      "logps/rejected": -401.8869934082031,
      "loss": 0.093,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.1991543769836426,
      "rewards/margins": 7.357363700866699,
      "rewards/rejected": -9.556517601013184,
      "step": 725
    },
    {
      "epoch": 1.5568,
      "grad_norm": 4.203859806060791,
      "learning_rate": 2.4769681216115384e-06,
      "logits/chosen": -1.8400808572769165,
      "logits/rejected": -1.7029167413711548,
      "logps/chosen": -242.50137329101562,
      "logps/rejected": -361.71673583984375,
      "loss": 0.1276,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.6351560354232788,
      "rewards/margins": 6.772261619567871,
      "rewards/rejected": -8.407417297363281,
      "step": 730
    },
    {
      "epoch": 1.5674666666666668,
      "grad_norm": 6.503302097320557,
      "learning_rate": 2.448181251730632e-06,
      "logits/chosen": -1.7422183752059937,
      "logits/rejected": -1.6272938251495361,
      "logps/chosen": -256.3648986816406,
      "logps/rejected": -386.4632873535156,
      "loss": 0.1335,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.1021742820739746,
      "rewards/margins": 6.539918422698975,
      "rewards/rejected": -8.64209270477295,
      "step": 735
    },
    {
      "epoch": 1.5781333333333334,
      "grad_norm": 1.11476469039917,
      "learning_rate": 2.419401254007658e-06,
      "logits/chosen": -1.715961217880249,
      "logits/rejected": -1.55876624584198,
      "logps/chosen": -257.96417236328125,
      "logps/rejected": -365.32354736328125,
      "loss": 0.0886,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.7964681386947632,
      "rewards/margins": 7.368740081787109,
      "rewards/rejected": -9.16520881652832,
      "step": 740
    },
    {
      "epoch": 1.5888,
      "grad_norm": 3.136431932449341,
      "learning_rate": 2.390631945221221e-06,
      "logits/chosen": -1.7085437774658203,
      "logits/rejected": -1.5970802307128906,
      "logps/chosen": -270.44158935546875,
      "logps/rejected": -397.62506103515625,
      "loss": 0.0853,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.0968310832977295,
      "rewards/margins": 7.725828647613525,
      "rewards/rejected": -9.82265853881836,
      "step": 745
    },
    {
      "epoch": 1.5994666666666668,
      "grad_norm": 5.395340442657471,
      "learning_rate": 2.361877140732366e-06,
      "logits/chosen": -1.708765983581543,
      "logits/rejected": -1.634315848350525,
      "logps/chosen": -262.7024841308594,
      "logps/rejected": -376.64984130859375,
      "loss": 0.0944,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.1605114936828613,
      "rewards/margins": 7.159433841705322,
      "rewards/rejected": -9.319944381713867,
      "step": 750
    },
    {
      "epoch": 1.6101333333333332,
      "grad_norm": 2.4793126583099365,
      "learning_rate": 2.333140653978592e-06,
      "logits/chosen": -1.7588707208633423,
      "logits/rejected": -1.6335443258285522,
      "logps/chosen": -247.37265014648438,
      "logps/rejected": -362.1492004394531,
      "loss": 0.1044,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.7570842504501343,
      "rewards/margins": 7.583054542541504,
      "rewards/rejected": -9.340139389038086,
      "step": 755
    },
    {
      "epoch": 1.6208,
      "grad_norm": 2.858823537826538,
      "learning_rate": 2.3044262959681156e-06,
      "logits/chosen": -1.648014783859253,
      "logits/rejected": -1.6031944751739502,
      "logps/chosen": -252.2265625,
      "logps/rejected": -396.5431823730469,
      "loss": 0.0798,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.9052528142929077,
      "rewards/margins": 7.492622375488281,
      "rewards/rejected": -9.397875785827637,
      "step": 760
    },
    {
      "epoch": 1.6314666666666666,
      "grad_norm": 0.9314205646514893,
      "learning_rate": 2.2757378747744575e-06,
      "logits/chosen": -1.7783981561660767,
      "logits/rejected": -1.6433979272842407,
      "logps/chosen": -264.9280700683594,
      "logps/rejected": -397.2356872558594,
      "loss": 0.0802,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.034813404083252,
      "rewards/margins": 7.162725925445557,
      "rewards/rejected": -9.197538375854492,
      "step": 765
    },
    {
      "epoch": 1.6421333333333332,
      "grad_norm": 1.140587329864502,
      "learning_rate": 2.24707919503142e-06,
      "logits/chosen": -1.7422771453857422,
      "logits/rejected": -1.6701154708862305,
      "logps/chosen": -259.47601318359375,
      "logps/rejected": -380.9583435058594,
      "loss": 0.0962,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.1725351810455322,
      "rewards/margins": 7.451235771179199,
      "rewards/rejected": -9.623769760131836,
      "step": 770
    },
    {
      "epoch": 1.6528,
      "grad_norm": 4.398436069488525,
      "learning_rate": 2.2184540574285213e-06,
      "logits/chosen": -1.7078425884246826,
      "logits/rejected": -1.6657629013061523,
      "logps/chosen": -278.24066162109375,
      "logps/rejected": -395.03900146484375,
      "loss": 0.1356,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.2757375240325928,
      "rewards/margins": 7.001380920410156,
      "rewards/rejected": -9.277118682861328,
      "step": 775
    },
    {
      "epoch": 1.6634666666666666,
      "grad_norm": 0.4673113226890564,
      "learning_rate": 2.1898662582069486e-06,
      "logits/chosen": -1.8044401407241821,
      "logits/rejected": -1.6835638284683228,
      "logps/chosen": -300.3841857910156,
      "logps/rejected": -386.4563903808594,
      "loss": 0.0949,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.046159505844116,
      "rewards/margins": 7.99750280380249,
      "rewards/rejected": -10.04366397857666,
      "step": 780
    },
    {
      "epoch": 1.6741333333333333,
      "grad_norm": 6.07424783706665,
      "learning_rate": 2.1613195886561044e-06,
      "logits/chosen": -1.7845760583877563,
      "logits/rejected": -1.6212952136993408,
      "logps/chosen": -241.0775146484375,
      "logps/rejected": -365.91839599609375,
      "loss": 0.1304,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.1666126251220703,
      "rewards/margins": 7.4346208572387695,
      "rewards/rejected": -9.601232528686523,
      "step": 785
    },
    {
      "epoch": 1.6848,
      "grad_norm": 3.622908353805542,
      "learning_rate": 2.132817834610811e-06,
      "logits/chosen": -1.7454599142074585,
      "logits/rejected": -1.5491307973861694,
      "logps/chosen": -267.7861328125,
      "logps/rejected": -377.89483642578125,
      "loss": 0.0918,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.376371145248413,
      "rewards/margins": 7.164327144622803,
      "rewards/rejected": -9.540698051452637,
      "step": 790
    },
    {
      "epoch": 1.6954666666666667,
      "grad_norm": 2.879439115524292,
      "learning_rate": 2.1043647759492338e-06,
      "logits/chosen": -1.6238399744033813,
      "logits/rejected": -1.6405404806137085,
      "logps/chosen": -264.1553649902344,
      "logps/rejected": -392.42059326171875,
      "loss": 0.1461,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.0837295055389404,
      "rewards/margins": 7.293753147125244,
      "rewards/rejected": -9.377483367919922,
      "step": 795
    },
    {
      "epoch": 1.7061333333333333,
      "grad_norm": 5.847136497497559,
      "learning_rate": 2.075964186091599e-06,
      "logits/chosen": -1.7471511363983154,
      "logits/rejected": -1.696508765220642,
      "logps/chosen": -247.0438690185547,
      "logps/rejected": -388.80731201171875,
      "loss": 0.0945,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.3469150066375732,
      "rewards/margins": 7.962810516357422,
      "rewards/rejected": -10.309725761413574,
      "step": 800
    },
    {
      "epoch": 1.7168,
      "grad_norm": 3.7853331565856934,
      "learning_rate": 2.047619831499765e-06,
      "logits/chosen": -1.6483488082885742,
      "logits/rejected": -1.5347471237182617,
      "logps/chosen": -260.1374816894531,
      "logps/rejected": -388.5803527832031,
      "loss": 0.1162,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.084120035171509,
      "rewards/margins": 7.762931823730469,
      "rewards/rejected": -9.847051620483398,
      "step": 805
    },
    {
      "epoch": 1.7274666666666667,
      "grad_norm": 0.991887629032135,
      "learning_rate": 2.0193354711777155e-06,
      "logits/chosen": -1.8399814367294312,
      "logits/rejected": -1.6649506092071533,
      "logps/chosen": -283.5755615234375,
      "logps/rejected": -396.29107666015625,
      "loss": 0.0995,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.306251049041748,
      "rewards/margins": 7.420642852783203,
      "rewards/rejected": -9.726892471313477,
      "step": 810
    },
    {
      "epoch": 1.7381333333333333,
      "grad_norm": 2.601606845855713,
      "learning_rate": 1.991114856173048e-06,
      "logits/chosen": -1.760546326637268,
      "logits/rejected": -1.6287925243377686,
      "logps/chosen": -257.34234619140625,
      "logps/rejected": -399.37896728515625,
      "loss": 0.0657,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.36808705329895,
      "rewards/margins": 8.1522855758667,
      "rewards/rejected": -10.520373344421387,
      "step": 815
    },
    {
      "epoch": 1.7488000000000001,
      "grad_norm": 3.6920084953308105,
      "learning_rate": 1.9629617290795115e-06,
      "logits/chosen": -1.757427453994751,
      "logits/rejected": -1.6638133525848389,
      "logps/chosen": -256.29473876953125,
      "logps/rejected": -385.53509521484375,
      "loss": 0.1288,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.197131395339966,
      "rewards/margins": 7.40597677230835,
      "rewards/rejected": -9.603108406066895,
      "step": 820
    },
    {
      "epoch": 1.7594666666666665,
      "grad_norm": 1.5328242778778076,
      "learning_rate": 1.934879823540663e-06,
      "logits/chosen": -1.787667989730835,
      "logits/rejected": -1.6369273662567139,
      "logps/chosen": -242.244873046875,
      "logps/rejected": -397.090087890625,
      "loss": 0.0861,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.16058087348938,
      "rewards/margins": 7.83211612701416,
      "rewards/rejected": -9.992696762084961,
      "step": 825
    },
    {
      "epoch": 1.7701333333333333,
      "grad_norm": 2.638007164001465,
      "learning_rate": 1.9068728637547195e-06,
      "logits/chosen": -1.7666289806365967,
      "logits/rejected": -1.7008821964263916,
      "logps/chosen": -272.5953674316406,
      "logps/rejected": -394.2113342285156,
      "loss": 0.0777,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.033980369567871,
      "rewards/margins": 7.657546043395996,
      "rewards/rejected": -9.691526412963867,
      "step": 830
    },
    {
      "epoch": 1.7808000000000002,
      "grad_norm": 4.389973163604736,
      "learning_rate": 1.878944563980658e-06,
      "logits/chosen": -1.7376337051391602,
      "logits/rejected": -1.6045913696289062,
      "logps/chosen": -255.71963500976562,
      "logps/rejected": -367.043212890625,
      "loss": 0.1418,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.31443452835083,
      "rewards/margins": 7.155198574066162,
      "rewards/rejected": -9.469633102416992,
      "step": 835
    },
    {
      "epoch": 1.7914666666666665,
      "grad_norm": 2.3055145740509033,
      "learning_rate": 1.85109862804563e-06,
      "logits/chosen": -1.7284475564956665,
      "logits/rejected": -1.675169587135315,
      "logps/chosen": -266.1928405761719,
      "logps/rejected": -362.9505920410156,
      "loss": 0.1054,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.2873363494873047,
      "rewards/margins": 6.60150671005249,
      "rewards/rejected": -8.888843536376953,
      "step": 840
    },
    {
      "epoch": 1.8021333333333334,
      "grad_norm": 3.743189811706543,
      "learning_rate": 1.823338748853766e-06,
      "logits/chosen": -1.7340991497039795,
      "logits/rejected": -1.6459306478500366,
      "logps/chosen": -271.4342346191406,
      "logps/rejected": -397.19488525390625,
      "loss": 0.1065,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.5071041584014893,
      "rewards/margins": 7.8493547439575195,
      "rewards/rejected": -10.35645866394043,
      "step": 845
    },
    {
      "epoch": 1.8128,
      "grad_norm": 5.754213333129883,
      "learning_rate": 1.7956686078964257e-06,
      "logits/chosen": -1.7911723852157593,
      "logits/rejected": -1.6511707305908203,
      "logps/chosen": -252.0098114013672,
      "logps/rejected": -386.8243408203125,
      "loss": 0.1497,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.3370299339294434,
      "rewards/margins": 7.750609397888184,
      "rewards/rejected": -10.087640762329102,
      "step": 850
    },
    {
      "epoch": 1.8234666666666666,
      "grad_norm": 8.280444145202637,
      "learning_rate": 1.7680918747639608e-06,
      "logits/chosen": -1.743274450302124,
      "logits/rejected": -1.6286823749542236,
      "logps/chosen": -269.9051513671875,
      "logps/rejected": -400.05938720703125,
      "loss": 0.1064,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.5208077430725098,
      "rewards/margins": 7.579964637756348,
      "rewards/rejected": -10.1007719039917,
      "step": 855
    },
    {
      "epoch": 1.8341333333333334,
      "grad_norm": 1.875370740890503,
      "learning_rate": 1.740612206659057e-06,
      "logits/chosen": -1.7147190570831299,
      "logits/rejected": -1.689009428024292,
      "logps/chosen": -254.6274871826172,
      "logps/rejected": -388.3784484863281,
      "loss": 0.1093,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.9739549160003662,
      "rewards/margins": 7.456284523010254,
      "rewards/rejected": -9.4302396774292,
      "step": 860
    },
    {
      "epoch": 1.8448,
      "grad_norm": 5.320019721984863,
      "learning_rate": 1.7132332479117204e-06,
      "logits/chosen": -1.728094458580017,
      "logits/rejected": -1.5693343877792358,
      "logps/chosen": -263.1184387207031,
      "logps/rejected": -371.22613525390625,
      "loss": 0.087,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.0235514640808105,
      "rewards/margins": 7.408260345458984,
      "rewards/rejected": -9.43181037902832,
      "step": 865
    },
    {
      "epoch": 1.8554666666666666,
      "grad_norm": 4.183524131774902,
      "learning_rate": 1.6859586294959666e-06,
      "logits/chosen": -1.8199039697647095,
      "logits/rejected": -1.6934419870376587,
      "logps/chosen": -253.3587646484375,
      "logps/rejected": -392.0380859375,
      "loss": 0.078,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.7781072854995728,
      "rewards/margins": 7.864907741546631,
      "rewards/rejected": -9.64301586151123,
      "step": 870
    },
    {
      "epoch": 1.8661333333333334,
      "grad_norm": 1.000540852546692,
      "learning_rate": 1.6587919685482884e-06,
      "logits/chosen": -1.7975037097930908,
      "logits/rejected": -1.6539417505264282,
      "logps/chosen": -246.7928466796875,
      "logps/rejected": -366.54071044921875,
      "loss": 0.1199,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.086042642593384,
      "rewards/margins": 7.348335266113281,
      "rewards/rejected": -9.434378623962402,
      "step": 875
    },
    {
      "epoch": 1.8768,
      "grad_norm": 2.6079916954040527,
      "learning_rate": 1.6317368678879497e-06,
      "logits/chosen": -1.6367311477661133,
      "logits/rejected": -1.5827094316482544,
      "logps/chosen": -256.6966247558594,
      "logps/rejected": -376.14373779296875,
      "loss": 0.0897,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.9041290283203125,
      "rewards/margins": 7.1437859535217285,
      "rewards/rejected": -9.047914505004883,
      "step": 880
    },
    {
      "epoch": 1.8874666666666666,
      "grad_norm": 3.0209317207336426,
      "learning_rate": 1.6047969155391856e-06,
      "logits/chosen": -1.7882978916168213,
      "logits/rejected": -1.6401067972183228,
      "logps/chosen": -260.91717529296875,
      "logps/rejected": -385.9642028808594,
      "loss": 0.0945,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.86648428440094,
      "rewards/margins": 8.293737411499023,
      "rewards/rejected": -10.160222053527832,
      "step": 885
    },
    {
      "epoch": 1.8981333333333335,
      "grad_norm": 0.41572609543800354,
      "learning_rate": 1.5779756842553617e-06,
      "logits/chosen": -1.7787930965423584,
      "logits/rejected": -1.6832020282745361,
      "logps/chosen": -259.8187561035156,
      "logps/rejected": -407.3184509277344,
      "loss": 0.0504,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.7974926233291626,
      "rewards/margins": 8.537004470825195,
      "rewards/rejected": -10.334497451782227,
      "step": 890
    },
    {
      "epoch": 1.9088,
      "grad_norm": 4.283809661865234,
      "learning_rate": 1.551276731045157e-06,
      "logits/chosen": -1.7304534912109375,
      "logits/rejected": -1.6511510610580444,
      "logps/chosen": -288.17987060546875,
      "logps/rejected": -412.12396240234375,
      "loss": 0.1044,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.9651966094970703,
      "rewards/margins": 7.937309265136719,
      "rewards/rejected": -9.902505874633789,
      "step": 895
    },
    {
      "epoch": 1.9194666666666667,
      "grad_norm": 1.008398413658142,
      "learning_rate": 1.5247035967008369e-06,
      "logits/chosen": -1.5831853151321411,
      "logits/rejected": -1.5248817205429077,
      "logps/chosen": -259.24981689453125,
      "logps/rejected": -378.78558349609375,
      "loss": 0.0808,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.084197521209717,
      "rewards/margins": 7.348579406738281,
      "rewards/rejected": -9.432775497436523,
      "step": 900
    },
    {
      "epoch": 1.9301333333333335,
      "grad_norm": 3.965376377105713,
      "learning_rate": 1.498259805328678e-06,
      "logits/chosen": -1.72213613986969,
      "logits/rejected": -1.5788049697875977,
      "logps/chosen": -264.9279479980469,
      "logps/rejected": -370.3101806640625,
      "loss": 0.0861,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.3263232707977295,
      "rewards/margins": 7.561483860015869,
      "rewards/rejected": -9.887807846069336,
      "step": 905
    },
    {
      "epoch": 1.9407999999999999,
      "grad_norm": 4.654145240783691,
      "learning_rate": 1.471948863881601e-06,
      "logits/chosen": -1.7574799060821533,
      "logits/rejected": -1.6463959217071533,
      "logps/chosen": -275.3285217285156,
      "logps/rejected": -394.51324462890625,
      "loss": 0.0984,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.8904142379760742,
      "rewards/margins": 7.419065952301025,
      "rewards/rejected": -9.309479713439941,
      "step": 910
    },
    {
      "epoch": 1.9514666666666667,
      "grad_norm": 2.2931063175201416,
      "learning_rate": 1.4457742616940864e-06,
      "logits/chosen": -1.6442797183990479,
      "logits/rejected": -1.578762412071228,
      "logps/chosen": -277.2344665527344,
      "logps/rejected": -399.81964111328125,
      "loss": 0.1103,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.1587257385253906,
      "rewards/margins": 7.75518798828125,
      "rewards/rejected": -9.913914680480957,
      "step": 915
    },
    {
      "epoch": 1.9621333333333333,
      "grad_norm": 3.6048293113708496,
      "learning_rate": 1.4197394700194165e-06,
      "logits/chosen": -1.7335107326507568,
      "logits/rejected": -1.616270661354065,
      "logps/chosen": -262.0077819824219,
      "logps/rejected": -375.99005126953125,
      "loss": 0.106,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.032534599304199,
      "rewards/margins": 7.5883941650390625,
      "rewards/rejected": -9.620927810668945,
      "step": 920
    },
    {
      "epoch": 1.9727999999999999,
      "grad_norm": 2.6519317626953125,
      "learning_rate": 1.3938479415693244e-06,
      "logits/chosen": -1.741246223449707,
      "logits/rejected": -1.6494983434677124,
      "logps/chosen": -271.10101318359375,
      "logps/rejected": -381.05450439453125,
      "loss": 0.0948,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.9012314081192017,
      "rewards/margins": 7.431715488433838,
      "rewards/rejected": -9.33294677734375,
      "step": 925
    },
    {
      "epoch": 1.9834666666666667,
      "grad_norm": 3.518383026123047,
      "learning_rate": 1.368103110056096e-06,
      "logits/chosen": -1.7782844305038452,
      "logits/rejected": -1.6442190408706665,
      "logps/chosen": -266.40045166015625,
      "logps/rejected": -369.31756591796875,
      "loss": 0.1255,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.061063289642334,
      "rewards/margins": 7.457045078277588,
      "rewards/rejected": -9.518108367919922,
      "step": 930
    },
    {
      "epoch": 1.9941333333333333,
      "grad_norm": 5.0803046226501465,
      "learning_rate": 1.3425083897371983e-06,
      "logits/chosen": -1.5946252346038818,
      "logits/rejected": -1.5125935077667236,
      "logps/chosen": -254.85342407226562,
      "logps/rejected": -365.8056945800781,
      "loss": 0.0898,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.6178001165390015,
      "rewards/margins": 7.518826961517334,
      "rewards/rejected": -9.136625289916992,
      "step": 935
    },
    {
      "epoch": 2.0042666666666666,
      "grad_norm": 2.6192548274993896,
      "learning_rate": 1.317067174962478e-06,
      "logits/chosen": -1.7343151569366455,
      "logits/rejected": -1.6101001501083374,
      "logps/chosen": -261.07666015625,
      "logps/rejected": -379.0164794921875,
      "loss": 0.0825,
      "rewards/accuracies": 0.9868420958518982,
      "rewards/chosen": -1.6997953653335571,
      "rewards/margins": 7.317230224609375,
      "rewards/rejected": -9.0170259475708,
      "step": 940
    },
    {
      "epoch": 2.0149333333333335,
      "grad_norm": 1.5539934635162354,
      "learning_rate": 1.291782839724009e-06,
      "logits/chosen": -1.718090295791626,
      "logits/rejected": -1.6289358139038086,
      "logps/chosen": -268.7707824707031,
      "logps/rejected": -378.489990234375,
      "loss": 0.0841,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.6988178491592407,
      "rewards/margins": 7.449611663818359,
      "rewards/rejected": -9.148429870605469,
      "step": 945
    },
    {
      "epoch": 2.0256,
      "grad_norm": 2.7771873474121094,
      "learning_rate": 1.2666587372086352e-06,
      "logits/chosen": -1.6812318563461304,
      "logits/rejected": -1.5723031759262085,
      "logps/chosen": -254.5369415283203,
      "logps/rejected": -378.9630126953125,
      "loss": 0.0768,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.7308629751205444,
      "rewards/margins": 7.691521644592285,
      "rewards/rejected": -9.422384262084961,
      "step": 950
    },
    {
      "epoch": 2.0362666666666667,
      "grad_norm": 1.9423669576644897,
      "learning_rate": 1.241698199353275e-06,
      "logits/chosen": -1.7462921142578125,
      "logits/rejected": -1.6803960800170898,
      "logps/chosen": -266.7231140136719,
      "logps/rejected": -418.3180236816406,
      "loss": 0.1006,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.0733628273010254,
      "rewards/margins": 7.935437202453613,
      "rewards/rejected": -10.008800506591797,
      "step": 955
    },
    {
      "epoch": 2.0469333333333335,
      "grad_norm": 1.0960689783096313,
      "learning_rate": 1.2169045364030404e-06,
      "logits/chosen": -1.7646312713623047,
      "logits/rejected": -1.6422895193099976,
      "logps/chosen": -249.3695526123047,
      "logps/rejected": -383.7786560058594,
      "loss": 0.0714,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.136165142059326,
      "rewards/margins": 7.901527404785156,
      "rewards/rejected": -10.037691116333008,
      "step": 960
    },
    {
      "epoch": 2.0576,
      "grad_norm": 1.9646838903427124,
      "learning_rate": 1.1922810364722402e-06,
      "logits/chosen": -1.744693398475647,
      "logits/rejected": -1.5728899240493774,
      "logps/chosen": -273.8901062011719,
      "logps/rejected": -388.103515625,
      "loss": 0.0702,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.5939252376556396,
      "rewards/margins": 7.756191253662109,
      "rewards/rejected": -9.350118637084961,
      "step": 965
    },
    {
      "epoch": 2.0682666666666667,
      "grad_norm": 2.2951996326446533,
      "learning_rate": 1.1678309651083055e-06,
      "logits/chosen": -1.7867664098739624,
      "logits/rejected": -1.6740038394927979,
      "logps/chosen": -274.0558166503906,
      "logps/rejected": -405.5430908203125,
      "loss": 0.0448,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.1959640979766846,
      "rewards/margins": 8.143606185913086,
      "rewards/rejected": -10.339570045471191,
      "step": 970
    },
    {
      "epoch": 2.0789333333333335,
      "grad_norm": 1.492812156677246,
      "learning_rate": 1.1435575648587224e-06,
      "logits/chosen": -1.6263166666030884,
      "logits/rejected": -1.5808578729629517,
      "logps/chosen": -273.41229248046875,
      "logps/rejected": -402.88348388671875,
      "loss": 0.0522,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.041072368621826,
      "rewards/margins": 8.062564849853516,
      "rewards/rejected": -10.1036376953125,
      "step": 975
    },
    {
      "epoch": 2.0896,
      "grad_norm": 2.0464603900909424,
      "learning_rate": 1.1194640548410039e-06,
      "logits/chosen": -1.6150739192962646,
      "logits/rejected": -1.6028960943222046,
      "logps/chosen": -255.7452850341797,
      "logps/rejected": -357.42822265625,
      "loss": 0.0904,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.7968528270721436,
      "rewards/margins": 7.02915096282959,
      "rewards/rejected": -8.826005935668945,
      "step": 980
    },
    {
      "epoch": 2.1002666666666667,
      "grad_norm": 3.042483329772949,
      "learning_rate": 1.095553630315774e-06,
      "logits/chosen": -1.7164863348007202,
      "logits/rejected": -1.6247742176055908,
      "logps/chosen": -276.2933654785156,
      "logps/rejected": -400.1717529296875,
      "loss": 0.0806,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.226152181625366,
      "rewards/margins": 7.620535850524902,
      "rewards/rejected": -9.846689224243164,
      "step": 985
    },
    {
      "epoch": 2.1109333333333336,
      "grad_norm": 2.414696216583252,
      "learning_rate": 1.0718294622630188e-06,
      "logits/chosen": -1.6823770999908447,
      "logits/rejected": -1.5639375448226929,
      "logps/chosen": -289.0705261230469,
      "logps/rejected": -421.67425537109375,
      "loss": 0.0784,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.4110684394836426,
      "rewards/margins": 8.174033164978027,
      "rewards/rejected": -10.585102081298828,
      "step": 990
    },
    {
      "epoch": 2.1216,
      "grad_norm": 2.291530132293701,
      "learning_rate": 1.0482946969615495e-06,
      "logits/chosen": -1.7374327182769775,
      "logits/rejected": -1.6161400079727173,
      "logps/chosen": -256.18450927734375,
      "logps/rejected": -393.5759582519531,
      "loss": 0.0668,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.667482852935791,
      "rewards/margins": 8.163459777832031,
      "rewards/rejected": -10.830942153930664,
      "step": 995
    },
    {
      "epoch": 2.1322666666666668,
      "grad_norm": 3.7659263610839844,
      "learning_rate": 1.0249524555717461e-06,
      "logits/chosen": -1.697607398033142,
      "logits/rejected": -1.5797332525253296,
      "logps/chosen": -263.2526550292969,
      "logps/rejected": -401.4608154296875,
      "loss": 0.0987,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.1764533519744873,
      "rewards/margins": 7.859429836273193,
      "rewards/rejected": -10.035882949829102,
      "step": 1000
    },
    {
      "epoch": 2.142933333333333,
      "grad_norm": 2.7956254482269287,
      "learning_rate": 1.0018058337216327e-06,
      "logits/chosen": -1.6498619318008423,
      "logits/rejected": -1.6099936962127686,
      "logps/chosen": -267.5693359375,
      "logps/rejected": -401.8230895996094,
      "loss": 0.0769,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.4292540550231934,
      "rewards/margins": 8.179109573364258,
      "rewards/rejected": -10.608363151550293,
      "step": 1005
    },
    {
      "epoch": 2.1536,
      "grad_norm": 1.6683810949325562,
      "learning_rate": 9.788579010963387e-07,
      "logits/chosen": -1.7626358270645142,
      "logits/rejected": -1.6691913604736328,
      "logps/chosen": -294.8406066894531,
      "logps/rejected": -387.442138671875,
      "loss": 0.0647,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.454096555709839,
      "rewards/margins": 7.879624843597412,
      "rewards/rejected": -10.333722114562988,
      "step": 1010
    },
    {
      "epoch": 2.164266666666667,
      "grad_norm": 2.0527291297912598,
      "learning_rate": 9.56111701030998e-07,
      "logits/chosen": -1.7989389896392822,
      "logits/rejected": -1.649007797241211,
      "logps/chosen": -257.8086242675781,
      "logps/rejected": -396.00946044921875,
      "loss": 0.0892,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.2390990257263184,
      "rewards/margins": 8.126911163330078,
      "rewards/rejected": -10.366010665893555,
      "step": 1015
    },
    {
      "epoch": 2.174933333333333,
      "grad_norm": 1.3809715509414673,
      "learning_rate": 9.335702501071459e-07,
      "logits/chosen": -1.727193832397461,
      "logits/rejected": -1.6351912021636963,
      "logps/chosen": -266.216796875,
      "logps/rejected": -408.1944885253906,
      "loss": 0.0571,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.289905548095703,
      "rewards/margins": 8.413689613342285,
      "rewards/rejected": -10.703596115112305,
      "step": 1020
    },
    {
      "epoch": 2.1856,
      "grad_norm": 2.758436441421509,
      "learning_rate": 9.112365377526636e-07,
      "logits/chosen": -1.7538105249404907,
      "logits/rejected": -1.6839125156402588,
      "logps/chosen": -251.68896484375,
      "logps/rejected": -405.6483459472656,
      "loss": 0.0612,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.4383013248443604,
      "rewards/margins": 8.36324691772461,
      "rewards/rejected": -10.801547050476074,
      "step": 1025
    },
    {
      "epoch": 2.196266666666667,
      "grad_norm": 1.1113784313201904,
      "learning_rate": 8.891135258453195e-07,
      "logits/chosen": -1.679633378982544,
      "logits/rejected": -1.6179072856903076,
      "logps/chosen": -247.2431182861328,
      "logps/rejected": -389.5404968261719,
      "loss": 0.0684,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.3117198944091797,
      "rewards/margins": 7.995427131652832,
      "rewards/rejected": -10.307147026062012,
      "step": 1030
    },
    {
      "epoch": 2.206933333333333,
      "grad_norm": 3.6547203063964844,
      "learning_rate": 8.672041483199709e-07,
      "logits/chosen": -1.726601243019104,
      "logits/rejected": -1.6291310787200928,
      "logps/chosen": -255.78787231445312,
      "logps/rejected": -379.4979553222656,
      "loss": 0.0652,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.340198040008545,
      "rewards/margins": 7.787562370300293,
      "rewards/rejected": -10.12775993347168,
      "step": 1035
    },
    {
      "epoch": 2.2176,
      "grad_norm": 2.5580599308013916,
      "learning_rate": 8.455113107794652e-07,
      "logits/chosen": -1.7268718481063843,
      "logits/rejected": -1.6617317199707031,
      "logps/chosen": -246.4077606201172,
      "logps/rejected": -419.98321533203125,
      "loss": 0.0552,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.661684036254883,
      "rewards/margins": 9.117815017700195,
      "rewards/rejected": -11.779499053955078,
      "step": 1040
    },
    {
      "epoch": 2.228266666666667,
      "grad_norm": 2.44766902923584,
      "learning_rate": 8.240378901093035e-07,
      "logits/chosen": -1.7534101009368896,
      "logits/rejected": -1.5951759815216064,
      "logps/chosen": -284.9955139160156,
      "logps/rejected": -385.4869384765625,
      "loss": 0.1098,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.5828628540039062,
      "rewards/margins": 7.782918453216553,
      "rewards/rejected": -10.365781784057617,
      "step": 1045
    },
    {
      "epoch": 2.238933333333333,
      "grad_norm": 3.560786724090576,
      "learning_rate": 8.027867340961113e-07,
      "logits/chosen": -1.662253975868225,
      "logits/rejected": -1.5555870532989502,
      "logps/chosen": -270.2218933105469,
      "logps/rejected": -380.3742370605469,
      "loss": 0.1164,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.591815710067749,
      "rewards/margins": 7.998503684997559,
      "rewards/rejected": -10.590320587158203,
      "step": 1050
    },
    {
      "epoch": 2.2496,
      "grad_norm": 2.0515780448913574,
      "learning_rate": 7.817606610499639e-07,
      "logits/chosen": -1.6859337091445923,
      "logits/rejected": -1.62407648563385,
      "logps/chosen": -270.041015625,
      "logps/rejected": -386.9949645996094,
      "loss": 0.0834,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.4862797260284424,
      "rewards/margins": 7.569838047027588,
      "rewards/rejected": -10.056117057800293,
      "step": 1055
    },
    {
      "epoch": 2.2602666666666664,
      "grad_norm": 3.088653564453125,
      "learning_rate": 7.609624594306278e-07,
      "logits/chosen": -1.7772037982940674,
      "logits/rejected": -1.6899620294570923,
      "logps/chosen": -271.0976257324219,
      "logps/rejected": -385.71441650390625,
      "loss": 0.0812,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.923471689224243,
      "rewards/margins": 7.496604919433594,
      "rewards/rejected": -10.420077323913574,
      "step": 1060
    },
    {
      "epoch": 2.2709333333333332,
      "grad_norm": 3.4833757877349854,
      "learning_rate": 7.403948874777556e-07,
      "logits/chosen": -1.7119638919830322,
      "logits/rejected": -1.6148226261138916,
      "logps/chosen": -261.0135192871094,
      "logps/rejected": -381.09100341796875,
      "loss": 0.1112,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.67107892036438,
      "rewards/margins": 7.778635501861572,
      "rewards/rejected": -10.449714660644531,
      "step": 1065
    },
    {
      "epoch": 2.2816,
      "grad_norm": 3.0664234161376953,
      "learning_rate": 7.20060672845091e-07,
      "logits/chosen": -1.6737985610961914,
      "logits/rejected": -1.6506000757217407,
      "logps/chosen": -269.85247802734375,
      "logps/rejected": -413.30511474609375,
      "loss": 0.0576,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.866260051727295,
      "rewards/margins": 8.69055461883545,
      "rewards/rejected": -11.556814193725586,
      "step": 1070
    },
    {
      "epoch": 2.2922666666666665,
      "grad_norm": 1.538775086402893,
      "learning_rate": 6.99962512238733e-07,
      "logits/chosen": -1.6387313604354858,
      "logits/rejected": -1.5764515399932861,
      "logps/chosen": -252.82766723632812,
      "logps/rejected": -385.333984375,
      "loss": 0.0883,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.387939691543579,
      "rewards/margins": 8.016633987426758,
      "rewards/rejected": -10.404573440551758,
      "step": 1075
    },
    {
      "epoch": 2.3029333333333333,
      "grad_norm": 1.7143809795379639,
      "learning_rate": 6.801030710594949e-07,
      "logits/chosen": -1.5965774059295654,
      "logits/rejected": -1.536005973815918,
      "logps/chosen": -261.13275146484375,
      "logps/rejected": -383.4078369140625,
      "loss": 0.0765,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.6068761348724365,
      "rewards/margins": 7.689870357513428,
      "rewards/rejected": -10.296747207641602,
      "step": 1080
    },
    {
      "epoch": 2.3136,
      "grad_norm": 4.500543117523193,
      "learning_rate": 6.604849830494264e-07,
      "logits/chosen": -1.7209281921386719,
      "logits/rejected": -1.6301357746124268,
      "logps/chosen": -252.2537384033203,
      "logps/rejected": -401.78277587890625,
      "loss": 0.0489,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.3812525272369385,
      "rewards/margins": 8.41871166229248,
      "rewards/rejected": -10.799964904785156,
      "step": 1085
    },
    {
      "epoch": 2.3242666666666665,
      "grad_norm": 1.3474994897842407,
      "learning_rate": 6.411108499425253e-07,
      "logits/chosen": -1.7181062698364258,
      "logits/rejected": -1.596534252166748,
      "logps/chosen": -276.0671081542969,
      "logps/rejected": -398.2869567871094,
      "loss": 0.0748,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.573090076446533,
      "rewards/margins": 7.86136531829834,
      "rewards/rejected": -10.434455871582031,
      "step": 1090
    },
    {
      "epoch": 2.3349333333333333,
      "grad_norm": 1.8286346197128296,
      "learning_rate": 6.219832411197013e-07,
      "logits/chosen": -1.6104246377944946,
      "logits/rejected": -1.5124198198318481,
      "logps/chosen": -272.6994934082031,
      "logps/rejected": -403.438232421875,
      "loss": 0.0775,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.20379638671875,
      "rewards/margins": 8.984777450561523,
      "rewards/rejected": -11.188573837280273,
      "step": 1095
    },
    {
      "epoch": 2.3456,
      "grad_norm": 0.6198655962944031,
      "learning_rate": 6.031046932680229e-07,
      "logits/chosen": -1.727940559387207,
      "logits/rejected": -1.5797127485275269,
      "logps/chosen": -262.6995849609375,
      "logps/rejected": -391.15765380859375,
      "loss": 0.0549,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.521888256072998,
      "rewards/margins": 8.586507797241211,
      "rewards/rejected": -11.108396530151367,
      "step": 1100
    },
    {
      "epoch": 2.3562666666666665,
      "grad_norm": 1.2429842948913574,
      "learning_rate": 5.844777100443091e-07,
      "logits/chosen": -1.6657642126083374,
      "logits/rejected": -1.6233059167861938,
      "logps/chosen": -265.207275390625,
      "logps/rejected": -429.60089111328125,
      "loss": 0.0806,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.748042583465576,
      "rewards/margins": 8.55555534362793,
      "rewards/rejected": -11.303597450256348,
      "step": 1105
    },
    {
      "epoch": 2.3669333333333333,
      "grad_norm": 2.5982189178466797,
      "learning_rate": 5.66104761743092e-07,
      "logits/chosen": -1.833526611328125,
      "logits/rejected": -1.7374675273895264,
      "logps/chosen": -254.8466339111328,
      "logps/rejected": -398.44610595703125,
      "loss": 0.0519,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.018444538116455,
      "rewards/margins": 8.912156105041504,
      "rewards/rejected": -11.930599212646484,
      "step": 1110
    },
    {
      "epoch": 2.3776,
      "grad_norm": 3.6712732315063477,
      "learning_rate": 5.479882849690102e-07,
      "logits/chosen": -1.6810152530670166,
      "logits/rejected": -1.617287278175354,
      "logps/chosen": -278.04498291015625,
      "logps/rejected": -404.9657287597656,
      "loss": 0.0683,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.8550310134887695,
      "rewards/margins": 7.767541408538818,
      "rewards/rejected": -10.62257194519043,
      "step": 1115
    },
    {
      "epoch": 2.3882666666666665,
      "grad_norm": 1.2086095809936523,
      "learning_rate": 5.301306823136687e-07,
      "logits/chosen": -1.6428139209747314,
      "logits/rejected": -1.569113850593567,
      "logps/chosen": -260.32574462890625,
      "logps/rejected": -399.7398986816406,
      "loss": 0.1033,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.7781503200531006,
      "rewards/margins": 8.46976089477539,
      "rewards/rejected": -11.247909545898438,
      "step": 1120
    },
    {
      "epoch": 2.3989333333333334,
      "grad_norm": 1.18802809715271,
      "learning_rate": 5.125343220370088e-07,
      "logits/chosen": -1.6931226253509521,
      "logits/rejected": -1.592712640762329,
      "logps/chosen": -253.77578735351562,
      "logps/rejected": -374.4710388183594,
      "loss": 0.0581,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.5571885108947754,
      "rewards/margins": 8.29760456085205,
      "rewards/rejected": -10.854792594909668,
      "step": 1125
    },
    {
      "epoch": 2.4096,
      "grad_norm": 3.1134982109069824,
      "learning_rate": 4.952015377532329e-07,
      "logits/chosen": -1.7127673625946045,
      "logits/rejected": -1.7162590026855469,
      "logps/chosen": -270.6441955566406,
      "logps/rejected": -401.02496337890625,
      "loss": 0.0903,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.0124757289886475,
      "rewards/margins": 8.173250198364258,
      "rewards/rejected": -11.185726165771484,
      "step": 1130
    },
    {
      "epoch": 2.4202666666666666,
      "grad_norm": 6.751755237579346,
      "learning_rate": 4.781346281213204e-07,
      "logits/chosen": -1.7349708080291748,
      "logits/rejected": -1.66085684299469,
      "logps/chosen": -260.246337890625,
      "logps/rejected": -419.47943115234375,
      "loss": 0.066,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.005652904510498,
      "rewards/margins": 9.477314949035645,
      "rewards/rejected": -12.482967376708984,
      "step": 1135
    },
    {
      "epoch": 2.4309333333333334,
      "grad_norm": 2.8330957889556885,
      "learning_rate": 4.613358565401818e-07,
      "logits/chosen": -1.615511178970337,
      "logits/rejected": -1.502151370048523,
      "logps/chosen": -264.5709533691406,
      "logps/rejected": -379.5435791015625,
      "loss": 0.0647,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.5337510108947754,
      "rewards/margins": 7.921186923980713,
      "rewards/rejected": -10.454936981201172,
      "step": 1140
    },
    {
      "epoch": 2.4416,
      "grad_norm": 2.5002682209014893,
      "learning_rate": 4.448074508484906e-07,
      "logits/chosen": -1.7348816394805908,
      "logits/rejected": -1.641600251197815,
      "logps/chosen": -261.57965087890625,
      "logps/rejected": -403.952880859375,
      "loss": 0.0556,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.5588488578796387,
      "rewards/margins": 8.548656463623047,
      "rewards/rejected": -11.107504844665527,
      "step": 1145
    },
    {
      "epoch": 2.4522666666666666,
      "grad_norm": 6.115179538726807,
      "learning_rate": 4.2855160302922777e-07,
      "logits/chosen": -1.697669267654419,
      "logits/rejected": -1.6198641061782837,
      "logps/chosen": -256.3226623535156,
      "logps/rejected": -405.70111083984375,
      "loss": 0.0852,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.8319969177246094,
      "rewards/margins": 8.845357894897461,
      "rewards/rejected": -11.67735481262207,
      "step": 1150
    },
    {
      "epoch": 2.4629333333333334,
      "grad_norm": 4.671123027801514,
      "learning_rate": 4.125704689189819e-07,
      "logits/chosen": -1.665469765663147,
      "logits/rejected": -1.5715079307556152,
      "logps/chosen": -243.5298309326172,
      "logps/rejected": -388.2669372558594,
      "loss": 0.0924,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.220625400543213,
      "rewards/margins": 8.359115600585938,
      "rewards/rejected": -10.579740524291992,
      "step": 1155
    },
    {
      "epoch": 2.4736000000000002,
      "grad_norm": 1.6122468709945679,
      "learning_rate": 3.9686616792204677e-07,
      "logits/chosen": -1.7475868463516235,
      "logits/rejected": -1.692379355430603,
      "logps/chosen": -283.0847473144531,
      "logps/rejected": -408.1714782714844,
      "loss": 0.0395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.709376573562622,
      "rewards/margins": 8.082962036132812,
      "rewards/rejected": -10.792339324951172,
      "step": 1160
    },
    {
      "epoch": 2.4842666666666666,
      "grad_norm": 2.0977368354797363,
      "learning_rate": 3.8144078272934676e-07,
      "logits/chosen": -1.654688835144043,
      "logits/rejected": -1.6329755783081055,
      "logps/chosen": -286.69622802734375,
      "logps/rejected": -413.87481689453125,
      "loss": 0.088,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.884509325027466,
      "rewards/margins": 8.137341499328613,
      "rewards/rejected": -11.021848678588867,
      "step": 1165
    },
    {
      "epoch": 2.4949333333333334,
      "grad_norm": 3.7184035778045654,
      "learning_rate": 3.6629635904222943e-07,
      "logits/chosen": -1.6443443298339844,
      "logits/rejected": -1.5297214984893799,
      "logps/chosen": -286.8924865722656,
      "logps/rejected": -398.5106506347656,
      "loss": 0.1043,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.0164103507995605,
      "rewards/margins": 7.627072334289551,
      "rewards/rejected": -10.64348316192627,
      "step": 1170
    },
    {
      "epoch": 2.5056000000000003,
      "grad_norm": 4.416862487792969,
      "learning_rate": 3.514349053011698e-07,
      "logits/chosen": -1.6677930355072021,
      "logits/rejected": -1.5294122695922852,
      "logps/chosen": -269.85345458984375,
      "logps/rejected": -371.2940979003906,
      "loss": 0.0723,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.5542593002319336,
      "rewards/margins": 7.903220176696777,
      "rewards/rejected": -10.457479476928711,
      "step": 1175
    },
    {
      "epoch": 2.5162666666666667,
      "grad_norm": 2.2231297492980957,
      "learning_rate": 3.3685839241941133e-07,
      "logits/chosen": -1.6161854267120361,
      "logits/rejected": -1.6047331094741821,
      "logps/chosen": -283.4258117675781,
      "logps/rejected": -387.2071228027344,
      "loss": 0.0765,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.8923115730285645,
      "rewards/margins": 8.108506202697754,
      "rewards/rejected": -11.000818252563477,
      "step": 1180
    },
    {
      "epoch": 2.5269333333333335,
      "grad_norm": 1.9496991634368896,
      "learning_rate": 3.2256875352158435e-07,
      "logits/chosen": -1.778193712234497,
      "logits/rejected": -1.6277202367782593,
      "logps/chosen": -268.64862060546875,
      "logps/rejected": -414.49481201171875,
      "loss": 0.0705,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.5748419761657715,
      "rewards/margins": 8.338712692260742,
      "rewards/rejected": -10.913556098937988,
      "step": 1185
    },
    {
      "epoch": 2.5376,
      "grad_norm": 1.1859385967254639,
      "learning_rate": 3.085678836873399e-07,
      "logits/chosen": -1.7131965160369873,
      "logits/rejected": -1.6504449844360352,
      "logps/chosen": -251.01370239257812,
      "logps/rejected": -401.9787292480469,
      "loss": 0.0762,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.776256799697876,
      "rewards/margins": 8.66688346862793,
      "rewards/rejected": -11.443140029907227,
      "step": 1190
    },
    {
      "epoch": 2.5482666666666667,
      "grad_norm": 1.0037485361099243,
      "learning_rate": 2.9485763970002106e-07,
      "logits/chosen": -1.6434129476547241,
      "logits/rejected": -1.5615208148956299,
      "logps/chosen": -273.0927734375,
      "logps/rejected": -408.0320739746094,
      "loss": 0.0603,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.6806273460388184,
      "rewards/margins": 8.583781242370605,
      "rewards/rejected": -11.264409065246582,
      "step": 1195
    },
    {
      "epoch": 2.558933333333333,
      "grad_norm": 2.8410496711730957,
      "learning_rate": 2.8143983980042174e-07,
      "logits/chosen": -1.6743905544281006,
      "logits/rejected": -1.6090198755264282,
      "logps/chosen": -269.591552734375,
      "logps/rejected": -399.52471923828125,
      "loss": 0.0858,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.9564836025238037,
      "rewards/margins": 7.9727783203125,
      "rewards/rejected": -10.929262161254883,
      "step": 1200
    },
    {
      "epoch": 2.5696,
      "grad_norm": 2.85349178314209,
      "learning_rate": 2.683162634456515e-07,
      "logits/chosen": -1.679058313369751,
      "logits/rejected": -1.529710292816162,
      "logps/chosen": -280.95941162109375,
      "logps/rejected": -372.44256591796875,
      "loss": 0.0859,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.4623801708221436,
      "rewards/margins": 7.8578782081604,
      "rewards/rejected": -10.320258140563965,
      "step": 1205
    },
    {
      "epoch": 2.5802666666666667,
      "grad_norm": 3.7565083503723145,
      "learning_rate": 2.5548865107314606e-07,
      "logits/chosen": -1.6767339706420898,
      "logits/rejected": -1.5683711767196655,
      "logps/chosen": -302.63616943359375,
      "logps/rejected": -418.7508850097656,
      "loss": 0.0827,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.8508572578430176,
      "rewards/margins": 8.236178398132324,
      "rewards/rejected": -11.0870361328125,
      "step": 1210
    },
    {
      "epoch": 2.590933333333333,
      "grad_norm": 3.984247922897339,
      "learning_rate": 2.429587038698511e-07,
      "logits/chosen": -1.7127422094345093,
      "logits/rejected": -1.6515560150146484,
      "logps/chosen": -257.34368896484375,
      "logps/rejected": -387.74627685546875,
      "loss": 0.0789,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.859555959701538,
      "rewards/margins": 8.351922988891602,
      "rewards/rejected": -11.211479187011719,
      "step": 1215
    },
    {
      "epoch": 2.6016,
      "grad_norm": 2.0166029930114746,
      "learning_rate": 2.307280835466144e-07,
      "logits/chosen": -1.6319023370742798,
      "logits/rejected": -1.5826950073242188,
      "logps/chosen": -260.2449951171875,
      "logps/rejected": -415.6009216308594,
      "loss": 0.0724,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.7726876735687256,
      "rewards/margins": 8.852060317993164,
      "rewards/rejected": -11.624747276306152,
      "step": 1220
    },
    {
      "epoch": 2.6122666666666667,
      "grad_norm": 1.6875503063201904,
      "learning_rate": 2.1879841211780856e-07,
      "logits/chosen": -1.6714298725128174,
      "logits/rejected": -1.5990326404571533,
      "logps/chosen": -252.33450317382812,
      "logps/rejected": -385.48095703125,
      "loss": 0.07,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.4542059898376465,
      "rewards/margins": 8.378938674926758,
      "rewards/rejected": -10.833145141601562,
      "step": 1225
    },
    {
      "epoch": 2.622933333333333,
      "grad_norm": 3.2986953258514404,
      "learning_rate": 2.0717127168622202e-07,
      "logits/chosen": -1.6889839172363281,
      "logits/rejected": -1.562585711479187,
      "logps/chosen": -326.72442626953125,
      "logps/rejected": -389.68267822265625,
      "loss": 0.0875,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.467181444168091,
      "rewards/margins": 8.251823425292969,
      "rewards/rejected": -10.71900463104248,
      "step": 1230
    },
    {
      "epoch": 2.6336,
      "grad_norm": 2.2151730060577393,
      "learning_rate": 1.958482042332424e-07,
      "logits/chosen": -1.699887990951538,
      "logits/rejected": -1.5633143186569214,
      "logps/chosen": -285.1319580078125,
      "logps/rejected": -402.71697998046875,
      "loss": 0.0948,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.002988338470459,
      "rewards/margins": 7.786578178405762,
      "rewards/rejected": -10.789566993713379,
      "step": 1235
    },
    {
      "epoch": 2.6442666666666668,
      "grad_norm": 4.6926655769348145,
      "learning_rate": 1.8483071141435937e-07,
      "logits/chosen": -1.6601158380508423,
      "logits/rejected": -1.5603011846542358,
      "logps/chosen": -263.892333984375,
      "logps/rejected": -367.2433166503906,
      "loss": 0.1363,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.742337942123413,
      "rewards/margins": 7.589631080627441,
      "rewards/rejected": -10.331968307495117,
      "step": 1240
    },
    {
      "epoch": 2.654933333333333,
      "grad_norm": 3.295954942703247,
      "learning_rate": 1.741202543600176e-07,
      "logits/chosen": -1.7194404602050781,
      "logits/rejected": -1.5846221446990967,
      "logps/chosen": -241.400146484375,
      "logps/rejected": -396.5201721191406,
      "loss": 0.0682,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.6485438346862793,
      "rewards/margins": 9.085619926452637,
      "rewards/rejected": -11.734164237976074,
      "step": 1245
    },
    {
      "epoch": 2.6656,
      "grad_norm": 6.180679798126221,
      "learning_rate": 1.6371825348184155e-07,
      "logits/chosen": -1.6426560878753662,
      "logits/rejected": -1.5631107091903687,
      "logps/chosen": -276.7149353027344,
      "logps/rejected": -389.96380615234375,
      "loss": 0.1129,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.8253414630889893,
      "rewards/margins": 8.019664764404297,
      "rewards/rejected": -10.845005989074707,
      "step": 1250
    },
    {
      "epoch": 2.676266666666667,
      "grad_norm": 0.9536997079849243,
      "learning_rate": 1.5362608828426246e-07,
      "logits/chosen": -1.7700952291488647,
      "logits/rejected": -1.7249057292938232,
      "logps/chosen": -256.5973815917969,
      "logps/rejected": -408.5018005371094,
      "loss": 0.0952,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.539684295654297,
      "rewards/margins": 8.208810806274414,
      "rewards/rejected": -10.748494148254395,
      "step": 1255
    },
    {
      "epoch": 2.686933333333333,
      "grad_norm": 2.8590283393859863,
      "learning_rate": 1.438450971815686e-07,
      "logits/chosen": -1.7384529113769531,
      "logits/rejected": -1.6048595905303955,
      "logps/chosen": -244.7389678955078,
      "logps/rejected": -378.0864562988281,
      "loss": 0.0781,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.570819854736328,
      "rewards/margins": 8.148038864135742,
      "rewards/rejected": -10.71885871887207,
      "step": 1260
    },
    {
      "epoch": 2.6976,
      "grad_norm": 3.8086490631103516,
      "learning_rate": 1.3437657732040783e-07,
      "logits/chosen": -1.6877682209014893,
      "logits/rejected": -1.6243743896484375,
      "logps/chosen": -265.04913330078125,
      "logps/rejected": -389.056640625,
      "loss": 0.1076,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.82427716255188,
      "rewards/margins": 7.797641754150391,
      "rewards/rejected": -10.621920585632324,
      "step": 1265
    },
    {
      "epoch": 2.708266666666667,
      "grad_norm": 1.2340490818023682,
      "learning_rate": 1.2522178440776022e-07,
      "logits/chosen": -1.6442654132843018,
      "logits/rejected": -1.5744202136993408,
      "logps/chosen": -265.56964111328125,
      "logps/rejected": -391.43963623046875,
      "loss": 0.0637,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.7177109718322754,
      "rewards/margins": 8.296258926391602,
      "rewards/rejected": -11.013969421386719,
      "step": 1270
    },
    {
      "epoch": 2.718933333333333,
      "grad_norm": 1.5882465839385986,
      "learning_rate": 1.1638193254440655e-07,
      "logits/chosen": -1.6713117361068726,
      "logits/rejected": -1.57658052444458,
      "logps/chosen": -279.78704833984375,
      "logps/rejected": -401.1554870605469,
      "loss": 0.0471,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.8568999767303467,
      "rewards/margins": 9.124058723449707,
      "rewards/rejected": -11.980958938598633,
      "step": 1275
    },
    {
      "epoch": 2.7296,
      "grad_norm": 5.013668060302734,
      "learning_rate": 1.0785819406391562e-07,
      "logits/chosen": -1.638289213180542,
      "logits/rejected": -1.6082589626312256,
      "logps/chosen": -245.79031372070312,
      "logps/rejected": -423.3377380371094,
      "loss": 0.0595,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.737645387649536,
      "rewards/margins": 9.465682983398438,
      "rewards/rejected": -12.203328132629395,
      "step": 1280
    },
    {
      "epoch": 2.740266666666667,
      "grad_norm": 4.975745677947998,
      "learning_rate": 9.965169937717095e-08,
      "logits/chosen": -1.6764233112335205,
      "logits/rejected": -1.6596872806549072,
      "logps/chosen": -258.89019775390625,
      "logps/rejected": -408.6078186035156,
      "loss": 0.0602,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.8898043632507324,
      "rewards/margins": 8.435752868652344,
      "rewards/rejected": -11.325556755065918,
      "step": 1285
    },
    {
      "epoch": 2.7509333333333332,
      "grad_norm": 1.6985228061676025,
      "learning_rate": 9.176353682245676e-08,
      "logits/chosen": -1.7202489376068115,
      "logits/rejected": -1.611146330833435,
      "logps/chosen": -250.17483520507812,
      "logps/rejected": -398.2389221191406,
      "loss": 0.0759,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.7857604026794434,
      "rewards/margins": 8.919150352478027,
      "rewards/rejected": -11.704911231994629,
      "step": 1290
    },
    {
      "epoch": 2.7616,
      "grad_norm": 3.58396577835083,
      "learning_rate": 8.41947525211223e-08,
      "logits/chosen": -1.5990982055664062,
      "logits/rejected": -1.5592104196548462,
      "logps/chosen": -269.1976623535156,
      "logps/rejected": -397.72674560546875,
      "loss": 0.0831,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.7801098823547363,
      "rewards/margins": 8.252833366394043,
      "rewards/rejected": -11.032944679260254,
      "step": 1295
    },
    {
      "epoch": 2.772266666666667,
      "grad_norm": 3.4320976734161377,
      "learning_rate": 7.694635023884789e-08,
      "logits/chosen": -1.6502788066864014,
      "logits/rejected": -1.634393334388733,
      "logps/chosen": -268.66253662109375,
      "logps/rejected": -408.7780456542969,
      "loss": 0.0665,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.4980883598327637,
      "rewards/margins": 8.380696296691895,
      "rewards/rejected": -10.8787841796875,
      "step": 1300
    },
    {
      "epoch": 2.7829333333333333,
      "grad_norm": 1.1601080894470215,
      "learning_rate": 7.00192912525241e-08,
      "logits/chosen": -1.707038164138794,
      "logits/rejected": -1.5817816257476807,
      "logps/chosen": -267.05767822265625,
      "logps/rejected": -411.76776123046875,
      "loss": 0.0579,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.5934057235717773,
      "rewards/margins": 9.068880081176758,
      "rewards/rejected": -11.662285804748535,
      "step": 1305
    },
    {
      "epoch": 2.7936,
      "grad_norm": 3.5436959266662598,
      "learning_rate": 6.341449422277047e-08,
      "logits/chosen": -1.6015123128890991,
      "logits/rejected": -1.546696662902832,
      "logps/chosen": -273.44683837890625,
      "logps/rejected": -400.2070007324219,
      "loss": 0.0551,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.4368176460266113,
      "rewards/margins": 8.266563415527344,
      "rewards/rejected": -10.703381538391113,
      "step": 1310
    },
    {
      "epoch": 2.804266666666667,
      "grad_norm": 2.056448459625244,
      "learning_rate": 5.7132835072101486e-08,
      "logits/chosen": -1.65818190574646,
      "logits/rejected": -1.6212623119354248,
      "logps/chosen": -271.24542236328125,
      "logps/rejected": -416.03265380859375,
      "loss": 0.0621,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.530935287475586,
      "rewards/margins": 9.317224502563477,
      "rewards/rejected": -11.848158836364746,
      "step": 1315
    },
    {
      "epoch": 2.8149333333333333,
      "grad_norm": 1.1611863374710083,
      "learning_rate": 5.117514686876379e-08,
      "logits/chosen": -1.6843284368515015,
      "logits/rejected": -1.570382833480835,
      "logps/chosen": -247.26882934570312,
      "logps/rejected": -382.57171630859375,
      "loss": 0.0613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.619004487991333,
      "rewards/margins": 8.069233894348145,
      "rewards/rejected": -10.688237190246582,
      "step": 1320
    },
    {
      "epoch": 2.8256,
      "grad_norm": 3.386897087097168,
      "learning_rate": 4.5542219716255274e-08,
      "logits/chosen": -1.6479991674423218,
      "logits/rejected": -1.647812843322754,
      "logps/chosen": -264.9526062011719,
      "logps/rejected": -393.6275939941406,
      "loss": 0.0578,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.8374381065368652,
      "rewards/margins": 8.005861282348633,
      "rewards/rejected": -10.84329891204834,
      "step": 1325
    },
    {
      "epoch": 2.836266666666667,
      "grad_norm": 2.9551398754119873,
      "learning_rate": 4.023480064854174e-08,
      "logits/chosen": -1.709986925125122,
      "logits/rejected": -1.658769965171814,
      "logps/chosen": -261.8425598144531,
      "logps/rejected": -389.1148681640625,
      "loss": 0.0814,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.9073290824890137,
      "rewards/margins": 7.660365104675293,
      "rewards/rejected": -10.567693710327148,
      "step": 1330
    },
    {
      "epoch": 2.8469333333333333,
      "grad_norm": 5.993083953857422,
      "learning_rate": 3.525359353098584e-08,
      "logits/chosen": -1.7044435739517212,
      "logits/rejected": -1.613206148147583,
      "logps/chosen": -260.56781005859375,
      "logps/rejected": -400.2635192871094,
      "loss": 0.0947,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.7162492275238037,
      "rewards/margins": 8.33190631866455,
      "rewards/rejected": -11.048154830932617,
      "step": 1335
    },
    {
      "epoch": 2.8576,
      "grad_norm": 0.7436990141868591,
      "learning_rate": 3.059925896700094e-08,
      "logits/chosen": -1.5562223196029663,
      "logits/rejected": -1.5099818706512451,
      "logps/chosen": -282.28704833984375,
      "logps/rejected": -367.90142822265625,
      "loss": 0.066,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.5948193073272705,
      "rewards/margins": 7.88361120223999,
      "rewards/rejected": -10.478429794311523,
      "step": 1340
    },
    {
      "epoch": 2.8682666666666665,
      "grad_norm": 1.7763005495071411,
      "learning_rate": 2.6272414210443964e-08,
      "logits/chosen": -1.6868610382080078,
      "logits/rejected": -1.6632941961288452,
      "logps/chosen": -266.9272766113281,
      "logps/rejected": -396.6302795410156,
      "loss": 0.1068,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.7057154178619385,
      "rewards/margins": 8.647436141967773,
      "rewards/rejected": -11.353150367736816,
      "step": 1345
    },
    {
      "epoch": 2.8789333333333333,
      "grad_norm": 3.7743892669677734,
      "learning_rate": 2.2273633083753632e-08,
      "logits/chosen": -1.6585218906402588,
      "logits/rejected": -1.613133430480957,
      "logps/chosen": -289.80340576171875,
      "logps/rejected": -405.9570007324219,
      "loss": 0.0767,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.0055086612701416,
      "rewards/margins": 8.358158111572266,
      "rewards/rejected": -11.363667488098145,
      "step": 1350
    },
    {
      "epoch": 2.8895999999999997,
      "grad_norm": 0.7690500020980835,
      "learning_rate": 1.8603445901851647e-08,
      "logits/chosen": -1.6909067630767822,
      "logits/rejected": -1.5630600452423096,
      "logps/chosen": -267.53607177734375,
      "logps/rejected": -386.2433166503906,
      "loss": 0.0566,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.2426962852478027,
      "rewards/margins": 8.468097686767578,
      "rewards/rejected": -10.710794448852539,
      "step": 1355
    },
    {
      "epoch": 2.9002666666666665,
      "grad_norm": 4.579198837280273,
      "learning_rate": 1.5262339401813375e-08,
      "logits/chosen": -1.647733449935913,
      "logits/rejected": -1.6072362661361694,
      "logps/chosen": -268.2415466308594,
      "logps/rejected": -409.1459655761719,
      "loss": 0.0983,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.9298815727233887,
      "rewards/margins": 8.94694709777832,
      "rewards/rejected": -11.876829147338867,
      "step": 1360
    },
    {
      "epoch": 2.9109333333333334,
      "grad_norm": 2.3603615760803223,
      "learning_rate": 1.2250756678315601e-08,
      "logits/chosen": -1.6706626415252686,
      "logits/rejected": -1.598555326461792,
      "logps/chosen": -275.09356689453125,
      "logps/rejected": -386.44921875,
      "loss": 0.0754,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.6181488037109375,
      "rewards/margins": 7.823951721191406,
      "rewards/rejected": -10.442100524902344,
      "step": 1365
    },
    {
      "epoch": 2.9215999999999998,
      "grad_norm": 3.3083324432373047,
      "learning_rate": 9.569097124875193e-09,
      "logits/chosen": -1.6924657821655273,
      "logits/rejected": -1.5973902940750122,
      "logps/chosen": -266.7765197753906,
      "logps/rejected": -410.7627868652344,
      "loss": 0.0799,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.732797145843506,
      "rewards/margins": 8.773340225219727,
      "rewards/rejected": -11.506136894226074,
      "step": 1370
    },
    {
      "epoch": 2.9322666666666666,
      "grad_norm": 4.005508899688721,
      "learning_rate": 7.217716380881479e-09,
      "logits/chosen": -1.6512973308563232,
      "logits/rejected": -1.5719560384750366,
      "logps/chosen": -287.5769958496094,
      "logps/rejected": -392.5569763183594,
      "loss": 0.0754,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.802158832550049,
      "rewards/margins": 8.369941711425781,
      "rewards/rejected": -11.172099113464355,
      "step": 1375
    },
    {
      "epoch": 2.9429333333333334,
      "grad_norm": 2.5721302032470703,
      "learning_rate": 5.196926284430359e-09,
      "logits/chosen": -1.6722768545150757,
      "logits/rejected": -1.5862112045288086,
      "logps/chosen": -276.39044189453125,
      "logps/rejected": -428.83062744140625,
      "loss": 0.0756,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.7317731380462646,
      "rewards/margins": 7.940549373626709,
      "rewards/rejected": -10.672322273254395,
      "step": 1380
    },
    {
      "epoch": 2.9536,
      "grad_norm": 1.1050063371658325,
      "learning_rate": 3.5069948309712775e-09,
      "logits/chosen": -1.563157320022583,
      "logits/rejected": -1.5820673704147339,
      "logps/chosen": -266.9634094238281,
      "logps/rejected": -400.5401611328125,
      "loss": 0.0924,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.8875529766082764,
      "rewards/margins": 7.935434818267822,
      "rewards/rejected": -10.822988510131836,
      "step": 1385
    },
    {
      "epoch": 2.9642666666666666,
      "grad_norm": 2.0641963481903076,
      "learning_rate": 2.1481461377634294e-09,
      "logits/chosen": -1.6637370586395264,
      "logits/rejected": -1.621468186378479,
      "logps/chosen": -285.5246887207031,
      "logps/rejected": -401.16314697265625,
      "loss": 0.0639,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.8416502475738525,
      "rewards/margins": 8.040994644165039,
      "rewards/rejected": -10.882644653320312,
      "step": 1390
    },
    {
      "epoch": 2.9749333333333334,
      "grad_norm": 1.0203123092651367,
      "learning_rate": 1.120560414154259e-09,
      "logits/chosen": -1.6394908428192139,
      "logits/rejected": -1.5650370121002197,
      "logps/chosen": -250.5342254638672,
      "logps/rejected": -398.98797607421875,
      "loss": 0.0769,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.790039539337158,
      "rewards/margins": 8.689876556396484,
      "rewards/rejected": -11.4799165725708,
      "step": 1395
    },
    {
      "epoch": 2.9856,
      "grad_norm": 3.153134346008301,
      "learning_rate": 4.2437393768079983e-10,
      "logits/chosen": -1.6397968530654907,
      "logits/rejected": -1.585146427154541,
      "logps/chosen": -279.3368225097656,
      "logps/rejected": -408.64739990234375,
      "loss": 0.0813,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.9198312759399414,
      "rewards/margins": 8.351078987121582,
      "rewards/rejected": -11.270910263061523,
      "step": 1400
    },
    {
      "epoch": 2.9962666666666666,
      "grad_norm": 4.959558010101318,
      "learning_rate": 5.967903599635171e-11,
      "logits/chosen": -1.687861680984497,
      "logits/rejected": -1.4993536472320557,
      "logps/chosen": -250.8742218017578,
      "logps/rejected": -405.6380920410156,
      "loss": 0.0809,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.5417988300323486,
      "rewards/margins": 8.413880348205566,
      "rewards/rejected": -10.955678939819336,
      "step": 1405
    }
  ],
  "logging_steps": 5,
  "max_steps": 1407,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}