{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 1407,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010666666666666666,
      "grad_norm": NaN,
      "learning_rate": 4.651162790697675e-07,
      "logits/chosen": -1.3544222116470337,
      "logits/rejected": -1.2236188650131226,
      "logps/chosen": -231.80038452148438,
      "logps/rejected": -251.7948760986328,
      "loss": 0.6931,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -7.575037307105958e-05,
      "rewards/margins": 6.424905586754903e-05,
      "rewards/rejected": -0.00013999939255882055,
      "step": 5
    },
    {
      "epoch": 0.021333333333333333,
      "grad_norm": 3.109471559524536,
      "learning_rate": 1.0465116279069768e-06,
      "logits/chosen": -1.308906078338623,
      "logits/rejected": -1.306926965713501,
      "logps/chosen": -224.56344604492188,
      "logps/rejected": -251.70864868164062,
      "loss": 0.6922,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.0001932097366079688,
      "rewards/margins": 0.0019199944799765944,
      "rewards/rejected": -0.0021132039837539196,
      "step": 10
    },
    {
      "epoch": 0.032,
      "grad_norm": NaN,
      "learning_rate": 1.6279069767441862e-06,
      "logits/chosen": -1.3518203496932983,
      "logits/rejected": -1.2913354635238647,
      "logps/chosen": -241.5867156982422,
      "logps/rejected": -274.6536865234375,
      "loss": 0.688,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.0011881353566423059,
      "rewards/margins": 0.010762467980384827,
      "rewards/rejected": -0.011950601823627949,
      "step": 15
    },
    {
      "epoch": 0.042666666666666665,
      "grad_norm": 2.744222402572632,
      "learning_rate": 2.2093023255813954e-06,
      "logits/chosen": -1.3305720090866089,
      "logits/rejected": -1.2945411205291748,
      "logps/chosen": -238.5529022216797,
      "logps/rejected": -261.58392333984375,
      "loss": 0.6826,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.005761843174695969,
      "rewards/margins": 0.02146998979151249,
      "rewards/rejected": -0.02723183110356331,
      "step": 20
    },
    {
      "epoch": 0.05333333333333334,
      "grad_norm": 4.8908371925354,
      "learning_rate": 2.790697674418605e-06,
      "logits/chosen": -1.3832628726959229,
      "logits/rejected": -1.3136870861053467,
      "logps/chosen": -220.1907958984375,
      "logps/rejected": -271.0519104003906,
      "loss": 0.6735,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.013803968206048012,
      "rewards/margins": 0.04084903746843338,
      "rewards/rejected": -0.05465300753712654,
      "step": 25
    },
    {
      "epoch": 0.064,
      "grad_norm": 3.333070755004883,
      "learning_rate": 3.372093023255814e-06,
      "logits/chosen": -1.3710800409317017,
      "logits/rejected": -1.2284512519836426,
      "logps/chosen": -249.5676727294922,
      "logps/rejected": -253.8758544921875,
      "loss": 0.6666,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.03833365440368652,
      "rewards/margins": 0.05555647611618042,
      "rewards/rejected": -0.09389013051986694,
      "step": 30
    },
    {
      "epoch": 0.07466666666666667,
      "grad_norm": 3.271761417388916,
      "learning_rate": 3.953488372093024e-06,
      "logits/chosen": -1.4493813514709473,
      "logits/rejected": -1.3014929294586182,
      "logps/chosen": -238.13302612304688,
      "logps/rejected": -246.8301239013672,
      "loss": 0.6531,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.05066124349832535,
      "rewards/margins": 0.08616013824939728,
      "rewards/rejected": -0.13682137429714203,
      "step": 35
    },
    {
      "epoch": 0.08533333333333333,
      "grad_norm": 2.7482733726501465,
      "learning_rate": 4.5348837209302326e-06,
      "logits/chosen": -1.3479039669036865,
      "logits/rejected": -1.28487229347229,
      "logps/chosen": -239.9534149169922,
      "logps/rejected": -261.44146728515625,
      "loss": 0.637,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07571917772293091,
      "rewards/margins": 0.12467632442712784,
      "rewards/rejected": -0.20039549469947815,
      "step": 40
    },
    {
      "epoch": 0.096,
      "grad_norm": 3.2934110164642334,
      "learning_rate": 4.99999336897255e-06,
      "logits/chosen": -1.452958345413208,
      "logits/rejected": -1.312406301498413,
      "logps/chosen": -239.06747436523438,
      "logps/rejected": -248.6891632080078,
      "loss": 0.6173,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.09406520426273346,
      "rewards/margins": 0.16908395290374756,
      "rewards/rejected": -0.2631491422653198,
      "step": 45
    },
    {
      "epoch": 0.10666666666666667,
      "grad_norm": 2.822927951812744,
      "learning_rate": 4.999761286705285e-06,
      "logits/chosen": -1.401163101196289,
      "logits/rejected": -1.3909813165664673,
      "logps/chosen": -242.43093872070312,
      "logps/rejected": -258.1756286621094,
      "loss": 0.5937,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.12050650268793106,
      "rewards/margins": 0.23688121140003204,
      "rewards/rejected": -0.3573877215385437,
      "step": 50
    },
    {
      "epoch": 0.11733333333333333,
      "grad_norm": 3.2107081413269043,
      "learning_rate": 4.999197688241076e-06,
      "logits/chosen": -1.420990228652954,
      "logits/rejected": -1.3618907928466797,
      "logps/chosen": -232.1669921875,
      "logps/rejected": -267.45782470703125,
      "loss": 0.5926,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.15464790165424347,
      "rewards/margins": 0.25124019384384155,
      "rewards/rejected": -0.4058881402015686,
      "step": 55
    },
    {
      "epoch": 0.128,
      "grad_norm": 3.5179314613342285,
      "learning_rate": 4.9983026483238685e-06,
      "logits/chosen": -1.4916331768035889,
      "logits/rejected": -1.3871724605560303,
      "logps/chosen": -236.6553497314453,
      "logps/rejected": -259.3423767089844,
      "loss": 0.5453,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.189641535282135,
      "rewards/margins": 0.38970229029655457,
      "rewards/rejected": -0.5793437957763672,
      "step": 60
    },
    {
      "epoch": 0.13866666666666666,
      "grad_norm": 2.8678946495056152,
      "learning_rate": 4.997076285653087e-06,
      "logits/chosen": -1.5075359344482422,
      "logits/rejected": -1.4784154891967773,
      "logps/chosen": -250.3011016845703,
      "logps/rejected": -271.4066467285156,
      "loss": 0.5349,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.25288641452789307,
      "rewards/margins": 0.4173409938812256,
      "rewards/rejected": -0.6702274084091187,
      "step": 65
    },
    {
      "epoch": 0.14933333333333335,
      "grad_norm": 3.1192572116851807,
      "learning_rate": 4.995518762867897e-06,
      "logits/chosen": -1.4870703220367432,
      "logits/rejected": -1.4476995468139648,
      "logps/chosen": -215.5365753173828,
      "logps/rejected": -257.3900146484375,
      "loss": 0.5312,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.3431021571159363,
      "rewards/margins": 0.4714565873146057,
      "rewards/rejected": -0.8145586848258972,
      "step": 70
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.9746811389923096,
      "learning_rate": 4.993630286525634e-06,
      "logits/chosen": -1.5514018535614014,
      "logits/rejected": -1.3919061422348022,
      "logps/chosen": -232.2204132080078,
      "logps/rejected": -262.62286376953125,
      "loss": 0.482,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.3241270184516907,
      "rewards/margins": 0.6433579325675964,
      "rewards/rejected": -0.9674848318099976,
      "step": 75
    },
    {
      "epoch": 0.17066666666666666,
      "grad_norm": 2.728930711746216,
      "learning_rate": 4.99141110707441e-06,
      "logits/chosen": -1.463017225265503,
      "logits/rejected": -1.3513896465301514,
      "logps/chosen": -257.9603576660156,
      "logps/rejected": -288.959716796875,
      "loss": 0.4826,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.379680335521698,
      "rewards/margins": 0.6570689082145691,
      "rewards/rejected": -1.036749243736267,
      "step": 80
    },
    {
      "epoch": 0.18133333333333335,
      "grad_norm": 2.79579758644104,
      "learning_rate": 4.9888615188199e-06,
      "logits/chosen": -1.5496175289154053,
      "logits/rejected": -1.4801855087280273,
      "logps/chosen": -238.9860382080078,
      "logps/rejected": -276.8408203125,
      "loss": 0.4282,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.3846990466117859,
      "rewards/margins": 0.8239493370056152,
      "rewards/rejected": -1.208648443222046,
      "step": 85
    },
    {
      "epoch": 0.192,
      "grad_norm": 2.9647085666656494,
      "learning_rate": 4.9859818598863104e-06,
      "logits/chosen": -1.4761499166488647,
      "logits/rejected": -1.3574707508087158,
      "logps/chosen": -235.34024047851562,
      "logps/rejected": -261.658203125,
      "loss": 0.462,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.4849798083305359,
      "rewards/margins": 0.7756170034408569,
      "rewards/rejected": -1.2605968713760376,
      "step": 90
    },
    {
      "epoch": 0.20266666666666666,
      "grad_norm": 3.024820327758789,
      "learning_rate": 4.982772512171538e-06,
      "logits/chosen": -1.5438058376312256,
      "logits/rejected": -1.4416005611419678,
      "logps/chosen": -239.1410369873047,
      "logps/rejected": -278.98419189453125,
      "loss": 0.4506,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.6898207664489746,
      "rewards/margins": 0.8674094080924988,
      "rewards/rejected": -1.5572302341461182,
      "step": 95
    },
    {
      "epoch": 0.21333333333333335,
      "grad_norm": 2.9155285358428955,
      "learning_rate": 4.979233901296523e-06,
      "logits/chosen": -1.5428584814071655,
      "logits/rejected": -1.4572758674621582,
      "logps/chosen": -210.7685546875,
      "logps/rejected": -254.948974609375,
      "loss": 0.3884,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.6256662011146545,
      "rewards/margins": 1.2459337711334229,
      "rewards/rejected": -1.8715999126434326,
      "step": 100
    },
    {
      "epoch": 0.224,
      "grad_norm": 2.731834650039673,
      "learning_rate": 4.975366496548804e-06,
      "logits/chosen": -1.5170609951019287,
      "logits/rejected": -1.4053804874420166,
      "logps/chosen": -241.0113983154297,
      "logps/rejected": -256.126220703125,
      "loss": 0.4358,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.6871055364608765,
      "rewards/margins": 0.9470389485359192,
      "rewards/rejected": -1.6341444253921509,
      "step": 105
    },
    {
      "epoch": 0.23466666666666666,
      "grad_norm": 2.4331085681915283,
      "learning_rate": 4.971170810820279e-06,
      "logits/chosen": -1.425811529159546,
      "logits/rejected": -1.4387760162353516,
      "logps/chosen": -232.7527313232422,
      "logps/rejected": -277.49896240234375,
      "loss": 0.3911,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.6859896779060364,
      "rewards/margins": 1.2273075580596924,
      "rewards/rejected": -1.9132972955703735,
      "step": 110
    },
    {
      "epoch": 0.24533333333333332,
      "grad_norm": 2.802173137664795,
      "learning_rate": 4.96664740053919e-06,
      "logits/chosen": -1.4978256225585938,
      "logits/rejected": -1.4406821727752686,
      "logps/chosen": -230.27685546875,
      "logps/rejected": -288.71697998046875,
      "loss": 0.3774,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.8096278309822083,
      "rewards/margins": 1.2959668636322021,
      "rewards/rejected": -2.1055946350097656,
      "step": 115
    },
    {
      "epoch": 0.256,
      "grad_norm": 2.4168198108673096,
      "learning_rate": 4.961796865596328e-06,
      "logits/chosen": -1.4427374601364136,
      "logits/rejected": -1.352508306503296,
      "logps/chosen": -242.40817260742188,
      "logps/rejected": -288.6490173339844,
      "loss": 0.3858,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.9087854623794556,
      "rewards/margins": 1.4612674713134766,
      "rewards/rejected": -2.3700530529022217,
      "step": 120
    },
    {
      "epoch": 0.26666666666666666,
      "grad_norm": 3.649536371231079,
      "learning_rate": 4.956619849265473e-06,
      "logits/chosen": -1.5077062845230103,
      "logits/rejected": -1.373518466949463,
      "logps/chosen": -241.6128387451172,
      "logps/rejected": -282.96295166015625,
      "loss": 0.4056,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.0533024072647095,
      "rewards/margins": 1.3088269233703613,
      "rewards/rejected": -2.3621296882629395,
      "step": 125
    },
    {
      "epoch": 0.2773333333333333,
      "grad_norm": 2.843197822570801,
      "learning_rate": 4.951117038118091e-06,
      "logits/chosen": -1.6640141010284424,
      "logits/rejected": -1.5276637077331543,
      "logps/chosen": -234.8096466064453,
      "logps/rejected": -284.8505554199219,
      "loss": 0.4381,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9368289709091187,
      "rewards/margins": 1.2585748434066772,
      "rewards/rejected": -2.195403575897217,
      "step": 130
    },
    {
      "epoch": 0.288,
      "grad_norm": 2.889314889907837,
      "learning_rate": 4.945289161932273e-06,
      "logits/chosen": -1.5789138078689575,
      "logits/rejected": -1.524099588394165,
      "logps/chosen": -248.79867553710938,
      "logps/rejected": -292.62481689453125,
      "loss": 0.3831,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.8927059173583984,
      "rewards/margins": 1.4558749198913574,
      "rewards/rejected": -2.348580837249756,
      "step": 135
    },
    {
      "epoch": 0.2986666666666667,
      "grad_norm": 3.7556686401367188,
      "learning_rate": 4.93913699359596e-06,
      "logits/chosen": -1.6477344036102295,
      "logits/rejected": -1.4557158946990967,
      "logps/chosen": -230.6923828125,
      "logps/rejected": -281.36468505859375,
      "loss": 0.3376,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.8048444986343384,
      "rewards/margins": 1.6911342144012451,
      "rewards/rejected": -2.495978593826294,
      "step": 140
    },
    {
      "epoch": 0.30933333333333335,
      "grad_norm": 2.8196702003479004,
      "learning_rate": 4.9326613490044364e-06,
      "logits/chosen": -1.5528147220611572,
      "logits/rejected": -1.4781819581985474,
      "logps/chosen": -224.55899047851562,
      "logps/rejected": -263.1064147949219,
      "loss": 0.3614,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.56626957654953,
      "rewards/margins": 1.4652245044708252,
      "rewards/rejected": -2.031494140625,
      "step": 145
    },
    {
      "epoch": 0.32,
      "grad_norm": 3.5620477199554443,
      "learning_rate": 4.925863086952132e-06,
      "logits/chosen": -1.6504319906234741,
      "logits/rejected": -1.5435733795166016,
      "logps/chosen": -255.0889129638672,
      "logps/rejected": -297.24017333984375,
      "loss": 0.3755,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.7335026264190674,
      "rewards/margins": 1.4152415990829468,
      "rewards/rejected": -2.1487443447113037,
      "step": 150
    },
    {
      "epoch": 0.33066666666666666,
      "grad_norm": 2.918090343475342,
      "learning_rate": 4.918743109018727e-06,
      "logits/chosen": -1.659012794494629,
      "logits/rejected": -1.5104398727416992,
      "logps/chosen": -234.96072387695312,
      "logps/rejected": -279.97979736328125,
      "loss": 0.3624,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.7696214914321899,
      "rewards/margins": 1.4934290647506714,
      "rewards/rejected": -2.2630505561828613,
      "step": 155
    },
    {
      "epoch": 0.3413333333333333,
      "grad_norm": 2.317545175552368,
      "learning_rate": 4.911302359449585e-06,
      "logits/chosen": -1.6991151571273804,
      "logits/rejected": -1.5082671642303467,
      "logps/chosen": -232.7823486328125,
      "logps/rejected": -265.2646789550781,
      "loss": 0.3063,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.5311766862869263,
      "rewards/margins": 1.7594959735870361,
      "rewards/rejected": -2.290672779083252,
      "step": 160
    },
    {
      "epoch": 0.352,
      "grad_norm": 2.7809877395629883,
      "learning_rate": 4.9035418250305314e-06,
      "logits/chosen": -1.5785572528839111,
      "logits/rejected": -1.440476655960083,
      "logps/chosen": -239.0006103515625,
      "logps/rejected": -274.723876953125,
      "loss": 0.3032,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.5675729513168335,
      "rewards/margins": 1.8231475353240967,
      "rewards/rejected": -2.3907203674316406,
      "step": 165
    },
    {
      "epoch": 0.3626666666666667,
      "grad_norm": 5.299256801605225,
      "learning_rate": 4.895462534956981e-06,
      "logits/chosen": -1.5343797206878662,
      "logits/rejected": -1.501845121383667,
      "logps/chosen": -249.54916381835938,
      "logps/rejected": -271.220703125,
      "loss": 0.4059,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.8906148672103882,
      "rewards/margins": 1.5632604360580444,
      "rewards/rejected": -2.4538753032684326,
      "step": 170
    },
    {
      "epoch": 0.37333333333333335,
      "grad_norm": 3.4545021057128906,
      "learning_rate": 4.887065560697449e-06,
      "logits/chosen": -1.5327939987182617,
      "logits/rejected": -1.5170584917068481,
      "logps/chosen": -257.06146240234375,
      "logps/rejected": -294.1949462890625,
      "loss": 0.3698,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8633643984794617,
      "rewards/margins": 1.7004616260528564,
      "rewards/rejected": -2.563826084136963,
      "step": 175
    },
    {
      "epoch": 0.384,
      "grad_norm": 3.70247745513916,
      "learning_rate": 4.878352015851459e-06,
      "logits/chosen": -1.652748703956604,
      "logits/rejected": -1.486902117729187,
      "logps/chosen": -253.253662109375,
      "logps/rejected": -273.6691589355469,
      "loss": 0.3075,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.6801033020019531,
      "rewards/margins": 1.8728739023208618,
      "rewards/rejected": -2.5529773235321045,
      "step": 180
    },
    {
      "epoch": 0.39466666666666667,
      "grad_norm": 2.5748231410980225,
      "learning_rate": 4.869323056001852e-06,
      "logits/chosen": -1.6040265560150146,
      "logits/rejected": -1.5244295597076416,
      "logps/chosen": -226.2984619140625,
      "logps/rejected": -279.1867980957031,
      "loss": 0.3129,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.4969603419303894,
      "rewards/margins": 1.9832227230072021,
      "rewards/rejected": -2.4801833629608154,
      "step": 185
    },
    {
      "epoch": 0.4053333333333333,
      "grad_norm": 3.6774702072143555,
      "learning_rate": 4.859979878561536e-06,
      "logits/chosen": -1.6129461526870728,
      "logits/rejected": -1.516275405883789,
      "logps/chosen": -237.805908203125,
      "logps/rejected": -278.765380859375,
      "loss": 0.3054,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.6979565620422363,
      "rewards/margins": 2.057680130004883,
      "rewards/rejected": -2.755636692047119,
      "step": 190
    },
    {
      "epoch": 0.416,
      "grad_norm": 3.621861219406128,
      "learning_rate": 4.850323722614685e-06,
      "logits/chosen": -1.588699460029602,
      "logits/rejected": -1.4784739017486572,
      "logps/chosen": -227.1220703125,
      "logps/rejected": -269.6943359375,
      "loss": 0.3308,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.7699406743049622,
      "rewards/margins": 1.8302541971206665,
      "rewards/rejected": -2.6001949310302734,
      "step": 195
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 3.025399923324585,
      "learning_rate": 4.840355868752416e-06,
      "logits/chosen": -1.523004174232483,
      "logits/rejected": -1.4143463373184204,
      "logps/chosen": -284.00787353515625,
      "logps/rejected": -284.97100830078125,
      "loss": 0.3458,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.6804647445678711,
      "rewards/margins": 1.930129051208496,
      "rewards/rejected": -2.610593795776367,
      "step": 200
    },
    {
      "epoch": 0.43733333333333335,
      "grad_norm": 3.7301924228668213,
      "learning_rate": 4.830077638902955e-06,
      "logits/chosen": -1.6758838891983032,
      "logits/rejected": -1.541736364364624,
      "logps/chosen": -213.0896759033203,
      "logps/rejected": -273.7264099121094,
      "loss": 0.3491,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8146641850471497,
      "rewards/margins": 2.013720989227295,
      "rewards/rejected": -2.828385353088379,
      "step": 205
    },
    {
      "epoch": 0.448,
      "grad_norm": 3.1502671241760254,
      "learning_rate": 4.8194903961563255e-06,
      "logits/chosen": -1.65460205078125,
      "logits/rejected": -1.5929137468338013,
      "logps/chosen": -220.838134765625,
      "logps/rejected": -291.26788330078125,
      "loss": 0.2393,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.652218222618103,
      "rewards/margins": 2.579854726791382,
      "rewards/rejected": -3.2320728302001953,
      "step": 210
    },
    {
      "epoch": 0.45866666666666667,
      "grad_norm": 3.069420337677002,
      "learning_rate": 4.808595544583578e-06,
      "logits/chosen": -1.6939719915390015,
      "logits/rejected": -1.6586183309555054,
      "logps/chosen": -225.1257781982422,
      "logps/rejected": -302.8365478515625,
      "loss": 0.274,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.784705638885498,
      "rewards/margins": 2.293743371963501,
      "rewards/rejected": -3.078449010848999,
      "step": 215
    },
    {
      "epoch": 0.4693333333333333,
      "grad_norm": 3.317854404449463,
      "learning_rate": 4.797394529050577e-06,
      "logits/chosen": -1.5986617803573608,
      "logits/rejected": -1.5012736320495605,
      "logps/chosen": -250.2396697998047,
      "logps/rejected": -291.0882263183594,
      "loss": 0.276,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.6932470798492432,
      "rewards/margins": 2.4106459617614746,
      "rewards/rejected": -3.1038930416107178,
      "step": 220
    },
    {
      "epoch": 0.48,
      "grad_norm": 14.15256118774414,
      "learning_rate": 4.785888835026393e-06,
      "logits/chosen": -1.5902974605560303,
      "logits/rejected": -1.534092664718628,
      "logps/chosen": -229.6944580078125,
      "logps/rejected": -281.76263427734375,
      "loss": 0.2861,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.8665626645088196,
      "rewards/margins": 2.3249261379241943,
      "rewards/rejected": -3.1914889812469482,
      "step": 225
    },
    {
      "epoch": 0.49066666666666664,
      "grad_norm": 5.102518081665039,
      "learning_rate": 4.7740799883862966e-06,
      "logits/chosen": -1.6663917303085327,
      "logits/rejected": -1.5469465255737305,
      "logps/chosen": -239.5758819580078,
      "logps/rejected": -291.83526611328125,
      "loss": 0.3142,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.139182686805725,
      "rewards/margins": 2.0994646549224854,
      "rewards/rejected": -3.2386474609375,
      "step": 230
    },
    {
      "epoch": 0.5013333333333333,
      "grad_norm": 2.747270345687866,
      "learning_rate": 4.761969555209394e-06,
      "logits/chosen": -1.6959060430526733,
      "logits/rejected": -1.7034603357315063,
      "logps/chosen": -218.6403350830078,
      "logps/rejected": -296.04327392578125,
      "loss": 0.3023,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.0236613750457764,
      "rewards/margins": 2.3380959033966064,
      "rewards/rejected": -3.361757278442383,
      "step": 235
    },
    {
      "epoch": 0.512,
      "grad_norm": 2.4223251342773438,
      "learning_rate": 4.7495591415709435e-06,
      "logits/chosen": -1.7274869680404663,
      "logits/rejected": -1.5388081073760986,
      "logps/chosen": -227.67236328125,
      "logps/rejected": -291.2453308105469,
      "loss": 0.2354,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.7354499101638794,
      "rewards/margins": 2.73541522026062,
      "rewards/rejected": -3.470865249633789,
      "step": 240
    },
    {
      "epoch": 0.5226666666666666,
      "grad_norm": 4.390448570251465,
      "learning_rate": 4.736850393329353e-06,
      "logits/chosen": -1.6266454458236694,
      "logits/rejected": -1.512935996055603,
      "logps/chosen": -244.3636474609375,
      "logps/rejected": -295.836181640625,
      "loss": 0.2993,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.9331833124160767,
      "rewards/margins": 2.6201255321502686,
      "rewards/rejected": -3.5533084869384766,
      "step": 245
    },
    {
      "epoch": 0.5333333333333333,
      "grad_norm": 3.507237672805786,
      "learning_rate": 4.7238449959079095e-06,
      "logits/chosen": -1.753446340560913,
      "logits/rejected": -1.6419095993041992,
      "logps/chosen": -233.2483367919922,
      "logps/rejected": -294.78887939453125,
      "loss": 0.2506,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.8268024325370789,
      "rewards/margins": 2.761507272720337,
      "rewards/rejected": -3.5883095264434814,
      "step": 250
    },
    {
      "epoch": 0.544,
      "grad_norm": 2.5926451683044434,
      "learning_rate": 4.710544674071259e-06,
      "logits/chosen": -1.6371052265167236,
      "logits/rejected": -1.5320228338241577,
      "logps/chosen": -230.4346923828125,
      "logps/rejected": -286.22198486328125,
      "loss": 0.2302,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7810002565383911,
      "rewards/margins": 2.938009262084961,
      "rewards/rejected": -3.7190093994140625,
      "step": 255
    },
    {
      "epoch": 0.5546666666666666,
      "grad_norm": 2.7341744899749756,
      "learning_rate": 4.696951191696673e-06,
      "logits/chosen": -1.6878139972686768,
      "logits/rejected": -1.575380563735962,
      "logps/chosen": -234.5509490966797,
      "logps/rejected": -295.4001159667969,
      "loss": 0.2846,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.2288405895233154,
      "rewards/margins": 2.4302990436553955,
      "rewards/rejected": -3.659140110015869,
      "step": 260
    },
    {
      "epoch": 0.5653333333333334,
      "grad_norm": 3.923440933227539,
      "learning_rate": 4.6830663515401196e-06,
      "logits/chosen": -1.7157907485961914,
      "logits/rejected": -1.5281307697296143,
      "logps/chosen": -255.41696166992188,
      "logps/rejected": -287.6669006347656,
      "loss": 0.2849,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.8545368909835815,
      "rewards/margins": 2.664400815963745,
      "rewards/rejected": -3.518937587738037,
      "step": 265
    },
    {
      "epoch": 0.576,
      "grad_norm": 2.7870829105377197,
      "learning_rate": 4.6688919949971855e-06,
      "logits/chosen": -1.6280025243759155,
      "logits/rejected": -1.5273808240890503,
      "logps/chosen": -256.6709899902344,
      "logps/rejected": -299.0733947753906,
      "loss": 0.2483,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.799675703048706,
      "rewards/margins": 2.8166801929473877,
      "rewards/rejected": -3.6163558959960938,
      "step": 270
    },
    {
      "epoch": 0.5866666666666667,
      "grad_norm": 3.013019561767578,
      "learning_rate": 4.654430001858874e-06,
      "logits/chosen": -1.5734882354736328,
      "logits/rejected": -1.5436983108520508,
      "logps/chosen": -248.3529815673828,
      "logps/rejected": -302.51617431640625,
      "loss": 0.2643,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.6249435544013977,
      "rewards/margins": 2.736476421356201,
      "rewards/rejected": -3.361420154571533,
      "step": 275
    },
    {
      "epoch": 0.5973333333333334,
      "grad_norm": 6.8396077156066895,
      "learning_rate": 4.6396822900623075e-06,
      "logits/chosen": -1.6659371852874756,
      "logits/rejected": -1.506557822227478,
      "logps/chosen": -238.4332733154297,
      "logps/rejected": -282.52227783203125,
      "loss": 0.3028,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.7782389521598816,
      "rewards/margins": 2.5956223011016846,
      "rewards/rejected": -3.373861312866211,
      "step": 280
    },
    {
      "epoch": 0.608,
      "grad_norm": 3.025742292404175,
      "learning_rate": 4.624650815436368e-06,
      "logits/chosen": -1.6658992767333984,
      "logits/rejected": -1.6078733205795288,
      "logps/chosen": -248.2996368408203,
      "logps/rejected": -303.9361572265625,
      "loss": 0.2461,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.6723818778991699,
      "rewards/margins": 2.8119044303894043,
      "rewards/rejected": -3.4842865467071533,
      "step": 285
    },
    {
      "epoch": 0.6186666666666667,
      "grad_norm": 2.623469591140747,
      "learning_rate": 4.609337571442324e-06,
      "logits/chosen": -1.6829065084457397,
      "logits/rejected": -1.5887826681137085,
      "logps/chosen": -226.87155151367188,
      "logps/rejected": -293.0638732910156,
      "loss": 0.2396,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.7441937923431396,
      "rewards/margins": 2.8528599739074707,
      "rewards/rejected": -3.5970540046691895,
      "step": 290
    },
    {
      "epoch": 0.6293333333333333,
      "grad_norm": 3.053109645843506,
      "learning_rate": 4.5937445889094535e-06,
      "logits/chosen": -1.663840651512146,
      "logits/rejected": -1.557969331741333,
      "logps/chosen": -237.5575714111328,
      "logps/rejected": -270.58392333984375,
      "loss": 0.3104,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.4625615179538727,
      "rewards/margins": 2.317203998565674,
      "rewards/rejected": -2.7797656059265137,
      "step": 295
    },
    {
      "epoch": 0.64,
      "grad_norm": 2.9096717834472656,
      "learning_rate": 4.577873935765722e-06,
      "logits/chosen": -1.7499110698699951,
      "logits/rejected": -1.525377631187439,
      "logps/chosen": -231.7384490966797,
      "logps/rejected": -274.1564636230469,
      "loss": 0.2208,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.593193531036377,
      "rewards/margins": 2.9766085147857666,
      "rewards/rejected": -3.5698020458221436,
      "step": 300
    },
    {
      "epoch": 0.6506666666666666,
      "grad_norm": 3.812568426132202,
      "learning_rate": 4.561727716763532e-06,
      "logits/chosen": -1.6329309940338135,
      "logits/rejected": -1.5051950216293335,
      "logps/chosen": -255.3319091796875,
      "logps/rejected": -280.5289001464844,
      "loss": 0.2537,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.8085314035415649,
      "rewards/margins": 2.630579948425293,
      "rewards/rejected": -3.4391112327575684,
      "step": 305
    },
    {
      "epoch": 0.6613333333333333,
      "grad_norm": 4.5825605392456055,
      "learning_rate": 4.545308073200591e-06,
      "logits/chosen": -1.618645429611206,
      "logits/rejected": -1.5387064218521118,
      "logps/chosen": -246.5149383544922,
      "logps/rejected": -281.2253112792969,
      "loss": 0.2933,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.7762895822525024,
      "rewards/margins": 2.5413150787353516,
      "rewards/rejected": -3.3176045417785645,
      "step": 310
    },
    {
      "epoch": 0.672,
      "grad_norm": 3.3296334743499756,
      "learning_rate": 4.528617182635943e-06,
      "logits/chosen": -1.7562048435211182,
      "logits/rejected": -1.7319389581680298,
      "logps/chosen": -232.3672637939453,
      "logps/rejected": -317.72369384765625,
      "loss": 0.2532,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.7930496335029602,
      "rewards/margins": 3.444871425628662,
      "rewards/rejected": -4.237921714782715,
      "step": 315
    },
    {
      "epoch": 0.6826666666666666,
      "grad_norm": 6.051812648773193,
      "learning_rate": 4.511657258601171e-06,
      "logits/chosen": -1.6694326400756836,
      "logits/rejected": -1.5220441818237305,
      "logps/chosen": -235.4072265625,
      "logps/rejected": -296.1033020019531,
      "loss": 0.2432,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.9599354863166809,
      "rewards/margins": 3.038886308670044,
      "rewards/rejected": -3.998821973800659,
      "step": 320
    },
    {
      "epoch": 0.6933333333333334,
      "grad_norm": 5.517160892486572,
      "learning_rate": 4.494430550306848e-06,
      "logits/chosen": -1.5575799942016602,
      "logits/rejected": -1.5139790773391724,
      "logps/chosen": -254.32772827148438,
      "logps/rejected": -290.1891174316406,
      "loss": 0.3125,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.8352974057197571,
      "rewards/margins": 2.8006937503814697,
      "rewards/rejected": -3.635990858078003,
      "step": 325
    },
    {
      "epoch": 0.704,
      "grad_norm": 2.6814022064208984,
      "learning_rate": 4.476939342344246e-06,
      "logits/chosen": -1.7075474262237549,
      "logits/rejected": -1.6463184356689453,
      "logps/chosen": -248.3602294921875,
      "logps/rejected": -305.9772033691406,
      "loss": 0.2713,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.1044809818267822,
      "rewards/margins": 2.6876957416534424,
      "rewards/rejected": -3.7921767234802246,
      "step": 330
    },
    {
      "epoch": 0.7146666666666667,
      "grad_norm": 6.655895709991455,
      "learning_rate": 4.45918595438236e-06,
      "logits/chosen": -1.6074548959732056,
      "logits/rejected": -1.5420688390731812,
      "logps/chosen": -235.84518432617188,
      "logps/rejected": -275.97943115234375,
      "loss": 0.3032,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.7538453340530396,
      "rewards/margins": 2.4696977138519287,
      "rewards/rejected": -3.223543643951416,
      "step": 335
    },
    {
      "epoch": 0.7253333333333334,
      "grad_norm": 2.56044602394104,
      "learning_rate": 4.441172740860267e-06,
      "logits/chosen": -1.6362565755844116,
      "logits/rejected": -1.5230262279510498,
      "logps/chosen": -252.67575073242188,
      "logps/rejected": -284.8129577636719,
      "loss": 0.2388,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.6972413063049316,
      "rewards/margins": 2.9790282249450684,
      "rewards/rejected": -3.676270008087158,
      "step": 340
    },
    {
      "epoch": 0.736,
      "grad_norm": 4.380439281463623,
      "learning_rate": 4.422902090674886e-06,
      "logits/chosen": -1.5679028034210205,
      "logits/rejected": -1.4958457946777344,
      "logps/chosen": -256.5689697265625,
      "logps/rejected": -279.14691162109375,
      "loss": 0.2577,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.9045771360397339,
      "rewards/margins": 2.8589415550231934,
      "rewards/rejected": -3.763519287109375,
      "step": 345
    },
    {
      "epoch": 0.7466666666666667,
      "grad_norm": 3.181389808654785,
      "learning_rate": 4.4043764268641705e-06,
      "logits/chosen": -1.6590477228164673,
      "logits/rejected": -1.5410702228546143,
      "logps/chosen": -251.5687255859375,
      "logps/rejected": -301.7498779296875,
      "loss": 0.2393,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.5267749428749084,
      "rewards/margins": 3.0952720642089844,
      "rewards/rejected": -3.622046947479248,
      "step": 350
    },
    {
      "epoch": 0.7573333333333333,
      "grad_norm": 3.395481824874878,
      "learning_rate": 4.385598206285758e-06,
      "logits/chosen": -1.7113549709320068,
      "logits/rejected": -1.656198501586914,
      "logps/chosen": -247.5892791748047,
      "logps/rejected": -303.3941345214844,
      "loss": 0.275,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.4766809344291687,
      "rewards/margins": 2.8971753120422363,
      "rewards/rejected": -3.3738560676574707,
      "step": 355
    },
    {
      "epoch": 0.768,
      "grad_norm": 71.72124481201172,
      "learning_rate": 4.3665699192911495e-06,
      "logits/chosen": -1.5628933906555176,
      "logits/rejected": -1.5330002307891846,
      "logps/chosen": -289.46942138671875,
      "logps/rejected": -281.2159423828125,
      "loss": 0.2754,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.38477277755737305,
      "rewards/margins": 2.6930577754974365,
      "rewards/rejected": -3.0778305530548096,
      "step": 360
    },
    {
      "epoch": 0.7786666666666666,
      "grad_norm": 2.965637445449829,
      "learning_rate": 4.347294089395438e-06,
      "logits/chosen": -1.672273874282837,
      "logits/rejected": -1.5576097965240479,
      "logps/chosen": -253.80630493164062,
      "logps/rejected": -281.87152099609375,
      "loss": 0.2398,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.35890644788742065,
      "rewards/margins": 2.8527824878692627,
      "rewards/rejected": -3.211688995361328,
      "step": 365
    },
    {
      "epoch": 0.7893333333333333,
      "grad_norm": 4.277189254760742,
      "learning_rate": 4.327773272942647e-06,
      "logits/chosen": -1.6600908041000366,
      "logits/rejected": -1.590300440788269,
      "logps/chosen": -231.3988800048828,
      "logps/rejected": -298.57745361328125,
      "loss": 0.2088,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.40050333738327026,
      "rewards/margins": 3.0887906551361084,
      "rewards/rejected": -3.4892940521240234,
      "step": 370
    },
    {
      "epoch": 0.8,
      "grad_norm": 2.961402654647827,
      "learning_rate": 4.308010058766703e-06,
      "logits/chosen": -1.660800576210022,
      "logits/rejected": -1.5310600996017456,
      "logps/chosen": -255.83114624023438,
      "logps/rejected": -295.5674743652344,
      "loss": 0.2092,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.6556711792945862,
      "rewards/margins": 3.229142427444458,
      "rewards/rejected": -3.8848137855529785,
      "step": 375
    },
    {
      "epoch": 0.8106666666666666,
      "grad_norm": 3.6059043407440186,
      "learning_rate": 4.2880070678481135e-06,
      "logits/chosen": -1.7003570795059204,
      "logits/rejected": -1.5770372152328491,
      "logps/chosen": -223.58554077148438,
      "logps/rejected": -292.36114501953125,
      "loss": 0.2584,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6750256419181824,
      "rewards/margins": 3.2846837043762207,
      "rewards/rejected": -3.9597091674804688,
      "step": 380
    },
    {
      "epoch": 0.8213333333333334,
      "grad_norm": 3.216796875,
      "learning_rate": 4.267766952966369e-06,
      "logits/chosen": -1.6272624731063843,
      "logits/rejected": -1.571340799331665,
      "logps/chosen": -241.29873657226562,
      "logps/rejected": -301.0836486816406,
      "loss": 0.2603,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.8578468561172485,
      "rewards/margins": 3.2342536449432373,
      "rewards/rejected": -4.092100620269775,
      "step": 385
    },
    {
      "epoch": 0.832,
      "grad_norm": 4.120914936065674,
      "learning_rate": 4.24729239834814e-06,
      "logits/chosen": -1.626132607460022,
      "logits/rejected": -1.52275550365448,
      "logps/chosen": -259.8028869628906,
      "logps/rejected": -295.94512939453125,
      "loss": 0.281,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6032441854476929,
      "rewards/margins": 3.1042075157165527,
      "rewards/rejected": -3.707451581954956,
      "step": 390
    },
    {
      "epoch": 0.8426666666666667,
      "grad_norm": 3.3638036251068115,
      "learning_rate": 4.226586119311291e-06,
      "logits/chosen": -1.6753953695297241,
      "logits/rejected": -1.6117645502090454,
      "logps/chosen": -241.83935546875,
      "logps/rejected": -291.6381530761719,
      "loss": 0.2388,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.4112634062767029,
      "rewards/margins": 3.3937580585479736,
      "rewards/rejected": -3.8050217628479004,
      "step": 395
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 2.242154359817505,
      "learning_rate": 4.205650861904782e-06,
      "logits/chosen": -1.6429239511489868,
      "logits/rejected": -1.5123769044876099,
      "logps/chosen": -222.09957885742188,
      "logps/rejected": -289.91717529296875,
      "loss": 0.2233,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.8481672406196594,
      "rewards/margins": 3.0394561290740967,
      "rewards/rejected": -3.8876235485076904,
      "step": 400
    },
    {
      "epoch": 0.864,
      "grad_norm": 4.096096992492676,
      "learning_rate": 4.184489402544486e-06,
      "logits/chosen": -1.7357295751571655,
      "logits/rejected": -1.6345306634902954,
      "logps/chosen": -232.3779754638672,
      "logps/rejected": -294.068115234375,
      "loss": 0.2335,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.7445335388183594,
      "rewards/margins": 3.0647544860839844,
      "rewards/rejected": -3.809288501739502,
      "step": 405
    },
    {
      "epoch": 0.8746666666666667,
      "grad_norm": 3.3298556804656982,
      "learning_rate": 4.1631045476449865e-06,
      "logits/chosen": -1.7491382360458374,
      "logits/rejected": -1.6363424062728882,
      "logps/chosen": -240.1590576171875,
      "logps/rejected": -309.88482666015625,
      "loss": 0.2004,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.7465359568595886,
      "rewards/margins": 3.701233386993408,
      "rewards/rejected": -4.4477691650390625,
      "step": 410
    },
    {
      "epoch": 0.8853333333333333,
      "grad_norm": 2.753004789352417,
      "learning_rate": 4.141499133247391e-06,
      "logits/chosen": -1.5934135913848877,
      "logits/rejected": -1.5748265981674194,
      "logps/chosen": -242.0540771484375,
      "logps/rejected": -309.4254455566406,
      "loss": 0.2532,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.6847050786018372,
      "rewards/margins": 3.4728336334228516,
      "rewards/rejected": -4.157538414001465,
      "step": 415
    },
    {
      "epoch": 0.896,
      "grad_norm": 3.4208219051361084,
      "learning_rate": 4.119676024643217e-06,
      "logits/chosen": -1.5917304754257202,
      "logits/rejected": -1.5950918197631836,
      "logps/chosen": -243.2762908935547,
      "logps/rejected": -299.3033447265625,
      "loss": 0.1765,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.5414136648178101,
      "rewards/margins": 3.837564468383789,
      "rewards/rejected": -4.378978252410889,
      "step": 420
    },
    {
      "epoch": 0.9066666666666666,
      "grad_norm": 5.384462833404541,
      "learning_rate": 4.0976381159944055e-06,
      "logits/chosen": -1.703701376914978,
      "logits/rejected": -1.5836318731307983,
      "logps/chosen": -228.7626190185547,
      "logps/rejected": -318.1847229003906,
      "loss": 0.2142,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.7460819482803345,
      "rewards/margins": 3.7437853813171387,
      "rewards/rejected": -4.489867210388184,
      "step": 425
    },
    {
      "epoch": 0.9173333333333333,
      "grad_norm": 2.9159977436065674,
      "learning_rate": 4.07538832994949e-06,
      "logits/chosen": -1.6830871105194092,
      "logits/rejected": -1.5406839847564697,
      "logps/chosen": -243.9796142578125,
      "logps/rejected": -309.07666015625,
      "loss": 0.2847,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.0015039443969727,
      "rewards/margins": 3.680140733718872,
      "rewards/rejected": -4.681644439697266,
      "step": 430
    },
    {
      "epoch": 0.928,
      "grad_norm": 3.914306402206421,
      "learning_rate": 4.0529296172560055e-06,
      "logits/chosen": -1.7112497091293335,
      "logits/rejected": -1.5582363605499268,
      "logps/chosen": -237.08541870117188,
      "logps/rejected": -321.8060302734375,
      "loss": 0.257,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2424498796463013,
      "rewards/margins": 3.5792899131774902,
      "rewards/rejected": -4.82174015045166,
      "step": 435
    },
    {
      "epoch": 0.9386666666666666,
      "grad_norm": 4.649114608764648,
      "learning_rate": 4.030264956369158e-06,
      "logits/chosen": -1.5580627918243408,
      "logits/rejected": -1.4607374668121338,
      "logps/chosen": -234.0313262939453,
      "logps/rejected": -297.1539306640625,
      "loss": 0.2303,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.0807582139968872,
      "rewards/margins": 3.3951945304870605,
      "rewards/rejected": -4.475952625274658,
      "step": 440
    },
    {
      "epoch": 0.9493333333333334,
      "grad_norm": 4.41303014755249,
      "learning_rate": 4.0073973530568274e-06,
      "logits/chosen": -1.495747447013855,
      "logits/rejected": -1.4580141305923462,
      "logps/chosen": -226.8778839111328,
      "logps/rejected": -291.4224548339844,
      "loss": 0.2653,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.8967695236206055,
      "rewards/margins": 3.4394097328186035,
      "rewards/rejected": -4.336178779602051,
      "step": 445
    },
    {
      "epoch": 0.96,
      "grad_norm": 2.877248525619507,
      "learning_rate": 3.9843298400009435e-06,
      "logits/chosen": -1.7053565979003906,
      "logits/rejected": -1.6477537155151367,
      "logps/chosen": -239.79800415039062,
      "logps/rejected": -309.544189453125,
      "loss": 0.2526,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.0235546827316284,
      "rewards/margins": 3.4787325859069824,
      "rewards/rejected": -4.502286911010742,
      "step": 450
    },
    {
      "epoch": 0.9706666666666667,
      "grad_norm": 6.4316725730896,
      "learning_rate": 3.961065476395296e-06,
      "logits/chosen": -1.768538236618042,
      "logits/rejected": -1.632031798362732,
      "logps/chosen": -250.4674530029297,
      "logps/rejected": -336.96759033203125,
      "loss": 0.1837,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.5413009524345398,
      "rewards/margins": 4.095879554748535,
      "rewards/rejected": -4.637180805206299,
      "step": 455
    },
    {
      "epoch": 0.9813333333333333,
      "grad_norm": 3.302759885787964,
      "learning_rate": 3.937607347539823e-06,
      "logits/chosen": -1.685744047164917,
      "logits/rejected": -1.5522499084472656,
      "logps/chosen": -227.767578125,
      "logps/rejected": -318.6248474121094,
      "loss": 0.2087,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.8970295190811157,
      "rewards/margins": 3.9703762531280518,
      "rewards/rejected": -4.867405891418457,
      "step": 460
    },
    {
      "epoch": 0.992,
      "grad_norm": 4.109023571014404,
      "learning_rate": 3.913958564431447e-06,
      "logits/chosen": -1.8247349262237549,
      "logits/rejected": -1.6116182804107666,
      "logps/chosen": -255.1387176513672,
      "logps/rejected": -299.3416748046875,
      "loss": 0.2102,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.7889047861099243,
      "rewards/margins": 4.012124538421631,
      "rewards/rejected": -4.801029682159424,
      "step": 465
    },
    {
      "epoch": 1.0021333333333333,
      "grad_norm": 4.297785758972168,
      "learning_rate": 3.89012226335149e-06,
      "logits/chosen": -1.5690182447433472,
      "logits/rejected": -1.5636067390441895,
      "logps/chosen": -235.01690673828125,
      "logps/rejected": -301.91552734375,
      "loss": 0.2782,
      "rewards/accuracies": 0.8618420958518982,
      "rewards/chosen": -1.062803030014038,
      "rewards/margins": 3.59759521484375,
      "rewards/rejected": -4.660398006439209,
      "step": 470
    },
    {
      "epoch": 1.0128,
      "grad_norm": 2.434410572052002,
      "learning_rate": 3.86610160544975e-06,
      "logits/chosen": -1.6619993448257446,
      "logits/rejected": -1.6416118144989014,
      "logps/chosen": -253.34414672851562,
      "logps/rejected": -312.0550842285156,
      "loss": 0.2136,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.7877097725868225,
      "rewards/margins": 3.9225335121154785,
      "rewards/rejected": -4.71024227142334,
      "step": 475
    },
    {
      "epoch": 1.0234666666666667,
      "grad_norm": 3.1037039756774902,
      "learning_rate": 3.841899776325267e-06,
      "logits/chosen": -1.5927152633666992,
      "logits/rejected": -1.4927324056625366,
      "logps/chosen": -250.84481811523438,
      "logps/rejected": -291.4174499511719,
      "loss": 0.1873,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.5820282101631165,
      "rewards/margins": 3.742619276046753,
      "rewards/rejected": -4.324646949768066,
      "step": 480
    },
    {
      "epoch": 1.0341333333333333,
      "grad_norm": 2.221498489379883,
      "learning_rate": 3.817519985603855e-06,
      "logits/chosen": -1.7725327014923096,
      "logits/rejected": -1.7125329971313477,
      "logps/chosen": -221.89370727539062,
      "logps/rejected": -311.96270751953125,
      "loss": 0.1119,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.5550159215927124,
      "rewards/margins": 4.137165546417236,
      "rewards/rejected": -4.692181587219238,
      "step": 485
    },
    {
      "epoch": 1.0448,
      "grad_norm": 3.163022756576538,
      "learning_rate": 3.792965466512437e-06,
      "logits/chosen": -1.7214984893798828,
      "logits/rejected": -1.6236610412597656,
      "logps/chosen": -243.5190887451172,
      "logps/rejected": -310.9081115722656,
      "loss": 0.2174,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.657336950302124,
      "rewards/margins": 3.4691410064697266,
      "rewards/rejected": -4.1264777183532715,
      "step": 490
    },
    {
      "epoch": 1.0554666666666668,
      "grad_norm": 1.9490315914154053,
      "learning_rate": 3.7682394754502687e-06,
      "logits/chosen": -1.7172479629516602,
      "logits/rejected": -1.6332595348358154,
      "logps/chosen": -227.83163452148438,
      "logps/rejected": -305.6872863769531,
      "loss": 0.189,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.819398045539856,
      "rewards/margins": 4.044649124145508,
      "rewards/rejected": -4.864047050476074,
      "step": 495
    },
    {
      "epoch": 1.0661333333333334,
      "grad_norm": 2.395745277404785,
      "learning_rate": 3.7433452915570685e-06,
      "logits/chosen": -1.7560336589813232,
      "logits/rejected": -1.5769942998886108,
      "logps/chosen": -240.0421905517578,
      "logps/rejected": -305.04827880859375,
      "loss": 0.1668,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.8382463455200195,
      "rewards/margins": 4.010151386260986,
      "rewards/rejected": -4.848397254943848,
      "step": 500
    },
    {
      "epoch": 1.0768,
      "grad_norm": 3.2161705493927,
      "learning_rate": 3.718286216278143e-06,
      "logits/chosen": -1.7935104370117188,
      "logits/rejected": -1.6894598007202148,
      "logps/chosen": -250.8125762939453,
      "logps/rejected": -325.422607421875,
      "loss": 0.1802,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3864238262176514,
      "rewards/margins": 4.100071907043457,
      "rewards/rejected": -5.486495018005371,
      "step": 505
    },
    {
      "epoch": 1.0874666666666666,
      "grad_norm": 3.6139273643493652,
      "learning_rate": 3.6930655729265557e-06,
      "logits/chosen": -1.659446120262146,
      "logits/rejected": -1.6318769454956055,
      "logps/chosen": -251.3518829345703,
      "logps/rejected": -334.08917236328125,
      "loss": 0.1798,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.2920472621917725,
      "rewards/margins": 4.446113586425781,
      "rewards/rejected": -5.738161087036133,
      "step": 510
    },
    {
      "epoch": 1.0981333333333334,
      "grad_norm": 4.280843257904053,
      "learning_rate": 3.667686706242386e-06,
      "logits/chosen": -1.5787420272827148,
      "logits/rejected": -1.4920685291290283,
      "logps/chosen": -254.0977325439453,
      "logps/rejected": -299.56256103515625,
      "loss": 0.2441,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.4882808923721313,
      "rewards/margins": 4.044205188751221,
      "rewards/rejected": -5.5324859619140625,
      "step": 515
    },
    {
      "epoch": 1.1088,
      "grad_norm": 2.1550538539886475,
      "learning_rate": 3.6421529819491576e-06,
      "logits/chosen": -1.6968841552734375,
      "logits/rejected": -1.6490501165390015,
      "logps/chosen": -250.90933227539062,
      "logps/rejected": -326.4671936035156,
      "loss": 0.188,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.3018690347671509,
      "rewards/margins": 4.414233207702637,
      "rewards/rejected": -5.716102600097656,
      "step": 520
    },
    {
      "epoch": 1.1194666666666666,
      "grad_norm": 1.9021104574203491,
      "learning_rate": 3.6164677863074783e-06,
      "logits/chosen": -1.6301161050796509,
      "logits/rejected": -1.6059134006500244,
      "logps/chosen": -228.54275512695312,
      "logps/rejected": -297.1298522949219,
      "loss": 0.2367,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.2790645360946655,
      "rewards/margins": 3.9954991340637207,
      "rewards/rejected": -5.274563789367676,
      "step": 525
    },
    {
      "epoch": 1.1301333333333332,
      "grad_norm": 3.255465030670166,
      "learning_rate": 3.590634525665955e-06,
      "logits/chosen": -1.7874761819839478,
      "logits/rejected": -1.6896089315414429,
      "logps/chosen": -233.70816040039062,
      "logps/rejected": -306.7872314453125,
      "loss": 0.1776,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.035360336303711,
      "rewards/margins": 4.155150413513184,
      "rewards/rejected": -5.190511703491211,
      "step": 530
    },
    {
      "epoch": 1.1408,
      "grad_norm": 2.013275623321533,
      "learning_rate": 3.564656626009446e-06,
      "logits/chosen": -1.8429739475250244,
      "logits/rejected": -1.6779648065567017,
      "logps/chosen": -247.69906616210938,
      "logps/rejected": -327.61865234375,
      "loss": 0.1586,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.3286124467849731,
      "rewards/margins": 4.181220054626465,
      "rewards/rejected": -5.509832382202148,
      "step": 535
    },
    {
      "epoch": 1.1514666666666666,
      "grad_norm": 4.79525089263916,
      "learning_rate": 3.5385375325047167e-06,
      "logits/chosen": -1.753906011581421,
      "logits/rejected": -1.6913907527923584,
      "logps/chosen": -251.79751586914062,
      "logps/rejected": -307.3892822265625,
      "loss": 0.1615,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2266600131988525,
      "rewards/margins": 4.172353267669678,
      "rewards/rejected": -5.399013519287109,
      "step": 540
    },
    {
      "epoch": 1.1621333333333332,
      "grad_norm": 3.9382004737854004,
      "learning_rate": 3.512280709043534e-06,
      "logits/chosen": -1.6286598443984985,
      "logits/rejected": -1.4853745698928833,
      "logps/chosen": -248.4619140625,
      "logps/rejected": -293.0491943359375,
      "loss": 0.2146,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.1233129501342773,
      "rewards/margins": 3.978351593017578,
      "rewards/rejected": -5.101665019989014,
      "step": 545
    },
    {
      "epoch": 1.1728,
      "grad_norm": 4.661540985107422,
      "learning_rate": 3.4858896377832966e-06,
      "logits/chosen": -1.7429920434951782,
      "logits/rejected": -1.6357555389404297,
      "logps/chosen": -245.42489624023438,
      "logps/rejected": -297.8553771972656,
      "loss": 0.1958,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1029307842254639,
      "rewards/margins": 3.8211007118225098,
      "rewards/rejected": -4.9240312576293945,
      "step": 550
    },
    {
      "epoch": 1.1834666666666667,
      "grad_norm": 2.63988995552063,
      "learning_rate": 3.459367818685231e-06,
      "logits/chosen": -1.7078949213027954,
      "logits/rejected": -1.5824353694915771,
      "logps/chosen": -244.9107666015625,
      "logps/rejected": -318.76898193359375,
      "loss": 0.1662,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.956513524055481,
      "rewards/margins": 4.618618011474609,
      "rewards/rejected": -5.575131416320801,
      "step": 555
    },
    {
      "epoch": 1.1941333333333333,
      "grad_norm": 4.890956878662109,
      "learning_rate": 3.432718769050228e-06,
      "logits/chosen": -1.6049926280975342,
      "logits/rejected": -1.4823522567749023,
      "logps/chosen": -243.07571411132812,
      "logps/rejected": -317.7985534667969,
      "loss": 0.1449,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.7478665113449097,
      "rewards/margins": 4.851959228515625,
      "rewards/rejected": -5.599825382232666,
      "step": 560
    },
    {
      "epoch": 1.2048,
      "grad_norm": 3.967740774154663,
      "learning_rate": 3.405946023052386e-06,
      "logits/chosen": -1.7225697040557861,
      "logits/rejected": -1.6431505680084229,
      "logps/chosen": -238.2488555908203,
      "logps/rejected": -307.4492492675781,
      "loss": 0.2182,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.2250194549560547,
      "rewards/margins": 4.405190467834473,
      "rewards/rejected": -5.630209922790527,
      "step": 565
    },
    {
      "epoch": 1.2154666666666667,
      "grad_norm": 3.474419355392456,
      "learning_rate": 3.3790531312703055e-06,
      "logits/chosen": -1.659584403038025,
      "logits/rejected": -1.6280295848846436,
      "logps/chosen": -239.4110107421875,
      "logps/rejected": -326.34466552734375,
      "loss": 0.1538,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.9404169917106628,
      "rewards/margins": 4.299707412719727,
      "rewards/rejected": -5.240124225616455,
      "step": 570
    },
    {
      "epoch": 1.2261333333333333,
      "grad_norm": 4.2344183921813965,
      "learning_rate": 3.35204366021622e-06,
      "logits/chosen": -1.6510120630264282,
      "logits/rejected": -1.5048983097076416,
      "logps/chosen": -240.78897094726562,
      "logps/rejected": -310.3196105957031,
      "loss": 0.2342,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.8230067491531372,
      "rewards/margins": 3.923809051513672,
      "rewards/rejected": -4.7468156814575195,
      "step": 575
    },
    {
      "epoch": 1.2368000000000001,
      "grad_norm": 3.379074811935425,
      "learning_rate": 3.3249211918630055e-06,
      "logits/chosen": -1.7805049419403076,
      "logits/rejected": -1.6977018117904663,
      "logps/chosen": -238.62423706054688,
      "logps/rejected": -301.8321228027344,
      "loss": 0.1984,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.016635537147522,
      "rewards/margins": 4.349543571472168,
      "rewards/rejected": -5.3661789894104,
      "step": 580
    },
    {
      "epoch": 1.2474666666666667,
      "grad_norm": 2.739449977874756,
      "learning_rate": 3.2976893231691424e-06,
      "logits/chosen": -1.6601759195327759,
      "logits/rejected": -1.619744062423706,
      "logps/chosen": -230.4942626953125,
      "logps/rejected": -320.5411071777344,
      "loss": 0.1424,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.6311012506484985,
      "rewards/margins": 4.800417900085449,
      "rewards/rejected": -5.431519508361816,
      "step": 585
    },
    {
      "epoch": 1.2581333333333333,
      "grad_norm": 3.436858654022217,
      "learning_rate": 3.270351665601691e-06,
      "logits/chosen": -1.682958960533142,
      "logits/rejected": -1.6138877868652344,
      "logps/chosen": -236.56436157226562,
      "logps/rejected": -306.29205322265625,
      "loss": 0.1444,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.0519824028015137,
      "rewards/margins": 4.614463806152344,
      "rewards/rejected": -5.666445732116699,
      "step": 590
    },
    {
      "epoch": 1.2688,
      "grad_norm": 1.9738092422485352,
      "learning_rate": 3.2429118446573417e-06,
      "logits/chosen": -1.6976039409637451,
      "logits/rejected": -1.568501353263855,
      "logps/chosen": -239.57424926757812,
      "logps/rejected": -324.35601806640625,
      "loss": 0.1958,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.3311951160430908,
      "rewards/margins": 4.367763042449951,
      "rewards/rejected": -5.698958396911621,
      "step": 595
    },
    {
      "epoch": 1.2794666666666665,
      "grad_norm": 3.606091260910034,
      "learning_rate": 3.215373499381602e-06,
      "logits/chosen": -1.6570765972137451,
      "logits/rejected": -1.557324767112732,
      "logps/chosen": -250.1347198486328,
      "logps/rejected": -305.9236755371094,
      "loss": 0.2033,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.1782742738723755,
      "rewards/margins": 4.263705730438232,
      "rewards/rejected": -5.441979885101318,
      "step": 600
    },
    {
      "epoch": 1.2901333333333334,
      "grad_norm": 4.431084156036377,
      "learning_rate": 3.1877402818861954e-06,
      "logits/chosen": -1.783603310585022,
      "logits/rejected": -1.6870628595352173,
      "logps/chosen": -252.9153289794922,
      "logps/rejected": -331.8052673339844,
      "loss": 0.1427,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4646859169006348,
      "rewards/margins": 4.739185333251953,
      "rewards/rejected": -6.203871250152588,
      "step": 605
    },
    {
      "epoch": 1.3008,
      "grad_norm": 2.1071999073028564,
      "learning_rate": 3.1600158568647145e-06,
      "logits/chosen": -1.6523160934448242,
      "logits/rejected": -1.64646315574646,
      "logps/chosen": -246.16506958007812,
      "logps/rejected": -309.9844665527344,
      "loss": 0.2131,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.479095220565796,
      "rewards/margins": 4.29295015335083,
      "rewards/rejected": -5.772045135498047,
      "step": 610
    },
    {
      "epoch": 1.3114666666666666,
      "grad_norm": 2.0734453201293945,
      "learning_rate": 3.1322039011066172e-06,
      "logits/chosen": -1.5979714393615723,
      "logits/rejected": -1.5535624027252197,
      "logps/chosen": -251.6847686767578,
      "logps/rejected": -300.205810546875,
      "loss": 0.1771,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.2251965999603271,
      "rewards/margins": 4.506723880767822,
      "rewards/rejected": -5.7319207191467285,
      "step": 615
    },
    {
      "epoch": 1.3221333333333334,
      "grad_norm": 3.522108554840088,
      "learning_rate": 3.1043081030096125e-06,
      "logits/chosen": -1.583038091659546,
      "logits/rejected": -1.4954283237457275,
      "logps/chosen": -244.6840362548828,
      "logps/rejected": -288.87139892578125,
      "loss": 0.1722,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.3493521213531494,
      "rewards/margins": 4.36538028717041,
      "rewards/rejected": -5.7147321701049805,
      "step": 620
    },
    {
      "epoch": 1.3328,
      "grad_norm": 3.5037546157836914,
      "learning_rate": 3.0763321620905072e-06,
      "logits/chosen": -1.68924880027771,
      "logits/rejected": -1.6171550750732422,
      "logps/chosen": -247.78756713867188,
      "logps/rejected": -317.5525207519531,
      "loss": 0.1445,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.0897579193115234,
      "rewards/margins": 4.7230424880981445,
      "rewards/rejected": -5.812800407409668,
      "step": 625
    },
    {
      "epoch": 1.3434666666666666,
      "grad_norm": 3.3278591632843018,
      "learning_rate": 3.048279788494583e-06,
      "logits/chosen": -1.7668014764785767,
      "logits/rejected": -1.611505150794983,
      "logps/chosen": -235.28744506835938,
      "logps/rejected": -308.88848876953125,
      "loss": 0.1671,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3909533023834229,
      "rewards/margins": 4.262879848480225,
      "rewards/rejected": -5.653833389282227,
      "step": 630
    },
    {
      "epoch": 1.3541333333333334,
      "grad_norm": 2.192690372467041,
      "learning_rate": 3.0201547025035553e-06,
      "logits/chosen": -1.7089593410491943,
      "logits/rejected": -1.5918943881988525,
      "logps/chosen": -247.35800170898438,
      "logps/rejected": -318.1089172363281,
      "loss": 0.1385,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.4722728729248047,
      "rewards/margins": 4.720250129699707,
      "rewards/rejected": -6.192523002624512,
      "step": 635
    },
    {
      "epoch": 1.3648,
      "grad_norm": 7.483966827392578,
      "learning_rate": 2.9919606340421963e-06,
      "logits/chosen": -1.6352918148040771,
      "logits/rejected": -1.5761418342590332,
      "logps/chosen": -245.5006103515625,
      "logps/rejected": -314.25115966796875,
      "loss": 0.1857,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.7750804424285889,
      "rewards/margins": 4.828191757202148,
      "rewards/rejected": -6.603272914886475,
      "step": 640
    },
    {
      "epoch": 1.3754666666666666,
      "grad_norm": 3.6300911903381348,
      "learning_rate": 2.9637013221836723e-06,
      "logits/chosen": -1.6962321996688843,
      "logits/rejected": -1.5770907402038574,
      "logps/chosen": -268.17938232421875,
      "logps/rejected": -325.52020263671875,
      "loss": 0.2036,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.6570656299591064,
      "rewards/margins": 4.438990592956543,
      "rewards/rejected": -6.0960564613342285,
      "step": 645
    },
    {
      "epoch": 1.3861333333333334,
      "grad_norm": 5.613654613494873,
      "learning_rate": 2.9353805146536762e-06,
      "logits/chosen": -1.6913630962371826,
      "logits/rejected": -1.583839774131775,
      "logps/chosen": -252.5726776123047,
      "logps/rejected": -320.84478759765625,
      "loss": 0.1636,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.6463943719863892,
      "rewards/margins": 4.656146049499512,
      "rewards/rejected": -6.302539825439453,
      "step": 650
    },
    {
      "epoch": 1.3968,
      "grad_norm": 1.9233977794647217,
      "learning_rate": 2.907001967333401e-06,
      "logits/chosen": -1.7078460454940796,
      "logits/rejected": -1.6775310039520264,
      "logps/chosen": -257.7984924316406,
      "logps/rejected": -340.76470947265625,
      "loss": 0.1728,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.483618140220642,
      "rewards/margins": 4.942142009735107,
      "rewards/rejected": -6.425759792327881,
      "step": 655
    },
    {
      "epoch": 1.4074666666666666,
      "grad_norm": 3.366171360015869,
      "learning_rate": 2.878569443761442e-06,
      "logits/chosen": -1.6747850179672241,
      "logits/rejected": -1.621994972229004,
      "logps/chosen": -237.207275390625,
      "logps/rejected": -312.04949951171875,
      "loss": 0.2247,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.444260835647583,
      "rewards/margins": 4.030089378356934,
      "rewards/rejected": -5.474350452423096,
      "step": 660
    },
    {
      "epoch": 1.4181333333333335,
      "grad_norm": 3.9642248153686523,
      "learning_rate": 2.85008671463468e-06,
      "logits/chosen": -1.74550461769104,
      "logits/rejected": -1.6358397006988525,
      "logps/chosen": -243.82864379882812,
      "logps/rejected": -310.81390380859375,
      "loss": 0.1717,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.506588101387024,
      "rewards/margins": 4.499510765075684,
      "rewards/rejected": -6.006098747253418,
      "step": 665
    },
    {
      "epoch": 1.4288,
      "grad_norm": 2.471996307373047,
      "learning_rate": 2.8215575573082127e-06,
      "logits/chosen": -1.6209523677825928,
      "logits/rejected": -1.6456940174102783,
      "logps/chosen": -247.5177764892578,
      "logps/rejected": -340.74420166015625,
      "loss": 0.1342,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.7314220666885376,
      "rewards/margins": 4.739981651306152,
      "rewards/rejected": -6.471404075622559,
      "step": 670
    },
    {
      "epoch": 1.4394666666666667,
      "grad_norm": 3.4441559314727783,
      "learning_rate": 2.792985755294404e-06,
      "logits/chosen": -1.8194706439971924,
      "logits/rejected": -1.696157455444336,
      "logps/chosen": -232.4257049560547,
      "logps/rejected": -337.1112365722656,
      "loss": 0.1154,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.5092469453811646,
      "rewards/margins": 5.188037872314453,
      "rewards/rejected": -6.697284698486328,
      "step": 675
    },
    {
      "epoch": 1.4501333333333333,
      "grad_norm": 1.8599908351898193,
      "learning_rate": 2.764375097761122e-06,
      "logits/chosen": -1.6593608856201172,
      "logits/rejected": -1.546225905418396,
      "logps/chosen": -250.6788787841797,
      "logps/rejected": -318.9678649902344,
      "loss": 0.1712,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.2948193550109863,
      "rewards/margins": 4.342173099517822,
      "rewards/rejected": -5.636992454528809,
      "step": 680
    },
    {
      "epoch": 1.4607999999999999,
      "grad_norm": 2.777001142501831,
      "learning_rate": 2.73572937902922e-06,
      "logits/chosen": -1.7056080102920532,
      "logits/rejected": -1.5910823345184326,
      "logps/chosen": -234.8707275390625,
      "logps/rejected": -323.86993408203125,
      "loss": 0.1672,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.3926390409469604,
      "rewards/margins": 4.474860191345215,
      "rewards/rejected": -5.867499351501465,
      "step": 685
    },
    {
      "epoch": 1.4714666666666667,
      "grad_norm": 2.0958902835845947,
      "learning_rate": 2.7070523980693362e-06,
      "logits/chosen": -1.5597681999206543,
      "logits/rejected": -1.4919359683990479,
      "logps/chosen": -258.60162353515625,
      "logps/rejected": -311.6105651855469,
      "loss": 0.1274,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.177411675453186,
      "rewards/margins": 4.901284217834473,
      "rewards/rejected": -6.078696250915527,
      "step": 690
    },
    {
      "epoch": 1.4821333333333333,
      "grad_norm": 4.915290832519531,
      "learning_rate": 2.678347957998081e-06,
      "logits/chosen": -1.740426778793335,
      "logits/rejected": -1.652065634727478,
      "logps/chosen": -250.73391723632812,
      "logps/rejected": -322.7600402832031,
      "loss": 0.1315,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.6637054681777954,
      "rewards/margins": 5.201625823974609,
      "rewards/rejected": -6.865331172943115,
      "step": 695
    },
    {
      "epoch": 1.4928,
      "grad_norm": 1.5418274402618408,
      "learning_rate": 2.649619865573666e-06,
      "logits/chosen": -1.5983693599700928,
      "logits/rejected": -1.5417187213897705,
      "logps/chosen": -245.4449462890625,
      "logps/rejected": -339.23028564453125,
      "loss": 0.1787,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.7123286724090576,
      "rewards/margins": 5.181328296661377,
      "rewards/rejected": -6.8936567306518555,
      "step": 700
    },
    {
      "epoch": 1.5034666666666667,
      "grad_norm": 2.4393539428710938,
      "learning_rate": 2.620871930691058e-06,
      "logits/chosen": -1.6640474796295166,
      "logits/rejected": -1.6505119800567627,
      "logps/chosen": -249.7317352294922,
      "logps/rejected": -331.77996826171875,
      "loss": 0.1866,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.970604658126831,
      "rewards/margins": 4.820348262786865,
      "rewards/rejected": -6.790953159332275,
      "step": 705
    },
    {
      "epoch": 1.5141333333333333,
      "grad_norm": 1.9373661279678345,
      "learning_rate": 2.5921079658767152e-06,
      "logits/chosen": -1.731371283531189,
      "logits/rejected": -1.6103847026824951,
      "logps/chosen": -265.63885498046875,
      "logps/rejected": -307.3009338378906,
      "loss": 0.15,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.724918007850647,
      "rewards/margins": 4.603318214416504,
      "rewards/rejected": -6.3282365798950195,
      "step": 710
    },
    {
      "epoch": 1.5248,
      "grad_norm": 3.6780591011047363,
      "learning_rate": 2.56333178578297e-06,
      "logits/chosen": -1.6388132572174072,
      "logits/rejected": -1.5583611726760864,
      "logps/chosen": -245.09841918945312,
      "logps/rejected": -328.18377685546875,
      "loss": 0.141,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.9475570917129517,
      "rewards/margins": 5.37711238861084,
      "rewards/rejected": -7.32466983795166,
      "step": 715
    },
    {
      "epoch": 1.5354666666666668,
      "grad_norm": 5.639969348907471,
      "learning_rate": 2.534547206682136e-06,
      "logits/chosen": -1.676372766494751,
      "logits/rejected": -1.610573172569275,
      "logps/chosen": -261.094482421875,
      "logps/rejected": -323.09112548828125,
      "loss": 0.1778,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.8512113094329834,
      "rewards/margins": 4.54202127456665,
      "rewards/rejected": -6.3932318687438965,
      "step": 720
    },
    {
      "epoch": 1.5461333333333334,
      "grad_norm": 4.020143508911133,
      "learning_rate": 2.5057580459603947e-06,
      "logits/chosen": -1.7616875171661377,
      "logits/rejected": -1.6894457340240479,
      "logps/chosen": -235.62210083007812,
      "logps/rejected": -308.05975341796875,
      "loss": 0.215,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.8900903463363647,
      "rewards/margins": 4.641322135925293,
      "rewards/rejected": -6.5314130783081055,
      "step": 725
    },
    {
      "epoch": 1.5568,
      "grad_norm": 4.262376308441162,
      "learning_rate": 2.4769681216115384e-06,
      "logits/chosen": -1.7237647771835327,
      "logits/rejected": -1.6486454010009766,
      "logps/chosen": -244.239013671875,
      "logps/rejected": -324.2884216308594,
      "loss": 0.1723,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6464115381240845,
      "rewards/margins": 4.711733818054199,
      "rewards/rejected": -6.358145713806152,
      "step": 730
    },
    {
      "epoch": 1.5674666666666668,
      "grad_norm": 1.481379508972168,
      "learning_rate": 2.448181251730632e-06,
      "logits/chosen": -1.6676714420318604,
      "logits/rejected": -1.584498643875122,
      "logps/chosen": -249.21066284179688,
      "logps/rejected": -323.2672119140625,
      "loss": 0.2,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.6483440399169922,
      "rewards/margins": 4.992440700531006,
      "rewards/rejected": -6.64078426361084,
      "step": 735
    },
    {
      "epoch": 1.5781333333333334,
      "grad_norm": 3.6630077362060547,
      "learning_rate": 2.419401254007658e-06,
      "logits/chosen": -1.5993918180465698,
      "logits/rejected": -1.5106697082519531,
      "logps/chosen": -246.16738891601562,
      "logps/rejected": -303.5965881347656,
      "loss": 0.1411,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.581565260887146,
      "rewards/margins": 4.666982650756836,
      "rewards/rejected": -6.248547554016113,
      "step": 740
    },
    {
      "epoch": 1.5888,
      "grad_norm": 4.977443218231201,
      "learning_rate": 2.390631945221221e-06,
      "logits/chosen": -1.6648800373077393,
      "logits/rejected": -1.535312294960022,
      "logps/chosen": -259.2499084472656,
      "logps/rejected": -321.27313232421875,
      "loss": 0.1764,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.6946322917938232,
      "rewards/margins": 4.95900297164917,
      "rewards/rejected": -6.653635501861572,
      "step": 745
    },
    {
      "epoch": 1.5994666666666668,
      "grad_norm": 3.6690218448638916,
      "learning_rate": 2.361877140732366e-06,
      "logits/chosen": -1.7277066707611084,
      "logits/rejected": -1.6841871738433838,
      "logps/chosen": -224.5117950439453,
      "logps/rejected": -330.66082763671875,
      "loss": 0.1727,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3350045680999756,
      "rewards/margins": 5.390060901641846,
      "rewards/rejected": -6.7250657081604,
      "step": 750
    },
    {
      "epoch": 1.6101333333333332,
      "grad_norm": 3.617159843444824,
      "learning_rate": 2.333140653978592e-06,
      "logits/chosen": -1.7293851375579834,
      "logits/rejected": -1.5459930896759033,
      "logps/chosen": -260.37548828125,
      "logps/rejected": -319.6600646972656,
      "loss": 0.1591,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.4235215187072754,
      "rewards/margins": 4.943792343139648,
      "rewards/rejected": -6.367314338684082,
      "step": 755
    },
    {
      "epoch": 1.6208,
      "grad_norm": 3.683688163757324,
      "learning_rate": 2.3044262959681156e-06,
      "logits/chosen": -1.6138556003570557,
      "logits/rejected": -1.472929835319519,
      "logps/chosen": -256.35125732421875,
      "logps/rejected": -320.8252868652344,
      "loss": 0.1434,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.7510179281234741,
      "rewards/margins": 5.087538242340088,
      "rewards/rejected": -6.838555812835693,
      "step": 760
    },
    {
      "epoch": 1.6314666666666666,
      "grad_norm": 2.7799344062805176,
      "learning_rate": 2.2757378747744575e-06,
      "logits/chosen": -1.6599740982055664,
      "logits/rejected": -1.521399736404419,
      "logps/chosen": -232.0360870361328,
      "logps/rejected": -297.9733581542969,
      "loss": 0.1437,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.280116319656372,
      "rewards/margins": 4.649960517883301,
      "rewards/rejected": -5.930076599121094,
      "step": 765
    },
    {
      "epoch": 1.6421333333333332,
      "grad_norm": 3.1140360832214355,
      "learning_rate": 2.24707919503142e-06,
      "logits/chosen": -1.7199656963348389,
      "logits/rejected": -1.620208501815796,
      "logps/chosen": -242.5529327392578,
      "logps/rejected": -318.6707763671875,
      "loss": 0.1247,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.4802429676055908,
      "rewards/margins": 4.666882514953613,
      "rewards/rejected": -6.147125244140625,
      "step": 770
    },
    {
      "epoch": 1.6528,
      "grad_norm": 6.060303688049316,
      "learning_rate": 2.2184540574285213e-06,
      "logits/chosen": -1.6107571125030518,
      "logits/rejected": -1.4901559352874756,
      "logps/chosen": -266.9434509277344,
      "logps/rejected": -303.53179931640625,
      "loss": 0.2051,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3911632299423218,
      "rewards/margins": 4.4641523361206055,
      "rewards/rejected": -5.855315208435059,
      "step": 775
    },
    {
      "epoch": 1.6634666666666666,
      "grad_norm": 2.654046058654785,
      "learning_rate": 2.1898662582069486e-06,
      "logits/chosen": -1.6774696111679077,
      "logits/rejected": -1.5776320695877075,
      "logps/chosen": -250.61331176757812,
      "logps/rejected": -306.73663330078125,
      "loss": 0.1373,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.7681554555892944,
      "rewards/margins": 5.010857582092285,
      "rewards/rejected": -6.779012203216553,
      "step": 780
    },
    {
      "epoch": 1.6741333333333333,
      "grad_norm": 3.1174325942993164,
      "learning_rate": 2.1613195886561044e-06,
      "logits/chosen": -1.7513983249664307,
      "logits/rejected": -1.6123911142349243,
      "logps/chosen": -230.7318878173828,
      "logps/rejected": -302.77874755859375,
      "loss": 0.113,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.284829020500183,
      "rewards/margins": 5.273435115814209,
      "rewards/rejected": -6.55826473236084,
      "step": 785
    },
    {
      "epoch": 1.6848,
      "grad_norm": 5.556603908538818,
      "learning_rate": 2.132817834610811e-06,
      "logits/chosen": -1.6932178735733032,
      "logits/rejected": -1.552677869796753,
      "logps/chosen": -241.80630493164062,
      "logps/rejected": -315.0558776855469,
      "loss": 0.1641,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.5892698764801025,
      "rewards/margins": 4.98702335357666,
      "rewards/rejected": -6.576292991638184,
      "step": 790
    },
    {
      "epoch": 1.6954666666666667,
      "grad_norm": 2.940763235092163,
      "learning_rate": 2.1043647759492338e-06,
      "logits/chosen": -1.5830045938491821,
      "logits/rejected": -1.4869303703308105,
      "logps/chosen": -264.3173828125,
      "logps/rejected": -319.78924560546875,
      "loss": 0.1699,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4739254713058472,
      "rewards/margins": 4.805297374725342,
      "rewards/rejected": -6.279223442077637,
      "step": 795
    },
    {
      "epoch": 1.7061333333333333,
      "grad_norm": 4.358272075653076,
      "learning_rate": 2.075964186091599e-06,
      "logits/chosen": -1.7524511814117432,
      "logits/rejected": -1.6442880630493164,
      "logps/chosen": -233.0540008544922,
      "logps/rejected": -352.62567138671875,
      "loss": 0.1132,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.8809230327606201,
      "rewards/margins": 5.72424840927124,
      "rewards/rejected": -7.605171203613281,
      "step": 800
    },
    {
      "epoch": 1.7168,
      "grad_norm": 3.5285327434539795,
      "learning_rate": 2.047619831499765e-06,
      "logits/chosen": -1.5862383842468262,
      "logits/rejected": -1.5314266681671143,
      "logps/chosen": -239.5761260986328,
      "logps/rejected": -314.5472106933594,
      "loss": 0.1659,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.4697277545928955,
      "rewards/margins": 4.891806125640869,
      "rewards/rejected": -6.3615336418151855,
      "step": 805
    },
    {
      "epoch": 1.7274666666666667,
      "grad_norm": 3.361513614654541,
      "learning_rate": 2.0193354711777155e-06,
      "logits/chosen": -1.7188619375228882,
      "logits/rejected": -1.5833302736282349,
      "logps/chosen": -264.5706481933594,
      "logps/rejected": -334.4622497558594,
      "loss": 0.1704,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.8755519390106201,
      "rewards/margins": 5.095242500305176,
      "rewards/rejected": -6.970794677734375,
      "step": 810
    },
    {
      "epoch": 1.7381333333333333,
      "grad_norm": 2.792012929916382,
      "learning_rate": 1.991114856173048e-06,
      "logits/chosen": -1.6884195804595947,
      "logits/rejected": -1.6442887783050537,
      "logps/chosen": -230.58377075195312,
      "logps/rejected": -339.1591796875,
      "loss": 0.171,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.912938117980957,
      "rewards/margins": 5.23507022857666,
      "rewards/rejected": -7.148007869720459,
      "step": 815
    },
    {
      "epoch": 1.7488000000000001,
      "grad_norm": 4.240962505340576,
      "learning_rate": 1.9629617290795115e-06,
      "logits/chosen": -1.5643692016601562,
      "logits/rejected": -1.526997685432434,
      "logps/chosen": -248.695556640625,
      "logps/rejected": -323.1645202636719,
      "loss": 0.1728,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.9166147708892822,
      "rewards/margins": 4.900598049163818,
      "rewards/rejected": -6.817213535308838,
      "step": 820
    },
    {
      "epoch": 1.7594666666666665,
      "grad_norm": 1.9399546384811401,
      "learning_rate": 1.934879823540663e-06,
      "logits/chosen": -1.6955827474594116,
      "logits/rejected": -1.568052053451538,
      "logps/chosen": -247.80142211914062,
      "logps/rejected": -331.73883056640625,
      "loss": 0.2002,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.013258934020996,
      "rewards/margins": 5.11118745803833,
      "rewards/rejected": -7.124446868896484,
      "step": 825
    },
    {
      "epoch": 1.7701333333333333,
      "grad_norm": 2.1034741401672363,
      "learning_rate": 1.9068728637547195e-06,
      "logits/chosen": -1.6912727355957031,
      "logits/rejected": -1.579400897026062,
      "logps/chosen": -254.18777465820312,
      "logps/rejected": -339.73016357421875,
      "loss": 0.1794,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.799628496170044,
      "rewards/margins": 5.339221000671387,
      "rewards/rejected": -7.138850212097168,
      "step": 830
    },
    {
      "epoch": 1.7808000000000002,
      "grad_norm": 2.8324687480926514,
      "learning_rate": 1.878944563980658e-06,
      "logits/chosen": -1.681926965713501,
      "logits/rejected": -1.5870113372802734,
      "logps/chosen": -248.7440948486328,
      "logps/rejected": -322.190673828125,
      "loss": 0.1347,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.6237030029296875,
      "rewards/margins": 5.433587551116943,
      "rewards/rejected": -7.057290554046631,
      "step": 835
    },
    {
      "epoch": 1.7914666666666665,
      "grad_norm": 2.730384111404419,
      "learning_rate": 1.85109862804563e-06,
      "logits/chosen": -1.6680930852890015,
      "logits/rejected": -1.6028831005096436,
      "logps/chosen": -299.27667236328125,
      "logps/rejected": -340.4490966796875,
      "loss": 0.1411,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.330984354019165,
      "rewards/margins": 5.942017555236816,
      "rewards/rejected": -7.273001194000244,
      "step": 840
    },
    {
      "epoch": 1.8021333333333334,
      "grad_norm": 3.7725889682769775,
      "learning_rate": 1.823338748853766e-06,
      "logits/chosen": -1.6155637502670288,
      "logits/rejected": -1.6243375539779663,
      "logps/chosen": -263.1604919433594,
      "logps/rejected": -347.5821228027344,
      "loss": 0.1621,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.142850399017334,
      "rewards/margins": 5.3936767578125,
      "rewards/rejected": -7.536526679992676,
      "step": 845
    },
    {
      "epoch": 1.8128,
      "grad_norm": 5.143514633178711,
      "learning_rate": 1.7956686078964257e-06,
      "logits/chosen": -1.7049986124038696,
      "logits/rejected": -1.6571338176727295,
      "logps/chosen": -238.5912322998047,
      "logps/rejected": -330.8258972167969,
      "loss": 0.2134,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6940962076187134,
      "rewards/margins": 4.851913928985596,
      "rewards/rejected": -6.5460100173950195,
      "step": 850
    },
    {
      "epoch": 1.8234666666666666,
      "grad_norm": 4.174952030181885,
      "learning_rate": 1.7680918747639608e-06,
      "logits/chosen": -1.7732454538345337,
      "logits/rejected": -1.6001640558242798,
      "logps/chosen": -235.79336547851562,
      "logps/rejected": -340.64886474609375,
      "loss": 0.1289,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.5047314167022705,
      "rewards/margins": 5.615990161895752,
      "rewards/rejected": -7.120721340179443,
      "step": 855
    },
    {
      "epoch": 1.8341333333333334,
      "grad_norm": 2.559386968612671,
      "learning_rate": 1.740612206659057e-06,
      "logits/chosen": -1.6925417184829712,
      "logits/rejected": -1.5877892971038818,
      "logps/chosen": -252.12435913085938,
      "logps/rejected": -337.1178894042969,
      "loss": 0.1112,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.6748367547988892,
      "rewards/margins": 5.5472025871276855,
      "rewards/rejected": -7.222039222717285,
      "step": 860
    },
    {
      "epoch": 1.8448,
      "grad_norm": 5.024665832519531,
      "learning_rate": 1.7132332479117204e-06,
      "logits/chosen": -1.757213830947876,
      "logits/rejected": -1.6775776147842407,
      "logps/chosen": -249.8083953857422,
      "logps/rejected": -341.9578857421875,
      "loss": 0.1382,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.523437738418579,
      "rewards/margins": 5.880698204040527,
      "rewards/rejected": -7.404136657714844,
      "step": 865
    },
    {
      "epoch": 1.8554666666666666,
      "grad_norm": 1.7119357585906982,
      "learning_rate": 1.6859586294959666e-06,
      "logits/chosen": -1.7067601680755615,
      "logits/rejected": -1.6395823955535889,
      "logps/chosen": -261.8993835449219,
      "logps/rejected": -340.2198791503906,
      "loss": 0.1338,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6026289463043213,
      "rewards/margins": 5.703084468841553,
      "rewards/rejected": -7.305713653564453,
      "step": 870
    },
    {
      "epoch": 1.8661333333333334,
      "grad_norm": 2.4370765686035156,
      "learning_rate": 1.6587919685482884e-06,
      "logits/chosen": -1.715688705444336,
      "logits/rejected": -1.5712188482284546,
      "logps/chosen": -242.3165740966797,
      "logps/rejected": -318.1778869628906,
      "loss": 0.128,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.9740794897079468,
      "rewards/margins": 5.140053749084473,
      "rewards/rejected": -7.114132881164551,
      "step": 875
    },
    {
      "epoch": 1.8768,
      "grad_norm": 5.989840984344482,
      "learning_rate": 1.6317368678879497e-06,
      "logits/chosen": -1.6727215051651,
      "logits/rejected": -1.5320321321487427,
      "logps/chosen": -242.89840698242188,
      "logps/rejected": -302.96954345703125,
      "loss": 0.1445,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.4183199405670166,
      "rewards/margins": 4.921168327331543,
      "rewards/rejected": -6.3394880294799805,
      "step": 880
    },
    {
      "epoch": 1.8874666666666666,
      "grad_norm": 3.716749429702759,
      "learning_rate": 1.6047969155391856e-06,
      "logits/chosen": -1.554337501525879,
      "logits/rejected": -1.4884510040283203,
      "logps/chosen": -256.3968200683594,
      "logps/rejected": -308.56573486328125,
      "loss": 0.1446,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.6531686782836914,
      "rewards/margins": 5.160852432250977,
      "rewards/rejected": -6.814022064208984,
      "step": 885
    },
    {
      "epoch": 1.8981333333333335,
      "grad_norm": 3.8302741050720215,
      "learning_rate": 1.5779756842553617e-06,
      "logits/chosen": -1.6652368307113647,
      "logits/rejected": -1.6241079568862915,
      "logps/chosen": -263.47515869140625,
      "logps/rejected": -331.0129089355469,
      "loss": 0.1351,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.0062878131866455,
      "rewards/margins": 5.339743614196777,
      "rewards/rejected": -7.34603214263916,
      "step": 890
    },
    {
      "epoch": 1.9088,
      "grad_norm": 7.920835971832275,
      "learning_rate": 1.551276731045157e-06,
      "logits/chosen": -1.5562691688537598,
      "logits/rejected": -1.5247366428375244,
      "logps/chosen": -275.4099426269531,
      "logps/rejected": -347.5796203613281,
      "loss": 0.1669,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.7762320041656494,
      "rewards/margins": 5.540648460388184,
      "rewards/rejected": -7.316880702972412,
      "step": 895
    },
    {
      "epoch": 1.9194666666666667,
      "grad_norm": 2.614849805831909,
      "learning_rate": 1.5247035967008369e-06,
      "logits/chosen": -1.5560858249664307,
      "logits/rejected": -1.503138542175293,
      "logps/chosen": -237.936767578125,
      "logps/rejected": -330.28277587890625,
      "loss": 0.152,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.6794102191925049,
      "rewards/margins": 5.538970947265625,
      "rewards/rejected": -7.218381404876709,
      "step": 900
    },
    {
      "epoch": 1.9301333333333335,
      "grad_norm": 3.9147305488586426,
      "learning_rate": 1.498259805328678e-06,
      "logits/chosen": -1.6482226848602295,
      "logits/rejected": -1.59519362449646,
      "logps/chosen": -239.3106689453125,
      "logps/rejected": -328.4599609375,
      "loss": 0.1567,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.219871997833252,
      "rewards/margins": 5.234225273132324,
      "rewards/rejected": -7.454097747802734,
      "step": 905
    },
    {
      "epoch": 1.9407999999999999,
      "grad_norm": 4.330874443054199,
      "learning_rate": 1.471948863881601e-06,
      "logits/chosen": -1.7839887142181396,
      "logits/rejected": -1.594058871269226,
      "logps/chosen": -255.79690551757812,
      "logps/rejected": -316.6875,
      "loss": 0.1776,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.6719051599502563,
      "rewards/margins": 5.126084327697754,
      "rewards/rejected": -6.797989845275879,
      "step": 910
    },
    {
      "epoch": 1.9514666666666667,
      "grad_norm": 3.5629780292510986,
      "learning_rate": 1.4457742616940864e-06,
      "logits/chosen": -1.6401689052581787,
      "logits/rejected": -1.5712366104125977,
      "logps/chosen": -250.49868774414062,
      "logps/rejected": -326.28741455078125,
      "loss": 0.1683,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.7338117361068726,
      "rewards/margins": 5.273244380950928,
      "rewards/rejected": -7.00705623626709,
      "step": 915
    },
    {
      "epoch": 1.9621333333333333,
      "grad_norm": 3.6759705543518066,
      "learning_rate": 1.4197394700194165e-06,
      "logits/chosen": -1.6852229833602905,
      "logits/rejected": -1.6576778888702393,
      "logps/chosen": -249.3162841796875,
      "logps/rejected": -325.59326171875,
      "loss": 0.1717,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.5520923137664795,
      "rewards/margins": 5.2563347816467285,
      "rewards/rejected": -6.808426856994629,
      "step": 920
    },
    {
      "epoch": 1.9727999999999999,
      "grad_norm": 3.09043288230896,
      "learning_rate": 1.3938479415693244e-06,
      "logits/chosen": -1.6909940242767334,
      "logits/rejected": -1.6325387954711914,
      "logps/chosen": -250.38870239257812,
      "logps/rejected": -348.135986328125,
      "loss": 0.109,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.6463645696640015,
      "rewards/margins": 5.957003593444824,
      "rewards/rejected": -7.603367805480957,
      "step": 925
    },
    {
      "epoch": 1.9834666666666667,
      "grad_norm": 3.553077220916748,
      "learning_rate": 1.368103110056096e-06,
      "logits/chosen": -1.7662723064422607,
      "logits/rejected": -1.6744906902313232,
      "logps/chosen": -252.43728637695312,
      "logps/rejected": -321.92413330078125,
      "loss": 0.1539,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.9222122430801392,
      "rewards/margins": 5.75597620010376,
      "rewards/rejected": -7.678188323974609,
      "step": 930
    },
    {
      "epoch": 1.9941333333333333,
      "grad_norm": 4.891976356506348,
      "learning_rate": 1.3425083897371983e-06,
      "logits/chosen": -1.6703214645385742,
      "logits/rejected": -1.5516163110733032,
      "logps/chosen": -262.6437683105469,
      "logps/rejected": -329.6023254394531,
      "loss": 0.1595,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.671091079711914,
      "rewards/margins": 5.250547409057617,
      "rewards/rejected": -6.921639442443848,
      "step": 935
    },
    {
      "epoch": 2.0042666666666666,
      "grad_norm": 3.132736921310425,
      "learning_rate": 1.317067174962478e-06,
      "logits/chosen": -1.6124461889266968,
      "logits/rejected": -1.465229868888855,
      "logps/chosen": -258.7012939453125,
      "logps/rejected": -305.882568359375,
      "loss": 0.176,
      "rewards/accuracies": 0.9539473652839661,
      "rewards/chosen": -2.025094985961914,
      "rewards/margins": 5.082742214202881,
      "rewards/rejected": -7.107837677001953,
      "step": 940
    },
    {
      "epoch": 2.0149333333333335,
      "grad_norm": 3.199899435043335,
      "learning_rate": 1.291782839724009e-06,
      "logits/chosen": -1.635519027709961,
      "logits/rejected": -1.5712947845458984,
      "logps/chosen": -258.9504699707031,
      "logps/rejected": -322.48211669921875,
      "loss": 0.1754,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.844058632850647,
      "rewards/margins": 4.864871978759766,
      "rewards/rejected": -6.708930969238281,
      "step": 945
    },
    {
      "epoch": 2.0256,
      "grad_norm": 3.8576583862304688,
      "learning_rate": 1.2666587372086352e-06,
      "logits/chosen": -1.575919508934021,
      "logits/rejected": -1.6220824718475342,
      "logps/chosen": -236.1714324951172,
      "logps/rejected": -340.546875,
      "loss": 0.1122,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.6586267948150635,
      "rewards/margins": 5.464929580688477,
      "rewards/rejected": -7.123556613922119,
      "step": 950
    },
    {
      "epoch": 2.0362666666666667,
      "grad_norm": 3.3838019371032715,
      "learning_rate": 1.241698199353275e-06,
      "logits/chosen": -1.6258891820907593,
      "logits/rejected": -1.5644147396087646,
      "logps/chosen": -243.3029022216797,
      "logps/rejected": -324.55096435546875,
      "loss": 0.145,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.659441590309143,
      "rewards/margins": 4.9935383796691895,
      "rewards/rejected": -6.652979850769043,
      "step": 955
    },
    {
      "epoch": 2.0469333333333335,
      "grad_norm": 4.255578994750977,
      "learning_rate": 1.2169045364030404e-06,
      "logits/chosen": -1.7650766372680664,
      "logits/rejected": -1.6293489933013916,
      "logps/chosen": -236.46322631835938,
      "logps/rejected": -319.8513488769531,
      "loss": 0.1216,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.9534380435943604,
      "rewards/margins": 5.467280864715576,
      "rewards/rejected": -7.420719146728516,
      "step": 960
    },
    {
      "epoch": 2.0576,
      "grad_norm": 6.157327651977539,
      "learning_rate": 1.1922810364722402e-06,
      "logits/chosen": -1.645727515220642,
      "logits/rejected": -1.5266873836517334,
      "logps/chosen": -258.87347412109375,
      "logps/rejected": -314.80267333984375,
      "loss": 0.1738,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.7486776113510132,
      "rewards/margins": 4.593234539031982,
      "rewards/rejected": -6.341912269592285,
      "step": 965
    },
    {
      "epoch": 2.0682666666666667,
      "grad_norm": 3.139277696609497,
      "learning_rate": 1.1678309651083055e-06,
      "logits/chosen": -1.7026522159576416,
      "logits/rejected": -1.6696736812591553,
      "logps/chosen": -236.3257293701172,
      "logps/rejected": -341.9847412109375,
      "loss": 0.1023,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.863978624343872,
      "rewards/margins": 5.8271098136901855,
      "rewards/rejected": -7.691088676452637,
      "step": 970
    },
    {
      "epoch": 2.0789333333333335,
      "grad_norm": 1.7706592082977295,
      "learning_rate": 1.1435575648587224e-06,
      "logits/chosen": -1.6053731441497803,
      "logits/rejected": -1.5967332124710083,
      "logps/chosen": -249.1739501953125,
      "logps/rejected": -336.7267761230469,
      "loss": 0.1229,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.9126554727554321,
      "rewards/margins": 5.3537068367004395,
      "rewards/rejected": -7.266362190246582,
      "step": 975
    },
    {
      "epoch": 2.0896,
      "grad_norm": 3.7354724407196045,
      "learning_rate": 1.1194640548410039e-06,
      "logits/chosen": -1.5527129173278809,
      "logits/rejected": -1.512411117553711,
      "logps/chosen": -259.1437683105469,
      "logps/rejected": -303.2730712890625,
      "loss": 0.1752,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.8620319366455078,
      "rewards/margins": 5.097782135009766,
      "rewards/rejected": -6.959813594818115,
      "step": 980
    },
    {
      "epoch": 2.1002666666666667,
      "grad_norm": 1.9574644565582275,
      "learning_rate": 1.095553630315774e-06,
      "logits/chosen": -1.608660101890564,
      "logits/rejected": -1.5342473983764648,
      "logps/chosen": -256.6988220214844,
      "logps/rejected": -332.3222351074219,
      "loss": 0.1137,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.5717804431915283,
      "rewards/margins": 5.8155951499938965,
      "rewards/rejected": -7.387376308441162,
      "step": 985
    },
    {
      "epoch": 2.1109333333333336,
      "grad_norm": 2.388463258743286,
      "learning_rate": 1.0718294622630188e-06,
      "logits/chosen": -1.7099485397338867,
      "logits/rejected": -1.5633583068847656,
      "logps/chosen": -273.2356262207031,
      "logps/rejected": -345.201171875,
      "loss": 0.1278,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.8504549264907837,
      "rewards/margins": 5.3531904220581055,
      "rewards/rejected": -7.2036452293396,
      "step": 990
    },
    {
      "epoch": 2.1216,
      "grad_norm": 3.036656618118286,
      "learning_rate": 1.0482946969615495e-06,
      "logits/chosen": -1.7514898777008057,
      "logits/rejected": -1.6838487386703491,
      "logps/chosen": -233.05953979492188,
      "logps/rejected": -331.9215393066406,
      "loss": 0.0959,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.013244867324829,
      "rewards/margins": 5.820936679840088,
      "rewards/rejected": -7.834181785583496,
      "step": 995
    },
    {
      "epoch": 2.1322666666666668,
      "grad_norm": 8.45803451538086,
      "learning_rate": 1.0249524555717461e-06,
      "logits/chosen": -1.6929506063461304,
      "logits/rejected": -1.5387563705444336,
      "logps/chosen": -242.8338623046875,
      "logps/rejected": -329.45965576171875,
      "loss": 0.1429,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.9728199243545532,
      "rewards/margins": 5.124451637268066,
      "rewards/rejected": -7.097271919250488,
      "step": 1000
    },
    {
      "epoch": 2.142933333333333,
      "grad_norm": 1.7584189176559448,
      "learning_rate": 1.0018058337216327e-06,
      "logits/chosen": -1.6556212902069092,
      "logits/rejected": -1.6083987951278687,
      "logps/chosen": -238.981689453125,
      "logps/rejected": -326.83062744140625,
      "loss": 0.1182,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.6057265996932983,
      "rewards/margins": 5.429762363433838,
      "rewards/rejected": -7.035489559173584,
      "step": 1005
    },
    {
      "epoch": 2.1536,
      "grad_norm": 4.888096809387207,
      "learning_rate": 9.788579010963387e-07,
      "logits/chosen": -1.6201951503753662,
      "logits/rejected": -1.5311352014541626,
      "logps/chosen": -267.74908447265625,
      "logps/rejected": -334.7906494140625,
      "loss": 0.1455,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.1402487754821777,
      "rewards/margins": 5.422475337982178,
      "rewards/rejected": -7.5627241134643555,
      "step": 1010
    },
    {
      "epoch": 2.164266666666667,
      "grad_norm": 2.1864840984344482,
      "learning_rate": 9.56111701030998e-07,
      "logits/chosen": -1.7070671319961548,
      "logits/rejected": -1.558129906654358,
      "logps/chosen": -247.9465789794922,
      "logps/rejected": -343.86029052734375,
      "loss": 0.1001,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.7870800495147705,
      "rewards/margins": 6.4040422439575195,
      "rewards/rejected": -8.191122055053711,
      "step": 1015
    },
    {
      "epoch": 2.174933333333333,
      "grad_norm": 3.70732045173645,
      "learning_rate": 9.335702501071459e-07,
      "logits/chosen": -1.6212594509124756,
      "logits/rejected": -1.4471334218978882,
      "logps/chosen": -265.968994140625,
      "logps/rejected": -326.27630615234375,
      "loss": 0.1046,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.8242906332015991,
      "rewards/margins": 5.466552257537842,
      "rewards/rejected": -7.2908430099487305,
      "step": 1020
    },
    {
      "epoch": 2.1856,
      "grad_norm": 4.0817084312438965,
      "learning_rate": 9.112365377526636e-07,
      "logits/chosen": -1.6601269245147705,
      "logits/rejected": -1.6235460042953491,
      "logps/chosen": -234.23635864257812,
      "logps/rejected": -316.6658630371094,
      "loss": 0.1544,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.0195419788360596,
      "rewards/margins": 5.3594865798950195,
      "rewards/rejected": -7.379029273986816,
      "step": 1025
    },
    {
      "epoch": 2.196266666666667,
      "grad_norm": 1.6365838050842285,
      "learning_rate": 8.891135258453195e-07,
      "logits/chosen": -1.603167176246643,
      "logits/rejected": -1.5676203966140747,
      "logps/chosen": -243.29550170898438,
      "logps/rejected": -324.26318359375,
      "loss": 0.1142,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.0350539684295654,
      "rewards/margins": 5.261748313903809,
      "rewards/rejected": -7.2968034744262695,
      "step": 1030
    },
    {
      "epoch": 2.206933333333333,
      "grad_norm": 3.2687931060791016,
      "learning_rate": 8.672041483199709e-07,
      "logits/chosen": -1.7075221538543701,
      "logits/rejected": -1.62152099609375,
      "logps/chosen": -260.42828369140625,
      "logps/rejected": -325.1893310546875,
      "loss": 0.1046,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.961230993270874,
      "rewards/margins": 5.438101291656494,
      "rewards/rejected": -7.399331569671631,
      "step": 1035
    },
    {
      "epoch": 2.2176,
      "grad_norm": 4.33626651763916,
      "learning_rate": 8.455113107794652e-07,
      "logits/chosen": -1.5977696180343628,
      "logits/rejected": -1.5143213272094727,
      "logps/chosen": -243.13815307617188,
      "logps/rejected": -315.7250061035156,
      "loss": 0.1049,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.2392735481262207,
      "rewards/margins": 5.705511093139648,
      "rewards/rejected": -7.944784641265869,
      "step": 1040
    },
    {
      "epoch": 2.228266666666667,
      "grad_norm": 3.568450927734375,
      "learning_rate": 8.240378901093035e-07,
      "logits/chosen": -1.7558826208114624,
      "logits/rejected": -1.6090519428253174,
      "logps/chosen": -253.54891967773438,
      "logps/rejected": -336.08343505859375,
      "loss": 0.1332,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.987137794494629,
      "rewards/margins": 5.502018928527832,
      "rewards/rejected": -7.489157199859619,
      "step": 1045
    },
    {
      "epoch": 2.238933333333333,
      "grad_norm": 2.2904350757598877,
      "learning_rate": 8.027867340961113e-07,
      "logits/chosen": -1.613086462020874,
      "logits/rejected": -1.5385499000549316,
      "logps/chosen": -251.39895629882812,
      "logps/rejected": -336.3780212402344,
      "loss": 0.1591,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.0407490730285645,
      "rewards/margins": 5.212506294250488,
      "rewards/rejected": -7.253255367279053,
      "step": 1050
    },
    {
      "epoch": 2.2496,
      "grad_norm": 3.7621541023254395,
      "learning_rate": 7.817606610499639e-07,
      "logits/chosen": -1.7154810428619385,
      "logits/rejected": -1.6074755191802979,
      "logps/chosen": -250.9788360595703,
      "logps/rejected": -348.1394958496094,
      "loss": 0.1059,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.8897058963775635,
      "rewards/margins": 6.101401329040527,
      "rewards/rejected": -7.991107940673828,
      "step": 1055
    },
    {
      "epoch": 2.2602666666666664,
      "grad_norm": 2.9305503368377686,
      "learning_rate": 7.609624594306278e-07,
      "logits/chosen": -1.6567564010620117,
      "logits/rejected": -1.6048027276992798,
      "logps/chosen": -247.90121459960938,
      "logps/rejected": -337.5819396972656,
      "loss": 0.1433,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.9860131740570068,
      "rewards/margins": 6.067047119140625,
      "rewards/rejected": -8.053060531616211,
      "step": 1060
    },
    {
      "epoch": 2.2709333333333332,
      "grad_norm": 4.816735744476318,
      "learning_rate": 7.403948874777556e-07,
      "logits/chosen": -1.6866226196289062,
      "logits/rejected": -1.6687873601913452,
      "logps/chosen": -259.2507019042969,
      "logps/rejected": -353.58197021484375,
      "loss": 0.1308,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.9822838306427002,
      "rewards/margins": 5.897215843200684,
      "rewards/rejected": -7.879499912261963,
      "step": 1065
    },
    {
      "epoch": 2.2816,
      "grad_norm": 5.037070274353027,
      "learning_rate": 7.20060672845091e-07,
      "logits/chosen": -1.5634400844573975,
      "logits/rejected": -1.5417492389678955,
      "logps/chosen": -253.765869140625,
      "logps/rejected": -333.6149597167969,
      "loss": 0.1106,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.216799259185791,
      "rewards/margins": 5.854150295257568,
      "rewards/rejected": -8.070950508117676,
      "step": 1070
    },
    {
      "epoch": 2.2922666666666665,
      "grad_norm": 3.694530487060547,
      "learning_rate": 6.99962512238733e-07,
      "logits/chosen": -1.507551908493042,
      "logits/rejected": -1.4376182556152344,
      "logps/chosen": -251.04052734375,
      "logps/rejected": -307.02593994140625,
      "loss": 0.1729,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.9417812824249268,
      "rewards/margins": 5.363430023193359,
      "rewards/rejected": -7.305211067199707,
      "step": 1075
    },
    {
      "epoch": 2.3029333333333333,
      "grad_norm": 1.9394572973251343,
      "learning_rate": 6.801030710594949e-07,
      "logits/chosen": -1.5492713451385498,
      "logits/rejected": -1.4976288080215454,
      "logps/chosen": -252.68612670898438,
      "logps/rejected": -325.3870544433594,
      "loss": 0.171,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.119556188583374,
      "rewards/margins": 5.055575847625732,
      "rewards/rejected": -7.175132751464844,
      "step": 1080
    },
    {
      "epoch": 2.3136,
      "grad_norm": 2.8549582958221436,
      "learning_rate": 6.604849830494264e-07,
      "logits/chosen": -1.7532446384429932,
      "logits/rejected": -1.6480261087417603,
      "logps/chosen": -248.76596069335938,
      "logps/rejected": -333.56781005859375,
      "loss": 0.1086,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.9362081289291382,
      "rewards/margins": 6.194217205047607,
      "rewards/rejected": -8.130426406860352,
      "step": 1085
    },
    {
      "epoch": 2.3242666666666665,
      "grad_norm": 2.9997100830078125,
      "learning_rate": 6.411108499425253e-07,
      "logits/chosen": -1.733490228652954,
      "logits/rejected": -1.5960662364959717,
      "logps/chosen": -264.21929931640625,
      "logps/rejected": -333.2695007324219,
      "loss": 0.1262,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.9376022815704346,
      "rewards/margins": 5.7414021492004395,
      "rewards/rejected": -7.6790056228637695,
      "step": 1090
    },
    {
      "epoch": 2.3349333333333333,
      "grad_norm": 3.0645434856414795,
      "learning_rate": 6.219832411197013e-07,
      "logits/chosen": -1.642669677734375,
      "logits/rejected": -1.4960535764694214,
      "logps/chosen": -248.64218139648438,
      "logps/rejected": -323.8027038574219,
      "loss": 0.1434,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.804839849472046,
      "rewards/margins": 5.383707523345947,
      "rewards/rejected": -7.188547611236572,
      "step": 1095
    },
    {
      "epoch": 2.3456,
      "grad_norm": 4.203052043914795,
      "learning_rate": 6.031046932680229e-07,
      "logits/chosen": -1.6723241806030273,
      "logits/rejected": -1.5729491710662842,
      "logps/chosen": -256.49395751953125,
      "logps/rejected": -353.57769775390625,
      "loss": 0.1144,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.992719292640686,
      "rewards/margins": 5.605996608734131,
      "rewards/rejected": -7.598715782165527,
      "step": 1100
    },
    {
      "epoch": 2.3562666666666665,
      "grad_norm": 3.4321813583374023,
      "learning_rate": 5.844777100443091e-07,
      "logits/chosen": -1.6482032537460327,
      "logits/rejected": -1.5842411518096924,
      "logps/chosen": -240.6206817626953,
      "logps/rejected": -331.48101806640625,
      "loss": 0.1226,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.8029711246490479,
      "rewards/margins": 5.810032367706299,
      "rewards/rejected": -7.613003730773926,
      "step": 1105
    },
    {
      "epoch": 2.3669333333333333,
      "grad_norm": 4.681034564971924,
      "learning_rate": 5.66104761743092e-07,
      "logits/chosen": -1.7696841955184937,
      "logits/rejected": -1.630789041519165,
      "logps/chosen": -255.98580932617188,
      "logps/rejected": -314.320556640625,
      "loss": 0.1541,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.265458583831787,
      "rewards/margins": 5.304887294769287,
      "rewards/rejected": -7.570345401763916,
      "step": 1110
    },
    {
      "epoch": 2.3776,
      "grad_norm": 2.437649726867676,
      "learning_rate": 5.479882849690102e-07,
      "logits/chosen": -1.6937763690948486,
      "logits/rejected": -1.6039565801620483,
      "logps/chosen": -248.5046844482422,
      "logps/rejected": -330.6466064453125,
      "loss": 0.1357,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.949684500694275,
      "rewards/margins": 5.519129276275635,
      "rewards/rejected": -7.468813896179199,
      "step": 1115
    },
    {
      "epoch": 2.3882666666666665,
      "grad_norm": 1.7076597213745117,
      "learning_rate": 5.301306823136687e-07,
      "logits/chosen": -1.5468543767929077,
      "logits/rejected": -1.4817711114883423,
      "logps/chosen": -242.590576171875,
      "logps/rejected": -331.3986511230469,
      "loss": 0.1559,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.899458885192871,
      "rewards/margins": 5.719172477722168,
      "rewards/rejected": -7.6186323165893555,
      "step": 1120
    },
    {
      "epoch": 2.3989333333333334,
      "grad_norm": 3.783508539199829,
      "learning_rate": 5.125343220370088e-07,
      "logits/chosen": -1.5996062755584717,
      "logits/rejected": -1.5361487865447998,
      "logps/chosen": -247.81689453125,
      "logps/rejected": -312.226806640625,
      "loss": 0.1241,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.627031683921814,
      "rewards/margins": 5.376161098480225,
      "rewards/rejected": -7.003192901611328,
      "step": 1125
    },
    {
      "epoch": 2.4096,
      "grad_norm": 3.6416852474212646,
      "learning_rate": 4.952015377532329e-07,
      "logits/chosen": -1.8114979267120361,
      "logits/rejected": -1.6875921487808228,
      "logps/chosen": -250.836669921875,
      "logps/rejected": -349.7911071777344,
      "loss": 0.1159,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.255504608154297,
      "rewards/margins": 5.732514381408691,
      "rewards/rejected": -7.9880194664001465,
      "step": 1130
    },
    {
      "epoch": 2.4202666666666666,
      "grad_norm": 3.7079567909240723,
      "learning_rate": 4.781346281213204e-07,
      "logits/chosen": -1.683234453201294,
      "logits/rejected": -1.6827503442764282,
      "logps/chosen": -234.7036895751953,
      "logps/rejected": -349.72784423828125,
      "loss": 0.1055,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.1054515838623047,
      "rewards/margins": 6.153875827789307,
      "rewards/rejected": -8.259326934814453,
      "step": 1135
    },
    {
      "epoch": 2.4309333333333334,
      "grad_norm": 3.7606658935546875,
      "learning_rate": 4.613358565401818e-07,
      "logits/chosen": -1.6943823099136353,
      "logits/rejected": -1.5724833011627197,
      "logps/chosen": -265.177734375,
      "logps/rejected": -340.174072265625,
      "loss": 0.1427,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.0506272315979004,
      "rewards/margins": 5.461020469665527,
      "rewards/rejected": -7.511648654937744,
      "step": 1140
    },
    {
      "epoch": 2.4416,
      "grad_norm": 3.089946985244751,
      "learning_rate": 4.448074508484906e-07,
      "logits/chosen": -1.686182975769043,
      "logits/rejected": -1.5479154586791992,
      "logps/chosen": -263.35321044921875,
      "logps/rejected": -339.89508056640625,
      "loss": 0.1014,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.8184881210327148,
      "rewards/margins": 5.920653820037842,
      "rewards/rejected": -7.739141941070557,
      "step": 1145
    },
    {
      "epoch": 2.4522666666666666,
      "grad_norm": 2.193614959716797,
      "learning_rate": 4.2855160302922777e-07,
      "logits/chosen": -1.6815516948699951,
      "logits/rejected": -1.5796687602996826,
      "logps/chosen": -254.398193359375,
      "logps/rejected": -342.8807678222656,
      "loss": 0.0996,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.013457775115967,
      "rewards/margins": 6.017300605773926,
      "rewards/rejected": -8.03075885772705,
      "step": 1150
    },
    {
      "epoch": 2.4629333333333334,
      "grad_norm": 6.938134670257568,
      "learning_rate": 4.125704689189819e-07,
      "logits/chosen": -1.5728188753128052,
      "logits/rejected": -1.5077226161956787,
      "logps/chosen": -253.8098602294922,
      "logps/rejected": -345.307373046875,
      "loss": 0.2014,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.9437261819839478,
      "rewards/margins": 5.033670902252197,
      "rewards/rejected": -6.977396488189697,
      "step": 1155
    },
    {
      "epoch": 2.4736000000000002,
      "grad_norm": 3.552980899810791,
      "learning_rate": 3.9686616792204677e-07,
      "logits/chosen": -1.617547631263733,
      "logits/rejected": -1.5896202325820923,
      "logps/chosen": -245.8852081298828,
      "logps/rejected": -349.4147644042969,
      "loss": 0.0944,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.8278429508209229,
      "rewards/margins": 5.722242832183838,
      "rewards/rejected": -7.550085544586182,
      "step": 1160
    },
    {
      "epoch": 2.4842666666666666,
      "grad_norm": 2.6436843872070312,
      "learning_rate": 3.8144078272934676e-07,
      "logits/chosen": -1.727230429649353,
      "logits/rejected": -1.6059099435806274,
      "logps/chosen": -269.52691650390625,
      "logps/rejected": -352.0751647949219,
      "loss": 0.0784,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.8891433477401733,
      "rewards/margins": 6.365373611450195,
      "rewards/rejected": -8.2545166015625,
      "step": 1165
    },
    {
      "epoch": 2.4949333333333334,
      "grad_norm": 3.0084333419799805,
      "learning_rate": 3.6629635904222943e-07,
      "logits/chosen": -1.6209266185760498,
      "logits/rejected": -1.6020692586898804,
      "logps/chosen": -255.9269561767578,
      "logps/rejected": -354.1142578125,
      "loss": 0.12,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.0139718055725098,
      "rewards/margins": 5.429463863372803,
      "rewards/rejected": -7.4434356689453125,
      "step": 1170
    },
    {
      "epoch": 2.5056000000000003,
      "grad_norm": 3.403123378753662,
      "learning_rate": 3.514349053011698e-07,
      "logits/chosen": -1.653505563735962,
      "logits/rejected": -1.4818648099899292,
      "logps/chosen": -240.80331420898438,
      "logps/rejected": -323.3808288574219,
      "loss": 0.12,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.5888588428497314,
      "rewards/margins": 6.088988304138184,
      "rewards/rejected": -7.677847385406494,
      "step": 1175
    },
    {
      "epoch": 2.5162666666666667,
      "grad_norm": 4.319390296936035,
      "learning_rate": 3.3685839241941133e-07,
      "logits/chosen": -1.6171585321426392,
      "logits/rejected": -1.6007347106933594,
      "logps/chosen": -258.7629089355469,
      "logps/rejected": -342.47998046875,
      "loss": 0.1118,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.1584792137145996,
      "rewards/margins": 5.684945583343506,
      "rewards/rejected": -7.843425750732422,
      "step": 1180
    },
    {
      "epoch": 2.5269333333333335,
      "grad_norm": 4.567004680633545,
      "learning_rate": 3.2256875352158435e-07,
      "logits/chosen": -1.6689363718032837,
      "logits/rejected": -1.5358021259307861,
      "logps/chosen": -236.81808471679688,
      "logps/rejected": -322.1048583984375,
      "loss": 0.1405,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.7008771896362305,
      "rewards/margins": 5.543111801147461,
      "rewards/rejected": -7.243988990783691,
      "step": 1185
    },
    {
      "epoch": 2.5376,
      "grad_norm": 6.191208362579346,
      "learning_rate": 3.085678836873399e-07,
      "logits/chosen": -1.702701210975647,
      "logits/rejected": -1.6787986755371094,
      "logps/chosen": -249.1476593017578,
      "logps/rejected": -344.263671875,
      "loss": 0.1209,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.9360427856445312,
      "rewards/margins": 5.811527729034424,
      "rewards/rejected": -7.7475690841674805,
      "step": 1190
    },
    {
      "epoch": 2.5482666666666667,
      "grad_norm": 2.056534767150879,
      "learning_rate": 2.9485763970002106e-07,
      "logits/chosen": -1.675588607788086,
      "logits/rejected": -1.6271533966064453,
      "logps/chosen": -233.121826171875,
      "logps/rejected": -334.0964050292969,
      "loss": 0.1223,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.9534991979599,
      "rewards/margins": 5.506706237792969,
      "rewards/rejected": -7.460206508636475,
      "step": 1195
    },
    {
      "epoch": 2.558933333333333,
      "grad_norm": 6.617891788482666,
      "learning_rate": 2.8143983980042174e-07,
      "logits/chosen": -1.6359453201293945,
      "logits/rejected": -1.5568664073944092,
      "logps/chosen": -262.3926086425781,
      "logps/rejected": -335.15667724609375,
      "loss": 0.1334,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.1467742919921875,
      "rewards/margins": 5.749007701873779,
      "rewards/rejected": -7.895781517028809,
      "step": 1200
    },
    {
      "epoch": 2.5696,
      "grad_norm": 3.7044594287872314,
      "learning_rate": 2.683162634456515e-07,
      "logits/chosen": -1.583653211593628,
      "logits/rejected": -1.6004180908203125,
      "logps/chosen": -250.63845825195312,
      "logps/rejected": -325.78436279296875,
      "loss": 0.1644,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.9192173480987549,
      "rewards/margins": 5.139154434204102,
      "rewards/rejected": -7.058372497558594,
      "step": 1205
    },
    {
      "epoch": 2.5802666666666667,
      "grad_norm": 2.7366485595703125,
      "learning_rate": 2.5548865107314606e-07,
      "logits/chosen": -1.557058572769165,
      "logits/rejected": -1.5571913719177246,
      "logps/chosen": -270.0423583984375,
      "logps/rejected": -337.73614501953125,
      "loss": 0.1198,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.8408619165420532,
      "rewards/margins": 5.491454601287842,
      "rewards/rejected": -7.3323163986206055,
      "step": 1210
    },
    {
      "epoch": 2.590933333333333,
      "grad_norm": 2.9448840618133545,
      "learning_rate": 2.429587038698511e-07,
      "logits/chosen": -1.641669511795044,
      "logits/rejected": -1.5695855617523193,
      "logps/chosen": -234.04360961914062,
      "logps/rejected": -311.2100524902344,
      "loss": 0.122,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.0470471382141113,
      "rewards/margins": 5.0656328201293945,
      "rewards/rejected": -7.112679481506348,
      "step": 1215
    },
    {
      "epoch": 2.6016,
      "grad_norm": 2.9614179134368896,
      "learning_rate": 2.307280835466144e-07,
      "logits/chosen": -1.749356985092163,
      "logits/rejected": -1.6079270839691162,
      "logps/chosen": -251.8319854736328,
      "logps/rejected": -315.96710205078125,
      "loss": 0.1393,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.129053831100464,
      "rewards/margins": 5.405560493469238,
      "rewards/rejected": -7.534613609313965,
      "step": 1220
    },
    {
      "epoch": 2.6122666666666667,
      "grad_norm": 2.680088996887207,
      "learning_rate": 2.1879841211780856e-07,
      "logits/chosen": -1.5628279447555542,
      "logits/rejected": -1.4973697662353516,
      "logps/chosen": -252.55484008789062,
      "logps/rejected": -329.0304260253906,
      "loss": 0.1142,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.107846736907959,
      "rewards/margins": 5.468822002410889,
      "rewards/rejected": -7.576669216156006,
      "step": 1225
    },
    {
      "epoch": 2.622933333333333,
      "grad_norm": 3.77762508392334,
      "learning_rate": 2.0717127168622202e-07,
      "logits/chosen": -1.564953088760376,
      "logits/rejected": -1.4988367557525635,
      "logps/chosen": -257.25506591796875,
      "logps/rejected": -330.61724853515625,
      "loss": 0.1235,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.027205467224121,
      "rewards/margins": 5.905071258544922,
      "rewards/rejected": -7.932276248931885,
      "step": 1230
    },
    {
      "epoch": 2.6336,
      "grad_norm": 3.186311721801758,
      "learning_rate": 1.958482042332424e-07,
      "logits/chosen": -1.663132667541504,
      "logits/rejected": -1.5677236318588257,
      "logps/chosen": -243.77468872070312,
      "logps/rejected": -324.66485595703125,
      "loss": 0.1043,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.8823124170303345,
      "rewards/margins": 5.564373970031738,
      "rewards/rejected": -7.446686744689941,
      "step": 1235
    },
    {
      "epoch": 2.6442666666666668,
      "grad_norm": 1.523322343826294,
      "learning_rate": 1.8483071141435937e-07,
      "logits/chosen": -1.591597318649292,
      "logits/rejected": -1.5641478300094604,
      "logps/chosen": -278.78485107421875,
      "logps/rejected": -347.0000305175781,
      "loss": 0.116,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.0552477836608887,
      "rewards/margins": 5.445364475250244,
      "rewards/rejected": -7.500611782073975,
      "step": 1240
    },
    {
      "epoch": 2.654933333333333,
      "grad_norm": 2.483313798904419,
      "learning_rate": 1.741202543600176e-07,
      "logits/chosen": -1.658911108970642,
      "logits/rejected": -1.600517988204956,
      "logps/chosen": -239.78817749023438,
      "logps/rejected": -339.4964294433594,
      "loss": 0.1052,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.173642635345459,
      "rewards/margins": 6.087950706481934,
      "rewards/rejected": -8.261592864990234,
      "step": 1245
    },
    {
      "epoch": 2.6656,
      "grad_norm": 2.3703670501708984,
      "learning_rate": 1.6371825348184155e-07,
      "logits/chosen": -1.5409141778945923,
      "logits/rejected": -1.4761722087860107,
      "logps/chosen": -240.65487670898438,
      "logps/rejected": -336.3206787109375,
      "loss": 0.0976,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.9273293018341064,
      "rewards/margins": 6.195014953613281,
      "rewards/rejected": -8.122344017028809,
      "step": 1250
    },
    {
      "epoch": 2.676266666666667,
      "grad_norm": 3.7135093212127686,
      "learning_rate": 1.5362608828426246e-07,
      "logits/chosen": -1.6882669925689697,
      "logits/rejected": -1.6413158178329468,
      "logps/chosen": -259.45892333984375,
      "logps/rejected": -355.84356689453125,
      "loss": 0.0932,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.087570905685425,
      "rewards/margins": 5.9243245124816895,
      "rewards/rejected": -8.011896133422852,
      "step": 1255
    },
    {
      "epoch": 2.686933333333333,
      "grad_norm": 4.499937057495117,
      "learning_rate": 1.438450971815686e-07,
      "logits/chosen": -1.6984704732894897,
      "logits/rejected": -1.5697863101959229,
      "logps/chosen": -293.70849609375,
      "logps/rejected": -330.1623229980469,
      "loss": 0.0973,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.8370702266693115,
      "rewards/margins": 6.17175817489624,
      "rewards/rejected": -8.008829116821289,
      "step": 1260
    },
    {
      "epoch": 2.6976,
      "grad_norm": 2.924912691116333,
      "learning_rate": 1.3437657732040783e-07,
      "logits/chosen": -1.6595064401626587,
      "logits/rejected": -1.5919541120529175,
      "logps/chosen": -272.3826599121094,
      "logps/rejected": -333.85504150390625,
      "loss": 0.1171,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.263150691986084,
      "rewards/margins": 6.062061309814453,
      "rewards/rejected": -8.325211524963379,
      "step": 1265
    },
    {
      "epoch": 2.708266666666667,
      "grad_norm": 3.1616270542144775,
      "learning_rate": 1.2522178440776022e-07,
      "logits/chosen": -1.5888944864273071,
      "logits/rejected": -1.5637750625610352,
      "logps/chosen": -259.8056640625,
      "logps/rejected": -316.7781982421875,
      "loss": 0.1181,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.2415337562561035,
      "rewards/margins": 5.334750175476074,
      "rewards/rejected": -7.576283931732178,
      "step": 1270
    },
    {
      "epoch": 2.718933333333333,
      "grad_norm": 2.0143356323242188,
      "learning_rate": 1.1638193254440655e-07,
      "logits/chosen": -1.6921374797821045,
      "logits/rejected": -1.6408201456069946,
      "logps/chosen": -256.64093017578125,
      "logps/rejected": -332.04278564453125,
      "loss": 0.1319,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.222224473953247,
      "rewards/margins": 5.89315938949585,
      "rewards/rejected": -8.115384101867676,
      "step": 1275
    },
    {
      "epoch": 2.7296,
      "grad_norm": 2.776367425918579,
      "learning_rate": 1.0785819406391562e-07,
      "logits/chosen": -1.6165263652801514,
      "logits/rejected": -1.5132572650909424,
      "logps/chosen": -248.881103515625,
      "logps/rejected": -339.60540771484375,
      "loss": 0.112,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.215090274810791,
      "rewards/margins": 5.569876670837402,
      "rewards/rejected": -7.784966468811035,
      "step": 1280
    },
    {
      "epoch": 2.740266666666667,
      "grad_norm": 2.7171905040740967,
      "learning_rate": 9.965169937717095e-08,
      "logits/chosen": -1.6130340099334717,
      "logits/rejected": -1.5577963590621948,
      "logps/chosen": -260.16143798828125,
      "logps/rejected": -337.82611083984375,
      "loss": 0.098,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.314629316329956,
      "rewards/margins": 5.716673851013184,
      "rewards/rejected": -8.031302452087402,
      "step": 1285
    },
    {
      "epoch": 2.7509333333333332,
      "grad_norm": 2.7834761142730713,
      "learning_rate": 9.176353682245676e-08,
      "logits/chosen": -1.6675822734832764,
      "logits/rejected": -1.5338008403778076,
      "logps/chosen": -257.9503479003906,
      "logps/rejected": -337.56756591796875,
      "loss": 0.1559,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.267277717590332,
      "rewards/margins": 5.890148162841797,
      "rewards/rejected": -8.157425880432129,
      "step": 1290
    },
    {
      "epoch": 2.7616,
      "grad_norm": 1.5055266618728638,
      "learning_rate": 8.41947525211223e-08,
      "logits/chosen": -1.5467194318771362,
      "logits/rejected": -1.4370124340057373,
      "logps/chosen": -245.481689453125,
      "logps/rejected": -307.88970947265625,
      "loss": 0.1115,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.0703835487365723,
      "rewards/margins": 5.803430557250977,
      "rewards/rejected": -7.873814582824707,
      "step": 1295
    },
    {
      "epoch": 2.772266666666667,
      "grad_norm": 3.726200580596924,
      "learning_rate": 7.694635023884789e-08,
      "logits/chosen": -1.615465521812439,
      "logits/rejected": -1.5055599212646484,
      "logps/chosen": -261.68084716796875,
      "logps/rejected": -333.84735107421875,
      "loss": 0.1108,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.922776460647583,
      "rewards/margins": 5.716401100158691,
      "rewards/rejected": -7.6391777992248535,
      "step": 1300
    },
    {
      "epoch": 2.7829333333333333,
      "grad_norm": 3.27950382232666,
      "learning_rate": 7.00192912525241e-08,
      "logits/chosen": -1.7413036823272705,
      "logits/rejected": -1.6767768859863281,
      "logps/chosen": -249.2274627685547,
      "logps/rejected": -324.9376525878906,
      "loss": 0.1504,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.0574612617492676,
      "rewards/margins": 5.184039115905762,
      "rewards/rejected": -7.2415008544921875,
      "step": 1305
    },
    {
      "epoch": 2.7936,
      "grad_norm": 2.958669900894165,
      "learning_rate": 6.341449422277047e-08,
      "logits/chosen": -1.6207433938980103,
      "logits/rejected": -1.5499709844589233,
      "logps/chosen": -271.83709716796875,
      "logps/rejected": -333.00177001953125,
      "loss": 0.0846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.2863690853118896,
      "rewards/margins": 5.1181640625,
      "rewards/rejected": -7.404533386230469,
      "step": 1310
    },
    {
      "epoch": 2.804266666666667,
      "grad_norm": 3.068798303604126,
      "learning_rate": 5.7132835072101486e-08,
      "logits/chosen": -1.7072492837905884,
      "logits/rejected": -1.6401913166046143,
      "logps/chosen": -248.870849609375,
      "logps/rejected": -345.37939453125,
      "loss": 0.1536,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.2537102699279785,
      "rewards/margins": 5.832470893859863,
      "rewards/rejected": -8.086182594299316,
      "step": 1315
    },
    {
      "epoch": 2.8149333333333333,
      "grad_norm": 2.465273141860962,
      "learning_rate": 5.117514686876379e-08,
      "logits/chosen": -1.6905529499053955,
      "logits/rejected": -1.5897334814071655,
      "logps/chosen": -234.72500610351562,
      "logps/rejected": -331.4391174316406,
      "loss": 0.1376,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.9037342071533203,
      "rewards/margins": 5.567811965942383,
      "rewards/rejected": -7.4715471267700195,
      "step": 1320
    },
    {
      "epoch": 2.8256,
      "grad_norm": 5.49542760848999,
      "learning_rate": 4.5542219716255274e-08,
      "logits/chosen": -1.62538743019104,
      "logits/rejected": -1.5213114023208618,
      "logps/chosen": -267.12176513671875,
      "logps/rejected": -337.9035949707031,
      "loss": 0.1046,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.3189761638641357,
      "rewards/margins": 5.719181060791016,
      "rewards/rejected": -8.03815746307373,
      "step": 1325
    },
    {
      "epoch": 2.836266666666667,
      "grad_norm": 2.2211387157440186,
      "learning_rate": 4.023480064854174e-08,
      "logits/chosen": -1.6421716213226318,
      "logits/rejected": -1.6016359329223633,
      "logps/chosen": -257.92681884765625,
      "logps/rejected": -335.91326904296875,
      "loss": 0.1023,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.207742214202881,
      "rewards/margins": 5.49573278427124,
      "rewards/rejected": -7.703474998474121,
      "step": 1330
    },
    {
      "epoch": 2.8469333333333333,
      "grad_norm": 1.8862336874008179,
      "learning_rate": 3.525359353098584e-08,
      "logits/chosen": -1.7027902603149414,
      "logits/rejected": -1.6094478368759155,
      "logps/chosen": -240.2790069580078,
      "logps/rejected": -345.55712890625,
      "loss": 0.0974,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.9166862964630127,
      "rewards/margins": 6.030681610107422,
      "rewards/rejected": -7.947368621826172,
      "step": 1335
    },
    {
      "epoch": 2.8576,
      "grad_norm": 1.444413661956787,
      "learning_rate": 3.059925896700094e-08,
      "logits/chosen": -1.6089794635772705,
      "logits/rejected": -1.5267149209976196,
      "logps/chosen": -269.1159362792969,
      "logps/rejected": -328.16534423828125,
      "loss": 0.1216,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.9404586553573608,
      "rewards/margins": 5.162356376647949,
      "rewards/rejected": -7.1028151512146,
      "step": 1340
    },
    {
      "epoch": 2.8682666666666665,
      "grad_norm": 5.427722454071045,
      "learning_rate": 2.6272414210443964e-08,
      "logits/chosen": -1.6677299737930298,
      "logits/rejected": -1.625317931175232,
      "logps/chosen": -269.1617736816406,
      "logps/rejected": -345.4956970214844,
      "loss": 0.1349,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.075646162033081,
      "rewards/margins": 5.521030426025391,
      "rewards/rejected": -7.596676826477051,
      "step": 1345
    },
    {
      "epoch": 2.8789333333333333,
      "grad_norm": 1.782738208770752,
      "learning_rate": 2.2273633083753632e-08,
      "logits/chosen": -1.7253984212875366,
      "logits/rejected": -1.5926178693771362,
      "logps/chosen": -249.1942596435547,
      "logps/rejected": -338.21722412109375,
      "loss": 0.1067,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.3312084674835205,
      "rewards/margins": 6.381017208099365,
      "rewards/rejected": -8.712225914001465,
      "step": 1350
    },
    {
      "epoch": 2.8895999999999997,
      "grad_norm": 5.87308931350708,
      "learning_rate": 1.8603445901851647e-08,
      "logits/chosen": -1.621783971786499,
      "logits/rejected": -1.5659376382827759,
      "logps/chosen": -252.837646484375,
      "logps/rejected": -325.2681884765625,
      "loss": 0.1463,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.2585935592651367,
      "rewards/margins": 5.560887336730957,
      "rewards/rejected": -7.819480895996094,
      "step": 1355
    },
    {
      "epoch": 2.9002666666666665,
      "grad_norm": 6.012791633605957,
      "learning_rate": 1.5262339401813375e-08,
      "logits/chosen": -1.684981346130371,
      "logits/rejected": -1.6372852325439453,
      "logps/chosen": -250.5609130859375,
      "logps/rejected": -345.0445556640625,
      "loss": 0.145,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.3670361042022705,
      "rewards/margins": 5.3330302238464355,
      "rewards/rejected": -7.700067043304443,
      "step": 1360
    },
    {
      "epoch": 2.9109333333333334,
      "grad_norm": 3.0086324214935303,
      "learning_rate": 1.2250756678315601e-08,
      "logits/chosen": -1.6575305461883545,
      "logits/rejected": -1.5837920904159546,
      "logps/chosen": -269.42578125,
      "logps/rejected": -344.31793212890625,
      "loss": 0.092,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.132127046585083,
      "rewards/margins": 5.720582962036133,
      "rewards/rejected": -7.852709770202637,
      "step": 1365
    },
    {
      "epoch": 2.9215999999999998,
      "grad_norm": 2.5158169269561768,
      "learning_rate": 9.569097124875193e-09,
      "logits/chosen": -1.636508584022522,
      "logits/rejected": -1.567469596862793,
      "logps/chosen": -243.44912719726562,
      "logps/rejected": -332.2774353027344,
      "loss": 0.1252,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.221357822418213,
      "rewards/margins": 5.588191032409668,
      "rewards/rejected": -7.809549808502197,
      "step": 1370
    },
    {
      "epoch": 2.9322666666666666,
      "grad_norm": 3.4159395694732666,
      "learning_rate": 7.217716380881479e-09,
      "logits/chosen": -1.6869888305664062,
      "logits/rejected": -1.615618348121643,
      "logps/chosen": -250.40011596679688,
      "logps/rejected": -348.78521728515625,
      "loss": 0.0966,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.015139102935791,
      "rewards/margins": 5.993285179138184,
      "rewards/rejected": -8.008424758911133,
      "step": 1375
    },
    {
      "epoch": 2.9429333333333334,
      "grad_norm": 2.5707411766052246,
      "learning_rate": 5.196926284430359e-09,
      "logits/chosen": -1.6021541357040405,
      "logits/rejected": -1.5102264881134033,
      "logps/chosen": -261.15020751953125,
      "logps/rejected": -347.87188720703125,
      "loss": 0.098,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.2800986766815186,
      "rewards/margins": 6.126519680023193,
      "rewards/rejected": -8.40661907196045,
      "step": 1380
    },
    {
      "epoch": 2.9536,
      "grad_norm": 2.2150466442108154,
      "learning_rate": 3.5069948309712775e-09,
      "logits/chosen": -1.6135914325714111,
      "logits/rejected": -1.5576746463775635,
      "logps/chosen": -262.16571044921875,
      "logps/rejected": -354.02056884765625,
      "loss": 0.1059,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.9849374294281006,
      "rewards/margins": 5.869291305541992,
      "rewards/rejected": -7.8542280197143555,
      "step": 1385
    },
    {
      "epoch": 2.9642666666666666,
      "grad_norm": 3.714444637298584,
      "learning_rate": 2.1481461377634294e-09,
      "logits/chosen": -1.6816928386688232,
      "logits/rejected": -1.6221061944961548,
      "logps/chosen": -265.137451171875,
      "logps/rejected": -331.76727294921875,
      "loss": 0.1451,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.0669548511505127,
      "rewards/margins": 5.562548637390137,
      "rewards/rejected": -7.629502773284912,
      "step": 1390
    },
    {
      "epoch": 2.9749333333333334,
      "grad_norm": 2.0670387744903564,
      "learning_rate": 1.120560414154259e-09,
      "logits/chosen": -1.5556447505950928,
      "logits/rejected": -1.5012569427490234,
      "logps/chosen": -235.62307739257812,
      "logps/rejected": -331.75531005859375,
      "loss": 0.1097,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.9400962591171265,
      "rewards/margins": 5.777553081512451,
      "rewards/rejected": -7.717649936676025,
      "step": 1395
    },
    {
      "epoch": 2.9856,
      "grad_norm": 3.319321632385254,
      "learning_rate": 4.2437393768079983e-10,
      "logits/chosen": -1.6425552368164062,
      "logits/rejected": -1.5908761024475098,
      "logps/chosen": -259.075439453125,
      "logps/rejected": -341.9222412109375,
      "loss": 0.1158,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.9800103902816772,
      "rewards/margins": 6.281598091125488,
      "rewards/rejected": -8.261608123779297,
      "step": 1400
    },
    {
      "epoch": 2.9962666666666666,
      "grad_norm": 3.756913423538208,
      "learning_rate": 5.967903599635171e-11,
      "logits/chosen": -1.6659742593765259,
      "logits/rejected": -1.5326024293899536,
      "logps/chosen": -264.480712890625,
      "logps/rejected": -333.2155456542969,
      "loss": 0.1354,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.168793201446533,
      "rewards/margins": 5.448444843292236,
      "rewards/rejected": -7.617238521575928,
      "step": 1405
    }
  ],
  "logging_steps": 5,
  "max_steps": 1407,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}