{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 1407,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010666666666666666,
      "grad_norm": 3.040982484817505,
      "learning_rate": 4.651162790697675e-07,
      "logits/chosen": -1.4189865589141846,
      "logits/rejected": -1.2718557119369507,
      "logps/chosen": -232.1872100830078,
      "logps/rejected": -285.8193359375,
      "loss": 0.693,
      "rewards/accuracies": 0.3187499940395355,
      "rewards/chosen": -0.0001921796938404441,
      "rewards/margins": 0.00023017881903797388,
      "rewards/rejected": -0.0004223585710860789,
      "step": 5
    },
    {
      "epoch": 0.021333333333333333,
      "grad_norm": 2.945732355117798,
      "learning_rate": 1.0465116279069768e-06,
      "logits/chosen": -1.5402610301971436,
      "logits/rejected": -1.3184393644332886,
      "logps/chosen": -230.44815063476562,
      "logps/rejected": -273.22674560546875,
      "loss": 0.69,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.00048116204561665654,
      "rewards/margins": 0.006424086634069681,
      "rewards/rejected": -0.0069052488543093204,
      "step": 10
    },
    {
      "epoch": 0.032,
      "grad_norm": 3.0436723232269287,
      "learning_rate": 1.6279069767441862e-06,
      "logits/chosen": -1.4266681671142578,
      "logits/rejected": -1.2699027061462402,
      "logps/chosen": -214.630859375,
      "logps/rejected": -266.53839111328125,
      "loss": 0.684,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.002722587902098894,
      "rewards/margins": 0.018564948812127113,
      "rewards/rejected": -0.02128753624856472,
      "step": 15
    },
    {
      "epoch": 0.042666666666666665,
      "grad_norm": 3.2988665103912354,
      "learning_rate": 2.2093023255813954e-06,
      "logits/chosen": -1.3134950399398804,
      "logits/rejected": -1.1439664363861084,
      "logps/chosen": -224.536376953125,
      "logps/rejected": -263.8288269042969,
      "loss": 0.6772,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.005389872007071972,
      "rewards/margins": 0.03269362449645996,
      "rewards/rejected": -0.03808349743485451,
      "step": 20
    },
    {
      "epoch": 0.05333333333333334,
      "grad_norm": 2.985452890396118,
      "learning_rate": 2.790697674418605e-06,
      "logits/chosen": -1.4989473819732666,
      "logits/rejected": -1.2572524547576904,
      "logps/chosen": -206.94247436523438,
      "logps/rejected": -274.64508056640625,
      "loss": 0.6598,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.002713642315939069,
      "rewards/margins": 0.0694461390376091,
      "rewards/rejected": -0.0721597820520401,
      "step": 25
    },
    {
      "epoch": 0.064,
      "grad_norm": 3.0096566677093506,
      "learning_rate": 3.372093023255814e-06,
      "logits/chosen": -1.3679146766662598,
      "logits/rejected": -1.1917188167572021,
      "logps/chosen": -244.6914825439453,
      "logps/rejected": -277.0018310546875,
      "loss": 0.6441,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.0334165021777153,
      "rewards/margins": 0.10440608114004135,
      "rewards/rejected": -0.13782256841659546,
      "step": 30
    },
    {
      "epoch": 0.07466666666666667,
      "grad_norm": 3.8459115028381348,
      "learning_rate": 3.953488372093024e-06,
      "logits/chosen": -1.4499976634979248,
      "logits/rejected": -1.2488772869110107,
      "logps/chosen": -236.5290069580078,
      "logps/rejected": -287.55169677734375,
      "loss": 0.6063,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.033719420433044434,
      "rewards/margins": 0.19089257717132568,
      "rewards/rejected": -0.2246120274066925,
      "step": 35
    },
    {
      "epoch": 0.08533333333333333,
      "grad_norm": 2.8168575763702393,
      "learning_rate": 4.5348837209302326e-06,
      "logits/chosen": -1.4405505657196045,
      "logits/rejected": -1.2370808124542236,
      "logps/chosen": -234.57730102539062,
      "logps/rejected": -293.7424621582031,
      "loss": 0.594,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.06413023173809052,
      "rewards/margins": 0.22726896405220032,
      "rewards/rejected": -0.291399210691452,
      "step": 40
    },
    {
      "epoch": 0.096,
      "grad_norm": 3.0759992599487305,
      "learning_rate": 4.99999336897255e-06,
      "logits/chosen": -1.4353830814361572,
      "logits/rejected": -1.2781504392623901,
      "logps/chosen": -232.90866088867188,
      "logps/rejected": -282.9027404785156,
      "loss": 0.5602,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.07559499889612198,
      "rewards/margins": 0.31765851378440857,
      "rewards/rejected": -0.39325350522994995,
      "step": 45
    },
    {
      "epoch": 0.10666666666666667,
      "grad_norm": 3.0262410640716553,
      "learning_rate": 4.999761286705285e-06,
      "logits/chosen": -1.5332512855529785,
      "logits/rejected": -1.4040536880493164,
      "logps/chosen": -221.7960662841797,
      "logps/rejected": -294.1759033203125,
      "loss": 0.4985,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.04457804188132286,
      "rewards/margins": 0.4884178638458252,
      "rewards/rejected": -0.5329959392547607,
      "step": 50
    },
    {
      "epoch": 0.11733333333333333,
      "grad_norm": 3.2148373126983643,
      "learning_rate": 4.999197688241076e-06,
      "logits/chosen": -1.4515631198883057,
      "logits/rejected": -1.2591352462768555,
      "logps/chosen": -216.49111938476562,
      "logps/rejected": -275.7051086425781,
      "loss": 0.4696,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.11984169483184814,
      "rewards/margins": 0.6132544279098511,
      "rewards/rejected": -0.7330960035324097,
      "step": 55
    },
    {
      "epoch": 0.128,
      "grad_norm": 2.543278694152832,
      "learning_rate": 4.9983026483238685e-06,
      "logits/chosen": -1.5313522815704346,
      "logits/rejected": -1.3514273166656494,
      "logps/chosen": -221.7779998779297,
      "logps/rejected": -289.889404296875,
      "loss": 0.4535,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.18022173643112183,
      "rewards/margins": 0.6861358880996704,
      "rewards/rejected": -0.8663576245307922,
      "step": 60
    },
    {
      "epoch": 0.13866666666666666,
      "grad_norm": 2.6283793449401855,
      "learning_rate": 4.997076285653087e-06,
      "logits/chosen": -1.5744426250457764,
      "logits/rejected": -1.3084551095962524,
      "logps/chosen": -235.96768188476562,
      "logps/rejected": -293.5603942871094,
      "loss": 0.4044,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.23008236289024353,
      "rewards/margins": 0.9606491923332214,
      "rewards/rejected": -1.1907316446304321,
      "step": 65
    },
    {
      "epoch": 0.14933333333333335,
      "grad_norm": 2.195939064025879,
      "learning_rate": 4.995518762867897e-06,
      "logits/chosen": -1.60664963722229,
      "logits/rejected": -1.3430724143981934,
      "logps/chosen": -218.2381591796875,
      "logps/rejected": -294.97222900390625,
      "loss": 0.359,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.27671200037002563,
      "rewards/margins": 1.1187210083007812,
      "rewards/rejected": -1.395432949066162,
      "step": 70
    },
    {
      "epoch": 0.16,
      "grad_norm": 3.54109263420105,
      "learning_rate": 4.993630286525634e-06,
      "logits/chosen": -1.5840482711791992,
      "logits/rejected": -1.3805081844329834,
      "logps/chosen": -232.5129852294922,
      "logps/rejected": -295.33099365234375,
      "loss": 0.3903,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.47087717056274414,
      "rewards/margins": 1.1671949625015259,
      "rewards/rejected": -1.6380720138549805,
      "step": 75
    },
    {
      "epoch": 0.17066666666666666,
      "grad_norm": 1.889794945716858,
      "learning_rate": 4.99141110707441e-06,
      "logits/chosen": -1.570359230041504,
      "logits/rejected": -1.364939570426941,
      "logps/chosen": -207.35525512695312,
      "logps/rejected": -281.3659973144531,
      "loss": 0.3623,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.21106550097465515,
      "rewards/margins": 1.3174269199371338,
      "rewards/rejected": -1.5284923315048218,
      "step": 80
    },
    {
      "epoch": 0.18133333333333335,
      "grad_norm": 3.4203381538391113,
      "learning_rate": 4.9888615188199e-06,
      "logits/chosen": -1.640369176864624,
      "logits/rejected": -1.4251915216445923,
      "logps/chosen": -222.8439178466797,
      "logps/rejected": -288.3473815917969,
      "loss": 0.3774,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.3367365002632141,
      "rewards/margins": 1.3324146270751953,
      "rewards/rejected": -1.6691510677337646,
      "step": 85
    },
    {
      "epoch": 0.192,
      "grad_norm": 3.1954681873321533,
      "learning_rate": 4.9859818598863104e-06,
      "logits/chosen": -1.55571711063385,
      "logits/rejected": -1.3716849088668823,
      "logps/chosen": -226.4812774658203,
      "logps/rejected": -290.22906494140625,
      "loss": 0.3714,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.3206635117530823,
      "rewards/margins": 1.3784153461456299,
      "rewards/rejected": -1.6990787982940674,
      "step": 90
    },
    {
      "epoch": 0.20266666666666666,
      "grad_norm": 2.557746648788452,
      "learning_rate": 4.982772512171538e-06,
      "logits/chosen": -1.6502959728240967,
      "logits/rejected": -1.453385353088379,
      "logps/chosen": -233.37594604492188,
      "logps/rejected": -294.97259521484375,
      "loss": 0.3449,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.4198358654975891,
      "rewards/margins": 1.5545928478240967,
      "rewards/rejected": -1.9744285345077515,
      "step": 95
    },
    {
      "epoch": 0.21333333333333335,
      "grad_norm": 2.5644826889038086,
      "learning_rate": 4.979233901296523e-06,
      "logits/chosen": -1.6967132091522217,
      "logits/rejected": -1.5241398811340332,
      "logps/chosen": -242.16000366210938,
      "logps/rejected": -305.0459289550781,
      "loss": 0.3319,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.45428210496902466,
      "rewards/margins": 1.6128562688827515,
      "rewards/rejected": -2.067138433456421,
      "step": 100
    },
    {
      "epoch": 0.224,
      "grad_norm": 3.155392646789551,
      "learning_rate": 4.975366496548804e-06,
      "logits/chosen": -1.6549123525619507,
      "logits/rejected": -1.411078691482544,
      "logps/chosen": -240.0164794921875,
      "logps/rejected": -305.4057922363281,
      "loss": 0.2697,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.26041409373283386,
      "rewards/margins": 1.9732940196990967,
      "rewards/rejected": -2.233708143234253,
      "step": 105
    },
    {
      "epoch": 0.23466666666666666,
      "grad_norm": 2.429647445678711,
      "learning_rate": 4.971170810820279e-06,
      "logits/chosen": -1.6650527715682983,
      "logits/rejected": -1.4914207458496094,
      "logps/chosen": -222.6490478515625,
      "logps/rejected": -295.0611877441406,
      "loss": 0.3103,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.23167634010314941,
      "rewards/margins": 1.8154058456420898,
      "rewards/rejected": -2.0470821857452393,
      "step": 110
    },
    {
      "epoch": 0.24533333333333332,
      "grad_norm": 3.5919618606567383,
      "learning_rate": 4.96664740053919e-06,
      "logits/chosen": -1.6617170572280884,
      "logits/rejected": -1.5017478466033936,
      "logps/chosen": -220.0016326904297,
      "logps/rejected": -293.1516418457031,
      "loss": 0.3044,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.28731054067611694,
      "rewards/margins": 1.8548088073730469,
      "rewards/rejected": -2.1421194076538086,
      "step": 115
    },
    {
      "epoch": 0.256,
      "grad_norm": 2.7757253646850586,
      "learning_rate": 4.961796865596328e-06,
      "logits/chosen": -1.6444618701934814,
      "logits/rejected": -1.3935641050338745,
      "logps/chosen": -221.69973754882812,
      "logps/rejected": -286.38458251953125,
      "loss": 0.2808,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.2497580498456955,
      "rewards/margins": 1.8792965412139893,
      "rewards/rejected": -2.129054546356201,
      "step": 120
    },
    {
      "epoch": 0.26666666666666666,
      "grad_norm": 1.7543736696243286,
      "learning_rate": 4.956619849265473e-06,
      "logits/chosen": -1.7066230773925781,
      "logits/rejected": -1.5209218263626099,
      "logps/chosen": -214.9864044189453,
      "logps/rejected": -305.9405212402344,
      "loss": 0.2312,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.12859247624874115,
      "rewards/margins": 2.200359344482422,
      "rewards/rejected": -2.3289520740509033,
      "step": 125
    },
    {
      "epoch": 0.2773333333333333,
      "grad_norm": 2.8612709045410156,
      "learning_rate": 4.951117038118091e-06,
      "logits/chosen": -1.7071641683578491,
      "logits/rejected": -1.5247122049331665,
      "logps/chosen": -246.58291625976562,
      "logps/rejected": -303.3542785644531,
      "loss": 0.2443,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.3323816657066345,
      "rewards/margins": 2.2684006690979004,
      "rewards/rejected": -2.6007821559906006,
      "step": 130
    },
    {
      "epoch": 0.288,
      "grad_norm": 3.071027994155884,
      "learning_rate": 4.945289161932273e-06,
      "logits/chosen": -1.7539116144180298,
      "logits/rejected": -1.6009010076522827,
      "logps/chosen": -239.4345245361328,
      "logps/rejected": -295.87200927734375,
      "loss": 0.2486,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.12532970309257507,
      "rewards/margins": 2.3635432720184326,
      "rewards/rejected": -2.48887300491333,
      "step": 135
    },
    {
      "epoch": 0.2986666666666667,
      "grad_norm": 2.4682374000549316,
      "learning_rate": 4.93913699359596e-06,
      "logits/chosen": -1.7155869007110596,
      "logits/rejected": -1.4812595844268799,
      "logps/chosen": -227.5341796875,
      "logps/rejected": -289.00030517578125,
      "loss": 0.2207,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.25302743911743164,
      "rewards/margins": 2.457817554473877,
      "rewards/rejected": -2.7108449935913086,
      "step": 140
    },
    {
      "epoch": 0.30933333333333335,
      "grad_norm": 3.2257938385009766,
      "learning_rate": 4.9326613490044364e-06,
      "logits/chosen": -1.6698166131973267,
      "logits/rejected": -1.486440896987915,
      "logps/chosen": -238.375,
      "logps/rejected": -297.738525390625,
      "loss": 0.2757,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.17056556046009064,
      "rewards/margins": 2.2673239707946777,
      "rewards/rejected": -2.437889575958252,
      "step": 145
    },
    {
      "epoch": 0.32,
      "grad_norm": 2.9405577182769775,
      "learning_rate": 4.925863086952132e-06,
      "logits/chosen": -1.7178294658660889,
      "logits/rejected": -1.480749249458313,
      "logps/chosen": -223.380615234375,
      "logps/rejected": -290.5135803222656,
      "loss": 0.2558,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.057668160647153854,
      "rewards/margins": 2.5294711589813232,
      "rewards/rejected": -2.587139129638672,
      "step": 150
    },
    {
      "epoch": 0.33066666666666666,
      "grad_norm": 1.0869413614273071,
      "learning_rate": 4.918743109018727e-06,
      "logits/chosen": -1.8033220767974854,
      "logits/rejected": -1.534440040588379,
      "logps/chosen": -211.77084350585938,
      "logps/rejected": -310.13201904296875,
      "loss": 0.1938,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.08430004119873047,
      "rewards/margins": 2.9461827278137207,
      "rewards/rejected": -3.030482530593872,
      "step": 155
    },
    {
      "epoch": 0.3413333333333333,
      "grad_norm": 1.6788111925125122,
      "learning_rate": 4.911302359449585e-06,
      "logits/chosen": -1.8236114978790283,
      "logits/rejected": -1.563982367515564,
      "logps/chosen": -213.33139038085938,
      "logps/rejected": -300.1676940917969,
      "loss": 0.1788,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.054857976734638214,
      "rewards/margins": 2.9758267402648926,
      "rewards/rejected": -3.03068470954895,
      "step": 160
    },
    {
      "epoch": 0.352,
      "grad_norm": 2.340954303741455,
      "learning_rate": 4.9035418250305314e-06,
      "logits/chosen": -1.6831834316253662,
      "logits/rejected": -1.4661271572113037,
      "logps/chosen": -228.6976318359375,
      "logps/rejected": -309.2475891113281,
      "loss": 0.2267,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.21103031933307648,
      "rewards/margins": 2.8673250675201416,
      "rewards/rejected": -3.078355312347412,
      "step": 165
    },
    {
      "epoch": 0.3626666666666667,
      "grad_norm": 3.3007848262786865,
      "learning_rate": 4.895462534956981e-06,
      "logits/chosen": -1.7243316173553467,
      "logits/rejected": -1.5944850444793701,
      "logps/chosen": -237.75094604492188,
      "logps/rejected": -310.4878234863281,
      "loss": 0.1998,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.272445410490036,
      "rewards/margins": 3.051952362060547,
      "rewards/rejected": -3.324397563934326,
      "step": 170
    },
    {
      "epoch": 0.37333333333333335,
      "grad_norm": 2.133695363998413,
      "learning_rate": 4.887065560697449e-06,
      "logits/chosen": -1.660601258277893,
      "logits/rejected": -1.5193674564361572,
      "logps/chosen": -235.02200317382812,
      "logps/rejected": -330.9132995605469,
      "loss": 0.2288,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.2509080469608307,
      "rewards/margins": 3.1449391841888428,
      "rewards/rejected": -3.3958473205566406,
      "step": 175
    },
    {
      "epoch": 0.384,
      "grad_norm": 2.8581511974334717,
      "learning_rate": 4.878352015851459e-06,
      "logits/chosen": -1.7492008209228516,
      "logits/rejected": -1.5329986810684204,
      "logps/chosen": -305.83502197265625,
      "logps/rejected": -326.94097900390625,
      "loss": 0.2059,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.5418329238891602,
      "rewards/margins": 3.5084152221679688,
      "rewards/rejected": -4.050248146057129,
      "step": 180
    },
    {
      "epoch": 0.39466666666666667,
      "grad_norm": 1.8751473426818848,
      "learning_rate": 4.869323056001852e-06,
      "logits/chosen": -1.639556884765625,
      "logits/rejected": -1.5379990339279175,
      "logps/chosen": -244.7671356201172,
      "logps/rejected": -285.7706298828125,
      "loss": 0.3078,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.567600429058075,
      "rewards/margins": 2.8157169818878174,
      "rewards/rejected": -3.383317470550537,
      "step": 185
    },
    {
      "epoch": 0.4053333333333333,
      "grad_norm": 2.4277961254119873,
      "learning_rate": 4.859979878561536e-06,
      "logits/chosen": -1.7614755630493164,
      "logits/rejected": -1.5140868425369263,
      "logps/chosen": -229.64779663085938,
      "logps/rejected": -282.7133483886719,
      "loss": 0.2575,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.49064379930496216,
      "rewards/margins": 3.128330945968628,
      "rewards/rejected": -3.6189746856689453,
      "step": 190
    },
    {
      "epoch": 0.416,
      "grad_norm": 3.492586612701416,
      "learning_rate": 4.850323722614685e-06,
      "logits/chosen": -1.8276838064193726,
      "logits/rejected": -1.6381219625473022,
      "logps/chosen": -224.7960205078125,
      "logps/rejected": -312.1609191894531,
      "loss": 0.2089,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.38933640718460083,
      "rewards/margins": 3.459027051925659,
      "rewards/rejected": -3.8483633995056152,
      "step": 195
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 2.13981556892395,
      "learning_rate": 4.840355868752416e-06,
      "logits/chosen": -1.7009217739105225,
      "logits/rejected": -1.4874917268753052,
      "logps/chosen": -281.18817138671875,
      "logps/rejected": -294.76593017578125,
      "loss": 0.277,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.39215925335884094,
      "rewards/margins": 2.851746082305908,
      "rewards/rejected": -3.2439048290252686,
      "step": 200
    },
    {
      "epoch": 0.43733333333333335,
      "grad_norm": 1.7804917097091675,
      "learning_rate": 4.830077638902955e-06,
      "logits/chosen": -1.8614517450332642,
      "logits/rejected": -1.6383779048919678,
      "logps/chosen": -205.98171997070312,
      "logps/rejected": -303.1836853027344,
      "loss": 0.1783,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.43484029173851013,
      "rewards/margins": 3.4785313606262207,
      "rewards/rejected": -3.9133713245391846,
      "step": 205
    },
    {
      "epoch": 0.448,
      "grad_norm": 3.0044808387756348,
      "learning_rate": 4.8194903961563255e-06,
      "logits/chosen": -1.9666965007781982,
      "logits/rejected": -1.7533382177352905,
      "logps/chosen": -202.64224243164062,
      "logps/rejected": -304.18115234375,
      "loss": 0.1671,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.2894894480705261,
      "rewards/margins": 3.478058338165283,
      "rewards/rejected": -3.767547607421875,
      "step": 210
    },
    {
      "epoch": 0.45866666666666667,
      "grad_norm": 2.3777949810028076,
      "learning_rate": 4.808595544583578e-06,
      "logits/chosen": -1.802603006362915,
      "logits/rejected": -1.5886794328689575,
      "logps/chosen": -240.7107391357422,
      "logps/rejected": -315.6220397949219,
      "loss": 0.1541,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.681304395198822,
      "rewards/margins": 3.757567882537842,
      "rewards/rejected": -4.438872337341309,
      "step": 215
    },
    {
      "epoch": 0.4693333333333333,
      "grad_norm": 2.1798269748687744,
      "learning_rate": 4.797394529050577e-06,
      "logits/chosen": -1.7322242259979248,
      "logits/rejected": -1.5891096591949463,
      "logps/chosen": -243.38955688476562,
      "logps/rejected": -320.7882080078125,
      "loss": 0.1923,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.6223587989807129,
      "rewards/margins": 3.5130057334899902,
      "rewards/rejected": -4.135365009307861,
      "step": 220
    },
    {
      "epoch": 0.48,
      "grad_norm": 2.2701711654663086,
      "learning_rate": 4.785888835026393e-06,
      "logits/chosen": -1.8017656803131104,
      "logits/rejected": -1.5656979084014893,
      "logps/chosen": -228.77890014648438,
      "logps/rejected": -307.7184143066406,
      "loss": 0.1845,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6147029995918274,
      "rewards/margins": 3.9243907928466797,
      "rewards/rejected": -4.539093971252441,
      "step": 225
    },
    {
      "epoch": 0.49066666666666664,
      "grad_norm": 2.7234058380126953,
      "learning_rate": 4.7740799883862966e-06,
      "logits/chosen": -1.8135614395141602,
      "logits/rejected": -1.704559564590454,
      "logps/chosen": -225.6403350830078,
      "logps/rejected": -312.3384704589844,
      "loss": 0.1871,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.6364536881446838,
      "rewards/margins": 3.503260374069214,
      "rewards/rejected": -4.139714241027832,
      "step": 230
    },
    {
      "epoch": 0.5013333333333333,
      "grad_norm": 1.9700164794921875,
      "learning_rate": 4.761969555209394e-06,
      "logits/chosen": -1.844793677330017,
      "logits/rejected": -1.7481451034545898,
      "logps/chosen": -218.34042358398438,
      "logps/rejected": -332.3699951171875,
      "loss": 0.1436,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.7182143926620483,
      "rewards/margins": 4.262960910797119,
      "rewards/rejected": -4.981175422668457,
      "step": 235
    },
    {
      "epoch": 0.512,
      "grad_norm": 2.5656299591064453,
      "learning_rate": 4.7495591415709435e-06,
      "logits/chosen": -1.8036201000213623,
      "logits/rejected": -1.6583620309829712,
      "logps/chosen": -238.70132446289062,
      "logps/rejected": -306.24420166015625,
      "loss": 0.2146,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.8436425924301147,
      "rewards/margins": 3.7423672676086426,
      "rewards/rejected": -4.586009979248047,
      "step": 240
    },
    {
      "epoch": 0.5226666666666666,
      "grad_norm": 1.9172959327697754,
      "learning_rate": 4.736850393329353e-06,
      "logits/chosen": -1.8544458150863647,
      "logits/rejected": -1.6191198825836182,
      "logps/chosen": -232.9375457763672,
      "logps/rejected": -318.1675720214844,
      "loss": 0.1511,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5402190089225769,
      "rewards/margins": 4.564599990844727,
      "rewards/rejected": -5.104818344116211,
      "step": 245
    },
    {
      "epoch": 0.5333333333333333,
      "grad_norm": 4.023538112640381,
      "learning_rate": 4.7238449959079095e-06,
      "logits/chosen": -1.9062020778656006,
      "logits/rejected": -1.7378747463226318,
      "logps/chosen": -229.92919921875,
      "logps/rejected": -324.6113586425781,
      "loss": 0.1818,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.6886031031608582,
      "rewards/margins": 4.174241542816162,
      "rewards/rejected": -4.862845420837402,
      "step": 250
    },
    {
      "epoch": 0.544,
      "grad_norm": 2.3114891052246094,
      "learning_rate": 4.710544674071259e-06,
      "logits/chosen": -1.920933723449707,
      "logits/rejected": -1.7430894374847412,
      "logps/chosen": -231.68533325195312,
      "logps/rejected": -307.49627685546875,
      "loss": 0.16,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.701662003993988,
      "rewards/margins": 4.109788417816162,
      "rewards/rejected": -4.811450481414795,
      "step": 255
    },
    {
      "epoch": 0.5546666666666666,
      "grad_norm": 1.7377828359603882,
      "learning_rate": 4.696951191696673e-06,
      "logits/chosen": -1.9457124471664429,
      "logits/rejected": -1.76227605342865,
      "logps/chosen": -222.2087860107422,
      "logps/rejected": -338.64349365234375,
      "loss": 0.1941,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.81779545545578,
      "rewards/margins": 4.531395435333252,
      "rewards/rejected": -5.349190711975098,
      "step": 260
    },
    {
      "epoch": 0.5653333333333334,
      "grad_norm": 2.8788559436798096,
      "learning_rate": 4.6830663515401196e-06,
      "logits/chosen": -1.9213287830352783,
      "logits/rejected": -1.7758309841156006,
      "logps/chosen": -231.01126098632812,
      "logps/rejected": -329.1012268066406,
      "loss": 0.2076,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.5432181358337402,
      "rewards/margins": 4.00408935546875,
      "rewards/rejected": -4.54730749130249,
      "step": 265
    },
    {
      "epoch": 0.576,
      "grad_norm": 2.318044900894165,
      "learning_rate": 4.6688919949971855e-06,
      "logits/chosen": -1.7594585418701172,
      "logits/rejected": -1.6069118976593018,
      "logps/chosen": -231.47994995117188,
      "logps/rejected": -327.43133544921875,
      "loss": 0.1632,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7389811277389526,
      "rewards/margins": 4.123358249664307,
      "rewards/rejected": -4.862339973449707,
      "step": 270
    },
    {
      "epoch": 0.5866666666666667,
      "grad_norm": 2.450094223022461,
      "learning_rate": 4.654430001858874e-06,
      "logits/chosen": -1.7364698648452759,
      "logits/rejected": -1.5289427042007446,
      "logps/chosen": -236.38003540039062,
      "logps/rejected": -317.5977478027344,
      "loss": 0.1806,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.30682843923568726,
      "rewards/margins": 4.335862159729004,
      "rewards/rejected": -4.642690181732178,
      "step": 275
    },
    {
      "epoch": 0.5973333333333334,
      "grad_norm": 2.3008522987365723,
      "learning_rate": 4.6396822900623075e-06,
      "logits/chosen": -1.8670647144317627,
      "logits/rejected": -1.6288022994995117,
      "logps/chosen": -248.07656860351562,
      "logps/rejected": -322.63616943359375,
      "loss": 0.1814,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.26228052377700806,
      "rewards/margins": 4.046158790588379,
      "rewards/rejected": -4.308439254760742,
      "step": 280
    },
    {
      "epoch": 0.608,
      "grad_norm": 2.12398362159729,
      "learning_rate": 4.624650815436368e-06,
      "logits/chosen": -1.8561731576919556,
      "logits/rejected": -1.6781187057495117,
      "logps/chosen": -256.2021179199219,
      "logps/rejected": -307.13232421875,
      "loss": 0.1669,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.36551186442375183,
      "rewards/margins": 3.9093899726867676,
      "rewards/rejected": -4.274901390075684,
      "step": 285
    },
    {
      "epoch": 0.6186666666666667,
      "grad_norm": 3.869758367538452,
      "learning_rate": 4.609337571442324e-06,
      "logits/chosen": -1.8233314752578735,
      "logits/rejected": -1.7314884662628174,
      "logps/chosen": -213.710205078125,
      "logps/rejected": -327.7879333496094,
      "loss": 0.1684,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.21804675459861755,
      "rewards/margins": 4.537252902984619,
      "rewards/rejected": -4.7552995681762695,
      "step": 290
    },
    {
      "epoch": 0.6293333333333333,
      "grad_norm": 3.0921249389648438,
      "learning_rate": 4.5937445889094535e-06,
      "logits/chosen": -1.8009964227676392,
      "logits/rejected": -1.5761785507202148,
      "logps/chosen": -236.89102172851562,
      "logps/rejected": -313.84295654296875,
      "loss": 0.1917,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.15902478992938995,
      "rewards/margins": 4.049332141876221,
      "rewards/rejected": -4.208357334136963,
      "step": 295
    },
    {
      "epoch": 0.64,
      "grad_norm": 2.362419843673706,
      "learning_rate": 4.577873935765722e-06,
      "logits/chosen": -1.9038022756576538,
      "logits/rejected": -1.6466445922851562,
      "logps/chosen": -222.8800506591797,
      "logps/rejected": -315.28875732421875,
      "loss": 0.1559,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.3516892194747925,
      "rewards/margins": 4.556002616882324,
      "rewards/rejected": -4.90769100189209,
      "step": 300
    },
    {
      "epoch": 0.6506666666666666,
      "grad_norm": 2.472935438156128,
      "learning_rate": 4.561727716763532e-06,
      "logits/chosen": -1.805426001548767,
      "logits/rejected": -1.6315984725952148,
      "logps/chosen": -244.3173370361328,
      "logps/rejected": -319.24176025390625,
      "loss": 0.1987,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.3080672025680542,
      "rewards/margins": 3.776719331741333,
      "rewards/rejected": -4.084786415100098,
      "step": 305
    },
    {
      "epoch": 0.6613333333333333,
      "grad_norm": 1.679807186126709,
      "learning_rate": 4.545308073200591e-06,
      "logits/chosen": -1.7192637920379639,
      "logits/rejected": -1.536003828048706,
      "logps/chosen": -238.8625946044922,
      "logps/rejected": -325.7824401855469,
      "loss": 0.1617,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.10333986580371857,
      "rewards/margins": 4.606169700622559,
      "rewards/rejected": -4.709508895874023,
      "step": 310
    },
    {
      "epoch": 0.672,
      "grad_norm": 1.7725813388824463,
      "learning_rate": 4.528617182635943e-06,
      "logits/chosen": -1.7725048065185547,
      "logits/rejected": -1.664609670639038,
      "logps/chosen": -244.0134735107422,
      "logps/rejected": -344.0420837402344,
      "loss": 0.2013,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.8228844404220581,
      "rewards/margins": 4.634818077087402,
      "rewards/rejected": -5.45770263671875,
      "step": 315
    },
    {
      "epoch": 0.6826666666666666,
      "grad_norm": 4.325881004333496,
      "learning_rate": 4.511657258601171e-06,
      "logits/chosen": -1.9085323810577393,
      "logits/rejected": -1.6071668863296509,
      "logps/chosen": -221.92404174804688,
      "logps/rejected": -292.5198669433594,
      "loss": 0.1981,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.2094966471195221,
      "rewards/margins": 4.443403244018555,
      "rewards/rejected": -4.652899742126465,
      "step": 320
    },
    {
      "epoch": 0.6933333333333334,
      "grad_norm": 1.9912614822387695,
      "learning_rate": 4.494430550306848e-06,
      "logits/chosen": -1.761791467666626,
      "logits/rejected": -1.6368515491485596,
      "logps/chosen": -225.29550170898438,
      "logps/rejected": -304.720703125,
      "loss": 0.2184,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.3698757588863373,
      "rewards/margins": 3.930206298828125,
      "rewards/rejected": -4.300081729888916,
      "step": 325
    },
    {
      "epoch": 0.704,
      "grad_norm": 2.0521910190582275,
      "learning_rate": 4.476939342344246e-06,
      "logits/chosen": -1.7833703756332397,
      "logits/rejected": -1.6641095876693726,
      "logps/chosen": -216.5574951171875,
      "logps/rejected": -319.1539306640625,
      "loss": 0.1463,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.6160684823989868,
      "rewards/margins": 4.553713321685791,
      "rewards/rejected": -5.169781684875488,
      "step": 330
    },
    {
      "epoch": 0.7146666666666667,
      "grad_norm": 2.2633090019226074,
      "learning_rate": 4.45918595438236e-06,
      "logits/chosen": -1.6932775974273682,
      "logits/rejected": -1.6147457361221313,
      "logps/chosen": -228.96987915039062,
      "logps/rejected": -333.4344482421875,
      "loss": 0.1349,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.269506573677063,
      "rewards/margins": 4.556288719177246,
      "rewards/rejected": -4.8257951736450195,
      "step": 335
    },
    {
      "epoch": 0.7253333333333334,
      "grad_norm": 1.9004288911819458,
      "learning_rate": 4.441172740860267e-06,
      "logits/chosen": -1.7839820384979248,
      "logits/rejected": -1.586343765258789,
      "logps/chosen": -231.0703582763672,
      "logps/rejected": -334.8110046386719,
      "loss": 0.1565,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.480475515127182,
      "rewards/margins": 4.550753116607666,
      "rewards/rejected": -5.031228065490723,
      "step": 340
    },
    {
      "epoch": 0.736,
      "grad_norm": 1.4590946435928345,
      "learning_rate": 4.422902090674886e-06,
      "logits/chosen": -1.7286231517791748,
      "logits/rejected": -1.5739670991897583,
      "logps/chosen": -230.1336669921875,
      "logps/rejected": -371.85638427734375,
      "loss": 0.1483,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.5395830869674683,
      "rewards/margins": 4.902421474456787,
      "rewards/rejected": -5.442005157470703,
      "step": 345
    },
    {
      "epoch": 0.7466666666666667,
      "grad_norm": 1.425997257232666,
      "learning_rate": 4.4043764268641705e-06,
      "logits/chosen": -1.7624887228012085,
      "logits/rejected": -1.5530593395233154,
      "logps/chosen": -236.2501983642578,
      "logps/rejected": -325.4139404296875,
      "loss": 0.1035,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.555716872215271,
      "rewards/margins": 4.979168891906738,
      "rewards/rejected": -5.534886360168457,
      "step": 350
    },
    {
      "epoch": 0.7573333333333333,
      "grad_norm": 2.0888102054595947,
      "learning_rate": 4.385598206285758e-06,
      "logits/chosen": -1.9180511236190796,
      "logits/rejected": -1.7492713928222656,
      "logps/chosen": -228.6175537109375,
      "logps/rejected": -325.20953369140625,
      "loss": 0.1149,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.6507972478866577,
      "rewards/margins": 5.017643451690674,
      "rewards/rejected": -5.668440818786621,
      "step": 355
    },
    {
      "epoch": 0.768,
      "grad_norm": 1.01185142993927,
      "learning_rate": 4.3665699192911495e-06,
      "logits/chosen": -1.6995750665664673,
      "logits/rejected": -1.5751515626907349,
      "logps/chosen": -218.4319305419922,
      "logps/rejected": -341.39703369140625,
      "loss": 0.1579,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.7953837513923645,
      "rewards/margins": 5.295377254486084,
      "rewards/rejected": -6.090761661529541,
      "step": 360
    },
    {
      "epoch": 0.7786666666666666,
      "grad_norm": 3.163583755493164,
      "learning_rate": 4.347294089395438e-06,
      "logits/chosen": -1.8674875497817993,
      "logits/rejected": -1.6801738739013672,
      "logps/chosen": -237.85214233398438,
      "logps/rejected": -345.7062683105469,
      "loss": 0.1076,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.8477768898010254,
      "rewards/margins": 6.111438751220703,
      "rewards/rejected": -6.959216117858887,
      "step": 365
    },
    {
      "epoch": 0.7893333333333333,
      "grad_norm": 2.2099385261535645,
      "learning_rate": 4.327773272942647e-06,
      "logits/chosen": -1.7721245288848877,
      "logits/rejected": -1.6586471796035767,
      "logps/chosen": -240.5192413330078,
      "logps/rejected": -312.6970520019531,
      "loss": 0.1863,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.2079604864120483,
      "rewards/margins": 4.806216239929199,
      "rewards/rejected": -6.014176845550537,
      "step": 370
    },
    {
      "epoch": 0.8,
      "grad_norm": 2.9571621417999268,
      "learning_rate": 4.308010058766703e-06,
      "logits/chosen": -1.8337215185165405,
      "logits/rejected": -1.675649642944336,
      "logps/chosen": -238.7671661376953,
      "logps/rejected": -331.68511962890625,
      "loss": 0.2111,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.2102830410003662,
      "rewards/margins": 5.248218536376953,
      "rewards/rejected": -6.45850133895874,
      "step": 375
    },
    {
      "epoch": 0.8106666666666666,
      "grad_norm": 2.9183189868927,
      "learning_rate": 4.2880070678481135e-06,
      "logits/chosen": -1.8363081216812134,
      "logits/rejected": -1.6459910869598389,
      "logps/chosen": -227.5572052001953,
      "logps/rejected": -328.91925048828125,
      "loss": 0.1342,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.3287819623947144,
      "rewards/margins": 5.704113006591797,
      "rewards/rejected": -7.032895088195801,
      "step": 380
    },
    {
      "epoch": 0.8213333333333334,
      "grad_norm": 2.6443653106689453,
      "learning_rate": 4.267766952966369e-06,
      "logits/chosen": -1.7400104999542236,
      "logits/rejected": -1.6507030725479126,
      "logps/chosen": -236.0244598388672,
      "logps/rejected": -360.4212951660156,
      "loss": 0.1261,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.1412465572357178,
      "rewards/margins": 5.938536643981934,
      "rewards/rejected": -7.0797834396362305,
      "step": 385
    },
    {
      "epoch": 0.832,
      "grad_norm": 2.404409408569336,
      "learning_rate": 4.24729239834814e-06,
      "logits/chosen": -1.7891966104507446,
      "logits/rejected": -1.6409313678741455,
      "logps/chosen": -242.6314697265625,
      "logps/rejected": -348.565673828125,
      "loss": 0.177,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.0923963785171509,
      "rewards/margins": 5.512354373931885,
      "rewards/rejected": -6.604750156402588,
      "step": 390
    },
    {
      "epoch": 0.8426666666666667,
      "grad_norm": 2.2743067741394043,
      "learning_rate": 4.226586119311291e-06,
      "logits/chosen": -1.8340394496917725,
      "logits/rejected": -1.7385257482528687,
      "logps/chosen": -238.93197631835938,
      "logps/rejected": -331.0224304199219,
      "loss": 0.17,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.9892851710319519,
      "rewards/margins": 4.964252471923828,
      "rewards/rejected": -5.953537464141846,
      "step": 395
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 3.5358939170837402,
      "learning_rate": 4.205650861904782e-06,
      "logits/chosen": -1.7910658121109009,
      "logits/rejected": -1.6007837057113647,
      "logps/chosen": -212.20230102539062,
      "logps/rejected": -335.0103759765625,
      "loss": 0.2147,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.0847444534301758,
      "rewards/margins": 5.438946723937988,
      "rewards/rejected": -6.523691654205322,
      "step": 400
    },
    {
      "epoch": 0.864,
      "grad_norm": 3.7709548473358154,
      "learning_rate": 4.184489402544486e-06,
      "logits/chosen": -1.8345584869384766,
      "logits/rejected": -1.7050797939300537,
      "logps/chosen": -233.6999969482422,
      "logps/rejected": -334.1361389160156,
      "loss": 0.1381,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.8633434176445007,
      "rewards/margins": 5.313804626464844,
      "rewards/rejected": -6.177147388458252,
      "step": 405
    },
    {
      "epoch": 0.8746666666666667,
      "grad_norm": 6.689227104187012,
      "learning_rate": 4.1631045476449865e-06,
      "logits/chosen": -1.8753330707550049,
      "logits/rejected": -1.6922645568847656,
      "logps/chosen": -227.6363525390625,
      "logps/rejected": -362.1102600097656,
      "loss": 0.1487,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.9522708058357239,
      "rewards/margins": 5.492487907409668,
      "rewards/rejected": -6.444759368896484,
      "step": 410
    },
    {
      "epoch": 0.8853333333333333,
      "grad_norm": 2.497356653213501,
      "learning_rate": 4.141499133247391e-06,
      "logits/chosen": -1.758281946182251,
      "logits/rejected": -1.60012948513031,
      "logps/chosen": -221.84115600585938,
      "logps/rejected": -317.8287353515625,
      "loss": 0.1434,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.5248526334762573,
      "rewards/margins": 5.158529758453369,
      "rewards/rejected": -5.6833815574646,
      "step": 415
    },
    {
      "epoch": 0.896,
      "grad_norm": 3.754014492034912,
      "learning_rate": 4.119676024643217e-06,
      "logits/chosen": -1.7635116577148438,
      "logits/rejected": -1.615441918373108,
      "logps/chosen": -227.84927368164062,
      "logps/rejected": -329.3919372558594,
      "loss": 0.2183,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.008711576461792,
      "rewards/margins": 4.943830966949463,
      "rewards/rejected": -5.952542781829834,
      "step": 420
    },
    {
      "epoch": 0.9066666666666666,
      "grad_norm": 4.455972194671631,
      "learning_rate": 4.0976381159944055e-06,
      "logits/chosen": -1.8479392528533936,
      "logits/rejected": -1.6086915731430054,
      "logps/chosen": -253.4595184326172,
      "logps/rejected": -353.01531982421875,
      "loss": 0.1233,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.9238975644111633,
      "rewards/margins": 5.841231346130371,
      "rewards/rejected": -6.765129089355469,
      "step": 425
    },
    {
      "epoch": 0.9173333333333333,
      "grad_norm": 3.814085006713867,
      "learning_rate": 4.07538832994949e-06,
      "logits/chosen": -1.7635217905044556,
      "logits/rejected": -1.5600554943084717,
      "logps/chosen": -235.2505340576172,
      "logps/rejected": -334.1875915527344,
      "loss": 0.1404,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.1111348867416382,
      "rewards/margins": 5.897833347320557,
      "rewards/rejected": -7.008968353271484,
      "step": 430
    },
    {
      "epoch": 0.928,
      "grad_norm": 1.5983116626739502,
      "learning_rate": 4.0529296172560055e-06,
      "logits/chosen": -1.7206604480743408,
      "logits/rejected": -1.6044199466705322,
      "logps/chosen": -255.4562225341797,
      "logps/rejected": -353.7682189941406,
      "loss": 0.1347,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.611911416053772,
      "rewards/margins": 5.771895408630371,
      "rewards/rejected": -7.383807182312012,
      "step": 435
    },
    {
      "epoch": 0.9386666666666666,
      "grad_norm": 3.102590322494507,
      "learning_rate": 4.030264956369158e-06,
      "logits/chosen": -1.7923877239227295,
      "logits/rejected": -1.6720997095108032,
      "logps/chosen": -229.66604614257812,
      "logps/rejected": -340.2394714355469,
      "loss": 0.1827,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.3031582832336426,
      "rewards/margins": 4.982819557189941,
      "rewards/rejected": -6.285977840423584,
      "step": 440
    },
    {
      "epoch": 0.9493333333333334,
      "grad_norm": 2.5847177505493164,
      "learning_rate": 4.0073973530568274e-06,
      "logits/chosen": -1.8058351278305054,
      "logits/rejected": -1.6101223230361938,
      "logps/chosen": -235.02920532226562,
      "logps/rejected": -333.5020751953125,
      "loss": 0.2007,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.0441210269927979,
      "rewards/margins": 5.034656047821045,
      "rewards/rejected": -6.078777313232422,
      "step": 445
    },
    {
      "epoch": 0.96,
      "grad_norm": 2.9892916679382324,
      "learning_rate": 3.9843298400009435e-06,
      "logits/chosen": -1.8426891565322876,
      "logits/rejected": -1.7364394664764404,
      "logps/chosen": -237.25210571289062,
      "logps/rejected": -343.7469177246094,
      "loss": 0.1625,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.9305593371391296,
      "rewards/margins": 5.86553955078125,
      "rewards/rejected": -6.7960991859436035,
      "step": 450
    },
    {
      "epoch": 0.9706666666666667,
      "grad_norm": 2.571195363998413,
      "learning_rate": 3.961065476395296e-06,
      "logits/chosen": -1.8120330572128296,
      "logits/rejected": -1.6824085712432861,
      "logps/chosen": -235.7975616455078,
      "logps/rejected": -355.3353576660156,
      "loss": 0.1486,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.8333516120910645,
      "rewards/margins": 5.53482723236084,
      "rewards/rejected": -6.368178844451904,
      "step": 455
    },
    {
      "epoch": 0.9813333333333333,
      "grad_norm": 1.5515739917755127,
      "learning_rate": 3.937607347539823e-06,
      "logits/chosen": -1.8604084253311157,
      "logits/rejected": -1.6165311336517334,
      "logps/chosen": -216.8756561279297,
      "logps/rejected": -344.33734130859375,
      "loss": 0.0933,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.6246806383132935,
      "rewards/margins": 5.950966835021973,
      "rewards/rejected": -6.575646877288818,
      "step": 460
    },
    {
      "epoch": 0.992,
      "grad_norm": 3.6848971843719482,
      "learning_rate": 3.913958564431447e-06,
      "logits/chosen": -1.8793195486068726,
      "logits/rejected": -1.6981279850006104,
      "logps/chosen": -242.66354370117188,
      "logps/rejected": -337.1656188964844,
      "loss": 0.1586,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.083747148513794,
      "rewards/margins": 5.684298992156982,
      "rewards/rejected": -6.7680463790893555,
      "step": 465
    },
    {
      "epoch": 1.0021333333333333,
      "grad_norm": 1.5835093259811401,
      "learning_rate": 3.89012226335149e-06,
      "logits/chosen": -1.67910635471344,
      "logits/rejected": -1.6301380395889282,
      "logps/chosen": -213.78390502929688,
      "logps/rejected": -342.0399475097656,
      "loss": 0.1175,
      "rewards/accuracies": 0.9473684430122375,
      "rewards/chosen": -1.068010687828064,
      "rewards/margins": 5.761088848114014,
      "rewards/rejected": -6.829100131988525,
      "step": 470
    },
    {
      "epoch": 1.0128,
      "grad_norm": 1.5585732460021973,
      "learning_rate": 3.86610160544975e-06,
      "logits/chosen": -1.7755944728851318,
      "logits/rejected": -1.6420894861221313,
      "logps/chosen": -241.9550018310547,
      "logps/rejected": -339.51776123046875,
      "loss": 0.106,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.1774930953979492,
      "rewards/margins": 5.725667476654053,
      "rewards/rejected": -6.90316104888916,
      "step": 475
    },
    {
      "epoch": 1.0234666666666667,
      "grad_norm": 2.624796152114868,
      "learning_rate": 3.841899776325267e-06,
      "logits/chosen": -1.8001434803009033,
      "logits/rejected": -1.6177053451538086,
      "logps/chosen": -245.3330078125,
      "logps/rejected": -339.3960266113281,
      "loss": 0.116,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.7908494472503662,
      "rewards/margins": 6.218578338623047,
      "rewards/rejected": -7.009427070617676,
      "step": 480
    },
    {
      "epoch": 1.0341333333333333,
      "grad_norm": 1.0212681293487549,
      "learning_rate": 3.817519985603855e-06,
      "logits/chosen": -1.9008468389511108,
      "logits/rejected": -1.785378098487854,
      "logps/chosen": -208.4319610595703,
      "logps/rejected": -339.15185546875,
      "loss": 0.1052,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.1869536638259888,
      "rewards/margins": 6.368255615234375,
      "rewards/rejected": -7.555208683013916,
      "step": 485
    },
    {
      "epoch": 1.0448,
      "grad_norm": 2.107454776763916,
      "learning_rate": 3.792965466512437e-06,
      "logits/chosen": -1.7998336553573608,
      "logits/rejected": -1.6306270360946655,
      "logps/chosen": -241.4481658935547,
      "logps/rejected": -336.5618591308594,
      "loss": 0.0921,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.0441654920578003,
      "rewards/margins": 6.187332630157471,
      "rewards/rejected": -7.231498718261719,
      "step": 490
    },
    {
      "epoch": 1.0554666666666668,
      "grad_norm": 2.4917521476745605,
      "learning_rate": 3.7682394754502687e-06,
      "logits/chosen": -1.8501405715942383,
      "logits/rejected": -1.7844288349151611,
      "logps/chosen": -233.9818115234375,
      "logps/rejected": -370.55322265625,
      "loss": 0.11,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.3015844821929932,
      "rewards/margins": 6.5641679763793945,
      "rewards/rejected": -7.86575174331665,
      "step": 495
    },
    {
      "epoch": 1.0661333333333334,
      "grad_norm": 4.674692153930664,
      "learning_rate": 3.7433452915570685e-06,
      "logits/chosen": -1.834409475326538,
      "logits/rejected": -1.7216771841049194,
      "logps/chosen": -224.1982421875,
      "logps/rejected": -330.5555114746094,
      "loss": 0.1307,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.9702581167221069,
      "rewards/margins": 5.728388786315918,
      "rewards/rejected": -6.698647499084473,
      "step": 500
    },
    {
      "epoch": 1.0768,
      "grad_norm": 1.135600209236145,
      "learning_rate": 3.718286216278143e-06,
      "logits/chosen": -1.8727703094482422,
      "logits/rejected": -1.6628506183624268,
      "logps/chosen": -232.24893188476562,
      "logps/rejected": -348.3692321777344,
      "loss": 0.0987,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.2046107053756714,
      "rewards/margins": 6.24137020111084,
      "rewards/rejected": -7.445981025695801,
      "step": 505
    },
    {
      "epoch": 1.0874666666666666,
      "grad_norm": 3.7477798461914062,
      "learning_rate": 3.6930655729265557e-06,
      "logits/chosen": -1.8030757904052734,
      "logits/rejected": -1.6741348505020142,
      "logps/chosen": -252.7802276611328,
      "logps/rejected": -363.14422607421875,
      "loss": 0.1047,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.1648820638656616,
      "rewards/margins": 6.590028285980225,
      "rewards/rejected": -7.754910469055176,
      "step": 510
    },
    {
      "epoch": 1.0981333333333334,
      "grad_norm": 1.919650673866272,
      "learning_rate": 3.667686706242386e-06,
      "logits/chosen": -1.7448360919952393,
      "logits/rejected": -1.6716108322143555,
      "logps/chosen": -255.5791473388672,
      "logps/rejected": -367.09210205078125,
      "loss": 0.0976,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.3545987606048584,
      "rewards/margins": 6.418627738952637,
      "rewards/rejected": -7.773226737976074,
      "step": 515
    },
    {
      "epoch": 1.1088,
      "grad_norm": 2.581798553466797,
      "learning_rate": 3.6421529819491576e-06,
      "logits/chosen": -1.8232694864273071,
      "logits/rejected": -1.7192691564559937,
      "logps/chosen": -248.962890625,
      "logps/rejected": -359.66497802734375,
      "loss": 0.098,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.4379160404205322,
      "rewards/margins": 6.792909145355225,
      "rewards/rejected": -8.230825424194336,
      "step": 520
    },
    {
      "epoch": 1.1194666666666666,
      "grad_norm": 1.842156171798706,
      "learning_rate": 3.6164677863074783e-06,
      "logits/chosen": -1.8928749561309814,
      "logits/rejected": -1.7210674285888672,
      "logps/chosen": -217.7656707763672,
      "logps/rejected": -342.0232238769531,
      "loss": 0.087,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.3847095966339111,
      "rewards/margins": 7.149325370788574,
      "rewards/rejected": -8.534035682678223,
      "step": 525
    },
    {
      "epoch": 1.1301333333333332,
      "grad_norm": 4.107654571533203,
      "learning_rate": 3.590634525665955e-06,
      "logits/chosen": -1.9062941074371338,
      "logits/rejected": -1.7494275569915771,
      "logps/chosen": -222.1142120361328,
      "logps/rejected": -378.0558776855469,
      "loss": 0.1408,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.640011191368103,
      "rewards/margins": 7.438580513000488,
      "rewards/rejected": -9.078592300415039,
      "step": 530
    },
    {
      "epoch": 1.1408,
      "grad_norm": 4.996397495269775,
      "learning_rate": 3.564656626009446e-06,
      "logits/chosen": -1.9074268341064453,
      "logits/rejected": -1.6736358404159546,
      "logps/chosen": -305.10498046875,
      "logps/rejected": -382.6112060546875,
      "loss": 0.1103,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.8150031566619873,
      "rewards/margins": 7.754342555999756,
      "rewards/rejected": -9.569345474243164,
      "step": 535
    },
    {
      "epoch": 1.1514666666666666,
      "grad_norm": 2.6492626667022705,
      "learning_rate": 3.5385375325047167e-06,
      "logits/chosen": -1.8353687524795532,
      "logits/rejected": -1.7401046752929688,
      "logps/chosen": -234.1809539794922,
      "logps/rejected": -361.4974060058594,
      "loss": 0.1014,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.9234129190444946,
      "rewards/margins": 6.884763240814209,
      "rewards/rejected": -8.80817699432373,
      "step": 540
    },
    {
      "epoch": 1.1621333333333332,
      "grad_norm": 1.1152234077453613,
      "learning_rate": 3.512280709043534e-06,
      "logits/chosen": -1.8246501684188843,
      "logits/rejected": -1.6780567169189453,
      "logps/chosen": -251.69308471679688,
      "logps/rejected": -354.8346252441406,
      "loss": 0.1077,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.4507404565811157,
      "rewards/margins": 6.514955043792725,
      "rewards/rejected": -7.965695381164551,
      "step": 545
    },
    {
      "epoch": 1.1728,
      "grad_norm": 4.050209999084473,
      "learning_rate": 3.4858896377832966e-06,
      "logits/chosen": -1.8190292119979858,
      "logits/rejected": -1.7231298685073853,
      "logps/chosen": -232.68844604492188,
      "logps/rejected": -375.8724365234375,
      "loss": 0.0755,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.4951999187469482,
      "rewards/margins": 7.632623195648193,
      "rewards/rejected": -9.127823829650879,
      "step": 550
    },
    {
      "epoch": 1.1834666666666667,
      "grad_norm": 5.159008026123047,
      "learning_rate": 3.459367818685231e-06,
      "logits/chosen": -1.8263816833496094,
      "logits/rejected": -1.6552646160125732,
      "logps/chosen": -237.4913330078125,
      "logps/rejected": -357.56719970703125,
      "loss": 0.0946,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.2089338302612305,
      "rewards/margins": 7.155152320861816,
      "rewards/rejected": -8.36408519744873,
      "step": 555
    },
    {
      "epoch": 1.1941333333333333,
      "grad_norm": 11.910748481750488,
      "learning_rate": 3.432718769050228e-06,
      "logits/chosen": -1.7615569829940796,
      "logits/rejected": -1.5479238033294678,
      "logps/chosen": -245.3880615234375,
      "logps/rejected": -345.82427978515625,
      "loss": 0.137,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.5121991634368896,
      "rewards/margins": 6.261626243591309,
      "rewards/rejected": -7.773825168609619,
      "step": 560
    },
    {
      "epoch": 1.2048,
      "grad_norm": 1.417484164237976,
      "learning_rate": 3.405946023052386e-06,
      "logits/chosen": -1.8306633234024048,
      "logits/rejected": -1.7571738958358765,
      "logps/chosen": -233.8412628173828,
      "logps/rejected": -352.0577087402344,
      "loss": 0.0998,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.8432447910308838,
      "rewards/margins": 6.6724748611450195,
      "rewards/rejected": -8.515719413757324,
      "step": 565
    },
    {
      "epoch": 1.2154666666666667,
      "grad_norm": 2.3207719326019287,
      "learning_rate": 3.3790531312703055e-06,
      "logits/chosen": -1.888755440711975,
      "logits/rejected": -1.743829369544983,
      "logps/chosen": -244.34909057617188,
      "logps/rejected": -382.1915588378906,
      "loss": 0.087,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.826398491859436,
      "rewards/margins": 7.315705299377441,
      "rewards/rejected": -9.14210319519043,
      "step": 570
    },
    {
      "epoch": 1.2261333333333333,
      "grad_norm": 3.761608123779297,
      "learning_rate": 3.35204366021622e-06,
      "logits/chosen": -1.7861427068710327,
      "logits/rejected": -1.6182992458343506,
      "logps/chosen": -231.2759552001953,
      "logps/rejected": -336.6302185058594,
      "loss": 0.1579,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.3482277393341064,
      "rewards/margins": 6.256989479064941,
      "rewards/rejected": -7.605217933654785,
      "step": 575
    },
    {
      "epoch": 1.2368000000000001,
      "grad_norm": 2.557394027709961,
      "learning_rate": 3.3249211918630055e-06,
      "logits/chosen": -1.8882964849472046,
      "logits/rejected": -1.7571732997894287,
      "logps/chosen": -220.7556610107422,
      "logps/rejected": -365.0335998535156,
      "loss": 0.095,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.1241406202316284,
      "rewards/margins": 7.271350860595703,
      "rewards/rejected": -8.395490646362305,
      "step": 580
    },
    {
      "epoch": 1.2474666666666667,
      "grad_norm": 3.2217822074890137,
      "learning_rate": 3.2976893231691424e-06,
      "logits/chosen": -1.7663228511810303,
      "logits/rejected": -1.6859546899795532,
      "logps/chosen": -249.8618927001953,
      "logps/rejected": -372.42169189453125,
      "loss": 0.115,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.5136463642120361,
      "rewards/margins": 6.937762260437012,
      "rewards/rejected": -8.451409339904785,
      "step": 585
    },
    {
      "epoch": 1.2581333333333333,
      "grad_norm": 0.9602277874946594,
      "learning_rate": 3.270351665601691e-06,
      "logits/chosen": -1.794925332069397,
      "logits/rejected": -1.7219619750976562,
      "logps/chosen": -232.79190063476562,
      "logps/rejected": -343.2873840332031,
      "loss": 0.1009,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.2487173080444336,
      "rewards/margins": 6.3407440185546875,
      "rewards/rejected": -7.589460849761963,
      "step": 590
    },
    {
      "epoch": 1.2688,
      "grad_norm": 2.6589019298553467,
      "learning_rate": 3.2429118446573417e-06,
      "logits/chosen": -1.8542613983154297,
      "logits/rejected": -1.6390371322631836,
      "logps/chosen": -232.03125,
      "logps/rejected": -352.66241455078125,
      "loss": 0.1434,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.5202168226242065,
      "rewards/margins": 6.778920650482178,
      "rewards/rejected": -8.299138069152832,
      "step": 595
    },
    {
      "epoch": 1.2794666666666665,
      "grad_norm": 1.2758371829986572,
      "learning_rate": 3.215373499381602e-06,
      "logits/chosen": -1.8230146169662476,
      "logits/rejected": -1.6330006122589111,
      "logps/chosen": -236.1676025390625,
      "logps/rejected": -346.3940734863281,
      "loss": 0.0952,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.3624433279037476,
      "rewards/margins": 7.068747520446777,
      "rewards/rejected": -8.431192398071289,
      "step": 600
    },
    {
      "epoch": 1.2901333333333334,
      "grad_norm": 1.0389660596847534,
      "learning_rate": 3.1877402818861954e-06,
      "logits/chosen": -1.808443307876587,
      "logits/rejected": -1.6811857223510742,
      "logps/chosen": -262.27532958984375,
      "logps/rejected": -381.2078552246094,
      "loss": 0.0504,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.750425100326538,
      "rewards/margins": 7.616028785705566,
      "rewards/rejected": -9.366454124450684,
      "step": 605
    },
    {
      "epoch": 1.3008,
      "grad_norm": 5.7284016609191895,
      "learning_rate": 3.1600158568647145e-06,
      "logits/chosen": -1.7468950748443604,
      "logits/rejected": -1.7098772525787354,
      "logps/chosen": -232.5611114501953,
      "logps/rejected": -350.3565673828125,
      "loss": 0.0777,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.3892863988876343,
      "rewards/margins": 6.9035325050354,
      "rewards/rejected": -8.292818069458008,
      "step": 610
    },
    {
      "epoch": 1.3114666666666666,
      "grad_norm": 2.848447561264038,
      "learning_rate": 3.1322039011066172e-06,
      "logits/chosen": -1.850782036781311,
      "logits/rejected": -1.6880401372909546,
      "logps/chosen": -247.71377563476562,
      "logps/rejected": -349.6388854980469,
      "loss": 0.1083,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.515555500984192,
      "rewards/margins": 6.665856838226318,
      "rewards/rejected": -8.181412696838379,
      "step": 615
    },
    {
      "epoch": 1.3221333333333334,
      "grad_norm": 2.5418522357940674,
      "learning_rate": 3.1043081030096125e-06,
      "logits/chosen": -1.7264868021011353,
      "logits/rejected": -1.5793980360031128,
      "logps/chosen": -243.20101928710938,
      "logps/rejected": -349.4842224121094,
      "loss": 0.1148,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.619227409362793,
      "rewards/margins": 7.061232089996338,
      "rewards/rejected": -8.680459022521973,
      "step": 620
    },
    {
      "epoch": 1.3328,
      "grad_norm": 1.7328300476074219,
      "learning_rate": 3.0763321620905072e-06,
      "logits/chosen": -1.7665011882781982,
      "logits/rejected": -1.5880069732666016,
      "logps/chosen": -263.8298034667969,
      "logps/rejected": -368.64764404296875,
      "loss": 0.1128,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.6416599750518799,
      "rewards/margins": 6.925388336181641,
      "rewards/rejected": -8.567049026489258,
      "step": 625
    },
    {
      "epoch": 1.3434666666666666,
      "grad_norm": 1.3818984031677246,
      "learning_rate": 3.048279788494583e-06,
      "logits/chosen": -1.8107048273086548,
      "logits/rejected": -1.6657230854034424,
      "logps/chosen": -231.5718231201172,
      "logps/rejected": -361.8509521484375,
      "loss": 0.0717,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.662607192993164,
      "rewards/margins": 7.121241569519043,
      "rewards/rejected": -8.783849716186523,
      "step": 630
    },
    {
      "epoch": 1.3541333333333334,
      "grad_norm": 1.9574915170669556,
      "learning_rate": 3.0201547025035553e-06,
      "logits/chosen": -1.7519423961639404,
      "logits/rejected": -1.6680078506469727,
      "logps/chosen": -265.30389404296875,
      "logps/rejected": -374.69158935546875,
      "loss": 0.1317,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.633040428161621,
      "rewards/margins": 7.419091701507568,
      "rewards/rejected": -9.052130699157715,
      "step": 635
    },
    {
      "epoch": 1.3648,
      "grad_norm": 4.3590898513793945,
      "learning_rate": 2.9919606340421963e-06,
      "logits/chosen": -1.8167431354522705,
      "logits/rejected": -1.582132339477539,
      "logps/chosen": -244.23715209960938,
      "logps/rejected": -375.7297668457031,
      "loss": 0.106,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.620737075805664,
      "rewards/margins": 7.2550506591796875,
      "rewards/rejected": -8.875787734985352,
      "step": 640
    },
    {
      "epoch": 1.3754666666666666,
      "grad_norm": 2.495652437210083,
      "learning_rate": 2.9637013221836723e-06,
      "logits/chosen": -1.8191438913345337,
      "logits/rejected": -1.6955486536026,
      "logps/chosen": -243.5186309814453,
      "logps/rejected": -383.9856872558594,
      "loss": 0.0928,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.691001534461975,
      "rewards/margins": 7.247744560241699,
      "rewards/rejected": -8.938745498657227,
      "step": 645
    },
    {
      "epoch": 1.3861333333333334,
      "grad_norm": 0.8766574859619141,
      "learning_rate": 2.9353805146536762e-06,
      "logits/chosen": -1.7703936100006104,
      "logits/rejected": -1.5947972536087036,
      "logps/chosen": -256.13348388671875,
      "logps/rejected": -367.19384765625,
      "loss": 0.1009,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.5907480716705322,
      "rewards/margins": 7.426206111907959,
      "rewards/rejected": -9.01695442199707,
      "step": 650
    },
    {
      "epoch": 1.3968,
      "grad_norm": 3.314081907272339,
      "learning_rate": 2.907001967333401e-06,
      "logits/chosen": -1.8712552785873413,
      "logits/rejected": -1.7601633071899414,
      "logps/chosen": -242.0836181640625,
      "logps/rejected": -389.406982421875,
      "loss": 0.0926,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.9015403985977173,
      "rewards/margins": 7.678501129150391,
      "rewards/rejected": -9.58004093170166,
      "step": 655
    },
    {
      "epoch": 1.4074666666666666,
      "grad_norm": 2.852855920791626,
      "learning_rate": 2.878569443761442e-06,
      "logits/chosen": -1.7180980443954468,
      "logits/rejected": -1.593620777130127,
      "logps/chosen": -245.14639282226562,
      "logps/rejected": -350.68536376953125,
      "loss": 0.0907,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.2707574367523193,
      "rewards/margins": 7.63345193862915,
      "rewards/rejected": -9.90420913696289,
      "step": 660
    },
    {
      "epoch": 1.4181333333333335,
      "grad_norm": 2.0365729331970215,
      "learning_rate": 2.85008671463468e-06,
      "logits/chosen": -1.8370532989501953,
      "logits/rejected": -1.7093183994293213,
      "logps/chosen": -242.4522247314453,
      "logps/rejected": -368.5684814453125,
      "loss": 0.0805,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.7941030263900757,
      "rewards/margins": 7.619574546813965,
      "rewards/rejected": -9.413678169250488,
      "step": 665
    },
    {
      "epoch": 1.4288,
      "grad_norm": 0.7839527726173401,
      "learning_rate": 2.8215575573082127e-06,
      "logits/chosen": -1.819196343421936,
      "logits/rejected": -1.6762737035751343,
      "logps/chosen": -235.5693817138672,
      "logps/rejected": -372.1368713378906,
      "loss": 0.0533,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.2108426094055176,
      "rewards/margins": 8.851494789123535,
      "rewards/rejected": -11.062335968017578,
      "step": 670
    },
    {
      "epoch": 1.4394666666666667,
      "grad_norm": 5.5416059494018555,
      "learning_rate": 2.792985755294404e-06,
      "logits/chosen": -1.8234074115753174,
      "logits/rejected": -1.6806237697601318,
      "logps/chosen": -235.62826538085938,
      "logps/rejected": -380.11541748046875,
      "loss": 0.1041,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.754598617553711,
      "rewards/margins": 8.41307258605957,
      "rewards/rejected": -11.167672157287598,
      "step": 675
    },
    {
      "epoch": 1.4501333333333333,
      "grad_norm": 6.455689907073975,
      "learning_rate": 2.764375097761122e-06,
      "logits/chosen": -1.627570390701294,
      "logits/rejected": -1.4764485359191895,
      "logps/chosen": -251.55331420898438,
      "logps/rejected": -352.8821105957031,
      "loss": 0.1401,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.669222593307495,
      "rewards/margins": 7.199682712554932,
      "rewards/rejected": -9.868905067443848,
      "step": 680
    },
    {
      "epoch": 1.4607999999999999,
      "grad_norm": 4.647602558135986,
      "learning_rate": 2.73572937902922e-06,
      "logits/chosen": -1.671311616897583,
      "logits/rejected": -1.548492431640625,
      "logps/chosen": -276.5353088378906,
      "logps/rejected": -366.04937744140625,
      "loss": 0.1593,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.8360071182250977,
      "rewards/margins": 7.552770137786865,
      "rewards/rejected": -10.388776779174805,
      "step": 685
    },
    {
      "epoch": 1.4714666666666667,
      "grad_norm": 0.9273281693458557,
      "learning_rate": 2.7070523980693362e-06,
      "logits/chosen": -1.532097339630127,
      "logits/rejected": -1.5424264669418335,
      "logps/chosen": -270.2457580566406,
      "logps/rejected": -377.8583679199219,
      "loss": 0.1488,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.7499938011169434,
      "rewards/margins": 7.535027980804443,
      "rewards/rejected": -10.285021781921387,
      "step": 690
    },
    {
      "epoch": 1.4821333333333333,
      "grad_norm": 3.152971029281616,
      "learning_rate": 2.678347957998081e-06,
      "logits/chosen": -1.8320585489273071,
      "logits/rejected": -1.6948597431182861,
      "logps/chosen": -241.94406127929688,
      "logps/rejected": -370.0408935546875,
      "loss": 0.1206,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.761826276779175,
      "rewards/margins": 7.769921779632568,
      "rewards/rejected": -10.531747817993164,
      "step": 695
    },
    {
      "epoch": 1.4928,
      "grad_norm": 3.29986834526062,
      "learning_rate": 2.649619865573666e-06,
      "logits/chosen": -1.8070224523544312,
      "logits/rejected": -1.633344292640686,
      "logps/chosen": -245.9867706298828,
      "logps/rejected": -370.3143005371094,
      "loss": 0.1189,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.6095986366271973,
      "rewards/margins": 7.841919898986816,
      "rewards/rejected": -10.451518058776855,
      "step": 700
    },
    {
      "epoch": 1.5034666666666667,
      "grad_norm": 5.474385738372803,
      "learning_rate": 2.620871930691058e-06,
      "logits/chosen": -1.7647151947021484,
      "logits/rejected": -1.6790416240692139,
      "logps/chosen": -238.916259765625,
      "logps/rejected": -353.8853759765625,
      "loss": 0.1491,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.5073959827423096,
      "rewards/margins": 6.695868015289307,
      "rewards/rejected": -9.203264236450195,
      "step": 705
    },
    {
      "epoch": 1.5141333333333333,
      "grad_norm": 1.038760781288147,
      "learning_rate": 2.5921079658767152e-06,
      "logits/chosen": -1.7665719985961914,
      "logits/rejected": -1.689576506614685,
      "logps/chosen": -251.5525360107422,
      "logps/rejected": -368.5316467285156,
      "loss": 0.0539,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.5829997062683105,
      "rewards/margins": 7.845690727233887,
      "rewards/rejected": -10.428690910339355,
      "step": 710
    },
    {
      "epoch": 1.5248,
      "grad_norm": 3.1464333534240723,
      "learning_rate": 2.56333178578297e-06,
      "logits/chosen": -1.7736793756484985,
      "logits/rejected": -1.6533575057983398,
      "logps/chosen": -244.5554962158203,
      "logps/rejected": -391.2909240722656,
      "loss": 0.0767,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.007012128829956,
      "rewards/margins": 8.108095169067383,
      "rewards/rejected": -11.115106582641602,
      "step": 715
    },
    {
      "epoch": 1.5354666666666668,
      "grad_norm": 5.089426040649414,
      "learning_rate": 2.534547206682136e-06,
      "logits/chosen": -1.8151180744171143,
      "logits/rejected": -1.696407675743103,
      "logps/chosen": -249.90390014648438,
      "logps/rejected": -380.46685791015625,
      "loss": 0.0861,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.8637096881866455,
      "rewards/margins": 8.219526290893555,
      "rewards/rejected": -11.083236694335938,
      "step": 720
    },
    {
      "epoch": 1.5461333333333334,
      "grad_norm": 1.6263580322265625,
      "learning_rate": 2.5057580459603947e-06,
      "logits/chosen": -1.8785078525543213,
      "logits/rejected": -1.7948106527328491,
      "logps/chosen": -234.8917694091797,
      "logps/rejected": -399.46337890625,
      "loss": 0.0607,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.608294725418091,
      "rewards/margins": 8.606790542602539,
      "rewards/rejected": -11.215084075927734,
      "step": 725
    },
    {
      "epoch": 1.5568,
      "grad_norm": 2.20981764793396,
      "learning_rate": 2.4769681216115384e-06,
      "logits/chosen": -1.7893867492675781,
      "logits/rejected": -1.732273817062378,
      "logps/chosen": -251.80380249023438,
      "logps/rejected": -365.3934631347656,
      "loss": 0.1178,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.6450085639953613,
      "rewards/margins": 7.855576515197754,
      "rewards/rejected": -10.500584602355957,
      "step": 730
    },
    {
      "epoch": 1.5674666666666668,
      "grad_norm": 0.7660602331161499,
      "learning_rate": 2.448181251730632e-06,
      "logits/chosen": -1.731345772743225,
      "logits/rejected": -1.5998746156692505,
      "logps/chosen": -263.6766662597656,
      "logps/rejected": -396.3465270996094,
      "loss": 0.0742,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.3900294303894043,
      "rewards/margins": 8.074613571166992,
      "rewards/rejected": -10.464642524719238,
      "step": 735
    },
    {
      "epoch": 1.5781333333333334,
      "grad_norm": 2.8943593502044678,
      "learning_rate": 2.419401254007658e-06,
      "logits/chosen": -1.7047817707061768,
      "logits/rejected": -1.6618448495864868,
      "logps/chosen": -270.00677490234375,
      "logps/rejected": -373.75115966796875,
      "loss": 0.0823,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.591595411300659,
      "rewards/margins": 7.437920570373535,
      "rewards/rejected": -10.029515266418457,
      "step": 740
    },
    {
      "epoch": 1.5888,
      "grad_norm": 1.2762200832366943,
      "learning_rate": 2.390631945221221e-06,
      "logits/chosen": -1.77741277217865,
      "logits/rejected": -1.6593185663223267,
      "logps/chosen": -259.33966064453125,
      "logps/rejected": -387.9762878417969,
      "loss": 0.0596,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.5528736114501953,
      "rewards/margins": 8.286169052124023,
      "rewards/rejected": -10.839041709899902,
      "step": 745
    },
    {
      "epoch": 1.5994666666666668,
      "grad_norm": 5.301697254180908,
      "learning_rate": 2.361877140732366e-06,
      "logits/chosen": -1.751280426979065,
      "logits/rejected": -1.6818625926971436,
      "logps/chosen": -244.2352752685547,
      "logps/rejected": -358.234375,
      "loss": 0.0983,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.585808277130127,
      "rewards/margins": 6.942679405212402,
      "rewards/rejected": -9.528488159179688,
      "step": 750
    },
    {
      "epoch": 1.6101333333333332,
      "grad_norm": 1.2639341354370117,
      "learning_rate": 2.333140653978592e-06,
      "logits/chosen": -1.8316749334335327,
      "logits/rejected": -1.6760244369506836,
      "logps/chosen": -247.3037567138672,
      "logps/rejected": -367.29510498046875,
      "loss": 0.11,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.250349760055542,
      "rewards/margins": 7.623064994812012,
      "rewards/rejected": -9.873414993286133,
      "step": 755
    },
    {
      "epoch": 1.6208,
      "grad_norm": 1.857134461402893,
      "learning_rate": 2.3044262959681156e-06,
      "logits/chosen": -1.769269585609436,
      "logits/rejected": -1.5758507251739502,
      "logps/chosen": -239.4264373779297,
      "logps/rejected": -377.9754638671875,
      "loss": 0.0973,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.9940210580825806,
      "rewards/margins": 7.984678745269775,
      "rewards/rejected": -9.978699684143066,
      "step": 760
    },
    {
      "epoch": 1.6314666666666666,
      "grad_norm": 0.692950963973999,
      "learning_rate": 2.2757378747744575e-06,
      "logits/chosen": -1.8032305240631104,
      "logits/rejected": -1.6365445852279663,
      "logps/chosen": -232.6775360107422,
      "logps/rejected": -371.04547119140625,
      "loss": 0.0768,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.1179115772247314,
      "rewards/margins": 8.067190170288086,
      "rewards/rejected": -10.185101509094238,
      "step": 765
    },
    {
      "epoch": 1.6421333333333332,
      "grad_norm": 1.407834529876709,
      "learning_rate": 2.24707919503142e-06,
      "logits/chosen": -1.7327646017074585,
      "logits/rejected": -1.648236870765686,
      "logps/chosen": -250.6312713623047,
      "logps/rejected": -385.9135437011719,
      "loss": 0.0714,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.2371888160705566,
      "rewards/margins": 7.962684631347656,
      "rewards/rejected": -10.199872970581055,
      "step": 770
    },
    {
      "epoch": 1.6528,
      "grad_norm": 4.060446739196777,
      "learning_rate": 2.2184540574285213e-06,
      "logits/chosen": -1.7848037481307983,
      "logits/rejected": -1.646002173423767,
      "logps/chosen": -259.26116943359375,
      "logps/rejected": -352.891845703125,
      "loss": 0.1054,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.8747847080230713,
      "rewards/margins": 7.4474196434021,
      "rewards/rejected": -9.322202682495117,
      "step": 775
    },
    {
      "epoch": 1.6634666666666666,
      "grad_norm": 2.409456968307495,
      "learning_rate": 2.1898662582069486e-06,
      "logits/chosen": -1.7882057428359985,
      "logits/rejected": -1.7148845195770264,
      "logps/chosen": -222.84060668945312,
      "logps/rejected": -384.2849426269531,
      "loss": 0.0724,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.966027855873108,
      "rewards/margins": 8.193787574768066,
      "rewards/rejected": -10.159815788269043,
      "step": 780
    },
    {
      "epoch": 1.6741333333333333,
      "grad_norm": 5.831104278564453,
      "learning_rate": 2.1613195886561044e-06,
      "logits/chosen": -1.7008816003799438,
      "logits/rejected": -1.5513235330581665,
      "logps/chosen": -243.56265258789062,
      "logps/rejected": -361.69219970703125,
      "loss": 0.1043,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.433258533477783,
      "rewards/margins": 7.737143039703369,
      "rewards/rejected": -10.170400619506836,
      "step": 785
    },
    {
      "epoch": 1.6848,
      "grad_norm": 5.545714378356934,
      "learning_rate": 2.132817834610811e-06,
      "logits/chosen": -1.7647472620010376,
      "logits/rejected": -1.5817819833755493,
      "logps/chosen": -237.57205200195312,
      "logps/rejected": -385.9304504394531,
      "loss": 0.1211,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.0809152126312256,
      "rewards/margins": 8.433000564575195,
      "rewards/rejected": -10.513916015625,
      "step": 790
    },
    {
      "epoch": 1.6954666666666667,
      "grad_norm": 6.176883220672607,
      "learning_rate": 2.1043647759492338e-06,
      "logits/chosen": -1.6110525131225586,
      "logits/rejected": -1.593756914138794,
      "logps/chosen": -256.33465576171875,
      "logps/rejected": -383.64947509765625,
      "loss": 0.1472,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.9406496286392212,
      "rewards/margins": 7.548769950866699,
      "rewards/rejected": -9.489419937133789,
      "step": 795
    },
    {
      "epoch": 1.7061333333333333,
      "grad_norm": 4.009058952331543,
      "learning_rate": 2.075964186091599e-06,
      "logits/chosen": -1.7719929218292236,
      "logits/rejected": -1.6545894145965576,
      "logps/chosen": -240.3562469482422,
      "logps/rejected": -371.539794921875,
      "loss": 0.1034,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.4147119522094727,
      "rewards/margins": 7.830977439880371,
      "rewards/rejected": -10.24569034576416,
      "step": 800
    },
    {
      "epoch": 1.7168,
      "grad_norm": 1.6909562349319458,
      "learning_rate": 2.047619831499765e-06,
      "logits/chosen": -1.6593084335327148,
      "logits/rejected": -1.5761816501617432,
      "logps/chosen": -255.69204711914062,
      "logps/rejected": -393.48992919921875,
      "loss": 0.1291,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.199592113494873,
      "rewards/margins": 7.6634626388549805,
      "rewards/rejected": -9.863054275512695,
      "step": 805
    },
    {
      "epoch": 1.7274666666666667,
      "grad_norm": 5.588451862335205,
      "learning_rate": 2.0193354711777155e-06,
      "logits/chosen": -1.7532539367675781,
      "logits/rejected": -1.6666532754898071,
      "logps/chosen": -245.4121551513672,
      "logps/rejected": -357.675048828125,
      "loss": 0.1185,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.0722060203552246,
      "rewards/margins": 7.035107612609863,
      "rewards/rejected": -9.10731315612793,
      "step": 810
    },
    {
      "epoch": 1.7381333333333333,
      "grad_norm": 2.641136646270752,
      "learning_rate": 1.991114856173048e-06,
      "logits/chosen": -1.8587242364883423,
      "logits/rejected": -1.757472276687622,
      "logps/chosen": -224.07275390625,
      "logps/rejected": -398.39373779296875,
      "loss": 0.0711,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.255558967590332,
      "rewards/margins": 8.534242630004883,
      "rewards/rejected": -10.789800643920898,
      "step": 815
    },
    {
      "epoch": 1.7488000000000001,
      "grad_norm": 1.2425836324691772,
      "learning_rate": 1.9629617290795115e-06,
      "logits/chosen": -1.7562110424041748,
      "logits/rejected": -1.6735032796859741,
      "logps/chosen": -225.7513885498047,
      "logps/rejected": -351.25933837890625,
      "loss": 0.0889,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.073871612548828,
      "rewards/margins": 7.723903656005859,
      "rewards/rejected": -9.797774314880371,
      "step": 820
    },
    {
      "epoch": 1.7594666666666665,
      "grad_norm": 3.0822105407714844,
      "learning_rate": 1.934879823540663e-06,
      "logits/chosen": -1.7950897216796875,
      "logits/rejected": -1.6102466583251953,
      "logps/chosen": -242.1965789794922,
      "logps/rejected": -376.4043884277344,
      "loss": 0.1112,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.1286075115203857,
      "rewards/margins": 8.270095825195312,
      "rewards/rejected": -10.398704528808594,
      "step": 825
    },
    {
      "epoch": 1.7701333333333333,
      "grad_norm": 3.5807950496673584,
      "learning_rate": 1.9068728637547195e-06,
      "logits/chosen": -1.7565339803695679,
      "logits/rejected": -1.6050052642822266,
      "logps/chosen": -266.1463623046875,
      "logps/rejected": -386.4732971191406,
      "loss": 0.0924,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.24617338180542,
      "rewards/margins": 8.325594902038574,
      "rewards/rejected": -10.571767807006836,
      "step": 830
    },
    {
      "epoch": 1.7808000000000002,
      "grad_norm": 2.8825063705444336,
      "learning_rate": 1.878944563980658e-06,
      "logits/chosen": -1.7978938817977905,
      "logits/rejected": -1.6335837841033936,
      "logps/chosen": -234.8115234375,
      "logps/rejected": -386.7230529785156,
      "loss": 0.0674,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.029653310775757,
      "rewards/margins": 8.498917579650879,
      "rewards/rejected": -10.528570175170898,
      "step": 835
    },
    {
      "epoch": 1.7914666666666665,
      "grad_norm": 2.1994361877441406,
      "learning_rate": 1.85109862804563e-06,
      "logits/chosen": -1.7133188247680664,
      "logits/rejected": -1.6955077648162842,
      "logps/chosen": -253.84384155273438,
      "logps/rejected": -391.4021301269531,
      "loss": 0.1069,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.2396225929260254,
      "rewards/margins": 7.77692174911499,
      "rewards/rejected": -10.016545295715332,
      "step": 840
    },
    {
      "epoch": 1.8021333333333334,
      "grad_norm": 4.0932769775390625,
      "learning_rate": 1.823338748853766e-06,
      "logits/chosen": -1.794632911682129,
      "logits/rejected": -1.7020469903945923,
      "logps/chosen": -257.8614501953125,
      "logps/rejected": -385.7154846191406,
      "loss": 0.0992,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.1005051136016846,
      "rewards/margins": 7.511053562164307,
      "rewards/rejected": -9.611557960510254,
      "step": 845
    },
    {
      "epoch": 1.8128,
      "grad_norm": 3.011971950531006,
      "learning_rate": 1.7956686078964257e-06,
      "logits/chosen": -1.7527557611465454,
      "logits/rejected": -1.655992865562439,
      "logps/chosen": -255.39816284179688,
      "logps/rejected": -365.300048828125,
      "loss": 0.1209,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.193533420562744,
      "rewards/margins": 7.039409637451172,
      "rewards/rejected": -9.232943534851074,
      "step": 850
    },
    {
      "epoch": 1.8234666666666666,
      "grad_norm": 2.4162063598632812,
      "learning_rate": 1.7680918747639608e-06,
      "logits/chosen": -1.7931512594223022,
      "logits/rejected": -1.7776679992675781,
      "logps/chosen": -250.3501434326172,
      "logps/rejected": -377.2782897949219,
      "loss": 0.1042,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.256478786468506,
      "rewards/margins": 7.774122714996338,
      "rewards/rejected": -10.030603408813477,
      "step": 855
    },
    {
      "epoch": 1.8341333333333334,
      "grad_norm": 2.753272533416748,
      "learning_rate": 1.740612206659057e-06,
      "logits/chosen": -1.7774744033813477,
      "logits/rejected": -1.6850471496582031,
      "logps/chosen": -249.3313751220703,
      "logps/rejected": -375.1966552734375,
      "loss": 0.1116,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.082430601119995,
      "rewards/margins": 7.262882232666016,
      "rewards/rejected": -9.345311164855957,
      "step": 860
    },
    {
      "epoch": 1.8448,
      "grad_norm": 5.52640438079834,
      "learning_rate": 1.7132332479117204e-06,
      "logits/chosen": -1.735599160194397,
      "logits/rejected": -1.6529653072357178,
      "logps/chosen": -242.7247314453125,
      "logps/rejected": -383.18292236328125,
      "loss": 0.102,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.9350744485855103,
      "rewards/margins": 8.235949516296387,
      "rewards/rejected": -10.171025276184082,
      "step": 865
    },
    {
      "epoch": 1.8554666666666666,
      "grad_norm": 3.504727840423584,
      "learning_rate": 1.6859586294959666e-06,
      "logits/chosen": -1.7679811716079712,
      "logits/rejected": -1.6120868921279907,
      "logps/chosen": -256.9747619628906,
      "logps/rejected": -386.1347351074219,
      "loss": 0.0554,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.9044342041015625,
      "rewards/margins": 8.597646713256836,
      "rewards/rejected": -10.502081871032715,
      "step": 870
    },
    {
      "epoch": 1.8661333333333334,
      "grad_norm": 2.296985626220703,
      "learning_rate": 1.6587919685482884e-06,
      "logits/chosen": -1.8257315158843994,
      "logits/rejected": -1.6303374767303467,
      "logps/chosen": -242.2867889404297,
      "logps/rejected": -359.695556640625,
      "loss": 0.0673,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.22623348236084,
      "rewards/margins": 7.586366176605225,
      "rewards/rejected": -9.812600135803223,
      "step": 875
    },
    {
      "epoch": 1.8768,
      "grad_norm": 1.439993143081665,
      "learning_rate": 1.6317368678879497e-06,
      "logits/chosen": -1.6686086654663086,
      "logits/rejected": -1.5566412210464478,
      "logps/chosen": -231.3364715576172,
      "logps/rejected": -358.9420166015625,
      "loss": 0.0984,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.624817132949829,
      "rewards/margins": 7.7060065269470215,
      "rewards/rejected": -9.33082389831543,
      "step": 880
    },
    {
      "epoch": 1.8874666666666666,
      "grad_norm": 2.0391299724578857,
      "learning_rate": 1.6047969155391856e-06,
      "logits/chosen": -1.6875035762786865,
      "logits/rejected": -1.5945326089859009,
      "logps/chosen": -264.5698547363281,
      "logps/rejected": -373.34814453125,
      "loss": 0.0744,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.7919718027114868,
      "rewards/margins": 7.827820777893066,
      "rewards/rejected": -9.619791984558105,
      "step": 885
    },
    {
      "epoch": 1.8981333333333335,
      "grad_norm": 2.268467426300049,
      "learning_rate": 1.5779756842553617e-06,
      "logits/chosen": -1.7344768047332764,
      "logits/rejected": -1.6171687841415405,
      "logps/chosen": -244.0048828125,
      "logps/rejected": -379.1364440917969,
      "loss": 0.1076,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.9673188924789429,
      "rewards/margins": 7.8214263916015625,
      "rewards/rejected": -9.788744926452637,
      "step": 890
    },
    {
      "epoch": 1.9088,
      "grad_norm": 1.3239772319793701,
      "learning_rate": 1.551276731045157e-06,
      "logits/chosen": -1.758827567100525,
      "logits/rejected": -1.612735390663147,
      "logps/chosen": -256.748779296875,
      "logps/rejected": -374.85943603515625,
      "loss": 0.0809,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.4343198537826538,
      "rewards/margins": 7.9875168800354,
      "rewards/rejected": -9.421835899353027,
      "step": 895
    },
    {
      "epoch": 1.9194666666666667,
      "grad_norm": 2.2805094718933105,
      "learning_rate": 1.5247035967008369e-06,
      "logits/chosen": -1.7128002643585205,
      "logits/rejected": -1.556166410446167,
      "logps/chosen": -239.7552490234375,
      "logps/rejected": -344.92205810546875,
      "loss": 0.1143,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.8794777393341064,
      "rewards/margins": 7.6836981773376465,
      "rewards/rejected": -9.563177108764648,
      "step": 900
    },
    {
      "epoch": 1.9301333333333335,
      "grad_norm": 0.7183341979980469,
      "learning_rate": 1.498259805328678e-06,
      "logits/chosen": -1.7574424743652344,
      "logits/rejected": -1.6206070184707642,
      "logps/chosen": -240.24465942382812,
      "logps/rejected": -349.49127197265625,
      "loss": 0.1213,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.0217463970184326,
      "rewards/margins": 7.537483215332031,
      "rewards/rejected": -9.559228897094727,
      "step": 905
    },
    {
      "epoch": 1.9407999999999999,
      "grad_norm": 3.615495443344116,
      "learning_rate": 1.471948863881601e-06,
      "logits/chosen": -1.759718656539917,
      "logits/rejected": -1.6743332147598267,
      "logps/chosen": -258.42999267578125,
      "logps/rejected": -365.8958435058594,
      "loss": 0.073,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.8100903034210205,
      "rewards/margins": 7.1778883934021,
      "rewards/rejected": -8.9879789352417,
      "step": 910
    },
    {
      "epoch": 1.9514666666666667,
      "grad_norm": 3.234266996383667,
      "learning_rate": 1.4457742616940864e-06,
      "logits/chosen": -1.6585487127304077,
      "logits/rejected": -1.5793458223342896,
      "logps/chosen": -274.3389587402344,
      "logps/rejected": -381.27581787109375,
      "loss": 0.0744,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.9123642444610596,
      "rewards/margins": 7.680113792419434,
      "rewards/rejected": -9.59247875213623,
      "step": 915
    },
    {
      "epoch": 1.9621333333333333,
      "grad_norm": 1.0968949794769287,
      "learning_rate": 1.4197394700194165e-06,
      "logits/chosen": -1.7973190546035767,
      "logits/rejected": -1.7022031545639038,
      "logps/chosen": -233.85910034179688,
      "logps/rejected": -363.92987060546875,
      "loss": 0.0566,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.6231153011322021,
      "rewards/margins": 8.063187599182129,
      "rewards/rejected": -9.68630313873291,
      "step": 920
    },
    {
      "epoch": 1.9727999999999999,
      "grad_norm": 5.4347100257873535,
      "learning_rate": 1.3938479415693244e-06,
      "logits/chosen": -1.8134357929229736,
      "logits/rejected": -1.7727657556533813,
      "logps/chosen": -241.57418823242188,
      "logps/rejected": -374.43780517578125,
      "loss": 0.107,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.7810213565826416,
      "rewards/margins": 7.694109916687012,
      "rewards/rejected": -9.475130081176758,
      "step": 925
    },
    {
      "epoch": 1.9834666666666667,
      "grad_norm": 3.721148729324341,
      "learning_rate": 1.368103110056096e-06,
      "logits/chosen": -1.7720524072647095,
      "logits/rejected": -1.690016508102417,
      "logps/chosen": -250.36215209960938,
      "logps/rejected": -392.93994140625,
      "loss": 0.0789,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.9710109233856201,
      "rewards/margins": 8.534833908081055,
      "rewards/rejected": -10.50584602355957,
      "step": 930
    },
    {
      "epoch": 1.9941333333333333,
      "grad_norm": 1.1054441928863525,
      "learning_rate": 1.3425083897371983e-06,
      "logits/chosen": -1.6688096523284912,
      "logits/rejected": -1.5874191522598267,
      "logps/chosen": -264.9595031738281,
      "logps/rejected": -363.69940185546875,
      "loss": 0.1013,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.360398530960083,
      "rewards/margins": 7.5831756591796875,
      "rewards/rejected": -8.943574905395508,
      "step": 935
    },
    {
      "epoch": 2.0042666666666666,
      "grad_norm": 1.7867152690887451,
      "learning_rate": 1.317067174962478e-06,
      "logits/chosen": -1.7479714155197144,
      "logits/rejected": -1.5715677738189697,
      "logps/chosen": -248.16004943847656,
      "logps/rejected": -350.427978515625,
      "loss": 0.0779,
      "rewards/accuracies": 0.9802631735801697,
      "rewards/chosen": -1.6614056825637817,
      "rewards/margins": 7.558930397033691,
      "rewards/rejected": -9.220335960388184,
      "step": 940
    },
    {
      "epoch": 2.0149333333333335,
      "grad_norm": 2.6329009532928467,
      "learning_rate": 1.291782839724009e-06,
      "logits/chosen": -1.724474549293518,
      "logits/rejected": -1.6523854732513428,
      "logps/chosen": -265.9844665527344,
      "logps/rejected": -377.27471923828125,
      "loss": 0.0861,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.9542032480239868,
      "rewards/margins": 7.510810852050781,
      "rewards/rejected": -9.46501350402832,
      "step": 945
    },
    {
      "epoch": 2.0256,
      "grad_norm": 3.648782968521118,
      "learning_rate": 1.2666587372086352e-06,
      "logits/chosen": -1.7386844158172607,
      "logits/rejected": -1.630690336227417,
      "logps/chosen": -201.80908203125,
      "logps/rejected": -352.9446716308594,
      "loss": 0.078,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.6233279705047607,
      "rewards/margins": 7.797116756439209,
      "rewards/rejected": -9.420445442199707,
      "step": 950
    },
    {
      "epoch": 2.0362666666666667,
      "grad_norm": 0.9289124011993408,
      "learning_rate": 1.241698199353275e-06,
      "logits/chosen": -1.8293838500976562,
      "logits/rejected": -1.7059133052825928,
      "logps/chosen": -230.53854370117188,
      "logps/rejected": -376.99884033203125,
      "loss": 0.0661,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.746546983718872,
      "rewards/margins": 8.739092826843262,
      "rewards/rejected": -10.485639572143555,
      "step": 955
    },
    {
      "epoch": 2.0469333333333335,
      "grad_norm": 0.8716382384300232,
      "learning_rate": 1.2169045364030404e-06,
      "logits/chosen": -1.8371355533599854,
      "logits/rejected": -1.6864582300186157,
      "logps/chosen": -228.82418823242188,
      "logps/rejected": -379.0071105957031,
      "loss": 0.0664,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.8618011474609375,
      "rewards/margins": 8.489458084106445,
      "rewards/rejected": -10.351259231567383,
      "step": 960
    },
    {
      "epoch": 2.0576,
      "grad_norm": 2.5190982818603516,
      "learning_rate": 1.1922810364722402e-06,
      "logits/chosen": -1.7426660060882568,
      "logits/rejected": -1.6516612768173218,
      "logps/chosen": -269.8039855957031,
      "logps/rejected": -393.49322509765625,
      "loss": 0.0921,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.9855560064315796,
      "rewards/margins": 7.847367286682129,
      "rewards/rejected": -9.832923889160156,
      "step": 965
    },
    {
      "epoch": 2.0682666666666667,
      "grad_norm": 0.5930553078651428,
      "learning_rate": 1.1678309651083055e-06,
      "logits/chosen": -1.806814193725586,
      "logits/rejected": -1.695652961730957,
      "logps/chosen": -225.7905731201172,
      "logps/rejected": -384.2733459472656,
      "loss": 0.0463,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.9584178924560547,
      "rewards/margins": 9.098180770874023,
      "rewards/rejected": -11.056598663330078,
      "step": 970
    },
    {
      "epoch": 2.0789333333333335,
      "grad_norm": 0.8087952136993408,
      "learning_rate": 1.1435575648587224e-06,
      "logits/chosen": -1.695840835571289,
      "logits/rejected": -1.6272556781768799,
      "logps/chosen": -250.8284454345703,
      "logps/rejected": -380.99847412109375,
      "loss": 0.0551,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.8700387477874756,
      "rewards/margins": 8.868024826049805,
      "rewards/rejected": -10.738062858581543,
      "step": 975
    },
    {
      "epoch": 2.0896,
      "grad_norm": 1.1131638288497925,
      "learning_rate": 1.1194640548410039e-06,
      "logits/chosen": -1.637101173400879,
      "logits/rejected": -1.589843511581421,
      "logps/chosen": -263.36175537109375,
      "logps/rejected": -357.9180603027344,
      "loss": 0.0818,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.8525962829589844,
      "rewards/margins": 7.566631317138672,
      "rewards/rejected": -9.419228553771973,
      "step": 980
    },
    {
      "epoch": 2.1002666666666667,
      "grad_norm": 1.2178566455841064,
      "learning_rate": 1.095553630315774e-06,
      "logits/chosen": -1.698826789855957,
      "logits/rejected": -1.611319899559021,
      "logps/chosen": -253.77139282226562,
      "logps/rejected": -394.76702880859375,
      "loss": 0.0705,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.7703306674957275,
      "rewards/margins": 8.478639602661133,
      "rewards/rejected": -10.248971939086914,
      "step": 985
    },
    {
      "epoch": 2.1109333333333336,
      "grad_norm": 1.0288853645324707,
      "learning_rate": 1.0718294622630188e-06,
      "logits/chosen": -1.676661729812622,
      "logits/rejected": -1.6332896947860718,
      "logps/chosen": -252.76467895507812,
      "logps/rejected": -379.791748046875,
      "loss": 0.0929,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.8333266973495483,
      "rewards/margins": 7.959515571594238,
      "rewards/rejected": -9.792842864990234,
      "step": 990
    },
    {
      "epoch": 2.1216,
      "grad_norm": 2.387310266494751,
      "learning_rate": 1.0482946969615495e-06,
      "logits/chosen": -1.8010175228118896,
      "logits/rejected": -1.6718385219573975,
      "logps/chosen": -241.35171508789062,
      "logps/rejected": -376.3408508300781,
      "loss": 0.0469,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.301366090774536,
      "rewards/margins": 8.878290176391602,
      "rewards/rejected": -11.179656982421875,
      "step": 995
    },
    {
      "epoch": 2.1322666666666668,
      "grad_norm": 3.4971799850463867,
      "learning_rate": 1.0249524555717461e-06,
      "logits/chosen": -1.787679672241211,
      "logits/rejected": -1.628638505935669,
      "logps/chosen": -230.5235137939453,
      "logps/rejected": -347.1987609863281,
      "loss": 0.0644,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.6777393817901611,
      "rewards/margins": 7.927115440368652,
      "rewards/rejected": -9.604854583740234,
      "step": 1000
    },
    {
      "epoch": 2.142933333333333,
      "grad_norm": 0.9783017635345459,
      "learning_rate": 1.0018058337216327e-06,
      "logits/chosen": -1.654268503189087,
      "logits/rejected": -1.5726966857910156,
      "logps/chosen": -252.07546997070312,
      "logps/rejected": -372.8138732910156,
      "loss": 0.0587,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.9420814514160156,
      "rewards/margins": 8.198111534118652,
      "rewards/rejected": -10.140192985534668,
      "step": 1005
    },
    {
      "epoch": 2.1536,
      "grad_norm": 2.6391026973724365,
      "learning_rate": 9.788579010963387e-07,
      "logits/chosen": -1.7386356592178345,
      "logits/rejected": -1.6452341079711914,
      "logps/chosen": -250.1106719970703,
      "logps/rejected": -376.52764892578125,
      "loss": 0.0753,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.8499062061309814,
      "rewards/margins": 8.821678161621094,
      "rewards/rejected": -10.671584129333496,
      "step": 1010
    },
    {
      "epoch": 2.164266666666667,
      "grad_norm": 4.805166721343994,
      "learning_rate": 9.56111701030998e-07,
      "logits/chosen": -1.8246469497680664,
      "logits/rejected": -1.643933892250061,
      "logps/chosen": -256.89422607421875,
      "logps/rejected": -388.6980895996094,
      "loss": 0.0949,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.3212974071502686,
      "rewards/margins": 8.553018569946289,
      "rewards/rejected": -10.874316215515137,
      "step": 1015
    },
    {
      "epoch": 2.174933333333333,
      "grad_norm": 1.8791966438293457,
      "learning_rate": 9.335702501071459e-07,
      "logits/chosen": -1.7584949731826782,
      "logits/rejected": -1.5843560695648193,
      "logps/chosen": -254.95584106445312,
      "logps/rejected": -365.45166015625,
      "loss": 0.1029,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.2223076820373535,
      "rewards/margins": 7.929934501647949,
      "rewards/rejected": -10.152241706848145,
      "step": 1020
    },
    {
      "epoch": 2.1856,
      "grad_norm": 1.4675019979476929,
      "learning_rate": 9.112365377526636e-07,
      "logits/chosen": -1.7484461069107056,
      "logits/rejected": -1.598569631576538,
      "logps/chosen": -268.4307861328125,
      "logps/rejected": -397.59100341796875,
      "loss": 0.0757,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.514547348022461,
      "rewards/margins": 8.377134323120117,
      "rewards/rejected": -10.891680717468262,
      "step": 1025
    },
    {
      "epoch": 2.196266666666667,
      "grad_norm": 2.629225015640259,
      "learning_rate": 8.891135258453195e-07,
      "logits/chosen": -1.6625139713287354,
      "logits/rejected": -1.5738680362701416,
      "logps/chosen": -246.3118133544922,
      "logps/rejected": -382.7955627441406,
      "loss": 0.0669,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.123089551925659,
      "rewards/margins": 8.360490798950195,
      "rewards/rejected": -10.483579635620117,
      "step": 1030
    },
    {
      "epoch": 2.206933333333333,
      "grad_norm": 1.3968008756637573,
      "learning_rate": 8.672041483199709e-07,
      "logits/chosen": -1.670629858970642,
      "logits/rejected": -1.5821272134780884,
      "logps/chosen": -268.6837158203125,
      "logps/rejected": -375.99835205078125,
      "loss": 0.071,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.3144733905792236,
      "rewards/margins": 8.026359558105469,
      "rewards/rejected": -10.340832710266113,
      "step": 1035
    },
    {
      "epoch": 2.2176,
      "grad_norm": 2.3471174240112305,
      "learning_rate": 8.455113107794652e-07,
      "logits/chosen": -1.736445665359497,
      "logits/rejected": -1.6636123657226562,
      "logps/chosen": -240.90966796875,
      "logps/rejected": -368.02459716796875,
      "loss": 0.0671,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.1975467205047607,
      "rewards/margins": 8.53138542175293,
      "rewards/rejected": -10.72893238067627,
      "step": 1040
    },
    {
      "epoch": 2.228266666666667,
      "grad_norm": 2.631293773651123,
      "learning_rate": 8.240378901093035e-07,
      "logits/chosen": -1.728139877319336,
      "logits/rejected": -1.565740942955017,
      "logps/chosen": -247.2626190185547,
      "logps/rejected": -365.80255126953125,
      "loss": 0.064,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.0829832553863525,
      "rewards/margins": 8.492073059082031,
      "rewards/rejected": -10.575056076049805,
      "step": 1045
    },
    {
      "epoch": 2.238933333333333,
      "grad_norm": 0.5728951692581177,
      "learning_rate": 8.027867340961113e-07,
      "logits/chosen": -1.673462152481079,
      "logits/rejected": -1.5609209537506104,
      "logps/chosen": -229.69677734375,
      "logps/rejected": -372.80145263671875,
      "loss": 0.047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.137375593185425,
      "rewards/margins": 8.91349983215332,
      "rewards/rejected": -11.050874710083008,
      "step": 1050
    },
    {
      "epoch": 2.2496,
      "grad_norm": 3.399742364883423,
      "learning_rate": 7.817606610499639e-07,
      "logits/chosen": -1.8059459924697876,
      "logits/rejected": -1.6398189067840576,
      "logps/chosen": -245.95883178710938,
      "logps/rejected": -367.82415771484375,
      "loss": 0.0761,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.1454949378967285,
      "rewards/margins": 8.3766450881958,
      "rewards/rejected": -10.522140502929688,
      "step": 1055
    },
    {
      "epoch": 2.2602666666666664,
      "grad_norm": 1.9253159761428833,
      "learning_rate": 7.609624594306278e-07,
      "logits/chosen": -1.772464394569397,
      "logits/rejected": -1.6853172779083252,
      "logps/chosen": -240.6079559326172,
      "logps/rejected": -407.1769104003906,
      "loss": 0.0615,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.328904628753662,
      "rewards/margins": 8.971172332763672,
      "rewards/rejected": -11.300076484680176,
      "step": 1060
    },
    {
      "epoch": 2.2709333333333332,
      "grad_norm": 0.7144515514373779,
      "learning_rate": 7.403948874777556e-07,
      "logits/chosen": -1.7335847616195679,
      "logits/rejected": -1.6609535217285156,
      "logps/chosen": -245.9636993408203,
      "logps/rejected": -373.9549560546875,
      "loss": 0.0584,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.3022031784057617,
      "rewards/margins": 8.158570289611816,
      "rewards/rejected": -10.460772514343262,
      "step": 1065
    },
    {
      "epoch": 2.2816,
      "grad_norm": 1.1433680057525635,
      "learning_rate": 7.20060672845091e-07,
      "logits/chosen": -1.7069857120513916,
      "logits/rejected": -1.6626923084259033,
      "logps/chosen": -247.3767547607422,
      "logps/rejected": -388.65264892578125,
      "loss": 0.0797,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.4776415824890137,
      "rewards/margins": 8.408472061157227,
      "rewards/rejected": -10.886113166809082,
      "step": 1070
    },
    {
      "epoch": 2.2922666666666665,
      "grad_norm": 2.0684447288513184,
      "learning_rate": 6.99962512238733e-07,
      "logits/chosen": -1.5935136079788208,
      "logits/rejected": -1.552176833152771,
      "logps/chosen": -240.5950927734375,
      "logps/rejected": -382.0034484863281,
      "loss": 0.0682,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.435925245285034,
      "rewards/margins": 8.210624694824219,
      "rewards/rejected": -10.646550178527832,
      "step": 1075
    },
    {
      "epoch": 2.3029333333333333,
      "grad_norm": 3.9305777549743652,
      "learning_rate": 6.801030710594949e-07,
      "logits/chosen": -1.6194684505462646,
      "logits/rejected": -1.521071195602417,
      "logps/chosen": -240.2565155029297,
      "logps/rejected": -384.17315673828125,
      "loss": 0.0681,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.3575754165649414,
      "rewards/margins": 9.207578659057617,
      "rewards/rejected": -11.565154075622559,
      "step": 1080
    },
    {
      "epoch": 2.3136,
      "grad_norm": 3.65750789642334,
      "learning_rate": 6.604849830494264e-07,
      "logits/chosen": -1.7517938613891602,
      "logits/rejected": -1.7062915563583374,
      "logps/chosen": -236.4104461669922,
      "logps/rejected": -396.84033203125,
      "loss": 0.0568,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.530136823654175,
      "rewards/margins": 9.359679222106934,
      "rewards/rejected": -11.889817237854004,
      "step": 1085
    },
    {
      "epoch": 2.3242666666666665,
      "grad_norm": 1.4848530292510986,
      "learning_rate": 6.411108499425253e-07,
      "logits/chosen": -1.7016388177871704,
      "logits/rejected": -1.6188669204711914,
      "logps/chosen": -243.3367462158203,
      "logps/rejected": -396.5169982910156,
      "loss": 0.0546,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.394667625427246,
      "rewards/margins": 8.645791053771973,
      "rewards/rejected": -11.040457725524902,
      "step": 1090
    },
    {
      "epoch": 2.3349333333333333,
      "grad_norm": 3.0178022384643555,
      "learning_rate": 6.219832411197013e-07,
      "logits/chosen": -1.6480319499969482,
      "logits/rejected": -1.4744436740875244,
      "logps/chosen": -265.7557067871094,
      "logps/rejected": -374.42816162109375,
      "loss": 0.0744,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.212043046951294,
      "rewards/margins": 8.311385154724121,
      "rewards/rejected": -10.523427963256836,
      "step": 1095
    },
    {
      "epoch": 2.3456,
      "grad_norm": 2.0852251052856445,
      "learning_rate": 6.031046932680229e-07,
      "logits/chosen": -1.7345670461654663,
      "logits/rejected": -1.609113335609436,
      "logps/chosen": -247.75338745117188,
      "logps/rejected": -375.01605224609375,
      "loss": 0.0828,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.2977194786071777,
      "rewards/margins": 7.741724967956543,
      "rewards/rejected": -10.039443969726562,
      "step": 1100
    },
    {
      "epoch": 2.3562666666666665,
      "grad_norm": 3.485975980758667,
      "learning_rate": 5.844777100443091e-07,
      "logits/chosen": -1.7265936136245728,
      "logits/rejected": -1.6472886800765991,
      "logps/chosen": -248.5940704345703,
      "logps/rejected": -372.21319580078125,
      "loss": 0.0872,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.629234790802002,
      "rewards/margins": 8.200715065002441,
      "rewards/rejected": -10.829948425292969,
      "step": 1105
    },
    {
      "epoch": 2.3669333333333333,
      "grad_norm": 0.6572098135948181,
      "learning_rate": 5.66104761743092e-07,
      "logits/chosen": -1.7917197942733765,
      "logits/rejected": -1.7144348621368408,
      "logps/chosen": -228.86181640625,
      "logps/rejected": -405.03009033203125,
      "loss": 0.0416,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.5428788661956787,
      "rewards/margins": 9.689337730407715,
      "rewards/rejected": -12.232217788696289,
      "step": 1110
    },
    {
      "epoch": 2.3776,
      "grad_norm": 4.4979777336120605,
      "learning_rate": 5.479882849690102e-07,
      "logits/chosen": -1.708397626876831,
      "logits/rejected": -1.6033588647842407,
      "logps/chosen": -244.21762084960938,
      "logps/rejected": -382.5280456542969,
      "loss": 0.0715,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.0844390392303467,
      "rewards/margins": 9.081504821777344,
      "rewards/rejected": -11.165945053100586,
      "step": 1115
    },
    {
      "epoch": 2.3882666666666665,
      "grad_norm": 1.708503007888794,
      "learning_rate": 5.301306823136687e-07,
      "logits/chosen": -1.6154632568359375,
      "logits/rejected": -1.5289294719696045,
      "logps/chosen": -249.3914794921875,
      "logps/rejected": -383.60150146484375,
      "loss": 0.0559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.4148969650268555,
      "rewards/margins": 8.664996147155762,
      "rewards/rejected": -11.079894065856934,
      "step": 1120
    },
    {
      "epoch": 2.3989333333333334,
      "grad_norm": 4.531334400177002,
      "learning_rate": 5.125343220370088e-07,
      "logits/chosen": -1.7035802602767944,
      "logits/rejected": -1.6139618158340454,
      "logps/chosen": -239.21157836914062,
      "logps/rejected": -351.17828369140625,
      "loss": 0.0821,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.9980465173721313,
      "rewards/margins": 8.510173797607422,
      "rewards/rejected": -10.508219718933105,
      "step": 1125
    },
    {
      "epoch": 2.4096,
      "grad_norm": 1.7990802526474,
      "learning_rate": 4.952015377532329e-07,
      "logits/chosen": -1.7770929336547852,
      "logits/rejected": -1.6199886798858643,
      "logps/chosen": -244.71224975585938,
      "logps/rejected": -403.7447509765625,
      "loss": 0.0349,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.51416015625,
      "rewards/margins": 8.962092399597168,
      "rewards/rejected": -11.476252555847168,
      "step": 1130
    },
    {
      "epoch": 2.4202666666666666,
      "grad_norm": 0.9469951391220093,
      "learning_rate": 4.781346281213204e-07,
      "logits/chosen": -1.7762229442596436,
      "logits/rejected": -1.6669120788574219,
      "logps/chosen": -256.1883544921875,
      "logps/rejected": -393.80706787109375,
      "loss": 0.0476,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.724867820739746,
      "rewards/margins": 9.057721138000488,
      "rewards/rejected": -11.78258991241455,
      "step": 1135
    },
    {
      "epoch": 2.4309333333333334,
      "grad_norm": 2.5484354496002197,
      "learning_rate": 4.613358565401818e-07,
      "logits/chosen": -1.6508283615112305,
      "logits/rejected": -1.5555996894836426,
      "logps/chosen": -270.6368408203125,
      "logps/rejected": -405.41363525390625,
      "loss": 0.0857,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.6728577613830566,
      "rewards/margins": 8.623315811157227,
      "rewards/rejected": -11.296173095703125,
      "step": 1140
    },
    {
      "epoch": 2.4416,
      "grad_norm": 3.9779036045074463,
      "learning_rate": 4.448074508484906e-07,
      "logits/chosen": -1.67898690700531,
      "logits/rejected": -1.5510108470916748,
      "logps/chosen": -245.95962524414062,
      "logps/rejected": -371.1490783691406,
      "loss": 0.0697,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.447220802307129,
      "rewards/margins": 8.701111793518066,
      "rewards/rejected": -11.148332595825195,
      "step": 1145
    },
    {
      "epoch": 2.4522666666666666,
      "grad_norm": 2.5895211696624756,
      "learning_rate": 4.2855160302922777e-07,
      "logits/chosen": -1.7640804052352905,
      "logits/rejected": -1.6478509902954102,
      "logps/chosen": -235.4073028564453,
      "logps/rejected": -381.1341247558594,
      "loss": 0.0629,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.591489791870117,
      "rewards/margins": 8.705648422241211,
      "rewards/rejected": -11.297138214111328,
      "step": 1150
    },
    {
      "epoch": 2.4629333333333334,
      "grad_norm": 2.4404029846191406,
      "learning_rate": 4.125704689189819e-07,
      "logits/chosen": -1.6468175649642944,
      "logits/rejected": -1.5339540243148804,
      "logps/chosen": -265.91986083984375,
      "logps/rejected": -380.76019287109375,
      "loss": 0.0784,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.748948574066162,
      "rewards/margins": 7.9840497970581055,
      "rewards/rejected": -10.732998847961426,
      "step": 1155
    },
    {
      "epoch": 2.4736000000000002,
      "grad_norm": 2.37691330909729,
      "learning_rate": 3.9686616792204677e-07,
      "logits/chosen": -1.718951940536499,
      "logits/rejected": -1.6564605236053467,
      "logps/chosen": -266.5101013183594,
      "logps/rejected": -405.48638916015625,
      "loss": 0.05,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.9024240970611572,
      "rewards/margins": 9.125859260559082,
      "rewards/rejected": -12.028284072875977,
      "step": 1160
    },
    {
      "epoch": 2.4842666666666666,
      "grad_norm": 1.4729171991348267,
      "learning_rate": 3.8144078272934676e-07,
      "logits/chosen": -1.6833305358886719,
      "logits/rejected": -1.5863052606582642,
      "logps/chosen": -255.60757446289062,
      "logps/rejected": -390.80413818359375,
      "loss": 0.045,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.29118275642395,
      "rewards/margins": 9.308106422424316,
      "rewards/rejected": -11.59929084777832,
      "step": 1165
    },
    {
      "epoch": 2.4949333333333334,
      "grad_norm": 2.2732648849487305,
      "learning_rate": 3.6629635904222943e-07,
      "logits/chosen": -1.6575851440429688,
      "logits/rejected": -1.6193211078643799,
      "logps/chosen": -245.96743774414062,
      "logps/rejected": -375.4206848144531,
      "loss": 0.0516,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.344364881515503,
      "rewards/margins": 8.420816421508789,
      "rewards/rejected": -10.765182495117188,
      "step": 1170
    },
    {
      "epoch": 2.5056000000000003,
      "grad_norm": 1.3393036127090454,
      "learning_rate": 3.514349053011698e-07,
      "logits/chosen": -1.6279098987579346,
      "logits/rejected": -1.5382213592529297,
      "logps/chosen": -235.5139923095703,
      "logps/rejected": -388.8031921386719,
      "loss": 0.1437,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.400181531906128,
      "rewards/margins": 8.55920124053955,
      "rewards/rejected": -10.959383010864258,
      "step": 1175
    },
    {
      "epoch": 2.5162666666666667,
      "grad_norm": 2.17390775680542,
      "learning_rate": 3.3685839241941133e-07,
      "logits/chosen": -1.6647920608520508,
      "logits/rejected": -1.5797088146209717,
      "logps/chosen": -253.29006958007812,
      "logps/rejected": -357.0965881347656,
      "loss": 0.0557,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.246269464492798,
      "rewards/margins": 8.394041061401367,
      "rewards/rejected": -10.640310287475586,
      "step": 1180
    },
    {
      "epoch": 2.5269333333333335,
      "grad_norm": 3.978947877883911,
      "learning_rate": 3.2256875352158435e-07,
      "logits/chosen": -1.7043958902359009,
      "logits/rejected": -1.5851062536239624,
      "logps/chosen": -250.3873748779297,
      "logps/rejected": -386.988037109375,
      "loss": 0.0816,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.7624685764312744,
      "rewards/margins": 8.765403747558594,
      "rewards/rejected": -11.527873039245605,
      "step": 1185
    },
    {
      "epoch": 2.5376,
      "grad_norm": 1.3339006900787354,
      "learning_rate": 3.085678836873399e-07,
      "logits/chosen": -1.7824872732162476,
      "logits/rejected": -1.6766750812530518,
      "logps/chosen": -248.77224731445312,
      "logps/rejected": -390.64532470703125,
      "loss": 0.0765,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.545628309249878,
      "rewards/margins": 8.543272972106934,
      "rewards/rejected": -11.088900566101074,
      "step": 1190
    },
    {
      "epoch": 2.5482666666666667,
      "grad_norm": 2.1457443237304688,
      "learning_rate": 2.9485763970002106e-07,
      "logits/chosen": -1.7480427026748657,
      "logits/rejected": -1.588936448097229,
      "logps/chosen": -245.76425170898438,
      "logps/rejected": -367.2337646484375,
      "loss": 0.0665,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.6265664100646973,
      "rewards/margins": 8.219732284545898,
      "rewards/rejected": -10.846298217773438,
      "step": 1195
    },
    {
      "epoch": 2.558933333333333,
      "grad_norm": 2.72080397605896,
      "learning_rate": 2.8143983980042174e-07,
      "logits/chosen": -1.7567298412322998,
      "logits/rejected": -1.6189864873886108,
      "logps/chosen": -238.1021728515625,
      "logps/rejected": -386.2373962402344,
      "loss": 0.0649,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.4193553924560547,
      "rewards/margins": 9.376398086547852,
      "rewards/rejected": -11.795753479003906,
      "step": 1200
    },
    {
      "epoch": 2.5696,
      "grad_norm": 3.7256271839141846,
      "learning_rate": 2.683162634456515e-07,
      "logits/chosen": -1.6302845478057861,
      "logits/rejected": -1.535885214805603,
      "logps/chosen": -259.4132385253906,
      "logps/rejected": -407.36553955078125,
      "loss": 0.0608,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.306084394454956,
      "rewards/margins": 9.6066312789917,
      "rewards/rejected": -11.91271686553955,
      "step": 1205
    },
    {
      "epoch": 2.5802666666666667,
      "grad_norm": 4.272429943084717,
      "learning_rate": 2.5548865107314606e-07,
      "logits/chosen": -1.6210415363311768,
      "logits/rejected": -1.532249927520752,
      "logps/chosen": -274.93890380859375,
      "logps/rejected": -375.2662353515625,
      "loss": 0.0568,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.343566417694092,
      "rewards/margins": 8.680849075317383,
      "rewards/rejected": -11.024415969848633,
      "step": 1210
    },
    {
      "epoch": 2.590933333333333,
      "grad_norm": 3.4025235176086426,
      "learning_rate": 2.429587038698511e-07,
      "logits/chosen": -1.687776803970337,
      "logits/rejected": -1.6682751178741455,
      "logps/chosen": -225.93594360351562,
      "logps/rejected": -393.0034484863281,
      "loss": 0.0495,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.5923023223876953,
      "rewards/margins": 9.531915664672852,
      "rewards/rejected": -12.124217987060547,
      "step": 1215
    },
    {
      "epoch": 2.6016,
      "grad_norm": 1.6154532432556152,
      "learning_rate": 2.307280835466144e-07,
      "logits/chosen": -1.658247709274292,
      "logits/rejected": -1.6115080118179321,
      "logps/chosen": -238.1659698486328,
      "logps/rejected": -411.9134826660156,
      "loss": 0.054,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.8724894523620605,
      "rewards/margins": 9.156352043151855,
      "rewards/rejected": -12.028841018676758,
      "step": 1220
    },
    {
      "epoch": 2.6122666666666667,
      "grad_norm": 1.3085912466049194,
      "learning_rate": 2.1879841211780856e-07,
      "logits/chosen": -1.6801115274429321,
      "logits/rejected": -1.6072133779525757,
      "logps/chosen": -246.29122924804688,
      "logps/rejected": -375.62725830078125,
      "loss": 0.0622,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.3886702060699463,
      "rewards/margins": 8.405167579650879,
      "rewards/rejected": -10.793837547302246,
      "step": 1225
    },
    {
      "epoch": 2.622933333333333,
      "grad_norm": 1.8979953527450562,
      "learning_rate": 2.0717127168622202e-07,
      "logits/chosen": -1.7112290859222412,
      "logits/rejected": -1.634482979774475,
      "logps/chosen": -248.1447296142578,
      "logps/rejected": -380.28729248046875,
      "loss": 0.0548,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.502384901046753,
      "rewards/margins": 8.42490005493164,
      "rewards/rejected": -10.927285194396973,
      "step": 1230
    },
    {
      "epoch": 2.6336,
      "grad_norm": 1.139436960220337,
      "learning_rate": 1.958482042332424e-07,
      "logits/chosen": -1.6665029525756836,
      "logits/rejected": -1.5504449605941772,
      "logps/chosen": -265.2987365722656,
      "logps/rejected": -388.6627502441406,
      "loss": 0.0446,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.944598436355591,
      "rewards/margins": 8.778532028198242,
      "rewards/rejected": -11.72313117980957,
      "step": 1235
    },
    {
      "epoch": 2.6442666666666668,
      "grad_norm": 6.541167736053467,
      "learning_rate": 1.8483071141435937e-07,
      "logits/chosen": -1.6075607538223267,
      "logits/rejected": -1.5328432321548462,
      "logps/chosen": -263.16680908203125,
      "logps/rejected": -365.10736083984375,
      "loss": 0.0794,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.3820810317993164,
      "rewards/margins": 8.6204252243042,
      "rewards/rejected": -11.002506256103516,
      "step": 1240
    },
    {
      "epoch": 2.654933333333333,
      "grad_norm": 3.5753588676452637,
      "learning_rate": 1.741202543600176e-07,
      "logits/chosen": -1.742559790611267,
      "logits/rejected": -1.6055104732513428,
      "logps/chosen": -245.0276336669922,
      "logps/rejected": -397.55206298828125,
      "loss": 0.0687,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.889216661453247,
      "rewards/margins": 9.328636169433594,
      "rewards/rejected": -12.217851638793945,
      "step": 1245
    },
    {
      "epoch": 2.6656,
      "grad_norm": 3.1118340492248535,
      "learning_rate": 1.6371825348184155e-07,
      "logits/chosen": -1.6370012760162354,
      "logits/rejected": -1.5552282333374023,
      "logps/chosen": -254.6912078857422,
      "logps/rejected": -362.8533020019531,
      "loss": 0.0585,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.709108829498291,
      "rewards/margins": 8.306825637817383,
      "rewards/rejected": -11.015934944152832,
      "step": 1250
    },
    {
      "epoch": 2.676266666666667,
      "grad_norm": 1.2242422103881836,
      "learning_rate": 1.5362608828426246e-07,
      "logits/chosen": -1.7631776332855225,
      "logits/rejected": -1.6615889072418213,
      "logps/chosen": -275.11981201171875,
      "logps/rejected": -398.39996337890625,
      "loss": 0.0543,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.662536382675171,
      "rewards/margins": 8.675864219665527,
      "rewards/rejected": -11.338400840759277,
      "step": 1255
    },
    {
      "epoch": 2.686933333333333,
      "grad_norm": 3.2008516788482666,
      "learning_rate": 1.438450971815686e-07,
      "logits/chosen": -1.717780351638794,
      "logits/rejected": -1.6827186346054077,
      "logps/chosen": -232.81961059570312,
      "logps/rejected": -409.2147521972656,
      "loss": 0.0843,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.2906346321105957,
      "rewards/margins": 9.57332706451416,
      "rewards/rejected": -12.863962173461914,
      "step": 1260
    },
    {
      "epoch": 2.6976,
      "grad_norm": 2.0423855781555176,
      "learning_rate": 1.3437657732040783e-07,
      "logits/chosen": -1.7342054843902588,
      "logits/rejected": -1.6255111694335938,
      "logps/chosen": -253.91708374023438,
      "logps/rejected": -384.605712890625,
      "loss": 0.0548,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.2692885398864746,
      "rewards/margins": 9.20631217956543,
      "rewards/rejected": -11.475601196289062,
      "step": 1265
    },
    {
      "epoch": 2.708266666666667,
      "grad_norm": 6.765120506286621,
      "learning_rate": 1.2522178440776022e-07,
      "logits/chosen": -1.704227089881897,
      "logits/rejected": -1.6806764602661133,
      "logps/chosen": -253.9882354736328,
      "logps/rejected": -389.9085998535156,
      "loss": 0.1024,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.5248208045959473,
      "rewards/margins": 8.699395179748535,
      "rewards/rejected": -11.224214553833008,
      "step": 1270
    },
    {
      "epoch": 2.718933333333333,
      "grad_norm": 1.7233941555023193,
      "learning_rate": 1.1638193254440655e-07,
      "logits/chosen": -1.7507209777832031,
      "logits/rejected": -1.6442060470581055,
      "logps/chosen": -235.013671875,
      "logps/rejected": -387.20709228515625,
      "loss": 0.0289,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.367755174636841,
      "rewards/margins": 9.576312065124512,
      "rewards/rejected": -11.944067001342773,
      "step": 1275
    },
    {
      "epoch": 2.7296,
      "grad_norm": 7.248271465301514,
      "learning_rate": 1.0785819406391562e-07,
      "logits/chosen": -1.65673828125,
      "logits/rejected": -1.6092407703399658,
      "logps/chosen": -228.1624755859375,
      "logps/rejected": -392.2158508300781,
      "loss": 0.1043,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.5122058391571045,
      "rewards/margins": 9.066630363464355,
      "rewards/rejected": -11.578836441040039,
      "step": 1280
    },
    {
      "epoch": 2.740266666666667,
      "grad_norm": 3.1239840984344482,
      "learning_rate": 9.965169937717095e-08,
      "logits/chosen": -1.7200371026992798,
      "logits/rejected": -1.6642881631851196,
      "logps/chosen": -239.06405639648438,
      "logps/rejected": -381.20440673828125,
      "loss": 0.0773,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.4310803413391113,
      "rewards/margins": 8.770525932312012,
      "rewards/rejected": -11.201605796813965,
      "step": 1285
    },
    {
      "epoch": 2.7509333333333332,
      "grad_norm": 2.2311618328094482,
      "learning_rate": 9.176353682245676e-08,
      "logits/chosen": -1.6120647192001343,
      "logits/rejected": -1.5454223155975342,
      "logps/chosen": -246.0549774169922,
      "logps/rejected": -379.45660400390625,
      "loss": 0.0733,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.705291271209717,
      "rewards/margins": 8.565595626831055,
      "rewards/rejected": -11.270886421203613,
      "step": 1290
    },
    {
      "epoch": 2.7616,
      "grad_norm": 0.7328024506568909,
      "learning_rate": 8.41947525211223e-08,
      "logits/chosen": -1.5624516010284424,
      "logits/rejected": -1.5214122533798218,
      "logps/chosen": -251.17922973632812,
      "logps/rejected": -388.30224609375,
      "loss": 0.063,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.610011339187622,
      "rewards/margins": 8.611980438232422,
      "rewards/rejected": -11.221991539001465,
      "step": 1295
    },
    {
      "epoch": 2.772266666666667,
      "grad_norm": 4.4146623611450195,
      "learning_rate": 7.694635023884789e-08,
      "logits/chosen": -1.6453464031219482,
      "logits/rejected": -1.5596413612365723,
      "logps/chosen": -261.37005615234375,
      "logps/rejected": -395.37274169921875,
      "loss": 0.1003,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.5829854011535645,
      "rewards/margins": 8.859365463256836,
      "rewards/rejected": -11.442350387573242,
      "step": 1300
    },
    {
      "epoch": 2.7829333333333333,
      "grad_norm": 3.653667688369751,
      "learning_rate": 7.00192912525241e-08,
      "logits/chosen": -1.6690375804901123,
      "logits/rejected": -1.559043526649475,
      "logps/chosen": -259.91229248046875,
      "logps/rejected": -400.84014892578125,
      "loss": 0.0712,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.8678531646728516,
      "rewards/margins": 9.116159439086914,
      "rewards/rejected": -11.98401165008545,
      "step": 1305
    },
    {
      "epoch": 2.7936,
      "grad_norm": 3.5960304737091064,
      "learning_rate": 6.341449422277047e-08,
      "logits/chosen": -1.6120831966400146,
      "logits/rejected": -1.5400011539459229,
      "logps/chosen": -263.94940185546875,
      "logps/rejected": -413.22259521484375,
      "loss": 0.0698,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.4007437229156494,
      "rewards/margins": 9.813273429870605,
      "rewards/rejected": -12.214017868041992,
      "step": 1310
    },
    {
      "epoch": 2.804266666666667,
      "grad_norm": 6.226192951202393,
      "learning_rate": 5.7132835072101486e-08,
      "logits/chosen": -1.7602459192276,
      "logits/rejected": -1.6944478750228882,
      "logps/chosen": -247.07528686523438,
      "logps/rejected": -404.52813720703125,
      "loss": 0.0735,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.5961649417877197,
      "rewards/margins": 9.362703323364258,
      "rewards/rejected": -11.958867073059082,
      "step": 1315
    },
    {
      "epoch": 2.8149333333333333,
      "grad_norm": 1.191611409187317,
      "learning_rate": 5.117514686876379e-08,
      "logits/chosen": -1.6201938390731812,
      "logits/rejected": -1.5016851425170898,
      "logps/chosen": -262.31060791015625,
      "logps/rejected": -399.95880126953125,
      "loss": 0.1033,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.614260196685791,
      "rewards/margins": 8.749483108520508,
      "rewards/rejected": -11.363743782043457,
      "step": 1320
    },
    {
      "epoch": 2.8256,
      "grad_norm": 0.6301141977310181,
      "learning_rate": 4.5542219716255274e-08,
      "logits/chosen": -1.6695480346679688,
      "logits/rejected": -1.5834704637527466,
      "logps/chosen": -268.4365234375,
      "logps/rejected": -412.6512756347656,
      "loss": 0.0368,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.7909514904022217,
      "rewards/margins": 9.708473205566406,
      "rewards/rejected": -12.499424934387207,
      "step": 1325
    },
    {
      "epoch": 2.836266666666667,
      "grad_norm": 1.9661916494369507,
      "learning_rate": 4.023480064854174e-08,
      "logits/chosen": -1.7694295644760132,
      "logits/rejected": -1.6655480861663818,
      "logps/chosen": -307.1631774902344,
      "logps/rejected": -419.0650939941406,
      "loss": 0.0428,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.6897337436676025,
      "rewards/margins": 9.9656982421875,
      "rewards/rejected": -12.655431747436523,
      "step": 1330
    },
    {
      "epoch": 2.8469333333333333,
      "grad_norm": 2.0274875164031982,
      "learning_rate": 3.525359353098584e-08,
      "logits/chosen": -1.7370857000350952,
      "logits/rejected": -1.6126264333724976,
      "logps/chosen": -258.9969177246094,
      "logps/rejected": -390.44873046875,
      "loss": 0.0781,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.651517391204834,
      "rewards/margins": 8.74222469329834,
      "rewards/rejected": -11.393742561340332,
      "step": 1335
    },
    {
      "epoch": 2.8576,
      "grad_norm": 0.6840654015541077,
      "learning_rate": 3.059925896700094e-08,
      "logits/chosen": -1.6134666204452515,
      "logits/rejected": -1.4918162822723389,
      "logps/chosen": -270.07086181640625,
      "logps/rejected": -397.61126708984375,
      "loss": 0.0576,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.371833324432373,
      "rewards/margins": 8.95458984375,
      "rewards/rejected": -11.326423645019531,
      "step": 1340
    },
    {
      "epoch": 2.8682666666666665,
      "grad_norm": 1.605663537979126,
      "learning_rate": 2.6272414210443964e-08,
      "logits/chosen": -1.6418520212173462,
      "logits/rejected": -1.5910496711730957,
      "logps/chosen": -270.58575439453125,
      "logps/rejected": -397.8790283203125,
      "loss": 0.0935,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.725590467453003,
      "rewards/margins": 8.987241744995117,
      "rewards/rejected": -11.712831497192383,
      "step": 1345
    },
    {
      "epoch": 2.8789333333333333,
      "grad_norm": 3.390230894088745,
      "learning_rate": 2.2273633083753632e-08,
      "logits/chosen": -1.756787657737732,
      "logits/rejected": -1.7060184478759766,
      "logps/chosen": -237.4143524169922,
      "logps/rejected": -390.0992431640625,
      "loss": 0.0485,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.657024621963501,
      "rewards/margins": 9.306405067443848,
      "rewards/rejected": -11.96342945098877,
      "step": 1350
    },
    {
      "epoch": 2.8895999999999997,
      "grad_norm": 1.4694249629974365,
      "learning_rate": 1.8603445901851647e-08,
      "logits/chosen": -1.6919931173324585,
      "logits/rejected": -1.6533225774765015,
      "logps/chosen": -259.6629638671875,
      "logps/rejected": -387.4039001464844,
      "loss": 0.0995,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.740100383758545,
      "rewards/margins": 8.346617698669434,
      "rewards/rejected": -11.08671760559082,
      "step": 1355
    },
    {
      "epoch": 2.9002666666666665,
      "grad_norm": 2.6060729026794434,
      "learning_rate": 1.5262339401813375e-08,
      "logits/chosen": -1.7318048477172852,
      "logits/rejected": -1.6237519979476929,
      "logps/chosen": -221.9154815673828,
      "logps/rejected": -387.20001220703125,
      "loss": 0.0549,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.4740569591522217,
      "rewards/margins": 9.128717422485352,
      "rewards/rejected": -11.602774620056152,
      "step": 1360
    },
    {
      "epoch": 2.9109333333333334,
      "grad_norm": 0.737140953540802,
      "learning_rate": 1.2250756678315601e-08,
      "logits/chosen": -1.7253105640411377,
      "logits/rejected": -1.6364980936050415,
      "logps/chosen": -254.7071533203125,
      "logps/rejected": -397.3651123046875,
      "loss": 0.058,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.506072521209717,
      "rewards/margins": 8.95616340637207,
      "rewards/rejected": -11.462237358093262,
      "step": 1365
    },
    {
      "epoch": 2.9215999999999998,
      "grad_norm": 2.5475025177001953,
      "learning_rate": 9.569097124875193e-09,
      "logits/chosen": -1.67425537109375,
      "logits/rejected": -1.5568588972091675,
      "logps/chosen": -247.0236358642578,
      "logps/rejected": -387.4052429199219,
      "loss": 0.0878,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.781790256500244,
      "rewards/margins": 9.150073051452637,
      "rewards/rejected": -11.931864738464355,
      "step": 1370
    },
    {
      "epoch": 2.9322666666666666,
      "grad_norm": 5.0270867347717285,
      "learning_rate": 7.217716380881479e-09,
      "logits/chosen": -1.7101695537567139,
      "logits/rejected": -1.637942910194397,
      "logps/chosen": -269.4486999511719,
      "logps/rejected": -391.3731384277344,
      "loss": 0.0823,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.8127238750457764,
      "rewards/margins": 8.60261058807373,
      "rewards/rejected": -11.415335655212402,
      "step": 1375
    },
    {
      "epoch": 2.9429333333333334,
      "grad_norm": 1.2320939302444458,
      "learning_rate": 5.196926284430359e-09,
      "logits/chosen": -1.6496827602386475,
      "logits/rejected": -1.570379614830017,
      "logps/chosen": -254.0030517578125,
      "logps/rejected": -404.6939392089844,
      "loss": 0.0626,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.8904614448547363,
      "rewards/margins": 8.672920227050781,
      "rewards/rejected": -11.563383102416992,
      "step": 1380
    },
    {
      "epoch": 2.9536,
      "grad_norm": 2.0147035121917725,
      "learning_rate": 3.5069948309712775e-09,
      "logits/chosen": -1.6875463724136353,
      "logits/rejected": -1.5886619091033936,
      "logps/chosen": -248.8668212890625,
      "logps/rejected": -395.17657470703125,
      "loss": 0.0602,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.4937005043029785,
      "rewards/margins": 9.028209686279297,
      "rewards/rejected": -11.521909713745117,
      "step": 1385
    },
    {
      "epoch": 2.9642666666666666,
      "grad_norm": 1.6050134897232056,
      "learning_rate": 2.1481461377634294e-09,
      "logits/chosen": -1.700669527053833,
      "logits/rejected": -1.642332673072815,
      "logps/chosen": -245.1533660888672,
      "logps/rejected": -398.7891845703125,
      "loss": 0.039,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.403536796569824,
      "rewards/margins": 9.711051940917969,
      "rewards/rejected": -12.114588737487793,
      "step": 1390
    },
    {
      "epoch": 2.9749333333333334,
      "grad_norm": 2.986335039138794,
      "learning_rate": 1.120560414154259e-09,
      "logits/chosen": -1.6862636804580688,
      "logits/rejected": -1.5859509706497192,
      "logps/chosen": -245.3629913330078,
      "logps/rejected": -389.30328369140625,
      "loss": 0.0603,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.827134132385254,
      "rewards/margins": 9.108485221862793,
      "rewards/rejected": -11.935619354248047,
      "step": 1395
    },
    {
      "epoch": 2.9856,
      "grad_norm": 1.4037060737609863,
      "learning_rate": 4.2437393768079983e-10,
      "logits/chosen": -1.5650889873504639,
      "logits/rejected": -1.5250481367111206,
      "logps/chosen": -265.4774475097656,
      "logps/rejected": -370.9742736816406,
      "loss": 0.0463,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.532733917236328,
      "rewards/margins": 8.557954788208008,
      "rewards/rejected": -11.090688705444336,
      "step": 1400
    },
    {
      "epoch": 2.9962666666666666,
      "grad_norm": 1.7923967838287354,
      "learning_rate": 5.967903599635171e-11,
      "logits/chosen": -1.688177466392517,
      "logits/rejected": -1.509224534034729,
      "logps/chosen": -274.61065673828125,
      "logps/rejected": -385.02349853515625,
      "loss": 0.0709,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.7587332725524902,
      "rewards/margins": 8.69700813293457,
      "rewards/rejected": -11.455740928649902,
      "step": 1405
    }
  ],
  "logging_steps": 5,
  "max_steps": 1407,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}