{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 1407,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010666666666666666,
      "grad_norm": 3.225456953048706,
      "learning_rate": 4.651162790697675e-07,
      "logits/chosen": -1.452316164970398,
      "logits/rejected": -1.3630012273788452,
      "logps/chosen": -216.8773651123047,
      "logps/rejected": -261.6772155761719,
      "loss": 0.6934,
      "rewards/accuracies": 0.29374998807907104,
      "rewards/chosen": -0.00024420738918706775,
      "rewards/margins": -0.0005015610950067639,
      "rewards/rejected": 0.00025735379313118756,
      "step": 5
    },
    {
      "epoch": 0.021333333333333333,
      "grad_norm": 3.985062599182129,
      "learning_rate": 1.0465116279069768e-06,
      "logits/chosen": -1.3076436519622803,
      "logits/rejected": -1.2246109247207642,
      "logps/chosen": -230.8943328857422,
      "logps/rejected": -299.6863708496094,
      "loss": 0.6899,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.001177475554868579,
      "rewards/margins": 0.006514736916869879,
      "rewards/rejected": -0.005337262060493231,
      "step": 10
    },
    {
      "epoch": 0.032,
      "grad_norm": 3.3894476890563965,
      "learning_rate": 1.6279069767441862e-06,
      "logits/chosen": -1.3111854791641235,
      "logits/rejected": -1.3214471340179443,
      "logps/chosen": -250.80496215820312,
      "logps/rejected": -299.97125244140625,
      "loss": 0.6834,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.0006973742274567485,
      "rewards/margins": 0.019777875393629074,
      "rewards/rejected": -0.020475249737501144,
      "step": 15
    },
    {
      "epoch": 0.042666666666666665,
      "grad_norm": 2.994020938873291,
      "learning_rate": 2.2093023255813954e-06,
      "logits/chosen": -1.3028817176818848,
      "logits/rejected": -1.1915266513824463,
      "logps/chosen": -255.93081665039062,
      "logps/rejected": -287.74224853515625,
      "loss": 0.6737,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.002559587825089693,
      "rewards/margins": 0.03989601880311966,
      "rewards/rejected": -0.042455606162548065,
      "step": 20
    },
    {
      "epoch": 0.05333333333333334,
      "grad_norm": 3.090261220932007,
      "learning_rate": 2.790697674418605e-06,
      "logits/chosen": -1.388347864151001,
      "logits/rejected": -1.328805923461914,
      "logps/chosen": -250.98336791992188,
      "logps/rejected": -322.8138122558594,
      "loss": 0.6594,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.002105569699779153,
      "rewards/margins": 0.07019330561161041,
      "rewards/rejected": -0.07229887694120407,
      "step": 25
    },
    {
      "epoch": 0.064,
      "grad_norm": 3.1172730922698975,
      "learning_rate": 3.372093023255814e-06,
      "logits/chosen": -1.4825924634933472,
      "logits/rejected": -1.3452062606811523,
      "logps/chosen": -253.34860229492188,
      "logps/rejected": -290.7360534667969,
      "loss": 0.6473,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.01585470139980316,
      "rewards/margins": 0.09762205183506012,
      "rewards/rejected": -0.11347675323486328,
      "step": 30
    },
    {
      "epoch": 0.07466666666666667,
      "grad_norm": 3.13596773147583,
      "learning_rate": 3.953488372093024e-06,
      "logits/chosen": -1.5015944242477417,
      "logits/rejected": -1.3286340236663818,
      "logps/chosen": -244.2925262451172,
      "logps/rejected": -298.30621337890625,
      "loss": 0.6208,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.01058957539498806,
      "rewards/margins": 0.15787732601165771,
      "rewards/rejected": -0.16846691071987152,
      "step": 35
    },
    {
      "epoch": 0.08533333333333333,
      "grad_norm": 2.9116740226745605,
      "learning_rate": 4.5348837209302326e-06,
      "logits/chosen": -1.4416661262512207,
      "logits/rejected": -1.2964617013931274,
      "logps/chosen": -243.5440216064453,
      "logps/rejected": -299.6397705078125,
      "loss": 0.5937,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.019765138626098633,
      "rewards/margins": 0.22198882699012756,
      "rewards/rejected": -0.2417539656162262,
      "step": 40
    },
    {
      "epoch": 0.096,
      "grad_norm": 3.0077366828918457,
      "learning_rate": 4.99999336897255e-06,
      "logits/chosen": -1.459515929222107,
      "logits/rejected": -1.3635624647140503,
      "logps/chosen": -238.49807739257812,
      "logps/rejected": -266.1226501464844,
      "loss": 0.5827,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.038423217833042145,
      "rewards/margins": 0.2630027234554291,
      "rewards/rejected": -0.3014259338378906,
      "step": 45
    },
    {
      "epoch": 0.10666666666666667,
      "grad_norm": 3.088587999343872,
      "learning_rate": 4.999761286705285e-06,
      "logits/chosen": -1.4864553213119507,
      "logits/rejected": -1.4152582883834839,
      "logps/chosen": -233.3901824951172,
      "logps/rejected": -296.8989562988281,
      "loss": 0.5212,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.0003606337995734066,
      "rewards/margins": 0.42451077699661255,
      "rewards/rejected": -0.42487144470214844,
      "step": 50
    },
    {
      "epoch": 0.11733333333333333,
      "grad_norm": 2.77523136138916,
      "learning_rate": 4.999197688241076e-06,
      "logits/chosen": -1.5394684076309204,
      "logits/rejected": -1.3223412036895752,
      "logps/chosen": -242.56759643554688,
      "logps/rejected": -281.634765625,
      "loss": 0.5129,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.07203203439712524,
      "rewards/margins": 0.5109471678733826,
      "rewards/rejected": -0.582979142665863,
      "step": 55
    },
    {
      "epoch": 0.128,
      "grad_norm": 2.9487898349761963,
      "learning_rate": 4.9983026483238685e-06,
      "logits/chosen": -1.4420719146728516,
      "logits/rejected": -1.2731995582580566,
      "logps/chosen": -224.287109375,
      "logps/rejected": -312.1568908691406,
      "loss": 0.461,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.2297721803188324,
      "rewards/margins": 0.6778525114059448,
      "rewards/rejected": -0.9076247215270996,
      "step": 60
    },
    {
      "epoch": 0.13866666666666666,
      "grad_norm": 2.45941162109375,
      "learning_rate": 4.997076285653087e-06,
      "logits/chosen": -1.5383515357971191,
      "logits/rejected": -1.4481399059295654,
      "logps/chosen": -274.0958557128906,
      "logps/rejected": -329.88018798828125,
      "loss": 0.4413,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.3417445719242096,
      "rewards/margins": 0.7747383713722229,
      "rewards/rejected": -1.1164829730987549,
      "step": 65
    },
    {
      "epoch": 0.14933333333333335,
      "grad_norm": 2.6682322025299072,
      "learning_rate": 4.995518762867897e-06,
      "logits/chosen": -1.5974746942520142,
      "logits/rejected": -1.3631844520568848,
      "logps/chosen": -217.626953125,
      "logps/rejected": -307.88116455078125,
      "loss": 0.3906,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.3570924401283264,
      "rewards/margins": 1.0183016061782837,
      "rewards/rejected": -1.3753941059112549,
      "step": 70
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.8534865379333496,
      "learning_rate": 4.993630286525634e-06,
      "logits/chosen": -1.6104965209960938,
      "logits/rejected": -1.4973475933074951,
      "logps/chosen": -241.7733154296875,
      "logps/rejected": -289.15728759765625,
      "loss": 0.3991,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.44355034828186035,
      "rewards/margins": 0.9931395649909973,
      "rewards/rejected": -1.4366897344589233,
      "step": 75
    },
    {
      "epoch": 0.17066666666666666,
      "grad_norm": 3.2399303913116455,
      "learning_rate": 4.99141110707441e-06,
      "logits/chosen": -1.5593732595443726,
      "logits/rejected": -1.3765180110931396,
      "logps/chosen": -244.89749145507812,
      "logps/rejected": -303.7383728027344,
      "loss": 0.3772,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.5426279306411743,
      "rewards/margins": 1.1284551620483398,
      "rewards/rejected": -1.6710832118988037,
      "step": 80
    },
    {
      "epoch": 0.18133333333333335,
      "grad_norm": 3.0792360305786133,
      "learning_rate": 4.9888615188199e-06,
      "logits/chosen": -1.6551406383514404,
      "logits/rejected": -1.4097787141799927,
      "logps/chosen": -245.085693359375,
      "logps/rejected": -293.0350036621094,
      "loss": 0.3911,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.5418190956115723,
      "rewards/margins": 1.2352501153945923,
      "rewards/rejected": -1.777069330215454,
      "step": 85
    },
    {
      "epoch": 0.192,
      "grad_norm": 2.767003059387207,
      "learning_rate": 4.9859818598863104e-06,
      "logits/chosen": -1.569866418838501,
      "logits/rejected": -1.4290711879730225,
      "logps/chosen": -244.1849822998047,
      "logps/rejected": -301.0859680175781,
      "loss": 0.3505,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.5658442974090576,
      "rewards/margins": 1.3717987537384033,
      "rewards/rejected": -1.9376428127288818,
      "step": 90
    },
    {
      "epoch": 0.20266666666666666,
      "grad_norm": 2.779585838317871,
      "learning_rate": 4.982772512171538e-06,
      "logits/chosen": -1.5476526021957397,
      "logits/rejected": -1.4095839262008667,
      "logps/chosen": -251.42025756835938,
      "logps/rejected": -317.84771728515625,
      "loss": 0.3451,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.6214931607246399,
      "rewards/margins": 1.5358760356903076,
      "rewards/rejected": -2.1573691368103027,
      "step": 95
    },
    {
      "epoch": 0.21333333333333335,
      "grad_norm": 2.5582046508789062,
      "learning_rate": 4.979233901296523e-06,
      "logits/chosen": -1.6822445392608643,
      "logits/rejected": -1.4913674592971802,
      "logps/chosen": -238.1501007080078,
      "logps/rejected": -317.0533142089844,
      "loss": 0.2784,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.4816685616970062,
      "rewards/margins": 1.7280359268188477,
      "rewards/rejected": -2.209704637527466,
      "step": 100
    },
    {
      "epoch": 0.224,
      "grad_norm": 2.492095470428467,
      "learning_rate": 4.975366496548804e-06,
      "logits/chosen": -1.58907151222229,
      "logits/rejected": -1.4344779253005981,
      "logps/chosen": -252.6648406982422,
      "logps/rejected": -321.97332763671875,
      "loss": 0.303,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.5509522557258606,
      "rewards/margins": 1.748511552810669,
      "rewards/rejected": -2.2994637489318848,
      "step": 105
    },
    {
      "epoch": 0.23466666666666666,
      "grad_norm": 3.3831777572631836,
      "learning_rate": 4.971170810820279e-06,
      "logits/chosen": -1.6393448114395142,
      "logits/rejected": -1.5223277807235718,
      "logps/chosen": -234.4925079345703,
      "logps/rejected": -295.9253845214844,
      "loss": 0.3034,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5105643272399902,
      "rewards/margins": 1.7626917362213135,
      "rewards/rejected": -2.2732558250427246,
      "step": 110
    },
    {
      "epoch": 0.24533333333333332,
      "grad_norm": 2.4063918590545654,
      "learning_rate": 4.96664740053919e-06,
      "logits/chosen": -1.647438406944275,
      "logits/rejected": -1.532233476638794,
      "logps/chosen": -236.671875,
      "logps/rejected": -318.50775146484375,
      "loss": 0.2571,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.8266940116882324,
      "rewards/margins": 2.115816593170166,
      "rewards/rejected": -2.9425106048583984,
      "step": 115
    },
    {
      "epoch": 0.256,
      "grad_norm": 2.415274143218994,
      "learning_rate": 4.961796865596328e-06,
      "logits/chosen": -1.7164440155029297,
      "logits/rejected": -1.5145854949951172,
      "logps/chosen": -237.99853515625,
      "logps/rejected": -316.65167236328125,
      "loss": 0.2933,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.7486313581466675,
      "rewards/margins": 2.1729655265808105,
      "rewards/rejected": -2.9215967655181885,
      "step": 120
    },
    {
      "epoch": 0.26666666666666666,
      "grad_norm": 1.8586480617523193,
      "learning_rate": 4.956619849265473e-06,
      "logits/chosen": -1.7343181371688843,
      "logits/rejected": -1.5120888948440552,
      "logps/chosen": -241.61019897460938,
      "logps/rejected": -336.1865234375,
      "loss": 0.2291,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.631386399269104,
      "rewards/margins": 2.4859204292297363,
      "rewards/rejected": -3.117306709289551,
      "step": 125
    },
    {
      "epoch": 0.2773333333333333,
      "grad_norm": 3.7649638652801514,
      "learning_rate": 4.951117038118091e-06,
      "logits/chosen": -1.7571346759796143,
      "logits/rejected": -1.582757592201233,
      "logps/chosen": -240.4564971923828,
      "logps/rejected": -326.0672302246094,
      "loss": 0.2523,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.7324283719062805,
      "rewards/margins": 2.460465908050537,
      "rewards/rejected": -3.192894458770752,
      "step": 130
    },
    {
      "epoch": 0.288,
      "grad_norm": 1.8477269411087036,
      "learning_rate": 4.945289161932273e-06,
      "logits/chosen": -1.7058353424072266,
      "logits/rejected": -1.5484061241149902,
      "logps/chosen": -234.3208465576172,
      "logps/rejected": -316.4405822753906,
      "loss": 0.2389,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.644821047782898,
      "rewards/margins": 2.5761804580688477,
      "rewards/rejected": -3.221001386642456,
      "step": 135
    },
    {
      "epoch": 0.2986666666666667,
      "grad_norm": 2.963148832321167,
      "learning_rate": 4.93913699359596e-06,
      "logits/chosen": -1.7283592224121094,
      "logits/rejected": -1.5544356107711792,
      "logps/chosen": -235.847900390625,
      "logps/rejected": -317.0403747558594,
      "loss": 0.2739,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.6758811473846436,
      "rewards/margins": 2.5437233448028564,
      "rewards/rejected": -3.219604015350342,
      "step": 140
    },
    {
      "epoch": 0.30933333333333335,
      "grad_norm": 3.0582330226898193,
      "learning_rate": 4.9326613490044364e-06,
      "logits/chosen": -1.6446880102157593,
      "logits/rejected": -1.4448646306991577,
      "logps/chosen": -258.02813720703125,
      "logps/rejected": -310.2739562988281,
      "loss": 0.2851,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.7733806371688843,
      "rewards/margins": 2.419772148132324,
      "rewards/rejected": -3.19315242767334,
      "step": 145
    },
    {
      "epoch": 0.32,
      "grad_norm": 2.466308832168579,
      "learning_rate": 4.925863086952132e-06,
      "logits/chosen": -1.6388626098632812,
      "logits/rejected": -1.455078363418579,
      "logps/chosen": -245.62631225585938,
      "logps/rejected": -345.78521728515625,
      "loss": 0.2196,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.46061477065086365,
      "rewards/margins": 2.8316876888275146,
      "rewards/rejected": -3.292302370071411,
      "step": 150
    },
    {
      "epoch": 0.33066666666666666,
      "grad_norm": 2.5503790378570557,
      "learning_rate": 4.918743109018727e-06,
      "logits/chosen": -1.7483993768692017,
      "logits/rejected": -1.5743939876556396,
      "logps/chosen": -231.1587371826172,
      "logps/rejected": -314.0552673339844,
      "loss": 0.2273,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.6226693987846375,
      "rewards/margins": 2.552253246307373,
      "rewards/rejected": -3.174922466278076,
      "step": 155
    },
    {
      "epoch": 0.3413333333333333,
      "grad_norm": 2.485719680786133,
      "learning_rate": 4.911302359449585e-06,
      "logits/chosen": -1.7262346744537354,
      "logits/rejected": -1.5447709560394287,
      "logps/chosen": -254.2543487548828,
      "logps/rejected": -323.9798278808594,
      "loss": 0.2383,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.7263298630714417,
      "rewards/margins": 2.8138160705566406,
      "rewards/rejected": -3.5401458740234375,
      "step": 160
    },
    {
      "epoch": 0.352,
      "grad_norm": 3.7984070777893066,
      "learning_rate": 4.9035418250305314e-06,
      "logits/chosen": -1.656524896621704,
      "logits/rejected": -1.4639983177185059,
      "logps/chosen": -234.14254760742188,
      "logps/rejected": -314.20318603515625,
      "loss": 0.206,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.35529714822769165,
      "rewards/margins": 2.8109519481658936,
      "rewards/rejected": -3.1662492752075195,
      "step": 165
    },
    {
      "epoch": 0.3626666666666667,
      "grad_norm": 2.783888339996338,
      "learning_rate": 4.895462534956981e-06,
      "logits/chosen": -1.6165010929107666,
      "logits/rejected": -1.5533854961395264,
      "logps/chosen": -242.182861328125,
      "logps/rejected": -333.15863037109375,
      "loss": 0.2425,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.5480275750160217,
      "rewards/margins": 2.9635632038116455,
      "rewards/rejected": -3.5115904808044434,
      "step": 170
    },
    {
      "epoch": 0.37333333333333335,
      "grad_norm": 6.773368835449219,
      "learning_rate": 4.887065560697449e-06,
      "logits/chosen": -1.6875731945037842,
      "logits/rejected": -1.507009744644165,
      "logps/chosen": -254.5814971923828,
      "logps/rejected": -332.4895935058594,
      "loss": 0.2905,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5142589807510376,
      "rewards/margins": 2.550226926803589,
      "rewards/rejected": -3.064485788345337,
      "step": 175
    },
    {
      "epoch": 0.384,
      "grad_norm": 2.7218058109283447,
      "learning_rate": 4.878352015851459e-06,
      "logits/chosen": -1.6414203643798828,
      "logits/rejected": -1.4779311418533325,
      "logps/chosen": -270.7060852050781,
      "logps/rejected": -332.2988586425781,
      "loss": 0.1879,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.36767151951789856,
      "rewards/margins": 3.0560736656188965,
      "rewards/rejected": -3.4237449169158936,
      "step": 180
    },
    {
      "epoch": 0.39466666666666667,
      "grad_norm": 2.207134962081909,
      "learning_rate": 4.869323056001852e-06,
      "logits/chosen": -1.675317406654358,
      "logits/rejected": -1.4597893953323364,
      "logps/chosen": -240.0108642578125,
      "logps/rejected": -306.22418212890625,
      "loss": 0.197,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.2930644154548645,
      "rewards/margins": 2.977311372756958,
      "rewards/rejected": -3.2703757286071777,
      "step": 185
    },
    {
      "epoch": 0.4053333333333333,
      "grad_norm": 2.3709778785705566,
      "learning_rate": 4.859979878561536e-06,
      "logits/chosen": -1.663538932800293,
      "logits/rejected": -1.516006588935852,
      "logps/chosen": -222.03439331054688,
      "logps/rejected": -327.6264343261719,
      "loss": 0.1997,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.18494608998298645,
      "rewards/margins": 3.1293203830718994,
      "rewards/rejected": -3.3142666816711426,
      "step": 190
    },
    {
      "epoch": 0.416,
      "grad_norm": 1.9043569564819336,
      "learning_rate": 4.850323722614685e-06,
      "logits/chosen": -1.7305800914764404,
      "logits/rejected": -1.612801194190979,
      "logps/chosen": -245.2909698486328,
      "logps/rejected": -351.641357421875,
      "loss": 0.1775,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5533159971237183,
      "rewards/margins": 3.2820746898651123,
      "rewards/rejected": -3.835390567779541,
      "step": 195
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 2.5433902740478516,
      "learning_rate": 4.840355868752416e-06,
      "logits/chosen": -1.571982741355896,
      "logits/rejected": -1.4637776613235474,
      "logps/chosen": -299.1954345703125,
      "logps/rejected": -337.9162292480469,
      "loss": 0.1962,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.3970106244087219,
      "rewards/margins": 3.3511834144592285,
      "rewards/rejected": -3.7481937408447266,
      "step": 200
    },
    {
      "epoch": 0.43733333333333335,
      "grad_norm": 2.6432783603668213,
      "learning_rate": 4.830077638902955e-06,
      "logits/chosen": -1.7969417572021484,
      "logits/rejected": -1.6737842559814453,
      "logps/chosen": -228.6658477783203,
      "logps/rejected": -323.2589416503906,
      "loss": 0.1855,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.45908913016319275,
      "rewards/margins": 3.377959728240967,
      "rewards/rejected": -3.8370490074157715,
      "step": 205
    },
    {
      "epoch": 0.448,
      "grad_norm": 2.609254837036133,
      "learning_rate": 4.8194903961563255e-06,
      "logits/chosen": -1.8882259130477905,
      "logits/rejected": -1.6982256174087524,
      "logps/chosen": -247.7525634765625,
      "logps/rejected": -331.72723388671875,
      "loss": 0.1426,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.49562868475914,
      "rewards/margins": 3.640789747238159,
      "rewards/rejected": -4.136418342590332,
      "step": 210
    },
    {
      "epoch": 0.45866666666666667,
      "grad_norm": 2.308016300201416,
      "learning_rate": 4.808595544583578e-06,
      "logits/chosen": -1.772505521774292,
      "logits/rejected": -1.6354877948760986,
      "logps/chosen": -234.55142211914062,
      "logps/rejected": -352.1875305175781,
      "loss": 0.1857,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.5159016251564026,
      "rewards/margins": 3.877976894378662,
      "rewards/rejected": -4.393878936767578,
      "step": 215
    },
    {
      "epoch": 0.4693333333333333,
      "grad_norm": 1.5003650188446045,
      "learning_rate": 4.797394529050577e-06,
      "logits/chosen": -1.6615060567855835,
      "logits/rejected": -1.5364525318145752,
      "logps/chosen": -272.47418212890625,
      "logps/rejected": -360.6487731933594,
      "loss": 0.1639,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.691543698310852,
      "rewards/margins": 3.9234442710876465,
      "rewards/rejected": -4.614988327026367,
      "step": 220
    },
    {
      "epoch": 0.48,
      "grad_norm": 2.611100435256958,
      "learning_rate": 4.785888835026393e-06,
      "logits/chosen": -1.745161771774292,
      "logits/rejected": -1.5977957248687744,
      "logps/chosen": -260.530029296875,
      "logps/rejected": -306.3468017578125,
      "loss": 0.2476,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.3127722144126892,
      "rewards/margins": 3.3172645568847656,
      "rewards/rejected": -3.6300368309020996,
      "step": 225
    },
    {
      "epoch": 0.49066666666666664,
      "grad_norm": 1.957841157913208,
      "learning_rate": 4.7740799883862966e-06,
      "logits/chosen": -1.8661781549453735,
      "logits/rejected": -1.6978685855865479,
      "logps/chosen": -233.84231567382812,
      "logps/rejected": -317.6667785644531,
      "loss": 0.1736,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.3665715754032135,
      "rewards/margins": 3.6203083992004395,
      "rewards/rejected": -3.986879825592041,
      "step": 230
    },
    {
      "epoch": 0.5013333333333333,
      "grad_norm": 1.5022276639938354,
      "learning_rate": 4.761969555209394e-06,
      "logits/chosen": -1.778978943824768,
      "logits/rejected": -1.6453516483306885,
      "logps/chosen": -249.01815795898438,
      "logps/rejected": -362.90460205078125,
      "loss": 0.1509,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8386176228523254,
      "rewards/margins": 4.385830402374268,
      "rewards/rejected": -5.224447727203369,
      "step": 235
    },
    {
      "epoch": 0.512,
      "grad_norm": 2.0542163848876953,
      "learning_rate": 4.7495591415709435e-06,
      "logits/chosen": -1.8296219110488892,
      "logits/rejected": -1.6568787097930908,
      "logps/chosen": -228.88412475585938,
      "logps/rejected": -314.2283630371094,
      "loss": 0.2659,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.5388306975364685,
      "rewards/margins": 3.918764591217041,
      "rewards/rejected": -4.457594871520996,
      "step": 240
    },
    {
      "epoch": 0.5226666666666666,
      "grad_norm": 3.47914719581604,
      "learning_rate": 4.736850393329353e-06,
      "logits/chosen": -1.779706358909607,
      "logits/rejected": -1.5548733472824097,
      "logps/chosen": -252.67044067382812,
      "logps/rejected": -338.41217041015625,
      "loss": 0.2123,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.5982939600944519,
      "rewards/margins": 3.9195141792297363,
      "rewards/rejected": -4.517808437347412,
      "step": 245
    },
    {
      "epoch": 0.5333333333333333,
      "grad_norm": 3.8012492656707764,
      "learning_rate": 4.7238449959079095e-06,
      "logits/chosen": -1.8602869510650635,
      "logits/rejected": -1.6974502801895142,
      "logps/chosen": -241.8807830810547,
      "logps/rejected": -328.1159362792969,
      "loss": 0.2123,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.4912596344947815,
      "rewards/margins": 4.016532897949219,
      "rewards/rejected": -4.5077924728393555,
      "step": 250
    },
    {
      "epoch": 0.544,
      "grad_norm": 1.548546552658081,
      "learning_rate": 4.710544674071259e-06,
      "logits/chosen": -1.8098742961883545,
      "logits/rejected": -1.6698535680770874,
      "logps/chosen": -236.11611938476562,
      "logps/rejected": -343.9376220703125,
      "loss": 0.1609,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.4053850769996643,
      "rewards/margins": 4.2683796882629395,
      "rewards/rejected": -4.673764228820801,
      "step": 255
    },
    {
      "epoch": 0.5546666666666666,
      "grad_norm": 3.3183388710021973,
      "learning_rate": 4.696951191696673e-06,
      "logits/chosen": -1.8579626083374023,
      "logits/rejected": -1.7205890417099,
      "logps/chosen": -230.24563598632812,
      "logps/rejected": -340.3718566894531,
      "loss": 0.1665,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.5163661241531372,
      "rewards/margins": 4.36639928817749,
      "rewards/rejected": -4.882765769958496,
      "step": 260
    },
    {
      "epoch": 0.5653333333333334,
      "grad_norm": 2.8505144119262695,
      "learning_rate": 4.6830663515401196e-06,
      "logits/chosen": -1.7807422876358032,
      "logits/rejected": -1.5978543758392334,
      "logps/chosen": -237.2873992919922,
      "logps/rejected": -336.2684326171875,
      "loss": 0.2242,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.35498833656311035,
      "rewards/margins": 3.7952804565429688,
      "rewards/rejected": -4.150269508361816,
      "step": 265
    },
    {
      "epoch": 0.576,
      "grad_norm": 2.377626895904541,
      "learning_rate": 4.6688919949971855e-06,
      "logits/chosen": -1.7819585800170898,
      "logits/rejected": -1.6391322612762451,
      "logps/chosen": -251.92874145507812,
      "logps/rejected": -349.6017761230469,
      "loss": 0.2408,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.44162049889564514,
      "rewards/margins": 4.0541276931762695,
      "rewards/rejected": -4.495748043060303,
      "step": 270
    },
    {
      "epoch": 0.5866666666666667,
      "grad_norm": 1.5478827953338623,
      "learning_rate": 4.654430001858874e-06,
      "logits/chosen": -1.7001129388809204,
      "logits/rejected": -1.5338956117630005,
      "logps/chosen": -251.0334014892578,
      "logps/rejected": -333.11163330078125,
      "loss": 0.1582,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.034758586436510086,
      "rewards/margins": 4.018908500671387,
      "rewards/rejected": -4.053666591644287,
      "step": 275
    },
    {
      "epoch": 0.5973333333333334,
      "grad_norm": 3.8507468700408936,
      "learning_rate": 4.6396822900623075e-06,
      "logits/chosen": -1.749701738357544,
      "logits/rejected": -1.6001421213150024,
      "logps/chosen": -250.6236114501953,
      "logps/rejected": -327.2859191894531,
      "loss": 0.1665,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.2926911413669586,
      "rewards/margins": 4.104784965515137,
      "rewards/rejected": -4.3974761962890625,
      "step": 280
    },
    {
      "epoch": 0.608,
      "grad_norm": 3.485337972640991,
      "learning_rate": 4.624650815436368e-06,
      "logits/chosen": -1.7875235080718994,
      "logits/rejected": -1.663147211074829,
      "logps/chosen": -275.81329345703125,
      "logps/rejected": -355.75921630859375,
      "loss": 0.1732,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.2872824966907501,
      "rewards/margins": 4.033877372741699,
      "rewards/rejected": -4.321159362792969,
      "step": 285
    },
    {
      "epoch": 0.6186666666666667,
      "grad_norm": 3.0503721237182617,
      "learning_rate": 4.609337571442324e-06,
      "logits/chosen": -1.8553688526153564,
      "logits/rejected": -1.7102324962615967,
      "logps/chosen": -227.72128295898438,
      "logps/rejected": -343.08148193359375,
      "loss": 0.1448,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3748090863227844,
      "rewards/margins": 4.319515228271484,
      "rewards/rejected": -4.694324970245361,
      "step": 290
    },
    {
      "epoch": 0.6293333333333333,
      "grad_norm": 1.912618637084961,
      "learning_rate": 4.5937445889094535e-06,
      "logits/chosen": -1.7128536701202393,
      "logits/rejected": -1.6339744329452515,
      "logps/chosen": -271.19769287109375,
      "logps/rejected": -340.0036926269531,
      "loss": 0.1723,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.5582373738288879,
      "rewards/margins": 4.0624799728393555,
      "rewards/rejected": -4.6207170486450195,
      "step": 295
    },
    {
      "epoch": 0.64,
      "grad_norm": 1.1806559562683105,
      "learning_rate": 4.577873935765722e-06,
      "logits/chosen": -1.789751648902893,
      "logits/rejected": -1.6350936889648438,
      "logps/chosen": -255.74234008789062,
      "logps/rejected": -362.6470642089844,
      "loss": 0.1218,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.6933242082595825,
      "rewards/margins": 4.8959221839904785,
      "rewards/rejected": -5.5892462730407715,
      "step": 300
    },
    {
      "epoch": 0.6506666666666666,
      "grad_norm": 2.678286075592041,
      "learning_rate": 4.561727716763532e-06,
      "logits/chosen": -1.8357679843902588,
      "logits/rejected": -1.708337426185608,
      "logps/chosen": -252.95004272460938,
      "logps/rejected": -354.3417053222656,
      "loss": 0.1762,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.6951386332511902,
      "rewards/margins": 4.543791770935059,
      "rewards/rejected": -5.2389302253723145,
      "step": 305
    },
    {
      "epoch": 0.6613333333333333,
      "grad_norm": 1.354732871055603,
      "learning_rate": 4.545308073200591e-06,
      "logits/chosen": -1.6892789602279663,
      "logits/rejected": -1.622051477432251,
      "logps/chosen": -254.616943359375,
      "logps/rejected": -349.6521301269531,
      "loss": 0.1573,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.8155484199523926,
      "rewards/margins": 4.775422096252441,
      "rewards/rejected": -5.590970039367676,
      "step": 310
    },
    {
      "epoch": 0.672,
      "grad_norm": 2.0910332202911377,
      "learning_rate": 4.528617182635943e-06,
      "logits/chosen": -1.925897240638733,
      "logits/rejected": -1.7414014339447021,
      "logps/chosen": -253.86239624023438,
      "logps/rejected": -356.8898620605469,
      "loss": 0.2282,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.9393094182014465,
      "rewards/margins": 4.897877216339111,
      "rewards/rejected": -5.83718729019165,
      "step": 315
    },
    {
      "epoch": 0.6826666666666666,
      "grad_norm": 4.3398566246032715,
      "learning_rate": 4.511657258601171e-06,
      "logits/chosen": -1.7808582782745361,
      "logits/rejected": -1.584148645401001,
      "logps/chosen": -245.6190643310547,
      "logps/rejected": -326.17950439453125,
      "loss": 0.1912,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.752561092376709,
      "rewards/margins": 4.876051425933838,
      "rewards/rejected": -5.628612518310547,
      "step": 320
    },
    {
      "epoch": 0.6933333333333334,
      "grad_norm": 1.8720273971557617,
      "learning_rate": 4.494430550306848e-06,
      "logits/chosen": -1.7456798553466797,
      "logits/rejected": -1.6327714920043945,
      "logps/chosen": -251.36740112304688,
      "logps/rejected": -327.3114013671875,
      "loss": 0.1792,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.30995193123817444,
      "rewards/margins": 4.125004291534424,
      "rewards/rejected": -4.434955596923828,
      "step": 325
    },
    {
      "epoch": 0.704,
      "grad_norm": 2.9589478969573975,
      "learning_rate": 4.476939342344246e-06,
      "logits/chosen": -1.8625526428222656,
      "logits/rejected": -1.7193504571914673,
      "logps/chosen": -234.44827270507812,
      "logps/rejected": -323.15924072265625,
      "loss": 0.1882,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.8839484453201294,
      "rewards/margins": 4.334848403930664,
      "rewards/rejected": -5.218796730041504,
      "step": 330
    },
    {
      "epoch": 0.7146666666666667,
      "grad_norm": 2.8924190998077393,
      "learning_rate": 4.45918595438236e-06,
      "logits/chosen": -1.7802038192749023,
      "logits/rejected": -1.6963586807250977,
      "logps/chosen": -247.6739044189453,
      "logps/rejected": -334.2060546875,
      "loss": 0.1904,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.8888412714004517,
      "rewards/margins": 4.769382953643799,
      "rewards/rejected": -5.658223628997803,
      "step": 335
    },
    {
      "epoch": 0.7253333333333334,
      "grad_norm": 2.6374990940093994,
      "learning_rate": 4.441172740860267e-06,
      "logits/chosen": -1.8051414489746094,
      "logits/rejected": -1.7025120258331299,
      "logps/chosen": -236.43789672851562,
      "logps/rejected": -326.9657287597656,
      "loss": 0.1302,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.7156514525413513,
      "rewards/margins": 4.83019495010376,
      "rewards/rejected": -5.545845985412598,
      "step": 340
    },
    {
      "epoch": 0.736,
      "grad_norm": 1.9260821342468262,
      "learning_rate": 4.422902090674886e-06,
      "logits/chosen": -1.7234573364257812,
      "logits/rejected": -1.61138916015625,
      "logps/chosen": -252.62173461914062,
      "logps/rejected": -356.4739685058594,
      "loss": 0.1349,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.8339887857437134,
      "rewards/margins": 4.9775390625,
      "rewards/rejected": -5.811528205871582,
      "step": 345
    },
    {
      "epoch": 0.7466666666666667,
      "grad_norm": 4.608032703399658,
      "learning_rate": 4.4043764268641705e-06,
      "logits/chosen": -1.8549143075942993,
      "logits/rejected": -1.7140591144561768,
      "logps/chosen": -234.22421264648438,
      "logps/rejected": -318.148681640625,
      "loss": 0.1986,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.8342403173446655,
      "rewards/margins": 4.861636161804199,
      "rewards/rejected": -5.6958770751953125,
      "step": 350
    },
    {
      "epoch": 0.7573333333333333,
      "grad_norm": 3.3214540481567383,
      "learning_rate": 4.385598206285758e-06,
      "logits/chosen": -1.80758535861969,
      "logits/rejected": -1.6905763149261475,
      "logps/chosen": -287.7499084472656,
      "logps/rejected": -365.01385498046875,
      "loss": 0.1779,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.3829190731048584,
      "rewards/margins": 4.985753536224365,
      "rewards/rejected": -6.368672847747803,
      "step": 355
    },
    {
      "epoch": 0.768,
      "grad_norm": 1.858853816986084,
      "learning_rate": 4.3665699192911495e-06,
      "logits/chosen": -1.7372462749481201,
      "logits/rejected": -1.6430860757827759,
      "logps/chosen": -241.73046875,
      "logps/rejected": -353.0447692871094,
      "loss": 0.1826,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.9526206254959106,
      "rewards/margins": 5.21450662612915,
      "rewards/rejected": -6.16712760925293,
      "step": 360
    },
    {
      "epoch": 0.7786666666666666,
      "grad_norm": 6.181103706359863,
      "learning_rate": 4.347294089395438e-06,
      "logits/chosen": -1.800432562828064,
      "logits/rejected": -1.683996558189392,
      "logps/chosen": -250.1630401611328,
      "logps/rejected": -342.5280456542969,
      "loss": 0.1389,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.913178563117981,
      "rewards/margins": 5.121844291687012,
      "rewards/rejected": -6.035023212432861,
      "step": 365
    },
    {
      "epoch": 0.7893333333333333,
      "grad_norm": 4.438066005706787,
      "learning_rate": 4.327773272942647e-06,
      "logits/chosen": -1.8669296503067017,
      "logits/rejected": -1.7096134424209595,
      "logps/chosen": -239.025146484375,
      "logps/rejected": -333.13409423828125,
      "loss": 0.2367,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.7618646621704102,
      "rewards/margins": 4.629305839538574,
      "rewards/rejected": -5.391171455383301,
      "step": 370
    },
    {
      "epoch": 0.8,
      "grad_norm": 3.4000484943389893,
      "learning_rate": 4.308010058766703e-06,
      "logits/chosen": -1.8593333959579468,
      "logits/rejected": -1.6820989847183228,
      "logps/chosen": -249.3157501220703,
      "logps/rejected": -353.73687744140625,
      "loss": 0.1273,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.8536429405212402,
      "rewards/margins": 5.058758735656738,
      "rewards/rejected": -5.9124016761779785,
      "step": 375
    },
    {
      "epoch": 0.8106666666666666,
      "grad_norm": 2.9082837104797363,
      "learning_rate": 4.2880070678481135e-06,
      "logits/chosen": -1.8044195175170898,
      "logits/rejected": -1.719753623008728,
      "logps/chosen": -253.3800048828125,
      "logps/rejected": -370.4222717285156,
      "loss": 0.1795,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.109043002128601,
      "rewards/margins": 4.930617809295654,
      "rewards/rejected": -6.039660930633545,
      "step": 380
    },
    {
      "epoch": 0.8213333333333334,
      "grad_norm": 3.509117841720581,
      "learning_rate": 4.267766952966369e-06,
      "logits/chosen": -1.7582893371582031,
      "logits/rejected": -1.6735187768936157,
      "logps/chosen": -248.4905242919922,
      "logps/rejected": -347.24755859375,
      "loss": 0.1581,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.7402669191360474,
      "rewards/margins": 5.071152687072754,
      "rewards/rejected": -5.811419486999512,
      "step": 385
    },
    {
      "epoch": 0.832,
      "grad_norm": 1.737369179725647,
      "learning_rate": 4.24729239834814e-06,
      "logits/chosen": -1.8210585117340088,
      "logits/rejected": -1.6675134897232056,
      "logps/chosen": -260.0174865722656,
      "logps/rejected": -339.507568359375,
      "loss": 0.191,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.807237446308136,
      "rewards/margins": 4.788012504577637,
      "rewards/rejected": -5.595250129699707,
      "step": 390
    },
    {
      "epoch": 0.8426666666666667,
      "grad_norm": 2.075969934463501,
      "learning_rate": 4.226586119311291e-06,
      "logits/chosen": -1.7811262607574463,
      "logits/rejected": -1.7235519886016846,
      "logps/chosen": -269.1385498046875,
      "logps/rejected": -357.7069091796875,
      "loss": 0.1703,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.7997838258743286,
      "rewards/margins": 4.9125871658325195,
      "rewards/rejected": -5.712370872497559,
      "step": 395
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 3.3996775150299072,
      "learning_rate": 4.205650861904782e-06,
      "logits/chosen": -1.8044359683990479,
      "logits/rejected": -1.686042070388794,
      "logps/chosen": -236.1574249267578,
      "logps/rejected": -339.8066711425781,
      "loss": 0.1439,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.5469352006912231,
      "rewards/margins": 4.560192108154297,
      "rewards/rejected": -5.107127666473389,
      "step": 400
    },
    {
      "epoch": 0.864,
      "grad_norm": 3.2350564002990723,
      "learning_rate": 4.184489402544486e-06,
      "logits/chosen": -1.7984288930892944,
      "logits/rejected": -1.7034324407577515,
      "logps/chosen": -244.17660522460938,
      "logps/rejected": -326.0493469238281,
      "loss": 0.1875,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.790968656539917,
      "rewards/margins": 4.585756778717041,
      "rewards/rejected": -5.376725196838379,
      "step": 405
    },
    {
      "epoch": 0.8746666666666667,
      "grad_norm": 2.342085599899292,
      "learning_rate": 4.1631045476449865e-06,
      "logits/chosen": -1.804836630821228,
      "logits/rejected": -1.6348615884780884,
      "logps/chosen": -240.3076934814453,
      "logps/rejected": -318.75494384765625,
      "loss": 0.1928,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.7394973039627075,
      "rewards/margins": 4.6910576820373535,
      "rewards/rejected": -5.430554389953613,
      "step": 410
    },
    {
      "epoch": 0.8853333333333333,
      "grad_norm": 3.078958511352539,
      "learning_rate": 4.141499133247391e-06,
      "logits/chosen": -1.7097524404525757,
      "logits/rejected": -1.6636264324188232,
      "logps/chosen": -234.5639190673828,
      "logps/rejected": -351.9319152832031,
      "loss": 0.1463,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.5261866450309753,
      "rewards/margins": 5.036253452301025,
      "rewards/rejected": -5.562439441680908,
      "step": 415
    },
    {
      "epoch": 0.896,
      "grad_norm": 2.330395221710205,
      "learning_rate": 4.119676024643217e-06,
      "logits/chosen": -1.6960022449493408,
      "logits/rejected": -1.5892505645751953,
      "logps/chosen": -237.07608032226562,
      "logps/rejected": -319.4405212402344,
      "loss": 0.1522,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.46109694242477417,
      "rewards/margins": 4.772611141204834,
      "rewards/rejected": -5.233708381652832,
      "step": 420
    },
    {
      "epoch": 0.9066666666666666,
      "grad_norm": 2.2593789100646973,
      "learning_rate": 4.0976381159944055e-06,
      "logits/chosen": -1.7869155406951904,
      "logits/rejected": -1.606461763381958,
      "logps/chosen": -273.7130432128906,
      "logps/rejected": -362.55267333984375,
      "loss": 0.1593,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.8516896963119507,
      "rewards/margins": 5.3220696449279785,
      "rewards/rejected": -6.173758506774902,
      "step": 425
    },
    {
      "epoch": 0.9173333333333333,
      "grad_norm": 3.2557544708251953,
      "learning_rate": 4.07538832994949e-06,
      "logits/chosen": -1.758851408958435,
      "logits/rejected": -1.6003106832504272,
      "logps/chosen": -267.98980712890625,
      "logps/rejected": -371.2702941894531,
      "loss": 0.1952,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.1154661178588867,
      "rewards/margins": 5.60113525390625,
      "rewards/rejected": -6.716601371765137,
      "step": 430
    },
    {
      "epoch": 0.928,
      "grad_norm": 3.192758321762085,
      "learning_rate": 4.0529296172560055e-06,
      "logits/chosen": -1.8879740238189697,
      "logits/rejected": -1.6979936361312866,
      "logps/chosen": -250.65975952148438,
      "logps/rejected": -344.8387451171875,
      "loss": 0.1664,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.0759499073028564,
      "rewards/margins": 5.6617536544799805,
      "rewards/rejected": -6.737703800201416,
      "step": 435
    },
    {
      "epoch": 0.9386666666666666,
      "grad_norm": 2.523839235305786,
      "learning_rate": 4.030264956369158e-06,
      "logits/chosen": -1.7966762781143188,
      "logits/rejected": -1.7065531015396118,
      "logps/chosen": -245.68002319335938,
      "logps/rejected": -358.82720947265625,
      "loss": 0.1692,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.0516901016235352,
      "rewards/margins": 5.241422176361084,
      "rewards/rejected": -6.293112277984619,
      "step": 440
    },
    {
      "epoch": 0.9493333333333334,
      "grad_norm": 3.1646761894226074,
      "learning_rate": 4.0073973530568274e-06,
      "logits/chosen": -1.843031644821167,
      "logits/rejected": -1.6471502780914307,
      "logps/chosen": -257.5235900878906,
      "logps/rejected": -372.1407165527344,
      "loss": 0.1023,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.7290087938308716,
      "rewards/margins": 6.054869651794434,
      "rewards/rejected": -6.783879280090332,
      "step": 445
    },
    {
      "epoch": 0.96,
      "grad_norm": 4.48504638671875,
      "learning_rate": 3.9843298400009435e-06,
      "logits/chosen": -1.7822949886322021,
      "logits/rejected": -1.6918121576309204,
      "logps/chosen": -251.33145141601562,
      "logps/rejected": -366.915283203125,
      "loss": 0.1788,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.2320232391357422,
      "rewards/margins": 5.393434047698975,
      "rewards/rejected": -6.625456809997559,
      "step": 450
    },
    {
      "epoch": 0.9706666666666667,
      "grad_norm": 2.657050848007202,
      "learning_rate": 3.961065476395296e-06,
      "logits/chosen": -1.7469505071640015,
      "logits/rejected": -1.6122243404388428,
      "logps/chosen": -268.603759765625,
      "logps/rejected": -365.6134338378906,
      "loss": 0.1598,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.2172812223434448,
      "rewards/margins": 5.346413612365723,
      "rewards/rejected": -6.563694953918457,
      "step": 455
    },
    {
      "epoch": 0.9813333333333333,
      "grad_norm": 2.8152854442596436,
      "learning_rate": 3.937607347539823e-06,
      "logits/chosen": -1.7389802932739258,
      "logits/rejected": -1.6374263763427734,
      "logps/chosen": -250.3136749267578,
      "logps/rejected": -358.8951110839844,
      "loss": 0.1151,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.1840031147003174,
      "rewards/margins": 6.090756416320801,
      "rewards/rejected": -7.274759769439697,
      "step": 460
    },
    {
      "epoch": 0.992,
      "grad_norm": 1.3589340448379517,
      "learning_rate": 3.913958564431447e-06,
      "logits/chosen": -1.9092724323272705,
      "logits/rejected": -1.728168249130249,
      "logps/chosen": -288.52215576171875,
      "logps/rejected": -378.2078552246094,
      "loss": 0.0895,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.3161489963531494,
      "rewards/margins": 6.398454666137695,
      "rewards/rejected": -7.714603424072266,
      "step": 465
    },
    {
      "epoch": 1.0021333333333333,
      "grad_norm": 3.265164375305176,
      "learning_rate": 3.89012226335149e-06,
      "logits/chosen": -1.7122092247009277,
      "logits/rejected": -1.6343815326690674,
      "logps/chosen": -256.2262878417969,
      "logps/rejected": -383.70245361328125,
      "loss": 0.1672,
      "rewards/accuracies": 0.9276315569877625,
      "rewards/chosen": -1.3532406091690063,
      "rewards/margins": 5.975292205810547,
      "rewards/rejected": -7.328533172607422,
      "step": 470
    },
    {
      "epoch": 1.0128,
      "grad_norm": 3.8232693672180176,
      "learning_rate": 3.86610160544975e-06,
      "logits/chosen": -1.8605287075042725,
      "logits/rejected": -1.6789414882659912,
      "logps/chosen": -258.9883728027344,
      "logps/rejected": -362.35162353515625,
      "loss": 0.0859,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.2631183862686157,
      "rewards/margins": 6.069647789001465,
      "rewards/rejected": -7.332766056060791,
      "step": 475
    },
    {
      "epoch": 1.0234666666666667,
      "grad_norm": 2.178936719894409,
      "learning_rate": 3.841899776325267e-06,
      "logits/chosen": -1.6484073400497437,
      "logits/rejected": -1.539050817489624,
      "logps/chosen": -269.6177978515625,
      "logps/rejected": -361.5688171386719,
      "loss": 0.1179,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.2398256063461304,
      "rewards/margins": 6.038259029388428,
      "rewards/rejected": -7.278084754943848,
      "step": 480
    },
    {
      "epoch": 1.0341333333333333,
      "grad_norm": 0.980421245098114,
      "learning_rate": 3.817519985603855e-06,
      "logits/chosen": -1.905623197555542,
      "logits/rejected": -1.7547218799591064,
      "logps/chosen": -250.7723846435547,
      "logps/rejected": -385.75262451171875,
      "loss": 0.0857,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.6823866367340088,
      "rewards/margins": 6.899411201477051,
      "rewards/rejected": -8.58179759979248,
      "step": 485
    },
    {
      "epoch": 1.0448,
      "grad_norm": 1.7659993171691895,
      "learning_rate": 3.792965466512437e-06,
      "logits/chosen": -1.7362083196640015,
      "logits/rejected": -1.6089004278182983,
      "logps/chosen": -272.89288330078125,
      "logps/rejected": -345.1723327636719,
      "loss": 0.1426,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.6883974075317383,
      "rewards/margins": 5.6151299476623535,
      "rewards/rejected": -7.303526878356934,
      "step": 490
    },
    {
      "epoch": 1.0554666666666668,
      "grad_norm": 2.1790196895599365,
      "learning_rate": 3.7682394754502687e-06,
      "logits/chosen": -1.8438827991485596,
      "logits/rejected": -1.7237058877944946,
      "logps/chosen": -248.16622924804688,
      "logps/rejected": -366.1410217285156,
      "loss": 0.0907,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.643491506576538,
      "rewards/margins": 6.8003034591674805,
      "rewards/rejected": -8.443795204162598,
      "step": 495
    },
    {
      "epoch": 1.0661333333333334,
      "grad_norm": 0.9579430222511292,
      "learning_rate": 3.7433452915570685e-06,
      "logits/chosen": -1.7718284130096436,
      "logits/rejected": -1.6880018711090088,
      "logps/chosen": -251.2505645751953,
      "logps/rejected": -377.408203125,
      "loss": 0.1142,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.4772014617919922,
      "rewards/margins": 6.702824592590332,
      "rewards/rejected": -8.180025100708008,
      "step": 500
    },
    {
      "epoch": 1.0768,
      "grad_norm": 2.1115434169769287,
      "learning_rate": 3.718286216278143e-06,
      "logits/chosen": -1.904197335243225,
      "logits/rejected": -1.7636644840240479,
      "logps/chosen": -245.1167449951172,
      "logps/rejected": -346.50262451171875,
      "loss": 0.1178,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.9693208932876587,
      "rewards/margins": 6.109748840332031,
      "rewards/rejected": -8.079069137573242,
      "step": 505
    },
    {
      "epoch": 1.0874666666666666,
      "grad_norm": 2.8808727264404297,
      "learning_rate": 3.6930655729265557e-06,
      "logits/chosen": -1.7936203479766846,
      "logits/rejected": -1.7006099224090576,
      "logps/chosen": -258.9440002441406,
      "logps/rejected": -360.27838134765625,
      "loss": 0.1172,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.7141954898834229,
      "rewards/margins": 6.435247898101807,
      "rewards/rejected": -8.149443626403809,
      "step": 510
    },
    {
      "epoch": 1.0981333333333334,
      "grad_norm": 2.3610522747039795,
      "learning_rate": 3.667686706242386e-06,
      "logits/chosen": -1.7922704219818115,
      "logits/rejected": -1.697859764099121,
      "logps/chosen": -243.87789916992188,
      "logps/rejected": -360.91607666015625,
      "loss": 0.1032,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.5759384632110596,
      "rewards/margins": 6.990079402923584,
      "rewards/rejected": -8.566018104553223,
      "step": 515
    },
    {
      "epoch": 1.1088,
      "grad_norm": 1.427386999130249,
      "learning_rate": 3.6421529819491576e-06,
      "logits/chosen": -1.8297468423843384,
      "logits/rejected": -1.7564407587051392,
      "logps/chosen": -249.23031616210938,
      "logps/rejected": -372.94390869140625,
      "loss": 0.077,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.6625449657440186,
      "rewards/margins": 6.893601417541504,
      "rewards/rejected": -8.556145668029785,
      "step": 520
    },
    {
      "epoch": 1.1194666666666666,
      "grad_norm": 2.358030319213867,
      "learning_rate": 3.6164677863074783e-06,
      "logits/chosen": -1.7567555904388428,
      "logits/rejected": -1.6881614923477173,
      "logps/chosen": -255.1883087158203,
      "logps/rejected": -393.2801818847656,
      "loss": 0.098,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.7937577962875366,
      "rewards/margins": 7.25093936920166,
      "rewards/rejected": -9.044698715209961,
      "step": 525
    },
    {
      "epoch": 1.1301333333333332,
      "grad_norm": 2.982337713241577,
      "learning_rate": 3.590634525665955e-06,
      "logits/chosen": -1.8906761407852173,
      "logits/rejected": -1.7957327365875244,
      "logps/chosen": -246.13357543945312,
      "logps/rejected": -395.7541198730469,
      "loss": 0.1017,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.8768450021743774,
      "rewards/margins": 7.057843208312988,
      "rewards/rejected": -8.934687614440918,
      "step": 530
    },
    {
      "epoch": 1.1408,
      "grad_norm": 2.2151243686676025,
      "learning_rate": 3.564656626009446e-06,
      "logits/chosen": -1.837033987045288,
      "logits/rejected": -1.7429161071777344,
      "logps/chosen": -263.468994140625,
      "logps/rejected": -405.4566955566406,
      "loss": 0.1206,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.9729989767074585,
      "rewards/margins": 6.8837761878967285,
      "rewards/rejected": -8.85677433013916,
      "step": 535
    },
    {
      "epoch": 1.1514666666666666,
      "grad_norm": 2.5153486728668213,
      "learning_rate": 3.5385375325047167e-06,
      "logits/chosen": -1.8511556386947632,
      "logits/rejected": -1.6896768808364868,
      "logps/chosen": -251.3939971923828,
      "logps/rejected": -361.03143310546875,
      "loss": 0.0876,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.2249603271484375,
      "rewards/margins": 7.108116149902344,
      "rewards/rejected": -8.333075523376465,
      "step": 540
    },
    {
      "epoch": 1.1621333333333332,
      "grad_norm": 2.5955557823181152,
      "learning_rate": 3.512280709043534e-06,
      "logits/chosen": -1.696006417274475,
      "logits/rejected": -1.5523803234100342,
      "logps/chosen": -264.4658203125,
      "logps/rejected": -375.38494873046875,
      "loss": 0.1307,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.334220290184021,
      "rewards/margins": 6.640855312347412,
      "rewards/rejected": -7.975076198577881,
      "step": 545
    },
    {
      "epoch": 1.1728,
      "grad_norm": 1.6547006368637085,
      "learning_rate": 3.4858896377832966e-06,
      "logits/chosen": -1.7349450588226318,
      "logits/rejected": -1.7120225429534912,
      "logps/chosen": -254.0203399658203,
      "logps/rejected": -399.4454345703125,
      "loss": 0.0766,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.3470942974090576,
      "rewards/margins": 7.173849582672119,
      "rewards/rejected": -8.520943641662598,
      "step": 550
    },
    {
      "epoch": 1.1834666666666667,
      "grad_norm": 1.4159612655639648,
      "learning_rate": 3.459367818685231e-06,
      "logits/chosen": -1.807586431503296,
      "logits/rejected": -1.7281315326690674,
      "logps/chosen": -265.9685974121094,
      "logps/rejected": -377.4097595214844,
      "loss": 0.1151,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.1236010789871216,
      "rewards/margins": 6.955357551574707,
      "rewards/rejected": -8.078958511352539,
      "step": 555
    },
    {
      "epoch": 1.1941333333333333,
      "grad_norm": 1.7317302227020264,
      "learning_rate": 3.432718769050228e-06,
      "logits/chosen": -1.661529779434204,
      "logits/rejected": -1.6265112161636353,
      "logps/chosen": -254.3848419189453,
      "logps/rejected": -353.63995361328125,
      "loss": 0.124,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.3399780988693237,
      "rewards/margins": 6.360163688659668,
      "rewards/rejected": -7.700140953063965,
      "step": 560
    },
    {
      "epoch": 1.2048,
      "grad_norm": 4.254868507385254,
      "learning_rate": 3.405946023052386e-06,
      "logits/chosen": -1.791841745376587,
      "logits/rejected": -1.6956441402435303,
      "logps/chosen": -242.0591278076172,
      "logps/rejected": -372.5768127441406,
      "loss": 0.1307,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.4513027667999268,
      "rewards/margins": 6.603933811187744,
      "rewards/rejected": -8.055237770080566,
      "step": 565
    },
    {
      "epoch": 1.2154666666666667,
      "grad_norm": 1.8187251091003418,
      "learning_rate": 3.3790531312703055e-06,
      "logits/chosen": -1.7333256006240845,
      "logits/rejected": -1.6957298517227173,
      "logps/chosen": -259.1826477050781,
      "logps/rejected": -400.0516662597656,
      "loss": 0.0561,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.6751782894134521,
      "rewards/margins": 7.766973972320557,
      "rewards/rejected": -9.442152976989746,
      "step": 570
    },
    {
      "epoch": 1.2261333333333333,
      "grad_norm": 2.6395459175109863,
      "learning_rate": 3.35204366021622e-06,
      "logits/chosen": -1.7824958562850952,
      "logits/rejected": -1.6265745162963867,
      "logps/chosen": -256.4351501464844,
      "logps/rejected": -372.00408935546875,
      "loss": 0.111,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.2362754344940186,
      "rewards/margins": 7.035012722015381,
      "rewards/rejected": -8.27128791809082,
      "step": 575
    },
    {
      "epoch": 1.2368000000000001,
      "grad_norm": 3.0449585914611816,
      "learning_rate": 3.3249211918630055e-06,
      "logits/chosen": -1.7625986337661743,
      "logits/rejected": -1.6265180110931396,
      "logps/chosen": -248.1802978515625,
      "logps/rejected": -383.77593994140625,
      "loss": 0.1145,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.305022954940796,
      "rewards/margins": 7.372431755065918,
      "rewards/rejected": -8.677454948425293,
      "step": 580
    },
    {
      "epoch": 1.2474666666666667,
      "grad_norm": 1.849378228187561,
      "learning_rate": 3.2976893231691424e-06,
      "logits/chosen": -1.751132607460022,
      "logits/rejected": -1.6718765497207642,
      "logps/chosen": -267.318359375,
      "logps/rejected": -364.69854736328125,
      "loss": 0.1285,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.278596043586731,
      "rewards/margins": 6.7831621170043945,
      "rewards/rejected": -8.06175708770752,
      "step": 585
    },
    {
      "epoch": 1.2581333333333333,
      "grad_norm": 2.360128164291382,
      "learning_rate": 3.270351665601691e-06,
      "logits/chosen": -1.7772048711776733,
      "logits/rejected": -1.6471363306045532,
      "logps/chosen": -258.44476318359375,
      "logps/rejected": -384.2423400878906,
      "loss": 0.0679,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.230088472366333,
      "rewards/margins": 7.426251411437988,
      "rewards/rejected": -8.656339645385742,
      "step": 590
    },
    {
      "epoch": 1.2688,
      "grad_norm": 2.0064170360565186,
      "learning_rate": 3.2429118446573417e-06,
      "logits/chosen": -1.7410202026367188,
      "logits/rejected": -1.6303008794784546,
      "logps/chosen": -251.84634399414062,
      "logps/rejected": -384.3974609375,
      "loss": 0.111,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.8702796697616577,
      "rewards/margins": 6.885188102722168,
      "rewards/rejected": -8.755468368530273,
      "step": 595
    },
    {
      "epoch": 1.2794666666666665,
      "grad_norm": 5.854542255401611,
      "learning_rate": 3.215373499381602e-06,
      "logits/chosen": -1.774011254310608,
      "logits/rejected": -1.6293554306030273,
      "logps/chosen": -270.77960205078125,
      "logps/rejected": -352.97100830078125,
      "loss": 0.2022,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.5606591701507568,
      "rewards/margins": 6.231796741485596,
      "rewards/rejected": -7.792455196380615,
      "step": 600
    },
    {
      "epoch": 1.2901333333333334,
      "grad_norm": 1.544241189956665,
      "learning_rate": 3.1877402818861954e-06,
      "logits/chosen": -1.8023827075958252,
      "logits/rejected": -1.7564594745635986,
      "logps/chosen": -256.7626953125,
      "logps/rejected": -388.5929870605469,
      "loss": 0.101,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.5191371440887451,
      "rewards/margins": 7.524176120758057,
      "rewards/rejected": -9.043313980102539,
      "step": 605
    },
    {
      "epoch": 1.3008,
      "grad_norm": 1.2384198904037476,
      "learning_rate": 3.1600158568647145e-06,
      "logits/chosen": -1.7687852382659912,
      "logits/rejected": -1.7042837142944336,
      "logps/chosen": -261.32220458984375,
      "logps/rejected": -395.2125549316406,
      "loss": 0.1093,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.778892159461975,
      "rewards/margins": 7.3676629066467285,
      "rewards/rejected": -9.146554946899414,
      "step": 610
    },
    {
      "epoch": 1.3114666666666666,
      "grad_norm": 3.274482011795044,
      "learning_rate": 3.1322039011066172e-06,
      "logits/chosen": -1.742261528968811,
      "logits/rejected": -1.6239099502563477,
      "logps/chosen": -272.2462463378906,
      "logps/rejected": -370.30645751953125,
      "loss": 0.139,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.2790501117706299,
      "rewards/margins": 6.3297247886657715,
      "rewards/rejected": -7.6087751388549805,
      "step": 615
    },
    {
      "epoch": 1.3221333333333334,
      "grad_norm": 3.0375425815582275,
      "learning_rate": 3.1043081030096125e-06,
      "logits/chosen": -1.7067797183990479,
      "logits/rejected": -1.5599234104156494,
      "logps/chosen": -254.21237182617188,
      "logps/rejected": -340.0577697753906,
      "loss": 0.1824,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.123809576034546,
      "rewards/margins": 5.8070549964904785,
      "rewards/rejected": -6.930863857269287,
      "step": 620
    },
    {
      "epoch": 1.3328,
      "grad_norm": 2.601759672164917,
      "learning_rate": 3.0763321620905072e-06,
      "logits/chosen": -1.6933444738388062,
      "logits/rejected": -1.5981481075286865,
      "logps/chosen": -262.598876953125,
      "logps/rejected": -358.65301513671875,
      "loss": 0.1607,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.4247936010360718,
      "rewards/margins": 5.9775896072387695,
      "rewards/rejected": -7.402383327484131,
      "step": 625
    },
    {
      "epoch": 1.3434666666666666,
      "grad_norm": 1.8564891815185547,
      "learning_rate": 3.048279788494583e-06,
      "logits/chosen": -1.825899362564087,
      "logits/rejected": -1.6926910877227783,
      "logps/chosen": -229.56484985351562,
      "logps/rejected": -356.9173278808594,
      "loss": 0.1117,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.2786517143249512,
      "rewards/margins": 6.977616310119629,
      "rewards/rejected": -8.256268501281738,
      "step": 630
    },
    {
      "epoch": 1.3541333333333334,
      "grad_norm": 2.049483060836792,
      "learning_rate": 3.0201547025035553e-06,
      "logits/chosen": -1.7682701349258423,
      "logits/rejected": -1.605406403541565,
      "logps/chosen": -273.62322998046875,
      "logps/rejected": -379.70440673828125,
      "loss": 0.0778,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.0921200513839722,
      "rewards/margins": 6.923942565917969,
      "rewards/rejected": -8.016063690185547,
      "step": 635
    },
    {
      "epoch": 1.3648,
      "grad_norm": 3.9953954219818115,
      "learning_rate": 2.9919606340421963e-06,
      "logits/chosen": -1.7687931060791016,
      "logits/rejected": -1.5742592811584473,
      "logps/chosen": -267.0946044921875,
      "logps/rejected": -376.08544921875,
      "loss": 0.0946,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.3415029048919678,
      "rewards/margins": 7.037326812744141,
      "rewards/rejected": -8.378829956054688,
      "step": 640
    },
    {
      "epoch": 1.3754666666666666,
      "grad_norm": 1.9268256425857544,
      "learning_rate": 2.9637013221836723e-06,
      "logits/chosen": -1.9088714122772217,
      "logits/rejected": -1.6881043910980225,
      "logps/chosen": -247.6619415283203,
      "logps/rejected": -370.9812316894531,
      "loss": 0.1035,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.0276373624801636,
      "rewards/margins": 6.4521050453186035,
      "rewards/rejected": -7.479742527008057,
      "step": 645
    },
    {
      "epoch": 1.3861333333333334,
      "grad_norm": 2.7961528301239014,
      "learning_rate": 2.9353805146536762e-06,
      "logits/chosen": -1.7485949993133545,
      "logits/rejected": -1.6905990839004517,
      "logps/chosen": -258.29876708984375,
      "logps/rejected": -363.9190979003906,
      "loss": 0.1327,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.9902496337890625,
      "rewards/margins": 6.535128593444824,
      "rewards/rejected": -7.525378227233887,
      "step": 650
    },
    {
      "epoch": 1.3968,
      "grad_norm": 2.9360063076019287,
      "learning_rate": 2.907001967333401e-06,
      "logits/chosen": -1.8041868209838867,
      "logits/rejected": -1.718971610069275,
      "logps/chosen": -231.60531616210938,
      "logps/rejected": -374.5210266113281,
      "loss": 0.1069,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.2519428730010986,
      "rewards/margins": 6.751153469085693,
      "rewards/rejected": -8.003096580505371,
      "step": 655
    },
    {
      "epoch": 1.4074666666666666,
      "grad_norm": 2.948979377746582,
      "learning_rate": 2.878569443761442e-06,
      "logits/chosen": -1.7529418468475342,
      "logits/rejected": -1.5835460424423218,
      "logps/chosen": -276.1628723144531,
      "logps/rejected": -353.0192565917969,
      "loss": 0.1458,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.3621124029159546,
      "rewards/margins": 5.750298500061035,
      "rewards/rejected": -7.112410545349121,
      "step": 660
    },
    {
      "epoch": 1.4181333333333335,
      "grad_norm": 3.252516031265259,
      "learning_rate": 2.85008671463468e-06,
      "logits/chosen": -1.7881982326507568,
      "logits/rejected": -1.6824554204940796,
      "logps/chosen": -278.9662780761719,
      "logps/rejected": -380.4771423339844,
      "loss": 0.1478,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5313074588775635,
      "rewards/margins": 6.1621809005737305,
      "rewards/rejected": -7.693489074707031,
      "step": 665
    },
    {
      "epoch": 1.4288,
      "grad_norm": 1.8899283409118652,
      "learning_rate": 2.8215575573082127e-06,
      "logits/chosen": -1.7420358657836914,
      "logits/rejected": -1.6729049682617188,
      "logps/chosen": -270.51031494140625,
      "logps/rejected": -392.34112548828125,
      "loss": 0.0952,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.5278486013412476,
      "rewards/margins": 6.879241943359375,
      "rewards/rejected": -8.40709114074707,
      "step": 670
    },
    {
      "epoch": 1.4394666666666667,
      "grad_norm": 1.3593741655349731,
      "learning_rate": 2.792985755294404e-06,
      "logits/chosen": -1.8171350955963135,
      "logits/rejected": -1.7248483896255493,
      "logps/chosen": -249.7633514404297,
      "logps/rejected": -382.9278259277344,
      "loss": 0.0659,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.6088777780532837,
      "rewards/margins": 7.664944648742676,
      "rewards/rejected": -9.273821830749512,
      "step": 675
    },
    {
      "epoch": 1.4501333333333333,
      "grad_norm": 3.274573564529419,
      "learning_rate": 2.764375097761122e-06,
      "logits/chosen": -1.5862953662872314,
      "logits/rejected": -1.532132863998413,
      "logps/chosen": -254.4930877685547,
      "logps/rejected": -369.3152160644531,
      "loss": 0.1444,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.3496243953704834,
      "rewards/margins": 6.570565223693848,
      "rewards/rejected": -7.92018985748291,
      "step": 680
    },
    {
      "epoch": 1.4607999999999999,
      "grad_norm": 2.0709228515625,
      "learning_rate": 2.73572937902922e-06,
      "logits/chosen": -1.7570823431015015,
      "logits/rejected": -1.5986483097076416,
      "logps/chosen": -265.8306579589844,
      "logps/rejected": -376.62188720703125,
      "loss": 0.099,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.4001950025558472,
      "rewards/margins": 6.844396114349365,
      "rewards/rejected": -8.244590759277344,
      "step": 685
    },
    {
      "epoch": 1.4714666666666667,
      "grad_norm": 3.2238757610321045,
      "learning_rate": 2.7070523980693362e-06,
      "logits/chosen": -1.6330959796905518,
      "logits/rejected": -1.511679768562317,
      "logps/chosen": -253.7584228515625,
      "logps/rejected": -350.31829833984375,
      "loss": 0.1107,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.0016086101531982,
      "rewards/margins": 6.8228440284729,
      "rewards/rejected": -7.8244524002075195,
      "step": 690
    },
    {
      "epoch": 1.4821333333333333,
      "grad_norm": 1.463384747505188,
      "learning_rate": 2.678347957998081e-06,
      "logits/chosen": -1.7908287048339844,
      "logits/rejected": -1.6633214950561523,
      "logps/chosen": -249.6637725830078,
      "logps/rejected": -363.63128662109375,
      "loss": 0.0945,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.6576259136199951,
      "rewards/margins": 7.192323207855225,
      "rewards/rejected": -8.849950790405273,
      "step": 695
    },
    {
      "epoch": 1.4928,
      "grad_norm": 2.0910377502441406,
      "learning_rate": 2.649619865573666e-06,
      "logits/chosen": -1.7448034286499023,
      "logits/rejected": -1.6555458307266235,
      "logps/chosen": -250.4905242919922,
      "logps/rejected": -397.0603942871094,
      "loss": 0.0926,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.482170820236206,
      "rewards/margins": 7.455416202545166,
      "rewards/rejected": -8.93758773803711,
      "step": 700
    },
    {
      "epoch": 1.5034666666666667,
      "grad_norm": 3.544623851776123,
      "learning_rate": 2.620871930691058e-06,
      "logits/chosen": -1.750417709350586,
      "logits/rejected": -1.6637424230575562,
      "logps/chosen": -259.9458312988281,
      "logps/rejected": -364.2375183105469,
      "loss": 0.0947,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.3259789943695068,
      "rewards/margins": 6.424938201904297,
      "rewards/rejected": -7.750917911529541,
      "step": 705
    },
    {
      "epoch": 1.5141333333333333,
      "grad_norm": 1.3879026174545288,
      "learning_rate": 2.5921079658767152e-06,
      "logits/chosen": -1.7787948846817017,
      "logits/rejected": -1.7029085159301758,
      "logps/chosen": -246.4046630859375,
      "logps/rejected": -381.16473388671875,
      "loss": 0.1354,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.7129465341567993,
      "rewards/margins": 7.154146671295166,
      "rewards/rejected": -8.867093086242676,
      "step": 710
    },
    {
      "epoch": 1.5248,
      "grad_norm": 2.2066287994384766,
      "learning_rate": 2.56333178578297e-06,
      "logits/chosen": -1.7623145580291748,
      "logits/rejected": -1.6360619068145752,
      "logps/chosen": -227.5376434326172,
      "logps/rejected": -355.4570617675781,
      "loss": 0.1361,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.6756374835968018,
      "rewards/margins": 7.0356597900390625,
      "rewards/rejected": -8.711296081542969,
      "step": 715
    },
    {
      "epoch": 1.5354666666666668,
      "grad_norm": 2.268791437149048,
      "learning_rate": 2.534547206682136e-06,
      "logits/chosen": -1.7191269397735596,
      "logits/rejected": -1.661130666732788,
      "logps/chosen": -258.20086669921875,
      "logps/rejected": -383.30535888671875,
      "loss": 0.094,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.680588960647583,
      "rewards/margins": 7.147183418273926,
      "rewards/rejected": -8.82777214050293,
      "step": 720
    },
    {
      "epoch": 1.5461333333333334,
      "grad_norm": 1.6195091009140015,
      "learning_rate": 2.5057580459603947e-06,
      "logits/chosen": -1.8001463413238525,
      "logits/rejected": -1.6632938385009766,
      "logps/chosen": -244.26486206054688,
      "logps/rejected": -393.6264343261719,
      "loss": 0.0438,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.2562836408615112,
      "rewards/margins": 7.85473108291626,
      "rewards/rejected": -9.111015319824219,
      "step": 725
    },
    {
      "epoch": 1.5568,
      "grad_norm": 3.6901800632476807,
      "learning_rate": 2.4769681216115384e-06,
      "logits/chosen": -1.7932723760604858,
      "logits/rejected": -1.6791099309921265,
      "logps/chosen": -266.78314208984375,
      "logps/rejected": -386.56964111328125,
      "loss": 0.1069,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.260191798210144,
      "rewards/margins": 7.321366786956787,
      "rewards/rejected": -8.581558227539062,
      "step": 730
    },
    {
      "epoch": 1.5674666666666668,
      "grad_norm": 1.7813483476638794,
      "learning_rate": 2.448181251730632e-06,
      "logits/chosen": -1.673689842224121,
      "logits/rejected": -1.5504353046417236,
      "logps/chosen": -266.9430236816406,
      "logps/rejected": -365.57745361328125,
      "loss": 0.0846,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.159118890762329,
      "rewards/margins": 7.304617404937744,
      "rewards/rejected": -8.463736534118652,
      "step": 735
    },
    {
      "epoch": 1.5781333333333334,
      "grad_norm": 3.4884979724884033,
      "learning_rate": 2.419401254007658e-06,
      "logits/chosen": -1.7130966186523438,
      "logits/rejected": -1.661629319190979,
      "logps/chosen": -244.4851837158203,
      "logps/rejected": -335.0830383300781,
      "loss": 0.1607,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.1306991577148438,
      "rewards/margins": 6.248559474945068,
      "rewards/rejected": -7.379258632659912,
      "step": 740
    },
    {
      "epoch": 1.5888,
      "grad_norm": 1.881792664527893,
      "learning_rate": 2.390631945221221e-06,
      "logits/chosen": -1.8140137195587158,
      "logits/rejected": -1.6874796152114868,
      "logps/chosen": -251.9772186279297,
      "logps/rejected": -405.4315185546875,
      "loss": 0.114,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.2588303089141846,
      "rewards/margins": 7.343568325042725,
      "rewards/rejected": -8.602398872375488,
      "step": 745
    },
    {
      "epoch": 1.5994666666666668,
      "grad_norm": 3.2374913692474365,
      "learning_rate": 2.361877140732366e-06,
      "logits/chosen": -1.7451698780059814,
      "logits/rejected": -1.6588739156723022,
      "logps/chosen": -244.10238647460938,
      "logps/rejected": -370.6669616699219,
      "loss": 0.0985,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.2883882522583008,
      "rewards/margins": 7.572729587554932,
      "rewards/rejected": -8.861119270324707,
      "step": 750
    },
    {
      "epoch": 1.6101333333333332,
      "grad_norm": 1.0592817068099976,
      "learning_rate": 2.333140653978592e-06,
      "logits/chosen": -1.7900444269180298,
      "logits/rejected": -1.6860978603363037,
      "logps/chosen": -236.3094482421875,
      "logps/rejected": -374.94378662109375,
      "loss": 0.0955,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.9188971519470215,
      "rewards/margins": 7.50311279296875,
      "rewards/rejected": -8.42201042175293,
      "step": 755
    },
    {
      "epoch": 1.6208,
      "grad_norm": 3.37741756439209,
      "learning_rate": 2.3044262959681156e-06,
      "logits/chosen": -1.7714325189590454,
      "logits/rejected": -1.6465963125228882,
      "logps/chosen": -236.4646759033203,
      "logps/rejected": -370.81890869140625,
      "loss": 0.1222,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.4455827474594116,
      "rewards/margins": 7.438872337341309,
      "rewards/rejected": -8.884454727172852,
      "step": 760
    },
    {
      "epoch": 1.6314666666666666,
      "grad_norm": 1.3188512325286865,
      "learning_rate": 2.2757378747744575e-06,
      "logits/chosen": -1.7838866710662842,
      "logits/rejected": -1.6729694604873657,
      "logps/chosen": -257.4605407714844,
      "logps/rejected": -385.75311279296875,
      "loss": 0.082,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.1314411163330078,
      "rewards/margins": 7.492845058441162,
      "rewards/rejected": -8.624286651611328,
      "step": 765
    },
    {
      "epoch": 1.6421333333333332,
      "grad_norm": 3.832404136657715,
      "learning_rate": 2.24707919503142e-06,
      "logits/chosen": -1.7909473180770874,
      "logits/rejected": -1.700435996055603,
      "logps/chosen": -225.17324829101562,
      "logps/rejected": -385.271484375,
      "loss": 0.061,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.3442027568817139,
      "rewards/margins": 7.862277984619141,
      "rewards/rejected": -9.206480026245117,
      "step": 770
    },
    {
      "epoch": 1.6528,
      "grad_norm": 1.5521408319473267,
      "learning_rate": 2.2184540574285213e-06,
      "logits/chosen": -1.6965129375457764,
      "logits/rejected": -1.5588537454605103,
      "logps/chosen": -269.69244384765625,
      "logps/rejected": -360.9696960449219,
      "loss": 0.1064,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.2951409816741943,
      "rewards/margins": 7.005385398864746,
      "rewards/rejected": -8.30052661895752,
      "step": 775
    },
    {
      "epoch": 1.6634666666666666,
      "grad_norm": 2.485806703567505,
      "learning_rate": 2.1898662582069486e-06,
      "logits/chosen": -1.7989184856414795,
      "logits/rejected": -1.713746428489685,
      "logps/chosen": -260.45458984375,
      "logps/rejected": -368.6556396484375,
      "loss": 0.1179,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.8484270572662354,
      "rewards/margins": 7.124408721923828,
      "rewards/rejected": -8.972835540771484,
      "step": 780
    },
    {
      "epoch": 1.6741333333333333,
      "grad_norm": 4.892862319946289,
      "learning_rate": 2.1613195886561044e-06,
      "logits/chosen": -1.7144527435302734,
      "logits/rejected": -1.664764404296875,
      "logps/chosen": -230.86538696289062,
      "logps/rejected": -350.9761657714844,
      "loss": 0.1218,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.329289197921753,
      "rewards/margins": 7.392025947570801,
      "rewards/rejected": -8.721314430236816,
      "step": 785
    },
    {
      "epoch": 1.6848,
      "grad_norm": 6.508184909820557,
      "learning_rate": 2.132817834610811e-06,
      "logits/chosen": -1.7057596445083618,
      "logits/rejected": -1.6182563304901123,
      "logps/chosen": -253.01986694335938,
      "logps/rejected": -384.61895751953125,
      "loss": 0.1046,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.4259307384490967,
      "rewards/margins": 7.484774589538574,
      "rewards/rejected": -8.910707473754883,
      "step": 790
    },
    {
      "epoch": 1.6954666666666667,
      "grad_norm": 2.587836265563965,
      "learning_rate": 2.1043647759492338e-06,
      "logits/chosen": -1.7026351690292358,
      "logits/rejected": -1.6045818328857422,
      "logps/chosen": -267.7701721191406,
      "logps/rejected": -374.3067321777344,
      "loss": 0.1003,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.1852244138717651,
      "rewards/margins": 7.090452671051025,
      "rewards/rejected": -8.275677680969238,
      "step": 795
    },
    {
      "epoch": 1.7061333333333333,
      "grad_norm": 3.841643810272217,
      "learning_rate": 2.075964186091599e-06,
      "logits/chosen": -1.7560237646102905,
      "logits/rejected": -1.7158886194229126,
      "logps/chosen": -267.0213928222656,
      "logps/rejected": -395.33831787109375,
      "loss": 0.1109,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.816641092300415,
      "rewards/margins": 7.103610992431641,
      "rewards/rejected": -8.920252799987793,
      "step": 800
    },
    {
      "epoch": 1.7168,
      "grad_norm": 3.8351080417633057,
      "learning_rate": 2.047619831499765e-06,
      "logits/chosen": -1.678251028060913,
      "logits/rejected": -1.561795949935913,
      "logps/chosen": -245.37661743164062,
      "logps/rejected": -384.1910400390625,
      "loss": 0.0985,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.9903218150138855,
      "rewards/margins": 7.894017219543457,
      "rewards/rejected": -8.884339332580566,
      "step": 805
    },
    {
      "epoch": 1.7274666666666667,
      "grad_norm": 3.4570720195770264,
      "learning_rate": 2.0193354711777155e-06,
      "logits/chosen": -1.7100801467895508,
      "logits/rejected": -1.5458927154541016,
      "logps/chosen": -284.8987731933594,
      "logps/rejected": -392.1943359375,
      "loss": 0.131,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.7098143100738525,
      "rewards/margins": 7.403327941894531,
      "rewards/rejected": -9.113142013549805,
      "step": 810
    },
    {
      "epoch": 1.7381333333333333,
      "grad_norm": 3.687009334564209,
      "learning_rate": 1.991114856173048e-06,
      "logits/chosen": -1.8737754821777344,
      "logits/rejected": -1.7086925506591797,
      "logps/chosen": -241.7539520263672,
      "logps/rejected": -365.61285400390625,
      "loss": 0.0872,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.461383581161499,
      "rewards/margins": 7.248700141906738,
      "rewards/rejected": -8.710083961486816,
      "step": 815
    },
    {
      "epoch": 1.7488000000000001,
      "grad_norm": 1.9755408763885498,
      "learning_rate": 1.9629617290795115e-06,
      "logits/chosen": -1.752289056777954,
      "logits/rejected": -1.6548378467559814,
      "logps/chosen": -236.4134979248047,
      "logps/rejected": -373.27874755859375,
      "loss": 0.094,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.2346256971359253,
      "rewards/margins": 7.46105432510376,
      "rewards/rejected": -8.695680618286133,
      "step": 820
    },
    {
      "epoch": 1.7594666666666665,
      "grad_norm": 1.1113483905792236,
      "learning_rate": 1.934879823540663e-06,
      "logits/chosen": -1.7978004217147827,
      "logits/rejected": -1.6400505304336548,
      "logps/chosen": -250.33908081054688,
      "logps/rejected": -393.55609130859375,
      "loss": 0.0734,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.497048258781433,
      "rewards/margins": 7.612870693206787,
      "rewards/rejected": -9.109919548034668,
      "step": 825
    },
    {
      "epoch": 1.7701333333333333,
      "grad_norm": 6.385664939880371,
      "learning_rate": 1.9068728637547195e-06,
      "logits/chosen": -1.766744613647461,
      "logits/rejected": -1.5946464538574219,
      "logps/chosen": -267.80242919921875,
      "logps/rejected": -375.2735900878906,
      "loss": 0.1224,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.3728525638580322,
      "rewards/margins": 7.434065818786621,
      "rewards/rejected": -8.806917190551758,
      "step": 830
    },
    {
      "epoch": 1.7808000000000002,
      "grad_norm": 3.626858949661255,
      "learning_rate": 1.878944563980658e-06,
      "logits/chosen": -1.7465204000473022,
      "logits/rejected": -1.6045042276382446,
      "logps/chosen": -252.1056671142578,
      "logps/rejected": -388.88665771484375,
      "loss": 0.0827,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.755131483078003,
      "rewards/margins": 7.423327445983887,
      "rewards/rejected": -9.178459167480469,
      "step": 835
    },
    {
      "epoch": 1.7914666666666665,
      "grad_norm": 1.526719093322754,
      "learning_rate": 1.85109862804563e-06,
      "logits/chosen": -1.7173770666122437,
      "logits/rejected": -1.6473115682601929,
      "logps/chosen": -275.9317932128906,
      "logps/rejected": -405.3008117675781,
      "loss": 0.0705,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.4180126190185547,
      "rewards/margins": 8.002815246582031,
      "rewards/rejected": -9.420827865600586,
      "step": 840
    },
    {
      "epoch": 1.8021333333333334,
      "grad_norm": 4.655531406402588,
      "learning_rate": 1.823338748853766e-06,
      "logits/chosen": -1.8110342025756836,
      "logits/rejected": -1.6926085948944092,
      "logps/chosen": -257.8267822265625,
      "logps/rejected": -390.58929443359375,
      "loss": 0.1022,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.7875226736068726,
      "rewards/margins": 7.7925848960876465,
      "rewards/rejected": -9.580107688903809,
      "step": 845
    },
    {
      "epoch": 1.8128,
      "grad_norm": 4.3262434005737305,
      "learning_rate": 1.7956686078964257e-06,
      "logits/chosen": -1.8735952377319336,
      "logits/rejected": -1.7246639728546143,
      "logps/chosen": -238.3771514892578,
      "logps/rejected": -385.5560302734375,
      "loss": 0.0842,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.257483720779419,
      "rewards/margins": 7.96505069732666,
      "rewards/rejected": -9.222535133361816,
      "step": 850
    },
    {
      "epoch": 1.8234666666666666,
      "grad_norm": 2.52714467048645,
      "learning_rate": 1.7680918747639608e-06,
      "logits/chosen": -1.8481733798980713,
      "logits/rejected": -1.7482507228851318,
      "logps/chosen": -270.3813171386719,
      "logps/rejected": -388.9419860839844,
      "loss": 0.0817,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.8209350109100342,
      "rewards/margins": 7.654992580413818,
      "rewards/rejected": -9.47592830657959,
      "step": 855
    },
    {
      "epoch": 1.8341333333333334,
      "grad_norm": 1.7736057043075562,
      "learning_rate": 1.740612206659057e-06,
      "logits/chosen": -1.8067420721054077,
      "logits/rejected": -1.7217512130737305,
      "logps/chosen": -244.73971557617188,
      "logps/rejected": -391.7164306640625,
      "loss": 0.0884,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.6112474203109741,
      "rewards/margins": 8.113238334655762,
      "rewards/rejected": -9.724488258361816,
      "step": 860
    },
    {
      "epoch": 1.8448,
      "grad_norm": 2.050666570663452,
      "learning_rate": 1.7132332479117204e-06,
      "logits/chosen": -1.8168611526489258,
      "logits/rejected": -1.6906824111938477,
      "logps/chosen": -247.4569091796875,
      "logps/rejected": -366.1076965332031,
      "loss": 0.1243,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.388657808303833,
      "rewards/margins": 7.390578269958496,
      "rewards/rejected": -8.77923583984375,
      "step": 865
    },
    {
      "epoch": 1.8554666666666666,
      "grad_norm": 3.22883677482605,
      "learning_rate": 1.6859586294959666e-06,
      "logits/chosen": -1.6801704168319702,
      "logits/rejected": -1.598153829574585,
      "logps/chosen": -260.2568664550781,
      "logps/rejected": -403.77679443359375,
      "loss": 0.0706,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.6268384456634521,
      "rewards/margins": 9.174223899841309,
      "rewards/rejected": -10.801061630249023,
      "step": 870
    },
    {
      "epoch": 1.8661333333333334,
      "grad_norm": 1.8140406608581543,
      "learning_rate": 1.6587919685482884e-06,
      "logits/chosen": -1.8333885669708252,
      "logits/rejected": -1.7453978061676025,
      "logps/chosen": -252.1963653564453,
      "logps/rejected": -375.7099609375,
      "loss": 0.1162,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.9302170276641846,
      "rewards/margins": 7.10568904876709,
      "rewards/rejected": -9.035905838012695,
      "step": 875
    },
    {
      "epoch": 1.8768,
      "grad_norm": 3.0864861011505127,
      "learning_rate": 1.6317368678879497e-06,
      "logits/chosen": -1.6215035915374756,
      "logits/rejected": -1.5060207843780518,
      "logps/chosen": -260.1852111816406,
      "logps/rejected": -365.7777099609375,
      "loss": 0.1116,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.2610294818878174,
      "rewards/margins": 7.266331672668457,
      "rewards/rejected": -8.527360916137695,
      "step": 880
    },
    {
      "epoch": 1.8874666666666666,
      "grad_norm": 6.364953994750977,
      "learning_rate": 1.6047969155391856e-06,
      "logits/chosen": -1.6732227802276611,
      "logits/rejected": -1.6195056438446045,
      "logps/chosen": -268.6829833984375,
      "logps/rejected": -414.5812072753906,
      "loss": 0.1082,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.7019331455230713,
      "rewards/margins": 8.101677894592285,
      "rewards/rejected": -9.80361270904541,
      "step": 885
    },
    {
      "epoch": 1.8981333333333335,
      "grad_norm": 1.6536486148834229,
      "learning_rate": 1.5779756842553617e-06,
      "logits/chosen": -1.7743663787841797,
      "logits/rejected": -1.684413194656372,
      "logps/chosen": -267.4327087402344,
      "logps/rejected": -410.91094970703125,
      "loss": 0.0409,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7548547983169556,
      "rewards/margins": 8.745696067810059,
      "rewards/rejected": -10.500551223754883,
      "step": 890
    },
    {
      "epoch": 1.9088,
      "grad_norm": 6.427159786224365,
      "learning_rate": 1.551276731045157e-06,
      "logits/chosen": -1.7461345195770264,
      "logits/rejected": -1.6096042394638062,
      "logps/chosen": -288.79302978515625,
      "logps/rejected": -365.92401123046875,
      "loss": 0.1409,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.4583120346069336,
      "rewards/margins": 7.0046586990356445,
      "rewards/rejected": -8.462970733642578,
      "step": 895
    },
    {
      "epoch": 1.9194666666666667,
      "grad_norm": 2.0357890129089355,
      "learning_rate": 1.5247035967008369e-06,
      "logits/chosen": -1.6722333431243896,
      "logits/rejected": -1.5082788467407227,
      "logps/chosen": -270.37371826171875,
      "logps/rejected": -395.6129150390625,
      "loss": 0.0574,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.7694507837295532,
      "rewards/margins": 8.128179550170898,
      "rewards/rejected": -9.897629737854004,
      "step": 900
    },
    {
      "epoch": 1.9301333333333335,
      "grad_norm": 0.9068081378936768,
      "learning_rate": 1.498259805328678e-06,
      "logits/chosen": -1.7914531230926514,
      "logits/rejected": -1.626260757446289,
      "logps/chosen": -251.34585571289062,
      "logps/rejected": -374.37982177734375,
      "loss": 0.054,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.6412708759307861,
      "rewards/margins": 8.070221900939941,
      "rewards/rejected": -9.711493492126465,
      "step": 905
    },
    {
      "epoch": 1.9407999999999999,
      "grad_norm": 4.50501012802124,
      "learning_rate": 1.471948863881601e-06,
      "logits/chosen": -1.7430839538574219,
      "logits/rejected": -1.6489261388778687,
      "logps/chosen": -265.7547912597656,
      "logps/rejected": -381.2185363769531,
      "loss": 0.0842,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.7131439447402954,
      "rewards/margins": 8.174077033996582,
      "rewards/rejected": -9.887219429016113,
      "step": 910
    },
    {
      "epoch": 1.9514666666666667,
      "grad_norm": 3.7587532997131348,
      "learning_rate": 1.4457742616940864e-06,
      "logits/chosen": -1.6547327041625977,
      "logits/rejected": -1.6035993099212646,
      "logps/chosen": -259.0414123535156,
      "logps/rejected": -401.49151611328125,
      "loss": 0.1169,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.150399684906006,
      "rewards/margins": 7.753369331359863,
      "rewards/rejected": -9.903768539428711,
      "step": 915
    },
    {
      "epoch": 1.9621333333333333,
      "grad_norm": 3.2942192554473877,
      "learning_rate": 1.4197394700194165e-06,
      "logits/chosen": -1.7956269979476929,
      "logits/rejected": -1.6683623790740967,
      "logps/chosen": -274.89825439453125,
      "logps/rejected": -404.5597839355469,
      "loss": 0.0832,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.0448083877563477,
      "rewards/margins": 8.11993408203125,
      "rewards/rejected": -10.164742469787598,
      "step": 920
    },
    {
      "epoch": 1.9727999999999999,
      "grad_norm": 3.37825608253479,
      "learning_rate": 1.3938479415693244e-06,
      "logits/chosen": -1.789982795715332,
      "logits/rejected": -1.6593557596206665,
      "logps/chosen": -247.4827117919922,
      "logps/rejected": -355.44805908203125,
      "loss": 0.0922,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.3679771423339844,
      "rewards/margins": 7.889315605163574,
      "rewards/rejected": -9.257293701171875,
      "step": 925
    },
    {
      "epoch": 1.9834666666666667,
      "grad_norm": 1.674501895904541,
      "learning_rate": 1.368103110056096e-06,
      "logits/chosen": -1.7774940729141235,
      "logits/rejected": -1.7311162948608398,
      "logps/chosen": -244.7096710205078,
      "logps/rejected": -395.4224548339844,
      "loss": 0.0729,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.9104505777359009,
      "rewards/margins": 8.02760124206543,
      "rewards/rejected": -9.9380521774292,
      "step": 930
    },
    {
      "epoch": 1.9941333333333333,
      "grad_norm": 2.0359997749328613,
      "learning_rate": 1.3425083897371983e-06,
      "logits/chosen": -1.6318652629852295,
      "logits/rejected": -1.5561574697494507,
      "logps/chosen": -274.3860778808594,
      "logps/rejected": -406.825439453125,
      "loss": 0.0966,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.8862063884735107,
      "rewards/margins": 8.324484825134277,
      "rewards/rejected": -10.210691452026367,
      "step": 935
    },
    {
      "epoch": 2.0042666666666666,
      "grad_norm": 1.8420169353485107,
      "learning_rate": 1.317067174962478e-06,
      "logits/chosen": -1.7264703512191772,
      "logits/rejected": -1.605676531791687,
      "logps/chosen": -273.5916442871094,
      "logps/rejected": -401.54718017578125,
      "loss": 0.0943,
      "rewards/accuracies": 0.9671052694320679,
      "rewards/chosen": -1.6913379430770874,
      "rewards/margins": 8.358253479003906,
      "rewards/rejected": -10.049591064453125,
      "step": 940
    },
    {
      "epoch": 2.0149333333333335,
      "grad_norm": 1.155405044555664,
      "learning_rate": 1.291782839724009e-06,
      "logits/chosen": -1.725711464881897,
      "logits/rejected": -1.6092923879623413,
      "logps/chosen": -265.33740234375,
      "logps/rejected": -369.3727111816406,
      "loss": 0.0769,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.9963560104370117,
      "rewards/margins": 7.6727800369262695,
      "rewards/rejected": -9.669134140014648,
      "step": 945
    },
    {
      "epoch": 2.0256,
      "grad_norm": 2.466005802154541,
      "learning_rate": 1.2666587372086352e-06,
      "logits/chosen": -1.678786039352417,
      "logits/rejected": -1.595643401145935,
      "logps/chosen": -237.52456665039062,
      "logps/rejected": -396.2528991699219,
      "loss": 0.0767,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.0811383724212646,
      "rewards/margins": 8.369074821472168,
      "rewards/rejected": -10.450213432312012,
      "step": 950
    },
    {
      "epoch": 2.0362666666666667,
      "grad_norm": 3.321977376937866,
      "learning_rate": 1.241698199353275e-06,
      "logits/chosen": -1.7931787967681885,
      "logits/rejected": -1.688396692276001,
      "logps/chosen": -245.0885467529297,
      "logps/rejected": -375.5834655761719,
      "loss": 0.0815,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.0435004234313965,
      "rewards/margins": 7.793776035308838,
      "rewards/rejected": -9.837276458740234,
      "step": 955
    },
    {
      "epoch": 2.0469333333333335,
      "grad_norm": 2.211179256439209,
      "learning_rate": 1.2169045364030404e-06,
      "logits/chosen": -1.8382015228271484,
      "logits/rejected": -1.7111326456069946,
      "logps/chosen": -244.8855743408203,
      "logps/rejected": -396.12042236328125,
      "loss": 0.0589,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9382930994033813,
      "rewards/margins": 7.813839912414551,
      "rewards/rejected": -9.7521333694458,
      "step": 960
    },
    {
      "epoch": 2.0576,
      "grad_norm": 0.9813823699951172,
      "learning_rate": 1.1922810364722402e-06,
      "logits/chosen": -1.760215163230896,
      "logits/rejected": -1.666748046875,
      "logps/chosen": -263.532958984375,
      "logps/rejected": -400.35064697265625,
      "loss": 0.0579,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.9762378931045532,
      "rewards/margins": 8.321720123291016,
      "rewards/rejected": -10.297958374023438,
      "step": 965
    },
    {
      "epoch": 2.0682666666666667,
      "grad_norm": 3.9575207233428955,
      "learning_rate": 1.1678309651083055e-06,
      "logits/chosen": -1.7971630096435547,
      "logits/rejected": -1.7192646265029907,
      "logps/chosen": -249.60342407226562,
      "logps/rejected": -400.5608825683594,
      "loss": 0.0614,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.3201982975006104,
      "rewards/margins": 8.795618057250977,
      "rewards/rejected": -11.115816116333008,
      "step": 970
    },
    {
      "epoch": 2.0789333333333335,
      "grad_norm": 1.4822748899459839,
      "learning_rate": 1.1435575648587224e-06,
      "logits/chosen": -1.725407361984253,
      "logits/rejected": -1.598661184310913,
      "logps/chosen": -253.1859893798828,
      "logps/rejected": -377.39361572265625,
      "loss": 0.0597,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.158353328704834,
      "rewards/margins": 8.470383644104004,
      "rewards/rejected": -10.62873649597168,
      "step": 975
    },
    {
      "epoch": 2.0896,
      "grad_norm": 1.8873692750930786,
      "learning_rate": 1.1194640548410039e-06,
      "logits/chosen": -1.668302297592163,
      "logits/rejected": -1.5627166032791138,
      "logps/chosen": -266.97283935546875,
      "logps/rejected": -375.2995910644531,
      "loss": 0.0735,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.951266884803772,
      "rewards/margins": 8.111071586608887,
      "rewards/rejected": -10.062339782714844,
      "step": 980
    },
    {
      "epoch": 2.1002666666666667,
      "grad_norm": 1.918198823928833,
      "learning_rate": 1.095553630315774e-06,
      "logits/chosen": -1.6616764068603516,
      "logits/rejected": -1.6129405498504639,
      "logps/chosen": -262.82342529296875,
      "logps/rejected": -405.90679931640625,
      "loss": 0.0624,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.9863789081573486,
      "rewards/margins": 8.869156837463379,
      "rewards/rejected": -10.855535507202148,
      "step": 985
    },
    {
      "epoch": 2.1109333333333336,
      "grad_norm": 1.5651233196258545,
      "learning_rate": 1.0718294622630188e-06,
      "logits/chosen": -1.673699140548706,
      "logits/rejected": -1.6072123050689697,
      "logps/chosen": -255.4591064453125,
      "logps/rejected": -394.9210510253906,
      "loss": 0.065,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.9069595336914062,
      "rewards/margins": 8.828472137451172,
      "rewards/rejected": -10.735431671142578,
      "step": 990
    },
    {
      "epoch": 2.1216,
      "grad_norm": 1.483293056488037,
      "learning_rate": 1.0482946969615495e-06,
      "logits/chosen": -1.774780511856079,
      "logits/rejected": -1.6786816120147705,
      "logps/chosen": -244.82205200195312,
      "logps/rejected": -408.83221435546875,
      "loss": 0.065,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.166687488555908,
      "rewards/margins": 9.450934410095215,
      "rewards/rejected": -11.617620468139648,
      "step": 995
    },
    {
      "epoch": 2.1322666666666668,
      "grad_norm": 1.5644961595535278,
      "learning_rate": 1.0249524555717461e-06,
      "logits/chosen": -1.7748655080795288,
      "logits/rejected": -1.5471773147583008,
      "logps/chosen": -278.0244140625,
      "logps/rejected": -402.07269287109375,
      "loss": 0.104,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.0994677543640137,
      "rewards/margins": 7.748255729675293,
      "rewards/rejected": -9.847723007202148,
      "step": 1000
    },
    {
      "epoch": 2.142933333333333,
      "grad_norm": 4.346913814544678,
      "learning_rate": 1.0018058337216327e-06,
      "logits/chosen": -1.7031605243682861,
      "logits/rejected": -1.6505285501480103,
      "logps/chosen": -259.60772705078125,
      "logps/rejected": -405.2123107910156,
      "loss": 0.1133,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.2629244327545166,
      "rewards/margins": 8.715912818908691,
      "rewards/rejected": -10.978836059570312,
      "step": 1005
    },
    {
      "epoch": 2.1536,
      "grad_norm": 0.6849018335342407,
      "learning_rate": 9.788579010963387e-07,
      "logits/chosen": -1.6680097579956055,
      "logits/rejected": -1.5711143016815186,
      "logps/chosen": -273.92779541015625,
      "logps/rejected": -412.5943908691406,
      "loss": 0.0395,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.9487603902816772,
      "rewards/margins": 9.590485572814941,
      "rewards/rejected": -11.53924560546875,
      "step": 1010
    },
    {
      "epoch": 2.164266666666667,
      "grad_norm": 3.6428449153900146,
      "learning_rate": 9.56111701030998e-07,
      "logits/chosen": -1.7994695901870728,
      "logits/rejected": -1.6355020999908447,
      "logps/chosen": -275.2525329589844,
      "logps/rejected": -391.162841796875,
      "loss": 0.0863,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.1008682250976562,
      "rewards/margins": 8.165130615234375,
      "rewards/rejected": -10.265998840332031,
      "step": 1015
    },
    {
      "epoch": 2.174933333333333,
      "grad_norm": 0.5808694958686829,
      "learning_rate": 9.335702501071459e-07,
      "logits/chosen": -1.7420467138290405,
      "logits/rejected": -1.5945088863372803,
      "logps/chosen": -268.7550964355469,
      "logps/rejected": -399.09405517578125,
      "loss": 0.0508,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.7931501865386963,
      "rewards/margins": 8.578863143920898,
      "rewards/rejected": -10.3720121383667,
      "step": 1020
    },
    {
      "epoch": 2.1856,
      "grad_norm": 4.649181842803955,
      "learning_rate": 9.112365377526636e-07,
      "logits/chosen": -1.733960747718811,
      "logits/rejected": -1.6852184534072876,
      "logps/chosen": -247.6918487548828,
      "logps/rejected": -402.8586730957031,
      "loss": 0.0551,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.897204041481018,
      "rewards/margins": 8.999551773071289,
      "rewards/rejected": -10.896756172180176,
      "step": 1025
    },
    {
      "epoch": 2.196266666666667,
      "grad_norm": 1.2479854822158813,
      "learning_rate": 8.891135258453195e-07,
      "logits/chosen": -1.7280356884002686,
      "logits/rejected": -1.6294937133789062,
      "logps/chosen": -255.19839477539062,
      "logps/rejected": -381.9560546875,
      "loss": 0.0714,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.0049684047698975,
      "rewards/margins": 8.376141548156738,
      "rewards/rejected": -10.381109237670898,
      "step": 1030
    },
    {
      "epoch": 2.206933333333333,
      "grad_norm": 1.556071400642395,
      "learning_rate": 8.672041483199709e-07,
      "logits/chosen": -1.7548717260360718,
      "logits/rejected": -1.6648685932159424,
      "logps/chosen": -274.25128173828125,
      "logps/rejected": -394.82037353515625,
      "loss": 0.0696,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.2940263748168945,
      "rewards/margins": 8.477774620056152,
      "rewards/rejected": -10.771800994873047,
      "step": 1035
    },
    {
      "epoch": 2.2176,
      "grad_norm": 2.772015333175659,
      "learning_rate": 8.455113107794652e-07,
      "logits/chosen": -1.6806873083114624,
      "logits/rejected": -1.6288923025131226,
      "logps/chosen": -262.681640625,
      "logps/rejected": -416.0440979003906,
      "loss": 0.0813,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.2407727241516113,
      "rewards/margins": 9.37430191040039,
      "rewards/rejected": -11.61507511138916,
      "step": 1040
    },
    {
      "epoch": 2.228266666666667,
      "grad_norm": 3.664442777633667,
      "learning_rate": 8.240378901093035e-07,
      "logits/chosen": -1.7520214319229126,
      "logits/rejected": -1.662095308303833,
      "logps/chosen": -269.2132568359375,
      "logps/rejected": -416.43389892578125,
      "loss": 0.1195,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.21559476852417,
      "rewards/margins": 8.348793029785156,
      "rewards/rejected": -10.564387321472168,
      "step": 1045
    },
    {
      "epoch": 2.238933333333333,
      "grad_norm": 2.8179938793182373,
      "learning_rate": 8.027867340961113e-07,
      "logits/chosen": -1.6288158893585205,
      "logits/rejected": -1.6131118535995483,
      "logps/chosen": -272.9729919433594,
      "logps/rejected": -413.86212158203125,
      "loss": 0.0648,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.159022569656372,
      "rewards/margins": 8.596202850341797,
      "rewards/rejected": -10.755226135253906,
      "step": 1050
    },
    {
      "epoch": 2.2496,
      "grad_norm": 1.2309969663619995,
      "learning_rate": 7.817606610499639e-07,
      "logits/chosen": -1.7523510456085205,
      "logits/rejected": -1.6217739582061768,
      "logps/chosen": -251.67251586914062,
      "logps/rejected": -424.4778747558594,
      "loss": 0.0427,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.8224045038223267,
      "rewards/margins": 9.106525421142578,
      "rewards/rejected": -10.928930282592773,
      "step": 1055
    },
    {
      "epoch": 2.2602666666666664,
      "grad_norm": 2.9990038871765137,
      "learning_rate": 7.609624594306278e-07,
      "logits/chosen": -1.7163537740707397,
      "logits/rejected": -1.7219412326812744,
      "logps/chosen": -276.01055908203125,
      "logps/rejected": -401.6703186035156,
      "loss": 0.094,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.0416719913482666,
      "rewards/margins": 8.327810287475586,
      "rewards/rejected": -10.369482040405273,
      "step": 1060
    },
    {
      "epoch": 2.2709333333333332,
      "grad_norm": 1.0522708892822266,
      "learning_rate": 7.403948874777556e-07,
      "logits/chosen": -1.6532328128814697,
      "logits/rejected": -1.6515743732452393,
      "logps/chosen": -241.5132293701172,
      "logps/rejected": -397.3147888183594,
      "loss": 0.0655,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.9390627145767212,
      "rewards/margins": 9.06468391418457,
      "rewards/rejected": -11.00374698638916,
      "step": 1065
    },
    {
      "epoch": 2.2816,
      "grad_norm": 2.331453800201416,
      "learning_rate": 7.20060672845091e-07,
      "logits/chosen": -1.7282425165176392,
      "logits/rejected": -1.6557228565216064,
      "logps/chosen": -250.88046264648438,
      "logps/rejected": -403.5778503417969,
      "loss": 0.054,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.550058603286743,
      "rewards/margins": 8.61311149597168,
      "rewards/rejected": -11.16317081451416,
      "step": 1070
    },
    {
      "epoch": 2.2922666666666665,
      "grad_norm": 4.555802822113037,
      "learning_rate": 6.99962512238733e-07,
      "logits/chosen": -1.697750449180603,
      "logits/rejected": -1.5816296339035034,
      "logps/chosen": -270.8791198730469,
      "logps/rejected": -407.1304016113281,
      "loss": 0.0667,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.9709125757217407,
      "rewards/margins": 8.97321891784668,
      "rewards/rejected": -10.944131851196289,
      "step": 1075
    },
    {
      "epoch": 2.3029333333333333,
      "grad_norm": 0.7534166574478149,
      "learning_rate": 6.801030710594949e-07,
      "logits/chosen": -1.6750532388687134,
      "logits/rejected": -1.5743725299835205,
      "logps/chosen": -272.9534606933594,
      "logps/rejected": -417.29534912109375,
      "loss": 0.0591,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.146669864654541,
      "rewards/margins": 8.45936393737793,
      "rewards/rejected": -10.606033325195312,
      "step": 1080
    },
    {
      "epoch": 2.3136,
      "grad_norm": 3.8812808990478516,
      "learning_rate": 6.604849830494264e-07,
      "logits/chosen": -1.7331759929656982,
      "logits/rejected": -1.6734988689422607,
      "logps/chosen": -273.17071533203125,
      "logps/rejected": -394.3504333496094,
      "loss": 0.0694,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.4721755981445312,
      "rewards/margins": 8.414241790771484,
      "rewards/rejected": -10.8864164352417,
      "step": 1085
    },
    {
      "epoch": 2.3242666666666665,
      "grad_norm": 2.18582820892334,
      "learning_rate": 6.411108499425253e-07,
      "logits/chosen": -1.8008339405059814,
      "logits/rejected": -1.637495756149292,
      "logps/chosen": -275.2874755859375,
      "logps/rejected": -391.81597900390625,
      "loss": 0.0681,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.0567822456359863,
      "rewards/margins": 8.40910816192627,
      "rewards/rejected": -10.465889930725098,
      "step": 1090
    },
    {
      "epoch": 2.3349333333333333,
      "grad_norm": 1.457015037536621,
      "learning_rate": 6.219832411197013e-07,
      "logits/chosen": -1.6465427875518799,
      "logits/rejected": -1.494945764541626,
      "logps/chosen": -260.6093444824219,
      "logps/rejected": -381.10870361328125,
      "loss": 0.0754,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.6909992694854736,
      "rewards/margins": 8.483034133911133,
      "rewards/rejected": -10.174033164978027,
      "step": 1095
    },
    {
      "epoch": 2.3456,
      "grad_norm": 3.5426363945007324,
      "learning_rate": 6.031046932680229e-07,
      "logits/chosen": -1.737248420715332,
      "logits/rejected": -1.5785925388336182,
      "logps/chosen": -246.08706665039062,
      "logps/rejected": -378.8399963378906,
      "loss": 0.0726,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.9671459197998047,
      "rewards/margins": 8.330504417419434,
      "rewards/rejected": -10.297648429870605,
      "step": 1100
    },
    {
      "epoch": 2.3562666666666665,
      "grad_norm": 1.9274934530258179,
      "learning_rate": 5.844777100443091e-07,
      "logits/chosen": -1.668204665184021,
      "logits/rejected": -1.669782280921936,
      "logps/chosen": -252.9781036376953,
      "logps/rejected": -419.4017028808594,
      "loss": 0.0667,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.1728081703186035,
      "rewards/margins": 9.157058715820312,
      "rewards/rejected": -11.329866409301758,
      "step": 1105
    },
    {
      "epoch": 2.3669333333333333,
      "grad_norm": 4.665882110595703,
      "learning_rate": 5.66104761743092e-07,
      "logits/chosen": -1.8508713245391846,
      "logits/rejected": -1.714739203453064,
      "logps/chosen": -245.087646484375,
      "logps/rejected": -413.19305419921875,
      "loss": 0.0469,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.481471538543701,
      "rewards/margins": 9.403627395629883,
      "rewards/rejected": -11.885098457336426,
      "step": 1110
    },
    {
      "epoch": 2.3776,
      "grad_norm": 2.681351900100708,
      "learning_rate": 5.479882849690102e-07,
      "logits/chosen": -1.7130721807479858,
      "logits/rejected": -1.639499306678772,
      "logps/chosen": -258.9979553222656,
      "logps/rejected": -405.5329284667969,
      "loss": 0.0864,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.2818868160247803,
      "rewards/margins": 8.402789115905762,
      "rewards/rejected": -10.684676170349121,
      "step": 1115
    },
    {
      "epoch": 2.3882666666666665,
      "grad_norm": 3.2716691493988037,
      "learning_rate": 5.301306823136687e-07,
      "logits/chosen": -1.7014468908309937,
      "logits/rejected": -1.5933825969696045,
      "logps/chosen": -249.73483276367188,
      "logps/rejected": -382.95709228515625,
      "loss": 0.0715,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.2783286571502686,
      "rewards/margins": 7.969732761383057,
      "rewards/rejected": -10.248061180114746,
      "step": 1120
    },
    {
      "epoch": 2.3989333333333334,
      "grad_norm": 0.9533172249794006,
      "learning_rate": 5.125343220370088e-07,
      "logits/chosen": -1.689549446105957,
      "logits/rejected": -1.6298614740371704,
      "logps/chosen": -261.215087890625,
      "logps/rejected": -415.8038635253906,
      "loss": 0.0652,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.2256734371185303,
      "rewards/margins": 8.733424186706543,
      "rewards/rejected": -10.959096908569336,
      "step": 1125
    },
    {
      "epoch": 2.4096,
      "grad_norm": 0.5545014142990112,
      "learning_rate": 4.952015377532329e-07,
      "logits/chosen": -1.784350037574768,
      "logits/rejected": -1.6608524322509766,
      "logps/chosen": -235.21969604492188,
      "logps/rejected": -393.8280029296875,
      "loss": 0.0702,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.7608108520507812,
      "rewards/margins": 8.938084602355957,
      "rewards/rejected": -10.698893547058105,
      "step": 1130
    },
    {
      "epoch": 2.4202666666666666,
      "grad_norm": 1.1242990493774414,
      "learning_rate": 4.781346281213204e-07,
      "logits/chosen": -1.7812182903289795,
      "logits/rejected": -1.6980056762695312,
      "logps/chosen": -263.33758544921875,
      "logps/rejected": -392.80078125,
      "loss": 0.0531,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.3382551670074463,
      "rewards/margins": 8.795969009399414,
      "rewards/rejected": -11.134222984313965,
      "step": 1135
    },
    {
      "epoch": 2.4309333333333334,
      "grad_norm": 1.811649203300476,
      "learning_rate": 4.613358565401818e-07,
      "logits/chosen": -1.714125394821167,
      "logits/rejected": -1.5603991746902466,
      "logps/chosen": -266.761962890625,
      "logps/rejected": -408.21099853515625,
      "loss": 0.0643,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.1256799697875977,
      "rewards/margins": 9.198551177978516,
      "rewards/rejected": -11.32423210144043,
      "step": 1140
    },
    {
      "epoch": 2.4416,
      "grad_norm": 2.120972156524658,
      "learning_rate": 4.448074508484906e-07,
      "logits/chosen": -1.698669195175171,
      "logits/rejected": -1.5931520462036133,
      "logps/chosen": -266.32000732421875,
      "logps/rejected": -399.5585021972656,
      "loss": 0.0568,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.22153377532959,
      "rewards/margins": 8.874009132385254,
      "rewards/rejected": -11.09554386138916,
      "step": 1145
    },
    {
      "epoch": 2.4522666666666666,
      "grad_norm": 5.987904071807861,
      "learning_rate": 4.2855160302922777e-07,
      "logits/chosen": -1.6893975734710693,
      "logits/rejected": -1.5851631164550781,
      "logps/chosen": -258.2315979003906,
      "logps/rejected": -399.9725646972656,
      "loss": 0.0611,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.5237483978271484,
      "rewards/margins": 8.855246543884277,
      "rewards/rejected": -11.378995895385742,
      "step": 1150
    },
    {
      "epoch": 2.4629333333333334,
      "grad_norm": 1.4711482524871826,
      "learning_rate": 4.125704689189819e-07,
      "logits/chosen": -1.6372854709625244,
      "logits/rejected": -1.5050592422485352,
      "logps/chosen": -248.77352905273438,
      "logps/rejected": -393.07489013671875,
      "loss": 0.0763,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.038905620574951,
      "rewards/margins": 8.548587799072266,
      "rewards/rejected": -10.587493896484375,
      "step": 1155
    },
    {
      "epoch": 2.4736000000000002,
      "grad_norm": 3.182465076446533,
      "learning_rate": 3.9686616792204677e-07,
      "logits/chosen": -1.7079054117202759,
      "logits/rejected": -1.668967604637146,
      "logps/chosen": -274.384765625,
      "logps/rejected": -426.66436767578125,
      "loss": 0.0522,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.5939784049987793,
      "rewards/margins": 9.367667198181152,
      "rewards/rejected": -11.961645126342773,
      "step": 1160
    },
    {
      "epoch": 2.4842666666666666,
      "grad_norm": 1.7841264009475708,
      "learning_rate": 3.8144078272934676e-07,
      "logits/chosen": -1.678052306175232,
      "logits/rejected": -1.5851737260818481,
      "logps/chosen": -289.3586120605469,
      "logps/rejected": -421.1358947753906,
      "loss": 0.0504,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.6983115673065186,
      "rewards/margins": 8.838812828063965,
      "rewards/rejected": -11.537124633789062,
      "step": 1165
    },
    {
      "epoch": 2.4949333333333334,
      "grad_norm": 4.349465370178223,
      "learning_rate": 3.6629635904222943e-07,
      "logits/chosen": -1.6927592754364014,
      "logits/rejected": -1.668840765953064,
      "logps/chosen": -268.197998046875,
      "logps/rejected": -403.91265869140625,
      "loss": 0.0527,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.0596611499786377,
      "rewards/margins": 8.410848617553711,
      "rewards/rejected": -10.470510482788086,
      "step": 1170
    },
    {
      "epoch": 2.5056000000000003,
      "grad_norm": 2.6785404682159424,
      "learning_rate": 3.514349053011698e-07,
      "logits/chosen": -1.6893892288208008,
      "logits/rejected": -1.5429222583770752,
      "logps/chosen": -263.0149230957031,
      "logps/rejected": -389.00592041015625,
      "loss": 0.0765,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.377685546875,
      "rewards/margins": 8.378408432006836,
      "rewards/rejected": -10.75609302520752,
      "step": 1175
    },
    {
      "epoch": 2.5162666666666667,
      "grad_norm": 1.3735431432724,
      "learning_rate": 3.3685839241941133e-07,
      "logits/chosen": -1.6811046600341797,
      "logits/rejected": -1.615979552268982,
      "logps/chosen": -282.908203125,
      "logps/rejected": -395.00885009765625,
      "loss": 0.1003,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.4607012271881104,
      "rewards/margins": 8.770505905151367,
      "rewards/rejected": -11.231206893920898,
      "step": 1180
    },
    {
      "epoch": 2.5269333333333335,
      "grad_norm": 4.778181076049805,
      "learning_rate": 3.2256875352158435e-07,
      "logits/chosen": -1.7965024709701538,
      "logits/rejected": -1.619570016860962,
      "logps/chosen": -269.8258056640625,
      "logps/rejected": -424.8388671875,
      "loss": 0.0941,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.287374973297119,
      "rewards/margins": 9.26796817779541,
      "rewards/rejected": -11.555343627929688,
      "step": 1185
    },
    {
      "epoch": 2.5376,
      "grad_norm": 1.0934377908706665,
      "learning_rate": 3.085678836873399e-07,
      "logits/chosen": -1.698390007019043,
      "logits/rejected": -1.6988176107406616,
      "logps/chosen": -260.91217041015625,
      "logps/rejected": -410.05828857421875,
      "loss": 0.0825,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.011669397354126,
      "rewards/margins": 8.771254539489746,
      "rewards/rejected": -10.782922744750977,
      "step": 1190
    },
    {
      "epoch": 2.5482666666666667,
      "grad_norm": 3.100398540496826,
      "learning_rate": 2.9485763970002106e-07,
      "logits/chosen": -1.7358124256134033,
      "logits/rejected": -1.630570650100708,
      "logps/chosen": -275.30572509765625,
      "logps/rejected": -406.6322937011719,
      "loss": 0.0762,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.0499629974365234,
      "rewards/margins": 8.547712326049805,
      "rewards/rejected": -10.597673416137695,
      "step": 1195
    },
    {
      "epoch": 2.558933333333333,
      "grad_norm": 2.3817620277404785,
      "learning_rate": 2.8143983980042174e-07,
      "logits/chosen": -1.7879635095596313,
      "logits/rejected": -1.6414146423339844,
      "logps/chosen": -265.8365783691406,
      "logps/rejected": -415.36688232421875,
      "loss": 0.0499,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.5079102516174316,
      "rewards/margins": 9.601276397705078,
      "rewards/rejected": -12.109187126159668,
      "step": 1200
    },
    {
      "epoch": 2.5696,
      "grad_norm": 1.1379503011703491,
      "learning_rate": 2.683162634456515e-07,
      "logits/chosen": -1.6539013385772705,
      "logits/rejected": -1.542270302772522,
      "logps/chosen": -282.34429931640625,
      "logps/rejected": -397.26898193359375,
      "loss": 0.041,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0202269554138184,
      "rewards/margins": 8.675008773803711,
      "rewards/rejected": -10.695237159729004,
      "step": 1205
    },
    {
      "epoch": 2.5802666666666667,
      "grad_norm": 1.2463977336883545,
      "learning_rate": 2.5548865107314606e-07,
      "logits/chosen": -1.5867834091186523,
      "logits/rejected": -1.495589256286621,
      "logps/chosen": -287.71478271484375,
      "logps/rejected": -385.6083068847656,
      "loss": 0.0634,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.2334399223327637,
      "rewards/margins": 8.365260124206543,
      "rewards/rejected": -10.598699569702148,
      "step": 1210
    },
    {
      "epoch": 2.590933333333333,
      "grad_norm": 3.3940725326538086,
      "learning_rate": 2.429587038698511e-07,
      "logits/chosen": -1.7194877862930298,
      "logits/rejected": -1.652475118637085,
      "logps/chosen": -231.0302276611328,
      "logps/rejected": -386.63714599609375,
      "loss": 0.0668,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.5545358657836914,
      "rewards/margins": 8.87263298034668,
      "rewards/rejected": -11.427169799804688,
      "step": 1215
    },
    {
      "epoch": 2.6016,
      "grad_norm": 1.4201222658157349,
      "learning_rate": 2.307280835466144e-07,
      "logits/chosen": -1.7387536764144897,
      "logits/rejected": -1.645634412765503,
      "logps/chosen": -272.35699462890625,
      "logps/rejected": -395.57843017578125,
      "loss": 0.0701,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.429680347442627,
      "rewards/margins": 7.889183044433594,
      "rewards/rejected": -10.318862915039062,
      "step": 1220
    },
    {
      "epoch": 2.6122666666666667,
      "grad_norm": 4.102688312530518,
      "learning_rate": 2.1879841211780856e-07,
      "logits/chosen": -1.6071255207061768,
      "logits/rejected": -1.6051759719848633,
      "logps/chosen": -253.9296112060547,
      "logps/rejected": -408.7915954589844,
      "loss": 0.0719,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.2510740756988525,
      "rewards/margins": 9.297788619995117,
      "rewards/rejected": -11.548863410949707,
      "step": 1225
    },
    {
      "epoch": 2.622933333333333,
      "grad_norm": 1.2044811248779297,
      "learning_rate": 2.0717127168622202e-07,
      "logits/chosen": -1.72200608253479,
      "logits/rejected": -1.6253166198730469,
      "logps/chosen": -265.43292236328125,
      "logps/rejected": -395.1199645996094,
      "loss": 0.0455,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.528899669647217,
      "rewards/margins": 8.821831703186035,
      "rewards/rejected": -11.350730895996094,
      "step": 1230
    },
    {
      "epoch": 2.6336,
      "grad_norm": 2.7064669132232666,
      "learning_rate": 1.958482042332424e-07,
      "logits/chosen": -1.7487704753875732,
      "logits/rejected": -1.5633618831634521,
      "logps/chosen": -256.0242919921875,
      "logps/rejected": -387.086181640625,
      "loss": 0.1128,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.0679194927215576,
      "rewards/margins": 9.494586944580078,
      "rewards/rejected": -11.562507629394531,
      "step": 1235
    },
    {
      "epoch": 2.6442666666666668,
      "grad_norm": 2.0025794506073,
      "learning_rate": 1.8483071141435937e-07,
      "logits/chosen": -1.5965923070907593,
      "logits/rejected": -1.5514066219329834,
      "logps/chosen": -288.4261474609375,
      "logps/rejected": -398.62908935546875,
      "loss": 0.0768,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.1800413131713867,
      "rewards/margins": 8.478593826293945,
      "rewards/rejected": -10.658636093139648,
      "step": 1240
    },
    {
      "epoch": 2.654933333333333,
      "grad_norm": 0.43382975459098816,
      "learning_rate": 1.741202543600176e-07,
      "logits/chosen": -1.7435810565948486,
      "logits/rejected": -1.6685746908187866,
      "logps/chosen": -252.3104248046875,
      "logps/rejected": -387.71044921875,
      "loss": 0.0762,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.4105584621429443,
      "rewards/margins": 8.391923904418945,
      "rewards/rejected": -10.802482604980469,
      "step": 1245
    },
    {
      "epoch": 2.6656,
      "grad_norm": 3.122593641281128,
      "learning_rate": 1.6371825348184155e-07,
      "logits/chosen": -1.7292636632919312,
      "logits/rejected": -1.6958808898925781,
      "logps/chosen": -266.6325378417969,
      "logps/rejected": -389.1386413574219,
      "loss": 0.1075,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.3547916412353516,
      "rewards/margins": 8.089277267456055,
      "rewards/rejected": -10.444067001342773,
      "step": 1250
    },
    {
      "epoch": 2.676266666666667,
      "grad_norm": 2.821847915649414,
      "learning_rate": 1.5362608828426246e-07,
      "logits/chosen": -1.7317949533462524,
      "logits/rejected": -1.6616737842559814,
      "logps/chosen": -273.41741943359375,
      "logps/rejected": -399.07659912109375,
      "loss": 0.0669,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.323963165283203,
      "rewards/margins": 8.672471046447754,
      "rewards/rejected": -10.996434211730957,
      "step": 1255
    },
    {
      "epoch": 2.686933333333333,
      "grad_norm": 5.389606952667236,
      "learning_rate": 1.438450971815686e-07,
      "logits/chosen": -1.7826637029647827,
      "logits/rejected": -1.6701968908309937,
      "logps/chosen": -266.99139404296875,
      "logps/rejected": -391.0360412597656,
      "loss": 0.0699,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.6968390941619873,
      "rewards/margins": 8.890096664428711,
      "rewards/rejected": -11.586936950683594,
      "step": 1260
    },
    {
      "epoch": 2.6976,
      "grad_norm": 2.9304444789886475,
      "learning_rate": 1.3437657732040783e-07,
      "logits/chosen": -1.6977288722991943,
      "logits/rejected": -1.575650691986084,
      "logps/chosen": -265.2109680175781,
      "logps/rejected": -393.3660888671875,
      "loss": 0.0733,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.047703266143799,
      "rewards/margins": 8.961397171020508,
      "rewards/rejected": -11.009099960327148,
      "step": 1265
    },
    {
      "epoch": 2.708266666666667,
      "grad_norm": 3.8689467906951904,
      "learning_rate": 1.2522178440776022e-07,
      "logits/chosen": -1.7184581756591797,
      "logits/rejected": -1.7143863439559937,
      "logps/chosen": -271.6422119140625,
      "logps/rejected": -383.397705078125,
      "loss": 0.0879,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.4679946899414062,
      "rewards/margins": 8.029095649719238,
      "rewards/rejected": -10.497090339660645,
      "step": 1270
    },
    {
      "epoch": 2.718933333333333,
      "grad_norm": 0.6974985003471375,
      "learning_rate": 1.1638193254440655e-07,
      "logits/chosen": -1.7665700912475586,
      "logits/rejected": -1.689286231994629,
      "logps/chosen": -255.50302124023438,
      "logps/rejected": -408.79461669921875,
      "loss": 0.066,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.648810863494873,
      "rewards/margins": 9.510984420776367,
      "rewards/rejected": -12.159795761108398,
      "step": 1275
    },
    {
      "epoch": 2.7296,
      "grad_norm": 1.31102454662323,
      "learning_rate": 1.0785819406391562e-07,
      "logits/chosen": -1.653638482093811,
      "logits/rejected": -1.6233888864517212,
      "logps/chosen": -266.7182922363281,
      "logps/rejected": -418.28192138671875,
      "loss": 0.056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.3988518714904785,
      "rewards/margins": 8.98476791381836,
      "rewards/rejected": -11.383620262145996,
      "step": 1280
    },
    {
      "epoch": 2.740266666666667,
      "grad_norm": 1.9283661842346191,
      "learning_rate": 9.965169937717095e-08,
      "logits/chosen": -1.6370359659194946,
      "logits/rejected": -1.631044626235962,
      "logps/chosen": -252.5908660888672,
      "logps/rejected": -401.8351745605469,
      "loss": 0.0834,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.1292572021484375,
      "rewards/margins": 8.970866203308105,
      "rewards/rejected": -11.10012435913086,
      "step": 1285
    },
    {
      "epoch": 2.7509333333333332,
      "grad_norm": 2.89962100982666,
      "learning_rate": 9.176353682245676e-08,
      "logits/chosen": -1.75473153591156,
      "logits/rejected": -1.631553292274475,
      "logps/chosen": -248.2369842529297,
      "logps/rejected": -367.32745361328125,
      "loss": 0.0781,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.326287269592285,
      "rewards/margins": 8.09610366821289,
      "rewards/rejected": -10.422391891479492,
      "step": 1290
    },
    {
      "epoch": 2.7616,
      "grad_norm": 3.716407299041748,
      "learning_rate": 8.41947525211223e-08,
      "logits/chosen": -1.6061655282974243,
      "logits/rejected": -1.5312092304229736,
      "logps/chosen": -253.99716186523438,
      "logps/rejected": -383.96124267578125,
      "loss": 0.1108,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.050841808319092,
      "rewards/margins": 8.109245300292969,
      "rewards/rejected": -10.160086631774902,
      "step": 1295
    },
    {
      "epoch": 2.772266666666667,
      "grad_norm": 1.4873884916305542,
      "learning_rate": 7.694635023884789e-08,
      "logits/chosen": -1.6627674102783203,
      "logits/rejected": -1.5674951076507568,
      "logps/chosen": -273.51275634765625,
      "logps/rejected": -383.80731201171875,
      "loss": 0.0602,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.287492036819458,
      "rewards/margins": 8.161794662475586,
      "rewards/rejected": -10.449285507202148,
      "step": 1300
    },
    {
      "epoch": 2.7829333333333333,
      "grad_norm": 3.9774677753448486,
      "learning_rate": 7.00192912525241e-08,
      "logits/chosen": -1.6725132465362549,
      "logits/rejected": -1.6510508060455322,
      "logps/chosen": -256.28558349609375,
      "logps/rejected": -408.4872741699219,
      "loss": 0.0857,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.293246030807495,
      "rewards/margins": 9.179945945739746,
      "rewards/rejected": -11.47319221496582,
      "step": 1305
    },
    {
      "epoch": 2.7936,
      "grad_norm": 3.7363460063934326,
      "learning_rate": 6.341449422277047e-08,
      "logits/chosen": -1.5966999530792236,
      "logits/rejected": -1.5942323207855225,
      "logps/chosen": -289.9300842285156,
      "logps/rejected": -430.7188415527344,
      "loss": 0.0602,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.317781686782837,
      "rewards/margins": 8.42338752746582,
      "rewards/rejected": -10.741169929504395,
      "step": 1310
    },
    {
      "epoch": 2.804266666666667,
      "grad_norm": 1.0523943901062012,
      "learning_rate": 5.7132835072101486e-08,
      "logits/chosen": -1.668763518333435,
      "logits/rejected": -1.6661746501922607,
      "logps/chosen": -245.1510009765625,
      "logps/rejected": -397.2869567871094,
      "loss": 0.0798,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.2015979290008545,
      "rewards/margins": 8.682696342468262,
      "rewards/rejected": -10.884294509887695,
      "step": 1315
    },
    {
      "epoch": 2.8149333333333333,
      "grad_norm": 1.1371092796325684,
      "learning_rate": 5.117514686876379e-08,
      "logits/chosen": -1.6409333944320679,
      "logits/rejected": -1.5510404109954834,
      "logps/chosen": -253.13516235351562,
      "logps/rejected": -394.7981262207031,
      "loss": 0.0875,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.084290027618408,
      "rewards/margins": 8.778473854064941,
      "rewards/rejected": -10.862764358520508,
      "step": 1320
    },
    {
      "epoch": 2.8256,
      "grad_norm": 4.370893955230713,
      "learning_rate": 4.5542219716255274e-08,
      "logits/chosen": -1.6426441669464111,
      "logits/rejected": -1.5962680578231812,
      "logps/chosen": -295.06854248046875,
      "logps/rejected": -409.53399658203125,
      "loss": 0.0983,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.649624824523926,
      "rewards/margins": 8.616082191467285,
      "rewards/rejected": -11.265706062316895,
      "step": 1325
    },
    {
      "epoch": 2.836266666666667,
      "grad_norm": 0.8870635032653809,
      "learning_rate": 4.023480064854174e-08,
      "logits/chosen": -1.773553490638733,
      "logits/rejected": -1.6232315301895142,
      "logps/chosen": -280.57781982421875,
      "logps/rejected": -411.35565185546875,
      "loss": 0.0473,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.6773681640625,
      "rewards/margins": 9.148324966430664,
      "rewards/rejected": -11.82569408416748,
      "step": 1330
    },
    {
      "epoch": 2.8469333333333333,
      "grad_norm": 2.6349360942840576,
      "learning_rate": 3.525359353098584e-08,
      "logits/chosen": -1.6854965686798096,
      "logits/rejected": -1.6002693176269531,
      "logps/chosen": -263.1238708496094,
      "logps/rejected": -407.7985534667969,
      "loss": 0.063,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.756936550140381,
      "rewards/margins": 8.892518043518066,
      "rewards/rejected": -11.649454116821289,
      "step": 1335
    },
    {
      "epoch": 2.8576,
      "grad_norm": 4.308375835418701,
      "learning_rate": 3.059925896700094e-08,
      "logits/chosen": -1.606723427772522,
      "logits/rejected": -1.4940506219863892,
      "logps/chosen": -289.65118408203125,
      "logps/rejected": -372.673583984375,
      "loss": 0.1077,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.2103538513183594,
      "rewards/margins": 7.562900543212891,
      "rewards/rejected": -9.773255348205566,
      "step": 1340
    },
    {
      "epoch": 2.8682666666666665,
      "grad_norm": 3.312288761138916,
      "learning_rate": 2.6272414210443964e-08,
      "logits/chosen": -1.6424680948257446,
      "logits/rejected": -1.656428337097168,
      "logps/chosen": -265.83258056640625,
      "logps/rejected": -385.98614501953125,
      "loss": 0.0882,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.3592121601104736,
      "rewards/margins": 7.7441887855529785,
      "rewards/rejected": -10.103400230407715,
      "step": 1345
    },
    {
      "epoch": 2.8789333333333333,
      "grad_norm": 6.28291130065918,
      "learning_rate": 2.2273633083753632e-08,
      "logits/chosen": -1.7576591968536377,
      "logits/rejected": -1.6805713176727295,
      "logps/chosen": -248.3701171875,
      "logps/rejected": -401.76904296875,
      "loss": 0.0964,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.8228938579559326,
      "rewards/margins": 9.486324310302734,
      "rewards/rejected": -12.309218406677246,
      "step": 1350
    },
    {
      "epoch": 2.8895999999999997,
      "grad_norm": 2.0318760871887207,
      "learning_rate": 1.8603445901851647e-08,
      "logits/chosen": -1.6315650939941406,
      "logits/rejected": -1.558574914932251,
      "logps/chosen": -266.645263671875,
      "logps/rejected": -393.82257080078125,
      "loss": 0.0685,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.1889877319335938,
      "rewards/margins": 8.80628776550293,
      "rewards/rejected": -10.995275497436523,
      "step": 1355
    },
    {
      "epoch": 2.9002666666666665,
      "grad_norm": 4.676877021789551,
      "learning_rate": 1.5262339401813375e-08,
      "logits/chosen": -1.785639762878418,
      "logits/rejected": -1.6920429468154907,
      "logps/chosen": -268.1185302734375,
      "logps/rejected": -410.142822265625,
      "loss": 0.1384,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.5416979789733887,
      "rewards/margins": 8.666342735290527,
      "rewards/rejected": -11.208040237426758,
      "step": 1360
    },
    {
      "epoch": 2.9109333333333334,
      "grad_norm": 4.1418538093566895,
      "learning_rate": 1.2250756678315601e-08,
      "logits/chosen": -1.7175476551055908,
      "logits/rejected": -1.6276105642318726,
      "logps/chosen": -279.15985107421875,
      "logps/rejected": -424.0628967285156,
      "loss": 0.0476,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.5548946857452393,
      "rewards/margins": 8.781354904174805,
      "rewards/rejected": -11.336250305175781,
      "step": 1365
    },
    {
      "epoch": 2.9215999999999998,
      "grad_norm": 1.394214153289795,
      "learning_rate": 9.569097124875193e-09,
      "logits/chosen": -1.7502952814102173,
      "logits/rejected": -1.6516965627670288,
      "logps/chosen": -252.2337188720703,
      "logps/rejected": -379.22149658203125,
      "loss": 0.1196,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.270294666290283,
      "rewards/margins": 7.760924339294434,
      "rewards/rejected": -10.031218528747559,
      "step": 1370
    },
    {
      "epoch": 2.9322666666666666,
      "grad_norm": 3.925764799118042,
      "learning_rate": 7.217716380881479e-09,
      "logits/chosen": -1.6725263595581055,
      "logits/rejected": -1.6312596797943115,
      "logps/chosen": -278.2488098144531,
      "logps/rejected": -412.33428955078125,
      "loss": 0.0922,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.1912899017333984,
      "rewards/margins": 8.727452278137207,
      "rewards/rejected": -10.918742179870605,
      "step": 1375
    },
    {
      "epoch": 2.9429333333333334,
      "grad_norm": 1.50179123878479,
      "learning_rate": 5.196926284430359e-09,
      "logits/chosen": -1.6912105083465576,
      "logits/rejected": -1.5623693466186523,
      "logps/chosen": -272.2286682128906,
      "logps/rejected": -398.8866271972656,
      "loss": 0.0806,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.530665397644043,
      "rewards/margins": 8.755317687988281,
      "rewards/rejected": -11.285982131958008,
      "step": 1380
    },
    {
      "epoch": 2.9536,
      "grad_norm": 2.6899476051330566,
      "learning_rate": 3.5069948309712775e-09,
      "logits/chosen": -1.6759449243545532,
      "logits/rejected": -1.6454025506973267,
      "logps/chosen": -288.8973083496094,
      "logps/rejected": -421.2940979003906,
      "loss": 0.0664,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.2580349445343018,
      "rewards/margins": 8.855852127075195,
      "rewards/rejected": -11.113885879516602,
      "step": 1385
    },
    {
      "epoch": 2.9642666666666666,
      "grad_norm": 1.0512514114379883,
      "learning_rate": 2.1481461377634294e-09,
      "logits/chosen": -1.6516395807266235,
      "logits/rejected": -1.6442527770996094,
      "logps/chosen": -271.1015625,
      "logps/rejected": -423.9894104003906,
      "loss": 0.061,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.8409223556518555,
      "rewards/margins": 9.194467544555664,
      "rewards/rejected": -11.035387992858887,
      "step": 1390
    },
    {
      "epoch": 2.9749333333333334,
      "grad_norm": 1.4273769855499268,
      "learning_rate": 1.120560414154259e-09,
      "logits/chosen": -1.6637353897094727,
      "logits/rejected": -1.5640382766723633,
      "logps/chosen": -255.4537811279297,
      "logps/rejected": -424.53680419921875,
      "loss": 0.0447,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.2763662338256836,
      "rewards/margins": 9.897512435913086,
      "rewards/rejected": -12.17387866973877,
      "step": 1395
    },
    {
      "epoch": 2.9856,
      "grad_norm": 3.6625378131866455,
      "learning_rate": 4.2437393768079983e-10,
      "logits/chosen": -1.672315001487732,
      "logits/rejected": -1.5887823104858398,
      "logps/chosen": -275.51129150390625,
      "logps/rejected": -381.4290466308594,
      "loss": 0.1196,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.458815813064575,
      "rewards/margins": 8.02712631225586,
      "rewards/rejected": -10.485941886901855,
      "step": 1400
    },
    {
      "epoch": 2.9962666666666666,
      "grad_norm": 2.525285243988037,
      "learning_rate": 5.967903599635171e-11,
      "logits/chosen": -1.67082941532135,
      "logits/rejected": -1.5557708740234375,
      "logps/chosen": -261.8987731933594,
      "logps/rejected": -392.11273193359375,
      "loss": 0.1019,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.351944923400879,
      "rewards/margins": 8.210747718811035,
      "rewards/rejected": -10.562692642211914,
      "step": 1405
    }
  ],
  "logging_steps": 5,
  "max_steps": 1407,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}