{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9871794871794872,
  "eval_steps": 500,
  "global_step": 699,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.021367521367521368,
      "grad_norm": 16.125,
      "learning_rate": 1.7556090538745385e-06,
      "logits/chosen": -3.5723044872283936,
      "logits/rejected": -3.5033211708068848,
      "logps/chosen": -41.10147476196289,
      "logps/rejected": -79.84379577636719,
      "loss": 0.6923,
      "rewards/accuracies": 0.3291666507720947,
      "rewards/chosen": 0.0008107174071483314,
      "rewards/margins": 0.0017922676634043455,
      "rewards/rejected": -0.0009815500816330314,
      "step": 5
    },
    {
      "epoch": 0.042735042735042736,
      "grad_norm": 16.75,
      "learning_rate": 3.950120371217711e-06,
      "logits/chosen": -3.5865039825439453,
      "logits/rejected": -3.506371021270752,
      "logps/chosen": -40.01554870605469,
      "logps/rejected": -78.2536849975586,
      "loss": 0.679,
      "rewards/accuracies": 0.8291667103767395,
      "rewards/chosen": -0.0026048908475786448,
      "rewards/margins": 0.028653645887970924,
      "rewards/rejected": -0.031258534640073776,
      "step": 10
    },
    {
      "epoch": 0.0641025641025641,
      "grad_norm": 14.6875,
      "learning_rate": 6.144631688560886e-06,
      "logits/chosen": -3.5993950366973877,
      "logits/rejected": -3.5078125,
      "logps/chosen": -39.8319206237793,
      "logps/rejected": -81.44599151611328,
      "loss": 0.6296,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.006084696389734745,
      "rewards/margins": 0.1335393637418747,
      "rewards/rejected": -0.13962405920028687,
      "step": 15
    },
    {
      "epoch": 0.08547008547008547,
      "grad_norm": 13.75,
      "learning_rate": 8.339143005904057e-06,
      "logits/chosen": -3.5526351928710938,
      "logits/rejected": -3.4919040203094482,
      "logps/chosen": -39.764862060546875,
      "logps/rejected": -80.73335266113281,
      "loss": 0.5434,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.015795540064573288,
      "rewards/margins": 0.33912044763565063,
      "rewards/rejected": -0.3549160361289978,
      "step": 20
    },
    {
      "epoch": 0.10683760683760683,
      "grad_norm": 11.1875,
      "learning_rate": 1.0533654323247232e-05,
      "logits/chosen": -3.5133590698242188,
      "logits/rejected": -3.472569227218628,
      "logps/chosen": -40.0025749206543,
      "logps/rejected": -85.53997802734375,
      "loss": 0.3966,
      "rewards/accuracies": 0.9791666865348816,
      "rewards/chosen": -0.04189577326178551,
      "rewards/margins": 0.7760334014892578,
      "rewards/rejected": -0.8179291486740112,
      "step": 25
    },
    {
      "epoch": 0.1282051282051282,
      "grad_norm": 7.59375,
      "learning_rate": 1.2728165640590407e-05,
      "logits/chosen": -3.44189190864563,
      "logits/rejected": -3.4465813636779785,
      "logps/chosen": -42.446632385253906,
      "logps/rejected": -93.0277328491211,
      "loss": 0.2592,
      "rewards/accuracies": 0.9958332777023315,
      "rewards/chosen": -0.1323380172252655,
      "rewards/margins": 1.4258702993392944,
      "rewards/rejected": -1.5582085847854614,
      "step": 30
    },
    {
      "epoch": 0.14957264957264957,
      "grad_norm": 4.28125,
      "learning_rate": 1.4922676957933578e-05,
      "logits/chosen": -3.223677158355713,
      "logits/rejected": -3.2687485218048096,
      "logps/chosen": -44.17692947387695,
      "logps/rejected": -108.72818756103516,
      "loss": 0.1195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.43831247091293335,
      "rewards/margins": 2.755509853363037,
      "rewards/rejected": -3.1938223838806152,
      "step": 35
    },
    {
      "epoch": 0.17094017094017094,
      "grad_norm": 2.03125,
      "learning_rate": 1.5360556888469565e-05,
      "logits/chosen": -2.780639410018921,
      "logits/rejected": -2.814786911010742,
      "logps/chosen": -52.43251419067383,
      "logps/rejected": -143.02699279785156,
      "loss": 0.0425,
      "rewards/accuracies": 0.9916666746139526,
      "rewards/chosen": -1.2548245191574097,
      "rewards/margins": 5.2223920822143555,
      "rewards/rejected": -6.477217197418213,
      "step": 40
    },
    {
      "epoch": 0.19230769230769232,
      "grad_norm": 1.6953125,
      "learning_rate": 1.535640428282884e-05,
      "logits/chosen": -2.3407504558563232,
      "logits/rejected": -2.3020548820495605,
      "logps/chosen": -51.26860427856445,
      "logps/rejected": -161.84654235839844,
      "loss": 0.0168,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.2653954029083252,
      "rewards/margins": 7.260884761810303,
      "rewards/rejected": -8.526280403137207,
      "step": 45
    },
    {
      "epoch": 0.21367521367521367,
      "grad_norm": 9.3125,
      "learning_rate": 1.5349059809872097e-05,
      "logits/chosen": -1.9962679147720337,
      "logits/rejected": -1.9376245737075806,
      "logps/chosen": -57.276153564453125,
      "logps/rejected": -180.43075561523438,
      "loss": 0.0199,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.6868196725845337,
      "rewards/margins": 8.579647064208984,
      "rewards/rejected": -10.266467094421387,
      "step": 50
    },
    {
      "epoch": 0.23504273504273504,
      "grad_norm": 1.484375,
      "learning_rate": 1.5338527542732884e-05,
      "logits/chosen": -1.9884917736053467,
      "logits/rejected": -1.9206396341323853,
      "logps/chosen": -58.84391403198242,
      "logps/rejected": -196.67623901367188,
      "loss": 0.009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.8948497772216797,
      "rewards/margins": 9.876777648925781,
      "rewards/rejected": -11.771627426147461,
      "step": 55
    },
    {
      "epoch": 0.2564102564102564,
      "grad_norm": 5.25,
      "learning_rate": 1.532481332244717e-05,
      "logits/chosen": -1.9375540018081665,
      "logits/rejected": -1.8217322826385498,
      "logps/chosen": -57.795997619628906,
      "logps/rejected": -198.72848510742188,
      "loss": 0.0046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.8656346797943115,
      "rewards/margins": 10.204580307006836,
      "rewards/rejected": -12.070215225219727,
      "step": 60
    },
    {
      "epoch": 0.2777777777777778,
      "grad_norm": 16.125,
      "learning_rate": 1.5307924754713968e-05,
      "logits/chosen": -1.9060790538787842,
      "logits/rejected": -1.8332831859588623,
      "logps/chosen": -58.628753662109375,
      "logps/rejected": -194.50588989257812,
      "loss": 0.0115,
      "rewards/accuracies": 0.9916666746139526,
      "rewards/chosen": -1.797798752784729,
      "rewards/margins": 9.891462326049805,
      "rewards/rejected": -11.689262390136719,
      "step": 65
    },
    {
      "epoch": 0.29914529914529914,
      "grad_norm": 0.2470703125,
      "learning_rate": 1.528787120567736e-05,
      "logits/chosen": -1.974802017211914,
      "logits/rejected": -1.877560019493103,
      "logps/chosen": -52.279327392578125,
      "logps/rejected": -189.9191131591797,
      "loss": 0.0049,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2973945140838623,
      "rewards/margins": 10.00140380859375,
      "rewards/rejected": -11.298797607421875,
      "step": 70
    },
    {
      "epoch": 0.32051282051282054,
      "grad_norm": 0.640625,
      "learning_rate": 1.526466379673215e-05,
      "logits/chosen": -1.9735548496246338,
      "logits/rejected": -1.870365858078003,
      "logps/chosen": -53.06673049926758,
      "logps/rejected": -199.06517028808594,
      "loss": 0.0071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3975543975830078,
      "rewards/margins": 10.824033737182617,
      "rewards/rejected": -12.221589088439941,
      "step": 75
    },
    {
      "epoch": 0.3418803418803419,
      "grad_norm": 10.5,
      "learning_rate": 1.5238315398356126e-05,
      "logits/chosen": -1.9223344326019287,
      "logits/rejected": -1.7829482555389404,
      "logps/chosen": -58.65262985229492,
      "logps/rejected": -205.0301055908203,
      "loss": 0.0075,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.9159812927246094,
      "rewards/margins": 10.972046852111816,
      "rewards/rejected": -12.888028144836426,
      "step": 80
    },
    {
      "epoch": 0.36324786324786323,
      "grad_norm": 0.055419921875,
      "learning_rate": 1.5208840622972272e-05,
      "logits/chosen": -1.9187581539154053,
      "logits/rejected": -1.742376685142517,
      "logps/chosen": -60.388633728027344,
      "logps/rejected": -209.2400360107422,
      "loss": 0.0215,
      "rewards/accuracies": 0.9916666746139526,
      "rewards/chosen": -1.904120683670044,
      "rewards/margins": 11.195540428161621,
      "rewards/rejected": -13.099660873413086,
      "step": 85
    },
    {
      "epoch": 0.38461538461538464,
      "grad_norm": 0.97265625,
      "learning_rate": 1.5176255816844948e-05,
      "logits/chosen": -1.870398759841919,
      "logits/rejected": -1.6664142608642578,
      "logps/chosen": -52.21706008911133,
      "logps/rejected": -206.40274047851562,
      "loss": 0.0065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3197202682495117,
      "rewards/margins": 11.56867504119873,
      "rewards/rejected": -12.888395309448242,
      "step": 90
    },
    {
      "epoch": 0.405982905982906,
      "grad_norm": 0.609375,
      "learning_rate": 1.5140579051014502e-05,
      "logits/chosen": -1.8215786218643188,
      "logits/rejected": -1.587632179260254,
      "logps/chosen": -56.239166259765625,
      "logps/rejected": -221.23486328125,
      "loss": 0.0027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6193969249725342,
      "rewards/margins": 12.424324035644531,
      "rewards/rejected": -14.043721199035645,
      "step": 95
    },
    {
      "epoch": 0.42735042735042733,
      "grad_norm": 0.19140625,
      "learning_rate": 1.5101830111275334e-05,
      "logits/chosen": -1.761718511581421,
      "logits/rejected": -1.5578025579452515,
      "logps/chosen": -58.93622970581055,
      "logps/rejected": -211.36245727539062,
      "loss": 0.0134,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.9031003713607788,
      "rewards/margins": 11.43458366394043,
      "rewards/rejected": -13.337686538696289,
      "step": 100
    },
    {
      "epoch": 0.44871794871794873,
      "grad_norm": 5.25,
      "learning_rate": 1.5060030487203004e-05,
      "logits/chosen": -1.7170766592025757,
      "logits/rejected": -1.4669135808944702,
      "logps/chosen": -59.824798583984375,
      "logps/rejected": -224.89785766601562,
      "loss": 0.0058,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.9724791049957275,
      "rewards/margins": 12.608641624450684,
      "rewards/rejected": -14.581120491027832,
      "step": 105
    },
    {
      "epoch": 0.4700854700854701,
      "grad_norm": 0.06396484375,
      "learning_rate": 1.501520336023643e-05,
      "logits/chosen": -1.6807842254638672,
      "logits/rejected": -1.377798318862915,
      "logps/chosen": -60.823036193847656,
      "logps/rejected": -220.12026977539062,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0682151317596436,
      "rewards/margins": 12.238883972167969,
      "rewards/rejected": -14.307100296020508,
      "step": 110
    },
    {
      "epoch": 0.49145299145299143,
      "grad_norm": 1.8984375,
      "learning_rate": 1.4967373590821828e-05,
      "logits/chosen": -1.6894537210464478,
      "logits/rejected": -1.387304425239563,
      "logps/chosen": -70.60054779052734,
      "logps/rejected": -224.40921020507812,
      "loss": 0.0135,
      "rewards/accuracies": 0.9916666746139526,
      "rewards/chosen": -3.0002286434173584,
      "rewards/margins": 11.61396598815918,
      "rewards/rejected": -14.6141939163208,
      "step": 115
    },
    {
      "epoch": 0.5128205128205128,
      "grad_norm": 0.07080078125,
      "learning_rate": 1.491656770462546e-05,
      "logits/chosen": -1.549338936805725,
      "logits/rejected": -1.1871496438980103,
      "logps/chosen": -62.74003219604492,
      "logps/rejected": -226.72341918945312,
      "loss": 0.0031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.2881884574890137,
      "rewards/margins": 12.516637802124023,
      "rewards/rejected": -14.804829597473145,
      "step": 120
    },
    {
      "epoch": 0.5341880341880342,
      "grad_norm": 9.5625,
      "learning_rate": 1.4862813877822923e-05,
      "logits/chosen": -1.5509467124938965,
      "logits/rejected": -1.188998818397522,
      "logps/chosen": -58.65150833129883,
      "logps/rejected": -226.0467071533203,
      "loss": 0.0104,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.832358956336975,
      "rewards/margins": 13.06682300567627,
      "rewards/rejected": -14.89918327331543,
      "step": 125
    },
    {
      "epoch": 0.5555555555555556,
      "grad_norm": 0.1904296875,
      "learning_rate": 1.4806141921473063e-05,
      "logits/chosen": -1.520416498184204,
      "logits/rejected": -1.0859997272491455,
      "logps/chosen": -57.10784149169922,
      "logps/rejected": -232.3367156982422,
      "loss": 0.0046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6952478885650635,
      "rewards/margins": 13.670125961303711,
      "rewards/rejected": -15.365373611450195,
      "step": 130
    },
    {
      "epoch": 0.5769230769230769,
      "grad_norm": 0.0791015625,
      "learning_rate": 1.4746583264985202e-05,
      "logits/chosen": -1.515005350112915,
      "logits/rejected": -1.0654128789901733,
      "logps/chosen": -57.121559143066406,
      "logps/rejected": -238.92703247070312,
      "loss": 0.0058,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.67409348487854,
      "rewards/margins": 14.308405876159668,
      "rewards/rejected": -15.982501029968262,
      "step": 135
    },
    {
      "epoch": 0.5982905982905983,
      "grad_norm": 0.033203125,
      "learning_rate": 1.468417093868888e-05,
      "logits/chosen": -1.547180414199829,
      "logits/rejected": -1.1224654912948608,
      "logps/chosen": -55.69971466064453,
      "logps/rejected": -235.5997772216797,
      "loss": 0.004,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.6552823781967163,
      "rewards/margins": 14.057337760925293,
      "rewards/rejected": -15.712621688842773,
      "step": 140
    },
    {
      "epoch": 0.6196581196581197,
      "grad_norm": 0.09326171875,
      "learning_rate": 1.4618939555515721e-05,
      "logits/chosen": -1.4791333675384521,
      "logits/rejected": -1.0679481029510498,
      "logps/chosen": -60.25483322143555,
      "logps/rejected": -235.65640258789062,
      "loss": 0.0038,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.0124049186706543,
      "rewards/margins": 13.800189018249512,
      "rewards/rejected": -15.812593460083008,
      "step": 145
    },
    {
      "epoch": 0.6410256410256411,
      "grad_norm": 0.0203857421875,
      "learning_rate": 1.455092529180363e-05,
      "logits/chosen": -1.5142757892608643,
      "logits/rejected": -1.0721313953399658,
      "logps/chosen": -58.559837341308594,
      "logps/rejected": -238.7608184814453,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9865844249725342,
      "rewards/margins": 14.183688163757324,
      "rewards/rejected": -16.170270919799805,
      "step": 150
    },
    {
      "epoch": 0.6623931623931624,
      "grad_norm": 4.375,
      "learning_rate": 1.4480165867233946e-05,
      "logits/chosen": -1.5100654363632202,
      "logits/rejected": -1.0968310832977295,
      "logps/chosen": -63.598304748535156,
      "logps/rejected": -246.39779663085938,
      "loss": 0.0081,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.3885955810546875,
      "rewards/margins": 14.358621597290039,
      "rewards/rejected": -16.747217178344727,
      "step": 155
    },
    {
      "epoch": 0.6837606837606838,
      "grad_norm": 6.375,
      "learning_rate": 1.440670052391267e-05,
      "logits/chosen": -1.5425626039505005,
      "logits/rejected": -1.1030757427215576,
      "logps/chosen": -59.99821090698242,
      "logps/rejected": -236.77932739257812,
      "loss": 0.0057,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.146557569503784,
      "rewards/margins": 13.929641723632812,
      "rewards/rejected": -16.07619857788086,
      "step": 160
    },
    {
      "epoch": 0.7051282051282052,
      "grad_norm": 2.140625,
      "learning_rate": 1.4330570004607398e-05,
      "logits/chosen": -1.5993397235870361,
      "logits/rejected": -1.1524969339370728,
      "logps/chosen": -58.561241149902344,
      "logps/rejected": -247.20388793945312,
      "loss": 0.0033,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.9490623474121094,
      "rewards/margins": 14.975687026977539,
      "rewards/rejected": -16.924747467041016,
      "step": 165
    },
    {
      "epoch": 0.7264957264957265,
      "grad_norm": 1.0,
      "learning_rate": 1.4251816530151986e-05,
      "logits/chosen": -1.550431489944458,
      "logits/rejected": -1.151962399482727,
      "logps/chosen": -62.471397399902344,
      "logps/rejected": -246.4335479736328,
      "loss": 0.0038,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.2203402519226074,
      "rewards/margins": 14.611509323120117,
      "rewards/rejected": -16.831850051879883,
      "step": 170
    },
    {
      "epoch": 0.7478632478632479,
      "grad_norm": 0.10791015625,
      "learning_rate": 1.4170483776031526e-05,
      "logits/chosen": -1.5044711828231812,
      "logits/rejected": -1.0406298637390137,
      "logps/chosen": -64.19065856933594,
      "logps/rejected": -249.743408203125,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.455766201019287,
      "rewards/margins": 14.67004680633545,
      "rewards/rejected": -17.125812530517578,
      "step": 175
    },
    {
      "epoch": 0.7692307692307693,
      "grad_norm": 0.30078125,
      "learning_rate": 1.4086616848160574e-05,
      "logits/chosen": -1.512284278869629,
      "logits/rejected": -1.0526224374771118,
      "logps/chosen": -68.66128540039062,
      "logps/rejected": -245.33120727539062,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.8888022899627686,
      "rewards/margins": 13.867166519165039,
      "rewards/rejected": -16.75596809387207,
      "step": 180
    },
    {
      "epoch": 0.7905982905982906,
      "grad_norm": 0.1064453125,
      "learning_rate": 1.4000262257868096e-05,
      "logits/chosen": -1.442132830619812,
      "logits/rejected": -0.9388168454170227,
      "logps/chosen": -61.00693893432617,
      "logps/rejected": -247.60067749023438,
      "loss": 0.0022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1700937747955322,
      "rewards/margins": 14.933375358581543,
      "rewards/rejected": -17.103466033935547,
      "step": 185
    },
    {
      "epoch": 0.811965811965812,
      "grad_norm": 0.65625,
      "learning_rate": 1.3911467896102994e-05,
      "logits/chosen": -1.3841662406921387,
      "logits/rejected": -0.9196082353591919,
      "logps/chosen": -56.78447341918945,
      "logps/rejected": -246.0129852294922,
      "loss": 0.0049,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -1.7434440851211548,
      "rewards/margins": 15.205429077148438,
      "rewards/rejected": -16.94887351989746,
      "step": 190
    },
    {
      "epoch": 0.8333333333333334,
      "grad_norm": 0.047119140625,
      "learning_rate": 1.3820283006874503e-05,
      "logits/chosen": -1.36122727394104,
      "logits/rejected": -0.899361789226532,
      "logps/chosen": -64.0156478881836,
      "logps/rejected": -245.7220458984375,
      "loss": 0.0077,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.193249464035034,
      "rewards/margins": 14.599599838256836,
      "rewards/rejected": -16.792850494384766,
      "step": 195
    },
    {
      "epoch": 0.8547008547008547,
      "grad_norm": 0.220703125,
      "learning_rate": 1.372675815994221e-05,
      "logits/chosen": -1.2733592987060547,
      "logits/rejected": -0.7683244347572327,
      "logps/chosen": -54.458702087402344,
      "logps/rejected": -250.61141967773438,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6900008916854858,
      "rewards/margins": 15.71391773223877,
      "rewards/rejected": -17.403919219970703,
      "step": 200
    },
    {
      "epoch": 0.8760683760683761,
      "grad_norm": 41.75,
      "learning_rate": 1.3630945222770829e-05,
      "logits/chosen": -1.3039919137954712,
      "logits/rejected": -0.7636764645576477,
      "logps/chosen": -61.440521240234375,
      "logps/rejected": -256.41033935546875,
      "loss": 0.0309,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.126218795776367,
      "rewards/margins": 15.685290336608887,
      "rewards/rejected": -17.811508178710938,
      "step": 205
    },
    {
      "epoch": 0.8974358974358975,
      "grad_norm": 0.8046875,
      "learning_rate": 1.3532897331765301e-05,
      "logits/chosen": -1.339553713798523,
      "logits/rejected": -0.7716963291168213,
      "logps/chosen": -62.3395881652832,
      "logps/rejected": -254.98672485351562,
      "loss": 0.0111,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.235722064971924,
      "rewards/margins": 15.472686767578125,
      "rewards/rejected": -17.70840835571289,
      "step": 210
    },
    {
      "epoch": 0.9188034188034188,
      "grad_norm": 0.314453125,
      "learning_rate": 1.3432668862802134e-05,
      "logits/chosen": -1.256614089012146,
      "logits/rejected": -0.6890861988067627,
      "logps/chosen": -62.10491943359375,
      "logps/rejected": -249.8031005859375,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.166273593902588,
      "rewards/margins": 14.911642074584961,
      "rewards/rejected": -17.07791519165039,
      "step": 215
    },
    {
      "epoch": 0.9401709401709402,
      "grad_norm": 0.68359375,
      "learning_rate": 1.3330315401073371e-05,
      "logits/chosen": -1.2784340381622314,
      "logits/rejected": -0.7194468379020691,
      "logps/chosen": -64.89430236816406,
      "logps/rejected": -246.3287811279297,
      "loss": 0.0024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.396063804626465,
      "rewards/margins": 14.473241806030273,
      "rewards/rejected": -16.869304656982422,
      "step": 220
    },
    {
      "epoch": 0.9615384615384616,
      "grad_norm": 11.5625,
      "learning_rate": 1.3225893710259887e-05,
      "logits/chosen": -1.1662753820419312,
      "logits/rejected": -0.5432911515235901,
      "logps/chosen": -66.24633026123047,
      "logps/rejected": -253.39212036132812,
      "loss": 0.01,
      "rewards/accuracies": 0.9916666746139526,
      "rewards/chosen": -2.7053072452545166,
      "rewards/margins": 14.921958923339844,
      "rewards/rejected": -17.62726402282715,
      "step": 225
    },
    {
      "epoch": 0.9829059829059829,
      "grad_norm": 1.5859375,
      "learning_rate": 1.3119461701051105e-05,
      "logits/chosen": -1.161176323890686,
      "logits/rejected": -0.45327743887901306,
      "logps/chosen": -62.70893096923828,
      "logps/rejected": -251.8865966796875,
      "loss": 0.0009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.506155252456665,
      "rewards/margins": 15.214022636413574,
      "rewards/rejected": -17.720178604125977,
      "step": 230
    },
    {
      "epoch": 0.9957264957264957,
      "eval_logits/chosen": -1.1960320472717285,
      "eval_logits/rejected": -0.4940774440765381,
      "eval_logps/chosen": -66.11089324951172,
      "eval_logps/rejected": -261.5491943359375,
      "eval_loss": 0.0009878784185275435,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -2.6667160987854004,
      "eval_rewards/margins": 15.686513900756836,
      "eval_rewards/rejected": -18.353229522705078,
      "eval_runtime": 9.4343,
      "eval_samples_per_second": 21.199,
      "eval_steps_per_second": 21.199,
      "step": 233
    },
    {
      "epoch": 1.0042735042735043,
      "grad_norm": 0.11669921875,
      "learning_rate": 1.3011078399028605e-05,
      "logits/chosen": -1.1641021966934204,
      "logits/rejected": -0.48939043283462524,
      "logps/chosen": -65.07087707519531,
      "logps/rejected": -266.9398498535156,
      "loss": 0.0032,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -2.5494394302368164,
      "rewards/margins": 16.291004180908203,
      "rewards/rejected": -18.840442657470703,
      "step": 235
    },
    {
      "epoch": 1.0256410256410255,
      "grad_norm": 0.130859375,
      "learning_rate": 1.2900803911931431e-05,
      "logits/chosen": -1.1983628273010254,
      "logits/rejected": -0.48308929800987244,
      "logps/chosen": -64.48258209228516,
      "logps/rejected": -265.9141540527344,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.472846746444702,
      "rewards/margins": 16.26525115966797,
      "rewards/rejected": -18.73809814453125,
      "step": 240
    },
    {
      "epoch": 1.047008547008547,
      "grad_norm": 0.052978515625,
      "learning_rate": 1.2788699396321252e-05,
      "logits/chosen": -1.1283172369003296,
      "logits/rejected": -0.419720321893692,
      "logps/chosen": -62.42974853515625,
      "logps/rejected": -256.7882385253906,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.3851864337921143,
      "rewards/margins": 15.727444648742676,
      "rewards/rejected": -18.11263084411621,
      "step": 245
    },
    {
      "epoch": 1.0683760683760684,
      "grad_norm": 0.000896453857421875,
      "learning_rate": 1.2674827023665853e-05,
      "logits/chosen": -1.1639864444732666,
      "logits/rejected": -0.4902980327606201,
      "logps/chosen": -67.04655456542969,
      "logps/rejected": -274.08026123046875,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.747849941253662,
      "rewards/margins": 16.89676284790039,
      "rewards/rejected": -19.64461326599121,
      "step": 250
    },
    {
      "epoch": 1.0897435897435896,
      "grad_norm": 0.208984375,
      "learning_rate": 1.255924994585978e-05,
      "logits/chosen": -1.1226640939712524,
      "logits/rejected": -0.4087928235530853,
      "logps/chosen": -69.7280044555664,
      "logps/rejected": -270.5373840332031,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.9504196643829346,
      "rewards/margins": 16.320491790771484,
      "rewards/rejected": -19.27090835571289,
      "step": 255
    },
    {
      "epoch": 1.1111111111111112,
      "grad_norm": 0.00885009765625,
      "learning_rate": 1.2442032260201255e-05,
      "logits/chosen": -1.1412135362625122,
      "logits/rejected": -0.4527861475944519,
      "logps/chosen": -69.53722381591797,
      "logps/rejected": -270.5766296386719,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.999143123626709,
      "rewards/margins": 16.23387336730957,
      "rewards/rejected": -19.233016967773438,
      "step": 260
    },
    {
      "epoch": 1.1324786324786325,
      "grad_norm": 0.263671875,
      "learning_rate": 1.2323238973844796e-05,
      "logits/chosen": -1.1780011653900146,
      "logits/rejected": -0.45843830704689026,
      "logps/chosen": -71.02696228027344,
      "logps/rejected": -280.1233825683594,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0659804344177246,
      "rewards/margins": 16.907106399536133,
      "rewards/rejected": -19.973085403442383,
      "step": 265
    },
    {
      "epoch": 1.1538461538461537,
      "grad_norm": 0.02197265625,
      "learning_rate": 1.2202935967749212e-05,
      "logits/chosen": -1.0917080640792847,
      "logits/rejected": -0.38812121748924255,
      "logps/chosen": -74.89707946777344,
      "logps/rejected": -274.8170166015625,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.3719239234924316,
      "rewards/margins": 16.13779067993164,
      "rewards/rejected": -19.509714126586914,
      "step": 270
    },
    {
      "epoch": 1.1752136752136753,
      "grad_norm": 0.72265625,
      "learning_rate": 1.2081189960141038e-05,
      "logits/chosen": -1.1228703260421753,
      "logits/rejected": -0.39134496450424194,
      "logps/chosen": -69.5420913696289,
      "logps/rejected": -274.7674255371094,
      "loss": 0.0015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.9194247722625732,
      "rewards/margins": 16.717750549316406,
      "rewards/rejected": -19.637174606323242,
      "step": 275
    },
    {
      "epoch": 1.1965811965811965,
      "grad_norm": 1.171875,
      "learning_rate": 1.1958068469513604e-05,
      "logits/chosen": -1.1384648084640503,
      "logits/rejected": -0.39668112993240356,
      "logps/chosen": -72.6049575805664,
      "logps/rejected": -283.6707458496094,
      "loss": 0.0092,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -3.210206985473633,
      "rewards/margins": 17.243507385253906,
      "rewards/rejected": -20.453712463378906,
      "step": 280
    },
    {
      "epoch": 1.217948717948718,
      "grad_norm": 0.013916015625,
      "learning_rate": 1.1833639777182316e-05,
      "logits/chosen": -1.0187714099884033,
      "logits/rejected": -0.2573033571243286,
      "logps/chosen": -70.54331970214844,
      "logps/rejected": -280.8934020996094,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.204702377319336,
      "rewards/margins": 17.17652702331543,
      "rewards/rejected": -20.3812313079834,
      "step": 285
    },
    {
      "epoch": 1.2393162393162394,
      "grad_norm": 0.012451171875,
      "learning_rate": 1.170797288941685e-05,
      "logits/chosen": -0.9472154378890991,
      "logits/rejected": -0.2180492877960205,
      "logps/chosen": -79.2474594116211,
      "logps/rejected": -289.6960144042969,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.919898271560669,
      "rewards/margins": 17.01470947265625,
      "rewards/rejected": -20.934606552124023,
      "step": 290
    },
    {
      "epoch": 1.2606837606837606,
      "grad_norm": 0.3359375,
      "learning_rate": 1.1581137499171342e-05,
      "logits/chosen": -0.9705740213394165,
      "logits/rejected": -0.21009401977062225,
      "logps/chosen": -79.57368469238281,
      "logps/rejected": -278.9507751464844,
      "loss": 0.0025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.9748988151550293,
      "rewards/margins": 16.132137298583984,
      "rewards/rejected": -20.10703468322754,
      "step": 295
    },
    {
      "epoch": 1.282051282051282,
      "grad_norm": 0.017822265625,
      "learning_rate": 1.145320394743371e-05,
      "logits/chosen": -0.9654294848442078,
      "logits/rejected": -0.22467419505119324,
      "logps/chosen": -74.53304290771484,
      "logps/rejected": -270.533447265625,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.5043253898620605,
      "rewards/margins": 15.963613510131836,
      "rewards/rejected": -19.467941284179688,
      "step": 300
    },
    {
      "epoch": 1.3034188034188035,
      "grad_norm": 0.0078125,
      "learning_rate": 1.1324243184215622e-05,
      "logits/chosen": -0.969558835029602,
      "logits/rejected": -0.21970291435718536,
      "logps/chosen": -73.47796630859375,
      "logps/rejected": -285.55706787109375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.5001537799835205,
      "rewards/margins": 17.251827239990234,
      "rewards/rejected": -20.75197982788086,
      "step": 305
    },
    {
      "epoch": 1.3247863247863247,
      "grad_norm": 0.0084228515625,
      "learning_rate": 1.1194326729204686e-05,
      "logits/chosen": -1.0194365978240967,
      "logits/rejected": -0.25868645310401917,
      "logps/chosen": -73.23660278320312,
      "logps/rejected": -280.81256103515625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4368844032287598,
      "rewards/margins": 16.979907989501953,
      "rewards/rejected": -20.416793823242188,
      "step": 310
    },
    {
      "epoch": 1.3461538461538463,
      "grad_norm": 0.00933837890625,
      "learning_rate": 1.1063526632100717e-05,
      "logits/chosen": -1.0158953666687012,
      "logits/rejected": -0.3026788830757141,
      "logps/chosen": -75.67848205566406,
      "logps/rejected": -277.49310302734375,
      "loss": 0.0025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.546459197998047,
      "rewards/margins": 16.51190948486328,
      "rewards/rejected": -20.058368682861328,
      "step": 315
    },
    {
      "epoch": 1.3675213675213675,
      "grad_norm": 0.005615234375,
      "learning_rate": 1.0931915432658055e-05,
      "logits/chosen": -1.023720383644104,
      "logits/rejected": -0.28601619601249695,
      "logps/chosen": -71.98751068115234,
      "logps/rejected": -279.75103759765625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2338900566101074,
      "rewards/margins": 16.945287704467773,
      "rewards/rejected": -20.17917823791504,
      "step": 320
    },
    {
      "epoch": 1.3888888888888888,
      "grad_norm": 0.09130859375,
      "learning_rate": 1.0799566120456133e-05,
      "logits/chosen": -1.0158692598342896,
      "logits/rejected": -0.25889790058135986,
      "logps/chosen": -74.53166198730469,
      "logps/rejected": -277.46502685546875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4598422050476074,
      "rewards/margins": 16.516708374023438,
      "rewards/rejected": -19.976551055908203,
      "step": 325
    },
    {
      "epoch": 1.4102564102564101,
      "grad_norm": 0.0211181640625,
      "learning_rate": 1.066655209442054e-05,
      "logits/chosen": -1.0357223749160767,
      "logits/rejected": -0.3092297315597534,
      "logps/chosen": -74.45147705078125,
      "logps/rejected": -280.28125,
      "loss": 0.0027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.444270372390747,
      "rewards/margins": 16.66710090637207,
      "rewards/rejected": -20.111371994018555,
      "step": 330
    },
    {
      "epoch": 1.4316239316239316,
      "grad_norm": 0.033447265625,
      "learning_rate": 1.0532947122117101e-05,
      "logits/chosen": -1.0367907285690308,
      "logits/rejected": -0.30450788140296936,
      "logps/chosen": -74.00047302246094,
      "logps/rejected": -280.2533264160156,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4167351722717285,
      "rewards/margins": 16.851306915283203,
      "rewards/rejected": -20.268043518066406,
      "step": 335
    },
    {
      "epoch": 1.452991452991453,
      "grad_norm": 0.05126953125,
      "learning_rate": 1.0398825298841499e-05,
      "logits/chosen": -1.0600817203521729,
      "logits/rejected": -0.41858386993408203,
      "logps/chosen": -77.15374755859375,
      "logps/rejected": -283.68585205078125,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.601487636566162,
      "rewards/margins": 16.883991241455078,
      "rewards/rejected": -20.4854793548584,
      "step": 340
    },
    {
      "epoch": 1.4743589743589745,
      "grad_norm": 0.0087890625,
      "learning_rate": 1.0264261006527144e-05,
      "logits/chosen": -1.1149675846099854,
      "logits/rejected": -0.4605252742767334,
      "logps/chosen": -73.10884094238281,
      "logps/rejected": -273.56329345703125,
      "loss": 0.0043,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -3.302705764770508,
      "rewards/margins": 16.43352508544922,
      "rewards/rejected": -19.73623275756836,
      "step": 345
    },
    {
      "epoch": 1.4957264957264957,
      "grad_norm": 0.056396484375,
      "learning_rate": 1.0129328872494075e-05,
      "logits/chosen": -1.2343705892562866,
      "logits/rejected": -0.5823469161987305,
      "logps/chosen": -70.9728775024414,
      "logps/rejected": -277.93389892578125,
      "loss": 0.0029,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -3.130044460296631,
      "rewards/margins": 16.93770980834961,
      "rewards/rejected": -20.067752838134766,
      "step": 350
    },
    {
      "epoch": 1.517094017094017,
      "grad_norm": 0.166015625,
      "learning_rate": 9.994103728061786e-06,
      "logits/chosen": -1.1977897882461548,
      "logits/rejected": -0.5562113523483276,
      "logps/chosen": -70.82041931152344,
      "logps/rejected": -274.3385009765625,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.062682867050171,
      "rewards/margins": 16.56722640991211,
      "rewards/rejected": -19.62990951538086,
      "step": 355
    },
    {
      "epoch": 1.5384615384615383,
      "grad_norm": 0.00537109375,
      "learning_rate": 9.858660567048902e-06,
      "logits/chosen": -1.2067844867706299,
      "logits/rejected": -0.5067782998085022,
      "logps/chosen": -74.36605834960938,
      "logps/rejected": -289.45306396484375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4267430305480957,
      "rewards/margins": 17.64672088623047,
      "rewards/rejected": -21.073461532592773,
      "step": 360
    },
    {
      "epoch": 1.5598290598290598,
      "grad_norm": 0.10302734375,
      "learning_rate": 9.72307450418274e-06,
      "logits/chosen": -1.2308647632598877,
      "logits/rejected": -0.5679563283920288,
      "logps/chosen": -69.85784912109375,
      "logps/rejected": -274.14263916015625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.9802072048187256,
      "rewards/margins": 16.728424072265625,
      "rewards/rejected": -19.708629608154297,
      "step": 365
    },
    {
      "epoch": 1.5811965811965814,
      "grad_norm": 0.041015625,
      "learning_rate": 9.587420733441835e-06,
      "logits/chosen": -1.227565050125122,
      "logits/rejected": -0.5805907249450684,
      "logps/chosen": -68.70391845703125,
      "logps/rejected": -279.85845947265625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.8893535137176514,
      "rewards/margins": 17.282146453857422,
      "rewards/rejected": -20.171499252319336,
      "step": 370
    },
    {
      "epoch": 1.6025641025641026,
      "grad_norm": 0.000698089599609375,
      "learning_rate": 9.45177448635447e-06,
      "logits/chosen": -1.251037836074829,
      "logits/rejected": -0.515083372592926,
      "logps/chosen": -68.63763427734375,
      "logps/rejected": -280.8656921386719,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.9085798263549805,
      "rewards/margins": 17.550891876220703,
      "rewards/rejected": -20.459474563598633,
      "step": 375
    },
    {
      "epoch": 1.623931623931624,
      "grad_norm": 0.00811767578125,
      "learning_rate": 9.316210990276434e-06,
      "logits/chosen": -1.174579381942749,
      "logits/rejected": -0.5305734276771545,
      "logps/chosen": -68.64695739746094,
      "logps/rejected": -270.04510498046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.869532346725464,
      "rewards/margins": 16.443248748779297,
      "rewards/rejected": -19.31278419494629,
      "step": 380
    },
    {
      "epoch": 1.6452991452991452,
      "grad_norm": 0.002349853515625,
      "learning_rate": 9.18080542667105e-06,
      "logits/chosen": -1.2183802127838135,
      "logits/rejected": -0.5133947134017944,
      "logps/chosen": -69.98764038085938,
      "logps/rejected": -290.1512756347656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.9592034816741943,
      "rewards/margins": 18.04782485961914,
      "rewards/rejected": -21.007028579711914,
      "step": 385
    },
    {
      "epoch": 1.6666666666666665,
      "grad_norm": 0.01348876953125,
      "learning_rate": 9.045632889414686e-06,
      "logits/chosen": -1.2081528902053833,
      "logits/rejected": -0.5276485681533813,
      "logps/chosen": -68.10844421386719,
      "logps/rejected": -281.4985046386719,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.8550543785095215,
      "rewards/margins": 17.489383697509766,
      "rewards/rejected": -20.344438552856445,
      "step": 390
    },
    {
      "epoch": 1.688034188034188,
      "grad_norm": 0.00848388671875,
      "learning_rate": 8.910768343150828e-06,
      "logits/chosen": -1.1923013925552368,
      "logits/rejected": -0.5268815755844116,
      "logps/chosen": -69.50736999511719,
      "logps/rejected": -282.33880615234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0535500049591064,
      "rewards/margins": 17.47011375427246,
      "rewards/rejected": -20.523662567138672,
      "step": 395
    },
    {
      "epoch": 1.7094017094017095,
      "grad_norm": 0.001434326171875,
      "learning_rate": 8.77628658171581e-06,
      "logits/chosen": -1.2103922367095947,
      "logits/rejected": -0.5478588342666626,
      "logps/chosen": -69.14801025390625,
      "logps/rejected": -277.53424072265625,
      "loss": 0.006,
      "rewards/accuracies": 0.9958332777023315,
      "rewards/chosen": -2.929593563079834,
      "rewards/margins": 16.97675895690918,
      "rewards/rejected": -19.906352996826172,
      "step": 400
    },
    {
      "epoch": 1.7307692307692308,
      "grad_norm": 0.091796875,
      "learning_rate": 8.642262186659298e-06,
      "logits/chosen": -1.1698075532913208,
      "logits/rejected": -0.520237147808075,
      "logps/chosen": -69.62812805175781,
      "logps/rejected": -279.01776123046875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.9473519325256348,
      "rewards/margins": 17.070568084716797,
      "rewards/rejected": -20.01791763305664,
      "step": 405
    },
    {
      "epoch": 1.7521367521367521,
      "grad_norm": 0.004058837890625,
      "learning_rate": 8.508769485882487e-06,
      "logits/chosen": -1.2100353240966797,
      "logits/rejected": -0.566001296043396,
      "logps/chosen": -73.57793426513672,
      "logps/rejected": -278.8910217285156,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.203927516937256,
      "rewards/margins": 16.867618560791016,
      "rewards/rejected": -20.07154655456543,
      "step": 410
    },
    {
      "epoch": 1.7735042735042734,
      "grad_norm": 0.78515625,
      "learning_rate": 8.375882512416969e-06,
      "logits/chosen": -1.1643860340118408,
      "logits/rejected": -0.4784732758998871,
      "logps/chosen": -72.14900207519531,
      "logps/rejected": -282.3783264160156,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.1646692752838135,
      "rewards/margins": 17.262126922607422,
      "rewards/rejected": -20.426794052124023,
      "step": 415
    },
    {
      "epoch": 1.7948717948717947,
      "grad_norm": 0.0245361328125,
      "learning_rate": 8.243674963367137e-06,
      "logits/chosen": -1.170971393585205,
      "logits/rejected": -0.4777548909187317,
      "logps/chosen": -74.64592742919922,
      "logps/rejected": -281.1402587890625,
      "loss": 0.0031,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -3.446406602859497,
      "rewards/margins": 16.716421127319336,
      "rewards/rejected": -20.162830352783203,
      "step": 420
    },
    {
      "epoch": 1.8162393162393162,
      "grad_norm": 0.00127410888671875,
      "learning_rate": 8.11222015903888e-06,
      "logits/chosen": -1.1845993995666504,
      "logits/rejected": -0.47238603234291077,
      "logps/chosen": -75.17137145996094,
      "logps/rejected": -286.2430725097656,
      "loss": 0.0046,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -3.5714709758758545,
      "rewards/margins": 17.298736572265625,
      "rewards/rejected": -20.87020492553711,
      "step": 425
    },
    {
      "epoch": 1.8376068376068377,
      "grad_norm": 0.0174560546875,
      "learning_rate": 7.981591002277265e-06,
      "logits/chosen": -1.1712948083877563,
      "logits/rejected": -0.4820891320705414,
      "logps/chosen": -74.11454010009766,
      "logps/rejected": -281.625732421875,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.407923460006714,
      "rewards/margins": 17.11835479736328,
      "rewards/rejected": -20.526281356811523,
      "step": 430
    },
    {
      "epoch": 1.858974358974359,
      "grad_norm": 0.01104736328125,
      "learning_rate": 7.851859938035712e-06,
      "logits/chosen": -1.149280309677124,
      "logits/rejected": -0.46685323119163513,
      "logps/chosen": -75.06929016113281,
      "logps/rejected": -291.274169921875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.454303026199341,
      "rewards/margins": 17.817852020263672,
      "rewards/rejected": -21.272151947021484,
      "step": 435
    },
    {
      "epoch": 1.8803418803418803,
      "grad_norm": 0.0277099609375,
      "learning_rate": 7.723098913199118e-06,
      "logits/chosen": -1.1872258186340332,
      "logits/rejected": -0.4977366328239441,
      "logps/chosen": -72.54876708984375,
      "logps/rejected": -279.2344665527344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.3534111976623535,
      "rewards/margins": 17.049516677856445,
      "rewards/rejected": -20.402929306030273,
      "step": 440
    },
    {
      "epoch": 1.9017094017094016,
      "grad_norm": 0.01104736328125,
      "learning_rate": 7.595379336683204e-06,
      "logits/chosen": -1.1503307819366455,
      "logits/rejected": -0.40190115571022034,
      "logps/chosen": -70.95316314697266,
      "logps/rejected": -287.2573547363281,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2571163177490234,
      "rewards/margins": 17.867298126220703,
      "rewards/rejected": -21.124412536621094,
      "step": 445
    },
    {
      "epoch": 1.9230769230769231,
      "grad_norm": 0.00060272216796875,
      "learning_rate": 7.468772039832218e-06,
      "logits/chosen": -1.1172707080841064,
      "logits/rejected": -0.3832516074180603,
      "logps/chosen": -66.7195816040039,
      "logps/rejected": -281.6941833496094,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.839646816253662,
      "rewards/margins": 17.725854873657227,
      "rewards/rejected": -20.56549835205078,
      "step": 450
    },
    {
      "epoch": 1.9444444444444444,
      "grad_norm": 0.0028839111328125,
      "learning_rate": 7.3433472371369404e-06,
      "logits/chosen": -1.1930923461914062,
      "logits/rejected": -0.48473644256591797,
      "logps/chosen": -72.68934631347656,
      "logps/rejected": -284.6398010253906,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.318150281906128,
      "rewards/margins": 17.357219696044922,
      "rewards/rejected": -20.675371170043945,
      "step": 455
    },
    {
      "epoch": 1.965811965811966,
      "grad_norm": 0.000492095947265625,
      "learning_rate": 7.219174487294784e-06,
      "logits/chosen": -1.1722862720489502,
      "logits/rejected": -0.47326725721359253,
      "logps/chosen": -70.49462890625,
      "logps/rejected": -287.0985107421875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0771074295043945,
      "rewards/margins": 17.8206844329834,
      "rewards/rejected": -20.89779281616211,
      "step": 460
    },
    {
      "epoch": 1.9871794871794872,
      "grad_norm": 0.00872802734375,
      "learning_rate": 7.0963226546336e-06,
      "logits/chosen": -1.20412278175354,
      "logits/rejected": -0.5105618238449097,
      "logps/chosen": -70.91376495361328,
      "logps/rejected": -280.29852294921875,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.1506786346435547,
      "rewards/margins": 17.16250228881836,
      "rewards/rejected": -20.313182830810547,
      "step": 465
    },
    {
      "epoch": 1.9914529914529915,
      "eval_logits/chosen": -1.2244133949279785,
      "eval_logits/rejected": -0.5135009288787842,
      "eval_logps/chosen": -72.20535278320312,
      "eval_logps/rejected": -285.3635559082031,
      "eval_loss": 4.7980323870433494e-05,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -3.276163101196289,
      "eval_rewards/margins": 17.45850372314453,
      "eval_rewards/rejected": -20.734668731689453,
      "eval_runtime": 9.4345,
      "eval_samples_per_second": 21.199,
      "eval_steps_per_second": 21.199,
      "step": 466
    },
    {
      "epoch": 2.0085470085470085,
      "grad_norm": 0.024169921875,
      "learning_rate": 6.974859870920561e-06,
      "logits/chosen": -1.1253283023834229,
      "logits/rejected": -0.4625110626220703,
      "logps/chosen": -72.798095703125,
      "logps/rejected": -279.7697448730469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.3227851390838623,
      "rewards/margins": 17.012950897216797,
      "rewards/rejected": -20.335737228393555,
      "step": 470
    },
    {
      "epoch": 2.02991452991453,
      "grad_norm": 0.00131988525390625,
      "learning_rate": 6.8548534975773135e-06,
      "logits/chosen": -1.179386854171753,
      "logits/rejected": -0.49411916732788086,
      "logps/chosen": -73.65827178955078,
      "logps/rejected": -285.2862243652344,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.295319080352783,
      "rewards/margins": 17.375682830810547,
      "rewards/rejected": -20.671003341674805,
      "step": 475
    },
    {
      "epoch": 2.051282051282051,
      "grad_norm": 0.189453125,
      "learning_rate": 6.736370088322359e-06,
      "logits/chosen": -1.1767910718917847,
      "logits/rejected": -0.4558785557746887,
      "logps/chosen": -72.45469665527344,
      "logps/rejected": -282.46148681640625,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.309537410736084,
      "rewards/margins": 17.340744018554688,
      "rewards/rejected": -20.65028190612793,
      "step": 480
    },
    {
      "epoch": 2.072649572649573,
      "grad_norm": 0.016357421875,
      "learning_rate": 6.619475352261356e-06,
      "logits/chosen": -1.1392714977264404,
      "logits/rejected": -0.45718201994895935,
      "logps/chosen": -76.55453491210938,
      "logps/rejected": -285.7558898925781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4542293548583984,
      "rewards/margins": 17.28127670288086,
      "rewards/rejected": -20.73550796508789,
      "step": 485
    },
    {
      "epoch": 2.094017094017094,
      "grad_norm": 0.0069580078125,
      "learning_rate": 6.504234117445857e-06,
      "logits/chosen": -1.1605439186096191,
      "logits/rejected": -0.4634723663330078,
      "logps/chosen": -72.77029418945312,
      "logps/rejected": -282.7125549316406,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.3726367950439453,
      "rewards/margins": 17.24945068359375,
      "rewards/rejected": -20.622087478637695,
      "step": 490
    },
    {
      "epoch": 2.1153846153846154,
      "grad_norm": 0.007354736328125,
      "learning_rate": 6.39071029492065e-06,
      "logits/chosen": -1.1251627206802368,
      "logits/rejected": -0.4109951853752136,
      "logps/chosen": -72.30722045898438,
      "logps/rejected": -281.81280517578125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.365689754486084,
      "rewards/margins": 17.244686126708984,
      "rewards/rejected": -20.610374450683594,
      "step": 495
    },
    {
      "epoch": 2.1367521367521367,
      "grad_norm": 0.13671875,
      "learning_rate": 6.2789668432796535e-06,
      "logits/chosen": -1.1276848316192627,
      "logits/rejected": -0.4581735134124756,
      "logps/chosen": -73.38197326660156,
      "logps/rejected": -283.4115295410156,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4196181297302246,
      "rewards/margins": 17.356618881225586,
      "rewards/rejected": -20.7762393951416,
      "step": 500
    },
    {
      "epoch": 2.1367521367521367,
      "eval_logits/chosen": -1.2213764190673828,
      "eval_logits/rejected": -0.508858859539032,
      "eval_logps/chosen": -72.31729125976562,
      "eval_logps/rejected": -285.5926818847656,
      "eval_loss": 4.8542991862632334e-05,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -3.2873549461364746,
      "eval_rewards/margins": 17.470226287841797,
      "eval_rewards/rejected": -20.757583618164062,
      "eval_runtime": 9.4725,
      "eval_samples_per_second": 21.114,
      "eval_steps_per_second": 21.114,
      "step": 500
    },
    {
      "epoch": 2.158119658119658,
      "grad_norm": 0.006744384765625,
      "learning_rate": 6.16906573375004e-06,
      "logits/chosen": -1.1647155284881592,
      "logits/rejected": -0.5106357336044312,
      "logps/chosen": -72.9385986328125,
      "logps/rejected": -281.58062744140625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.3363139629364014,
      "rewards/margins": 17.115955352783203,
      "rewards/rejected": -20.452272415161133,
      "step": 505
    },
    {
      "epoch": 2.1794871794871793,
      "grad_norm": 0.033203125,
      "learning_rate": 6.061067915823923e-06,
      "logits/chosen": -1.102561593055725,
      "logits/rejected": -0.4104360044002533,
      "logps/chosen": -70.00373077392578,
      "logps/rejected": -283.4765319824219,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.082702159881592,
      "rewards/margins": 17.501201629638672,
      "rewards/rejected": -20.583904266357422,
      "step": 510
    },
    {
      "epoch": 2.200854700854701,
      "grad_norm": 0.04150390625,
      "learning_rate": 5.955033283456711e-06,
      "logits/chosen": -1.136200189590454,
      "logits/rejected": -0.4127614498138428,
      "logps/chosen": -77.00238037109375,
      "logps/rejected": -293.89617919921875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.632338762283325,
      "rewards/margins": 17.84152603149414,
      "rewards/rejected": -21.473865509033203,
      "step": 515
    },
    {
      "epoch": 2.2222222222222223,
      "grad_norm": 0.00136566162109375,
      "learning_rate": 5.8510206418507914e-06,
      "logits/chosen": -1.2033765316009521,
      "logits/rejected": -0.5239652395248413,
      "logps/chosen": -74.33930969238281,
      "logps/rejected": -299.2998352050781,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.386591672897339,
      "rewards/margins": 18.531057357788086,
      "rewards/rejected": -21.91765022277832,
      "step": 520
    },
    {
      "epoch": 2.2435897435897436,
      "grad_norm": 0.003875732421875,
      "learning_rate": 5.749087674843095e-06,
      "logits/chosen": -1.139147400856018,
      "logits/rejected": -0.46916326880455017,
      "logps/chosen": -68.5783920288086,
      "logps/rejected": -284.41552734375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0137269496917725,
      "rewards/margins": 17.694652557373047,
      "rewards/rejected": -20.708377838134766,
      "step": 525
    },
    {
      "epoch": 2.264957264957265,
      "grad_norm": 0.01904296875,
      "learning_rate": 5.649290912914482e-06,
      "logits/chosen": -1.1451623439788818,
      "logits/rejected": -0.47735461592674255,
      "logps/chosen": -77.09215545654297,
      "logps/rejected": -298.22796630859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.572205066680908,
      "rewards/margins": 18.168258666992188,
      "rewards/rejected": -21.740463256835938,
      "step": 530
    },
    {
      "epoch": 2.286324786324786,
      "grad_norm": 0.036865234375,
      "learning_rate": 5.5516857018388144e-06,
      "logits/chosen": -1.187374472618103,
      "logits/rejected": -0.5284038782119751,
      "logps/chosen": -71.90570831298828,
      "logps/rejected": -280.32940673828125,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.206554412841797,
      "rewards/margins": 17.105445861816406,
      "rewards/rejected": -20.312000274658203,
      "step": 535
    },
    {
      "epoch": 2.3076923076923075,
      "grad_norm": 0.00921630859375,
      "learning_rate": 5.456326171989005e-06,
      "logits/chosen": -1.15079927444458,
      "logits/rejected": -0.4923780858516693,
      "logps/chosen": -70.75343322753906,
      "logps/rejected": -297.9629821777344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.109229564666748,
      "rewards/margins": 18.49727439880371,
      "rewards/rejected": -21.60650634765625,
      "step": 540
    },
    {
      "epoch": 2.3290598290598292,
      "grad_norm": 0.00830078125,
      "learning_rate": 5.363265208317156e-06,
      "logits/chosen": -1.1182730197906494,
      "logits/rejected": -0.4831443727016449,
      "logps/chosen": -71.95735931396484,
      "logps/rejected": -281.10089111328125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.26196551322937,
      "rewards/margins": 17.25033950805664,
      "rewards/rejected": -20.512304306030273,
      "step": 545
    },
    {
      "epoch": 2.3504273504273505,
      "grad_norm": 0.006072998046875,
      "learning_rate": 5.272554421025347e-06,
      "logits/chosen": -1.1618000268936157,
      "logits/rejected": -0.45643243193626404,
      "logps/chosen": -72.98902893066406,
      "logps/rejected": -289.97998046875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.336233139038086,
      "rewards/margins": 17.833538055419922,
      "rewards/rejected": -21.169771194458008,
      "step": 550
    },
    {
      "epoch": 2.371794871794872,
      "grad_norm": 0.008544921875,
      "learning_rate": 5.184244116943411e-06,
      "logits/chosen": -1.151729941368103,
      "logits/rejected": -0.4680960774421692,
      "logps/chosen": -72.05565643310547,
      "logps/rejected": -286.6762390136719,
      "loss": 0.003,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -3.189112901687622,
      "rewards/margins": 17.703588485717773,
      "rewards/rejected": -20.892702102661133,
      "step": 555
    },
    {
      "epoch": 2.393162393162393,
      "grad_norm": 0.00457763671875,
      "learning_rate": 5.098383271629512e-06,
      "logits/chosen": -1.169447660446167,
      "logits/rejected": -0.4685635566711426,
      "logps/chosen": -72.01811218261719,
      "logps/rejected": -279.0551452636719,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2659707069396973,
      "rewards/margins": 17.044937133789062,
      "rewards/rejected": -20.3109073638916,
      "step": 560
    },
    {
      "epoch": 2.4145299145299144,
      "grad_norm": 0.002227783203125,
      "learning_rate": 5.015019502209056e-06,
      "logits/chosen": -1.1631286144256592,
      "logits/rejected": -0.46378326416015625,
      "logps/chosen": -70.7663803100586,
      "logps/rejected": -276.07537841796875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2191970348358154,
      "rewards/margins": 16.797992706298828,
      "rewards/rejected": -20.017189025878906,
      "step": 565
    },
    {
      "epoch": 2.435897435897436,
      "grad_norm": 0.000713348388671875,
      "learning_rate": 4.934199040966955e-06,
      "logits/chosen": -1.185304045677185,
      "logits/rejected": -0.4863820970058441,
      "logps/chosen": -71.80037689208984,
      "logps/rejected": -280.21148681640625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2629001140594482,
      "rewards/margins": 17.172740936279297,
      "rewards/rejected": -20.435644149780273,
      "step": 570
    },
    {
      "epoch": 2.4572649572649574,
      "grad_norm": 0.00372314453125,
      "learning_rate": 4.855966709707881e-06,
      "logits/chosen": -1.1501516103744507,
      "logits/rejected": -0.5059275031089783,
      "logps/chosen": -75.91123962402344,
      "logps/rejected": -286.08929443359375,
      "loss": 0.0029,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -3.5484020709991455,
      "rewards/margins": 17.334753036499023,
      "rewards/rejected": -20.88315773010254,
      "step": 575
    },
    {
      "epoch": 2.4786324786324787,
      "grad_norm": 0.01190185546875,
      "learning_rate": 4.780365894898799e-06,
      "logits/chosen": -1.1519519090652466,
      "logits/rejected": -0.47328823804855347,
      "logps/chosen": -73.0169906616211,
      "logps/rejected": -286.8009948730469,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.299699068069458,
      "rewards/margins": 17.556556701660156,
      "rewards/rejected": -20.856258392333984,
      "step": 580
    },
    {
      "epoch": 2.5,
      "grad_norm": 0.007354736328125,
      "learning_rate": 4.7074385236074684e-06,
      "logits/chosen": -1.1783647537231445,
      "logits/rejected": -0.45893925428390503,
      "logps/chosen": -77.3306655883789,
      "logps/rejected": -294.39862060546875,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.5838446617126465,
      "rewards/margins": 17.903881072998047,
      "rewards/rejected": -21.487728118896484,
      "step": 585
    },
    {
      "epoch": 2.5213675213675213,
      "grad_norm": 0.0615234375,
      "learning_rate": 4.63722504025034e-06,
      "logits/chosen": -1.1400740146636963,
      "logits/rejected": -0.455253541469574,
      "logps/chosen": -70.24212646484375,
      "logps/rejected": -285.1997985839844,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0748324394226074,
      "rewards/margins": 17.681386947631836,
      "rewards/rejected": -20.7562198638916,
      "step": 590
    },
    {
      "epoch": 2.5427350427350426,
      "grad_norm": 0.00384521484375,
      "learning_rate": 4.569764384162676e-06,
      "logits/chosen": -1.1541723012924194,
      "logits/rejected": -0.43652376532554626,
      "logps/chosen": -65.93037414550781,
      "logps/rejected": -284.7110290527344,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.755821943283081,
      "rewards/margins": 17.971193313598633,
      "rewards/rejected": -20.727014541625977,
      "step": 595
    },
    {
      "epoch": 2.564102564102564,
      "grad_norm": 0.0198974609375,
      "learning_rate": 4.50509396800341e-06,
      "logits/chosen": -1.0580496788024902,
      "logits/rejected": -0.334017276763916,
      "logps/chosen": -72.97813415527344,
      "logps/rejected": -287.36724853515625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.329857587814331,
      "rewards/margins": 17.7404727935791,
      "rewards/rejected": -21.070331573486328,
      "step": 600
    },
    {
      "epoch": 2.5854700854700856,
      "grad_norm": 0.02490234375,
      "learning_rate": 4.443249657006627e-06,
      "logits/chosen": -1.0786378383636475,
      "logits/rejected": -0.3501695990562439,
      "logps/chosen": -70.572509765625,
      "logps/rejected": -291.306396484375,
      "loss": 0.0029,
      "rewards/accuracies": 0.9958333969116211,
      "rewards/chosen": -3.165311813354492,
      "rewards/margins": 18.223491668701172,
      "rewards/rejected": -21.388805389404297,
      "step": 605
    },
    {
      "epoch": 2.606837606837607,
      "grad_norm": 0.00946044921875,
      "learning_rate": 4.384265749091266e-06,
      "logits/chosen": -1.050445318222046,
      "logits/rejected": -0.34452471137046814,
      "logps/chosen": -78.69644927978516,
      "logps/rejected": -289.40362548828125,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.7829060554504395,
      "rewards/margins": 17.385984420776367,
      "rewards/rejected": -21.16888999938965,
      "step": 610
    },
    {
      "epoch": 2.628205128205128,
      "grad_norm": 0.054443359375,
      "learning_rate": 4.328174955840002e-06,
      "logits/chosen": -1.0618460178375244,
      "logits/rejected": -0.329507052898407,
      "logps/chosen": -67.98468017578125,
      "logps/rejected": -282.68719482421875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0549356937408447,
      "rewards/margins": 17.611309051513672,
      "rewards/rejected": -20.666244506835938,
      "step": 615
    },
    {
      "epoch": 2.6495726495726495,
      "grad_norm": 0.006683349609375,
      "learning_rate": 4.275008384357902e-06,
      "logits/chosen": -1.0847241878509521,
      "logits/rejected": -0.37132930755615234,
      "logps/chosen": -72.32841491699219,
      "logps/rejected": -284.59033203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2704505920410156,
      "rewards/margins": 17.539974212646484,
      "rewards/rejected": -20.810426712036133,
      "step": 620
    },
    {
      "epoch": 2.6709401709401708,
      "grad_norm": 0.0023651123046875,
      "learning_rate": 4.224795520020898e-06,
      "logits/chosen": -1.0495777130126953,
      "logits/rejected": -0.3218225836753845,
      "logps/chosen": -75.63634490966797,
      "logps/rejected": -283.78515625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4399490356445312,
      "rewards/margins": 17.156658172607422,
      "rewards/rejected": -20.596609115600586,
      "step": 625
    },
    {
      "epoch": 2.6923076923076925,
      "grad_norm": 0.00848388671875,
      "learning_rate": 4.177564210123634e-06,
      "logits/chosen": -1.0822218656539917,
      "logits/rejected": -0.37178927659988403,
      "logps/chosen": -71.94633483886719,
      "logps/rejected": -292.45965576171875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2282912731170654,
      "rewards/margins": 18.118127822875977,
      "rewards/rejected": -21.346416473388672,
      "step": 630
    },
    {
      "epoch": 2.713675213675214,
      "grad_norm": 0.0595703125,
      "learning_rate": 4.133340648435789e-06,
      "logits/chosen": -1.0735520124435425,
      "logits/rejected": -0.33007147908210754,
      "logps/chosen": -74.36048889160156,
      "logps/rejected": -289.21795654296875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.496561050415039,
      "rewards/margins": 17.668859481811523,
      "rewards/rejected": -21.16541862487793,
      "step": 635
    },
    {
      "epoch": 2.735042735042735,
      "grad_norm": 0.0281982421875,
      "learning_rate": 4.092149360675402e-06,
      "logits/chosen": -1.0645346641540527,
      "logits/rejected": -0.37463390827178955,
      "logps/chosen": -78.34007263183594,
      "logps/rejected": -292.8553771972656,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.6229121685028076,
      "rewards/margins": 17.589675903320312,
      "rewards/rejected": -21.212587356567383,
      "step": 640
    },
    {
      "epoch": 2.7564102564102564,
      "grad_norm": 0.034912109375,
      "learning_rate": 4.054013190907282e-06,
      "logits/chosen": -1.039671540260315,
      "logits/rejected": -0.3333393931388855,
      "logps/chosen": -69.4139404296875,
      "logps/rejected": -283.67291259765625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0979743003845215,
      "rewards/margins": 17.693653106689453,
      "rewards/rejected": -20.7916259765625,
      "step": 645
    },
    {
      "epoch": 2.7777777777777777,
      "grad_norm": 0.0034332275390625,
      "learning_rate": 4.018953288874035e-06,
      "logits/chosen": -1.0656228065490723,
      "logits/rejected": -0.3747131824493408,
      "logps/chosen": -73.5749282836914,
      "logps/rejected": -289.0091247558594,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.367324113845825,
      "rewards/margins": 17.781917572021484,
      "rewards/rejected": -21.149242401123047,
      "step": 650
    },
    {
      "epoch": 2.799145299145299,
      "grad_norm": 0.00098419189453125,
      "learning_rate": 3.9869890982667385e-06,
      "logits/chosen": -1.0841352939605713,
      "logits/rejected": -0.30320778489112854,
      "logps/chosen": -71.91044616699219,
      "logps/rejected": -290.2735595703125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.256523609161377,
      "rewards/margins": 17.801498413085938,
      "rewards/rejected": -21.058025360107422,
      "step": 655
    },
    {
      "epoch": 2.8205128205128203,
      "grad_norm": 0.00970458984375,
      "learning_rate": 3.9581383459417625e-06,
      "logits/chosen": -1.052257776260376,
      "logits/rejected": -0.34810546040534973,
      "logps/chosen": -78.00955963134766,
      "logps/rejected": -298.2610168457031,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.769831418991089,
      "rewards/margins": 18.104028701782227,
      "rewards/rejected": -21.873859405517578,
      "step": 660
    },
    {
      "epoch": 2.841880341880342,
      "grad_norm": 0.004974365234375,
      "learning_rate": 3.932417032089722e-06,
      "logits/chosen": -1.109933614730835,
      "logits/rejected": -0.38616496324539185,
      "logps/chosen": -75.1268081665039,
      "logps/rejected": -294.04522705078125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.484327793121338,
      "rewards/margins": 18.015382766723633,
      "rewards/rejected": -21.499710083007812,
      "step": 665
    },
    {
      "epoch": 2.8632478632478633,
      "grad_norm": 0.00125885009765625,
      "learning_rate": 3.909839421362017e-06,
      "logits/chosen": -1.0365312099456787,
      "logits/rejected": -0.3303161561489105,
      "logps/chosen": -75.15225219726562,
      "logps/rejected": -289.403076171875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.5623269081115723,
      "rewards/margins": 17.46319007873535,
      "rewards/rejected": -21.025516510009766,
      "step": 670
    },
    {
      "epoch": 2.8846153846153846,
      "grad_norm": 0.039794921875,
      "learning_rate": 3.890418034959871e-06,
      "logits/chosen": -1.0185749530792236,
      "logits/rejected": -0.24934515357017517,
      "logps/chosen": -73.84525299072266,
      "logps/rejected": -287.80670166015625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.451679229736328,
      "rewards/margins": 17.665843963623047,
      "rewards/rejected": -21.117523193359375,
      "step": 675
    },
    {
      "epoch": 2.905982905982906,
      "grad_norm": 0.05517578125,
      "learning_rate": 3.874163643690263e-06,
      "logits/chosen": -1.0194957256317139,
      "logits/rejected": -0.2993074655532837,
      "logps/chosen": -80.23220825195312,
      "logps/rejected": -291.32550048828125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.9078011512756348,
      "rewards/margins": 17.403505325317383,
      "rewards/rejected": -21.31130599975586,
      "step": 680
    },
    {
      "epoch": 2.9273504273504276,
      "grad_norm": 0.01055908203125,
      "learning_rate": 3.861085261992599e-06,
      "logits/chosen": -1.0856374502182007,
      "logits/rejected": -0.4421593248844147,
      "logps/chosen": -77.59326171875,
      "logps/rejected": -291.57843017578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.7359116077423096,
      "rewards/margins": 17.531463623046875,
      "rewards/rejected": -21.267375946044922,
      "step": 685
    },
    {
      "epoch": 2.948717948717949,
      "grad_norm": 0.068359375,
      "learning_rate": 3.851190142939442e-06,
      "logits/chosen": -1.0751783847808838,
      "logits/rejected": -0.3822060823440552,
      "logps/chosen": -71.5597915649414,
      "logps/rejected": -287.9521179199219,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.1508712768554688,
      "rewards/margins": 17.788990020751953,
      "rewards/rejected": -20.939861297607422,
      "step": 690
    },
    {
      "epoch": 2.97008547008547,
      "grad_norm": 0.005615234375,
      "learning_rate": 3.844483774214069e-06,
      "logits/chosen": -1.044440507888794,
      "logits/rejected": -0.2854236960411072,
      "logps/chosen": -70.91199493408203,
      "logps/rejected": -288.72283935546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.184840679168701,
      "rewards/margins": 17.9717960357666,
      "rewards/rejected": -21.156635284423828,
      "step": 695
    },
    {
      "epoch": 2.9871794871794872,
      "eval_logits/chosen": -1.1473212242126465,
      "eval_logits/rejected": -0.4121095538139343,
      "eval_logps/chosen": -73.0550308227539,
      "eval_logps/rejected": -287.1957092285156,
      "eval_loss": 4.4729193177772686e-05,
      "eval_rewards/accuracies": 1.0,
      "eval_rewards/chosen": -3.3611295223236084,
      "eval_rewards/margins": 17.5567569732666,
      "eval_rewards/rejected": -20.917884826660156,
      "eval_runtime": 9.4413,
      "eval_samples_per_second": 21.183,
      "eval_steps_per_second": 21.183,
      "step": 699
    }
  ],
  "logging_steps": 5,
  "max_steps": 702,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 12,
  "trial_name": null,
  "trial_params": null
}