{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9765925925925925,
  "eval_steps": 500,
  "global_step": 315,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.047407407407407405,
      "grad_norm": 1114.319091796875,
      "learning_rate": 7.8125e-06,
      "log_odds_chosen": -1.007889747619629,
      "log_odds_ratio": -10.178816795349121,
      "logps/chosen": -21.696243286132812,
      "logps/rejected": -20.688587188720703,
      "loss": 547.5699,
      "nll_loss": 9.67590618133545,
      "rewards/accuracies": 0.4781250059604645,
      "rewards/chosen": -10.848121643066406,
      "rewards/margins": -0.503828227519989,
      "rewards/rejected": -10.344293594360352,
      "step": 5
    },
    {
      "epoch": 0.09481481481481481,
      "grad_norm": 923.0885620117188,
      "learning_rate": 1.5625e-05,
      "log_odds_chosen": -2.801370143890381,
      "log_odds_ratio": -10.164667129516602,
      "logps/chosen": -20.971328735351562,
      "logps/rejected": -18.17032241821289,
      "loss": 553.2887,
      "nll_loss": 8.679121971130371,
      "rewards/accuracies": 0.4468750059604645,
      "rewards/chosen": -10.485664367675781,
      "rewards/margins": -1.4005038738250732,
      "rewards/rejected": -9.085161209106445,
      "step": 10
    },
    {
      "epoch": 0.14222222222222222,
      "grad_norm": 460.3068542480469,
      "learning_rate": 2.34375e-05,
      "log_odds_chosen": -1.2739719152450562,
      "log_odds_ratio": -10.46790599822998,
      "logps/chosen": -20.88275909423828,
      "logps/rejected": -19.607295989990234,
      "loss": 521.1988,
      "nll_loss": 8.076498031616211,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -10.44137954711914,
      "rewards/margins": -0.6377328038215637,
      "rewards/rejected": -9.803647994995117,
      "step": 15
    },
    {
      "epoch": 0.18962962962962962,
      "grad_norm": 4484.80712890625,
      "learning_rate": 3.125e-05,
      "log_odds_chosen": -3.9785079956054688,
      "log_odds_ratio": -8.86056900024414,
      "logps/chosen": -16.492053985595703,
      "logps/rejected": -12.51595401763916,
      "loss": 449.7023,
      "nll_loss": 6.58371639251709,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -8.246026992797852,
      "rewards/margins": -1.9880508184432983,
      "rewards/rejected": -6.25797700881958,
      "step": 20
    },
    {
      "epoch": 0.23703703703703705,
      "grad_norm": 703.4686279296875,
      "learning_rate": 3.90625e-05,
      "log_odds_chosen": -0.08368232101202011,
      "log_odds_ratio": -1.9139995574951172,
      "logps/chosen": -4.356338977813721,
      "logps/rejected": -4.261002540588379,
      "loss": 110.5851,
      "nll_loss": 3.422860622406006,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.1781694889068604,
      "rewards/margins": -0.0476679690182209,
      "rewards/rejected": -2.1305012702941895,
      "step": 25
    },
    {
      "epoch": 0.28444444444444444,
      "grad_norm": 436.39642333984375,
      "learning_rate": 4.6875e-05,
      "log_odds_chosen": -0.037819117307662964,
      "log_odds_ratio": -1.0239157676696777,
      "logps/chosen": -2.210418462753296,
      "logps/rejected": -2.175981044769287,
      "loss": 55.4786,
      "nll_loss": 2.731612205505371,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.105209231376648,
      "rewards/margins": -0.017218593508005142,
      "rewards/rejected": -1.0879905223846436,
      "step": 30
    },
    {
      "epoch": 0.33185185185185184,
      "grad_norm": 202.19808959960938,
      "learning_rate": 4.998613757348784e-05,
      "log_odds_chosen": 0.19444182515144348,
      "log_odds_ratio": -0.8455008268356323,
      "logps/chosen": -1.7893317937850952,
      "logps/rejected": -1.9684311151504517,
      "loss": 41.4203,
      "nll_loss": 2.4831597805023193,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.8946658968925476,
      "rewards/margins": 0.08954959362745285,
      "rewards/rejected": -0.9842155575752258,
      "step": 35
    },
    {
      "epoch": 0.37925925925925924,
      "grad_norm": 168.00384521484375,
      "learning_rate": 4.990147841143462e-05,
      "log_odds_chosen": 0.29362189769744873,
      "log_odds_ratio": -0.7308410406112671,
      "logps/chosen": -1.5209834575653076,
      "logps/rejected": -1.7699060440063477,
      "loss": 33.351,
      "nll_loss": 2.3416779041290283,
      "rewards/accuracies": 0.565625011920929,
      "rewards/chosen": -0.7604917287826538,
      "rewards/margins": 0.12446129322052002,
      "rewards/rejected": -0.8849530220031738,
      "step": 40
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 193.2564239501953,
      "learning_rate": 4.97401218720448e-05,
      "log_odds_chosen": 0.298764705657959,
      "log_odds_ratio": -0.7091392278671265,
      "logps/chosen": -1.45993971824646,
      "logps/rejected": -1.725881576538086,
      "loss": 31.5672,
      "nll_loss": 2.40507173538208,
      "rewards/accuracies": 0.5843750238418579,
      "rewards/chosen": -0.72996985912323,
      "rewards/margins": 0.1329708844423294,
      "rewards/rejected": -0.862940788269043,
      "step": 45
    },
    {
      "epoch": 0.4740740740740741,
      "grad_norm": 265.47442626953125,
      "learning_rate": 4.9502564938797946e-05,
      "log_odds_chosen": 0.3887616693973541,
      "log_odds_ratio": -0.6871553063392639,
      "logps/chosen": -1.3525534868240356,
      "logps/rejected": -1.6923545598983765,
      "loss": 27.2478,
      "nll_loss": 2.172515392303467,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.6762767434120178,
      "rewards/margins": 0.16990050673484802,
      "rewards/rejected": -0.8461772799491882,
      "step": 50
    },
    {
      "epoch": 0.5214814814814814,
      "grad_norm": 976.0902709960938,
      "learning_rate": 4.918953929490768e-05,
      "log_odds_chosen": 1.4601755142211914,
      "log_odds_ratio": -0.7356270551681519,
      "logps/chosen": -1.7127513885498047,
      "logps/rejected": -3.102419137954712,
      "loss": 21.1464,
      "nll_loss": 2.375514268875122,
      "rewards/accuracies": 0.6156250238418579,
      "rewards/chosen": -0.8563756942749023,
      "rewards/margins": 0.6948337554931641,
      "rewards/rejected": -1.551209568977356,
      "step": 55
    },
    {
      "epoch": 0.5688888888888889,
      "grad_norm": 19477.037109375,
      "learning_rate": 4.88020090697132e-05,
      "log_odds_chosen": 9.887539863586426,
      "log_odds_ratio": -2.272916316986084,
      "logps/chosen": -5.333150863647461,
      "logps/rejected": -15.112531661987305,
      "loss": -21.8917,
      "nll_loss": 5.349360466003418,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -2.6665754318237305,
      "rewards/margins": 4.889690399169922,
      "rewards/rejected": -7.556265830993652,
      "step": 60
    },
    {
      "epoch": 0.6162962962962963,
      "grad_norm": 63504.7578125,
      "learning_rate": 4.834116786912897e-05,
      "log_odds_chosen": 6.005472183227539,
      "log_odds_ratio": -5.739134788513184,
      "logps/chosen": -10.86668872833252,
      "logps/rejected": -16.817562103271484,
      "loss": 161.1276,
      "nll_loss": 9.975183486938477,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": -5.43334436416626,
      "rewards/margins": 2.975436210632324,
      "rewards/rejected": -8.408781051635742,
      "step": 65
    },
    {
      "epoch": 0.6637037037037037,
      "grad_norm": 3978.453857421875,
      "learning_rate": 4.7808435099299045e-05,
      "log_odds_chosen": 17.832883834838867,
      "log_odds_ratio": -5.360915184020996,
      "logps/chosen": -8.222649574279785,
      "logps/rejected": -25.95966148376465,
      "loss": -106.2073,
      "nll_loss": 7.058934211730957,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -4.111324787139893,
      "rewards/margins": 8.868505477905273,
      "rewards/rejected": -12.979830741882324,
      "step": 70
    },
    {
      "epoch": 0.7111111111111111,
      "grad_norm": 16509.505859375,
      "learning_rate": 4.720545159477922e-05,
      "log_odds_chosen": 20.426956176757812,
      "log_odds_ratio": -9.292292594909668,
      "logps/chosen": -15.002415657043457,
      "logps/rejected": -35.350364685058594,
      "loss": 6.2434,
      "nll_loss": 11.920351028442383,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -7.5012078285217285,
      "rewards/margins": 10.173975944519043,
      "rewards/rejected": -17.675182342529297,
      "step": 75
    },
    {
      "epoch": 0.7585185185185185,
      "grad_norm": 3993.2314453125,
      "learning_rate": 4.653407456471222e-05,
      "log_odds_chosen": 21.786739349365234,
      "log_odds_ratio": -9.150012969970703,
      "logps/chosen": -12.506328582763672,
      "logps/rejected": -34.199073791503906,
      "loss": -92.8843,
      "nll_loss": 10.512908935546875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -6.253164291381836,
      "rewards/margins": 10.8463716506958,
      "rewards/rejected": -17.099536895751953,
      "step": 80
    },
    {
      "epoch": 0.8059259259259259,
      "grad_norm": 3684.40771484375,
      "learning_rate": 4.579637187256222e-05,
      "log_odds_chosen": 21.78746795654297,
      "log_odds_ratio": -7.730111122131348,
      "logps/chosen": -8.821676254272461,
      "logps/rejected": -30.544620513916016,
      "loss": -188.5252,
      "nll_loss": 7.775321006774902,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -4.4108381271362305,
      "rewards/margins": 10.861473083496094,
      "rewards/rejected": -15.272310256958008,
      "step": 85
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 4620.634765625,
      "learning_rate": 4.499461566702685e-05,
      "log_odds_chosen": 28.149227142333984,
      "log_odds_ratio": -15.921751022338867,
      "logps/chosen": -24.887939453125,
      "logps/rejected": -52.94123077392578,
      "loss": 93.227,
      "nll_loss": 20.595924377441406,
      "rewards/accuracies": 0.6656249761581421,
      "rewards/chosen": -12.4439697265625,
      "rewards/margins": 14.026643753051758,
      "rewards/rejected": -26.47061538696289,
      "step": 90
    },
    {
      "epoch": 0.9007407407407407,
      "grad_norm": 2822.56982421875,
      "learning_rate": 4.413127538374411e-05,
      "log_odds_chosen": 48.35707473754883,
      "log_odds_ratio": -11.363774299621582,
      "logps/chosen": -19.054187774658203,
      "logps/rejected": -67.31429290771484,
      "loss": -343.8322,
      "nll_loss": 13.582868576049805,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -9.527093887329102,
      "rewards/margins": 24.130054473876953,
      "rewards/rejected": -33.65714645385742,
      "step": 95
    },
    {
      "epoch": 0.9481481481481482,
      "grad_norm": 11888.7197265625,
      "learning_rate": 4.320901013934887e-05,
      "log_odds_chosen": 37.94886016845703,
      "log_odds_ratio": -13.369104385375977,
      "logps/chosen": -25.14851951599121,
      "logps/rejected": -62.99921798706055,
      "loss": -14.4122,
      "nll_loss": 19.4227294921875,
      "rewards/accuracies": 0.6656249761581421,
      "rewards/chosen": -12.574259757995605,
      "rewards/margins": 18.925350189208984,
      "rewards/rejected": -31.499608993530273,
      "step": 100
    },
    {
      "epoch": 0.9955555555555555,
      "grad_norm": 17781.978515625,
      "learning_rate": 4.223066054130568e-05,
      "log_odds_chosen": 32.50678253173828,
      "log_odds_ratio": -13.40135669708252,
      "logps/chosen": -26.233570098876953,
      "logps/rejected": -58.676490783691406,
      "loss": 106.4254,
      "nll_loss": 19.13003158569336,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -13.116785049438477,
      "rewards/margins": 16.22146224975586,
      "rewards/rejected": -29.338245391845703,
      "step": 105
    },
    {
      "epoch": 1.037925925925926,
      "grad_norm": 8946.7998046875,
      "learning_rate": 4.1199239938743797e-05,
      "log_odds_chosen": 11.558371543884277,
      "log_odds_ratio": -7.9565300941467285,
      "logps/chosen": -13.197209358215332,
      "logps/rejected": -24.688472747802734,
      "loss": 99.8268,
      "nll_loss": 10.583297729492188,
      "rewards/accuracies": 0.618881106376648,
      "rewards/chosen": -6.598604679107666,
      "rewards/margins": 5.745632171630859,
      "rewards/rejected": -12.344236373901367,
      "step": 110
    },
    {
      "epoch": 1.0853333333333333,
      "grad_norm": 28189.90625,
      "learning_rate": 4.0117925141242174e-05,
      "log_odds_chosen": 23.58847427368164,
      "log_odds_ratio": -8.089736938476562,
      "logps/chosen": -9.830774307250977,
      "logps/rejected": -33.307220458984375,
      "loss": -189.9675,
      "nll_loss": 7.8513503074646,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -4.915387153625488,
      "rewards/margins": 11.7382230758667,
      "rewards/rejected": -16.653610229492188,
      "step": 115
    },
    {
      "epoch": 1.1327407407407408,
      "grad_norm": 4763.630859375,
      "learning_rate": 3.899004663415084e-05,
      "log_odds_chosen": 37.111053466796875,
      "log_odds_ratio": -8.056583404541016,
      "logps/chosen": -9.871607780456543,
      "logps/rejected": -46.8720588684082,
      "loss": -404.5778,
      "nll_loss": 8.503942489624023,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -4.9358038902282715,
      "rewards/margins": 18.500225067138672,
      "rewards/rejected": -23.4360294342041,
      "step": 120
    },
    {
      "epoch": 1.1801481481481482,
      "grad_norm": 5928.169921875,
      "learning_rate": 3.781907832058587e-05,
      "log_odds_chosen": 35.93215560913086,
      "log_odds_ratio": -9.4933443069458,
      "logps/chosen": -11.483396530151367,
      "logps/rejected": -47.30681610107422,
      "loss": -357.2632,
      "nll_loss": 10.28132438659668,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -5.741698265075684,
      "rewards/margins": 17.911710739135742,
      "rewards/rejected": -23.65340805053711,
      "step": 125
    },
    {
      "epoch": 1.2275555555555555,
      "grad_norm": 11281.740234375,
      "learning_rate": 3.660862682169282e-05,
      "log_odds_chosen": 40.6527214050293,
      "log_odds_ratio": -11.877891540527344,
      "logps/chosen": -16.65973472595215,
      "logps/rejected": -57.17784881591797,
      "loss": -304.9178,
      "nll_loss": 12.847620964050293,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -8.329867362976074,
      "rewards/margins": 20.259056091308594,
      "rewards/rejected": -28.588924407958984,
      "step": 130
    },
    {
      "epoch": 1.274962962962963,
      "grad_norm": 7086.1142578125,
      "learning_rate": 3.5362420368134356e-05,
      "log_odds_chosen": 11.272117614746094,
      "log_odds_ratio": -6.918790340423584,
      "logps/chosen": -7.792928218841553,
      "logps/rejected": -18.96177101135254,
      "loss": -39.8691,
      "nll_loss": 6.798020362854004,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -3.8964641094207764,
      "rewards/margins": 5.584420204162598,
      "rewards/rejected": -9.48088550567627,
      "step": 135
    },
    {
      "epoch": 1.3223703703703704,
      "grad_norm": 137599.171875,
      "learning_rate": 3.408429731701635e-05,
      "log_odds_chosen": 21.463497161865234,
      "log_odds_ratio": -2.791048049926758,
      "logps/chosen": -5.230650424957275,
      "logps/rejected": -26.6016902923584,
      "loss": -218.9189,
      "nll_loss": 4.977573394775391,
      "rewards/accuracies": 0.6656249761581421,
      "rewards/chosen": -2.6153252124786377,
      "rewards/margins": 10.68552017211914,
      "rewards/rejected": -13.3008451461792,
      "step": 140
    },
    {
      "epoch": 1.3697777777777778,
      "grad_norm": 20666.33203125,
      "learning_rate": 3.2778194329621104e-05,
      "log_odds_chosen": 40.393245697021484,
      "log_odds_ratio": -13.250892639160156,
      "logps/chosen": -19.773197174072266,
      "logps/rejected": -60.044952392578125,
      "loss": -223.2017,
      "nll_loss": 16.38017463684082,
      "rewards/accuracies": 0.684374988079071,
      "rewards/chosen": -9.886598587036133,
      "rewards/margins": 20.135875701904297,
      "rewards/rejected": -30.022476196289062,
      "step": 145
    },
    {
      "epoch": 1.417185185185185,
      "grad_norm": 19448.947265625,
      "learning_rate": 3.144813424636031e-05,
      "log_odds_chosen": 45.739837646484375,
      "log_odds_ratio": -10.681756973266602,
      "logps/chosen": -15.583732604980469,
      "logps/rejected": -61.16387176513672,
      "loss": -401.1902,
      "nll_loss": 12.756240844726562,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -7.791866302490234,
      "rewards/margins": 22.790069580078125,
      "rewards/rejected": -30.58193588256836,
      "step": 150
    },
    {
      "epoch": 1.4645925925925927,
      "grad_norm": 7016.2177734375,
      "learning_rate": 3.0098213696293542e-05,
      "log_odds_chosen": 26.974849700927734,
      "log_odds_ratio": -9.331676483154297,
      "logps/chosen": -11.568083763122559,
      "logps/rejected": -38.42928695678711,
      "loss": -208.4774,
      "nll_loss": 10.451626777648926,
      "rewards/accuracies": 0.6781250238418579,
      "rewards/chosen": -5.784041881561279,
      "rewards/margins": 13.430601119995117,
      "rewards/rejected": -19.214643478393555,
      "step": 155
    },
    {
      "epoch": 1.512,
      "grad_norm": 2596.490234375,
      "learning_rate": 2.8732590479375165e-05,
      "log_odds_chosen": 17.883655548095703,
      "log_odds_ratio": -1.8087437152862549,
      "logps/chosen": -3.404132843017578,
      "logps/rejected": -21.178518295288086,
      "loss": -204.2252,
      "nll_loss": 3.781926393508911,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.702066421508789,
      "rewards/margins": 8.88719367980957,
      "rewards/rejected": -10.589259147644043,
      "step": 160
    },
    {
      "epoch": 1.5594074074074074,
      "grad_norm": 15875.6162109375,
      "learning_rate": 2.7355470760292956e-05,
      "log_odds_chosen": 17.571182250976562,
      "log_odds_ratio": -6.643049716949463,
      "logps/chosen": -9.500158309936523,
      "logps/rejected": -26.94256591796875,
      "loss": -81.0908,
      "nll_loss": 8.642666816711426,
      "rewards/accuracies": 0.6781250238418579,
      "rewards/chosen": -4.750079154968262,
      "rewards/margins": 8.72120475769043,
      "rewards/rejected": -13.471282958984375,
      "step": 165
    },
    {
      "epoch": 1.6068148148148147,
      "grad_norm": 6882.3740234375,
      "learning_rate": 2.597109611334169e-05,
      "log_odds_chosen": 37.06153106689453,
      "log_odds_ratio": -7.4608635902404785,
      "logps/chosen": -15.66374683380127,
      "logps/rejected": -52.575416564941406,
      "loss": -208.487,
      "nll_loss": 13.647031784057617,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -7.831873416900635,
      "rewards/margins": 18.45583152770996,
      "rewards/rejected": -26.287708282470703,
      "step": 170
    },
    {
      "epoch": 1.6542222222222223,
      "grad_norm": 6623.11572265625,
      "learning_rate": 2.458373045823404e-05,
      "log_odds_chosen": 21.147552490234375,
      "log_odds_ratio": -7.808111667633057,
      "logps/chosen": -9.466218948364258,
      "logps/rejected": -30.470600128173828,
      "loss": -157.8435,
      "nll_loss": 7.549225807189941,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -4.733109474182129,
      "rewards/margins": 10.502188682556152,
      "rewards/rejected": -15.235300064086914,
      "step": 175
    },
    {
      "epoch": 1.7016296296296296,
      "grad_norm": 5010.78564453125,
      "learning_rate": 2.3197646927086697e-05,
      "log_odds_chosen": 24.123126983642578,
      "log_odds_ratio": -1.9697444438934326,
      "logps/chosen": -2.780689001083374,
      "logps/rejected": -26.779048919677734,
      "loss": -326.2735,
      "nll_loss": 3.6041767597198486,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": -1.390344500541687,
      "rewards/margins": 11.99917984008789,
      "rewards/rejected": -13.389524459838867,
      "step": 180
    },
    {
      "epoch": 1.749037037037037,
      "grad_norm": 6793.22412109375,
      "learning_rate": 2.1817114703032176e-05,
      "log_odds_chosen": 38.25811004638672,
      "log_odds_ratio": -7.5069379806518555,
      "logps/chosen": -9.548910140991211,
      "logps/rejected": -47.6781120300293,
      "loss": -424.2517,
      "nll_loss": 7.210787296295166,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -4.7744550704956055,
      "rewards/margins": 19.06460189819336,
      "rewards/rejected": -23.83905601501465,
      "step": 185
    },
    {
      "epoch": 1.7964444444444445,
      "grad_norm": 68079.1953125,
      "learning_rate": 2.0446385870993467e-05,
      "log_odds_chosen": 41.46270751953125,
      "log_odds_ratio": -9.237896919250488,
      "logps/chosen": -11.56389331817627,
      "logps/rejected": -52.86220169067383,
      "loss": -438.3831,
      "nll_loss": 10.06884765625,
      "rewards/accuracies": 0.753125011920929,
      "rewards/chosen": -5.781946659088135,
      "rewards/margins": 20.649154663085938,
      "rewards/rejected": -26.431100845336914,
      "step": 190
    },
    {
      "epoch": 1.8438518518518519,
      "grad_norm": 19936.81640625,
      "learning_rate": 1.9089682321121834e-05,
      "log_odds_chosen": 43.99555206298828,
      "log_odds_ratio": -7.328845977783203,
      "logps/chosen": -9.099332809448242,
      "logps/rejected": -52.933860778808594,
      "loss": -527.35,
      "nll_loss": 8.21430778503418,
      "rewards/accuracies": 0.7406250238418579,
      "rewards/chosen": -4.549666404724121,
      "rewards/margins": 21.91726303100586,
      "rewards/rejected": -26.466930389404297,
      "step": 195
    },
    {
      "epoch": 1.8912592592592592,
      "grad_norm": 291176.21875,
      "learning_rate": 1.775118274523545e-05,
      "log_odds_chosen": 32.236446380615234,
      "log_odds_ratio": -11.109225273132324,
      "logps/chosen": -14.257405281066895,
      "logps/rejected": -46.36336898803711,
      "loss": -234.8545,
      "nll_loss": 11.66010856628418,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -7.128702640533447,
      "rewards/margins": 16.052982330322266,
      "rewards/rejected": -23.181684494018555,
      "step": 200
    },
    {
      "epoch": 1.9386666666666668,
      "grad_norm": 3509.22021484375,
      "learning_rate": 1.643500976631037e-05,
      "log_odds_chosen": 30.14800453186035,
      "log_odds_ratio": -12.824560165405273,
      "logps/chosen": -15.561635971069336,
      "logps/rejected": -45.55414581298828,
      "loss": -186.7682,
      "nll_loss": 12.363306045532227,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -7.780817985534668,
      "rewards/margins": 14.996258735656738,
      "rewards/rejected": -22.77707290649414,
      "step": 205
    },
    {
      "epoch": 1.986074074074074,
      "grad_norm": 2383.834228515625,
      "learning_rate": 1.514521724066537e-05,
      "log_odds_chosen": 41.85120391845703,
      "log_odds_ratio": -13.449541091918945,
      "logps/chosen": -15.545976638793945,
      "logps/rejected": -57.27630615234375,
      "loss": -384.8493,
      "nll_loss": 11.846510887145996,
      "rewards/accuracies": 0.6656249761581421,
      "rewards/chosen": -7.772988319396973,
      "rewards/margins": 20.86516761779785,
      "rewards/rejected": -28.638153076171875,
      "step": 210
    },
    {
      "epoch": 2.0284444444444443,
      "grad_norm": 24200.37109375,
      "learning_rate": 1.3885777771950348e-05,
      "log_odds_chosen": 33.1331787109375,
      "log_odds_ratio": -8.699053764343262,
      "logps/chosen": -9.583888053894043,
      "logps/rejected": -42.54216384887695,
      "loss": -321.3827,
      "nll_loss": 8.21044921875,
      "rewards/accuracies": 0.7272727489471436,
      "rewards/chosen": -4.7919440269470215,
      "rewards/margins": 16.479137420654297,
      "rewards/rejected": -21.271081924438477,
      "step": 215
    },
    {
      "epoch": 2.075851851851852,
      "grad_norm": 6451.2578125,
      "learning_rate": 1.2660570475395683e-05,
      "log_odds_chosen": 19.57032585144043,
      "log_odds_ratio": -11.807035446166992,
      "logps/chosen": -13.269747734069824,
      "logps/rejected": -32.721336364746094,
      "loss": -75.1553,
      "nll_loss": 10.17901611328125,
      "rewards/accuracies": 0.684374988079071,
      "rewards/chosen": -6.634873867034912,
      "rewards/margins": 9.725793838500977,
      "rewards/rejected": -16.360668182373047,
      "step": 220
    },
    {
      "epoch": 2.1232592592592594,
      "grad_norm": 3175.251708984375,
      "learning_rate": 1.1473369030008974e-05,
      "log_odds_chosen": 44.743412017822266,
      "log_odds_ratio": -6.601927280426025,
      "logps/chosen": -7.938143730163574,
      "logps/rejected": -52.494384765625,
      "loss": -564.2982,
      "nll_loss": 7.033768653869629,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": -3.969071865081787,
      "rewards/margins": 22.278118133544922,
      "rewards/rejected": -26.2471923828125,
      "step": 225
    },
    {
      "epoch": 2.1706666666666665,
      "grad_norm": 10925.5673828125,
      "learning_rate": 1.0327830055518842e-05,
      "log_odds_chosen": 37.528343200683594,
      "log_odds_ratio": -7.298922061920166,
      "logps/chosen": -8.199933052062988,
      "logps/rejected": -45.548763275146484,
      "loss": -451.7604,
      "nll_loss": 6.6627702713012695,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -4.099966526031494,
      "rewards/margins": 18.674419403076172,
      "rewards/rejected": -22.774381637573242,
      "step": 230
    },
    {
      "epoch": 2.218074074074074,
      "grad_norm": 5350.28662109375,
      "learning_rate": 9.227481849865235e-06,
      "log_odds_chosen": 37.19614791870117,
      "log_odds_ratio": -10.001937866210938,
      "logps/chosen": -12.3423490524292,
      "logps/rejected": -49.37834930419922,
      "loss": -357.3079,
      "nll_loss": 9.177508354187012,
      "rewards/accuracies": 0.684374988079071,
      "rewards/chosen": -6.1711745262146,
      "rewards/margins": 18.518001556396484,
      "rewards/rejected": -24.68917465209961,
      "step": 235
    },
    {
      "epoch": 2.2654814814814817,
      "grad_norm": 10187.1015625,
      "learning_rate": 8.175713521924978e-06,
      "log_odds_chosen": 42.718074798583984,
      "log_odds_ratio": -11.093441009521484,
      "logps/chosen": -14.150568008422852,
      "logps/rejected": -56.68629837036133,
      "loss": -405.1544,
      "nll_loss": 11.543497085571289,
      "rewards/accuracies": 0.7406250238418579,
      "rewards/chosen": -7.075284004211426,
      "rewards/margins": 21.267864227294922,
      "rewards/rejected": -28.343149185180664,
      "step": 240
    },
    {
      "epoch": 2.3128888888888888,
      "grad_norm": 5219.4873046875,
      "learning_rate": 7.1757645529443665e-06,
      "log_odds_chosen": 44.08445358276367,
      "log_odds_ratio": -11.732454299926758,
      "logps/chosen": -13.956028938293457,
      "logps/rejected": -57.879173278808594,
      "loss": -443.6736,
      "nll_loss": 10.992512702941895,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -6.9780144691467285,
      "rewards/margins": 21.96157455444336,
      "rewards/rejected": -28.939586639404297,
      "step": 245
    },
    {
      "epoch": 2.3602962962962963,
      "grad_norm": 11797.4462890625,
      "learning_rate": 6.230714818829733e-06,
      "log_odds_chosen": 35.4526252746582,
      "log_odds_ratio": -8.070879936218262,
      "logps/chosen": -9.394768714904785,
      "logps/rejected": -44.66648483276367,
      "loss": -392.5384,
      "nll_loss": 7.098433017730713,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -4.697384357452393,
      "rewards/margins": 17.6358585357666,
      "rewards/rejected": -22.333242416381836,
      "step": 250
    },
    {
      "epoch": 2.407703703703704,
      "grad_norm": 11236.10546875,
      "learning_rate": 5.343475104027743e-06,
      "log_odds_chosen": 37.37392807006836,
      "log_odds_ratio": -9.121585845947266,
      "logps/chosen": -12.584083557128906,
      "logps/rejected": -49.78114318847656,
      "loss": -338.2161,
      "nll_loss": 9.931722640991211,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -6.292041778564453,
      "rewards/margins": 18.598529815673828,
      "rewards/rejected": -24.89057159423828,
      "step": 255
    },
    {
      "epoch": 2.455111111111111,
      "grad_norm": 15344.7255859375,
      "learning_rate": 4.516778136213037e-06,
      "log_odds_chosen": 34.853729248046875,
      "log_odds_ratio": -6.747254371643066,
      "logps/chosen": -9.025346755981445,
      "logps/rejected": -43.715370178222656,
      "loss": -374.0381,
      "nll_loss": 8.037846565246582,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -4.512673377990723,
      "rewards/margins": 17.345016479492188,
      "rewards/rejected": -21.857685089111328,
      "step": 260
    },
    {
      "epoch": 2.5025185185185186,
      "grad_norm": 8925.0234375,
      "learning_rate": 3.7531701693965554e-06,
      "log_odds_chosen": 52.23662567138672,
      "log_odds_ratio": -5.923990726470947,
      "logps/chosen": -7.497511386871338,
      "logps/rejected": -59.578468322753906,
      "loss": -687.7863,
      "nll_loss": 6.113022327423096,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": -3.748755693435669,
      "rewards/margins": 26.040477752685547,
      "rewards/rejected": -29.789234161376953,
      "step": 265
    },
    {
      "epoch": 2.549925925925926,
      "grad_norm": 3612.017333984375,
      "learning_rate": 3.055003141378948e-06,
      "log_odds_chosen": 44.58516311645508,
      "log_odds_ratio": -5.719125747680664,
      "logps/chosen": -10.139649391174316,
      "logps/rejected": -54.54728317260742,
      "loss": -477.472,
      "nll_loss": 8.543855667114258,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -5.069824695587158,
      "rewards/margins": 22.203815460205078,
      "rewards/rejected": -27.27364158630371,
      "step": 270
    },
    {
      "epoch": 2.5973333333333333,
      "grad_norm": 9717.0439453125,
      "learning_rate": 2.424427429704365e-06,
      "log_odds_chosen": 38.22019577026367,
      "log_odds_ratio": -7.6910576820373535,
      "logps/chosen": -10.25065803527832,
      "logps/rejected": -48.30152130126953,
      "loss": -403.615,
      "nll_loss": 8.413914680480957,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -5.12532901763916,
      "rewards/margins": 19.025432586669922,
      "rewards/rejected": -24.150760650634766,
      "step": 275
    },
    {
      "epoch": 2.644740740740741,
      "grad_norm": 7232.400390625,
      "learning_rate": 1.8633852284264508e-06,
      "log_odds_chosen": 48.507225036621094,
      "log_odds_ratio": -8.356045722961426,
      "logps/chosen": -10.951539993286133,
      "logps/rejected": -59.291465759277344,
      "loss": -556.3883,
      "nll_loss": 8.764814376831055,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -5.475769996643066,
      "rewards/margins": 24.169963836669922,
      "rewards/rejected": -29.645732879638672,
      "step": 280
    },
    {
      "epoch": 2.6921481481481484,
      "grad_norm": 8219.6298828125,
      "learning_rate": 1.3736045660864034e-06,
      "log_odds_chosen": 41.27231979370117,
      "log_odds_ratio": -6.389247417449951,
      "logps/chosen": -11.282487869262695,
      "logps/rejected": -52.3975830078125,
      "loss": -398.777,
      "nll_loss": 9.017220497131348,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -5.641243934631348,
      "rewards/margins": 20.55754852294922,
      "rewards/rejected": -26.19879150390625,
      "step": 285
    },
    {
      "epoch": 2.7395555555555555,
      "grad_norm": 9359.0595703125,
      "learning_rate": 9.565939833279192e-07,
      "log_odds_chosen": 24.906938552856445,
      "log_odds_ratio": -18.328092575073242,
      "logps/chosen": -23.42938804626465,
      "logps/rejected": -48.18115997314453,
      "loss": 60.6635,
      "nll_loss": 17.129901885986328,
      "rewards/accuracies": 0.6968749761581421,
      "rewards/chosen": -11.714694023132324,
      "rewards/margins": 12.375885963439941,
      "rewards/rejected": -24.090579986572266,
      "step": 290
    },
    {
      "epoch": 2.786962962962963,
      "grad_norm": 4202.4697265625,
      "learning_rate": 6.136378865420872e-07,
      "log_odds_chosen": 37.57846450805664,
      "log_odds_ratio": -9.300204277038574,
      "logps/chosen": -13.682713508605957,
      "logps/rejected": -51.1092414855957,
      "loss": -309.3759,
      "nll_loss": 10.95567512512207,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -6.8413567543029785,
      "rewards/margins": 18.713260650634766,
      "rewards/rejected": -25.55462074279785,
      "step": 295
    },
    {
      "epoch": 2.83437037037037,
      "grad_norm": 29844.5703125,
      "learning_rate": 3.45792591853214e-07,
      "log_odds_chosen": 43.591487884521484,
      "log_odds_ratio": -10.49733829498291,
      "logps/chosen": -14.3878755569458,
      "logps/rejected": -57.82908248901367,
      "loss": -402.1579,
      "nll_loss": 10.638388633728027,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -7.1939377784729,
      "rewards/margins": 21.720603942871094,
      "rewards/rejected": -28.914541244506836,
      "step": 300
    },
    {
      "epoch": 2.8817777777777778,
      "grad_norm": 16960.9296875,
      "learning_rate": 1.538830716302092e-07,
      "log_odds_chosen": 41.2000732421875,
      "log_odds_ratio": -11.432271957397461,
      "logps/chosen": -13.90955924987793,
      "logps/rejected": -54.91774368286133,
      "loss": -393.7107,
      "nll_loss": 10.265416145324707,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -6.954779624938965,
      "rewards/margins": 20.504093170166016,
      "rewards/rejected": -27.458871841430664,
      "step": 305
    },
    {
      "epoch": 2.9291851851851853,
      "grad_norm": 9071.34375,
      "learning_rate": 3.8500413544415025e-08,
      "log_odds_chosen": 41.77397918701172,
      "log_odds_ratio": -10.353241920471191,
      "logps/chosen": -12.447257995605469,
      "logps/rejected": -54.037437438964844,
      "loss": -432.6522,
      "nll_loss": 10.377059936523438,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -6.223628997802734,
      "rewards/margins": 20.795089721679688,
      "rewards/rejected": -27.018718719482422,
      "step": 310
    },
    {
      "epoch": 2.9765925925925925,
      "grad_norm": 8651.380859375,
      "learning_rate": 0.0,
      "log_odds_chosen": 53.18922805786133,
      "log_odds_ratio": -10.290987014770508,
      "logps/chosen": -12.473676681518555,
      "logps/rejected": -65.4810791015625,
      "loss": -613.3333,
      "nll_loss": 10.335878372192383,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -6.236838340759277,
      "rewards/margins": 26.50370216369629,
      "rewards/rejected": -32.74053955078125,
      "step": 315
    },
    {
      "epoch": 2.9765925925925925,
      "step": 315,
      "total_flos": 0.0,
      "train_loss": -186.8132874746171,
      "train_runtime": 9331.4928,
      "train_samples_per_second": 2.17,
      "train_steps_per_second": 0.034
    }
  ],
  "logging_steps": 5,
  "max_steps": 315,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}