{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.0,
  "eval_steps": 500,
  "global_step": 10000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0002,
      "grad_norm": 153.0,
      "learning_rate": 0.0,
      "logits/chosen": 0.38140869140625,
      "logits/rejected": -0.0750732421875,
      "logps/chosen": -179.625,
      "logps/rejected": -175.5,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.01251220703125,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.01251220703125,
      "step": 1
    },
    {
      "epoch": 0.01,
      "grad_norm": 152.0,
      "learning_rate": 1.6333333333333334e-07,
      "logits/chosen": -0.07141295820474625,
      "logits/rejected": -0.49311113357543945,
      "logps/chosen": -169.4234619140625,
      "logps/rejected": -166.2244873046875,
      "loss": 0.6897,
      "rewards/accuracies": 0.32397958636283875,
      "rewards/chosen": 0.022099709138274193,
      "rewards/margins": 0.00855239573866129,
      "rewards/rejected": 0.013539839535951614,
      "step": 50
    },
    {
      "epoch": 0.02,
      "grad_norm": 173.0,
      "learning_rate": 3.3e-07,
      "logits/chosen": -0.19355152547359467,
      "logits/rejected": -0.6559900045394897,
      "logps/chosen": -169.9737548828125,
      "logps/rejected": -168.53875732421875,
      "loss": 0.6714,
      "rewards/accuracies": 0.47999998927116394,
      "rewards/chosen": 0.056079406291246414,
      "rewards/margins": 0.04595337063074112,
      "rewards/rejected": 0.010139770805835724,
      "step": 100
    },
    {
      "epoch": 0.03,
      "grad_norm": 159.0,
      "learning_rate": 4.966666666666666e-07,
      "logits/chosen": -0.1707330346107483,
      "logits/rejected": -0.6063339114189148,
      "logps/chosen": -168.2937469482422,
      "logps/rejected": -166.5187530517578,
      "loss": 0.6156,
      "rewards/accuracies": 0.7900000214576721,
      "rewards/chosen": 0.1966903656721115,
      "rewards/margins": 0.16668151319026947,
      "rewards/rejected": 0.030118407681584358,
      "step": 150
    },
    {
      "epoch": 0.04,
      "grad_norm": 87.0,
      "learning_rate": 6.633333333333334e-07,
      "logits/chosen": -0.08012771606445312,
      "logits/rejected": -0.5313219428062439,
      "logps/chosen": -163.4462432861328,
      "logps/rejected": -166.76124572753906,
      "loss": 0.5115,
      "rewards/accuracies": 0.8550000190734863,
      "rewards/chosen": 0.4547726511955261,
      "rewards/margins": 0.4213232398033142,
      "rewards/rejected": 0.03348724544048309,
      "step": 200
    },
    {
      "epoch": 0.05,
      "grad_norm": 67.5,
      "learning_rate": 8.300000000000001e-07,
      "logits/chosen": -0.17801956832408905,
      "logits/rejected": -0.6736994981765747,
      "logps/chosen": -162.8975067138672,
      "logps/rejected": -169.00250244140625,
      "loss": 0.3636,
      "rewards/accuracies": 0.8974999785423279,
      "rewards/chosen": 0.7460748553276062,
      "rewards/margins": 0.8772411942481995,
      "rewards/rejected": -0.1307925432920456,
      "step": 250
    },
    {
      "epoch": 0.06,
      "grad_norm": 29.625,
      "learning_rate": 9.966666666666667e-07,
      "logits/chosen": -0.3131498694419861,
      "logits/rejected": -0.8370306491851807,
      "logps/chosen": -156.57749938964844,
      "logps/rejected": -171.12249755859375,
      "loss": 0.2411,
      "rewards/accuracies": 0.8725000023841858,
      "rewards/chosen": 0.9195745587348938,
      "rewards/margins": 1.5089257955551147,
      "rewards/rejected": -0.5890390276908875,
      "step": 300
    },
    {
      "epoch": 0.07,
      "grad_norm": 8.625,
      "learning_rate": 1.1633333333333333e-06,
      "logits/chosen": -0.809218168258667,
      "logits/rejected": -1.343685269355774,
      "logps/chosen": -165.8387451171875,
      "logps/rejected": -189.9462432861328,
      "loss": 0.1339,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.9223541021347046,
      "rewards/margins": 2.6102733612060547,
      "rewards/rejected": -1.6883777379989624,
      "step": 350
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.234375,
      "learning_rate": 1.3300000000000002e-06,
      "logits/chosen": -1.2586804628372192,
      "logits/rejected": -1.9825918674468994,
      "logps/chosen": -158.1649932861328,
      "logps/rejected": -198.03750610351562,
      "loss": 0.0961,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": 1.2969841957092285,
      "rewards/margins": 4.182460784912109,
      "rewards/rejected": -2.8857672214508057,
      "step": 400
    },
    {
      "epoch": 0.09,
      "grad_norm": 0.349609375,
      "learning_rate": 1.4966666666666668e-06,
      "logits/chosen": -1.5829663276672363,
      "logits/rejected": -2.467480421066284,
      "logps/chosen": -147.5625,
      "logps/rejected": -204.00750732421875,
      "loss": 0.1038,
      "rewards/accuracies": 0.8525000214576721,
      "rewards/chosen": 1.9529736042022705,
      "rewards/margins": 5.97265625,
      "rewards/rejected": -4.018417835235596,
      "step": 450
    },
    {
      "epoch": 0.1,
      "grad_norm": 0.5625,
      "learning_rate": 1.6633333333333334e-06,
      "logits/chosen": -1.8108301162719727,
      "logits/rejected": -2.774589776992798,
      "logps/chosen": -147.88250732421875,
      "logps/rejected": -219.89625549316406,
      "loss": 0.0802,
      "rewards/accuracies": 0.8849999904632568,
      "rewards/chosen": 2.4084813594818115,
      "rewards/margins": 7.462968826293945,
      "rewards/rejected": -5.055732250213623,
      "step": 500
    },
    {
      "epoch": 0.11,
      "grad_norm": 0.21484375,
      "learning_rate": 1.83e-06,
      "logits/chosen": -1.8340039253234863,
      "logits/rejected": -2.798632860183716,
      "logps/chosen": -138.77374267578125,
      "logps/rejected": -212.2449951171875,
      "loss": 0.1111,
      "rewards/accuracies": 0.8399999737739563,
      "rewards/chosen": 2.4734740257263184,
      "rewards/margins": 7.538750171661377,
      "rewards/rejected": -5.063173770904541,
      "step": 550
    },
    {
      "epoch": 0.12,
      "grad_norm": 0.1806640625,
      "learning_rate": 1.996666666666667e-06,
      "logits/chosen": -1.9733397960662842,
      "logits/rejected": -2.937753915786743,
      "logps/chosen": -142.70875549316406,
      "logps/rejected": -223.96624755859375,
      "loss": 0.0953,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.598142147064209,
      "rewards/margins": 8.3950777053833,
      "rewards/rejected": -5.798213005065918,
      "step": 600
    },
    {
      "epoch": 0.13,
      "grad_norm": 0.2265625,
      "learning_rate": 2.1633333333333335e-06,
      "logits/chosen": -2.0656299591064453,
      "logits/rejected": -3.027539014816284,
      "logps/chosen": -142.24249267578125,
      "logps/rejected": -230.47250366210938,
      "loss": 0.0883,
      "rewards/accuracies": 0.8725000023841858,
      "rewards/chosen": 2.734023332595825,
      "rewards/margins": 9.052812576293945,
      "rewards/rejected": -6.316962718963623,
      "step": 650
    },
    {
      "epoch": 0.14,
      "grad_norm": 0.30859375,
      "learning_rate": 2.33e-06,
      "logits/chosen": -1.9706201553344727,
      "logits/rejected": -2.9912109375,
      "logps/chosen": -138.24249267578125,
      "logps/rejected": -228.1925048828125,
      "loss": 0.09,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": 2.9716455936431885,
      "rewards/margins": 9.053828239440918,
      "rewards/rejected": -6.082441329956055,
      "step": 700
    },
    {
      "epoch": 0.15,
      "grad_norm": 0.01446533203125,
      "learning_rate": 2.4966666666666668e-06,
      "logits/chosen": -1.9857901334762573,
      "logits/rejected": -2.991640567779541,
      "logps/chosen": -134.00999450683594,
      "logps/rejected": -224.47625732421875,
      "loss": 0.0848,
      "rewards/accuracies": 0.8774999976158142,
      "rewards/chosen": 3.2274608612060547,
      "rewards/margins": 9.318828582763672,
      "rewards/rejected": -6.091513633728027,
      "step": 750
    },
    {
      "epoch": 0.16,
      "grad_norm": 0.15625,
      "learning_rate": 2.6633333333333334e-06,
      "logits/chosen": -1.9683300256729126,
      "logits/rejected": -3.0144922733306885,
      "logps/chosen": -142.32000732421875,
      "logps/rejected": -238.3975067138672,
      "loss": 0.071,
      "rewards/accuracies": 0.8974999785423279,
      "rewards/chosen": 3.2487499713897705,
      "rewards/margins": 9.865859031677246,
      "rewards/rejected": -6.616034984588623,
      "step": 800
    },
    {
      "epoch": 0.17,
      "grad_norm": 0.00604248046875,
      "learning_rate": 2.83e-06,
      "logits/chosen": -2.0511231422424316,
      "logits/rejected": -3.087148427963257,
      "logps/chosen": -138.8387451171875,
      "logps/rejected": -237.4425048828125,
      "loss": 0.0813,
      "rewards/accuracies": 0.8824999928474426,
      "rewards/chosen": 3.2463672161102295,
      "rewards/margins": 10.090624809265137,
      "rewards/rejected": -6.84329080581665,
      "step": 850
    },
    {
      "epoch": 0.18,
      "grad_norm": 0.01470947265625,
      "learning_rate": 2.996666666666667e-06,
      "logits/chosen": -2.1478612422943115,
      "logits/rejected": -3.173710823059082,
      "logps/chosen": -143.4824981689453,
      "logps/rejected": -246.1737518310547,
      "loss": 0.0726,
      "rewards/accuracies": 0.8949999809265137,
      "rewards/chosen": 3.197275400161743,
      "rewards/margins": 10.487030982971191,
      "rewards/rejected": -7.29049825668335,
      "step": 900
    },
    {
      "epoch": 0.19,
      "grad_norm": 0.0067138671875,
      "learning_rate": 3.1633333333333337e-06,
      "logits/chosen": -1.9968359470367432,
      "logits/rejected": -3.075078010559082,
      "logps/chosen": -136.73875427246094,
      "logps/rejected": -234.65499877929688,
      "loss": 0.0813,
      "rewards/accuracies": 0.8824999928474426,
      "rewards/chosen": 3.270380973815918,
      "rewards/margins": 10.061562538146973,
      "rewards/rejected": -6.791113376617432,
      "step": 950
    },
    {
      "epoch": 0.2,
      "grad_norm": 0.055908203125,
      "learning_rate": 3.3300000000000003e-06,
      "logits/chosen": -2.032480478286743,
      "logits/rejected": -3.120234489440918,
      "logps/chosen": -141.06625366210938,
      "logps/rejected": -247.29124450683594,
      "loss": 0.0813,
      "rewards/accuracies": 0.8824999928474426,
      "rewards/chosen": 3.2919139862060547,
      "rewards/margins": 10.692343711853027,
      "rewards/rejected": -7.398417949676514,
      "step": 1000
    },
    {
      "epoch": 0.21,
      "grad_norm": 0.017822265625,
      "learning_rate": 3.496666666666667e-06,
      "logits/chosen": -2.1911962032318115,
      "logits/rejected": -3.1728124618530273,
      "logps/chosen": -143.2687530517578,
      "logps/rejected": -246.8125,
      "loss": 0.0847,
      "rewards/accuracies": 0.8774999976158142,
      "rewards/chosen": 3.0637621879577637,
      "rewards/margins": 10.748281478881836,
      "rewards/rejected": -7.685234546661377,
      "step": 1050
    },
    {
      "epoch": 0.22,
      "grad_norm": 0.060546875,
      "learning_rate": 3.6633333333333336e-06,
      "logits/chosen": -2.195253849029541,
      "logits/rejected": -3.108535051345825,
      "logps/chosen": -134.12750244140625,
      "logps/rejected": -233.7412567138672,
      "loss": 0.1124,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.9126367568969727,
      "rewards/margins": 10.165234565734863,
      "rewards/rejected": -7.252851486206055,
      "step": 1100
    },
    {
      "epoch": 0.23,
      "grad_norm": 0.00811767578125,
      "learning_rate": 3.830000000000001e-06,
      "logits/chosen": -2.157832145690918,
      "logits/rejected": -3.157460927963257,
      "logps/chosen": -140.08250427246094,
      "logps/rejected": -249.22000122070312,
      "loss": 0.0709,
      "rewards/accuracies": 0.8974999785423279,
      "rewards/chosen": 3.393681526184082,
      "rewards/margins": 11.291093826293945,
      "rewards/rejected": -7.8983154296875,
      "step": 1150
    },
    {
      "epoch": 0.24,
      "grad_norm": 0.37109375,
      "learning_rate": 3.996666666666667e-06,
      "logits/chosen": -2.176523447036743,
      "logits/rejected": -3.145156145095825,
      "logps/chosen": -136.72625732421875,
      "logps/rejected": -240.5812530517578,
      "loss": 0.0847,
      "rewards/accuracies": 0.8774999976158142,
      "rewards/chosen": 3.007997989654541,
      "rewards/margins": 10.645390510559082,
      "rewards/rejected": -7.636513710021973,
      "step": 1200
    },
    {
      "epoch": 0.25,
      "grad_norm": 0.0303955078125,
      "learning_rate": 4.163333333333334e-06,
      "logits/chosen": -2.346796989440918,
      "logits/rejected": -3.239375114440918,
      "logps/chosen": -141.64999389648438,
      "logps/rejected": -254.55624389648438,
      "loss": 0.0709,
      "rewards/accuracies": 0.8974999785423279,
      "rewards/chosen": 3.1400487422943115,
      "rewards/margins": 11.464765548706055,
      "rewards/rejected": -8.323661804199219,
      "step": 1250
    },
    {
      "epoch": 0.26,
      "grad_norm": 0.8828125,
      "learning_rate": 4.33e-06,
      "logits/chosen": -2.233154296875,
      "logits/rejected": -3.121083974838257,
      "logps/chosen": -137.37875366210938,
      "logps/rejected": -247.23875427246094,
      "loss": 0.0882,
      "rewards/accuracies": 0.8725000023841858,
      "rewards/chosen": 2.990058660507202,
      "rewards/margins": 11.343280792236328,
      "rewards/rejected": -8.351679801940918,
      "step": 1300
    },
    {
      "epoch": 0.27,
      "grad_norm": 0.0081787109375,
      "learning_rate": 4.496666666666667e-06,
      "logits/chosen": -2.3371288776397705,
      "logits/rejected": -3.1724023818969727,
      "logps/chosen": -142.37249755859375,
      "logps/rejected": -257.1512451171875,
      "loss": 0.0519,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.159780263900757,
      "rewards/margins": 11.788749694824219,
      "rewards/rejected": -8.627890586853027,
      "step": 1350
    },
    {
      "epoch": 0.28,
      "grad_norm": 1.15625,
      "learning_rate": 4.663333333333333e-06,
      "logits/chosen": -2.136476993560791,
      "logits/rejected": -3.0350780487060547,
      "logps/chosen": -141.83624267578125,
      "logps/rejected": -258.1824951171875,
      "loss": 0.0882,
      "rewards/accuracies": 0.8725000023841858,
      "rewards/chosen": 2.5560545921325684,
      "rewards/margins": 11.789999961853027,
      "rewards/rejected": -9.23446273803711,
      "step": 1400
    },
    {
      "epoch": 0.29,
      "grad_norm": 0.001434326171875,
      "learning_rate": 4.83e-06,
      "logits/chosen": -2.327712297439575,
      "logits/rejected": -3.039235830307007,
      "logps/chosen": -151.52749633789062,
      "logps/rejected": -260.1824951171875,
      "loss": 0.0916,
      "rewards/accuracies": 0.8675000071525574,
      "rewards/chosen": 1.725927710533142,
      "rewards/margins": 11.106093406677246,
      "rewards/rejected": -9.376816749572754,
      "step": 1450
    },
    {
      "epoch": 0.3,
      "grad_norm": 0.000873565673828125,
      "learning_rate": 4.9966666666666665e-06,
      "logits/chosen": -2.236884832382202,
      "logits/rejected": -2.9620020389556885,
      "logps/chosen": -152.15249633789062,
      "logps/rejected": -266.6524963378906,
      "loss": 0.083,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": 1.9792224168777466,
      "rewards/margins": 11.753125190734863,
      "rewards/rejected": -9.775390625,
      "step": 1500
    },
    {
      "epoch": 0.31,
      "grad_norm": 1.1484375,
      "learning_rate": 4.99983747144442e-06,
      "logits/chosen": -2.1749095916748047,
      "logits/rejected": -3.0108959674835205,
      "logps/chosen": -158.8925018310547,
      "logps/rejected": -274.04998779296875,
      "loss": 0.0795,
      "rewards/accuracies": 0.8849999904632568,
      "rewards/chosen": 1.5659692287445068,
      "rewards/margins": 11.828437805175781,
      "rewards/rejected": -10.266562461853027,
      "step": 1550
    },
    {
      "epoch": 0.32,
      "grad_norm": 0.00445556640625,
      "learning_rate": 4.999336572604176e-06,
      "logits/chosen": -2.3599512577056885,
      "logits/rejected": -2.979990243911743,
      "logps/chosen": -150.02499389648438,
      "logps/rejected": -264.9649963378906,
      "loss": 0.0934,
      "rewards/accuracies": 0.8650000095367432,
      "rewards/chosen": 1.7431994676589966,
      "rewards/margins": 11.767656326293945,
      "rewards/rejected": -10.0248441696167,
      "step": 1600
    },
    {
      "epoch": 0.33,
      "grad_norm": 0.07763671875,
      "learning_rate": 4.998497303600191e-06,
      "logits/chosen": -2.491284132003784,
      "logits/rejected": -3.098569393157959,
      "logps/chosen": -151.4949951171875,
      "logps/rejected": -261.3475036621094,
      "loss": 0.0916,
      "rewards/accuracies": 0.8675000071525574,
      "rewards/chosen": 1.6169995069503784,
      "rewards/margins": 11.283594131469727,
      "rewards/rejected": -9.667499542236328,
      "step": 1650
    },
    {
      "epoch": 0.34,
      "grad_norm": 0.001434326171875,
      "learning_rate": 4.997319778056057e-06,
      "logits/chosen": -2.4677734375,
      "logits/rejected": -3.164921760559082,
      "logps/chosen": -160.6125030517578,
      "logps/rejected": -291.8324890136719,
      "loss": 0.0779,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.3262377977371216,
      "rewards/margins": 13.2251558303833,
      "rewards/rejected": -11.900468826293945,
      "step": 1700
    },
    {
      "epoch": 0.35,
      "grad_norm": 0.0142822265625,
      "learning_rate": 4.995804155389881e-06,
      "logits/chosen": -2.2496564388275146,
      "logits/rejected": -2.910781145095825,
      "logps/chosen": -163.3000030517578,
      "logps/rejected": -282.75,
      "loss": 0.0899,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": 1.067590355873108,
      "rewards/margins": 12.181406021118164,
      "rewards/rejected": -11.118515968322754,
      "step": 1750
    },
    {
      "epoch": 0.36,
      "grad_norm": 1.4921875,
      "learning_rate": 4.9939506407927115e-06,
      "logits/chosen": -2.496284246444702,
      "logits/rejected": -3.016840934753418,
      "logps/chosen": -165.72999572753906,
      "logps/rejected": -291.8699951171875,
      "loss": 0.0812,
      "rewards/accuracies": 0.8824999928474426,
      "rewards/chosen": 0.7725061178207397,
      "rewards/margins": 12.811249732971191,
      "rewards/rejected": -12.039375305175781,
      "step": 1800
    },
    {
      "epoch": 0.37,
      "grad_norm": 0.0125732421875,
      "learning_rate": 4.991759485200754e-06,
      "logits/chosen": -2.5551586151123047,
      "logits/rejected": -2.9110498428344727,
      "logps/chosen": -162.89500427246094,
      "logps/rejected": -287.5174865722656,
      "loss": 0.0951,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.4655786156654358,
      "rewards/margins": 12.595937728881836,
      "rewards/rejected": -12.12906265258789,
      "step": 1850
    },
    {
      "epoch": 0.38,
      "grad_norm": 1.140625,
      "learning_rate": 4.989230985261403e-06,
      "logits/chosen": -2.6131153106689453,
      "logits/rejected": -2.968681573867798,
      "logps/chosen": -168.32749938964844,
      "logps/rejected": -289.5,
      "loss": 0.0795,
      "rewards/accuracies": 0.8849999904632568,
      "rewards/chosen": 0.41573241353034973,
      "rewards/margins": 12.315781593322754,
      "rewards/rejected": -11.899062156677246,
      "step": 1900
    },
    {
      "epoch": 0.39,
      "grad_norm": 0.0218505859375,
      "learning_rate": 4.986365483293072e-06,
      "logits/chosen": -2.2319605350494385,
      "logits/rejected": -2.6794629096984863,
      "logps/chosen": -165.3937530517578,
      "logps/rejected": -280.6925048828125,
      "loss": 0.0832,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": 0.6209277510643005,
      "rewards/margins": 11.815312385559082,
      "rewards/rejected": -11.191679954528809,
      "step": 1950
    },
    {
      "epoch": 0.4,
      "grad_norm": 1.2578125,
      "learning_rate": 4.9831633672388605e-06,
      "logits/chosen": -2.148073673248291,
      "logits/rejected": -2.8818554878234863,
      "logps/chosen": -156.14500427246094,
      "logps/rejected": -290.29998779296875,
      "loss": 0.0674,
      "rewards/accuracies": 0.9024999737739563,
      "rewards/chosen": 1.61712646484375,
      "rewards/margins": 13.622812271118164,
      "rewards/rejected": -12.008906364440918,
      "step": 2000
    },
    {
      "epoch": 0.41,
      "grad_norm": 0.00494384765625,
      "learning_rate": 4.979625070614023e-06,
      "logits/chosen": -1.928134799003601,
      "logits/rejected": -2.771103620529175,
      "logps/chosen": -149.2050018310547,
      "logps/rejected": -288.3500061035156,
      "loss": 0.0674,
      "rewards/accuracies": 0.9024999737739563,
      "rewards/chosen": 2.3412108421325684,
      "rewards/margins": 14.15218734741211,
      "rewards/rejected": -11.807969093322754,
      "step": 2050
    },
    {
      "epoch": 0.42,
      "grad_norm": 0.047607421875,
      "learning_rate": 4.975751072447283e-06,
      "logits/chosen": -1.7242242097854614,
      "logits/rejected": -2.762929677963257,
      "logps/chosen": -138.24374389648438,
      "logps/rejected": -269.7225036621094,
      "loss": 0.0622,
      "rewards/accuracies": 0.9100000262260437,
      "rewards/chosen": 3.4145116806030273,
      "rewards/margins": 13.523906707763672,
      "rewards/rejected": -10.108515739440918,
      "step": 2100
    },
    {
      "epoch": 0.43,
      "grad_norm": 0.93359375,
      "learning_rate": 4.9715418972159794e-06,
      "logits/chosen": -1.8216897249221802,
      "logits/rejected": -2.836932420730591,
      "logps/chosen": -136.2324981689453,
      "logps/rejected": -263.7225036621094,
      "loss": 0.102,
      "rewards/accuracies": 0.8525000214576721,
      "rewards/chosen": 3.042714834213257,
      "rewards/margins": 12.95718765258789,
      "rewards/rejected": -9.909453392028809,
      "step": 2150
    },
    {
      "epoch": 0.44,
      "grad_norm": 0.000362396240234375,
      "learning_rate": 4.96699811477506e-06,
      "logits/chosen": -2.153923273086548,
      "logits/rejected": -3.0781617164611816,
      "logps/chosen": -139.92250061035156,
      "logps/rejected": -269.2449951171875,
      "loss": 0.0882,
      "rewards/accuracies": 0.8725000023841858,
      "rewards/chosen": 2.6978321075439453,
      "rewards/margins": 13.111719131469727,
      "rewards/rejected": -10.4213285446167,
      "step": 2200
    },
    {
      "epoch": 0.45,
      "grad_norm": 0.00933837890625,
      "learning_rate": 4.962120340279933e-06,
      "logits/chosen": -2.206913948059082,
      "logits/rejected": -3.013432502746582,
      "logps/chosen": -149.11500549316406,
      "logps/rejected": -284.74749755859375,
      "loss": 0.0778,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.5205469131469727,
      "rewards/margins": 13.883281707763672,
      "rewards/rejected": -11.363203048706055,
      "step": 2250
    },
    {
      "epoch": 0.46,
      "grad_norm": 0.007232666015625,
      "learning_rate": 4.956909234103184e-06,
      "logits/chosen": -2.232741594314575,
      "logits/rejected": -2.846698045730591,
      "logps/chosen": -147.88999938964844,
      "logps/rejected": -290.8949890136719,
      "loss": 0.0691,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.5749120712280273,
      "rewards/margins": 14.527656555175781,
      "rewards/rejected": -11.950780868530273,
      "step": 2300
    },
    {
      "epoch": 0.47,
      "grad_norm": 0.6015625,
      "learning_rate": 4.951365501745172e-06,
      "logits/chosen": -2.397939443588257,
      "logits/rejected": -2.8929004669189453,
      "logps/chosen": -144.75750732421875,
      "logps/rejected": -275.7850036621094,
      "loss": 0.0899,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": 2.490410089492798,
      "rewards/margins": 13.264843940734863,
      "rewards/rejected": -10.7734375,
      "step": 2350
    },
    {
      "epoch": 0.48,
      "grad_norm": 0.02099609375,
      "learning_rate": 4.945489893738518e-06,
      "logits/chosen": -2.36920166015625,
      "logits/rejected": -2.889394521713257,
      "logps/chosen": -136.7274932861328,
      "logps/rejected": -272.572509765625,
      "loss": 0.0951,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.901992082595825,
      "rewards/margins": 13.673593521118164,
      "rewards/rejected": -10.7650785446167,
      "step": 2400
    },
    {
      "epoch": 0.49,
      "grad_norm": 0.007110595703125,
      "learning_rate": 4.93928320554649e-06,
      "logits/chosen": -1.9525684118270874,
      "logits/rejected": -2.8195831775665283,
      "logps/chosen": -135.28875732421875,
      "logps/rejected": -266.6099853515625,
      "loss": 0.0882,
      "rewards/accuracies": 0.8725000023841858,
      "rewards/chosen": 3.462538957595825,
      "rewards/margins": 13.346015930175781,
      "rewards/rejected": -9.8804292678833,
      "step": 2450
    },
    {
      "epoch": 0.5,
      "grad_norm": 0.94140625,
      "learning_rate": 4.932746277455317e-06,
      "logits/chosen": -1.662766456604004,
      "logits/rejected": -2.691631555557251,
      "logps/chosen": -133.22000122070312,
      "logps/rejected": -272.822509765625,
      "loss": 0.1037,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 3.5004687309265137,
      "rewards/margins": 14.248437881469727,
      "rewards/rejected": -10.7514066696167,
      "step": 2500
    },
    {
      "epoch": 0.51,
      "grad_norm": 0.00982666015625,
      "learning_rate": 4.9258799944604215e-06,
      "logits/chosen": -1.7904162406921387,
      "logits/rejected": -2.6735968589782715,
      "logps/chosen": -132.4512481689453,
      "logps/rejected": -281.63751220703125,
      "loss": 0.0743,
      "rewards/accuracies": 0.8924999833106995,
      "rewards/chosen": 3.9552342891693115,
      "rewards/margins": 15.276874542236328,
      "rewards/rejected": -11.324609756469727,
      "step": 2550
    },
    {
      "epoch": 0.52,
      "grad_norm": 0.013916015625,
      "learning_rate": 4.918685286146611e-06,
      "logits/chosen": -1.7311677932739258,
      "logits/rejected": -2.691605567932129,
      "logps/chosen": -132.29249572753906,
      "logps/rejected": -285.6025085449219,
      "loss": 0.0834,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": 3.967148542404175,
      "rewards/margins": 15.4975004196167,
      "rewards/rejected": -11.534062385559082,
      "step": 2600
    },
    {
      "epoch": 0.53,
      "grad_norm": 0.005035400390625,
      "learning_rate": 4.911163126562218e-06,
      "logits/chosen": -1.7594763040542603,
      "logits/rejected": -2.734794855117798,
      "logps/chosen": -134.93499755859375,
      "logps/rejected": -289.4525146484375,
      "loss": 0.0933,
      "rewards/accuracies": 0.8650000095367432,
      "rewards/chosen": 3.4932031631469727,
      "rewards/margins": 15.765155792236328,
      "rewards/rejected": -12.267890930175781,
      "step": 2650
    },
    {
      "epoch": 0.54,
      "grad_norm": 0.00014495849609375,
      "learning_rate": 4.903314534087243e-06,
      "logits/chosen": -1.64252507686615,
      "logits/rejected": -2.694523811340332,
      "logps/chosen": -137.2375030517578,
      "logps/rejected": -277.3800048828125,
      "loss": 0.0743,
      "rewards/accuracies": 0.8924999833106995,
      "rewards/chosen": 3.0264551639556885,
      "rewards/margins": 14.148124694824219,
      "rewards/rejected": -11.121874809265137,
      "step": 2700
    },
    {
      "epoch": 0.55,
      "grad_norm": 0.73046875,
      "learning_rate": 4.895140571295469e-06,
      "logits/chosen": -1.6052197217941284,
      "logits/rejected": -2.6878418922424316,
      "logps/chosen": -130.99374389648438,
      "logps/rejected": -274.2774963378906,
      "loss": 0.0933,
      "rewards/accuracies": 0.8650000095367432,
      "rewards/chosen": 3.450859308242798,
      "rewards/margins": 14.58187484741211,
      "rewards/rejected": -11.129921913146973,
      "step": 2750
    },
    {
      "epoch": 0.56,
      "grad_norm": 0.001068115234375,
      "learning_rate": 4.886642344810612e-06,
      "logits/chosen": -2.1891088485717773,
      "logits/rejected": -2.992116689682007,
      "logps/chosen": -131.99749755859375,
      "logps/rejected": -261.6474914550781,
      "loss": 0.1072,
      "rewards/accuracies": 0.8450000286102295,
      "rewards/chosen": 3.183906316757202,
      "rewards/margins": 13.224843978881836,
      "rewards/rejected": -10.036718368530273,
      "step": 2800
    },
    {
      "epoch": 0.57,
      "grad_norm": 0.0004825592041015625,
      "learning_rate": 4.877821005156504e-06,
      "logits/chosen": -2.3619725704193115,
      "logits/rejected": -3.226367235183716,
      "logps/chosen": -138.77000427246094,
      "logps/rejected": -263.8999938964844,
      "loss": 0.0951,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.243339776992798,
      "rewards/margins": 12.722031593322754,
      "rewards/rejected": -9.478281021118164,
      "step": 2850
    },
    {
      "epoch": 0.58,
      "grad_norm": 0.0098876953125,
      "learning_rate": 4.868677746601325e-06,
      "logits/chosen": -2.1765477657318115,
      "logits/rejected": -3.075366258621216,
      "logps/chosen": -134.6649932861328,
      "logps/rejected": -251.0850067138672,
      "loss": 0.102,
      "rewards/accuracies": 0.8525000214576721,
      "rewards/chosen": 3.067031145095825,
      "rewards/margins": 11.9115629196167,
      "rewards/rejected": -8.846875190734863,
      "step": 2900
    },
    {
      "epoch": 0.59,
      "grad_norm": 1.1796875,
      "learning_rate": 4.859213806995924e-06,
      "logits/chosen": -2.022216796875,
      "logits/rejected": -3.042172908782959,
      "logps/chosen": -135.00750732421875,
      "logps/rejected": -262.7250061035156,
      "loss": 0.0812,
      "rewards/accuracies": 0.8824999928474426,
      "rewards/chosen": 3.4410157203674316,
      "rewards/margins": 13.021562576293945,
      "rewards/rejected": -9.582109451293945,
      "step": 2950
    },
    {
      "epoch": 0.6,
      "grad_norm": 0.0026092529296875,
      "learning_rate": 4.849430467606228e-06,
      "logits/chosen": -1.728432059288025,
      "logits/rejected": -2.991831064224243,
      "logps/chosen": -129.99374389648438,
      "logps/rejected": -245.56500244140625,
      "loss": 0.1003,
      "rewards/accuracies": 0.8550000190734863,
      "rewards/chosen": 3.511894464492798,
      "rewards/margins": 11.818437576293945,
      "rewards/rejected": -8.303203582763672,
      "step": 3000
    },
    {
      "epoch": 0.61,
      "grad_norm": 0.00012111663818359375,
      "learning_rate": 4.839329052939784e-06,
      "logits/chosen": -1.3972948789596558,
      "logits/rejected": -2.9753711223602295,
      "logps/chosen": -133.10374450683594,
      "logps/rejected": -260.8924865722656,
      "loss": 0.0778,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.828476667404175,
      "rewards/margins": 12.948124885559082,
      "rewards/rejected": -9.120469093322754,
      "step": 3050
    },
    {
      "epoch": 0.62,
      "grad_norm": 0.53515625,
      "learning_rate": 4.82891093056644e-06,
      "logits/chosen": -1.142480492591858,
      "logits/rejected": -2.7538769245147705,
      "logps/chosen": -126.52874755859375,
      "logps/rejected": -251.2100067138672,
      "loss": 0.0985,
      "rewards/accuracies": 0.8575000166893005,
      "rewards/chosen": 4.20089864730835,
      "rewards/margins": 12.569218635559082,
      "rewards/rejected": -8.369375228881836,
      "step": 3100
    },
    {
      "epoch": 0.63,
      "grad_norm": 0.0034332275390625,
      "learning_rate": 4.818177510933194e-06,
      "logits/chosen": -1.1324691772460938,
      "logits/rejected": -2.7247815132141113,
      "logps/chosen": -128.28125,
      "logps/rejected": -250.6425018310547,
      "loss": 0.0933,
      "rewards/accuracies": 0.8650000095367432,
      "rewards/chosen": 4.047187328338623,
      "rewards/margins": 12.607500076293945,
      "rewards/rejected": -8.559687614440918,
      "step": 3150
    },
    {
      "epoch": 0.64,
      "grad_norm": 0.015380859375,
      "learning_rate": 4.807130247173252e-06,
      "logits/chosen": -0.9328582882881165,
      "logits/rejected": -2.533745050430298,
      "logps/chosen": -124.90499877929688,
      "logps/rejected": -245.25250244140625,
      "loss": 0.0916,
      "rewards/accuracies": 0.8675000071525574,
      "rewards/chosen": 4.320508003234863,
      "rewards/margins": 12.207812309265137,
      "rewards/rejected": -7.890781402587891,
      "step": 3200
    },
    {
      "epoch": 0.65,
      "grad_norm": 0.546875,
      "learning_rate": 4.795770634909287e-06,
      "logits/chosen": -0.9098020792007446,
      "logits/rejected": -2.5764403343200684,
      "logps/chosen": -127.57749938964844,
      "logps/rejected": -257.6549987792969,
      "loss": 0.0657,
      "rewards/accuracies": 0.9049999713897705,
      "rewards/chosen": 4.306952953338623,
      "rewards/margins": 13.306875228881836,
      "rewards/rejected": -8.999530792236328,
      "step": 3250
    },
    {
      "epoch": 0.66,
      "grad_norm": 0.00775146484375,
      "learning_rate": 4.784100212050959e-06,
      "logits/chosen": -1.165554165840149,
      "logits/rejected": -2.720310688018799,
      "logps/chosen": -135.34750366210938,
      "logps/rejected": -268.3550109863281,
      "loss": 0.0795,
      "rewards/accuracies": 0.8849999904632568,
      "rewards/chosen": 3.560781240463257,
      "rewards/margins": 13.581093788146973,
      "rewards/rejected": -10.021249771118164,
      "step": 3300
    },
    {
      "epoch": 0.67,
      "grad_norm": 0.67578125,
      "learning_rate": 4.772120558586711e-06,
      "logits/chosen": -1.4726300239562988,
      "logits/rejected": -2.9082860946655273,
      "logps/chosen": -136.72500610351562,
      "logps/rejected": -266.2174987792969,
      "loss": 0.0864,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.1438281536102295,
      "rewards/margins": 13.237656593322754,
      "rewards/rejected": -10.093280792236328,
      "step": 3350
    },
    {
      "epoch": 0.68,
      "grad_norm": 0.000457763671875,
      "learning_rate": 4.759833296369855e-06,
      "logits/chosen": -1.7112644910812378,
      "logits/rejected": -2.9018359184265137,
      "logps/chosen": -144.86500549316406,
      "logps/rejected": -274.8800048828125,
      "loss": 0.0864,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.424466609954834,
      "rewards/margins": 13.268437385559082,
      "rewards/rejected": -10.842421531677246,
      "step": 3400
    },
    {
      "epoch": 0.69,
      "grad_norm": 0.00023937225341796875,
      "learning_rate": 4.747240088899007e-06,
      "logits/chosen": -1.8725781440734863,
      "logits/rejected": -2.871826171875,
      "logps/chosen": -145.64500427246094,
      "logps/rejected": -284.7799987792969,
      "loss": 0.083,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": 2.372001886367798,
      "rewards/margins": 13.883749961853027,
      "rewards/rejected": -11.511327743530273,
      "step": 3450
    },
    {
      "epoch": 0.7,
      "grad_norm": 0.0089111328125,
      "learning_rate": 4.734342641092873e-06,
      "logits/chosen": -1.76885986328125,
      "logits/rejected": -2.764961004257202,
      "logps/chosen": -133.86749267578125,
      "logps/rejected": -264.427490234375,
      "loss": 0.0882,
      "rewards/accuracies": 0.8725000023841858,
      "rewards/chosen": 3.250908136367798,
      "rewards/margins": 13.281562805175781,
      "rewards/rejected": -10.029687881469727,
      "step": 3500
    },
    {
      "epoch": 0.71,
      "grad_norm": 0.009765625,
      "learning_rate": 4.72114269905943e-06,
      "logits/chosen": -1.7043505907058716,
      "logits/rejected": -2.7690234184265137,
      "logps/chosen": -130.1699981689453,
      "logps/rejected": -268.5350036621094,
      "loss": 0.0761,
      "rewards/accuracies": 0.8899999856948853,
      "rewards/chosen": 3.886406183242798,
      "rewards/margins": 13.834218978881836,
      "rewards/rejected": -9.946484565734863,
      "step": 3550
    },
    {
      "epoch": 0.72,
      "grad_norm": 0.004180908203125,
      "learning_rate": 4.70764204985953e-06,
      "logits/chosen": -1.7190561294555664,
      "logits/rejected": -2.7930660247802734,
      "logps/chosen": -132.5762481689453,
      "logps/rejected": -261.9324951171875,
      "loss": 0.0951,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.5982422828674316,
      "rewards/margins": 13.246718406677246,
      "rewards/rejected": -9.651249885559082,
      "step": 3600
    },
    {
      "epoch": 0.73,
      "grad_norm": 0.3984375,
      "learning_rate": 4.693842521264963e-06,
      "logits/chosen": -1.6216427087783813,
      "logits/rejected": -2.808516263961792,
      "logps/chosen": -126.45999908447266,
      "logps/rejected": -260.43499755859375,
      "loss": 0.0968,
      "rewards/accuracies": 0.8600000143051147,
      "rewards/chosen": 3.8330078125,
      "rewards/margins": 13.640625,
      "rewards/rejected": -9.81070327758789,
      "step": 3650
    },
    {
      "epoch": 0.74,
      "grad_norm": 0.0030670166015625,
      "learning_rate": 4.679745981511005e-06,
      "logits/chosen": -1.497025728225708,
      "logits/rejected": -2.7889842987060547,
      "logps/chosen": -127.35624694824219,
      "logps/rejected": -263.0849914550781,
      "loss": 0.0812,
      "rewards/accuracies": 0.8824999928474426,
      "rewards/chosen": 4.245625019073486,
      "rewards/margins": 13.9360933303833,
      "rewards/rejected": -9.694062232971191,
      "step": 3700
    },
    {
      "epoch": 0.75,
      "grad_norm": 0.4609375,
      "learning_rate": 4.665354339043487e-06,
      "logits/chosen": -1.61189603805542,
      "logits/rejected": -2.818247079849243,
      "logps/chosen": -125.53874969482422,
      "logps/rejected": -262.56500244140625,
      "loss": 0.0882,
      "rewards/accuracies": 0.8725000023841858,
      "rewards/chosen": 4.035273551940918,
      "rewards/margins": 13.862030982971191,
      "rewards/rejected": -9.828046798706055,
      "step": 3750
    },
    {
      "epoch": 0.76,
      "grad_norm": 0.005584716796875,
      "learning_rate": 4.650669542260426e-06,
      "logits/chosen": -1.8283697366714478,
      "logits/rejected": -3.02060604095459,
      "logps/chosen": -132.30624389648438,
      "logps/rejected": -265.7875061035156,
      "loss": 0.0882,
      "rewards/accuracies": 0.8725000023841858,
      "rewards/chosen": 3.852304697036743,
      "rewards/margins": 13.544843673706055,
      "rewards/rejected": -9.695077896118164,
      "step": 3800
    },
    {
      "epoch": 0.77,
      "grad_norm": 0.006256103515625,
      "learning_rate": 4.635693579248238e-06,
      "logits/chosen": -1.77850341796875,
      "logits/rejected": -2.859311580657959,
      "logps/chosen": -131.875,
      "logps/rejected": -267.4624938964844,
      "loss": 0.0882,
      "rewards/accuracies": 0.8725000023841858,
      "rewards/chosen": 3.6142187118530273,
      "rewards/margins": 13.896875381469727,
      "rewards/rejected": -10.281562805175781,
      "step": 3850
    },
    {
      "epoch": 0.78,
      "grad_norm": 0.0023345947265625,
      "learning_rate": 4.620428477512588e-06,
      "logits/chosen": -1.6927603483200073,
      "logits/rejected": -2.8547582626342773,
      "logps/chosen": -132.7550048828125,
      "logps/rejected": -273.0150146484375,
      "loss": 0.0847,
      "rewards/accuracies": 0.8774999976158142,
      "rewards/chosen": 3.655820369720459,
      "rewards/margins": 14.208906173706055,
      "rewards/rejected": -10.555000305175781,
      "step": 3900
    },
    {
      "epoch": 0.79,
      "grad_norm": 0.8671875,
      "learning_rate": 4.604876303703892e-06,
      "logits/chosen": -1.5046154260635376,
      "logits/rejected": -2.71980357170105,
      "logps/chosen": -127.0512466430664,
      "logps/rejected": -260.4525146484375,
      "loss": 0.0882,
      "rewards/accuracies": 0.8725000023841858,
      "rewards/chosen": 3.969980478286743,
      "rewards/margins": 13.666093826293945,
      "rewards/rejected": -9.696015357971191,
      "step": 3950
    },
    {
      "epoch": 0.8,
      "grad_norm": 0.91796875,
      "learning_rate": 4.5890391633375345e-06,
      "logits/chosen": -1.6800073385238647,
      "logits/rejected": -2.810532331466675,
      "logps/chosen": -141.1374969482422,
      "logps/rejected": -285.43499755859375,
      "loss": 0.0812,
      "rewards/accuracies": 0.8824999928474426,
      "rewards/chosen": 3.1419239044189453,
      "rewards/margins": 14.576562881469727,
      "rewards/rejected": -11.4341402053833,
      "step": 4000
    },
    {
      "epoch": 0.81,
      "grad_norm": 1.171875,
      "learning_rate": 4.572919200508805e-06,
      "logits/chosen": -1.7100884914398193,
      "logits/rejected": -2.719179630279541,
      "logps/chosen": -143.1024932861328,
      "logps/rejected": -274.3575134277344,
      "loss": 0.1037,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.4612793922424316,
      "rewards/margins": 13.33968734741211,
      "rewards/rejected": -10.880234718322754,
      "step": 4050
    },
    {
      "epoch": 0.82,
      "grad_norm": 0.78515625,
      "learning_rate": 4.556518597602633e-06,
      "logits/chosen": -1.8037463426589966,
      "logits/rejected": -2.8130078315734863,
      "logps/chosen": -149.1824951171875,
      "logps/rejected": -280.9200134277344,
      "loss": 0.0916,
      "rewards/accuracies": 0.8675000071525574,
      "rewards/chosen": 2.325624942779541,
      "rewards/margins": 13.437812805175781,
      "rewards/rejected": -11.111093521118164,
      "step": 4100
    },
    {
      "epoch": 0.83,
      "grad_norm": 0.00099945068359375,
      "learning_rate": 4.539839574998117e-06,
      "logits/chosen": -1.8497778177261353,
      "logits/rejected": -2.873305559158325,
      "logps/chosen": -151.7624969482422,
      "logps/rejected": -289.9024963378906,
      "loss": 0.0847,
      "rewards/accuracies": 0.8774999976158142,
      "rewards/chosen": 2.024599552154541,
      "rewards/margins": 14.141249656677246,
      "rewards/rejected": -12.12093734741211,
      "step": 4150
    },
    {
      "epoch": 0.84,
      "grad_norm": 5.90625,
      "learning_rate": 4.522884390767928e-06,
      "logits/chosen": -2.027651309967041,
      "logits/rejected": -2.663203239440918,
      "logps/chosen": -154.8524932861328,
      "logps/rejected": -288.4624938964844,
      "loss": 0.1054,
      "rewards/accuracies": 0.8475000262260437,
      "rewards/chosen": 0.9060644507408142,
      "rewards/margins": 13.541406631469727,
      "rewards/rejected": -12.634062767028809,
      "step": 4200
    },
    {
      "epoch": 0.85,
      "grad_norm": 0.671875,
      "learning_rate": 4.5056553403726014e-06,
      "logits/chosen": -1.9349523782730103,
      "logits/rejected": -2.789375066757202,
      "logps/chosen": -148.86500549316406,
      "logps/rejected": -268.6025085449219,
      "loss": 0.0847,
      "rewards/accuracies": 0.8774999976158142,
      "rewards/chosen": 2.1701366901397705,
      "rewards/margins": 12.2095308303833,
      "rewards/rejected": -10.041191101074219,
      "step": 4250
    },
    {
      "epoch": 0.86,
      "grad_norm": 5.78125,
      "learning_rate": 4.488154756349765e-06,
      "logits/chosen": -1.8653271198272705,
      "logits/rejected": -2.8621387481689453,
      "logps/chosen": -143.14625549316406,
      "logps/rejected": -274.0362548828125,
      "loss": 0.0882,
      "rewards/accuracies": 0.8725000023841858,
      "rewards/chosen": 2.540623664855957,
      "rewards/margins": 13.264374732971191,
      "rewards/rejected": -10.722421646118164,
      "step": 4300
    },
    {
      "epoch": 0.87,
      "grad_norm": 0.65234375,
      "learning_rate": 4.470385007998354e-06,
      "logits/chosen": -2.208327054977417,
      "logits/rejected": -3.1323602199554443,
      "logps/chosen": -146.74249267578125,
      "logps/rejected": -301.1075134277344,
      "loss": 0.0968,
      "rewards/accuracies": 0.8600000143051147,
      "rewards/chosen": 2.182009220123291,
      "rewards/margins": 15.532031059265137,
      "rewards/rejected": -13.352360725402832,
      "step": 4350
    },
    {
      "epoch": 0.88,
      "grad_norm": 0.93359375,
      "learning_rate": 4.452348501057847e-06,
      "logits/chosen": -2.0818140506744385,
      "logits/rejected": -3.202207088470459,
      "logps/chosen": -157.0050048828125,
      "logps/rejected": -315.1000061035156,
      "loss": 0.0951,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.374355435371399,
      "rewards/margins": 15.971718788146973,
      "rewards/rejected": -14.5912504196167,
      "step": 4400
    },
    {
      "epoch": 0.89,
      "grad_norm": 0.0002765655517578125,
      "learning_rate": 4.434047677382563e-06,
      "logits/chosen": -1.5668798685073853,
      "logits/rejected": -2.893112897872925,
      "logps/chosen": -147.7725067138672,
      "logps/rejected": -312.5950012207031,
      "loss": 0.0916,
      "rewards/accuracies": 0.8675000071525574,
      "rewards/chosen": 1.8955810070037842,
      "rewards/margins": 16.66828155517578,
      "rewards/rejected": -14.76968765258789,
      "step": 4450
    },
    {
      "epoch": 0.9,
      "grad_norm": 7.724761962890625e-05,
      "learning_rate": 4.415485014611076e-06,
      "logits/chosen": -1.6306848526000977,
      "logits/rejected": -3.0001611709594727,
      "logps/chosen": -143.4774932861328,
      "logps/rejected": -301.7674865722656,
      "loss": 0.1037,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.233813524246216,
      "rewards/margins": 16.112031936645508,
      "rewards/rejected": -13.87906265258789,
      "step": 4500
    },
    {
      "epoch": 0.91,
      "grad_norm": 1.3984375,
      "learning_rate": 4.396663025830785e-06,
      "logits/chosen": -1.5060467720031738,
      "logits/rejected": -3.0440375804901123,
      "logps/chosen": -146.45249938964844,
      "logps/rejected": -308.010009765625,
      "loss": 0.0882,
      "rewards/accuracies": 0.8725000023841858,
      "rewards/chosen": 2.640126943588257,
      "rewards/margins": 16.458906173706055,
      "rewards/rejected": -13.8149995803833,
      "step": 4550
    },
    {
      "epoch": 0.92,
      "grad_norm": 0.00164031982421875,
      "learning_rate": 4.377584259237676e-06,
      "logits/chosen": -1.5768399238586426,
      "logits/rejected": -3.0472412109375,
      "logps/chosen": -144.15249633789062,
      "logps/rejected": -300.93499755859375,
      "loss": 0.0951,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.490654230117798,
      "rewards/margins": 16.099843978881836,
      "rewards/rejected": -13.607500076293945,
      "step": 4600
    },
    {
      "epoch": 0.93,
      "grad_norm": 0.012451171875,
      "learning_rate": 4.358251297791342e-06,
      "logits/chosen": -1.487817406654358,
      "logits/rejected": -3.055607795715332,
      "logps/chosen": -156.3074951171875,
      "logps/rejected": -315.9574890136719,
      "loss": 0.0743,
      "rewards/accuracies": 0.8924999833106995,
      "rewards/chosen": 1.7694629430770874,
      "rewards/margins": 16.351093292236328,
      "rewards/rejected": -14.579843521118164,
      "step": 4650
    },
    {
      "epoch": 0.94,
      "grad_norm": 0.8359375,
      "learning_rate": 4.338666758865291e-06,
      "logits/chosen": -1.5760504007339478,
      "logits/rejected": -3.079218864440918,
      "logps/chosen": -161.78250122070312,
      "logps/rejected": -324.3299865722656,
      "loss": 0.0778,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.9159045219421387,
      "rewards/margins": 16.36734390258789,
      "rewards/rejected": -15.450937271118164,
      "step": 4700
    },
    {
      "epoch": 0.95,
      "grad_norm": 0.000255584716796875,
      "learning_rate": 4.318833293892593e-06,
      "logits/chosen": -1.5287631750106812,
      "logits/rejected": -2.8774261474609375,
      "logps/chosen": -165.83999633789062,
      "logps/rejected": -315.05499267578125,
      "loss": 0.0916,
      "rewards/accuracies": 0.8675000071525574,
      "rewards/chosen": 0.16832397878170013,
      "rewards/margins": 15.089219093322754,
      "rewards/rejected": -14.9165620803833,
      "step": 4750
    },
    {
      "epoch": 0.96,
      "grad_norm": 0.82421875,
      "learning_rate": 4.2987535880069194e-06,
      "logits/chosen": -1.7194920778274536,
      "logits/rejected": -3.0331249237060547,
      "logps/chosen": -173.74000549316406,
      "logps/rejected": -324.07501220703125,
      "loss": 0.0864,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08797118812799454,
      "rewards/margins": 15.2670316696167,
      "rewards/rejected": -15.170312881469727,
      "step": 4800
    },
    {
      "epoch": 0.97,
      "grad_norm": 0.68359375,
      "learning_rate": 4.278430359679022e-06,
      "logits/chosen": -1.6618307828903198,
      "logits/rejected": -2.7870311737060547,
      "logps/chosen": -171.0500030517578,
      "logps/rejected": -332.36248779296875,
      "loss": 0.0761,
      "rewards/accuracies": 0.8899999856948853,
      "rewards/chosen": -0.015126952901482582,
      "rewards/margins": 16.22640609741211,
      "rewards/rejected": -16.239219665527344,
      "step": 4850
    },
    {
      "epoch": 0.98,
      "grad_norm": 0.0081787109375,
      "learning_rate": 4.2578663603486916e-06,
      "logits/chosen": -1.4954382181167603,
      "logits/rejected": -2.641606330871582,
      "logps/chosen": -165.57749938964844,
      "logps/rejected": -321.75,
      "loss": 0.1158,
      "rewards/accuracies": 0.8324999809265137,
      "rewards/chosen": -0.45833495259284973,
      "rewards/margins": 15.786406517028809,
      "rewards/rejected": -16.246719360351562,
      "step": 4900
    },
    {
      "epoch": 0.99,
      "grad_norm": 0.009765625,
      "learning_rate": 4.23706437405226e-06,
      "logits/chosen": -1.6313989162445068,
      "logits/rejected": -2.8603711128234863,
      "logps/chosen": -175.71499633789062,
      "logps/rejected": -339.1875,
      "loss": 0.0812,
      "rewards/accuracies": 0.8824999928474426,
      "rewards/chosen": 0.04604126140475273,
      "rewards/margins": 16.614530563354492,
      "rewards/rejected": -16.570938110351562,
      "step": 4950
    },
    {
      "epoch": 1.0,
      "grad_norm": 0.0008697509765625,
      "learning_rate": 4.21602721704568e-06,
      "logits/chosen": -1.7376703023910522,
      "logits/rejected": -2.959348678588867,
      "logps/chosen": -163.42250061035156,
      "logps/rejected": -324.82000732421875,
      "loss": 0.0812,
      "rewards/accuracies": 0.8824999928474426,
      "rewards/chosen": 0.5958447456359863,
      "rewards/margins": 16.27703094482422,
      "rewards/rejected": -15.682969093322754,
      "step": 5000
    },
    {
      "epoch": 1.01,
      "grad_norm": 0.00130462646484375,
      "learning_rate": 4.194757737423261e-06,
      "logits/chosen": -1.9893441200256348,
      "logits/rejected": -3.1228907108306885,
      "logps/chosen": -161.6875,
      "logps/rejected": -316.3900146484375,
      "loss": 0.083,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": 0.8289526104927063,
      "rewards/margins": 15.733905792236328,
      "rewards/rejected": -14.90640640258789,
      "step": 5050
    },
    {
      "epoch": 1.02,
      "grad_norm": 0.003387451171875,
      "learning_rate": 4.1732588147320705e-06,
      "logits/chosen": -1.9210351705551147,
      "logits/rejected": -3.0717577934265137,
      "logps/chosen": -151.19000244140625,
      "logps/rejected": -294.67999267578125,
      "loss": 0.1054,
      "rewards/accuracies": 0.8475000262260437,
      "rewards/chosen": 0.8471630811691284,
      "rewards/margins": 14.576250076293945,
      "rewards/rejected": -13.729218482971191,
      "step": 5100
    },
    {
      "epoch": 1.03,
      "grad_norm": 0.91015625,
      "learning_rate": 4.1515333595820975e-06,
      "logits/chosen": -1.982885718345642,
      "logits/rejected": -3.0044140815734863,
      "logps/chosen": -163.9550018310547,
      "logps/rejected": -320.55999755859375,
      "loss": 0.0916,
      "rewards/accuracies": 0.8675000071525574,
      "rewards/chosen": 0.7394506931304932,
      "rewards/margins": 15.829375267028809,
      "rewards/rejected": -15.094843864440918,
      "step": 5150
    },
    {
      "epoch": 1.04,
      "grad_norm": 9.393692016601562e-05,
      "learning_rate": 4.129584313252198e-06,
      "logits/chosen": -1.7410473823547363,
      "logits/rejected": -2.6847314834594727,
      "logps/chosen": -158.25999450683594,
      "logps/rejected": -317.07000732421875,
      "loss": 0.0899,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": 1.0339257717132568,
      "rewards/margins": 15.897500038146973,
      "rewards/rejected": -14.865156173706055,
      "step": 5200
    },
    {
      "epoch": 1.05,
      "grad_norm": 0.8984375,
      "learning_rate": 4.107414647291893e-06,
      "logits/chosen": -1.84354829788208,
      "logits/rejected": -2.650254011154175,
      "logps/chosen": -160.79750061035156,
      "logps/rejected": -312.51251220703125,
      "loss": 0.0899,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": 1.0474340915679932,
      "rewards/margins": 15.432499885559082,
      "rewards/rejected": -14.383749961853027,
      "step": 5250
    },
    {
      "epoch": 1.06,
      "grad_norm": 0.0002899169921875,
      "learning_rate": 4.085027363119076e-06,
      "logits/chosen": -1.7570642232894897,
      "logits/rejected": -2.6784496307373047,
      "logps/chosen": -159.75750732421875,
      "logps/rejected": -307.239990234375,
      "loss": 0.0985,
      "rewards/accuracies": 0.8575000166893005,
      "rewards/chosen": 0.8954675197601318,
      "rewards/margins": 15.072968482971191,
      "rewards/rejected": -14.178437232971191,
      "step": 5300
    },
    {
      "epoch": 1.07,
      "grad_norm": 0.0067138671875,
      "learning_rate": 4.062425491613656e-06,
      "logits/chosen": -1.6061410903930664,
      "logits/rejected": -2.5976123809814453,
      "logps/chosen": -157.30250549316406,
      "logps/rejected": -299.4425048828125,
      "loss": 0.0968,
      "rewards/accuracies": 0.8600000143051147,
      "rewards/chosen": 0.9188379049301147,
      "rewards/margins": 14.39859390258789,
      "rewards/rejected": -13.4799222946167,
      "step": 5350
    },
    {
      "epoch": 1.08,
      "grad_norm": 2.578125,
      "learning_rate": 4.039612092707236e-06,
      "logits/chosen": -1.7808789014816284,
      "logits/rejected": -2.769533634185791,
      "logps/chosen": -161.10499572753906,
      "logps/rejected": -301.5849914550781,
      "loss": 0.0847,
      "rewards/accuracies": 0.8774999976158142,
      "rewards/chosen": 0.6963549852371216,
      "rewards/margins": 14.181875228881836,
      "rewards/rejected": -13.483905792236328,
      "step": 5400
    },
    {
      "epoch": 1.09,
      "grad_norm": 1.0,
      "learning_rate": 4.016590254968842e-06,
      "logits/chosen": -1.786810278892517,
      "logits/rejected": -2.6385669708251953,
      "logps/chosen": -168.65750122070312,
      "logps/rejected": -320.4674987792969,
      "loss": 0.0795,
      "rewards/accuracies": 0.8849999904632568,
      "rewards/chosen": 0.3265624940395355,
      "rewards/margins": 15.369843482971191,
      "rewards/rejected": -15.044530868530273,
      "step": 5450
    },
    {
      "epoch": 1.1,
      "grad_norm": 0.000202178955078125,
      "learning_rate": 3.993363095186781e-06,
      "logits/chosen": -1.775051236152649,
      "logits/rejected": -2.604965925216675,
      "logps/chosen": -175.4250030517578,
      "logps/rejected": -334.4700012207031,
      "loss": 0.083,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": -0.4224267601966858,
      "rewards/margins": 16.112499237060547,
      "rewards/rejected": -16.530624389648438,
      "step": 5500
    },
    {
      "epoch": 1.11,
      "grad_norm": 0.0089111328125,
      "learning_rate": 3.9699337579466765e-06,
      "logits/chosen": -1.4332165718078613,
      "logits/rejected": -2.4917151927948,
      "logps/chosen": -174.8524932861328,
      "logps/rejected": -330.2449951171875,
      "loss": 0.0916,
      "rewards/accuracies": 0.8675000071525574,
      "rewards/chosen": -0.360107421875,
      "rewards/margins": 15.817968368530273,
      "rewards/rejected": -16.182968139648438,
      "step": 5550
    },
    {
      "epoch": 1.12,
      "grad_norm": 0.000827789306640625,
      "learning_rate": 3.946305415205748e-06,
      "logits/chosen": -1.3821977376937866,
      "logits/rejected": -2.346054792404175,
      "logps/chosen": -173.88250732421875,
      "logps/rejected": -325.8175048828125,
      "loss": 0.1003,
      "rewards/accuracies": 0.8550000190734863,
      "rewards/chosen": -0.9856836199760437,
      "rewards/margins": 15.3203125,
      "rewards/rejected": -16.30843734741211,
      "step": 5600
    },
    {
      "epoch": 1.13,
      "grad_norm": 1.171875,
      "learning_rate": 3.922481265863371e-06,
      "logits/chosen": -1.5632950067520142,
      "logits/rejected": -2.638700008392334,
      "logps/chosen": -176.65750122070312,
      "logps/rejected": -339.3949890136719,
      "loss": 0.0812,
      "rewards/accuracies": 0.8824999928474426,
      "rewards/chosen": -0.8098046779632568,
      "rewards/margins": 16.346094131469727,
      "rewards/rejected": -17.149063110351562,
      "step": 5650
    },
    {
      "epoch": 1.1400000000000001,
      "grad_norm": 0.8984375,
      "learning_rate": 3.898464535327997e-06,
      "logits/chosen": -1.559891700744629,
      "logits/rejected": -2.6719970703125,
      "logps/chosen": -175.10499572753906,
      "logps/rejected": -332.5799865722656,
      "loss": 0.0847,
      "rewards/accuracies": 0.8774999976158142,
      "rewards/chosen": -0.6723046898841858,
      "rewards/margins": 16.07062530517578,
      "rewards/rejected": -16.74625015258789,
      "step": 5700
    },
    {
      "epoch": 1.15,
      "grad_norm": 1.52587890625e-05,
      "learning_rate": 3.874258475080497e-06,
      "logits/chosen": -1.848596215248108,
      "logits/rejected": -2.9883642196655273,
      "logps/chosen": -171.56500244140625,
      "logps/rejected": -336.0675048828125,
      "loss": 0.0812,
      "rewards/accuracies": 0.8824999928474426,
      "rewards/chosen": -0.05261474475264549,
      "rewards/margins": 16.615468978881836,
      "rewards/rejected": -16.658750534057617,
      "step": 5750
    },
    {
      "epoch": 1.16,
      "grad_norm": 0.0010528564453125,
      "learning_rate": 3.849866362233947e-06,
      "logits/chosen": -1.624019742012024,
      "logits/rejected": -2.9648828506469727,
      "logps/chosen": -168.875,
      "logps/rejected": -331.17999267578125,
      "loss": 0.0726,
      "rewards/accuracies": 0.8949999809265137,
      "rewards/chosen": 0.45997437834739685,
      "rewards/margins": 16.332500457763672,
      "rewards/rejected": -15.879219055175781,
      "step": 5800
    },
    {
      "epoch": 1.17,
      "grad_norm": 0.001678466796875,
      "learning_rate": 3.8252914990899695e-06,
      "logits/chosen": -1.573280692100525,
      "logits/rejected": -2.7242345809936523,
      "logps/chosen": -169.05499267578125,
      "logps/rejected": -322.07501220703125,
      "loss": 0.0726,
      "rewards/accuracies": 0.8949999809265137,
      "rewards/chosen": -0.01105346716940403,
      "rewards/margins": 15.53531265258789,
      "rewards/rejected": -15.5521879196167,
      "step": 5850
    },
    {
      "epoch": 1.18,
      "grad_norm": 0.01416015625,
      "learning_rate": 3.800537212691651e-06,
      "logits/chosen": -1.745273470878601,
      "logits/rejected": -2.6969921588897705,
      "logps/chosen": -178.67250061035156,
      "logps/rejected": -330.9800109863281,
      "loss": 0.0812,
      "rewards/accuracies": 0.8824999928474426,
      "rewards/chosen": -0.30939698219299316,
      "rewards/margins": 15.374062538146973,
      "rewards/rejected": -15.685625076293945,
      "step": 5900
    },
    {
      "epoch": 1.19,
      "grad_norm": 0.000164031982421875,
      "learning_rate": 3.775606854373115e-06,
      "logits/chosen": -1.655666470527649,
      "logits/rejected": -2.6190404891967773,
      "logps/chosen": -174.1925048828125,
      "logps/rejected": -326.572509765625,
      "loss": 0.0726,
      "rewards/accuracies": 0.8949999809265137,
      "rewards/chosen": 0.033031005412340164,
      "rewards/margins": 15.436562538146973,
      "rewards/rejected": -15.404999732971191,
      "step": 5950
    },
    {
      "epoch": 1.2,
      "grad_norm": 0.546875,
      "learning_rate": 3.7505037993058046e-06,
      "logits/chosen": -1.7215648889541626,
      "logits/rejected": -2.621614933013916,
      "logps/chosen": -175.375,
      "logps/rejected": -334.135009765625,
      "loss": 0.0847,
      "rewards/accuracies": 0.8774999976158142,
      "rewards/chosen": -0.3496679663658142,
      "rewards/margins": 16.133594512939453,
      "rewards/rejected": -16.4854679107666,
      "step": 6000
    },
    {
      "epoch": 1.21,
      "grad_norm": 1.0859375,
      "learning_rate": 3.7252314460415396e-06,
      "logits/chosen": -1.5741479396820068,
      "logits/rejected": -2.679941415786743,
      "logps/chosen": -172.75250244140625,
      "logps/rejected": -326.0625,
      "loss": 0.0743,
      "rewards/accuracies": 0.8924999833106995,
      "rewards/chosen": 0.0013964843237772584,
      "rewards/margins": 15.612030982971191,
      "rewards/rejected": -15.608750343322754,
      "step": 6050
    },
    {
      "epoch": 1.22,
      "grad_norm": 2.3245811462402344e-05,
      "learning_rate": 3.6997932160524018e-06,
      "logits/chosen": -1.6118944883346558,
      "logits/rejected": -2.7812891006469727,
      "logps/chosen": -174.97500610351562,
      "logps/rejected": -325.45001220703125,
      "loss": 0.0778,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.1942773461341858,
      "rewards/margins": 15.537187576293945,
      "rewards/rejected": -15.733750343322754,
      "step": 6100
    },
    {
      "epoch": 1.23,
      "grad_norm": 6.961822509765625e-05,
      "learning_rate": 3.6741925532675297e-06,
      "logits/chosen": -1.480682373046875,
      "logits/rejected": -2.7474658489227295,
      "logps/chosen": -165.65750122070312,
      "logps/rejected": -317.760009765625,
      "loss": 0.0726,
      "rewards/accuracies": 0.8949999809265137,
      "rewards/chosen": 0.29445311427116394,
      "rewards/margins": 15.585000038146973,
      "rewards/rejected": -15.289531707763672,
      "step": 6150
    },
    {
      "epoch": 1.24,
      "grad_norm": 0.0042724609375,
      "learning_rate": 3.648432923606862e-06,
      "logits/chosen": -1.3990026712417603,
      "logits/rejected": -2.915020704269409,
      "logps/chosen": -164.90750122070312,
      "logps/rejected": -323.4700012207031,
      "loss": 0.0795,
      "rewards/accuracies": 0.8849999904632568,
      "rewards/chosen": 0.8015918135643005,
      "rewards/margins": 16.133438110351562,
      "rewards/rejected": -15.328437805175781,
      "step": 6200
    },
    {
      "epoch": 1.25,
      "grad_norm": 0.00024127960205078125,
      "learning_rate": 3.622517814511906e-06,
      "logits/chosen": -1.320135474205017,
      "logits/rejected": -2.860288143157959,
      "logps/chosen": -163.79750061035156,
      "logps/rejected": -321.0450134277344,
      "loss": 0.0916,
      "rewards/accuracies": 0.8675000071525574,
      "rewards/chosen": 0.6940441727638245,
      "rewards/margins": 16.037656784057617,
      "rewards/rejected": -15.3423433303833,
      "step": 6250
    },
    {
      "epoch": 1.26,
      "grad_norm": 0.98828125,
      "learning_rate": 3.5964507344735965e-06,
      "logits/chosen": -1.176222324371338,
      "logits/rejected": -2.6737987995147705,
      "logps/chosen": -160.44500732421875,
      "logps/rejected": -313.2349853515625,
      "loss": 0.1003,
      "rewards/accuracies": 0.8550000190734863,
      "rewards/chosen": 0.5672607421875,
      "rewards/margins": 15.430312156677246,
      "rewards/rejected": -14.865625381469727,
      "step": 6300
    },
    {
      "epoch": 1.27,
      "grad_norm": 0.000461578369140625,
      "learning_rate": 3.5702352125573015e-06,
      "logits/chosen": -1.2549536228179932,
      "logits/rejected": -2.675966739654541,
      "logps/chosen": -159.0675048828125,
      "logps/rejected": -311.6025085449219,
      "loss": 0.0882,
      "rewards/accuracies": 0.8725000023841858,
      "rewards/chosen": 0.6797509789466858,
      "rewards/margins": 15.3959379196167,
      "rewards/rejected": -14.717031478881836,
      "step": 6350
    },
    {
      "epoch": 1.28,
      "grad_norm": 0.0001392364501953125,
      "learning_rate": 3.543874797925042e-06,
      "logits/chosen": -1.5146979093551636,
      "logits/rejected": -2.7764551639556885,
      "logps/chosen": -170.92750549316406,
      "logps/rejected": -323.7449951171875,
      "loss": 0.0847,
      "rewards/accuracies": 0.8774999976158142,
      "rewards/chosen": 0.3626416027545929,
      "rewards/margins": 15.621562957763672,
      "rewards/rejected": -15.259843826293945,
      "step": 6400
    },
    {
      "epoch": 1.29,
      "grad_norm": 0.0126953125,
      "learning_rate": 3.5173730593549947e-06,
      "logits/chosen": -1.8136059045791626,
      "logits/rejected": -2.790639638900757,
      "logps/chosen": -169.0,
      "logps/rejected": -316.1000061035156,
      "loss": 0.0812,
      "rewards/accuracies": 0.8824999928474426,
      "rewards/chosen": 0.23876464366912842,
      "rewards/margins": 14.961406707763672,
      "rewards/rejected": -14.722187042236328,
      "step": 6450
    },
    {
      "epoch": 1.3,
      "grad_norm": 0.00799560546875,
      "learning_rate": 3.4907335847583356e-06,
      "logits/chosen": -1.9238842725753784,
      "logits/rejected": -2.8575196266174316,
      "logps/chosen": -170.00999450683594,
      "logps/rejected": -319.3324890136719,
      "loss": 0.1003,
      "rewards/accuracies": 0.8550000190734863,
      "rewards/chosen": -0.1691601574420929,
      "rewards/margins": 15.154375076293945,
      "rewards/rejected": -15.327031135559082,
      "step": 6500
    },
    {
      "epoch": 1.31,
      "grad_norm": 0.0074462890625,
      "learning_rate": 3.463959980693492e-06,
      "logits/chosen": -1.840087890625,
      "logits/rejected": -2.790390729904175,
      "logps/chosen": -169.75750732421875,
      "logps/rejected": -326.3450012207031,
      "loss": 0.0709,
      "rewards/accuracies": 0.8974999785423279,
      "rewards/chosen": 0.2927929759025574,
      "rewards/margins": 15.926562309265137,
      "rewards/rejected": -15.626562118530273,
      "step": 6550
    },
    {
      "epoch": 1.32,
      "grad_norm": 6.341934204101562e-05,
      "learning_rate": 3.4370558718778753e-06,
      "logits/chosen": -1.810239315032959,
      "logits/rejected": -2.8067736625671387,
      "logps/chosen": -168.5050048828125,
      "logps/rejected": -325.92498779296875,
      "loss": 0.0726,
      "rewards/accuracies": 0.8949999809265137,
      "rewards/chosen": 0.42659667134284973,
      "rewards/margins": 16.05078125,
      "rewards/rejected": -15.6264066696167,
      "step": 6600
    },
    {
      "epoch": 1.33,
      "grad_norm": 0.0002422332763671875,
      "learning_rate": 3.4100249006971514e-06,
      "logits/chosen": -2.036135196685791,
      "logits/rejected": -2.8965821266174316,
      "logps/chosen": -168.71499633789062,
      "logps/rejected": -319.447509765625,
      "loss": 0.0812,
      "rewards/accuracies": 0.8824999928474426,
      "rewards/chosen": 0.32066404819488525,
      "rewards/margins": 15.444375038146973,
      "rewards/rejected": -15.124062538146973,
      "step": 6650
    },
    {
      "epoch": 1.34,
      "grad_norm": 0.00063323974609375,
      "learning_rate": 3.3828707267121185e-06,
      "logits/chosen": -1.9675488471984863,
      "logits/rejected": -2.8052685260772705,
      "logps/chosen": -165.78250122070312,
      "logps/rejected": -315.56500244140625,
      "loss": 0.0882,
      "rewards/accuracies": 0.8725000023841858,
      "rewards/chosen": 0.36207762360572815,
      "rewards/margins": 15.29671859741211,
      "rewards/rejected": -14.934218406677246,
      "step": 6700
    },
    {
      "epoch": 1.35,
      "grad_norm": 1.15625,
      "learning_rate": 3.355597026163264e-06,
      "logits/chosen": -1.784468412399292,
      "logits/rejected": -2.6825194358825684,
      "logps/chosen": -162.88250732421875,
      "logps/rejected": -311.5199890136719,
      "loss": 0.1037,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.07732421904802322,
      "rewards/margins": 15.029531478881836,
      "rewards/rejected": -14.949531555175781,
      "step": 6750
    },
    {
      "epoch": 1.3599999999999999,
      "grad_norm": 0.8828125,
      "learning_rate": 3.3282074914730577e-06,
      "logits/chosen": -1.9187493324279785,
      "logits/rejected": -2.836562395095825,
      "logps/chosen": -158.41000366210938,
      "logps/rejected": -310.2674865722656,
      "loss": 0.0726,
      "rewards/accuracies": 0.8949999809265137,
      "rewards/chosen": 1.508481502532959,
      "rewards/margins": 15.624530792236328,
      "rewards/rejected": -14.117656707763672,
      "step": 6800
    },
    {
      "epoch": 1.37,
      "grad_norm": 0.0068359375,
      "learning_rate": 3.300705830746057e-06,
      "logits/chosen": -1.7984619140625,
      "logits/rejected": -2.8072216510772705,
      "logps/chosen": -147.9375,
      "logps/rejected": -300.1875,
      "loss": 0.0743,
      "rewards/accuracies": 0.8924999833106995,
      "rewards/chosen": 2.547187566757202,
      "rewards/margins": 15.463281631469727,
      "rewards/rejected": -12.914375305175781,
      "step": 6850
    },
    {
      "epoch": 1.38,
      "grad_norm": 0.01104736328125,
      "learning_rate": 3.2730957672668917e-06,
      "logits/chosen": -1.6440536975860596,
      "logits/rejected": -2.777017593383789,
      "logps/chosen": -143.3625030517578,
      "logps/rejected": -295.2699890136719,
      "loss": 0.0726,
      "rewards/accuracies": 0.8949999809265137,
      "rewards/chosen": 2.895751953125,
      "rewards/margins": 15.431718826293945,
      "rewards/rejected": -12.537031173706055,
      "step": 6900
    },
    {
      "epoch": 1.3900000000000001,
      "grad_norm": 0.0013580322265625,
      "learning_rate": 3.245381038996188e-06,
      "logits/chosen": -1.747806429862976,
      "logits/rejected": -2.857853889465332,
      "logps/chosen": -139.8975067138672,
      "logps/rejected": -288.42999267578125,
      "loss": 0.0795,
      "rewards/accuracies": 0.8849999904632568,
      "rewards/chosen": 3.0215137004852295,
      "rewards/margins": 15.106406211853027,
      "rewards/rejected": -12.087187767028809,
      "step": 6950
    },
    {
      "epoch": 1.4,
      "grad_norm": 0.0035247802734375,
      "learning_rate": 3.2175653980645096e-06,
      "logits/chosen": -1.6131787300109863,
      "logits/rejected": -2.7717015743255615,
      "logps/chosen": -134.77499389648438,
      "logps/rejected": -281.7250061035156,
      "loss": 0.0916,
      "rewards/accuracies": 0.8675000071525574,
      "rewards/chosen": 3.087773323059082,
      "rewards/margins": 14.971718788146973,
      "rewards/rejected": -11.884687423706055,
      "step": 7000
    },
    {
      "epoch": 1.41,
      "grad_norm": 0.003173828125,
      "learning_rate": 3.189652610264379e-06,
      "logits/chosen": -1.6364532709121704,
      "logits/rejected": -2.7611522674560547,
      "logps/chosen": -148.1074981689453,
      "logps/rejected": -292.260009765625,
      "loss": 0.0985,
      "rewards/accuracies": 0.8575000166893005,
      "rewards/chosen": 2.152085065841675,
      "rewards/margins": 14.622968673706055,
      "rewards/rejected": -12.472969055175781,
      "step": 7050
    },
    {
      "epoch": 1.42,
      "grad_norm": 0.00909423828125,
      "learning_rate": 3.1616464545404486e-06,
      "logits/chosen": -1.6323034763336182,
      "logits/rejected": -2.8021483421325684,
      "logps/chosen": -152.21749877929688,
      "logps/rejected": -307.4024963378906,
      "loss": 0.0709,
      "rewards/accuracies": 0.8974999785423279,
      "rewards/chosen": 2.286259651184082,
      "rewards/margins": 15.906719207763672,
      "rewards/rejected": -13.616406440734863,
      "step": 7100
    },
    {
      "epoch": 1.43,
      "grad_norm": 0.00014591217041015625,
      "learning_rate": 3.133550722477896e-06,
      "logits/chosen": -1.5869457721710205,
      "logits/rejected": -2.831690788269043,
      "logps/chosen": -151.6699981689453,
      "logps/rejected": -312.9800109863281,
      "loss": 0.0709,
      "rewards/accuracies": 0.8974999785423279,
      "rewards/chosen": 2.021599054336548,
      "rewards/margins": 16.44562530517578,
      "rewards/rejected": -14.4232816696167,
      "step": 7150
    },
    {
      "epoch": 1.44,
      "grad_norm": 0.0057373046875,
      "learning_rate": 3.105369217789099e-06,
      "logits/chosen": -1.5832836627960205,
      "logits/rejected": -2.826430559158325,
      "logps/chosen": -147.8800048828125,
      "logps/rejected": -299.2049865722656,
      "loss": 0.102,
      "rewards/accuracies": 0.8525000214576721,
      "rewards/chosen": 1.6627343893051147,
      "rewards/margins": 15.454687118530273,
      "rewards/rejected": -13.7876558303833,
      "step": 7200
    },
    {
      "epoch": 1.45,
      "grad_norm": 1.3359375,
      "learning_rate": 3.077105755798675e-06,
      "logits/chosen": -1.5947095155715942,
      "logits/rejected": -2.803586483001709,
      "logps/chosen": -149.30250549316406,
      "logps/rejected": -300.635009765625,
      "loss": 0.1037,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.6690233945846558,
      "rewards/margins": 15.402812957763672,
      "rewards/rejected": -13.731562614440918,
      "step": 7250
    },
    {
      "epoch": 1.46,
      "grad_norm": 0.0003814697265625,
      "learning_rate": 3.0487641629269515e-06,
      "logits/chosen": -1.3740723133087158,
      "logits/rejected": -2.788759708404541,
      "logps/chosen": -149.9949951171875,
      "logps/rejected": -307.3275146484375,
      "loss": 0.0864,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.7442187070846558,
      "rewards/margins": 15.862968444824219,
      "rewards/rejected": -14.121718406677246,
      "step": 7300
    },
    {
      "epoch": 1.47,
      "grad_norm": 0.6015625,
      "learning_rate": 3.0203482761719226e-06,
      "logits/chosen": -1.3477171659469604,
      "logits/rejected": -2.810493230819702,
      "logps/chosen": -152.57249450683594,
      "logps/rejected": -303.9750061035156,
      "loss": 0.1054,
      "rewards/accuracies": 0.8475000262260437,
      "rewards/chosen": 1.5104199647903442,
      "rewards/margins": 15.333281517028809,
      "rewards/rejected": -13.822187423706055,
      "step": 7350
    },
    {
      "epoch": 1.48,
      "grad_norm": 0.59375,
      "learning_rate": 2.991861942589788e-06,
      "logits/chosen": -1.4101388454437256,
      "logits/rejected": -2.9320101737976074,
      "logps/chosen": -151.7100067138672,
      "logps/rejected": -306.79998779296875,
      "loss": 0.0761,
      "rewards/accuracies": 0.8899999856948853,
      "rewards/chosen": 2.018501043319702,
      "rewards/margins": 15.747187614440918,
      "rewards/rejected": -13.726093292236328,
      "step": 7400
    },
    {
      "epoch": 1.49,
      "grad_norm": 4.76837158203125e-05,
      "learning_rate": 2.9633090187741186e-06,
      "logits/chosen": -1.3069872856140137,
      "logits/rejected": -2.8863134384155273,
      "logps/chosen": -145.5574951171875,
      "logps/rejected": -290.4125061035156,
      "loss": 0.1037,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.1462109088897705,
      "rewards/margins": 14.840781211853027,
      "rewards/rejected": -12.689844131469727,
      "step": 7450
    },
    {
      "epoch": 1.5,
      "grad_norm": 0.0001659393310546875,
      "learning_rate": 2.934693370333739e-06,
      "logits/chosen": -1.3929543495178223,
      "logits/rejected": -2.97314453125,
      "logps/chosen": -147.5225067138672,
      "logps/rejected": -305.05999755859375,
      "loss": 0.0709,
      "rewards/accuracies": 0.8974999785423279,
      "rewards/chosen": 2.6558544635772705,
      "rewards/margins": 15.836562156677246,
      "rewards/rejected": -13.185937881469727,
      "step": 7500
    },
    {
      "epoch": 1.51,
      "grad_norm": 0.98828125,
      "learning_rate": 2.9060188713693794e-06,
      "logits/chosen": -1.3370373249053955,
      "logits/rejected": -2.9380431175231934,
      "logps/chosen": -141.3574981689453,
      "logps/rejected": -294.44500732421875,
      "loss": 0.0761,
      "rewards/accuracies": 0.8899999856948853,
      "rewards/chosen": 2.661679744720459,
      "rewards/margins": 15.43765640258789,
      "rewards/rejected": -12.771562576293945,
      "step": 7550
    },
    {
      "epoch": 1.52,
      "grad_norm": 0.78515625,
      "learning_rate": 2.8772894039491938e-06,
      "logits/chosen": -1.4408868551254272,
      "logits/rejected": -2.962076425552368,
      "logps/chosen": -142.13999938964844,
      "logps/rejected": -295.55999755859375,
      "loss": 0.0847,
      "rewards/accuracies": 0.8774999976158142,
      "rewards/chosen": 2.679374933242798,
      "rewards/margins": 15.471875190734863,
      "rewards/rejected": -12.795391082763672,
      "step": 7600
    },
    {
      "epoch": 1.53,
      "grad_norm": 0.01397705078125,
      "learning_rate": 2.848508857583183e-06,
      "logits/chosen": -1.521622896194458,
      "logits/rejected": -3.0157811641693115,
      "logps/chosen": -142.84500122070312,
      "logps/rejected": -294.93499755859375,
      "loss": 0.0864,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.692207098007202,
      "rewards/margins": 15.400468826293945,
      "rewards/rejected": -12.703906059265137,
      "step": 7650
    },
    {
      "epoch": 1.54,
      "grad_norm": 0.0037689208984375,
      "learning_rate": 2.81968112869662e-06,
      "logits/chosen": -1.334303379058838,
      "logits/rejected": -2.9068212509155273,
      "logps/chosen": -137.8925018310547,
      "logps/rejected": -280.385009765625,
      "loss": 0.1037,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.869335889816284,
      "rewards/margins": 14.509531021118164,
      "rewards/rejected": -11.638280868530273,
      "step": 7700
    },
    {
      "epoch": 1.55,
      "grad_norm": 1.1015625,
      "learning_rate": 2.790810120102534e-06,
      "logits/chosen": -1.453762173652649,
      "logits/rejected": -2.9707226753234863,
      "logps/chosen": -146.41000366210938,
      "logps/rejected": -292.67498779296875,
      "loss": 0.083,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": 2.3908984661102295,
      "rewards/margins": 14.83578109741211,
      "rewards/rejected": -12.446874618530273,
      "step": 7750
    },
    {
      "epoch": 1.56,
      "grad_norm": 0.00040435791015625,
      "learning_rate": 2.7618997404733365e-06,
      "logits/chosen": -1.6041711568832397,
      "logits/rejected": -2.9843311309814453,
      "logps/chosen": -150.46749877929688,
      "logps/rejected": -298.6700134277344,
      "loss": 0.0847,
      "rewards/accuracies": 0.8774999976158142,
      "rewards/chosen": 2.042372941970825,
      "rewards/margins": 15.013437271118164,
      "rewards/rejected": -12.966405868530273,
      "step": 7800
    },
    {
      "epoch": 1.5699999999999998,
      "grad_norm": 0.0004024505615234375,
      "learning_rate": 2.7329539038116453e-06,
      "logits/chosen": -1.4521881341934204,
      "logits/rejected": -2.926464796066284,
      "logps/chosen": -150.1074981689453,
      "logps/rejected": -298.0425109863281,
      "loss": 0.0795,
      "rewards/accuracies": 0.8849999904632568,
      "rewards/chosen": 2.3667969703674316,
      "rewards/margins": 14.889687538146973,
      "rewards/rejected": -12.527030944824219,
      "step": 7850
    },
    {
      "epoch": 1.58,
      "grad_norm": 0.8671875,
      "learning_rate": 2.7039765289203947e-06,
      "logits/chosen": -1.412445068359375,
      "logits/rejected": -2.8667969703674316,
      "logps/chosen": -146.1300048828125,
      "logps/rejected": -291.2850036621094,
      "loss": 0.0812,
      "rewards/accuracies": 0.8824999928474426,
      "rewards/chosen": 2.3307225704193115,
      "rewards/margins": 14.735937118530273,
      "rewards/rejected": -12.403437614440918,
      "step": 7900
    },
    {
      "epoch": 1.5899999999999999,
      "grad_norm": 0.67578125,
      "learning_rate": 2.6749715388722865e-06,
      "logits/chosen": -1.2646269798278809,
      "logits/rejected": -2.821697473526001,
      "logps/chosen": -145.84750366210938,
      "logps/rejected": -289.67498779296875,
      "loss": 0.0899,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": 2.281503915786743,
      "rewards/margins": 14.5806245803833,
      "rewards/rejected": -12.300156593322754,
      "step": 7950
    },
    {
      "epoch": 1.6,
      "grad_norm": 0.0015411376953125,
      "learning_rate": 2.6459428604786757e-06,
      "logits/chosen": -1.311349630355835,
      "logits/rejected": -2.7577929496765137,
      "logps/chosen": -150.4425048828125,
      "logps/rejected": -293.7149963378906,
      "loss": 0.0985,
      "rewards/accuracies": 0.8575000166893005,
      "rewards/chosen": 1.655849575996399,
      "rewards/margins": 14.475312232971191,
      "rewards/rejected": -12.818437576293945,
      "step": 8000
    },
    {
      "epoch": 1.6099999999999999,
      "grad_norm": 0.00147247314453125,
      "learning_rate": 2.616894423757941e-06,
      "logits/chosen": -1.4159927368164062,
      "logits/rejected": -2.9021289348602295,
      "logps/chosen": -151.3925018310547,
      "logps/rejected": -296.8399963378906,
      "loss": 0.0795,
      "rewards/accuracies": 0.8849999904632568,
      "rewards/chosen": 2.0074121952056885,
      "rewards/margins": 14.760781288146973,
      "rewards/rejected": -12.754219055175781,
      "step": 8050
    },
    {
      "epoch": 1.62,
      "grad_norm": 0.000164031982421875,
      "learning_rate": 2.587830161403419e-06,
      "logits/chosen": -1.3917722702026367,
      "logits/rejected": -2.887305974960327,
      "logps/chosen": -152.03500366210938,
      "logps/rejected": -288.260009765625,
      "loss": 0.0916,
      "rewards/accuracies": 0.8675000071525574,
      "rewards/chosen": 1.9093506336212158,
      "rewards/margins": 13.984375,
      "rewards/rejected": -12.0795316696167,
      "step": 8100
    },
    {
      "epoch": 1.63,
      "grad_norm": 0.90625,
      "learning_rate": 2.5587540082509864e-06,
      "logits/chosen": -1.3086663484573364,
      "logits/rejected": -2.910728693008423,
      "logps/chosen": -148.97250366210938,
      "logps/rejected": -287.9200134277344,
      "loss": 0.0795,
      "rewards/accuracies": 0.8849999904632568,
      "rewards/chosen": 2.129091739654541,
      "rewards/margins": 14.16812515258789,
      "rewards/rejected": -12.041093826293945,
      "step": 8150
    },
    {
      "epoch": 1.6400000000000001,
      "grad_norm": 0.0361328125,
      "learning_rate": 2.5296699007463434e-06,
      "logits/chosen": -1.198205590248108,
      "logits/rejected": -2.7841992378234863,
      "logps/chosen": -147.5749969482422,
      "logps/rejected": -290.5050048828125,
      "loss": 0.0657,
      "rewards/accuracies": 0.9049999713897705,
      "rewards/chosen": 2.2004687786102295,
      "rewards/margins": 14.422344207763672,
      "rewards/rejected": -12.222969055175781,
      "step": 8200
    },
    {
      "epoch": 1.65,
      "grad_norm": 0.0004787445068359375,
      "learning_rate": 2.500581776412081e-06,
      "logits/chosen": -1.3178759813308716,
      "logits/rejected": -2.763364315032959,
      "logps/chosen": -153.17999267578125,
      "logps/rejected": -296.3699951171875,
      "loss": 0.083,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": 1.7891894578933716,
      "rewards/margins": 14.428750038146973,
      "rewards/rejected": -12.639843940734863,
      "step": 8250
    },
    {
      "epoch": 1.6600000000000001,
      "grad_norm": 0.7578125,
      "learning_rate": 2.471493573314605e-06,
      "logits/chosen": -1.273976445198059,
      "logits/rejected": -2.638751745223999,
      "logps/chosen": -147.61749267578125,
      "logps/rejected": -281.2300109863281,
      "loss": 0.1089,
      "rewards/accuracies": 0.8424999713897705,
      "rewards/chosen": 1.724277377128601,
      "rewards/margins": 13.661250114440918,
      "rewards/rejected": -11.94156265258789,
      "step": 8300
    },
    {
      "epoch": 1.67,
      "grad_norm": 0.5234375,
      "learning_rate": 2.442409229530985e-06,
      "logits/chosen": -1.3456776142120361,
      "logits/rejected": -2.6645333766937256,
      "logps/chosen": -148.5449981689453,
      "logps/rejected": -290.0400085449219,
      "loss": 0.0968,
      "rewards/accuracies": 0.8600000143051147,
      "rewards/chosen": 1.7859375476837158,
      "rewards/margins": 14.419530868530273,
      "rewards/rejected": -12.630781173706055,
      "step": 8350
    },
    {
      "epoch": 1.6800000000000002,
      "grad_norm": 1.046875,
      "learning_rate": 2.4133326826158006e-06,
      "logits/chosen": -1.452473759651184,
      "logits/rejected": -2.719834089279175,
      "logps/chosen": -151.13999938964844,
      "logps/rejected": -297.2650146484375,
      "loss": 0.0847,
      "rewards/accuracies": 0.8774999976158142,
      "rewards/chosen": 1.8502148389816284,
      "rewards/margins": 14.8579683303833,
      "rewards/rejected": -13.008437156677246,
      "step": 8400
    },
    {
      "epoch": 1.69,
      "grad_norm": 0.0001697540283203125,
      "learning_rate": 2.3842678690680612e-06,
      "logits/chosen": -1.2888891696929932,
      "logits/rejected": -2.6560449600219727,
      "logps/chosen": -150.96499633789062,
      "logps/rejected": -290.1199951171875,
      "loss": 0.0968,
      "rewards/accuracies": 0.8600000143051147,
      "rewards/chosen": 1.7348560094833374,
      "rewards/margins": 14.02734375,
      "rewards/rejected": -12.290937423706055,
      "step": 8450
    },
    {
      "epoch": 1.7,
      "grad_norm": 0.00016498565673828125,
      "learning_rate": 2.355218723798264e-06,
      "logits/chosen": -1.3379980325698853,
      "logits/rejected": -2.7865869998931885,
      "logps/chosen": -153.50999450683594,
      "logps/rejected": -295.2149963378906,
      "loss": 0.0778,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.0467870235443115,
      "rewards/margins": 14.391094207763672,
      "rewards/rejected": -12.346562385559082,
      "step": 8500
    },
    {
      "epoch": 1.71,
      "grad_norm": 1.0,
      "learning_rate": 2.326189179595676e-06,
      "logits/chosen": -1.2621526718139648,
      "logits/rejected": -2.6848363876342773,
      "logps/chosen": -151.13999938964844,
      "logps/rejected": -294.8949890136719,
      "loss": 0.0812,
      "rewards/accuracies": 0.8824999928474426,
      "rewards/chosen": 2.0117383003234863,
      "rewards/margins": 14.526562690734863,
      "rewards/rejected": -12.5053129196167,
      "step": 8550
    },
    {
      "epoch": 1.72,
      "grad_norm": 0.0203857421875,
      "learning_rate": 2.297183166595889e-06,
      "logits/chosen": -1.3091638088226318,
      "logits/rejected": -2.7887303829193115,
      "logps/chosen": -155.8625030517578,
      "logps/rejected": -300.9750061035156,
      "loss": 0.0588,
      "rewards/accuracies": 0.9150000214576721,
      "rewards/chosen": 2.133901357650757,
      "rewards/margins": 14.717499732971191,
      "rewards/rejected": -12.588281631469727,
      "step": 8600
    },
    {
      "epoch": 1.73,
      "grad_norm": 1.25,
      "learning_rate": 2.26820461174875e-06,
      "logits/chosen": -1.2939709424972534,
      "logits/rejected": -2.6905393600463867,
      "logps/chosen": -148.60499572753906,
      "logps/rejected": -286.3699951171875,
      "loss": 0.0933,
      "rewards/accuracies": 0.8650000095367432,
      "rewards/chosen": 1.790956974029541,
      "rewards/margins": 13.9829683303833,
      "rewards/rejected": -12.193750381469727,
      "step": 8650
    },
    {
      "epoch": 1.74,
      "grad_norm": 0.93359375,
      "learning_rate": 2.23925743828671e-06,
      "logits/chosen": -1.1340380907058716,
      "logits/rejected": -2.585568904876709,
      "logps/chosen": -146.1999969482422,
      "logps/rejected": -279.0899963378906,
      "loss": 0.1037,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.663818359375,
      "rewards/margins": 13.498906135559082,
      "rewards/rejected": -11.834218978881836,
      "step": 8700
    },
    {
      "epoch": 1.75,
      "grad_norm": 0.00543212890625,
      "learning_rate": 2.2103455651936824e-06,
      "logits/chosen": -1.1339178085327148,
      "logits/rejected": -2.6299610137939453,
      "logps/chosen": -146.9774932861328,
      "logps/rejected": -285.3175048828125,
      "loss": 0.0968,
      "rewards/accuracies": 0.8600000143051147,
      "rewards/chosen": 1.9546289443969727,
      "rewards/margins": 14.03499984741211,
      "rewards/rejected": -12.078594207763672,
      "step": 8750
    },
    {
      "epoch": 1.76,
      "grad_norm": 0.005828857421875,
      "learning_rate": 2.181472906674478e-06,
      "logits/chosen": -1.313624382019043,
      "logits/rejected": -2.7783007621765137,
      "logps/chosen": -154.29750061035156,
      "logps/rejected": -303.6675109863281,
      "loss": 0.0553,
      "rewards/accuracies": 0.9200000166893005,
      "rewards/chosen": 2.0906200408935547,
      "rewards/margins": 15.23062515258789,
      "rewards/rejected": -13.136249542236328,
      "step": 8800
    },
    {
      "epoch": 1.77,
      "grad_norm": 0.00531005859375,
      "learning_rate": 2.152643371624878e-06,
      "logits/chosen": -1.257965087890625,
      "logits/rejected": -2.8671875,
      "logps/chosen": -148.28500366210938,
      "logps/rejected": -293.7850036621094,
      "loss": 0.0743,
      "rewards/accuracies": 0.8924999833106995,
      "rewards/chosen": 2.257265567779541,
      "rewards/margins": 14.864843368530273,
      "rewards/rejected": -12.609844207763672,
      "step": 8850
    },
    {
      "epoch": 1.78,
      "grad_norm": 0.005889892578125,
      "learning_rate": 2.1238608631024416e-06,
      "logits/chosen": -1.217246651649475,
      "logits/rejected": -2.843193292617798,
      "logps/chosen": -151.90249633789062,
      "logps/rejected": -294.79998779296875,
      "loss": 0.083,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": 1.9859668016433716,
      "rewards/margins": 14.599687576293945,
      "rewards/rejected": -12.60953140258789,
      "step": 8900
    },
    {
      "epoch": 1.79,
      "grad_norm": 0.79296875,
      "learning_rate": 2.095129277798084e-06,
      "logits/chosen": -1.2338311672210693,
      "logits/rejected": -2.8333849906921387,
      "logps/chosen": -149.3524932861328,
      "logps/rejected": -293.375,
      "loss": 0.0951,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.1244139671325684,
      "rewards/margins": 14.682812690734863,
      "rewards/rejected": -12.559218406677246,
      "step": 8950
    },
    {
      "epoch": 1.8,
      "grad_norm": 0.00011968612670898438,
      "learning_rate": 2.0664525055085353e-06,
      "logits/chosen": -1.25933837890625,
      "logits/rejected": -2.8884618282318115,
      "logps/chosen": -149.42750549316406,
      "logps/rejected": -296.8999938964844,
      "loss": 0.0916,
      "rewards/accuracies": 0.8675000071525574,
      "rewards/chosen": 2.1980907917022705,
      "rewards/margins": 14.94124984741211,
      "rewards/rejected": -12.743906021118164,
      "step": 9000
    },
    {
      "epoch": 1.81,
      "grad_norm": 0.005859375,
      "learning_rate": 2.037834428609718e-06,
      "logits/chosen": -1.220767855644226,
      "logits/rejected": -2.8833789825439453,
      "logps/chosen": -150.5800018310547,
      "logps/rejected": -299.7900085449219,
      "loss": 0.0709,
      "rewards/accuracies": 0.8974999785423279,
      "rewards/chosen": 2.273066520690918,
      "rewards/margins": 15.19124984741211,
      "rewards/rejected": -12.915781021118164,
      "step": 9050
    },
    {
      "epoch": 1.8199999999999998,
      "grad_norm": 1.265625,
      "learning_rate": 2.009278921531141e-06,
      "logits/chosen": -1.1782631874084473,
      "logits/rejected": -2.8433496952056885,
      "logps/chosen": -148.50250244140625,
      "logps/rejected": -299.1025085449219,
      "loss": 0.0743,
      "rewards/accuracies": 0.8924999833106995,
      "rewards/chosen": 2.413759708404541,
      "rewards/margins": 15.21875,
      "rewards/rejected": -12.807656288146973,
      "step": 9100
    },
    {
      "epoch": 1.83,
      "grad_norm": 0.0002918243408203125,
      "learning_rate": 1.9807898502313577e-06,
      "logits/chosen": -1.0487598180770874,
      "logits/rejected": -2.6698436737060547,
      "logps/chosen": -143.33250427246094,
      "logps/rejected": -277.25,
      "loss": 0.1227,
      "rewards/accuracies": 0.8224999904632568,
      "rewards/chosen": 1.6386914253234863,
      "rewards/margins": 13.599374771118164,
      "rewards/rejected": -11.961718559265137,
      "step": 9150
    },
    {
      "epoch": 1.8399999999999999,
      "grad_norm": 0.00640869140625,
      "learning_rate": 1.9523710716745846e-06,
      "logits/chosen": -1.1239213943481445,
      "logits/rejected": -2.814990282058716,
      "logps/chosen": -145.6374969482422,
      "logps/rejected": -289.4549865722656,
      "loss": 0.0916,
      "rewards/accuracies": 0.8675000071525574,
      "rewards/chosen": 2.229843854904175,
      "rewards/margins": 14.695937156677246,
      "rewards/rejected": -12.4609375,
      "step": 9200
    },
    {
      "epoch": 1.85,
      "grad_norm": 1.1328125,
      "learning_rate": 1.9240264333085247e-06,
      "logits/chosen": -1.1269491910934448,
      "logits/rejected": -2.812253475189209,
      "logps/chosen": -142.22000122070312,
      "logps/rejected": -289.4075012207031,
      "loss": 0.0882,
      "rewards/accuracies": 0.8725000023841858,
      "rewards/chosen": 2.428847551345825,
      "rewards/margins": 14.7670316696167,
      "rewards/rejected": -12.332500457763672,
      "step": 9250
    },
    {
      "epoch": 1.8599999999999999,
      "grad_norm": 0.0002460479736328125,
      "learning_rate": 1.8957597725434814e-06,
      "logits/chosen": -1.052548885345459,
      "logits/rejected": -2.777416944503784,
      "logps/chosen": -144.0449981689453,
      "logps/rejected": -290.26251220703125,
      "loss": 0.083,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": 2.608515739440918,
      "rewards/margins": 14.833125114440918,
      "rewards/rejected": -12.229687690734863,
      "step": 9300
    },
    {
      "epoch": 1.87,
      "grad_norm": 0.000820159912109375,
      "learning_rate": 1.8675749162328472e-06,
      "logits/chosen": -0.9933964610099792,
      "logits/rejected": -2.699892520904541,
      "logps/chosen": -144.44000244140625,
      "logps/rejected": -286.8374938964844,
      "loss": 0.0882,
      "rewards/accuracies": 0.8725000023841858,
      "rewards/chosen": 2.7115039825439453,
      "rewards/margins": 14.4439058303833,
      "rewards/rejected": -11.732656478881836,
      "step": 9350
    },
    {
      "epoch": 1.88,
      "grad_norm": 1.1484375,
      "learning_rate": 1.839475680154994e-06,
      "logits/chosen": -0.9599626064300537,
      "logits/rejected": -2.6507763862609863,
      "logps/chosen": -139.96249389648438,
      "logps/rejected": -276.3900146484375,
      "loss": 0.1106,
      "rewards/accuracies": 0.8399999737739563,
      "rewards/chosen": 2.6600000858306885,
      "rewards/margins": 13.848750114440918,
      "rewards/rejected": -11.1899995803833,
      "step": 9400
    },
    {
      "epoch": 1.8900000000000001,
      "grad_norm": 0.0001621246337890625,
      "learning_rate": 1.8114658684966893e-06,
      "logits/chosen": -0.9667956829071045,
      "logits/rejected": -2.65411376953125,
      "logps/chosen": -140.2100067138672,
      "logps/rejected": -285.072509765625,
      "loss": 0.0812,
      "rewards/accuracies": 0.8824999928474426,
      "rewards/chosen": 2.996386766433716,
      "rewards/margins": 14.584375381469727,
      "rewards/rejected": -11.588281631469727,
      "step": 9450
    },
    {
      "epoch": 1.9,
      "grad_norm": 0.00021076202392578125,
      "learning_rate": 1.7835492733380621e-06,
      "logits/chosen": -0.9071944952011108,
      "logits/rejected": -2.6002306938171387,
      "logps/chosen": -140.1374969482422,
      "logps/rejected": -280.19000244140625,
      "loss": 0.0951,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.7462353706359863,
      "rewards/margins": 14.235312461853027,
      "rewards/rejected": -11.487500190734863,
      "step": 9500
    },
    {
      "epoch": 1.9100000000000001,
      "grad_norm": 8.487701416015625e-05,
      "learning_rate": 1.755729674139224e-06,
      "logits/chosen": -0.9953001141548157,
      "logits/rejected": -2.6695971488952637,
      "logps/chosen": -138.27499389648438,
      "logps/rejected": -274.2049865722656,
      "loss": 0.1106,
      "rewards/accuracies": 0.8399999737739563,
      "rewards/chosen": 2.5620312690734863,
      "rewards/margins": 13.859375,
      "rewards/rejected": -11.292655944824219,
      "step": 9550
    },
    {
      "epoch": 1.92,
      "grad_norm": 0.578125,
      "learning_rate": 1.7280108372285804e-06,
      "logits/chosen": -1.1185680627822876,
      "logits/rejected": -2.6991472244262695,
      "logps/chosen": -143.2100067138672,
      "logps/rejected": -288.0799865722656,
      "loss": 0.0847,
      "rewards/accuracies": 0.8774999976158142,
      "rewards/chosen": 2.851914167404175,
      "rewards/margins": 14.767969131469727,
      "rewards/rejected": -11.918749809265137,
      "step": 9600
    },
    {
      "epoch": 1.9300000000000002,
      "grad_norm": 1.53125,
      "learning_rate": 1.700396515292942e-06,
      "logits/chosen": -1.051497220993042,
      "logits/rejected": -2.6542186737060547,
      "logps/chosen": -140.2550048828125,
      "logps/rejected": -285.32000732421875,
      "loss": 0.0933,
      "rewards/accuracies": 0.8650000095367432,
      "rewards/chosen": 2.637402296066284,
      "rewards/margins": 14.5482816696167,
      "rewards/rejected": -11.911328315734863,
      "step": 9650
    },
    {
      "epoch": 1.94,
      "grad_norm": 0.0093994140625,
      "learning_rate": 1.67289044686946e-06,
      "logits/chosen": -1.104941725730896,
      "logits/rejected": -2.7016260623931885,
      "logps/chosen": -141.42750549316406,
      "logps/rejected": -286.2049865722656,
      "loss": 0.0812,
      "rewards/accuracies": 0.8824999928474426,
      "rewards/chosen": 2.8697266578674316,
      "rewards/margins": 14.7017183303833,
      "rewards/rejected": -11.830156326293945,
      "step": 9700
    },
    {
      "epoch": 1.95,
      "grad_norm": 0.0042724609375,
      "learning_rate": 1.6454963558394954e-06,
      "logits/chosen": -1.0008597373962402,
      "logits/rejected": -2.603630304336548,
      "logps/chosen": -137.0625,
      "logps/rejected": -274.5350036621094,
      "loss": 0.0916,
      "rewards/accuracies": 0.8675000071525574,
      "rewards/chosen": 2.792910099029541,
      "rewards/margins": 13.903437614440918,
      "rewards/rejected": -11.112500190734863,
      "step": 9750
    },
    {
      "epoch": 1.96,
      "grad_norm": 0.00189971923828125,
      "learning_rate": 1.6182179509244623e-06,
      "logits/chosen": -0.8819994926452637,
      "logits/rejected": -2.5757317543029785,
      "logps/chosen": -136.65750122070312,
      "logps/rejected": -267.739990234375,
      "loss": 0.1054,
      "rewards/accuracies": 0.8475000262260437,
      "rewards/chosen": 2.8574609756469727,
      "rewards/margins": 13.30062484741211,
      "rewards/rejected": -10.443046569824219,
      "step": 9800
    },
    {
      "epoch": 1.97,
      "grad_norm": 0.001617431640625,
      "learning_rate": 1.5910589251837258e-06,
      "logits/chosen": -0.9589361548423767,
      "logits/rejected": -2.654125928878784,
      "logps/chosen": -141.08250427246094,
      "logps/rejected": -274.427490234375,
      "loss": 0.0899,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": 2.935683488845825,
      "rewards/margins": 13.803203582763672,
      "rewards/rejected": -10.866328239440918,
      "step": 9850
    },
    {
      "epoch": 1.98,
      "grad_norm": 0.000946044921875,
      "learning_rate": 1.5640229555146237e-06,
      "logits/chosen": -0.9323858618736267,
      "logits/rejected": -2.6372363567352295,
      "logps/chosen": -140.74000549316406,
      "logps/rejected": -276.75,
      "loss": 0.0882,
      "rewards/accuracies": 0.8725000023841858,
      "rewards/chosen": 3.178281307220459,
      "rewards/margins": 13.927734375,
      "rewards/rejected": -10.7514066696167,
      "step": 9900
    },
    {
      "epoch": 1.99,
      "grad_norm": 1.265625,
      "learning_rate": 1.537113702154668e-06,
      "logits/chosen": -1.0329382419586182,
      "logits/rejected": -2.759021520614624,
      "logps/chosen": -140.61500549316406,
      "logps/rejected": -277.2174987792969,
      "loss": 0.0847,
      "rewards/accuracies": 0.8774999976158142,
      "rewards/chosen": 2.995312452316284,
      "rewards/margins": 13.927968978881836,
      "rewards/rejected": -10.926405906677246,
      "step": 9950
    },
    {
      "epoch": 2.0,
      "grad_norm": 0.69140625,
      "learning_rate": 1.5103348081860159e-06,
      "logits/chosen": -0.9912976026535034,
      "logits/rejected": -2.705747127532959,
      "logps/chosen": -140.69749450683594,
      "logps/rejected": -283.8374938964844,
      "loss": 0.0709,
      "rewards/accuracies": 0.8974999785423279,
      "rewards/chosen": 3.1512889862060547,
      "rewards/margins": 14.588281631469727,
      "rewards/rejected": -11.444062232971191,
      "step": 10000
    }
  ],
  "logging_steps": 50,
  "max_steps": 15000,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}