{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.0,
  "eval_steps": 50,
  "global_step": 1334,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.015012197410395946,
      "grad_norm": 0.48664575815200806,
      "kl": 0.05595267936587334,
      "learning_rate": 2.686567164179105e-06,
      "logits/chosen": -105151202.07792208,
      "logits/rejected": -100224814.26506025,
      "logps/chosen": -32.888798701298704,
      "logps/rejected": -30.840290850903614,
      "loss": 0.5002947807312011,
      "rewards/chosen": -0.0017914044392573368,
      "rewards/margins": -0.003978508190451383,
      "rewards/rejected": 0.0021871037511940463,
      "step": 10
    },
    {
      "epoch": 0.030024394820791892,
      "grad_norm": 0.7029150128364563,
      "kl": 0.08854874223470688,
      "learning_rate": 5.671641791044776e-06,
      "logits/chosen": -90880536.38095239,
      "logits/rejected": -110196500.21052632,
      "logps/chosen": -39.982947939918155,
      "logps/rejected": -31.82884457236842,
      "loss": 0.4987481594085693,
      "rewards/chosen": 0.007288790174892971,
      "rewards/margins": 0.008891924347420385,
      "rewards/rejected": -0.0016031341725274136,
      "step": 20
    },
    {
      "epoch": 0.04503659223118784,
      "grad_norm": 0.7273629903793335,
      "kl": 0.26002588868141174,
      "learning_rate": 8.656716417910447e-06,
      "logits/chosen": -98148453.05263157,
      "logits/rejected": -101435257.90476191,
      "logps/chosen": -34.69019197162829,
      "logps/rejected": -27.827526274181547,
      "loss": 0.49477639198303225,
      "rewards/chosen": 0.038317783882743435,
      "rewards/margins": 0.04702380150183102,
      "rewards/rejected": -0.008706017619087583,
      "step": 30
    },
    {
      "epoch": 0.060048789641583784,
      "grad_norm": 0.8483396172523499,
      "kl": 0.9080759286880493,
      "learning_rate": 1.1641791044776121e-05,
      "logits/chosen": -79740646.0289855,
      "logits/rejected": -91644567.91208792,
      "logps/chosen": -36.05313773777174,
      "logps/rejected": -33.51322115384615,
      "loss": 0.4763810157775879,
      "rewards/chosen": 0.14765035933342532,
      "rewards/margins": 0.1901713592418726,
      "rewards/rejected": -0.042520999908447266,
      "step": 40
    },
    {
      "epoch": 0.07506098705197974,
      "grad_norm": 1.2962456941604614,
      "kl": 2.3752338886260986,
      "learning_rate": 1.4626865671641792e-05,
      "logits/chosen": -82581724.27906977,
      "logits/rejected": -105110202.8108108,
      "logps/chosen": -26.611898732739824,
      "logps/rejected": -32.0681812183277,
      "loss": 0.4478025436401367,
      "rewards/chosen": 0.3435138879820358,
      "rewards/margins": 0.5002580459278234,
      "rewards/rejected": -0.15674415794578758,
      "step": 50
    },
    {
      "epoch": 0.07506098705197974,
      "eval_kl": 1.266696572303772,
      "eval_logits/chosen": -55813307.64397906,
      "eval_logits/rejected": -56554646.11518325,
      "eval_logps/chosen": -31.56733679646597,
      "eval_logps/rejected": -35.57937765870419,
      "eval_loss": 0.3994504511356354,
      "eval_rewards/chosen": 0.5163626146566181,
      "eval_rewards/margins": 0.8252790161452368,
      "eval_rewards/rejected": -0.3089164014886187,
      "eval_runtime": 464.3345,
      "eval_samples_per_second": 1.645,
      "eval_steps_per_second": 0.823,
      "step": 50
    },
    {
      "epoch": 0.09007318446237568,
      "grad_norm": 0.775963306427002,
      "kl": 4.466587066650391,
      "learning_rate": 1.7611940298507464e-05,
      "logits/chosen": -83022252.65116279,
      "logits/rejected": -83638320.43243243,
      "logps/chosen": -29.599955714026162,
      "logps/rejected": -38.7579246727196,
      "loss": 0.3778259754180908,
      "rewards/chosen": 0.6707564509192179,
      "rewards/margins": 1.2162449115630893,
      "rewards/rejected": -0.5454884606438715,
      "step": 60
    },
    {
      "epoch": 0.10508538187277162,
      "grad_norm": 0.43517357110977173,
      "kl": 5.635891914367676,
      "learning_rate": 1.9999877036668485e-05,
      "logits/chosen": -86946379.63636364,
      "logits/rejected": -74469447.1111111,
      "logps/chosen": -25.857749245383523,
      "logps/rejected": -35.837982177734375,
      "loss": 0.3408613443374634,
      "rewards/chosen": 0.8303727236661044,
      "rewards/margins": 1.9862453865282461,
      "rewards/rejected": -1.1558726628621419,
      "step": 70
    },
    {
      "epoch": 0.12009757928316757,
      "grad_norm": 0.37438932061195374,
      "kl": 5.818446636199951,
      "learning_rate": 1.999557363757665e-05,
      "logits/chosen": -97060782.08,
      "logits/rejected": -77247716.89411765,
      "logps/chosen": -30.320673828125,
      "logps/rejected": -56.52916475183824,
      "loss": 0.2436617136001587,
      "rewards/chosen": 1.0769728597005208,
      "rewards/margins": 3.3871797329771756,
      "rewards/rejected": -2.3102068732766545,
      "step": 80
    },
    {
      "epoch": 0.13510977669356353,
      "grad_norm": 0.4023344814777374,
      "kl": 5.037129878997803,
      "learning_rate": 1.9985125095570854e-05,
      "logits/chosen": -88758926.58227848,
      "logits/rejected": -90645194.27160494,
      "logps/chosen": -28.795388523536392,
      "logps/rejected": -63.26287012924383,
      "loss": 0.2622586011886597,
      "rewards/chosen": 0.9623794555664062,
      "rewards/margins": 4.024350295832128,
      "rewards/rejected": -3.0619708402657215,
      "step": 90
    },
    {
      "epoch": 0.15012197410395947,
      "grad_norm": 0.3208625614643097,
      "kl": 4.7330522537231445,
      "learning_rate": 1.996853783427282e-05,
      "logits/chosen": -78684659.8095238,
      "logits/rejected": -79006693.05263157,
      "logps/chosen": -23.673159644717263,
      "logps/rejected": -65.49229029605263,
      "loss": 0.27015070915222167,
      "rewards/chosen": 0.9256639026460194,
      "rewards/margins": 4.187760106901775,
      "rewards/rejected": -3.2620962042557564,
      "step": 100
    },
    {
      "epoch": 0.15012197410395947,
      "eval_kl": 0.14395534992218018,
      "eval_logits/chosen": -47605652.77486911,
      "eval_logits/rejected": -38367379.43455497,
      "eval_logps/chosen": -27.383620337041886,
      "eval_logps/rejected": -67.5675719895288,
      "eval_loss": 0.20005758106708527,
      "eval_rewards/chosen": 0.9347344043991328,
      "eval_rewards/margins": 4.442470530564872,
      "eval_rewards/rejected": -3.5077361261657396,
      "eval_runtime": 472.543,
      "eval_samples_per_second": 1.617,
      "eval_steps_per_second": 0.808,
      "step": 100
    },
    {
      "epoch": 0.16513417151435542,
      "grad_norm": 0.44810113310813904,
      "kl": 4.619333267211914,
      "learning_rate": 1.994582205130551e-05,
      "logits/chosen": -76966491.1780822,
      "logits/rejected": -71586910.16091955,
      "logps/chosen": -26.702449432791095,
      "logps/rejected": -66.4336218121408,
      "loss": 0.2479196548461914,
      "rewards/chosen": 0.9265415766467787,
      "rewards/margins": 4.2691862790758375,
      "rewards/rejected": -3.342644702429059,
      "step": 110
    },
    {
      "epoch": 0.18014636892475136,
      "grad_norm": 0.34525591135025024,
      "kl": 5.025256156921387,
      "learning_rate": 1.991699171202376e-05,
      "logits/chosen": -84793181.36470588,
      "logits/rejected": -70789720.74666667,
      "logps/chosen": -27.854406020220587,
      "logps/rejected": -62.20436197916667,
      "loss": 0.2833747148513794,
      "rewards/chosen": 0.8931190939510569,
      "rewards/margins": 4.17925760306564,
      "rewards/rejected": -3.286138509114583,
      "step": 120
    },
    {
      "epoch": 0.1951585663351473,
      "grad_norm": 0.38824889063835144,
      "kl": 5.002859592437744,
      "learning_rate": 1.9882064540928598e-05,
      "logits/chosen": -80103908.47311828,
      "logits/rejected": -67727787.94029851,
      "logps/chosen": -24.253184328797044,
      "logps/rejected": -76.31905900186567,
      "loss": 0.2879242658615112,
      "rewards/chosen": 0.9358312135101646,
      "rewards/margins": 4.883322759685538,
      "rewards/rejected": -3.9474915461753732,
      "step": 130
    },
    {
      "epoch": 0.21017076374554325,
      "grad_norm": 0.4769190847873688,
      "kl": 4.515671730041504,
      "learning_rate": 1.9841062010770435e-05,
      "logits/chosen": -79682590.47619048,
      "logits/rejected": -53528855.578947365,
      "logps/chosen": -31.625252859933035,
      "logps/rejected": -65.04932925575658,
      "loss": 0.268853759765625,
      "rewards/chosen": 0.9304619743710473,
      "rewards/margins": 4.450175526745636,
      "rewards/rejected": -3.5197135523745886,
      "step": 140
    },
    {
      "epoch": 0.2251829611559392,
      "grad_norm": 0.6323740482330322,
      "kl": 4.37551736831665,
      "learning_rate": 1.9794009329347926e-05,
      "logits/chosen": -86433298.73170732,
      "logits/rejected": -85731268.92307693,
      "logps/chosen": -33.249347965891765,
      "logps/rejected": -68.0100348056891,
      "loss": 0.25741305351257326,
      "rewards/chosen": 0.9401206970214844,
      "rewards/margins": 4.5076380998660355,
      "rewards/rejected": -3.567517402844551,
      "step": 150
    },
    {
      "epoch": 0.2251829611559392,
      "eval_kl": 0.17389115691184998,
      "eval_logits/chosen": -45713702.86910995,
      "eval_logits/rejected": -34640582.366492145,
      "eval_logps/chosen": -26.543809820844242,
      "eval_logps/rejected": -71.86549513252618,
      "eval_loss": 0.1908683031797409,
      "eval_rewards/chosen": 1.0187157436191099,
      "eval_rewards/margins": 4.956243544973004,
      "eval_rewards/rejected": -3.937527801353894,
      "eval_runtime": 464.1447,
      "eval_samples_per_second": 1.646,
      "eval_steps_per_second": 0.823,
      "step": 150
    },
    {
      "epoch": 0.24019515856633514,
      "grad_norm": 0.3375397026538849,
      "kl": 4.020479202270508,
      "learning_rate": 1.974093542401054e-05,
      "logits/chosen": -62679096.88888889,
      "logits/rejected": -77540241.82278481,
      "logps/chosen": -22.955916039737655,
      "logps/rejected": -71.9782127670095,
      "loss": 0.26079225540161133,
      "rewards/chosen": 0.8830830138406636,
      "rewards/margins": 4.736599025139121,
      "rewards/rejected": -3.8535160112984572,
      "step": 160
    },
    {
      "epoch": 0.2552073559767311,
      "grad_norm": 0.4141367971897125,
      "kl": 3.990988254547119,
      "learning_rate": 1.968187292387444e-05,
      "logits/chosen": -84086227.2,
      "logits/rejected": -68478252.8,
      "logps/chosen": -29.576809692382813,
      "logps/rejected": -76.20518798828125,
      "loss": 0.22698066234588624,
      "rewards/chosen": 1.1348169326782227,
      "rewards/margins": 5.321233177185058,
      "rewards/rejected": -4.186416244506836,
      "step": 170
    },
    {
      "epoch": 0.27021955338712705,
      "grad_norm": 0.2972460687160492,
      "kl": 3.929959774017334,
      "learning_rate": 1.9616858139762534e-05,
      "logits/chosen": -99423652.8219178,
      "logits/rejected": -75411597.2413793,
      "logps/chosen": -23.80077288901969,
      "logps/rejected": -63.870745779454026,
      "loss": 0.23734564781188966,
      "rewards/chosen": 0.9383544921875,
      "rewards/margins": 4.621817051679239,
      "rewards/rejected": -3.6834625594917383,
      "step": 180
    },
    {
      "epoch": 0.285231750797523,
      "grad_norm": 0.41057029366493225,
      "kl": 4.216883659362793,
      "learning_rate": 1.954593104188107e-05,
      "logits/chosen": -75821622.61333333,
      "logits/rejected": -63695125.08235294,
      "logps/chosen": -24.083863932291667,
      "logps/rejected": -68.76608455882354,
      "loss": 0.2280487060546875,
      "rewards/chosen": 1.1367711385091146,
      "rewards/margins": 5.0540437286975335,
      "rewards/rejected": -3.917272590188419,
      "step": 190
    },
    {
      "epoch": 0.30024394820791894,
      "grad_norm": 0.32265180349349976,
      "kl": 4.959702491760254,
      "learning_rate": 1.9469135235246503e-05,
      "logits/chosen": -70969464.80898876,
      "logits/rejected": -63905417.01408451,
      "logps/chosen": -26.195647164676966,
      "logps/rejected": -72.58189370598592,
      "loss": 0.2571362018585205,
      "rewards/chosen": 1.1260040797544328,
      "rewards/margins": 5.329004430491668,
      "rewards/rejected": -4.203000350737236,
      "step": 200
    },
    {
      "epoch": 0.30024394820791894,
      "eval_kl": 0.15900468826293945,
      "eval_logits/chosen": -43982228.77486911,
      "eval_logits/rejected": -31387007.329842933,
      "eval_logps/chosen": -26.374332767506544,
      "eval_logps/rejected": -77.79594445353403,
      "eval_loss": 0.18420633673667908,
      "eval_rewards/chosen": 1.0356630654859293,
      "eval_rewards/margins": 5.566235287651342,
      "eval_rewards/rejected": -4.5305722221654126,
      "eval_runtime": 418.7669,
      "eval_samples_per_second": 1.824,
      "eval_steps_per_second": 0.912,
      "step": 200
    },
    {
      "epoch": 0.3152561456183149,
      "grad_norm": 0.3199129104614258,
      "kl": 3.8654441833496094,
      "learning_rate": 1.9386517932877725e-05,
      "logits/chosen": -63076752.69565217,
      "logits/rejected": -68469287.38461539,
      "logps/chosen": -26.9500927026721,
      "logps/rejected": -77.23451450892857,
      "loss": 0.23405146598815918,
      "rewards/chosen": 0.9223934671153193,
      "rewards/margins": 5.440813340698767,
      "rewards/rejected": -4.518419873583448,
      "step": 210
    },
    {
      "epoch": 0.33026834302871083,
      "grad_norm": 0.4808659851551056,
      "kl": 4.875772476196289,
      "learning_rate": 1.9298129926770132e-05,
      "logits/chosen": -67988052.3956044,
      "logits/rejected": -63445236.86956522,
      "logps/chosen": -26.67523931146978,
      "logps/rejected": -78.15327785326087,
      "loss": 0.2701714277267456,
      "rewards/chosen": 1.0167858417217548,
      "rewards/margins": 5.61166012699123,
      "rewards/rejected": -4.594874285269475,
      "step": 220
    },
    {
      "epoch": 0.3452805404391068,
      "grad_norm": 0.2898392081260681,
      "kl": 3.1590735912323,
      "learning_rate": 1.920402555666941e-05,
      "logits/chosen": -67118973.96825397,
      "logits/rejected": -70008605.03092784,
      "logps/chosen": -28.408491830977184,
      "logps/rejected": -76.45369382248711,
      "loss": 0.19473642110824585,
      "rewards/chosen": 1.0573667495969743,
      "rewards/margins": 5.643574188586987,
      "rewards/rejected": -4.586207438990013,
      "step": 230
    },
    {
      "epoch": 0.3602927378495027,
      "grad_norm": 0.36392441391944885,
      "kl": 4.529810905456543,
      "learning_rate": 1.910426267666418e-05,
      "logits/chosen": -78076474.85057472,
      "logits/rejected": -74812261.69863014,
      "logps/chosen": -22.985556416127874,
      "logps/rejected": -74.91467786815069,
      "loss": 0.2578288555145264,
      "rewards/chosen": 1.0158540572243175,
      "rewards/margins": 5.46974408692252,
      "rewards/rejected": -4.4538900296982025,
      "step": 240
    },
    {
      "epoch": 0.37530493525989866,
      "grad_norm": 0.3606679439544678,
      "kl": 3.946249485015869,
      "learning_rate": 1.8998902619618117e-05,
      "logits/chosen": -75268391.38461539,
      "logits/rejected": -61223561.36585366,
      "logps/chosen": -24.976700220352566,
      "logps/rejected": -75.95566763528963,
      "loss": 0.23691632747650146,
      "rewards/chosen": 0.9890132806239984,
      "rewards/margins": 5.471519265046636,
      "rewards/rejected": -4.482505984422637,
      "step": 250
    },
    {
      "epoch": 0.37530493525989866,
      "eval_kl": 0.16116125881671906,
      "eval_logits/chosen": -45084184.12565445,
      "eval_logits/rejected": -31980281.298429318,
      "eval_logps/chosen": -26.101974087859947,
      "eval_logps/rejected": -80.09241553501309,
      "eval_loss": 0.18029513955116272,
      "eval_rewards/chosen": 1.0628989654061682,
      "eval_rewards/margins": 5.823118679186436,
      "eval_rewards/rejected": -4.760219713780268,
      "eval_runtime": 425.436,
      "eval_samples_per_second": 1.796,
      "eval_steps_per_second": 0.898,
      "step": 250
    },
    {
      "epoch": 0.3903171326702946,
      "grad_norm": 0.4404217302799225,
      "kl": 5.508103370666504,
      "learning_rate": 1.8888010159463294e-05,
      "logits/chosen": -64772149.89473684,
      "logits/rejected": -65391466.33846154,
      "logps/chosen": -25.71417814555921,
      "logps/rejected": -63.81729266826923,
      "loss": 0.28806486129760744,
      "rewards/chosen": 1.0554629677220395,
      "rewards/margins": 4.9188026706216785,
      "rewards/rejected": -3.8633397028996392,
      "step": 260
    },
    {
      "epoch": 0.40532933008069055,
      "grad_norm": 0.6277490258216858,
      "kl": 4.712449073791504,
      "learning_rate": 1.877165347137809e-05,
      "logits/chosen": -90587110.07594937,
      "logits/rejected": -68866022.71604939,
      "logps/chosen": -26.297134592563292,
      "logps/rejected": -75.93940489969135,
      "loss": 0.24233634471893312,
      "rewards/chosen": 1.0065902516811709,
      "rewards/margins": 5.576781434889119,
      "rewards/rejected": -4.570191183207948,
      "step": 270
    },
    {
      "epoch": 0.4203415274910865,
      "grad_norm": 0.32237356901168823,
      "kl": 3.980494737625122,
      "learning_rate": 1.864990408987401e-05,
      "logits/chosen": -70893069.83783785,
      "logits/rejected": -69000977.86046511,
      "logps/chosen": -28.647619298986488,
      "logps/rejected": -82.09332985101744,
      "loss": 0.2172463655471802,
      "rewards/chosen": 1.1764121184477936,
      "rewards/margins": 6.09591881244307,
      "rewards/rejected": -4.9195066939952765,
      "step": 280
    },
    {
      "epoch": 0.43535372490148244,
      "grad_norm": 0.3504326641559601,
      "kl": 4.214869022369385,
      "learning_rate": 1.852283686481721e-05,
      "logits/chosen": -68802216.58536585,
      "logits/rejected": -67867707.07692307,
      "logps/chosen": -27.0248398199314,
      "logps/rejected": -81.77991661658653,
      "loss": 0.242197847366333,
      "rewards/chosen": 1.0362097577350895,
      "rewards/margins": 5.799550634387733,
      "rewards/rejected": -4.763340876652644,
      "step": 290
    },
    {
      "epoch": 0.4503659223118784,
      "grad_norm": 0.33721521496772766,
      "kl": 4.173134803771973,
      "learning_rate": 1.8390529915411853e-05,
      "logits/chosen": -65034005.590361446,
      "logits/rejected": -67509134.96103896,
      "logps/chosen": -24.494224456419428,
      "logps/rejected": -84.23314478490259,
      "loss": 0.22235071659088135,
      "rewards/chosen": 1.1591761945241905,
      "rewards/margins": 6.0879243872494335,
      "rewards/rejected": -4.9287481927252434,
      "step": 300
    },
    {
      "epoch": 0.4503659223118784,
      "eval_kl": 0.17067986726760864,
      "eval_logits/chosen": -46740898.17801047,
      "eval_logits/rejected": -33324983.62303665,
      "eval_logps/chosen": -25.849294932100786,
      "eval_logps/rejected": -80.77142813318063,
      "eval_loss": 0.17754967510700226,
      "eval_rewards/chosen": 1.0881668809820844,
      "eval_rewards/margins": 5.916288685424165,
      "eval_rewards/rejected": -4.828121804442081,
      "eval_runtime": 427.9477,
      "eval_samples_per_second": 1.785,
      "eval_steps_per_second": 0.893,
      "step": 300
    },
    {
      "epoch": 0.46537811972227433,
      "grad_norm": 0.4849989116191864,
      "kl": 4.441267967224121,
      "learning_rate": 1.8253064582173457e-05,
      "logits/chosen": -69129365.06329113,
      "logits/rejected": -72012464.98765433,
      "logps/chosen": -21.719686387460442,
      "logps/rejected": -72.88394579475309,
      "loss": 0.24629390239715576,
      "rewards/chosen": 0.9837675456759296,
      "rewards/margins": 5.237387834666092,
      "rewards/rejected": -4.253620288990162,
      "step": 310
    },
    {
      "epoch": 0.4803903171326703,
      "grad_norm": 0.4396725594997406,
      "kl": 4.207592487335205,
      "learning_rate": 1.8110525376921863e-05,
      "logits/chosen": -79998642.60465117,
      "logits/rejected": -74981417.51351352,
      "logps/chosen": -25.47288335755814,
      "logps/rejected": -80.42956213048987,
      "loss": 0.2393803358078003,
      "rewards/chosen": 1.1084918532260628,
      "rewards/margins": 5.974426070524416,
      "rewards/rejected": -4.865934217298353,
      "step": 320
    },
    {
      "epoch": 0.4954025145430662,
      "grad_norm": 0.5366594195365906,
      "kl": 3.9820334911346436,
      "learning_rate": 1.796299993082451e-05,
      "logits/chosen": -74307654.1369863,
      "logits/rejected": -68484590.34482759,
      "logps/chosen": -26.794069055008563,
      "logps/rejected": -78.43782552083333,
      "loss": 0.22449307441711425,
      "rewards/chosen": 1.1078702325690282,
      "rewards/margins": 5.943478353191334,
      "rewards/rejected": -4.835608120622306,
      "step": 330
    },
    {
      "epoch": 0.5104147119534622,
      "grad_norm": 0.4778093099594116,
      "kl": 4.551473140716553,
      "learning_rate": 1.7810578940521998e-05,
      "logits/chosen": -98528541.76744185,
      "logits/rejected": -74760185.08108108,
      "logps/chosen": -20.286565736282704,
      "logps/rejected": -77.08236776815879,
      "loss": 0.2551869869232178,
      "rewards/chosen": 1.1039025949877361,
      "rewards/margins": 5.889928183564594,
      "rewards/rejected": -4.786025588576858,
      "step": 340
    },
    {
      "epoch": 0.5254269093638582,
      "grad_norm": 0.4355256259441376,
      "kl": 4.321000099182129,
      "learning_rate": 1.7653356112369e-05,
      "logits/chosen": -100444391.61904761,
      "logits/rejected": -81269827.36842105,
      "logps/chosen": -25.645176478794642,
      "logps/rejected": -87.44900030838816,
      "loss": 0.24073328971862792,
      "rewards/chosen": 1.0453554789225261,
      "rewards/margins": 6.272355263693291,
      "rewards/rejected": -5.226999784770765,
      "step": 350
    },
    {
      "epoch": 0.5254269093638582,
      "eval_kl": 0.1573699414730072,
      "eval_logits/chosen": -47666427.979057595,
      "eval_logits/rejected": -33615502.07329843,
      "eval_logps/chosen": -25.851928071825917,
      "eval_logps/rejected": -83.59740571825917,
      "eval_loss": 0.17489400506019592,
      "eval_rewards/chosen": 1.0879038865653632,
      "eval_rewards/margins": 6.198621979558655,
      "eval_rewards/rejected": -5.110718092993292,
      "eval_runtime": 422.5798,
      "eval_samples_per_second": 1.808,
      "eval_steps_per_second": 0.904,
      "step": 350
    },
    {
      "epoch": 0.5404391067742541,
      "grad_norm": 0.6740466356277466,
      "kl": 4.984511375427246,
      "learning_rate": 1.749142810482488e-05,
      "logits/chosen": -96085843.34883721,
      "logits/rejected": -83732369.2972973,
      "logps/chosen": -30.92190338844477,
      "logps/rejected": -78.17299672719595,
      "loss": 0.25183792114257814,
      "rewards/chosen": 1.143318264983421,
      "rewards/margins": 5.906809921312601,
      "rewards/rejected": -4.7634916563291805,
      "step": 360
    },
    {
      "epoch": 0.55545130418465,
      "grad_norm": 0.42306992411613464,
      "kl": 4.017538547515869,
      "learning_rate": 1.7324894469029405e-05,
      "logits/chosen": -83550086.4,
      "logits/rejected": -69219065.6,
      "logps/chosen": -21.729214477539063,
      "logps/rejected": -78.86099243164062,
      "loss": 0.2265170097351074,
      "rewards/chosen": 1.150559902191162,
      "rewards/margins": 5.680189037322998,
      "rewards/rejected": -4.529629135131836,
      "step": 370
    },
    {
      "epoch": 0.570463501595046,
      "grad_norm": 0.7183542251586914,
      "kl": 4.302075386047363,
      "learning_rate": 1.715385758760002e-05,
      "logits/chosen": -94022080.87671232,
      "logits/rejected": -67760139.77011494,
      "logps/chosen": -28.821867642337327,
      "logps/rejected": -88.21133598239942,
      "loss": 0.22173850536346434,
      "rewards/chosen": 1.1270856465378853,
      "rewards/margins": 6.429633759078295,
      "rewards/rejected": -5.30254811254041,
      "step": 380
    },
    {
      "epoch": 0.5854756990054419,
      "grad_norm": 0.4068809449672699,
      "kl": 4.413468360900879,
      "learning_rate": 1.697842261168843e-05,
      "logits/chosen": -89074789.20930232,
      "logits/rejected": -74542716.54054055,
      "logps/chosen": -24.867312409156977,
      "logps/rejected": -70.55822423986487,
      "loss": 0.24221227169036866,
      "rewards/chosen": 1.105927312096884,
      "rewards/margins": 5.485582892359764,
      "rewards/rejected": -4.37965558026288,
      "step": 390
    },
    {
      "epoch": 0.6004878964158379,
      "grad_norm": 0.3094225525856018,
      "kl": 3.9395923614501953,
      "learning_rate": 1.6798697396335102e-05,
      "logits/chosen": -84558361.28395061,
      "logits/rejected": -80714233.51898734,
      "logps/chosen": -30.648084852430557,
      "logps/rejected": -83.06282140031645,
      "loss": 0.22035319805145265,
      "rewards/chosen": 1.1816176426263503,
      "rewards/margins": 6.228112524944389,
      "rewards/rejected": -5.046494882318038,
      "step": 400
    },
    {
      "epoch": 0.6004878964158379,
      "eval_kl": 0.1634005606174469,
      "eval_logits/chosen": -50731554.84816754,
      "eval_logits/rejected": -36390836.942408375,
      "eval_logps/chosen": -25.668382894306284,
      "eval_logps/rejected": -84.24621134653141,
      "eval_loss": 0.1733660250902176,
      "eval_rewards/chosen": 1.1062583723617474,
      "eval_rewards/margins": 6.281857475560373,
      "eval_rewards/rejected": -5.175599103198626,
      "eval_runtime": 427.5005,
      "eval_samples_per_second": 1.787,
      "eval_steps_per_second": 0.894,
      "step": 400
    },
    {
      "epoch": 0.6155000938262338,
      "grad_norm": 0.4934229254722595,
      "kl": 3.653895616531372,
      "learning_rate": 1.6614792434161418e-05,
      "logits/chosen": -90473977.17333333,
      "logits/rejected": -89471861.45882353,
      "logps/chosen": -22.688899739583334,
      "logps/rejected": -79.36219554227941,
      "loss": 0.21901493072509765,
      "rewards/chosen": 1.0243033854166668,
      "rewards/margins": 5.917456317976409,
      "rewards/rejected": -4.893152932559743,
      "step": 410
    },
    {
      "epoch": 0.6305122912366298,
      "grad_norm": 0.4996384382247925,
      "kl": 3.6179280281066895,
      "learning_rate": 1.6426820787440366e-05,
      "logits/chosen": -85964416.0,
      "logits/rejected": -84499870.47619048,
      "logps/chosen": -26.152443333675986,
      "logps/rejected": -78.37731352306548,
      "loss": 0.2094388484954834,
      "rewards/chosen": 1.1747771815249795,
      "rewards/margins": 6.022950777135098,
      "rewards/rejected": -4.848173595610119,
      "step": 420
    },
    {
      "epoch": 0.6455244886470257,
      "grad_norm": 0.6519464254379272,
      "kl": 3.751269817352295,
      "learning_rate": 1.6234898018587336e-05,
      "logits/chosen": -82877628.63157895,
      "logits/rejected": -67029967.23809524,
      "logps/chosen": -25.796669407894736,
      "logps/rejected": -80.34830729166667,
      "loss": 0.21852760314941405,
      "rewards/chosen": 0.9713470057437295,
      "rewards/margins": 5.867524025135471,
      "rewards/rejected": -4.896177019391741,
      "step": 430
    },
    {
      "epoch": 0.6605366860574217,
      "grad_norm": 0.5263959765434265,
      "kl": 4.094634056091309,
      "learning_rate": 1.6039142119113956e-05,
      "logits/chosen": -80433825.68421052,
      "logits/rejected": -77660050.28571428,
      "logps/chosen": -23.38715402703536,
      "logps/rejected": -83.7175525483631,
      "loss": 0.2311333656311035,
      "rewards/chosen": 1.0536013151469983,
      "rewards/margins": 6.101049609650346,
      "rewards/rejected": -5.047448294503348,
      "step": 440
    },
    {
      "epoch": 0.6755488834678176,
      "grad_norm": 0.58863765001297,
      "kl": 5.108607292175293,
      "learning_rate": 1.58396734370885e-05,
      "logits/chosen": -92037412.57142857,
      "logits/rejected": -84992687.15789473,
      "logps/chosen": -34.8051031203497,
      "logps/rejected": -72.51608115748355,
      "loss": 0.25075249671936034,
      "rewards/chosen": 1.194018046061198,
      "rewards/margins": 5.641956663968271,
      "rewards/rejected": -4.447938617907073,
      "step": 450
    },
    {
      "epoch": 0.6755488834678176,
      "eval_kl": 0.15371429920196533,
      "eval_logits/chosen": -53600052.27225131,
      "eval_logits/rejected": -38773534.827225134,
      "eval_logps/chosen": -25.656866103566752,
      "eval_logps/rejected": -85.39284399541884,
      "eval_loss": 0.17146842181682587,
      "eval_rewards/chosen": 1.1074100514357004,
      "eval_rewards/margins": 6.397672483434228,
      "eval_rewards/rejected": -5.290262431998528,
      "eval_runtime": 428.4403,
      "eval_samples_per_second": 1.783,
      "eval_steps_per_second": 0.892,
      "step": 450
    },
    {
      "epoch": 0.6905610808782136,
      "grad_norm": 0.49697065353393555,
      "kl": 3.1813266277313232,
      "learning_rate": 1.5636614603147513e-05,
      "logits/chosen": -114965474.31884058,
      "logits/rejected": -84445184.0,
      "logps/chosen": -22.76505710767663,
      "logps/rejected": -89.04121415693682,
      "loss": 0.20205256938934327,
      "rewards/chosen": 0.9410718834918478,
      "rewards/margins": 6.368589281864959,
      "rewards/rejected": -5.427517398373111,
      "step": 460
    },
    {
      "epoch": 0.7055732782886095,
      "grad_norm": 0.6644816994667053,
      "kl": 3.073093891143799,
      "learning_rate": 1.543009045510419e-05,
      "logits/chosen": -89053725.25714286,
      "logits/rejected": -82642397.86666666,
      "logps/chosen": -27.184453473772322,
      "logps/rejected": -74.28021918402777,
      "loss": 0.1959690809249878,
      "rewards/chosen": 1.1658863612583705,
      "rewards/margins": 5.783280702621218,
      "rewards/rejected": -4.617394341362847,
      "step": 470
    },
    {
      "epoch": 0.7205854756990054,
      "grad_norm": 0.3856127858161926,
      "kl": 3.390268325805664,
      "learning_rate": 1.5220227961199767e-05,
      "logits/chosen": -105331782.1369863,
      "logits/rejected": -82775881.56321838,
      "logps/chosen": -26.920950275577912,
      "logps/rejected": -86.48434132543103,
      "loss": 0.1951006054878235,
      "rewards/chosen": 1.159071830854024,
      "rewards/margins": 6.306170415960884,
      "rewards/rejected": -5.1470985851068605,
      "step": 480
    },
    {
      "epoch": 0.7355976731094014,
      "grad_norm": 0.5393089056015015,
      "kl": 3.25313138961792,
      "learning_rate": 1.5007156142045176e-05,
      "logits/chosen": -91164856.8888889,
      "logits/rejected": -86136715.63636364,
      "logps/chosen": -23.249182807074654,
      "logps/rejected": -84.9217529296875,
      "loss": 0.21610660552978517,
      "rewards/chosen": 0.9365202585856119,
      "rewards/margins": 6.319411826856209,
      "rewards/rejected": -5.382891568270597,
      "step": 490
    },
    {
      "epoch": 0.7506098705197973,
      "grad_norm": 0.7707648873329163,
      "kl": 3.6630606651306152,
      "learning_rate": 1.4791005991300923e-05,
      "logits/chosen": -110857683.75308642,
      "logits/rejected": -82150322.2278481,
      "logps/chosen": -24.525343906732253,
      "logps/rejected": -82.56521954113924,
      "loss": 0.22861866950988768,
      "rewards/chosen": 0.9733524087034626,
      "rewards/margins": 6.094431865660543,
      "rewards/rejected": -5.121079456957081,
      "step": 500
    },
    {
      "epoch": 0.7506098705197973,
      "eval_kl": 0.1316855102777481,
      "eval_logits/chosen": -54686553.801047124,
      "eval_logits/rejected": -39620532.942408375,
      "eval_logps/chosen": -25.837093013743456,
      "eval_logps/rejected": -88.067878763089,
      "eval_loss": 0.16978919506072998,
      "eval_rewards/chosen": 1.0893871047733965,
      "eval_rewards/margins": 6.647152566160831,
      "eval_rewards/rejected": -5.557765461387435,
      "eval_runtime": 424.0565,
      "eval_samples_per_second": 1.802,
      "eval_steps_per_second": 0.901,
      "step": 500
    },
    {
      "epoch": 0.7656220679301933,
      "grad_norm": 0.5762482285499573,
      "kl": 3.2514896392822266,
      "learning_rate": 1.4571910395143964e-05,
      "logits/chosen": -123910669.4736842,
      "logits/rejected": -82550308.57142857,
      "logps/chosen": -20.17646066766036,
      "logps/rejected": -92.69414411272321,
      "loss": 0.21347689628601074,
      "rewards/chosen": 1.0528185995001542,
      "rewards/margins": 6.815291206340742,
      "rewards/rejected": -5.762472606840587,
      "step": 510
    },
    {
      "epoch": 0.7806342653405892,
      "grad_norm": 0.6658563017845154,
      "kl": 3.7189126014709473,
      "learning_rate": 1.4350004050571093e-05,
      "logits/chosen": -96275552.0,
      "logits/rejected": -91569664.0,
      "logps/chosen": -24.03918914794922,
      "logps/rejected": -82.615673828125,
      "loss": 0.2262340307235718,
      "rewards/chosen": 1.1256752014160156,
      "rewards/margins": 6.1439353942871096,
      "rewards/rejected": -5.018260192871094,
      "step": 520
    },
    {
      "epoch": 0.7956464627509852,
      "grad_norm": 0.7213343977928162,
      "kl": 4.146698951721191,
      "learning_rate": 1.412542338258905e-05,
      "logits/chosen": -71005795.90243903,
      "logits/rejected": -80605669.74358974,
      "logps/chosen": -21.523729277820124,
      "logps/rejected": -77.89060621995192,
      "loss": 0.22721762657165528,
      "rewards/chosen": 1.1042013400938453,
      "rewards/margins": 6.000391885591641,
      "rewards/rejected": -4.896190545497796,
      "step": 530
    },
    {
      "epoch": 0.8106586601613811,
      "grad_norm": 0.6627029180526733,
      "kl": 4.362208366394043,
      "learning_rate": 1.3898306460342295e-05,
      "logits/chosen": -103514886.91891892,
      "logits/rejected": -70722119.44186047,
      "logps/chosen": -30.26540725295608,
      "logps/rejected": -79.42020008175872,
      "loss": 0.21057591438293458,
      "rewards/chosen": 1.2330357319599874,
      "rewards/margins": 6.344258110003228,
      "rewards/rejected": -5.111222378043241,
      "step": 540
    },
    {
      "epoch": 0.825670857571777,
      "grad_norm": 0.6026250720024109,
      "kl": 4.500165939331055,
      "learning_rate": 1.366879291222995e-05,
      "logits/chosen": -97725642.41860466,
      "logits/rejected": -77692900.32432432,
      "logps/chosen": -19.79853856286337,
      "logps/rejected": -86.36903505067568,
      "loss": 0.23624191284179688,
      "rewards/chosen": 1.1637908580691316,
      "rewards/margins": 6.564948546519301,
      "rewards/rejected": -5.401157688450169,
      "step": 550
    },
    {
      "epoch": 0.825670857571777,
      "eval_kl": 0.13565272092819214,
      "eval_logits/chosen": -52988113.08900524,
      "eval_logits/rejected": -37333402.136125654,
      "eval_logps/chosen": -25.6715835651178,
      "eval_logps/rejected": -88.75424370091623,
      "eval_loss": 0.16850697994232178,
      "eval_rewards/chosen": 1.1059383372361746,
      "eval_rewards/margins": 6.7323411871625485,
      "eval_rewards/rejected": -5.626402849926374,
      "eval_runtime": 436.4536,
      "eval_samples_per_second": 1.75,
      "eval_steps_per_second": 0.875,
      "step": 550
    },
    {
      "epoch": 0.840683054982173,
      "grad_norm": 0.5151755213737488,
      "kl": 3.809490203857422,
      "learning_rate": 1.3437023840064178e-05,
      "logits/chosen": -93870756.97777778,
      "logits/rejected": -93916525.71428572,
      "logps/chosen": -24.47983127170139,
      "logps/rejected": -81.66769670758929,
      "loss": 0.2295558214187622,
      "rewards/chosen": 1.177097405327691,
      "rewards/margins": 6.296527511354477,
      "rewards/rejected": -5.119430106026786,
      "step": 560
    },
    {
      "epoch": 0.8556952523925689,
      "grad_norm": 0.5849828124046326,
      "kl": 3.8631434440612793,
      "learning_rate": 1.3203141732322693e-05,
      "logits/chosen": -99688978.96296297,
      "logits/rejected": -75819280.20253165,
      "logps/chosen": -28.64499240451389,
      "logps/rejected": -93.65779519382912,
      "loss": 0.2268832206726074,
      "rewards/chosen": 1.1103750158239294,
      "rewards/margins": 7.005714402196555,
      "rewards/rejected": -5.895339386372626,
      "step": 570
    },
    {
      "epoch": 0.8707074498029649,
      "grad_norm": 0.6972293257713318,
      "kl": 4.144640922546387,
      "learning_rate": 1.2967290376548783e-05,
      "logits/chosen": -105536728.84705882,
      "logits/rejected": -104656288.42666666,
      "logps/chosen": -25.18662109375,
      "logps/rejected": -73.02625,
      "loss": 0.23414719104766846,
      "rewards/chosen": 1.136143762925092,
      "rewards/margins": 5.7962092739928,
      "rewards/rejected": -4.6600655110677085,
      "step": 580
    },
    {
      "epoch": 0.8857196472133608,
      "grad_norm": 0.6593893766403198,
      "kl": 4.090695858001709,
      "learning_rate": 1.2729614770952672e-05,
      "logits/chosen": -82203424.0,
      "logits/rejected": -89865196.8,
      "logps/chosen": -24.66193389892578,
      "logps/rejected": -81.38163452148437,
      "loss": 0.215281081199646,
      "rewards/chosen": 1.1623407363891602,
      "rewards/margins": 6.0649263381958,
      "rewards/rejected": -4.90258560180664,
      "step": 590
    },
    {
      "epoch": 0.9007318446237568,
      "grad_norm": 0.30154913663864136,
      "kl": 4.011666297912598,
      "learning_rate": 1.2490261035268614e-05,
      "logits/chosen": -103245324.64197531,
      "logits/rejected": -64040046.17721519,
      "logps/chosen": -25.815553264853396,
      "logps/rejected": -85.59282288370254,
      "loss": 0.22003612518310547,
      "rewards/chosen": 1.1088857768494407,
      "rewards/margins": 6.442291476760437,
      "rewards/rejected": -5.333405699910997,
      "step": 600
    },
    {
      "epoch": 0.9007318446237568,
      "eval_kl": 0.1486717015504837,
      "eval_logits/chosen": -52628898.17801047,
      "eval_logits/rejected": -37037149.82198953,
      "eval_logps/chosen": -25.418955538285342,
      "eval_logps/rejected": -87.40471613219896,
      "eval_loss": 0.1679808497428894,
      "eval_rewards/chosen": 1.131200820363629,
      "eval_rewards/margins": 6.622650146484375,
      "eval_rewards/rejected": -5.491449326120746,
      "eval_runtime": 434.8716,
      "eval_samples_per_second": 1.757,
      "eval_steps_per_second": 0.878,
      "step": 600
    },
    {
      "epoch": 0.9157440420341527,
      "grad_norm": 0.8136588931083679,
      "kl": 4.867345333099365,
      "learning_rate": 1.2249376320922435e-05,
      "logits/chosen": -96478865.38271604,
      "logits/rejected": -88826089.31645569,
      "logps/chosen": -24.912190755208332,
      "logps/rejected": -73.71896014636076,
      "loss": 0.2281566619873047,
      "rewards/chosen": 1.2302150961793499,
      "rewards/margins": 5.906620990784769,
      "rewards/rejected": -4.676405894605419,
      "step": 610
    },
    {
      "epoch": 0.9307562394445487,
      "grad_norm": 0.34996625781059265,
      "kl": 3.6859207153320312,
      "learning_rate": 1.2007108720564852e-05,
      "logits/chosen": -104111360.0,
      "logits/rejected": -78802962.73170732,
      "logps/chosen": -17.025127704326923,
      "logps/rejected": -87.1907274199695,
      "loss": 0.21024839878082274,
      "rewards/chosen": 1.2063480279384515,
      "rewards/margins": 6.545918236232088,
      "rewards/rejected": -5.339570208293636,
      "step": 620
    },
    {
      "epoch": 0.9457684368549446,
      "grad_norm": 0.5199890732765198,
      "kl": 4.800234794616699,
      "learning_rate": 1.1763607177026131e-05,
      "logits/chosen": -101240820.74725275,
      "logits/rejected": -65516358.492753625,
      "logps/chosen": -25.177771935096153,
      "logps/rejected": -84.0569095335145,
      "loss": 0.2449950933456421,
      "rewards/chosen": 1.216908716893458,
      "rewards/margins": 6.35318651000374,
      "rewards/rejected": -5.136277793110281,
      "step": 630
    },
    {
      "epoch": 0.9607806342653405,
      "grad_norm": 0.5771164298057556,
      "kl": 4.2080159187316895,
      "learning_rate": 1.1519021391748024e-05,
      "logits/chosen": -97543491.67816092,
      "logits/rejected": -95486464.0,
      "logps/chosen": -25.306859509698278,
      "logps/rejected": -79.20900444135275,
      "loss": 0.2462038516998291,
      "rewards/chosen": 1.1318982880691002,
      "rewards/margins": 6.078684828897439,
      "rewards/rejected": -4.946786540828339,
      "step": 640
    },
    {
      "epoch": 0.9757928316757365,
      "grad_norm": 0.6379005312919617,
      "kl": 4.442286014556885,
      "learning_rate": 1.12735017327494e-05,
      "logits/chosen": -89721952.46376811,
      "logits/rejected": -92305312.35164835,
      "logps/chosen": -26.444505774456523,
      "logps/rejected": -81.37484975961539,
      "loss": 0.20378758907318115,
      "rewards/chosen": 1.1361859086631,
      "rewards/margins": 6.409890353632193,
      "rewards/rejected": -5.273704444969093,
      "step": 650
    },
    {
      "epoch": 0.9757928316757365,
      "eval_kl": 0.14565975964069366,
      "eval_logits/chosen": -52615848.87958115,
      "eval_logits/rejected": -36491556.18848167,
      "eval_logps/chosen": -25.423930382853403,
      "eval_logps/rejected": -88.12435577552355,
      "eval_loss": 0.16747047007083893,
      "eval_rewards/chosen": 1.1307032719956642,
      "eval_rewards/margins": 6.694117201560454,
      "eval_rewards/rejected": -5.56341392956479,
      "eval_runtime": 433.8353,
      "eval_samples_per_second": 1.761,
      "eval_steps_per_second": 0.881,
      "step": 650
    },
    {
      "epoch": 0.9908050290861324,
      "grad_norm": 0.466814786195755,
      "kl": 5.0540056228637695,
      "learning_rate": 1.1027199142181999e-05,
      "logits/chosen": -101186585.6,
      "logits/rejected": -91657702.4,
      "logps/chosen": -21.362211608886717,
      "logps/rejected": -79.55250854492188,
      "loss": 0.23439102172851561,
      "rewards/chosen": 1.1953808784484863,
      "rewards/margins": 6.219569492340088,
      "rewards/rejected": -5.024188613891601,
      "step": 660
    },
    {
      "epoch": 1.0045036592231187,
      "grad_norm": 0.6553441286087036,
      "kl": 3.510671615600586,
      "learning_rate": 1.0780265043533252e-05,
      "logits/chosen": -106023401.07462686,
      "logits/rejected": -84727529.31645569,
      "logps/chosen": -23.81528575384795,
      "logps/rejected": -83.47320633900317,
      "loss": 0.21112267971038817,
      "rewards/chosen": 1.0875146211083255,
      "rewards/margins": 6.22628809260259,
      "rewards/rejected": -5.1387734714942646,
      "step": 670
    },
    {
      "epoch": 1.0195158566335147,
      "grad_norm": 0.7766548991203308,
      "kl": 4.419243812561035,
      "learning_rate": 1.0532851248533187e-05,
      "logits/chosen": -140482802.52631578,
      "logits/rejected": -84243260.95238096,
      "logps/chosen": -26.12240921823602,
      "logps/rejected": -79.62333170572917,
      "loss": 0.2175424337387085,
      "rewards/chosen": 1.0876518048738177,
      "rewards/margins": 6.136545324684086,
      "rewards/rejected": -5.048893519810268,
      "step": 680
    },
    {
      "epoch": 1.0345280540439106,
      "grad_norm": 0.6276580095291138,
      "kl": 3.7814064025878906,
      "learning_rate": 1.0285109863822632e-05,
      "logits/chosen": -85505976.70886075,
      "logits/rejected": -103004653.03703703,
      "logps/chosen": -25.645328570015824,
      "logps/rejected": -85.05873239776234,
      "loss": 0.22568042278289796,
      "rewards/chosen": 1.1665521935571599,
      "rewards/margins": 6.489668543142423,
      "rewards/rejected": -5.323116349585263,
      "step": 690
    },
    {
      "epoch": 1.0495402514543066,
      "grad_norm": 0.4705566167831421,
      "kl": 4.468962669372559,
      "learning_rate": 1.0037193197440117e-05,
      "logits/chosen": -93083011.28205128,
      "logits/rejected": -83736931.90243903,
      "logps/chosen": -26.694182567107372,
      "logps/rejected": -77.2444502667683,
      "loss": 0.23378393650054932,
      "rewards/chosen": 1.1905487256172376,
      "rewards/margins": 6.1078944704247835,
      "rewards/rejected": -4.9173457448075455,
      "step": 700
    },
    {
      "epoch": 1.0495402514543066,
      "eval_kl": 0.13426250219345093,
      "eval_logits/chosen": -53473505.172774866,
      "eval_logits/rejected": -37231283.60209424,
      "eval_logps/chosen": -25.48450537876309,
      "eval_logps/rejected": -88.80958565117801,
      "eval_loss": 0.1667330414056778,
      "eval_rewards/chosen": 1.1246460120715396,
      "eval_rewards/margins": 6.756583568313359,
      "eval_rewards/rejected": -5.6319375562418195,
      "eval_runtime": 433.1925,
      "eval_samples_per_second": 1.764,
      "eval_steps_per_second": 0.882,
      "step": 700
    },
    {
      "epoch": 1.0645524488647027,
      "grad_norm": 0.4909619390964508,
      "kl": 3.246283769607544,
      "learning_rate": 9.789253665184971e-06,
      "logits/chosen": -87369889.3150685,
      "logits/rejected": -85285476.04597701,
      "logps/chosen": -21.669224569242296,
      "logps/rejected": -90.8365548670977,
      "loss": 0.19179887771606446,
      "rewards/chosen": 1.123171035557577,
      "rewards/margins": 6.876017238188684,
      "rewards/rejected": -5.7528462026311065,
      "step": 710
    },
    {
      "epoch": 1.0795646462750985,
      "grad_norm": 0.5964953899383545,
      "kl": 4.596374034881592,
      "learning_rate": 9.541443696914142e-06,
      "logits/chosen": -87436251.85882352,
      "logits/rejected": -91590417.06666666,
      "logps/chosen": -19.859073414522058,
      "logps/rejected": -85.19293619791667,
      "loss": 0.24003658294677735,
      "rewards/chosen": 1.0800759708180148,
      "rewards/margins": 6.487529991000306,
      "rewards/rejected": -5.407454020182292,
      "step": 720
    },
    {
      "epoch": 1.0945768436854943,
      "grad_norm": 0.7163007855415344,
      "kl": 4.345240592956543,
      "learning_rate": 9.293915642830407e-06,
      "logits/chosen": -100356619.37777779,
      "logits/rejected": -67504727.77142857,
      "logps/chosen": -23.59975043402778,
      "logps/rejected": -85.96699916294642,
      "loss": 0.24338247776031494,
      "rewards/chosen": 1.1390112135145398,
      "rewards/margins": 6.603775617811415,
      "rewards/rejected": -5.464764404296875,
      "step": 730
    },
    {
      "epoch": 1.1095890410958904,
      "grad_norm": 0.4801461398601532,
      "kl": 4.036700248718262,
      "learning_rate": 9.046821679819528e-06,
      "logits/chosen": -99535643.75903614,
      "logits/rejected": -75779311.37662338,
      "logps/chosen": -23.405419039439007,
      "logps/rejected": -80.26659522118507,
      "loss": 0.22335414886474608,
      "rewards/chosen": 1.214461085308029,
      "rewards/margins": 6.408107801141024,
      "rewards/rejected": -5.193646715832995,
      "step": 740
    },
    {
      "epoch": 1.1246012385062865,
      "grad_norm": 0.8056963682174683,
      "kl": 3.960627794265747,
      "learning_rate": 8.800313717893959e-06,
      "logits/chosen": -105539597.4736842,
      "logits/rejected": -83317174.85714285,
      "logps/chosen": -28.418807180304277,
      "logps/rejected": -83.92525809151786,
      "loss": 0.18643591403961182,
      "rewards/chosen": 1.4098168423301296,
      "rewards/margins": 6.669779015065434,
      "rewards/rejected": -5.259962172735305,
      "step": 750
    },
    {
      "epoch": 1.1246012385062865,
      "eval_kl": 0.13305240869522095,
      "eval_logits/chosen": -54042077.15183246,
      "eval_logits/rejected": -37955905.67539267,
      "eval_logps/chosen": -25.507919870746072,
      "eval_logps/rejected": -89.86107248036649,
      "eval_loss": 0.16618458926677704,
      "eval_rewards/chosen": 1.1223043871175555,
      "eval_rewards/margins": 6.859389220232739,
      "eval_rewards/rejected": -5.737084833115183,
      "eval_runtime": 433.0131,
      "eval_samples_per_second": 1.764,
      "eval_steps_per_second": 0.882,
      "step": 750
    },
    {
      "epoch": 1.1396134359166823,
      "grad_norm": 0.5588005781173706,
      "kl": 3.8682758808135986,
      "learning_rate": 8.554543306800668e-06,
      "logits/chosen": -115601833.55844156,
      "logits/rejected": -101657297.73493975,
      "logps/chosen": -26.970395571225648,
      "logps/rejected": -90.736375188253,
      "loss": 0.20916638374328614,
      "rewards/chosen": 1.233702325201654,
      "rewards/margins": 6.799562200248717,
      "rewards/rejected": -5.565859875047063,
      "step": 760
    },
    {
      "epoch": 1.1546256333270781,
      "grad_norm": 0.6878859996795654,
      "kl": 3.435823917388916,
      "learning_rate": 8.309661542850376e-06,
      "logits/chosen": -85981894.4,
      "logits/rejected": -88435206.4,
      "logps/chosen": -18.470010375976564,
      "logps/rejected": -87.00191650390624,
      "loss": 0.21931285858154298,
      "rewards/chosen": 1.0244508743286134,
      "rewards/margins": 6.513032722473145,
      "rewards/rejected": -5.488581848144531,
      "step": 770
    },
    {
      "epoch": 1.1696378307374742,
      "grad_norm": 0.6787204742431641,
      "kl": 4.617588520050049,
      "learning_rate": 8.065818976025607e-06,
      "logits/chosen": -100501036.8,
      "logits/rejected": -93501510.4,
      "logps/chosen": -23.78038635253906,
      "logps/rejected": -80.3609375,
      "loss": 0.22347478866577147,
      "rewards/chosen": 1.2114054679870605,
      "rewards/margins": 6.213579082489014,
      "rewards/rejected": -5.002173614501953,
      "step": 780
    },
    {
      "epoch": 1.1846500281478702,
      "grad_norm": 0.8261892795562744,
      "kl": 4.476070404052734,
      "learning_rate": 7.823165517424608e-06,
      "logits/chosen": -109933940.36363636,
      "logits/rejected": -87284124.44444445,
      "logps/chosen": -26.592254638671875,
      "logps/rejected": -86.63107638888889,
      "loss": 0.24246513843536377,
      "rewards/chosen": 1.1668794805353337,
      "rewards/margins": 6.552800534951567,
      "rewards/rejected": -5.385921054416233,
      "step": 790
    },
    {
      "epoch": 1.199662225558266,
      "grad_norm": 0.6302554607391357,
      "kl": 4.160749435424805,
      "learning_rate": 7.581850347098052e-06,
      "logits/chosen": -106630573.41935484,
      "logits/rejected": -80972089.31343284,
      "logps/chosen": -22.595963016633064,
      "logps/rejected": -93.17525652985074,
      "loss": 0.26153428554534913,
      "rewards/chosen": 0.970795416062878,
      "rewards/margins": 6.776746890380742,
      "rewards/rejected": -5.8059514743178635,
      "step": 800
    },
    {
      "epoch": 1.199662225558266,
      "eval_kl": 0.1418178677558899,
      "eval_logits/chosen": -54769111.790575914,
      "eval_logits/rejected": -38202357.27748691,
      "eval_logps/chosen": -25.350251043030106,
      "eval_logps/rejected": -89.12186068390052,
      "eval_loss": 0.16583210229873657,
      "eval_rewards/chosen": 1.1380715095559963,
      "eval_rewards/margins": 6.801235418669217,
      "eval_rewards/rejected": -5.66316390911322,
      "eval_runtime": 439.3783,
      "eval_samples_per_second": 1.739,
      "eval_steps_per_second": 0.869,
      "step": 800
    },
    {
      "epoch": 1.2146744229686621,
      "grad_norm": 0.866295337677002,
      "kl": 3.919668197631836,
      "learning_rate": 7.342021822335144e-06,
      "logits/chosen": -100262202.02666667,
      "logits/rejected": -68069791.6235294,
      "logps/chosen": -35.465442708333335,
      "logps/rejected": -78.75108570772059,
      "loss": 0.2064718246459961,
      "rewards/chosen": 1.1432391357421876,
      "rewards/margins": 6.094182667451746,
      "rewards/rejected": -4.950943531709559,
      "step": 810
    },
    {
      "epoch": 1.229686620379058,
      "grad_norm": 0.5769370198249817,
      "kl": 4.099992275238037,
      "learning_rate": 7.1038273864555865e-06,
      "logits/chosen": -97576707.0117647,
      "logits/rejected": -107417941.33333333,
      "logps/chosen": -25.98314280790441,
      "logps/rejected": -87.1881640625,
      "loss": 0.23273687362670897,
      "rewards/chosen": 1.1329481237074908,
      "rewards/margins": 6.32427869011374,
      "rewards/rejected": -5.19133056640625,
      "step": 820
    },
    {
      "epoch": 1.244698817789454,
      "grad_norm": 0.7531806230545044,
      "kl": 3.6843295097351074,
      "learning_rate": 6.867413478163445e-06,
      "logits/chosen": -78316946.28571428,
      "logits/rejected": -93422969.26315789,
      "logps/chosen": -23.63427007765997,
      "logps/rejected": -83.31368857935855,
      "loss": 0.23035976886749268,
      "rewards/chosen": 1.1488791874476842,
      "rewards/margins": 6.381895352126961,
      "rewards/rejected": -5.2330161646792765,
      "step": 830
    },
    {
      "epoch": 1.2597110151998498,
      "grad_norm": 0.6657658219337463,
      "kl": 3.697706699371338,
      "learning_rate": 6.632925441518594e-06,
      "logits/chosen": -121944169.02564102,
      "logits/rejected": -104590610.73170732,
      "logps/chosen": -25.239830603966347,
      "logps/rejected": -85.82620760289635,
      "loss": 0.20410506725311278,
      "rewards/chosen": 1.2310838943872697,
      "rewards/margins": 6.589877571740548,
      "rewards/rejected": -5.358793677353278,
      "step": 840
    },
    {
      "epoch": 1.274723212610246,
      "grad_norm": 0.718334972858429,
      "kl": 3.1744580268859863,
      "learning_rate": 6.400507436581178e-06,
      "logits/chosen": -118265521.77777778,
      "logits/rejected": -87497675.63636364,
      "logps/chosen": -22.66006130642361,
      "logps/rejected": -80.14004239169034,
      "loss": 0.2051610231399536,
      "rewards/chosen": 1.1418331993950739,
      "rewards/margins": 6.250424799292979,
      "rewards/rejected": -5.108591599897905,
      "step": 850
    },
    {
      "epoch": 1.274723212610246,
      "eval_kl": 0.13580438494682312,
      "eval_logits/chosen": -54494524.314136125,
      "eval_logits/rejected": -38059876.52356021,
      "eval_logps/chosen": -25.360545852421467,
      "eval_logps/rejected": -90.01760880235602,
      "eval_loss": 0.1653730720281601,
      "eval_rewards/chosen": 1.1370419806834915,
      "eval_rewards/margins": 6.889781212931529,
      "eval_rewards/rejected": -5.752739232248037,
      "eval_runtime": 436.4699,
      "eval_samples_per_second": 1.75,
      "eval_steps_per_second": 0.875,
      "step": 850
    },
    {
      "epoch": 1.2897354100206417,
      "grad_norm": 0.8406268954277039,
      "kl": 3.3996193408966064,
      "learning_rate": 6.17030235078397e-06,
      "logits/chosen": -103157228.05194806,
      "logits/rejected": -76576545.92771085,
      "logps/chosen": -23.79248046875,
      "logps/rejected": -83.25042945218374,
      "loss": 0.1881808042526245,
      "rewards/chosen": 1.337797784186029,
      "rewards/margins": 6.671168866170216,
      "rewards/rejected": -5.333371081984187,
      "step": 860
    },
    {
      "epoch": 1.3047476074310378,
      "grad_norm": 0.5345896482467651,
      "kl": 4.3026533126831055,
      "learning_rate": 5.942451711087103e-06,
      "logits/chosen": -91860792.19512194,
      "logits/rejected": -88229369.43589744,
      "logps/chosen": -25.162978753810975,
      "logps/rejected": -85.17337114383012,
      "loss": 0.2208491086959839,
      "rewards/chosen": 1.286998004448123,
      "rewards/margins": 6.5604120293880275,
      "rewards/rejected": -5.273414024939904,
      "step": 870
    },
    {
      "epoch": 1.3197598048414336,
      "grad_norm": 0.774782121181488,
      "kl": 4.132607460021973,
      "learning_rate": 5.717095596969226e-06,
      "logits/chosen": -90163577.26315789,
      "logits/rejected": -97945203.8095238,
      "logps/chosen": -25.397158974095394,
      "logps/rejected": -81.13313220796131,
      "loss": 0.23573668003082277,
      "rewards/chosen": 1.0274274725663035,
      "rewards/margins": 6.3404724293185355,
      "rewards/rejected": -5.313044956752232,
      "step": 880
    },
    {
      "epoch": 1.3347720022518297,
      "grad_norm": 0.7283003926277161,
      "kl": 4.143404006958008,
      "learning_rate": 5.494372554308554e-06,
      "logits/chosen": -98379586.37037037,
      "logits/rejected": -108482605.3670886,
      "logps/chosen": -26.02851019965278,
      "logps/rejected": -84.22099362143987,
      "loss": 0.2245171546936035,
      "rewards/chosen": 1.1866206416377314,
      "rewards/margins": 6.487602380537059,
      "rewards/rejected": -5.300981738899328,
      "step": 890
    },
    {
      "epoch": 1.3497841996622255,
      "grad_norm": 0.8141359686851501,
      "kl": 2.8307042121887207,
      "learning_rate": 5.274419510206713e-06,
      "logits/chosen": -99083153.2972973,
      "logits/rejected": -92244980.09302326,
      "logps/chosen": -26.87116303315034,
      "logps/rejected": -93.76043559229652,
      "loss": 0.1964324712753296,
      "rewards/chosen": 1.0808814796241555,
      "rewards/margins": 7.159457855895957,
      "rewards/rejected": -6.078576376271802,
      "step": 900
    },
    {
      "epoch": 1.3497841996622255,
      "eval_kl": 0.13515853881835938,
      "eval_logits/chosen": -55153846.28272251,
      "eval_logits/rejected": -38913712.92146597,
      "eval_logps/chosen": -25.354182857493456,
      "eval_logps/rejected": -90.31069003599477,
      "eval_loss": 0.16496765613555908,
      "eval_rewards/chosen": 1.1376782162651342,
      "eval_rewards/margins": 6.919725188410094,
      "eval_rewards/rejected": -5.78204697214496,
      "eval_runtime": 421.2799,
      "eval_samples_per_second": 1.814,
      "eval_steps_per_second": 0.907,
      "step": 900
    },
    {
      "epoch": 1.3647963970726216,
      "grad_norm": 0.7665517926216125,
      "kl": 4.347356796264648,
      "learning_rate": 5.057371688807849e-06,
      "logits/chosen": -97613490.08695652,
      "logits/rejected": -81990761.41176471,
      "logps/chosen": -20.336362092391305,
      "logps/rejected": -81.53354061351104,
      "loss": 0.2507650375366211,
      "rewards/chosen": 1.0822478584621265,
      "rewards/margins": 6.298613897065067,
      "rewards/rejected": -5.216366038602941,
      "step": 910
    },
    {
      "epoch": 1.3798085944830174,
      "grad_norm": 0.7740744352340698,
      "kl": 3.4625542163848877,
      "learning_rate": 4.843362528164653e-06,
      "logits/chosen": -89836807.1111111,
      "logits/rejected": -76998423.27272727,
      "logps/chosen": -26.65255567762587,
      "logps/rejected": -82.26154119318181,
      "loss": 0.19089690446853638,
      "rewards/chosen": 1.2612567477756076,
      "rewards/margins": 6.494063907199436,
      "rewards/rejected": -5.232807159423828,
      "step": 920
    },
    {
      "epoch": 1.3948207918934135,
      "grad_norm": 0.9630061984062195,
      "kl": 5.1812615394592285,
      "learning_rate": 4.632523598202435e-06,
      "logits/chosen": -93226178.56,
      "logits/rejected": -92916608.0,
      "logps/chosen": -22.01804443359375,
      "logps/rejected": -70.4688232421875,
      "loss": 0.2744077205657959,
      "rewards/chosen": 1.1364972686767578,
      "rewards/margins": 5.4957613881429035,
      "rewards/rejected": -4.359264119466146,
      "step": 930
    },
    {
      "epoch": 1.4098329893038093,
      "grad_norm": 0.6798044443130493,
      "kl": 3.6221089363098145,
      "learning_rate": 4.4249845198317595e-06,
      "logits/chosen": -105949212.05479452,
      "logits/rejected": -94108677.88505746,
      "logps/chosen": -25.15871147260274,
      "logps/rejected": -91.54803116020115,
      "loss": 0.1968392014503479,
      "rewards/chosen": 1.132062206529591,
      "rewards/margins": 7.05869023162837,
      "rewards/rejected": -5.926628025098779,
      "step": 940
    },
    {
      "epoch": 1.4248451867142053,
      "grad_norm": 0.8968099355697632,
      "kl": 4.054515838623047,
      "learning_rate": 4.2208728852592475e-06,
      "logits/chosen": -93645798.4,
      "logits/rejected": -91873120.0,
      "logps/chosen": -30.97816162109375,
      "logps/rejected": -87.55768432617188,
      "loss": 0.20583555698394776,
      "rewards/chosen": 1.347819995880127,
      "rewards/margins": 7.024199390411377,
      "rewards/rejected": -5.67637939453125,
      "step": 950
    },
    {
      "epoch": 1.4248451867142053,
      "eval_kl": 0.13550503551959991,
      "eval_logits/chosen": -55356973.570680626,
      "eval_logits/rejected": -39265502.4921466,
      "eval_logps/chosen": -25.34827491001309,
      "eval_logps/rejected": -90.32589577879581,
      "eval_loss": 0.16480737924575806,
      "eval_rewards/chosen": 1.1382690749243292,
      "eval_rewards/margins": 6.921837132638662,
      "eval_rewards/rejected": -5.783568057714333,
      "eval_runtime": 421.7775,
      "eval_samples_per_second": 1.811,
      "eval_steps_per_second": 0.906,
      "step": 950
    },
    {
      "epoch": 1.4398573841246012,
      "grad_norm": 0.9420547485351562,
      "kl": 3.70204496383667,
      "learning_rate": 4.020314179545609e-06,
      "logits/chosen": -85311225.75609756,
      "logits/rejected": -102413193.84615384,
      "logps/chosen": -23.1737060546875,
      "logps/rejected": -92.63661232972757,
      "loss": 0.22018556594848632,
      "rewards/chosen": 1.1600302719488376,
      "rewards/margins": 6.917001582295392,
      "rewards/rejected": -5.756971310346555,
      "step": 960
    },
    {
      "epoch": 1.4548695815349972,
      "grad_norm": 0.6381933689117432,
      "kl": 3.5098605155944824,
      "learning_rate": 3.823431703459164e-06,
      "logits/chosen": -91596382.31578948,
      "logits/rejected": -75453647.23809524,
      "logps/chosen": -22.27265046772204,
      "logps/rejected": -97.9072265625,
      "loss": 0.19860419034957885,
      "rewards/chosen": 1.1405779186048006,
      "rewards/margins": 7.267270917581735,
      "rewards/rejected": -6.126692998976934,
      "step": 970
    },
    {
      "epoch": 1.469881778945393,
      "grad_norm": 0.801608145236969,
      "kl": 4.244645118713379,
      "learning_rate": 3.6303464976721835e-06,
      "logits/chosen": -83721495.81395349,
      "logits/rejected": -104408437.62162162,
      "logps/chosen": -23.19318336664244,
      "logps/rejected": -76.96543760557432,
      "loss": 0.23014593124389648,
      "rewards/chosen": 1.1628135858580124,
      "rewards/margins": 6.100889708245048,
      "rewards/rejected": -4.938076122387035,
      "step": 980
    },
    {
      "epoch": 1.4848939763557891,
      "grad_norm": 0.6242425441741943,
      "kl": 4.364640712738037,
      "learning_rate": 3.4411772683467235e-06,
      "logits/chosen": -100387865.92405063,
      "logits/rejected": -97113922.37037037,
      "logps/chosen": -22.812269766119464,
      "logps/rejected": -83.23450159143519,
      "loss": 0.20972530841827391,
      "rewards/chosen": 1.2276213440714003,
      "rewards/margins": 6.537462170412065,
      "rewards/rejected": -5.309840826340664,
      "step": 990
    },
    {
      "epoch": 1.499906173766185,
      "grad_norm": 0.6481726765632629,
      "kl": 3.0429587364196777,
      "learning_rate": 3.2560403141557085e-06,
      "logits/chosen": -108133474.19178082,
      "logits/rejected": -89296595.86206897,
      "logps/chosen": -25.251272541202912,
      "logps/rejected": -89.57416823814656,
      "loss": 0.19645323753356933,
      "rewards/chosen": 1.1944668913540775,
      "rewards/margins": 7.028910433132992,
      "rewards/rejected": -5.834443541778915,
      "step": 1000
    },
    {
      "epoch": 1.499906173766185,
      "eval_kl": 0.12928542494773865,
      "eval_logits/chosen": -55498553.633507855,
      "eval_logits/rejected": -39448407.12041885,
      "eval_logps/chosen": -25.369434616328533,
      "eval_logps/rejected": -90.84591786649214,
      "eval_loss": 0.16446402668952942,
      "eval_rewards/chosen": 1.13615289658152,
      "eval_rewards/margins": 6.971722268309269,
      "eval_rewards/rejected": -5.835569371727749,
      "eval_runtime": 422.5452,
      "eval_samples_per_second": 1.808,
      "eval_steps_per_second": 0.904,
      "step": 1000
    },
    {
      "epoch": 1.514918371176581,
      "grad_norm": 0.8008208274841309,
      "kl": 4.670881271362305,
      "learning_rate": 3.0750494547840737e-06,
      "logits/chosen": -125810654.24175824,
      "logits/rejected": -89304412.75362319,
      "logps/chosen": -21.928126073145606,
      "logps/rejected": -89.32652570199275,
      "loss": 0.24952163696289062,
      "rewards/chosen": 1.1456220018994676,
      "rewards/margins": 6.920434119766315,
      "rewards/rejected": -5.774812117866848,
      "step": 1010
    },
    {
      "epoch": 1.5299305685869768,
      "grad_norm": 0.7026108503341675,
      "kl": 3.7687172889709473,
      "learning_rate": 2.8983159609539634e-06,
      "logits/chosen": -110794227.2,
      "logits/rejected": -108752460.8,
      "logps/chosen": -27.65631103515625,
      "logps/rejected": -90.34964599609376,
      "loss": 0.21506090164184571,
      "rewards/chosen": 1.2219769477844238,
      "rewards/margins": 6.7797516822814945,
      "rewards/rejected": -5.5577747344970705,
      "step": 1020
    },
    {
      "epoch": 1.544942765997373,
      "grad_norm": 0.7361843585968018,
      "kl": 5.285219669342041,
      "learning_rate": 2.7259484860170206e-06,
      "logits/chosen": -122753947.82608695,
      "logits/rejected": -96830245.64705883,
      "logps/chosen": -23.716366975203805,
      "logps/rejected": -79.43409639246323,
      "loss": 0.25809249877929685,
      "rewards/chosen": 1.178453196649966,
      "rewards/margins": 6.409470365480389,
      "rewards/rejected": -5.231017168830423,
      "step": 1030
    },
    {
      "epoch": 1.559954963407769,
      "grad_norm": 0.49050334095954895,
      "kl": 3.2523837089538574,
      "learning_rate": 2.558052999155768e-06,
      "logits/chosen": -92674758.68656716,
      "logits/rejected": -81995770.49462366,
      "logps/chosen": -23.637330923507463,
      "logps/rejected": -95.77403603830645,
      "loss": 0.16697038412094117,
      "rewards/chosen": 1.3456653196420243,
      "rewards/margins": 7.250253588291218,
      "rewards/rejected": -5.904588268649194,
      "step": 1040
    },
    {
      "epoch": 1.5749671608181648,
      "grad_norm": 0.872729480266571,
      "kl": 4.101504802703857,
      "learning_rate": 2.3947327202351887e-06,
      "logits/chosen": -114927466.14634146,
      "logits/rejected": -84980342.15384616,
      "logps/chosen": -27.76390410632622,
      "logps/rejected": -88.0575420673077,
      "loss": 0.2262244701385498,
      "rewards/chosen": 1.1740545877596227,
      "rewards/margins": 6.741694452167675,
      "rewards/rejected": -5.5676398644080525,
      "step": 1050
    },
    {
      "epoch": 1.5749671608181648,
      "eval_kl": 0.13142813742160797,
      "eval_logits/chosen": -55718252.56544503,
      "eval_logits/rejected": -39580859.64397906,
      "eval_logps/chosen": -25.34311088841623,
      "eval_logps/rejected": -90.90671016034031,
      "eval_loss": 0.16429825127124786,
      "eval_rewards/chosen": 1.1387850776392752,
      "eval_rewards/margins": 6.980434317863424,
      "eval_rewards/rejected": -5.841649240224149,
      "eval_runtime": 421.6304,
      "eval_samples_per_second": 1.812,
      "eval_steps_per_second": 0.906,
      "step": 1050
    },
    {
      "epoch": 1.5899793582285606,
      "grad_norm": 0.7624918222427368,
      "kl": 4.680811882019043,
      "learning_rate": 2.2360880563445763e-06,
      "logits/chosen": -109705061.20930232,
      "logits/rejected": -81854505.51351352,
      "logps/chosen": -22.512400072674417,
      "logps/rejected": -72.15485114020271,
      "loss": 0.2276033878326416,
      "rewards/chosen": 1.2336682164391806,
      "rewards/margins": 5.973305849066925,
      "rewards/rejected": -4.739637632627745,
      "step": 1060
    },
    {
      "epoch": 1.6049915556389567,
      "grad_norm": 0.4747796654701233,
      "kl": 3.267906904220581,
      "learning_rate": 2.0822165400686033e-06,
      "logits/chosen": -104269369.6,
      "logits/rejected": -106300960.0,
      "logps/chosen": -30.304464721679686,
      "logps/rejected": -98.04397583007812,
      "loss": 0.19671732187271118,
      "rewards/chosen": 1.3054424285888673,
      "rewards/margins": 7.5222820281982425,
      "rewards/rejected": -6.216839599609375,
      "step": 1070
    },
    {
      "epoch": 1.6200037530493527,
      "grad_norm": 0.9367624521255493,
      "kl": 3.2648837566375732,
      "learning_rate": 1.933212769525613e-06,
      "logits/chosen": -89395676.44444445,
      "logits/rejected": -104650240.0,
      "logps/chosen": -21.667002360026043,
      "logps/rejected": -84.4835371537642,
      "loss": 0.20028886795043946,
      "rewards/chosen": 1.037461280822754,
      "rewards/margins": 6.473727399652654,
      "rewards/rejected": -5.4362661188299,
      "step": 1080
    },
    {
      "epoch": 1.6350159504597486,
      "grad_norm": 0.6833193302154541,
      "kl": 3.370619297027588,
      "learning_rate": 1.7891683502099832e-06,
      "logits/chosen": -98081811.2,
      "logits/rejected": -81132697.6,
      "logps/chosen": -23.849533081054688,
      "logps/rejected": -80.27091674804687,
      "loss": 0.2054957628250122,
      "rewards/chosen": 1.2119413375854493,
      "rewards/margins": 6.449519538879395,
      "rewards/rejected": -5.237578201293945,
      "step": 1090
    },
    {
      "epoch": 1.6500281478701444,
      "grad_norm": 0.6855626702308655,
      "kl": 3.7833855152130127,
      "learning_rate": 1.6501718386743282e-06,
      "logits/chosen": -90577152.0,
      "logits/rejected": -74229938.60465117,
      "logps/chosen": -25.44064743454392,
      "logps/rejected": -92.05196788699128,
      "loss": 0.18979684114456177,
      "rewards/chosen": 1.2005912162162162,
      "rewards/margins": 7.079025153466398,
      "rewards/rejected": -5.878433937250182,
      "step": 1100
    },
    {
      "epoch": 1.6500281478701444,
      "eval_kl": 0.12844021618366241,
      "eval_logits/chosen": -56039509.78010471,
      "eval_logits/rejected": -39965537.84293193,
      "eval_logps/chosen": -25.35427488956152,
      "eval_logps/rejected": -91.16684391361257,
      "eval_loss": 0.16429360210895538,
      "eval_rewards/chosen": 1.1376691089250655,
      "eval_rewards/margins": 7.0053321079433895,
      "eval_rewards/rejected": -5.867662999018324,
      "eval_runtime": 421.252,
      "eval_samples_per_second": 1.814,
      "eval_steps_per_second": 0.907,
      "step": 1100
    },
    {
      "epoch": 1.6650403452805405,
      "grad_norm": 1.1428500413894653,
      "kl": 4.261361122131348,
      "learning_rate": 1.5163086880861045e-06,
      "logits/chosen": -90409088.0,
      "logits/rejected": -88890406.4,
      "logps/chosen": -27.13056640625,
      "logps/rejected": -81.66841430664063,
      "loss": 0.2300961971282959,
      "rewards/chosen": 1.1836955070495605,
      "rewards/margins": 6.292539882659912,
      "rewards/rejected": -5.108844375610351,
      "step": 1110
    },
    {
      "epoch": 1.6800525426909365,
      "grad_norm": 0.7117341160774231,
      "kl": 3.3029885292053223,
      "learning_rate": 1.3876611956921793e-06,
      "logits/chosen": -111789738.66666667,
      "logits/rejected": -96844811.63636364,
      "logps/chosen": -29.337073432074654,
      "logps/rejected": -84.19040749289773,
      "loss": 0.1927316427230835,
      "rewards/chosen": 1.232086393568251,
      "rewards/margins": 6.685429524893713,
      "rewards/rejected": -5.453343131325462,
      "step": 1120
    },
    {
      "epoch": 1.6950647401013323,
      "grad_norm": 0.5888846516609192,
      "kl": 3.230769395828247,
      "learning_rate": 1.264308452223616e-06,
      "logits/chosen": -111363690.98507462,
      "logits/rejected": -93282744.43010753,
      "logps/chosen": -23.554046175373134,
      "logps/rejected": -93.44833669354838,
      "loss": 0.17531968355178834,
      "rewards/chosen": 1.087284258942106,
      "rewards/margins": 7.054321993107093,
      "rewards/rejected": -5.967037734164987,
      "step": 1130
    },
    {
      "epoch": 1.7100769375117282,
      "grad_norm": 0.6945241093635559,
      "kl": 3.231876850128174,
      "learning_rate": 1.1463262932717323e-06,
      "logits/chosen": -100743661.26829268,
      "logits/rejected": -99711829.33333333,
      "logps/chosen": -25.812577410442074,
      "logps/rejected": -81.0398199619391,
      "loss": 0.21396026611328126,
      "rewards/chosen": 1.2160039762171304,
      "rewards/margins": 6.219116769185881,
      "rewards/rejected": -5.00311279296875,
      "step": 1140
    },
    {
      "epoch": 1.7250891349221242,
      "grad_norm": 0.7128337621688843,
      "kl": 2.7168688774108887,
      "learning_rate": 1.0337872526654357e-06,
      "logits/chosen": -115620558.22222222,
      "logits/rejected": -87219781.81818181,
      "logps/chosen": -23.565806070963543,
      "logps/rejected": -90.92021040482955,
      "loss": 0.1860736846923828,
      "rewards/chosen": 1.1724183824327257,
      "rewards/margins": 6.89070298936632,
      "rewards/rejected": -5.718284606933594,
      "step": 1150
    },
    {
      "epoch": 1.7250891349221242,
      "eval_kl": 0.12663534283638,
      "eval_logits/chosen": -56202390.11518325,
      "eval_logits/rejected": -40215629.738219894,
      "eval_logps/chosen": -25.382503169993456,
      "eval_logps/rejected": -91.49146146924083,
      "eval_loss": 0.16411779820919037,
      "eval_rewards/chosen": 1.134846392726399,
      "eval_rewards/margins": 7.034971147307552,
      "eval_rewards/rejected": -5.900124754581152,
      "eval_runtime": 419.4694,
      "eval_samples_per_second": 1.821,
      "eval_steps_per_second": 0.911,
      "step": 1150
    },
    {
      "epoch": 1.7401013323325203,
      "grad_norm": 0.7305581569671631,
      "kl": 3.2470200061798096,
      "learning_rate": 9.267605178784033e-07,
      "logits/chosen": -93602140.44444445,
      "logits/rejected": -86012282.18181819,
      "logps/chosen": -22.607952541775173,
      "logps/rejected": -88.2861328125,
      "loss": 0.1894938588142395,
      "rewards/chosen": 1.1574607425265842,
      "rewards/margins": 6.9352643369424225,
      "rewards/rejected": -5.777803594415838,
      "step": 1160
    },
    {
      "epoch": 1.7551135297429161,
      "grad_norm": 0.7399213314056396,
      "kl": 3.665466785430908,
      "learning_rate": 8.253118874935628e-07,
      "logits/chosen": -131597628.04938272,
      "logits/rejected": -97115894.278481,
      "logps/chosen": -22.83131992669753,
      "logps/rejected": -88.38763350474683,
      "loss": 0.22316858768463135,
      "rewards/chosen": 1.0157588440694927,
      "rewards/margins": 6.545139545834722,
      "rewards/rejected": -5.52938070176523,
      "step": 1170
    },
    {
      "epoch": 1.770125727153312,
      "grad_norm": 0.8386373519897461,
      "kl": 3.3606925010681152,
      "learning_rate": 7.2950373075101e-07,
      "logits/chosen": -123296072.20512821,
      "logits/rejected": -92810976.7804878,
      "logps/chosen": -24.13685803535657,
      "logps/rejected": -94.89799685594512,
      "loss": 0.2036738395690918,
      "rewards/chosen": 1.2165361062074318,
      "rewards/margins": 7.279486426567569,
      "rewards/rejected": -6.062950320360137,
      "step": 1180
    },
    {
      "epoch": 1.785137924563708,
      "grad_norm": 0.5710262656211853,
      "kl": 3.399351119995117,
      "learning_rate": 6.393949492042606e-07,
      "logits/chosen": -89650777.6,
      "logits/rejected": -79596396.8,
      "logps/chosen": -20.27527313232422,
      "logps/rejected": -84.6680419921875,
      "loss": 0.20696897506713868,
      "rewards/chosen": 1.1883172035217284,
      "rewards/margins": 6.458553409576416,
      "rewards/rejected": -5.270236206054688,
      "step": 1190
    },
    {
      "epoch": 1.800150121974104,
      "grad_norm": 0.7187512516975403,
      "kl": 4.214442729949951,
      "learning_rate": 5.550409405083656e-07,
      "logits/chosen": -86440611.84,
      "logits/rejected": -87346730.16470589,
      "logps/chosen": -23.561062825520832,
      "logps/rejected": -78.86371783088235,
      "loss": 0.21113624572753906,
      "rewards/chosen": 1.2850724283854167,
      "rewards/margins": 6.377582337622549,
      "rewards/rejected": -5.092509909237132,
      "step": 1200
    },
    {
      "epoch": 1.800150121974104,
      "eval_kl": 0.12559062242507935,
      "eval_logits/chosen": -56251209.71727749,
      "eval_logits/rejected": -40232681.21465968,
      "eval_logps/chosen": -25.399662037794503,
      "eval_logps/rejected": -91.77921506871728,
      "eval_loss": 0.16403362154960632,
      "eval_rewards/chosen": 1.1331302183460814,
      "eval_rewards/margins": 7.06202905465171,
      "eval_rewards/rejected": -5.9288988363056285,
      "eval_runtime": 416.8935,
      "eval_samples_per_second": 1.833,
      "eval_steps_per_second": 0.916,
      "step": 1200
    },
    {
      "epoch": 1.8151623193845,
      "grad_norm": 0.6832463145256042,
      "kl": 4.084317207336426,
      "learning_rate": 4.7649356436218484e-07,
      "logits/chosen": -100126576.3902439,
      "logits/rejected": -74491890.87179486,
      "logps/chosen": -26.829598775724087,
      "logps/rejected": -91.56940479767628,
      "loss": 0.20048983097076417,
      "rewards/chosen": 1.2928543090820312,
      "rewards/margins": 7.3325772407727365,
      "rewards/rejected": -6.039722931690705,
      "step": 1210
    },
    {
      "epoch": 1.8301745167948957,
      "grad_norm": 0.6473249793052673,
      "kl": 4.0460524559021,
      "learning_rate": 4.0380111062575555e-07,
      "logits/chosen": -102995694.13953489,
      "logits/rejected": -78813273.94594595,
      "logps/chosen": -18.408095248909884,
      "logps/rejected": -83.04783836570945,
      "loss": 0.22513113021850586,
      "rewards/chosen": 1.1756272426871366,
      "rewards/margins": 6.415878113675312,
      "rewards/rejected": -5.240250870988175,
      "step": 1220
    },
    {
      "epoch": 1.8451867142052918,
      "grad_norm": 0.9530460238456726,
      "kl": 3.9541709423065186,
      "learning_rate": 3.3700826963233735e-07,
      "logits/chosen": -109047211.6043956,
      "logits/rejected": -95853879.6521739,
      "logps/chosen": -23.388164813701923,
      "logps/rejected": -93.71621659873189,
      "loss": 0.22243864536285402,
      "rewards/chosen": 1.2722159584799966,
      "rewards/margins": 7.281630410189689,
      "rewards/rejected": -6.009414451709692,
      "step": 1230
    },
    {
      "epoch": 1.8601989116156878,
      "grad_norm": 0.6990249752998352,
      "kl": 3.6704108715057373,
      "learning_rate": 2.761561047134009e-07,
      "logits/chosen": -101100076.24691358,
      "logits/rejected": -93491297.21518987,
      "logps/chosen": -31.321445794753085,
      "logps/rejected": -85.77536590189874,
      "loss": 0.2213526725769043,
      "rewards/chosen": 1.1806782851984472,
      "rewards/margins": 6.7673752479654565,
      "rewards/rejected": -5.586696962767009,
      "step": 1240
    },
    {
      "epoch": 1.8752111090260837,
      "grad_norm": 0.6529566645622253,
      "kl": 3.52073335647583,
      "learning_rate": 2.2128202695345324e-07,
      "logits/chosen": -108384976.5925926,
      "logits/rejected": -110889893.26582278,
      "logps/chosen": -22.77246997974537,
      "logps/rejected": -85.0575059335443,
      "loss": 0.20152571201324462,
      "rewards/chosen": 1.2122876202618633,
      "rewards/margins": 6.816732293051618,
      "rewards/rejected": -5.6044446727897546,
      "step": 1250
    },
    {
      "epoch": 1.8752111090260837,
      "eval_kl": 0.12627530097961426,
      "eval_logits/chosen": -56267920.7539267,
      "eval_logits/rejected": -40249947.14136126,
      "eval_logps/chosen": -25.39533653059555,
      "eval_logps/rejected": -91.83618291884817,
      "eval_loss": 0.16398374736309052,
      "eval_rewards/chosen": 1.1335626572214497,
      "eval_rewards/margins": 7.068159173296385,
      "eval_rewards/rejected": -5.934596516074935,
      "eval_runtime": 434.1736,
      "eval_samples_per_second": 1.76,
      "eval_steps_per_second": 0.88,
      "step": 1250
    },
    {
      "epoch": 1.8902233064364795,
      "grad_norm": 0.8459712862968445,
      "kl": 3.8880774974823,
      "learning_rate": 1.7241977219020988e-07,
      "logits/chosen": -91639609.22352941,
      "logits/rejected": -86516210.34666666,
      "logps/chosen": -24.805669806985293,
      "logps/rejected": -82.68663411458333,
      "loss": 0.22068817615509034,
      "rewards/chosen": 1.1558401668772977,
      "rewards/margins": 6.285211911668965,
      "rewards/rejected": -5.1293717447916665,
      "step": 1260
    },
    {
      "epoch": 1.9052355038468756,
      "grad_norm": 0.6756543517112732,
      "kl": 3.553884983062744,
      "learning_rate": 1.29599380274259e-07,
      "logits/chosen": -110346801.09589042,
      "logits/rejected": -77342543.44827586,
      "logps/chosen": -23.693068412885275,
      "logps/rejected": -85.32204112787356,
      "loss": 0.19014220237731932,
      "rewards/chosen": 1.2435979973779965,
      "rewards/margins": 6.845515132682414,
      "rewards/rejected": -5.601917135304418,
      "step": 1270
    },
    {
      "epoch": 1.9202477012572716,
      "grad_norm": 0.47155070304870605,
      "kl": 4.422393798828125,
      "learning_rate": 9.284717660097087e-08,
      "logits/chosen": -105391794.89156626,
      "logits/rejected": -89061336.10389611,
      "logps/chosen": -21.691320947853917,
      "logps/rejected": -87.40567928165585,
      "loss": 0.2130509376525879,
      "rewards/chosen": 1.323310484369117,
      "rewards/margins": 6.870470447209012,
      "rewards/rejected": -5.547159962839895,
      "step": 1280
    },
    {
      "epoch": 1.9352598986676675,
      "grad_norm": 0.4472443163394928,
      "kl": 3.7577896118164062,
      "learning_rate": 6.218575592600085e-08,
      "logits/chosen": -110973014.36144578,
      "logits/rejected": -77297936.62337662,
      "logps/chosen": -20.669974821159638,
      "logps/rejected": -91.66918628246754,
      "loss": 0.21389095783233641,
      "rewards/chosen": 1.1915134292051017,
      "rewards/margins": 6.848199790939072,
      "rewards/rejected": -5.6566863617339695,
      "step": 1290
    },
    {
      "epoch": 1.9502720960780633,
      "grad_norm": 0.6200128197669983,
      "kl": 4.125483512878418,
      "learning_rate": 3.7633968474338756e-08,
      "logits/chosen": -104873574.4,
      "logits/rejected": -72432032.91428572,
      "logps/chosen": -22.444955783420138,
      "logps/rejected": -87.90431780133929,
      "loss": 0.22912445068359374,
      "rewards/chosen": 1.2457435607910157,
      "rewards/margins": 6.887794167654855,
      "rewards/rejected": -5.642050606863839,
      "step": 1300
    },
    {
      "epoch": 1.9502720960780633,
      "eval_kl": 0.1256106048822403,
      "eval_logits/chosen": -56246100.43979058,
      "eval_logits/rejected": -40258029.235602096,
      "eval_logps/chosen": -25.398680362401834,
      "eval_logps/rejected": -91.83032354384817,
      "eval_loss": 0.16401667892932892,
      "eval_rewards/chosen": 1.1332284817520861,
      "eval_rewards/margins": 7.067238932504704,
      "eval_rewards/rejected": -5.934010450752618,
      "eval_runtime": 423.6676,
      "eval_samples_per_second": 1.803,
      "eval_steps_per_second": 0.902,
      "step": 1300
    },
    {
      "epoch": 1.9652842934884593,
      "grad_norm": 0.702017605304718,
      "kl": 4.159964561462402,
      "learning_rate": 1.9206908351453447e-08,
      "logits/chosen": -90640182.85714285,
      "logits/rejected": -113604419.36842105,
      "logps/chosen": -23.79085431780134,
      "logps/rejected": -79.48878238075658,
      "loss": 0.22954988479614258,
      "rewards/chosen": 1.2310051690964472,
      "rewards/margins": 6.36179146252778,
      "rewards/rejected": -5.130786293431332,
      "step": 1310
    },
    {
      "epoch": 1.9802964908988554,
      "grad_norm": 0.4877679944038391,
      "kl": 4.048553943634033,
      "learning_rate": 6.915904263634643e-09,
      "logits/chosen": -131888278.58823529,
      "logits/rejected": -94457789.2173913,
      "logps/chosen": -28.130503934972428,
      "logps/rejected": -86.42369543987772,
      "loss": 0.1838226318359375,
      "rewards/chosen": 1.2597647274241728,
      "rewards/margins": 6.924890035253656,
      "rewards/rejected": -5.665125307829483,
      "step": 1320
    },
    {
      "epoch": 1.9953086883092512,
      "grad_norm": 0.5606428384780884,
      "kl": 4.262922286987305,
      "learning_rate": 7.685125532586047e-10,
      "logits/chosen": -103120477.0909091,
      "logits/rejected": -96393641.63855422,
      "logps/chosen": -19.540289544439936,
      "logps/rejected": -81.25472985692771,
      "loss": 0.21868836879730225,
      "rewards/chosen": 1.1489427244508421,
      "rewards/margins": 6.39600752172588,
      "rewards/rejected": -5.247064797275038,
      "step": 1330
    },
    {
      "epoch": 2.0,
      "eval_kl": 0.12448398768901825,
      "eval_logits/chosen": -56288711.70680628,
      "eval_logits/rejected": -40230820.85863874,
      "eval_logps/chosen": -25.394838023560208,
      "eval_logps/rejected": -91.83712369109948,
      "eval_loss": 0.16396363079547882,
      "eval_rewards/chosen": 1.1336126677028795,
      "eval_rewards/margins": 7.068303133180628,
      "eval_rewards/rejected": -5.934690465477749,
      "eval_runtime": 432.8328,
      "eval_samples_per_second": 1.765,
      "eval_steps_per_second": 0.883,
      "step": 1334
    },
    {
      "epoch": 2.0,
      "step": 1334,
      "total_flos": 0.0,
      "train_loss": 0.23600534109518803,
      "train_runtime": 42516.8613,
      "train_samples_per_second": 0.501,
      "train_steps_per_second": 0.031
    }
  ],
  "logging_steps": 10,
  "max_steps": 1334,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}