{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 200,
  "global_step": 3884,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "entropy": 1.0695502400398254,
      "epoch": 0.0025749967812540232,
      "grad_norm": 58.40303421020508,
      "learning_rate": 3.846153846153846e-08,
      "logits/chosen": 4.762095425386862,
      "logits/rejected": 5.165317430713165,
      "logps/chosen": -326.04670944213865,
      "logps/rejected": -287.5275459289551,
      "loss": 0.7019648075103759,
      "mean_token_accuracy": 0.7099196255207062,
      "num_tokens": 138906.0,
      "rewards/accuracies": 0.44375,
      "rewards/chosen": -0.014671237254515291,
      "rewards/margins": -0.013322968035936356,
      "rewards/rejected": -0.0013482690206728875,
      "step": 10
    },
    {
      "entropy": 1.051201492547989,
      "epoch": 0.0051499935625080465,
      "grad_norm": 60.10802459716797,
      "learning_rate": 8.119658119658119e-08,
      "logits/chosen": 5.049522462404801,
      "logits/rejected": 5.092235104276472,
      "logps/chosen": -330.7694374084473,
      "logps/rejected": -300.17331314086914,
      "loss": 0.689347505569458,
      "mean_token_accuracy": 0.7070325821638107,
      "num_tokens": 280553.0,
      "rewards/accuracies": 0.575,
      "rewards/chosen": 0.006806808104738593,
      "rewards/margins": 0.011950487177819014,
      "rewards/rejected": -0.005143680749461055,
      "step": 20
    },
    {
      "entropy": 1.111130267381668,
      "epoch": 0.00772499034376207,
      "grad_norm": 64.04685974121094,
      "learning_rate": 1.2393162393162394e-07,
      "logits/chosen": 5.153677836715249,
      "logits/rejected": 5.129135751846538,
      "logps/chosen": -318.63440856933596,
      "logps/rejected": -291.53640899658205,
      "loss": 0.702676248550415,
      "mean_token_accuracy": 0.6977999448776245,
      "num_tokens": 402696.0,
      "rewards/accuracies": 0.51875,
      "rewards/chosen": 0.002496657450683415,
      "rewards/margins": -0.014800068037584425,
      "rewards/rejected": 0.017296723928302525,
      "step": 30
    },
    {
      "entropy": 1.0413395404815673,
      "epoch": 0.010299987125016093,
      "grad_norm": 61.53692626953125,
      "learning_rate": 1.6666666666666665e-07,
      "logits/chosen": 4.89737151113117,
      "logits/rejected": 4.989192161185598,
      "logps/chosen": -317.6526679992676,
      "logps/rejected": -299.5892158508301,
      "loss": 0.6864168167114257,
      "mean_token_accuracy": 0.7229504019021988,
      "num_tokens": 535659.0,
      "rewards/accuracies": 0.54375,
      "rewards/chosen": 0.005982771259732544,
      "rewards/margins": 0.01693534429650754,
      "rewards/rejected": -0.010952573386020958,
      "step": 40
    },
    {
      "entropy": 1.0844607055187225,
      "epoch": 0.012874983906270118,
      "grad_norm": 61.18706512451172,
      "learning_rate": 2.0940170940170939e-07,
      "logits/chosen": 5.037751540173592,
      "logits/rejected": 5.022813626757577,
      "logps/chosen": -335.12218589782714,
      "logps/rejected": -308.31434173583983,
      "loss": 0.6873773574829102,
      "mean_token_accuracy": 0.7006662994623184,
      "num_tokens": 667089.0,
      "rewards/accuracies": 0.51875,
      "rewards/chosen": 0.01746939681470394,
      "rewards/margins": 0.016403451142832638,
      "rewards/rejected": 0.0010659459047019481,
      "step": 50
    },
    {
      "entropy": 1.0144690483808518,
      "epoch": 0.01544998068752414,
      "grad_norm": 75.81221771240234,
      "learning_rate": 2.521367521367521e-07,
      "logits/chosen": 4.896282725285022,
      "logits/rejected": 4.959213862117638,
      "logps/chosen": -303.1107292175293,
      "logps/rejected": -293.32022323608396,
      "loss": 0.6931710720062256,
      "mean_token_accuracy": 0.7159741580486297,
      "num_tokens": 810423.0,
      "rewards/accuracies": 0.4875,
      "rewards/chosen": 0.004395447322167456,
      "rewards/margins": 0.0054800987243652345,
      "rewards/rejected": -0.0010846516583114863,
      "step": 60
    },
    {
      "entropy": 0.9541261970996857,
      "epoch": 0.018024977468778163,
      "grad_norm": 73.47660827636719,
      "learning_rate": 2.948717948717949e-07,
      "logits/chosen": 4.878544510221824,
      "logits/rejected": 5.001038510432101,
      "logps/chosen": -328.04087524414064,
      "logps/rejected": -302.33535919189455,
      "loss": 0.6965915203094483,
      "mean_token_accuracy": 0.7324783325195312,
      "num_tokens": 954004.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003411970962770283,
      "rewards/margins": -0.00023952401243150235,
      "rewards/rejected": 0.0036514950916171074,
      "step": 70
    },
    {
      "entropy": 1.0023272186517715,
      "epoch": 0.020599974250032186,
      "grad_norm": 84.27365112304688,
      "learning_rate": 3.376068376068376e-07,
      "logits/chosen": 4.891619277720404,
      "logits/rejected": 4.941218434899946,
      "logps/chosen": -382.654524230957,
      "logps/rejected": -341.0330238342285,
      "loss": 0.6920312881469727,
      "mean_token_accuracy": 0.7118763566017151,
      "num_tokens": 1108886.0,
      "rewards/accuracies": 0.55625,
      "rewards/chosen": 0.02751660463400185,
      "rewards/margins": 0.008006168599240481,
      "rewards/rejected": 0.01951043652370572,
      "step": 80
    },
    {
      "entropy": 1.0618479609489442,
      "epoch": 0.023174971031286212,
      "grad_norm": 70.65860748291016,
      "learning_rate": 3.803418803418803e-07,
      "logits/chosen": 4.614978686245202,
      "logits/rejected": 4.850742337522037,
      "logps/chosen": -320.1764625549316,
      "logps/rejected": -306.739794921875,
      "loss": 0.6981391906738281,
      "mean_token_accuracy": 0.7010324835777283,
      "num_tokens": 1240538.0,
      "rewards/accuracies": 0.48125,
      "rewards/chosen": 0.02626001196913421,
      "rewards/margins": -0.004135513259097934,
      "rewards/rejected": 0.03039552546106279,
      "step": 90
    },
    {
      "entropy": 1.0384016394615174,
      "epoch": 0.025749967812540235,
      "grad_norm": 59.127098083496094,
      "learning_rate": 4.2307692307692304e-07,
      "logits/chosen": 4.8460533212294745,
      "logits/rejected": 4.824443955007636,
      "logps/chosen": -348.85643463134767,
      "logps/rejected": -312.45501098632815,
      "loss": 0.6569533348083496,
      "mean_token_accuracy": 0.7068787574768066,
      "num_tokens": 1382625.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.08904398875311018,
      "rewards/margins": 0.08204479101113975,
      "rewards/rejected": 0.0069991969503462315,
      "step": 100
    },
    {
      "entropy": 1.0480757176876068,
      "epoch": 0.028324964593794258,
      "grad_norm": 64.17548370361328,
      "learning_rate": 4.658119658119658e-07,
      "logits/chosen": 4.7895138574416185,
      "logits/rejected": 4.869067111731783,
      "logps/chosen": -348.2337989807129,
      "logps/rejected": -308.14727478027345,
      "loss": 0.6725531578063965,
      "mean_token_accuracy": 0.7132146149873734,
      "num_tokens": 1514173.0,
      "rewards/accuracies": 0.6,
      "rewards/chosen": 0.12068244460970164,
      "rewards/margins": 0.05134738911874592,
      "rewards/rejected": 0.06933505749329924,
      "step": 110
    },
    {
      "entropy": 1.0579831272363662,
      "epoch": 0.03089996137504828,
      "grad_norm": 74.17851257324219,
      "learning_rate": 4.999996522409758e-07,
      "logits/chosen": 4.79368538551119,
      "logits/rejected": 4.86757195450835,
      "logps/chosen": -369.29692764282225,
      "logps/rejected": -331.0557464599609,
      "loss": 0.6710665702819825,
      "mean_token_accuracy": 0.7087177097797394,
      "num_tokens": 1660343.0,
      "rewards/accuracies": 0.6375,
      "rewards/chosen": 0.1534318385645747,
      "rewards/margins": 0.06275590499863029,
      "rewards/rejected": 0.09067593847867102,
      "step": 120
    },
    {
      "entropy": 1.0356961339712143,
      "epoch": 0.033474958156302304,
      "grad_norm": 59.33082580566406,
      "learning_rate": 4.999874807767132e-07,
      "logits/chosen": 4.807214629060647,
      "logits/rejected": 4.723712813209219,
      "logps/chosen": -326.1460441589355,
      "logps/rejected": -299.46656799316406,
      "loss": 0.6591967582702637,
      "mean_token_accuracy": 0.7120212823152542,
      "num_tokens": 1802352.0,
      "rewards/accuracies": 0.6,
      "rewards/chosen": 0.16570830587297677,
      "rewards/margins": 0.08986456668935716,
      "rewards/rejected": 0.07584373848512768,
      "step": 130
    },
    {
      "entropy": 1.0639059126377106,
      "epoch": 0.036049954937556326,
      "grad_norm": 46.880950927734375,
      "learning_rate": 4.999579223287168e-07,
      "logits/chosen": 4.783742518251371,
      "logits/rejected": 4.773145675630588,
      "logps/chosen": -344.53967132568357,
      "logps/rejected": -279.5810417175293,
      "loss": 0.6366758346557617,
      "mean_token_accuracy": 0.7052115440368653,
      "num_tokens": 1935694.0,
      "rewards/accuracies": 0.66875,
      "rewards/chosen": 0.20478311218321324,
      "rewards/margins": 0.13913744706660508,
      "rewards/rejected": 0.06564566306769848,
      "step": 140
    },
    {
      "entropy": 1.019757255911827,
      "epoch": 0.03862495171881035,
      "grad_norm": 61.628849029541016,
      "learning_rate": 4.999109789528186e-07,
      "logits/chosen": 4.742164159736975,
      "logits/rejected": 4.946638459164037,
      "logps/chosen": -329.5531692504883,
      "logps/rejected": -289.46395416259764,
      "loss": 0.6572101593017579,
      "mean_token_accuracy": 0.7132875680923462,
      "num_tokens": 2071315.0,
      "rewards/accuracies": 0.61875,
      "rewards/chosen": 0.21405786070972682,
      "rewards/margins": 0.10857852287590504,
      "rewards/rejected": 0.1054793362505734,
      "step": 150
    },
    {
      "entropy": 1.0863682091236115,
      "epoch": 0.04119994850006437,
      "grad_norm": 52.7701301574707,
      "learning_rate": 4.998466539139966e-07,
      "logits/chosen": 4.762784820500906,
      "logits/rejected": 4.796325654968387,
      "logps/chosen": -273.65675354003906,
      "logps/rejected": -278.2464324951172,
      "loss": 0.6301429271697998,
      "mean_token_accuracy": 0.7196185618638993,
      "num_tokens": 2205535.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": 0.22666821889579297,
      "rewards/margins": 0.16818621791899205,
      "rewards/rejected": 0.058482004422694446,
      "step": 160
    },
    {
      "entropy": 1.028375071287155,
      "epoch": 0.0437749452813184,
      "grad_norm": 56.66053009033203,
      "learning_rate": 4.997649516861489e-07,
      "logits/chosen": 4.541193592999439,
      "logits/rejected": 4.769671591019123,
      "logps/chosen": -325.7829078674316,
      "logps/rejected": -333.5553314208984,
      "loss": 0.6510717391967773,
      "mean_token_accuracy": 0.7231594532728195,
      "num_tokens": 2345876.0,
      "rewards/accuracies": 0.63125,
      "rewards/chosen": 0.23217825312167406,
      "rewards/margins": 0.13302348610013723,
      "rewards/rejected": 0.09915476976893842,
      "step": 170
    },
    {
      "entropy": 1.0339969098567963,
      "epoch": 0.046349942062572425,
      "grad_norm": 55.01676559448242,
      "learning_rate": 4.996658779517809e-07,
      "logits/chosen": 4.670579991176632,
      "logits/rejected": 4.806632291205961,
      "logps/chosen": -300.85853881835936,
      "logps/rejected": -313.85018997192384,
      "loss": 0.6131317138671875,
      "mean_token_accuracy": 0.7170714914798737,
      "num_tokens": 2486861.0,
      "rewards/accuracies": 0.7,
      "rewards/chosen": 0.2972709834575653,
      "rewards/margins": 0.24978987369686365,
      "rewards/rejected": 0.04748110845685005,
      "step": 180
    },
    {
      "entropy": 1.0799639374017715,
      "epoch": 0.04892493884382645,
      "grad_norm": 47.156986236572266,
      "learning_rate": 4.995494396016116e-07,
      "logits/chosen": 4.792294079993604,
      "logits/rejected": 4.859750700385524,
      "logps/chosen": -313.53778610229494,
      "logps/rejected": -324.3677551269531,
      "loss": 0.6257365703582763,
      "mean_token_accuracy": 0.7067189812660217,
      "num_tokens": 2614373.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.29387753307819364,
      "rewards/margins": 0.20768443727865815,
      "rewards/rejected": 0.08619309198111295,
      "step": 190
    },
    {
      "entropy": 1.0718048125505448,
      "epoch": 0.05149993562508047,
      "grad_norm": 59.598426818847656,
      "learning_rate": 4.994156447340932e-07,
      "logits/chosen": 4.765590034251408,
      "logits/rejected": 4.673178326492318,
      "logps/chosen": -296.8489875793457,
      "logps/rejected": -295.27930374145507,
      "loss": 0.6396026611328125,
      "mean_token_accuracy": 0.7119928777217865,
      "num_tokens": 2740292.0,
      "rewards/accuracies": 0.64375,
      "rewards/chosen": 0.24728852249681949,
      "rewards/margins": 0.18662491645663976,
      "rewards/rejected": 0.06066360287368298,
      "step": 200
    },
    {
      "entropy": 1.123349404335022,
      "epoch": 0.05407493240633449,
      "grad_norm": 77.97437286376953,
      "learning_rate": 4.992645026548485e-07,
      "logits/chosen": 4.384905356176568,
      "logits/rejected": 4.5785579199739725,
      "logps/chosen": -337.41533813476565,
      "logps/rejected": -335.0264404296875,
      "loss": 0.5705361843109131,
      "mean_token_accuracy": 0.6963643610477448,
      "num_tokens": 2867334.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": 0.471590805798769,
      "rewards/margins": 0.3648594722151756,
      "rewards/rejected": 0.10673133693635464,
      "step": 210
    },
    {
      "entropy": 1.1385729312896729,
      "epoch": 0.056649929187588516,
      "grad_norm": 51.946537017822266,
      "learning_rate": 4.990960238760234e-07,
      "logits/chosen": 4.721716537520671,
      "logits/rejected": 4.601918369467295,
      "logps/chosen": -351.25513687133787,
      "logps/rejected": -302.89658813476564,
      "loss": 0.5859468460083008,
      "mean_token_accuracy": 0.6919317305088043,
      "num_tokens": 2993069.0,
      "rewards/accuracies": 0.675,
      "rewards/chosen": 0.39948597028851507,
      "rewards/margins": 0.37867568135261537,
      "rewards/rejected": 0.020810306444764136,
      "step": 220
    },
    {
      "entropy": 1.0420648545026778,
      "epoch": 0.05922492596884254,
      "grad_norm": 63.54645538330078,
      "learning_rate": 4.989102201155557e-07,
      "logits/chosen": 4.890828809020718,
      "logits/rejected": 4.79452456459243,
      "logps/chosen": -343.53161773681643,
      "logps/rejected": -325.0349151611328,
      "loss": 0.6073165893554687,
      "mean_token_accuracy": 0.7045987546443939,
      "num_tokens": 3141079.0,
      "rewards/accuracies": 0.68125,
      "rewards/chosen": 0.4077755454927683,
      "rewards/margins": 0.32392252162098883,
      "rewards/rejected": 0.08385302387177944,
      "step": 230
    },
    {
      "entropy": 1.0679937094449996,
      "epoch": 0.06179992275009656,
      "grad_norm": 52.09357833862305,
      "learning_rate": 4.987071042963607e-07,
      "logits/chosen": 4.418793507089909,
      "logits/rejected": 4.522803089162048,
      "logps/chosen": -355.0448799133301,
      "logps/rejected": -339.4919242858887,
      "loss": 0.5773736953735351,
      "mean_token_accuracy": 0.704371201992035,
      "num_tokens": 3286962.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.545511468499899,
      "rewards/margins": 0.40970403887331486,
      "rewards/rejected": 0.13580743744969367,
      "step": 240
    },
    {
      "entropy": 0.9754935801029205,
      "epoch": 0.06437491953135059,
      "grad_norm": 55.18085861206055,
      "learning_rate": 4.984866905454313e-07,
      "logits/chosen": 4.734972345059665,
      "logits/rejected": 4.7931226182846345,
      "logps/chosen": -290.1737594604492,
      "logps/rejected": -316.17198638916017,
      "loss": 0.6571999073028565,
      "mean_token_accuracy": 0.728389224410057,
      "num_tokens": 3426548.0,
      "rewards/accuracies": 0.58125,
      "rewards/chosen": 0.4490304827690125,
      "rewards/margins": 0.2581039540469646,
      "rewards/rejected": 0.19092654064297676,
      "step": 250
    },
    {
      "entropy": 1.0574446082115174,
      "epoch": 0.06694991631260461,
      "grad_norm": 45.543148040771484,
      "learning_rate": 4.982489941928565e-07,
      "logits/chosen": 4.519881946840187,
      "logits/rejected": 4.961670236968278,
      "logps/chosen": -318.65669174194335,
      "logps/rejected": -312.55185317993164,
      "loss": 0.5793586730957031,
      "mean_token_accuracy": 0.7170014798641204,
      "num_tokens": 3571612.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.45276781246066095,
      "rewards/margins": 0.4238900698721409,
      "rewards/rejected": 0.028877736534923315,
      "step": 260
    },
    {
      "entropy": 1.0360048443078995,
      "epoch": 0.06952491309385864,
      "grad_norm": 71.59954071044922,
      "learning_rate": 4.979940317707547e-07,
      "logits/chosen": 4.615665879567825,
      "logits/rejected": 4.5951095572387946,
      "logps/chosen": -347.95579528808594,
      "logps/rejected": -298.5147361755371,
      "loss": 0.5869906425476075,
      "mean_token_accuracy": 0.695509958267212,
      "num_tokens": 3707243.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": 0.44863925948739053,
      "rewards/margins": 0.3824565585702658,
      "rewards/rejected": 0.06618269477039576,
      "step": 270
    },
    {
      "entropy": 1.0836758971214295,
      "epoch": 0.07209990987511265,
      "grad_norm": 53.48826599121094,
      "learning_rate": 4.977218210121237e-07,
      "logits/chosen": 4.381913341564861,
      "logits/rejected": 4.4011498928625565,
      "logps/chosen": -347.5009132385254,
      "logps/rejected": -301.47351303100584,
      "loss": 0.5558026790618896,
      "mean_token_accuracy": 0.6883328318595886,
      "num_tokens": 3836639.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.5033460140228272,
      "rewards/margins": 0.45913518890738486,
      "rewards/rejected": 0.044210815243422984,
      "step": 280
    },
    {
      "entropy": 1.019636595249176,
      "epoch": 0.07467490665636668,
      "grad_norm": 58.595001220703125,
      "learning_rate": 4.974323808496079e-07,
      "logits/chosen": 4.617471123727146,
      "logits/rejected": 4.759611293000644,
      "logps/chosen": -324.77361526489256,
      "logps/rejected": -318.6325256347656,
      "loss": 0.5850511074066163,
      "mean_token_accuracy": 0.7116295546293259,
      "num_tokens": 3967890.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.377638790756464,
      "rewards/margins": 0.4224536880850792,
      "rewards/rejected": -0.044814901798963545,
      "step": 290
    },
    {
      "entropy": 1.0210690379142762,
      "epoch": 0.0772499034376207,
      "grad_norm": 51.6544189453125,
      "learning_rate": 4.971257314141808e-07,
      "logits/chosen": 4.724067558444576,
      "logits/rejected": 4.807996827326434,
      "logps/chosen": -347.1739959716797,
      "logps/rejected": -319.8342315673828,
      "loss": 0.605327033996582,
      "mean_token_accuracy": 0.7064412742853164,
      "num_tokens": 4109051.0,
      "rewards/accuracies": 0.68125,
      "rewards/chosen": 0.37119412459433077,
      "rewards/margins": 0.3749303735792637,
      "rewards/rejected": -0.003736240416765213,
      "step": 300
    },
    {
      "entropy": 1.0039023548364638,
      "epoch": 0.07982490021887473,
      "grad_norm": 101.59368896484375,
      "learning_rate": 4.968018940337457e-07,
      "logits/chosen": 4.615459057356064,
      "logits/rejected": 4.763312215236125,
      "logps/chosen": -324.7437110900879,
      "logps/rejected": -302.2521469116211,
      "loss": 0.6154751777648926,
      "mean_token_accuracy": 0.7176793605089188,
      "num_tokens": 4247174.0,
      "rewards/accuracies": 0.65,
      "rewards/chosen": 0.4162539541721344,
      "rewards/margins": 0.39256238043308256,
      "rewards/rejected": 0.02369157038629055,
      "step": 310
    },
    {
      "entropy": 1.0267376393079757,
      "epoch": 0.08239989700012874,
      "grad_norm": 62.431922912597656,
      "learning_rate": 4.964608912316513e-07,
      "logits/chosen": 4.860332914675701,
      "logits/rejected": 4.91242366277054,
      "logps/chosen": -331.2693698883057,
      "logps/rejected": -319.29067611694336,
      "loss": 0.6155694961547852,
      "mean_token_accuracy": 0.7094589233398437,
      "num_tokens": 4384787.0,
      "rewards/accuracies": 0.66875,
      "rewards/chosen": 0.5132191546261311,
      "rewards/margins": 0.36463063918054106,
      "rewards/rejected": 0.14858851991593838,
      "step": 320
    },
    {
      "entropy": 0.9868817120790482,
      "epoch": 0.08497489378138277,
      "grad_norm": 61.427738189697266,
      "learning_rate": 4.961027467251261e-07,
      "logits/chosen": 4.638749258286775,
      "logits/rejected": 4.5989357091926495,
      "logps/chosen": -318.4152290344238,
      "logps/rejected": -305.52441024780273,
      "loss": 0.604922866821289,
      "mean_token_accuracy": 0.7185680598020554,
      "num_tokens": 4523458.0,
      "rewards/accuracies": 0.68125,
      "rewards/chosen": 0.5514455199241638,
      "rewards/margins": 0.40419873856008054,
      "rewards/rejected": 0.14724678825587034,
      "step": 330
    },
    {
      "entropy": 1.0020043164491654,
      "epoch": 0.0875498905626368,
      "grad_norm": 52.15379333496094,
      "learning_rate": 4.957274854236283e-07,
      "logits/chosen": 4.362704721137801,
      "logits/rejected": 4.4858774130028864,
      "logps/chosen": -298.45718536376955,
      "logps/rejected": -275.92749252319334,
      "loss": 0.6286430358886719,
      "mean_token_accuracy": 0.7173390120267868,
      "num_tokens": 4655707.0,
      "rewards/accuracies": 0.6125,
      "rewards/chosen": 0.5156877748668194,
      "rewards/margins": 0.3501556381583214,
      "rewards/rejected": 0.16553213112056256,
      "step": 340
    },
    {
      "entropy": 0.9908010929822921,
      "epoch": 0.09012488734389082,
      "grad_norm": 49.259765625,
      "learning_rate": 4.953351334271134e-07,
      "logits/chosen": 4.406444560691517,
      "logits/rejected": 4.548902690320075,
      "logps/chosen": -296.87431106567385,
      "logps/rejected": -289.5457908630371,
      "loss": 0.5424035072326661,
      "mean_token_accuracy": 0.7123640716075897,
      "num_tokens": 4790294.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.5531937591731548,
      "rewards/margins": 0.5116788432002067,
      "rewards/rejected": 0.04151492342352867,
      "step": 350
    },
    {
      "entropy": 1.0101291090250015,
      "epoch": 0.09269988412514485,
      "grad_norm": 59.696929931640625,
      "learning_rate": 4.949257180242192e-07,
      "logits/chosen": 4.385342873631919,
      "logits/rejected": 4.272401614914889,
      "logps/chosen": -329.9344657897949,
      "logps/rejected": -303.8744956970215,
      "loss": 0.5772665023803711,
      "mean_token_accuracy": 0.7187704712152481,
      "num_tokens": 4928018.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": 0.44092832542955873,
      "rewards/margins": 0.43197181150317193,
      "rewards/rejected": 0.008956506475806236,
      "step": 360
    },
    {
      "entropy": 1.016788786649704,
      "epoch": 0.09527488090639886,
      "grad_norm": 50.9133415222168,
      "learning_rate": 4.944992676903674e-07,
      "logits/chosen": 4.506991812732053,
      "logits/rejected": 4.551043911449192,
      "logps/chosen": -341.2626007080078,
      "logps/rejected": -321.827197265625,
      "loss": 0.5512761116027832,
      "mean_token_accuracy": 0.713101840019226,
      "num_tokens": 5081614.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.4262397438287735,
      "rewards/margins": 0.5591157540678978,
      "rewards/rejected": -0.13287600912153721,
      "step": 370
    },
    {
      "entropy": 0.985381743311882,
      "epoch": 0.0978498776876529,
      "grad_norm": 101.32551574707031,
      "learning_rate": 4.940558120857834e-07,
      "logits/chosen": 4.556351895973922,
      "logits/rejected": 4.600648669219739,
      "logps/chosen": -338.7858177185059,
      "logps/rejected": -305.161669921875,
      "loss": 0.5784684181213379,
      "mean_token_accuracy": 0.7156250983476639,
      "num_tokens": 5222452.0,
      "rewards/accuracies": 0.7,
      "rewards/chosen": 0.43173090666532515,
      "rewards/margins": 0.4866475585848093,
      "rewards/rejected": -0.05491665340960026,
      "step": 380
    },
    {
      "entropy": 1.0119995504617691,
      "epoch": 0.10042487446890691,
      "grad_norm": 66.73582458496094,
      "learning_rate": 4.935953820534332e-07,
      "logits/chosen": 4.700755701463852,
      "logits/rejected": 4.563956382940839,
      "logps/chosen": -350.07495574951173,
      "logps/rejected": -295.9588035583496,
      "loss": 0.5613166809082031,
      "mean_token_accuracy": 0.7058805853128434,
      "num_tokens": 5361783.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.432503791898489,
      "rewards/margins": 0.5063534714281559,
      "rewards/rejected": -0.07384968474507332,
      "step": 390
    },
    {
      "entropy": 0.9758326828479766,
      "epoch": 0.10299987125016094,
      "grad_norm": 44.98636245727539,
      "learning_rate": 4.931180096168784e-07,
      "logits/chosen": 4.683912871421698,
      "logits/rejected": 4.797076860659791,
      "logps/chosen": -325.4021278381348,
      "logps/rejected": -319.83669128417966,
      "loss": 0.5478048324584961,
      "mean_token_accuracy": 0.7172370463609695,
      "num_tokens": 5498760.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": 0.3713498543947935,
      "rewards/margins": 0.5634564943611622,
      "rewards/rejected": -0.1921066589653492,
      "step": 400
    },
    {
      "entropy": 0.9767284691333771,
      "epoch": 0.10557486803141496,
      "grad_norm": 47.32701110839844,
      "learning_rate": 4.926237279780488e-07,
      "logits/chosen": 4.4485483092319855,
      "logits/rejected": 4.88916482277722,
      "logps/chosen": -312.9255798339844,
      "logps/rejected": -301.4211082458496,
      "loss": 0.5510597229003906,
      "mean_token_accuracy": 0.7192813426256179,
      "num_tokens": 5638362.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.4443170979619026,
      "rewards/margins": 0.5508027624338865,
      "rewards/rejected": -0.1064856756478548,
      "step": 410
    },
    {
      "entropy": 1.038025939464569,
      "epoch": 0.10814986481266899,
      "grad_norm": 59.02484893798828,
      "learning_rate": 4.921125715149331e-07,
      "logits/chosen": 4.492031042311866,
      "logits/rejected": 4.7620465682604465,
      "logps/chosen": -304.71643142700196,
      "logps/rejected": -302.98192901611327,
      "loss": 0.5734364509582519,
      "mean_token_accuracy": 0.7094101399183274,
      "num_tokens": 5774772.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": 0.575700556486845,
      "rewards/margins": 0.6073308132588864,
      "rewards/rejected": -0.031630271673202516,
      "step": 420
    },
    {
      "entropy": 0.9706959962844849,
      "epoch": 0.110724861593923,
      "grad_norm": 50.152259826660156,
      "learning_rate": 4.915845757791882e-07,
      "logits/chosen": 4.301352728074184,
      "logits/rejected": 4.374018651463462,
      "logps/chosen": -309.12598037719727,
      "logps/rejected": -329.5872093200684,
      "loss": 0.5827741622924805,
      "mean_token_accuracy": 0.7238694995641708,
      "num_tokens": 5915818.0,
      "rewards/accuracies": 0.66875,
      "rewards/chosen": 0.3628706641495228,
      "rewards/margins": 0.526591594517231,
      "rewards/rejected": -0.16372093437239527,
      "step": 430
    },
    {
      "entropy": 0.9883161246776581,
      "epoch": 0.11329985837517703,
      "grad_norm": 63.66497802734375,
      "learning_rate": 4.910397774936662e-07,
      "logits/chosen": 4.324055190943431,
      "logits/rejected": 4.455768599008789,
      "logps/chosen": -339.16539764404297,
      "logps/rejected": -317.9355972290039,
      "loss": 0.60983304977417,
      "mean_token_accuracy": 0.7213237226009369,
      "num_tokens": 6057267.0,
      "rewards/accuracies": 0.63125,
      "rewards/chosen": 0.3570538476109505,
      "rewards/margins": 0.4920197740197182,
      "rewards/rejected": -0.1349659183062613,
      "step": 440
    },
    {
      "entropy": 1.0575228780508041,
      "epoch": 0.11587485515643106,
      "grad_norm": 77.4422378540039,
      "learning_rate": 4.904782145498601e-07,
      "logits/chosen": 4.483779070580654,
      "logits/rejected": 4.545815018757674,
      "logps/chosen": -311.60619049072267,
      "logps/rejected": -317.6271324157715,
      "loss": 0.5732481479644775,
      "mean_token_accuracy": 0.7105006694793701,
      "num_tokens": 6185910.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": 0.271255973726511,
      "rewards/margins": 0.5383441850543023,
      "rewards/rejected": -0.2670882109552622,
      "step": 450
    },
    {
      "entropy": 1.0017959356307984,
      "epoch": 0.11844985193768508,
      "grad_norm": 52.255638122558594,
      "learning_rate": 4.89899926005269e-07,
      "logits/chosen": 4.416739378206386,
      "logits/rejected": 4.47387991202081,
      "logps/chosen": -306.1744514465332,
      "logps/rejected": -295.7477088928223,
      "loss": 0.5897578239440918,
      "mean_token_accuracy": 0.7148655951023102,
      "num_tokens": 6315045.0,
      "rewards/accuracies": 0.66875,
      "rewards/chosen": 0.1683360619470477,
      "rewards/margins": 0.4851381786167622,
      "rewards/rejected": -0.31680211387574675,
      "step": 460
    },
    {
      "entropy": 0.9547203719615937,
      "epoch": 0.12102484871893911,
      "grad_norm": 65.27824401855469,
      "learning_rate": 4.893049520806809e-07,
      "logits/chosen": 4.526039482298581,
      "logits/rejected": 4.724331283409848,
      "logps/chosen": -318.0591064453125,
      "logps/rejected": -308.0281555175781,
      "loss": 0.5524198532104492,
      "mean_token_accuracy": 0.7198063462972641,
      "num_tokens": 6448792.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.2623703435063362,
      "rewards/margins": 0.555498230457306,
      "rewards/rejected": -0.29312787503004073,
      "step": 470
    },
    {
      "entropy": 0.9980389684438705,
      "epoch": 0.12359984550019312,
      "grad_norm": 41.32709884643555,
      "learning_rate": 4.886933341573762e-07,
      "logits/chosen": 4.443947690470212,
      "logits/rejected": 4.704294055958519,
      "logps/chosen": -322.1322135925293,
      "logps/rejected": -324.39286651611326,
      "loss": 0.5217737197875977,
      "mean_token_accuracy": 0.7148303747177124,
      "num_tokens": 6582968.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.1742788452655077,
      "rewards/margins": 0.75794066041708,
      "rewards/rejected": -0.5836618229746818,
      "step": 480
    },
    {
      "entropy": 0.9905156761407852,
      "epoch": 0.12617484228144715,
      "grad_norm": 63.09601974487305,
      "learning_rate": 4.880651147742484e-07,
      "logits/chosen": 4.403596984868758,
      "logits/rejected": 4.697439525919373,
      "logps/chosen": -305.22482986450194,
      "logps/rejected": -354.5737548828125,
      "loss": 0.5686468124389649,
      "mean_token_accuracy": 0.7282920807600022,
      "num_tokens": 6712612.0,
      "rewards/accuracies": 0.7,
      "rewards/chosen": 0.2129853243008256,
      "rewards/margins": 0.6234859123826026,
      "rewards/rejected": -0.4105006046593189,
      "step": 490
    },
    {
      "entropy": 0.9388987690210342,
      "epoch": 0.12874983906270118,
      "grad_norm": 42.92222595214844,
      "learning_rate": 4.874203376248463e-07,
      "logits/chosen": 4.488846695070381,
      "logits/rejected": 4.778856944495108,
      "logps/chosen": -290.40800247192385,
      "logps/rejected": -312.62304458618166,
      "loss": 0.518910264968872,
      "mean_token_accuracy": 0.726474717259407,
      "num_tokens": 6847702.0,
      "rewards/accuracies": 0.76875,
      "rewards/chosen": 0.246230723336339,
      "rewards/margins": 0.7769725546240807,
      "rewards/rejected": -0.5307418283075094,
      "step": 500
    },
    {
      "entropy": 0.9601681470870972,
      "epoch": 0.13132483584395518,
      "grad_norm": 39.82841110229492,
      "learning_rate": 4.86759047554335e-07,
      "logits/chosen": 4.2579053387255765,
      "logits/rejected": 4.4132857090626825,
      "logps/chosen": -317.2354568481445,
      "logps/rejected": -294.6973342895508,
      "loss": 0.5153061866760253,
      "mean_token_accuracy": 0.7181984424591065,
      "num_tokens": 6984334.0,
      "rewards/accuracies": 0.76875,
      "rewards/chosen": 0.34739205725491046,
      "rewards/margins": 0.7444691531360149,
      "rewards/rejected": -0.39707708433270456,
      "step": 510
    },
    {
      "entropy": 0.9526659429073334,
      "epoch": 0.13389983262520921,
      "grad_norm": 39.31724548339844,
      "learning_rate": 4.860812905563763e-07,
      "logits/chosen": 4.304689935360902,
      "logits/rejected": 4.547928502321976,
      "logps/chosen": -318.8589202880859,
      "logps/rejected": -332.00280685424804,
      "loss": 0.47237606048583985,
      "mean_token_accuracy": 0.727465134859085,
      "num_tokens": 7138124.0,
      "rewards/accuracies": 0.7875,
      "rewards/chosen": 0.18707619085907937,
      "rewards/margins": 0.8195438928902149,
      "rewards/rejected": -0.6324677051045,
      "step": 520
    },
    {
      "entropy": 0.9723318427801132,
      "epoch": 0.13647482940646324,
      "grad_norm": 56.447967529296875,
      "learning_rate": 4.853871137699307e-07,
      "logits/chosen": 4.589381324158332,
      "logits/rejected": 4.651859008699785,
      "logps/chosen": -347.7586387634277,
      "logps/rejected": -314.0169731140137,
      "loss": 0.5345488548278808,
      "mean_token_accuracy": 0.7160911679267883,
      "num_tokens": 7271610.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": 0.03413632959127426,
      "rewards/margins": 0.7016147773712873,
      "rewards/rejected": -0.6674784570932388,
      "step": 530
    },
    {
      "entropy": 0.9283442407846451,
      "epoch": 0.13904982618771727,
      "grad_norm": 81.77369689941406,
      "learning_rate": 4.846765654759777e-07,
      "logits/chosen": 4.534017620164606,
      "logits/rejected": 4.336357329893327,
      "logps/chosen": -333.11379013061526,
      "logps/rejected": -350.09332885742185,
      "loss": 0.6328543663024903,
      "mean_token_accuracy": 0.7222778290510178,
      "num_tokens": 7418090.0,
      "rewards/accuracies": 0.6375,
      "rewards/chosen": -0.07262884676456452,
      "rewards/margins": 0.5407668419182301,
      "rewards/rejected": -0.6133956845849753,
      "step": 540
    },
    {
      "entropy": 0.9569640517234802,
      "epoch": 0.14162482296897128,
      "grad_norm": 60.66325759887695,
      "learning_rate": 4.839496950941585e-07,
      "logits/chosen": 4.465449477689448,
      "logits/rejected": 4.338006664381983,
      "logps/chosen": -324.85693740844727,
      "logps/rejected": -300.08666076660154,
      "loss": 0.5704984664916992,
      "mean_token_accuracy": 0.7159777522087097,
      "num_tokens": 7551340.0,
      "rewards/accuracies": 0.66875,
      "rewards/chosen": 0.20383757278323172,
      "rewards/margins": 0.6437537096440792,
      "rewards/rejected": -0.43991615511476995,
      "step": 550
    },
    {
      "entropy": 0.9533803671598434,
      "epoch": 0.1441998197502253,
      "grad_norm": 73.0498046875,
      "learning_rate": 4.832065531793389e-07,
      "logits/chosen": 4.39996322075107,
      "logits/rejected": 4.554421709155159,
      "logps/chosen": -300.69481964111327,
      "logps/rejected": -292.7109878540039,
      "loss": 0.5687302589416504,
      "mean_token_accuracy": 0.7131290882825851,
      "num_tokens": 7683673.0,
      "rewards/accuracies": 0.68125,
      "rewards/chosen": 0.1011431023478508,
      "rewards/margins": 0.6272410407662392,
      "rewards/rejected": -0.5260979343205691,
      "step": 560
    },
    {
      "entropy": 0.9646992802619934,
      "epoch": 0.14677481653147934,
      "grad_norm": 54.05545425415039,
      "learning_rate": 4.824471914180928e-07,
      "logits/chosen": 4.462153498340667,
      "logits/rejected": 4.351227799524455,
      "logps/chosen": -334.3186973571777,
      "logps/rejected": -304.96626739501954,
      "loss": 0.6647938251495361,
      "mean_token_accuracy": 0.7061104267835617,
      "num_tokens": 7812237.0,
      "rewards/accuracies": 0.68125,
      "rewards/chosen": -0.020775986835360527,
      "rewards/margins": 0.530225483328104,
      "rewards/rejected": -0.5510014675557613,
      "step": 570
    },
    {
      "entropy": 1.0479345887899398,
      "epoch": 0.14934981331273336,
      "grad_norm": 37.81373977661133,
      "learning_rate": 4.816716626251072e-07,
      "logits/chosen": 4.184722720620551,
      "logits/rejected": 4.135806228787347,
      "logps/chosen": -335.03366928100587,
      "logps/rejected": -359.79450912475585,
      "loss": 0.572507095336914,
      "mean_token_accuracy": 0.6949407935142518,
      "num_tokens": 7949638.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.14069565683603286,
      "rewards/margins": 0.6246777504682541,
      "rewards/rejected": -0.4839820809662342,
      "step": 580
    },
    {
      "entropy": 1.011368888616562,
      "epoch": 0.1519248100939874,
      "grad_norm": 44.93890380859375,
      "learning_rate": 4.808800207395095e-07,
      "logits/chosen": 4.431858594614932,
      "logits/rejected": 4.494303066742516,
      "logps/chosen": -359.7134948730469,
      "logps/rejected": -331.4195068359375,
      "loss": 0.5715437412261963,
      "mean_token_accuracy": 0.7035282164812088,
      "num_tokens": 8086979.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": 0.1286509022116661,
      "rewards/margins": 0.6396637663245202,
      "rewards/rejected": -0.5110128573141992,
      "step": 590
    },
    {
      "entropy": 1.0390411615371704,
      "epoch": 0.1544998068752414,
      "grad_norm": 41.1849250793457,
      "learning_rate": 4.80072320821115e-07,
      "logits/chosen": 4.509100731375415,
      "logits/rejected": 4.498800666009457,
      "logps/chosen": -343.8060928344727,
      "logps/rejected": -301.508959197998,
      "loss": 0.5278671741485595,
      "mean_token_accuracy": 0.6904387384653091,
      "num_tokens": 8215969.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.2609533634036779,
      "rewards/margins": 0.7143490366637707,
      "rewards/rejected": -0.4533956632018089,
      "step": 600
    },
    {
      "entropy": 1.0100758999586106,
      "epoch": 0.15707480365649543,
      "grad_norm": 45.980743408203125,
      "learning_rate": 4.792486190465983e-07,
      "logits/chosen": 4.436240458904658,
      "logits/rejected": 4.483234245575103,
      "logps/chosen": -351.39085540771487,
      "logps/rejected": -322.53857345581054,
      "loss": 0.5409737586975097,
      "mean_token_accuracy": 0.7018147617578506,
      "num_tokens": 8353478.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1759142629802227,
      "rewards/margins": 0.6746158372610808,
      "rewards/rejected": -0.49870156645774844,
      "step": 610
    },
    {
      "entropy": 1.0743782013654708,
      "epoch": 0.15964980043774946,
      "grad_norm": 73.07006072998047,
      "learning_rate": 4.784089727055858e-07,
      "logits/chosen": 4.279494154259277,
      "logits/rejected": 4.438133950466845,
      "logps/chosen": -329.93651657104493,
      "logps/rejected": -338.3123947143555,
      "loss": 0.5927572250366211,
      "mean_token_accuracy": 0.6971615046262741,
      "num_tokens": 8494676.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.1258799023926258,
      "rewards/margins": 0.5927445106208324,
      "rewards/rejected": -0.466864612698555,
      "step": 620
    },
    {
      "entropy": 1.0129943102598191,
      "epoch": 0.16222479721900349,
      "grad_norm": 57.14754867553711,
      "learning_rate": 4.77553440196671e-07,
      "logits/chosen": 4.545317591931587,
      "logits/rejected": 4.609596492216733,
      "logps/chosen": -314.0488555908203,
      "logps/rejected": -280.8207015991211,
      "loss": 0.5136913299560547,
      "mean_token_accuracy": 0.7004855394363403,
      "num_tokens": 8624139.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.26730198562145235,
      "rewards/margins": 0.799684862792492,
      "rewards/rejected": -0.532382888533175,
      "step": 630
    },
    {
      "entropy": 0.9850052386522293,
      "epoch": 0.1647997940002575,
      "grad_norm": 58.24784469604492,
      "learning_rate": 4.766820810233528e-07,
      "logits/chosen": 4.495559711713429,
      "logits/rejected": 4.5218358684585676,
      "logps/chosen": -324.5868515014648,
      "logps/rejected": -319.23336639404295,
      "loss": 0.5680853843688964,
      "mean_token_accuracy": 0.7023002177476882,
      "num_tokens": 8768480.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.1788150530308485,
      "rewards/margins": 0.5665261905640364,
      "rewards/rejected": -0.3877111442387104,
      "step": 640
    },
    {
      "entropy": 0.8995870023965835,
      "epoch": 0.16737479078151152,
      "grad_norm": 56.94486999511719,
      "learning_rate": 4.757949557898972e-07,
      "logits/chosen": 4.750891546224381,
      "logits/rejected": 4.5728916398525445,
      "logps/chosen": -327.8912872314453,
      "logps/rejected": -321.7278022766113,
      "loss": 0.57937593460083,
      "mean_token_accuracy": 0.7331031769514084,
      "num_tokens": 8916927.0,
      "rewards/accuracies": 0.675,
      "rewards/chosen": 0.29649644866585734,
      "rewards/margins": 0.7233392115682363,
      "rewards/rejected": -0.42684278152883054,
      "step": 650
    },
    {
      "entropy": 0.963553175330162,
      "epoch": 0.16994978756276555,
      "grad_norm": 67.72953796386719,
      "learning_rate": 4.7489212619712206e-07,
      "logits/chosen": 4.477993553359102,
      "logits/rejected": 4.419784017374876,
      "logps/chosen": -332.0310920715332,
      "logps/rejected": -299.7566711425781,
      "loss": 0.5437621116638184,
      "mean_token_accuracy": 0.7131728023290634,
      "num_tokens": 9059558.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.2984531581401825,
      "rewards/margins": 0.7288076810538768,
      "rewards/rejected": -0.43035454135388135,
      "step": 660
    },
    {
      "entropy": 0.9489945232868194,
      "epoch": 0.17252478434401958,
      "grad_norm": 41.158843994140625,
      "learning_rate": 4.739736550381056e-07,
      "logits/chosen": 4.682270884264778,
      "logits/rejected": 4.493197512040618,
      "logps/chosen": -304.6476936340332,
      "logps/rejected": -325.6847457885742,
      "loss": 0.534097957611084,
      "mean_token_accuracy": 0.7255629748106003,
      "num_tokens": 9197633.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.28304303996264935,
      "rewards/margins": 0.7079235155135393,
      "rewards/rejected": -0.4248804687522352,
      "step": 670
    },
    {
      "entropy": 1.0493477791547776,
      "epoch": 0.1750997811252736,
      "grad_norm": 43.2928352355957,
      "learning_rate": 4.730396061938192e-07,
      "logits/chosen": 4.510271629390098,
      "logits/rejected": 4.432055280745351,
      "logps/chosen": -365.35415878295896,
      "logps/rejected": -337.8881271362305,
      "loss": 0.5792159080505371,
      "mean_token_accuracy": 0.7000991433858872,
      "num_tokens": 9333452.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": 0.14067796915769576,
      "rewards/margins": 0.613553061336279,
      "rewards/rejected": -0.4728750824928284,
      "step": 680
    },
    {
      "entropy": 0.9644862800836563,
      "epoch": 0.1776747779065276,
      "grad_norm": 69.288818359375,
      "learning_rate": 4.720900446286844e-07,
      "logits/chosen": 4.534953701150827,
      "logits/rejected": 4.728891639666656,
      "logps/chosen": -331.7658752441406,
      "logps/rejected": -320.43864288330076,
      "loss": 0.5897587776184082,
      "mean_token_accuracy": 0.7107261806726456,
      "num_tokens": 9470396.0,
      "rewards/accuracies": 0.7,
      "rewards/chosen": 0.10997514789924026,
      "rewards/margins": 0.6469641901552677,
      "rewards/rejected": -0.5369890414178371,
      "step": 690
    },
    {
      "entropy": 0.9663564383983612,
      "epoch": 0.18024977468778164,
      "grad_norm": 56.91593933105469,
      "learning_rate": 4.711250363860543e-07,
      "logits/chosen": 4.553612024481287,
      "logits/rejected": 4.682406648335661,
      "logps/chosen": -354.33360290527344,
      "logps/rejected": -330.5455451965332,
      "loss": 0.5636067390441895,
      "mean_token_accuracy": 0.7119427442550659,
      "num_tokens": 9616899.0,
      "rewards/accuracies": 0.675,
      "rewards/chosen": 0.12148317620158196,
      "rewards/margins": 0.6505684278905391,
      "rewards/rejected": -0.5290852546691894,
      "step": 700
    },
    {
      "entropy": 0.9706416428089142,
      "epoch": 0.18282477146903567,
      "grad_norm": 84.57388305664062,
      "learning_rate": 4.7014464858362034e-07,
      "logits/chosen": 4.64987863105768,
      "logits/rejected": 4.537342444882523,
      "logps/chosen": -335.3706756591797,
      "logps/rejected": -332.0774948120117,
      "loss": 0.5970035552978515,
      "mean_token_accuracy": 0.7182563871145249,
      "num_tokens": 9761467.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": 0.09402619935572147,
      "rewards/margins": 0.6252138413488865,
      "rewards/rejected": -0.5311876498162746,
      "step": 710
    },
    {
      "entropy": 1.034638649225235,
      "epoch": 0.1853997682502897,
      "grad_norm": 54.29759979248047,
      "learning_rate": 4.6914894940874437e-07,
      "logits/chosen": 4.266203497733936,
      "logits/rejected": 4.523968467287711,
      "logps/chosen": -325.725659942627,
      "logps/rejected": -323.9387481689453,
      "loss": 0.5491708278656006,
      "mean_token_accuracy": 0.7071366012096405,
      "num_tokens": 9903698.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": 0.22243431201204658,
      "rewards/margins": 0.649902455508709,
      "rewards/rejected": -0.42746814265847205,
      "step": 720
    },
    {
      "entropy": 0.9635148346424103,
      "epoch": 0.1879747650315437,
      "grad_norm": 65.74314880371094,
      "learning_rate": 4.6813800811371574e-07,
      "logits/chosen": 4.393643202445704,
      "logits/rejected": 4.69651795868508,
      "logps/chosen": -316.67277908325195,
      "logps/rejected": -307.452326965332,
      "loss": 0.549955940246582,
      "mean_token_accuracy": 0.7105065941810608,
      "num_tokens": 10030969.0,
      "rewards/accuracies": 0.7,
      "rewards/chosen": 0.1749096255749464,
      "rewards/margins": 0.706392652913928,
      "rewards/rejected": -0.5314830277115107,
      "step": 730
    },
    {
      "entropy": 1.0183222383260726,
      "epoch": 0.19054976181279773,
      "grad_norm": 86.1967544555664,
      "learning_rate": 4.671118950109348e-07,
      "logits/chosen": 4.368303356172162,
      "logits/rejected": 4.454061870111907,
      "logps/chosen": -336.9675567626953,
      "logps/rejected": -344.4869667053223,
      "loss": 0.5904175758361816,
      "mean_token_accuracy": 0.7114386945962906,
      "num_tokens": 10174551.0,
      "rewards/accuracies": 0.6625,
      "rewards/chosen": 0.1169948361814022,
      "rewards/margins": 0.6056915517896414,
      "rewards/rejected": -0.48869672007858755,
      "step": 740
    },
    {
      "entropy": 0.9779737055301666,
      "epoch": 0.19312475859405176,
      "grad_norm": 81.3215103149414,
      "learning_rate": 4.6607068146802284e-07,
      "logits/chosen": 4.34994022328082,
      "logits/rejected": 4.692101749793964,
      "logps/chosen": -331.00280303955077,
      "logps/rejected": -304.03379211425784,
      "loss": 0.5423304557800293,
      "mean_token_accuracy": 0.7130115896463394,
      "num_tokens": 10309026.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.28315746448934076,
      "rewards/margins": 0.6935677349567413,
      "rewards/rejected": -0.41041027642786504,
      "step": 750
    },
    {
      "entropy": 0.9288118928670883,
      "epoch": 0.1956997553753058,
      "grad_norm": 64.25709533691406,
      "learning_rate": 4.65014439902858e-07,
      "logits/chosen": 4.646212096793471,
      "logits/rejected": 4.713876319349232,
      "logps/chosen": -328.4228065490723,
      "logps/rejected": -325.85131683349607,
      "loss": 0.597814416885376,
      "mean_token_accuracy": 0.715106874704361,
      "num_tokens": 10454313.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.2064698487520218,
      "rewards/margins": 0.5923826836049557,
      "rewards/rejected": -0.38591282814741135,
      "step": 760
    },
    {
      "entropy": 1.0085820376873016,
      "epoch": 0.1982747521565598,
      "grad_norm": 60.60913848876953,
      "learning_rate": 4.639432437785387e-07,
      "logits/chosen": 4.624968581831299,
      "logits/rejected": 4.833529951127492,
      "logps/chosen": -336.28843994140624,
      "logps/rejected": -309.39982376098635,
      "loss": 0.5225073814392089,
      "mean_token_accuracy": 0.7082129001617432,
      "num_tokens": 10590291.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.3072497993707657,
      "rewards/margins": 0.7669197037816048,
      "rewards/rejected": -0.45966988801956177,
      "step": 770
    },
    {
      "entropy": 0.9794066548347473,
      "epoch": 0.20084974893781382,
      "grad_norm": 58.9998664855957,
      "learning_rate": 4.6285716759827424e-07,
      "logits/chosen": 4.6144834186546575,
      "logits/rejected": 4.793606974197976,
      "logps/chosen": -327.2888931274414,
      "logps/rejected": -325.06821594238284,
      "loss": 0.5766840934753418,
      "mean_token_accuracy": 0.7179515421390533,
      "num_tokens": 10740192.0,
      "rewards/accuracies": 0.675,
      "rewards/chosen": 0.14814186543226243,
      "rewards/margins": 0.6016824893653393,
      "rewards/rejected": -0.4535406146198511,
      "step": 780
    },
    {
      "entropy": 1.0347339719533921,
      "epoch": 0.20342474571906785,
      "grad_norm": 51.062068939208984,
      "learning_rate": 4.6175628690020285e-07,
      "logits/chosen": 4.346229749477498,
      "logits/rejected": 4.816758360147,
      "logps/chosen": -305.7140487670898,
      "logps/rejected": -290.80119094848635,
      "loss": 0.531673812866211,
      "mean_token_accuracy": 0.7126160234212875,
      "num_tokens": 10871228.0,
      "rewards/accuracies": 0.8,
      "rewards/chosen": 0.2662514135241508,
      "rewards/margins": 0.765745447576046,
      "rewards/rejected": -0.49949402883648875,
      "step": 790
    },
    {
      "entropy": 1.0405800729990005,
      "epoch": 0.20599974250032188,
      "grad_norm": 43.19007110595703,
      "learning_rate": 4.60640678252138e-07,
      "logits/chosen": 4.161079633620957,
      "logits/rejected": 4.3872889124713,
      "logps/chosen": -328.34368896484375,
      "logps/rejected": -308.5476463317871,
      "loss": 0.5606516838073731,
      "mean_token_accuracy": 0.6936956197023392,
      "num_tokens": 10997163.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.2265689242631197,
      "rewards/margins": 0.6632086362689733,
      "rewards/rejected": -0.4366397067904472,
      "step": 800
    },
    {
      "entropy": 0.9820335626602172,
      "epoch": 0.2085747392815759,
      "grad_norm": 46.55604553222656,
      "learning_rate": 4.5951041924624286e-07,
      "logits/chosen": 4.517027936970296,
      "logits/rejected": 4.5808484038104424,
      "logps/chosen": -318.0545349121094,
      "logps/rejected": -309.64340591430664,
      "loss": 0.5541488647460937,
      "mean_token_accuracy": 0.7265620589256286,
      "num_tokens": 11129908.0,
      "rewards/accuracies": 0.675,
      "rewards/chosen": 0.3501715624704957,
      "rewards/margins": 0.7329499304294587,
      "rewards/rejected": -0.3827783614397049,
      "step": 810
    },
    {
      "entropy": 1.0183537423610687,
      "epoch": 0.2111497360628299,
      "grad_norm": 43.764923095703125,
      "learning_rate": 4.5836558849363384e-07,
      "logits/chosen": 4.702848554606683,
      "logits/rejected": 4.6702062948790415,
      "logps/chosen": -318.51319122314453,
      "logps/rejected": -289.55165786743163,
      "loss": 0.4766195297241211,
      "mean_token_accuracy": 0.6965174585580826,
      "num_tokens": 11260989.0,
      "rewards/accuracies": 0.7875,
      "rewards/chosen": 0.40055517572909594,
      "rewards/margins": 0.8507210377603769,
      "rewards/rejected": -0.45016585290431976,
      "step": 820
    },
    {
      "entropy": 1.0276739478111268,
      "epoch": 0.21372473284408394,
      "grad_norm": 93.66756439208984,
      "learning_rate": 4.5720626561891306e-07,
      "logits/chosen": 4.369238347744605,
      "logits/rejected": 4.546188272271925,
      "logps/chosen": -362.52023162841795,
      "logps/rejected": -339.30326080322266,
      "loss": 0.5464986324310303,
      "mean_token_accuracy": 0.7055434852838516,
      "num_tokens": 11400548.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.20844534412026405,
      "rewards/margins": 0.6820525597780943,
      "rewards/rejected": -0.47360720857977867,
      "step": 830
    },
    {
      "entropy": 1.0048970103263855,
      "epoch": 0.21629972962533797,
      "grad_norm": 39.34489822387695,
      "learning_rate": 4.5603253125463013e-07,
      "logits/chosen": 4.482431808569223,
      "logits/rejected": 4.461442789980365,
      "logps/chosen": -344.8500915527344,
      "logps/rejected": -313.5320655822754,
      "loss": 0.6112454414367676,
      "mean_token_accuracy": 0.7048583775758743,
      "num_tokens": 11532663.0,
      "rewards/accuracies": 0.65,
      "rewards/chosen": 0.17530339695513247,
      "rewards/margins": 0.6294477324932813,
      "rewards/rejected": -0.45414433553814887,
      "step": 840
    },
    {
      "entropy": 1.0157939851284028,
      "epoch": 0.218874726406592,
      "grad_norm": 50.64561080932617,
      "learning_rate": 4.548444670356743e-07,
      "logits/chosen": 4.558716367494338,
      "logits/rejected": 4.622169300415546,
      "logps/chosen": -295.56880950927734,
      "logps/rejected": -281.37263641357424,
      "loss": 0.5642531871795654,
      "mean_token_accuracy": 0.6964393675327301,
      "num_tokens": 11662432.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.2496818548068404,
      "rewards/margins": 0.6377269882708788,
      "rewards/rejected": -0.3880451485514641,
      "step": 850
    },
    {
      "entropy": 0.9604170709848404,
      "epoch": 0.221449723187846,
      "grad_norm": 49.767303466796875,
      "learning_rate": 4.536421555935965e-07,
      "logits/chosen": 4.543303368009663,
      "logits/rejected": 4.610104424823825,
      "logps/chosen": -352.4889404296875,
      "logps/rejected": -309.16469802856443,
      "loss": 0.5757847785949707,
      "mean_token_accuracy": 0.7144225656986236,
      "num_tokens": 11799597.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.25048075765371325,
      "rewards/margins": 0.6514817461371422,
      "rewards/rejected": -0.40100100487470625,
      "step": 860
    },
    {
      "entropy": 0.9212517082691193,
      "epoch": 0.22402471996910003,
      "grad_norm": 44.281654357910156,
      "learning_rate": 4.524256805508623e-07,
      "logits/chosen": 4.4288841618762085,
      "logits/rejected": 4.522083677187061,
      "logps/chosen": -284.46302261352537,
      "logps/rejected": -299.4017608642578,
      "loss": 0.5767347335815429,
      "mean_token_accuracy": 0.7335207104682923,
      "num_tokens": 11932852.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": 0.31010360531508924,
      "rewards/margins": 0.6455920591950417,
      "rewards/rejected": -0.33548846244812014,
      "step": 870
    },
    {
      "entropy": 1.0051367849111557,
      "epoch": 0.22659971675035406,
      "grad_norm": 59.0626335144043,
      "learning_rate": 4.5119512651503554e-07,
      "logits/chosen": 4.433084857610975,
      "logits/rejected": 4.87764424226992,
      "logps/chosen": -325.5847366333008,
      "logps/rejected": -313.316064453125,
      "loss": 0.532440185546875,
      "mean_token_accuracy": 0.7039438933134079,
      "num_tokens": 12057229.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": 0.3609015665948391,
      "rewards/margins": 0.9245390221476555,
      "rewards/rejected": -0.5636374469846487,
      "step": 880
    },
    {
      "entropy": 0.9578097075223923,
      "epoch": 0.2291747135316081,
      "grad_norm": 82.41941833496094,
      "learning_rate": 4.499505790728943e-07,
      "logits/chosen": 4.321722093432549,
      "logits/rejected": 4.647098786096395,
      "logps/chosen": -321.9654907226562,
      "logps/rejected": -307.7423271179199,
      "loss": 0.537187147140503,
      "mean_token_accuracy": 0.7209613204002381,
      "num_tokens": 12205196.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.33095078393816946,
      "rewards/margins": 0.769245932251215,
      "rewards/rejected": -0.4382951531559229,
      "step": 890
    },
    {
      "entropy": 0.9880760461091995,
      "epoch": 0.23174971031286212,
      "grad_norm": 44.68686294555664,
      "learning_rate": 4.486921247844777e-07,
      "logits/chosen": 4.494354658873168,
      "logits/rejected": 4.507425478709289,
      "logps/chosen": -305.7351020812988,
      "logps/rejected": -276.03930740356446,
      "loss": 0.5129434585571289,
      "mean_token_accuracy": 0.7095896989107132,
      "num_tokens": 12334089.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.36211476698517797,
      "rewards/margins": 0.8058889979496598,
      "rewards/rejected": -0.4437742181122303,
      "step": 900
    },
    {
      "entropy": 1.0145466029644012,
      "epoch": 0.23432470709411612,
      "grad_norm": 84.48532104492188,
      "learning_rate": 4.474198511770659e-07,
      "logits/chosen": 4.206393560107395,
      "logits/rejected": 4.203391496213106,
      "logps/chosen": -380.53197174072267,
      "logps/rejected": -352.91199493408203,
      "loss": 0.6008936882019043,
      "mean_token_accuracy": 0.709510189294815,
      "num_tokens": 12477504.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.3297712683677673,
      "rewards/margins": 0.7578461959958076,
      "rewards/rejected": -0.42807492688298227,
      "step": 910
    },
    {
      "entropy": 0.9396564811468124,
      "epoch": 0.23689970387537015,
      "grad_norm": 59.96947479248047,
      "learning_rate": 4.4613384673909206e-07,
      "logits/chosen": 4.477632786838245,
      "logits/rejected": 4.655172462661108,
      "logps/chosen": -306.4482566833496,
      "logps/rejected": -309.2312377929687,
      "loss": 0.5252578735351563,
      "mean_token_accuracy": 0.7247987031936646,
      "num_tokens": 12620019.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.4170497950166464,
      "rewards/margins": 0.7426737532019615,
      "rewards/rejected": -0.32562397196888926,
      "step": 920
    },
    {
      "entropy": 0.9839500367641449,
      "epoch": 0.23947470065662418,
      "grad_norm": 51.99100875854492,
      "learning_rate": 4.448342009139883e-07,
      "logits/chosen": 4.521783636596377,
      "logits/rejected": 4.74909994208237,
      "logps/chosen": -304.9866119384766,
      "logps/rejected": -305.9053276062012,
      "loss": 0.6351909637451172,
      "mean_token_accuracy": 0.7072886407375336,
      "num_tokens": 12746450.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3074321709573269,
      "rewards/margins": 0.5390587091445923,
      "rewards/rejected": -0.23162655010819436,
      "step": 930
    },
    {
      "entropy": 0.9272339820861817,
      "epoch": 0.24204969743787821,
      "grad_norm": 50.78702926635742,
      "learning_rate": 4.435210040939644e-07,
      "logits/chosen": 4.49347232077586,
      "logits/rejected": 4.544626873447705,
      "logps/chosen": -298.0924583435059,
      "logps/rejected": -311.46923828125,
      "loss": 0.5839786529541016,
      "mean_token_accuracy": 0.724303224682808,
      "num_tokens": 12897894.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.3771603710949421,
      "rewards/margins": 0.6256467439234257,
      "rewards/rejected": -0.2484863717108965,
      "step": 940
    },
    {
      "entropy": 0.9667321592569351,
      "epoch": 0.24462469421913222,
      "grad_norm": 67.10762786865234,
      "learning_rate": 4.421943476137212e-07,
      "logits/chosen": 4.021056633771452,
      "logits/rejected": 4.102243549720436,
      "logps/chosen": -342.9797004699707,
      "logps/rejected": -341.395401763916,
      "loss": 0.5504098892211914,
      "mean_token_accuracy": 0.7200477600097657,
      "num_tokens": 13040090.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": 0.2565610365010798,
      "rewards/margins": 0.7125073965638876,
      "rewards/rejected": -0.45594635233283043,
      "step": 950
    },
    {
      "entropy": 0.9595152884721756,
      "epoch": 0.24719969100038625,
      "grad_norm": 47.71066665649414,
      "learning_rate": 4.408543237440978e-07,
      "logits/chosen": 4.39034454098633,
      "logits/rejected": 4.761796312543795,
      "logps/chosen": -335.67313919067385,
      "logps/rejected": -313.7569305419922,
      "loss": 0.5047685623168945,
      "mean_token_accuracy": 0.7175188481807708,
      "num_tokens": 13177252.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.28581122644245627,
      "rewards/margins": 0.8943164490163327,
      "rewards/rejected": -0.6085052292793989,
      "step": 960
    },
    {
      "entropy": 0.9980293989181519,
      "epoch": 0.24977468778164028,
      "grad_norm": 52.98997497558594,
      "learning_rate": 4.395010256856544e-07,
      "logits/chosen": 4.402309676885595,
      "logits/rejected": 4.488470372113004,
      "logps/chosen": -338.8411064147949,
      "logps/rejected": -308.0730743408203,
      "loss": 0.5538735389709473,
      "mean_token_accuracy": 0.711775341629982,
      "num_tokens": 13308594.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": 0.38073269501328466,
      "rewards/margins": 0.6910081386566163,
      "rewards/rejected": -0.3102754525840282,
      "step": 970
    },
    {
      "entropy": 0.9796093523502349,
      "epoch": 0.2523496845628943,
      "grad_norm": 72.55328369140625,
      "learning_rate": 4.381345475621897e-07,
      "logits/chosen": 4.3565436245435745,
      "logits/rejected": 4.169848814716963,
      "logps/chosen": -346.8167358398438,
      "logps/rejected": -293.40828170776365,
      "loss": 0.5575485706329346,
      "mean_token_accuracy": 0.702203905582428,
      "num_tokens": 13441680.0,
      "rewards/accuracies": 0.675,
      "rewards/chosen": 0.48410786855965854,
      "rewards/margins": 0.6693111173808575,
      "rewards/rejected": -0.18520323652774096,
      "step": 980
    },
    {
      "entropy": 1.045410931110382,
      "epoch": 0.2549246813441483,
      "grad_norm": 38.534996032714844,
      "learning_rate": 4.367549844141946e-07,
      "logits/chosen": 4.20656935170718,
      "logits/rejected": 4.301880442441847,
      "logps/chosen": -330.08270797729494,
      "logps/rejected": -319.90564880371096,
      "loss": 0.5081873416900635,
      "mean_token_accuracy": 0.7055037915706635,
      "num_tokens": 13570019.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.5979794513434171,
      "rewards/margins": 0.8163566261529922,
      "rewards/rejected": -0.21837717592716216,
      "step": 990
    },
    {
      "entropy": 0.9823104649782181,
      "epoch": 0.25749967812540236,
      "grad_norm": 50.6489372253418,
      "learning_rate": 4.353624321922421e-07,
      "logits/chosen": 4.286461799453784,
      "logits/rejected": 4.583997969007628,
      "logps/chosen": -316.2755645751953,
      "logps/rejected": -288.8273071289062,
      "loss": 0.5642371654510498,
      "mean_token_accuracy": 0.7020954936742783,
      "num_tokens": 13703625.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": 0.4080685555934906,
      "rewards/margins": 0.6109554670751095,
      "rewards/rejected": -0.20288692265748978,
      "step": 1000
    },
    {
      "entropy": 1.0343029111623765,
      "epoch": 0.26007467490665637,
      "grad_norm": 66.1665267944336,
      "learning_rate": 4.3395698775031373e-07,
      "logits/chosen": 4.268548969788038,
      "logits/rejected": 4.119006348300696,
      "logps/chosen": -357.6877983093262,
      "logps/rejected": -311.49485168457034,
      "loss": 0.5452314376831054,
      "mean_token_accuracy": 0.7012816995382309,
      "num_tokens": 13846374.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.5230955958366394,
      "rewards/margins": 0.7875805646181107,
      "rewards/rejected": -0.26448496505618097,
      "step": 1010
    },
    {
      "entropy": 0.9775324791669846,
      "epoch": 0.26264967168791037,
      "grad_norm": 72.12793731689453,
      "learning_rate": 4.3253874883906305e-07,
      "logits/chosen": 4.074687471749744,
      "logits/rejected": 4.233662853315577,
      "logps/chosen": -340.43592529296876,
      "logps/rejected": -325.31726837158203,
      "loss": 0.5151017189025879,
      "mean_token_accuracy": 0.7066153079271317,
      "num_tokens": 13989379.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.5307824924588204,
      "rewards/margins": 0.7811033889651299,
      "rewards/rejected": -0.25032090246677396,
      "step": 1020
    },
    {
      "entropy": 0.9814414829015732,
      "epoch": 0.2652246684691644,
      "grad_norm": 64.95217895507812,
      "learning_rate": 4.311078140990173e-07,
      "logits/chosen": 4.479037367671388,
      "logits/rejected": 4.75286217597192,
      "logps/chosen": -325.9545074462891,
      "logps/rejected": -284.51468658447266,
      "loss": 0.5569169998168946,
      "mean_token_accuracy": 0.7042251348495483,
      "num_tokens": 14123739.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": 0.32408043220639227,
      "rewards/margins": 0.7771117240190506,
      "rewards/rejected": -0.45303131118416784,
      "step": 1030
    },
    {
      "entropy": 0.9413322359323502,
      "epoch": 0.26779966525041843,
      "grad_norm": 80.55500030517578,
      "learning_rate": 4.2966428305371636e-07,
      "logits/chosen": 4.377699481875437,
      "logits/rejected": 4.389535161226265,
      "logps/chosen": -313.2835182189941,
      "logps/rejected": -316.5220802307129,
      "loss": 0.5732664108276367,
      "mean_token_accuracy": 0.7221273809671402,
      "num_tokens": 14265000.0,
      "rewards/accuracies": 0.68125,
      "rewards/chosen": 0.4250858768820763,
      "rewards/margins": 0.7080753080546855,
      "rewards/rejected": -0.2829894308000803,
      "step": 1040
    },
    {
      "entropy": 0.9776986628770828,
      "epoch": 0.2703746620316725,
      "grad_norm": 51.449913024902344,
      "learning_rate": 4.2820825610279133e-07,
      "logits/chosen": 4.661412588919254,
      "logits/rejected": 4.593736025078175,
      "logps/chosen": -339.15430603027346,
      "logps/rejected": -318.2574661254883,
      "loss": 0.4670609474182129,
      "mean_token_accuracy": 0.7175067514181137,
      "num_tokens": 14400976.0,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.38831937648355963,
      "rewards/margins": 0.949066436290741,
      "rewards/rejected": -0.5607470721006393,
      "step": 1050
    },
    {
      "entropy": 1.020277938246727,
      "epoch": 0.2729496588129265,
      "grad_norm": 55.74995040893555,
      "learning_rate": 4.2673983451498095e-07,
      "logits/chosen": 4.402389798338661,
      "logits/rejected": 4.487614058865336,
      "logps/chosen": -313.8472129821777,
      "logps/rejected": -328.69329528808595,
      "loss": 0.614504051208496,
      "mean_token_accuracy": 0.7057614833116531,
      "num_tokens": 14543371.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.3766850482672453,
      "rewards/margins": 0.6233583599328995,
      "rewards/rejected": -0.24667333364486693,
      "step": 1060
    },
    {
      "entropy": 0.9874202489852906,
      "epoch": 0.2755246555941805,
      "grad_norm": 59.98944091796875,
      "learning_rate": 4.252591204210887e-07,
      "logits/chosen": 4.263326733993418,
      "logits/rejected": 4.384204437259707,
      "logps/chosen": -322.31773834228517,
      "logps/rejected": -316.5559844970703,
      "loss": 0.5732818126678467,
      "mean_token_accuracy": 0.7176870882511139,
      "num_tokens": 14693676.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.4322884988039732,
      "rewards/margins": 0.7766366615891457,
      "rewards/rejected": -0.34434815123677254,
      "step": 1070
    },
    {
      "entropy": 0.9693405300378799,
      "epoch": 0.27809965237543455,
      "grad_norm": 45.86904525756836,
      "learning_rate": 4.2376621680687917e-07,
      "logits/chosen": 4.387672304444868,
      "logits/rejected": 4.787616007597828,
      "logps/chosen": -320.0915107727051,
      "logps/rejected": -304.12927017211916,
      "loss": 0.4748681545257568,
      "mean_token_accuracy": 0.7157050430774688,
      "num_tokens": 14828096.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.45522386208176613,
      "rewards/margins": 0.992213748395443,
      "rewards/rejected": -0.5369898892939091,
      "step": 1080
    },
    {
      "entropy": 1.0544966250658034,
      "epoch": 0.28067464915668855,
      "grad_norm": 59.417510986328125,
      "learning_rate": 4.222612275059156e-07,
      "logits/chosen": 4.638836982554174,
      "logits/rejected": 4.424454369076098,
      "logps/chosen": -385.91309356689453,
      "logps/rejected": -338.5951675415039,
      "loss": 0.5437905311584472,
      "mean_token_accuracy": 0.6905349731445313,
      "num_tokens": 14963351.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.5071282530203461,
      "rewards/margins": 0.8644575633108615,
      "rewards/rejected": -0.3573293127119541,
      "step": 1090
    },
    {
      "entropy": 0.9879767268896102,
      "epoch": 0.28324964593794255,
      "grad_norm": 63.74018859863281,
      "learning_rate": 4.2074425719233766e-07,
      "logits/chosen": 4.488359396642159,
      "logits/rejected": 4.643684549172062,
      "logps/chosen": -331.8841720581055,
      "logps/rejected": -321.42944717407227,
      "loss": 0.5911607265472412,
      "mean_token_accuracy": 0.7219341546297073,
      "num_tokens": 15108671.0,
      "rewards/accuracies": 0.675,
      "rewards/chosen": 0.40470806770026685,
      "rewards/margins": 0.7509341590106487,
      "rewards/rejected": -0.3462261028587818,
      "step": 1100
    },
    {
      "entropy": 0.9788440942764283,
      "epoch": 0.2858246427191966,
      "grad_norm": 76.77590942382812,
      "learning_rate": 4.192154113735814e-07,
      "logits/chosen": 4.421786018683489,
      "logits/rejected": 4.490996666769295,
      "logps/chosen": -314.8667678833008,
      "logps/rejected": -286.60582275390624,
      "loss": 0.577817440032959,
      "mean_token_accuracy": 0.7135196805000306,
      "num_tokens": 15247587.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.42707300968468187,
      "rewards/margins": 0.6557801038026809,
      "rewards/rejected": -0.22870709635317327,
      "step": 1110
    },
    {
      "entropy": 0.9727748841047287,
      "epoch": 0.2883996395004506,
      "grad_norm": 57.815582275390625,
      "learning_rate": 4.1767479638304136e-07,
      "logits/chosen": 4.1840160801063835,
      "logits/rejected": 4.439244682258356,
      "logps/chosen": -317.1370674133301,
      "logps/rejected": -316.2620231628418,
      "loss": 0.46355476379394533,
      "mean_token_accuracy": 0.7234881222248077,
      "num_tokens": 15386538.0,
      "rewards/accuracies": 0.76875,
      "rewards/chosen": 0.4425433840602636,
      "rewards/margins": 1.1140578791499138,
      "rewards/rejected": -0.6715144798159599,
      "step": 1120
    },
    {
      "entropy": 1.0229500859975815,
      "epoch": 0.29097463628170467,
      "grad_norm": 70.49669647216797,
      "learning_rate": 4.161225193726744e-07,
      "logits/chosen": 4.247028300686612,
      "logits/rejected": 4.342862855944697,
      "logps/chosen": -329.0539245605469,
      "logps/rejected": -294.3794647216797,
      "loss": 0.7115816593170166,
      "mean_token_accuracy": 0.7092172116041183,
      "num_tokens": 15526554.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.16585742607712745,
      "rewards/margins": 0.5084913685917855,
      "rewards/rejected": -0.34263395629823207,
      "step": 1130
    },
    {
      "entropy": 1.0134399235248566,
      "epoch": 0.29354963306295867,
      "grad_norm": 49.2717399597168,
      "learning_rate": 4.1455868830554753e-07,
      "logits/chosen": 4.687399029849365,
      "logits/rejected": 4.804292485515946,
      "logps/chosen": -313.1935607910156,
      "logps/rejected": -308.4917694091797,
      "loss": 0.47522659301757814,
      "mean_token_accuracy": 0.7117983579635621,
      "num_tokens": 15663517.0,
      "rewards/accuracies": 0.775,
      "rewards/chosen": 0.3684566400945187,
      "rewards/margins": 0.9885708034038544,
      "rewards/rejected": -0.6201141782104969,
      "step": 1140
    },
    {
      "entropy": 0.9662601947784424,
      "epoch": 0.2961246298442127,
      "grad_norm": 46.927555084228516,
      "learning_rate": 4.1298341194832874e-07,
      "logits/chosen": 4.2654172623623,
      "logits/rejected": 4.462777736977772,
      "logps/chosen": -332.715348815918,
      "logps/rejected": -314.6715911865234,
      "loss": 0.5290184020996094,
      "mean_token_accuracy": 0.7249146103858948,
      "num_tokens": 15804723.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.17502789571881294,
      "rewards/margins": 0.9185936354100704,
      "rewards/rejected": -0.7435657449066639,
      "step": 1150
    },
    {
      "entropy": 0.9269982099533081,
      "epoch": 0.29869962662546673,
      "grad_norm": 43.15007400512695,
      "learning_rate": 4.1139679986372217e-07,
      "logits/chosen": 4.3429068940834075,
      "logits/rejected": 4.419197189323593,
      "logps/chosen": -319.6227722167969,
      "logps/rejected": -288.4176681518555,
      "loss": 0.5703027248382568,
      "mean_token_accuracy": 0.7230746954679489,
      "num_tokens": 15940851.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": 0.15694138407707214,
      "rewards/margins": 0.802145817130804,
      "rewards/rejected": -0.6452044431120157,
      "step": 1160
    },
    {
      "entropy": 1.0573670476675034,
      "epoch": 0.30127462340672073,
      "grad_norm": 59.737918853759766,
      "learning_rate": 4.0979896240284794e-07,
      "logits/chosen": 4.556767603242213,
      "logits/rejected": 4.557870560061238,
      "logps/chosen": -357.4036674499512,
      "logps/rejected": -333.00369415283205,
      "loss": 0.6517429351806641,
      "mean_token_accuracy": 0.6915355741977691,
      "num_tokens": 16077951.0,
      "rewards/accuracies": 0.6625,
      "rewards/chosen": 0.0758826458826661,
      "rewards/margins": 0.5559393465518951,
      "rewards/rejected": -0.4800567090511322,
      "step": 1170
    },
    {
      "entropy": 0.9671867489814758,
      "epoch": 0.3038496201879748,
      "grad_norm": 63.33560562133789,
      "learning_rate": 4.0819001069756693e-07,
      "logits/chosen": 4.426397280001497,
      "logits/rejected": 4.558353541171919,
      "logps/chosen": -340.3173118591309,
      "logps/rejected": -305.58366165161135,
      "loss": 0.6510348796844483,
      "mean_token_accuracy": 0.7134972125291824,
      "num_tokens": 16219068.0,
      "rewards/accuracies": 0.6625,
      "rewards/chosen": 0.09389384500682355,
      "rewards/margins": 0.5385100539773703,
      "rewards/rejected": -0.44461619537323716,
      "step": 1180
    },
    {
      "entropy": 0.994009718298912,
      "epoch": 0.3064246169692288,
      "grad_norm": 39.513572692871094,
      "learning_rate": 4.0657005665275145e-07,
      "logits/chosen": 4.424516966732384,
      "logits/rejected": 4.583500376884508,
      "logps/chosen": -367.8083465576172,
      "logps/rejected": -335.01172103881834,
      "loss": 0.5242624759674073,
      "mean_token_accuracy": 0.7107499301433563,
      "num_tokens": 16359511.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": 0.2147902548313141,
      "rewards/margins": 0.7728373639285564,
      "rewards/rejected": -0.558047103881836,
      "step": 1190
    },
    {
      "entropy": 0.9978521913290024,
      "epoch": 0.3089996137504828,
      "grad_norm": 71.0284423828125,
      "learning_rate": 4.0493921293850226e-07,
      "logits/chosen": 4.563816555341373,
      "logits/rejected": 4.472360028324781,
      "logps/chosen": -350.8389205932617,
      "logps/rejected": -326.0243545532227,
      "loss": 0.5175440311431885,
      "mean_token_accuracy": 0.7131787866353989,
      "num_tokens": 16499453.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.046732995100319384,
      "rewards/margins": 0.7765126742422581,
      "rewards/rejected": -0.729779689013958,
      "step": 1200
    },
    {
      "entropy": 1.031485903263092,
      "epoch": 0.31157461053173685,
      "grad_norm": 46.33605194091797,
      "learning_rate": 4.032975929823119e-07,
      "logits/chosen": 4.554355201437686,
      "logits/rejected": 4.689299279318398,
      "logps/chosen": -358.5169845581055,
      "logps/rejected": -337.0670585632324,
      "loss": 0.5396876335144043,
      "mean_token_accuracy": 0.6951931953430176,
      "num_tokens": 16638240.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.0658013928681612,
      "rewards/margins": 0.8561005435883999,
      "rewards/rejected": -0.790299154818058,
      "step": 1210
    },
    {
      "entropy": 0.9911634683609009,
      "epoch": 0.31414960731299085,
      "grad_norm": 41.01152420043945,
      "learning_rate": 4.01645310961176e-07,
      "logits/chosen": 4.613249416296803,
      "logits/rejected": 4.713178879447741,
      "logps/chosen": -323.03479080200196,
      "logps/rejected": -302.49776458740234,
      "loss": 0.44008312225341795,
      "mean_token_accuracy": 0.7046775192022323,
      "num_tokens": 16762530.0,
      "rewards/accuracies": 0.81875,
      "rewards/chosen": 0.050583545863628385,
      "rewards/margins": 1.049423012137413,
      "rewards/rejected": -0.9988394796848297,
      "step": 1220
    },
    {
      "entropy": 0.9754254132509231,
      "epoch": 0.31672460409424485,
      "grad_norm": 66.57528686523438,
      "learning_rate": 3.999824817936518e-07,
      "logits/chosen": 4.369396370883204,
      "logits/rejected": 4.565405825816332,
      "logps/chosen": -309.5333885192871,
      "logps/rejected": -310.53377532958984,
      "loss": 0.5147846221923829,
      "mean_token_accuracy": 0.7178112894296647,
      "num_tokens": 16888973.0,
      "rewards/accuracies": 0.75625,
      "rewards/chosen": 0.13259877637028694,
      "rewards/margins": 0.9616010505706072,
      "rewards/rejected": -0.8290023021399975,
      "step": 1230
    },
    {
      "entropy": 1.001852485537529,
      "epoch": 0.3192996008754989,
      "grad_norm": 93.46293640136719,
      "learning_rate": 3.9830922113186546e-07,
      "logits/chosen": 4.4342765187607505,
      "logits/rejected": 4.428724925246377,
      "logps/chosen": -339.5704040527344,
      "logps/rejected": -337.2366798400879,
      "loss": 0.5879923820495605,
      "mean_token_accuracy": 0.7226002395153046,
      "num_tokens": 17035377.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": -0.16235284619033336,
      "rewards/margins": 0.7105340275913476,
      "rewards/rejected": -0.8728868663311005,
      "step": 1240
    },
    {
      "entropy": 0.9102651327848434,
      "epoch": 0.3218745976567529,
      "grad_norm": 64.64633178710938,
      "learning_rate": 3.9662564535346855e-07,
      "logits/chosen": 4.385838511702223,
      "logits/rejected": 4.631670609424354,
      "logps/chosen": -317.2974662780762,
      "logps/rejected": -309.2920379638672,
      "loss": 0.5887946128845215,
      "mean_token_accuracy": 0.7323829025030136,
      "num_tokens": 17171999.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": -0.03849662467837334,
      "rewards/margins": 0.7032071411609649,
      "rewards/rejected": -0.7417037427425385,
      "step": 1250
    },
    {
      "entropy": 1.0048375755548478,
      "epoch": 0.32444959443800697,
      "grad_norm": 69.2200698852539,
      "learning_rate": 3.9493187155354346e-07,
      "logits/chosen": 4.491866410164571,
      "logits/rejected": 4.637912356655077,
      "logps/chosen": -348.9070739746094,
      "logps/rejected": -355.15400009155275,
      "loss": 0.5262919902801514,
      "mean_token_accuracy": 0.7100594013929367,
      "num_tokens": 17308547.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": 0.006792543455958366,
      "rewards/margins": 0.9484427526593209,
      "rewards/rejected": -0.9416502088308334,
      "step": 1260
    },
    {
      "entropy": 1.0846450448036193,
      "epoch": 0.327024591219261,
      "grad_norm": 75.90949249267578,
      "learning_rate": 3.932280175364595e-07,
      "logits/chosen": 4.498207897925901,
      "logits/rejected": 4.635352861664876,
      "logps/chosen": -319.4342674255371,
      "logps/rejected": -311.4297561645508,
      "loss": 0.4927949905395508,
      "mean_token_accuracy": 0.6965655654668808,
      "num_tokens": 17433786.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.13532297760248185,
      "rewards/margins": 0.943923632055521,
      "rewards/rejected": -0.808600626140833,
      "step": 1270
    },
    {
      "entropy": 1.0548436164855957,
      "epoch": 0.329599588000515,
      "grad_norm": 58.16775131225586,
      "learning_rate": 3.915142018076794e-07,
      "logits/chosen": 4.290384003766279,
      "logits/rejected": 4.634074884239211,
      "logps/chosen": -356.6481246948242,
      "logps/rejected": -339.0784294128418,
      "loss": 0.5433180809020997,
      "mean_token_accuracy": 0.7074305713176727,
      "num_tokens": 17564705.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.11096932906657457,
      "rewards/margins": 0.8167812839150429,
      "rewards/rejected": -0.7058119729161263,
      "step": 1280
    },
    {
      "entropy": 1.0007994264364242,
      "epoch": 0.33217458478176903,
      "grad_norm": 44.396663665771484,
      "learning_rate": 3.897905435655171e-07,
      "logits/chosen": 4.342449735180563,
      "logits/rejected": 4.586052330703876,
      "logps/chosen": -333.563484954834,
      "logps/rejected": -305.6269012451172,
      "loss": 0.4774781703948975,
      "mean_token_accuracy": 0.7060365378856659,
      "num_tokens": 17698024.0,
      "rewards/accuracies": 0.7875,
      "rewards/chosen": 0.344276400655508,
      "rewards/margins": 1.051966665685177,
      "rewards/rejected": -0.7076902478933335,
      "step": 1290
    },
    {
      "entropy": 1.0160721749067307,
      "epoch": 0.33474958156302304,
      "grad_norm": 61.9525260925293,
      "learning_rate": 3.8805716269284714e-07,
      "logits/chosen": 4.30281845903861,
      "logits/rejected": 4.424997012653698,
      "logps/chosen": -355.3688453674316,
      "logps/rejected": -352.252995300293,
      "loss": 0.5308104991912842,
      "mean_token_accuracy": 0.7141296565532684,
      "num_tokens": 17836151.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.3676468536257744,
      "rewards/margins": 0.8782583214342594,
      "rewards/rejected": -0.5106114767491817,
      "step": 1300
    },
    {
      "entropy": 1.0081484764814377,
      "epoch": 0.3373245783442771,
      "grad_norm": 68.5790023803711,
      "learning_rate": 3.863141797487669e-07,
      "logits/chosen": 4.18447886895724,
      "logits/rejected": 4.236553585044361,
      "logps/chosen": -295.8922348022461,
      "logps/rejected": -309.9488128662109,
      "loss": 0.5453038215637207,
      "mean_token_accuracy": 0.7150882959365845,
      "num_tokens": 17973049.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.25624501071870326,
      "rewards/margins": 0.9517513796687126,
      "rewards/rejected": -0.6955063514411449,
      "step": 1310
    },
    {
      "entropy": 1.0145260840654373,
      "epoch": 0.3398995751255311,
      "grad_norm": 73.98221588134766,
      "learning_rate": 3.8456171596021145e-07,
      "logits/chosen": 4.320287637804293,
      "logits/rejected": 4.558357023509025,
      "logps/chosen": -334.7983528137207,
      "logps/rejected": -328.91476211547854,
      "loss": 0.5630992889404297,
      "mean_token_accuracy": 0.7118307948112488,
      "num_tokens": 18120880.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": 0.2879774123430252,
      "rewards/margins": 0.9250785499811173,
      "rewards/rejected": -0.6371011391282082,
      "step": 1320
    },
    {
      "entropy": 1.0538078635931014,
      "epoch": 0.3424745719067851,
      "grad_norm": 54.6861686706543,
      "learning_rate": 3.827998932135222e-07,
      "logits/chosen": 4.52472381130375,
      "logits/rejected": 4.5028530702015725,
      "logps/chosen": -345.592847442627,
      "logps/rejected": -305.8528594970703,
      "loss": 0.475890064239502,
      "mean_token_accuracy": 0.7067580848932267,
      "num_tokens": 18253011.0,
      "rewards/accuracies": 0.775,
      "rewards/chosen": 0.43196534886956217,
      "rewards/margins": 0.9813137754797936,
      "rewards/rejected": -0.5493484251201153,
      "step": 1330
    },
    {
      "entropy": 1.0047302454710008,
      "epoch": 0.34504956868803915,
      "grad_norm": 69.02996826171875,
      "learning_rate": 3.810288340459689e-07,
      "logits/chosen": 4.361984383670553,
      "logits/rejected": 4.361619629463221,
      "logps/chosen": -350.0064292907715,
      "logps/rejected": -344.76106185913085,
      "loss": 0.6270887374877929,
      "mean_token_accuracy": 0.7146157413721085,
      "num_tokens": 18397200.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0734201766550541,
      "rewards/margins": 0.6079637352377176,
      "rewards/rejected": -0.5345435634255409,
      "step": 1340
    },
    {
      "entropy": 0.9514402151107788,
      "epoch": 0.34762456546929316,
      "grad_norm": 80.87300872802734,
      "learning_rate": 3.792486616372279e-07,
      "logits/chosen": 4.443909030636971,
      "logits/rejected": 4.460360202042288,
      "logps/chosen": -329.0527824401855,
      "logps/rejected": -343.86602249145506,
      "loss": 0.5502840995788574,
      "mean_token_accuracy": 0.7286958664655685,
      "num_tokens": 18539055.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": 0.1653471700847149,
      "rewards/margins": 0.7919495921581984,
      "rewards/rejected": -0.6266024131327868,
      "step": 1350
    },
    {
      "entropy": 0.9700424164533615,
      "epoch": 0.3501995622505472,
      "grad_norm": 57.26204299926758,
      "learning_rate": 3.7745949980081425e-07,
      "logits/chosen": 4.3594056296038275,
      "logits/rejected": 4.593635610352538,
      "logps/chosen": -282.6548042297363,
      "logps/rejected": -285.6140678405762,
      "loss": 0.5153086662292481,
      "mean_token_accuracy": 0.7336988478899003,
      "num_tokens": 18669012.0,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.40514476522803305,
      "rewards/margins": 0.9818641506135464,
      "rewards/rejected": -0.5767194028943777,
      "step": 1360
    },
    {
      "entropy": 0.9874075561761856,
      "epoch": 0.3527745590318012,
      "grad_norm": 65.18054962158203,
      "learning_rate": 3.7566147297547047e-07,
      "logits/chosen": 4.258672967953727,
      "logits/rejected": 4.426676722125068,
      "logps/chosen": -297.9508171081543,
      "logps/rejected": -308.9977172851562,
      "loss": 0.6036244392395019,
      "mean_token_accuracy": 0.7138050466775894,
      "num_tokens": 18801198.0,
      "rewards/accuracies": 0.66875,
      "rewards/chosen": 0.3781992755830288,
      "rewards/margins": 0.6525024998933077,
      "rewards/rejected": -0.27430321276187897,
      "step": 1370
    },
    {
      "entropy": 0.9998229205608368,
      "epoch": 0.3553495558130552,
      "grad_norm": 49.18499755859375,
      "learning_rate": 3.738547062165116e-07,
      "logits/chosen": 4.28802518209317,
      "logits/rejected": 4.515345606367839,
      "logps/chosen": -344.1229049682617,
      "logps/rejected": -343.2588516235352,
      "loss": 0.605926513671875,
      "mean_token_accuracy": 0.7159162849187851,
      "num_tokens": 18949925.0,
      "rewards/accuracies": 0.675,
      "rewards/chosen": 0.18270959258079528,
      "rewards/margins": 0.7173453290015459,
      "rewards/rejected": -0.5346357382833957,
      "step": 1380
    },
    {
      "entropy": 1.0593062907457351,
      "epoch": 0.3579245525943093,
      "grad_norm": 69.1773910522461,
      "learning_rate": 3.7203932518712734e-07,
      "logits/chosen": 4.3985165466469995,
      "logits/rejected": 4.648432641296062,
      "logps/chosen": -331.2413200378418,
      "logps/rejected": -317.4773864746094,
      "loss": 0.6042040824890137,
      "mean_token_accuracy": 0.7046501845121383,
      "num_tokens": 19071642.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.26102426275610924,
      "rewards/margins": 0.6165750399231911,
      "rewards/rejected": -0.35555077604949475,
      "step": 1390
    },
    {
      "entropy": 1.0222451746463777,
      "epoch": 0.3604995493755633,
      "grad_norm": 61.763160705566406,
      "learning_rate": 3.702154561496423e-07,
      "logits/chosen": 4.1127401210487795,
      "logits/rejected": 4.284693891063448,
      "logps/chosen": -318.31056213378906,
      "logps/rejected": -295.82591705322267,
      "loss": 0.5423140048980712,
      "mean_token_accuracy": 0.7130143046379089,
      "num_tokens": 19213285.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.2595692839473486,
      "rewards/margins": 0.8101298615336419,
      "rewards/rejected": -0.5505605675280094,
      "step": 1400
    },
    {
      "entropy": 1.0160330265760422,
      "epoch": 0.3630745461568173,
      "grad_norm": 77.30133819580078,
      "learning_rate": 3.683832259567341e-07,
      "logits/chosen": 4.499462280513063,
      "logits/rejected": 4.526303443830601,
      "logps/chosen": -337.373348236084,
      "logps/rejected": -337.92723617553713,
      "loss": 0.5844311237335205,
      "mean_token_accuracy": 0.7160362660884857,
      "num_tokens": 19348248.0,
      "rewards/accuracies": 0.66875,
      "rewards/chosen": 0.39422787353396416,
      "rewards/margins": 0.6997637458145618,
      "rewards/rejected": -0.3055358748883009,
      "step": 1410
    },
    {
      "entropy": 0.9416633933782578,
      "epoch": 0.36564954293807134,
      "grad_norm": 43.26420974731445,
      "learning_rate": 3.6654276204261043e-07,
      "logits/chosen": 4.5233009591843665,
      "logits/rejected": 4.598622006642471,
      "logps/chosen": -323.8805061340332,
      "logps/rejected": -315.74465255737306,
      "loss": 0.5712813854217529,
      "mean_token_accuracy": 0.7368648052215576,
      "num_tokens": 19493233.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.342887281998992,
      "rewards/margins": 0.748133927769959,
      "rewards/rejected": -0.4052466508001089,
      "step": 1420
    },
    {
      "entropy": 1.0600703418254853,
      "epoch": 0.36822453971932534,
      "grad_norm": 60.04607009887695,
      "learning_rate": 3.64694192414146e-07,
      "logits/chosen": 4.482818825251574,
      "logits/rejected": 4.374184376101063,
      "logps/chosen": -346.8372741699219,
      "logps/rejected": -318.7594619750977,
      "loss": 0.5119295597076416,
      "mean_token_accuracy": 0.7020805537700653,
      "num_tokens": 19627479.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": 0.5068976059556007,
      "rewards/margins": 0.9839373663067817,
      "rewards/rejected": -0.4770397573709488,
      "step": 1430
    },
    {
      "entropy": 1.0695737957954408,
      "epoch": 0.3707995365005794,
      "grad_norm": 66.99427795410156,
      "learning_rate": 3.6283764564197947e-07,
      "logits/chosen": 4.178093219183054,
      "logits/rejected": 4.626961773469877,
      "logps/chosen": -291.58864135742186,
      "logps/rejected": -294.04699249267577,
      "loss": 0.496781587600708,
      "mean_token_accuracy": 0.7073402464389801,
      "num_tokens": 19752558.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.40894790813326837,
      "rewards/margins": 0.8948969356715679,
      "rewards/rejected": -0.485949020460248,
      "step": 1440
    },
    {
      "entropy": 1.0322244733572006,
      "epoch": 0.3733745332818334,
      "grad_norm": 43.88212585449219,
      "learning_rate": 3.6097325085157126e-07,
      "logits/chosen": 4.574426650801122,
      "logits/rejected": 4.561950068538189,
      "logps/chosen": -354.1114860534668,
      "logps/rejected": -308.34269485473635,
      "loss": 0.5143645286560059,
      "mean_token_accuracy": 0.7042175978422165,
      "num_tokens": 19892893.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.3972885079681873,
      "rewards/margins": 0.8085891950875521,
      "rewards/rejected": -0.4113006766885519,
      "step": 1450
    },
    {
      "entropy": 0.9956100940704345,
      "epoch": 0.3759495300630874,
      "grad_norm": 31.5433406829834,
      "learning_rate": 3.5910113771422234e-07,
      "logits/chosen": 4.565448950462079,
      "logits/rejected": 4.586300662800534,
      "logps/chosen": -320.2218307495117,
      "logps/rejected": -307.1070892333984,
      "loss": 0.5700956821441651,
      "mean_token_accuracy": 0.7226622521877288,
      "num_tokens": 20032058.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.3020356763154268,
      "rewards/margins": 0.6896195448935032,
      "rewards/rejected": -0.38758386448025706,
      "step": 1460
    },
    {
      "entropy": 1.0120248764753341,
      "epoch": 0.37852452684434146,
      "grad_norm": 59.42012405395508,
      "learning_rate": 3.572214364380559e-07,
      "logits/chosen": 4.384914371666054,
      "logits/rejected": 4.491667970276556,
      "logps/chosen": -344.7418045043945,
      "logps/rejected": -315.87948684692384,
      "loss": 0.47869324684143066,
      "mean_token_accuracy": 0.7147729307413101,
      "num_tokens": 20168549.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.47410065568983556,
      "rewards/margins": 1.035487013310194,
      "rewards/rejected": -0.5613863591104746,
      "step": 1470
    },
    {
      "entropy": 0.9572999686002731,
      "epoch": 0.38109952362559546,
      "grad_norm": 34.17436218261719,
      "learning_rate": 3.5533427775896073e-07,
      "logits/chosen": 4.303627878264292,
      "logits/rejected": 4.463746868966824,
      "logps/chosen": -319.0563430786133,
      "logps/rejected": -348.01474838256837,
      "loss": 0.5226101875305176,
      "mean_token_accuracy": 0.7151152938604355,
      "num_tokens": 20315971.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.36733451671898365,
      "rewards/margins": 0.8375594437122345,
      "rewards/rejected": -0.4702249463647604,
      "step": 1480
    },
    {
      "entropy": 0.9907828450202942,
      "epoch": 0.3836745204068495,
      "grad_norm": 35.07666015625,
      "learning_rate": 3.534397929314988e-07,
      "logits/chosen": 4.499505547386105,
      "logits/rejected": 4.559010892291426,
      "logps/chosen": -343.87607803344724,
      "logps/rejected": -307.31992568969724,
      "loss": 0.46512527465820314,
      "mean_token_accuracy": 0.7122376918792724,
      "num_tokens": 20448199.0,
      "rewards/accuracies": 0.7875,
      "rewards/chosen": 0.41770453490316867,
      "rewards/margins": 1.0182910978794097,
      "rewards/rejected": -0.6005865626037121,
      "step": 1490
    },
    {
      "entropy": 0.9885411828756332,
      "epoch": 0.3862495171881035,
      "grad_norm": 33.83668518066406,
      "learning_rate": 3.51538113719776e-07,
      "logits/chosen": 4.585875986271088,
      "logits/rejected": 4.692986495571829,
      "logps/chosen": -342.3488739013672,
      "logps/rejected": -335.82422943115233,
      "loss": 0.5086318969726562,
      "mean_token_accuracy": 0.7093215554952621,
      "num_tokens": 20592523.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.42292424775660037,
      "rewards/margins": 0.9259717024862766,
      "rewards/rejected": -0.5030474781990051,
      "step": 1500
    },
    {
      "entropy": 0.9731161743402481,
      "epoch": 0.3888245139693575,
      "grad_norm": 163.65415954589844,
      "learning_rate": 3.496293723882776e-07,
      "logits/chosen": 4.448863749394176,
      "logits/rejected": 4.3950092747068945,
      "logps/chosen": -330.829727935791,
      "logps/rejected": -316.89535598754884,
      "loss": 0.5070469379425049,
      "mean_token_accuracy": 0.7174982994794845,
      "num_tokens": 20735839.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.4557771056890488,
      "rewards/margins": 0.8968748055398464,
      "rewards/rejected": -0.44109772704541683,
      "step": 1510
    },
    {
      "entropy": 1.0547097712755202,
      "epoch": 0.3913995107506116,
      "grad_norm": 50.195186614990234,
      "learning_rate": 3.4771370169266973e-07,
      "logits/chosen": 4.553992445601459,
      "logits/rejected": 4.251664231203607,
      "logps/chosen": -335.4343605041504,
      "logps/rejected": -325.768017578125,
      "loss": 0.5149908542633057,
      "mean_token_accuracy": 0.6950643092393876,
      "num_tokens": 20876847.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.42625158168375493,
      "rewards/margins": 0.9463345900177955,
      "rewards/rejected": -0.5200830094516278,
      "step": 1520
    },
    {
      "entropy": 1.037656196951866,
      "epoch": 0.3939745075318656,
      "grad_norm": 59.47888946533203,
      "learning_rate": 3.457912348705653e-07,
      "logits/chosen": 4.438172884191337,
      "logits/rejected": 4.536922108194634,
      "logps/chosen": -335.8483200073242,
      "logps/rejected": -330.42218856811525,
      "loss": 0.522545337677002,
      "mean_token_accuracy": 0.7065093517303467,
      "num_tokens": 21006575.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.2367303378880024,
      "rewards/margins": 0.9508240945637226,
      "rewards/rejected": -0.7140937626361847,
      "step": 1530
    },
    {
      "entropy": 0.9364173501729965,
      "epoch": 0.3965495043131196,
      "grad_norm": 82.34601593017578,
      "learning_rate": 3.438621056322576e-07,
      "logits/chosen": 4.373839777033968,
      "logits/rejected": 4.7794054819041225,
      "logps/chosen": -298.2203536987305,
      "logps/rejected": -336.3901840209961,
      "loss": 0.5983014106750488,
      "mean_token_accuracy": 0.7462542623281478,
      "num_tokens": 21167374.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.2251519788056612,
      "rewards/margins": 0.813858188688755,
      "rewards/rejected": -0.5887062273919582,
      "step": 1540
    },
    {
      "entropy": 1.0086449295282365,
      "epoch": 0.39912450109437364,
      "grad_norm": 60.29326629638672,
      "learning_rate": 3.4192644815142035e-07,
      "logits/chosen": 4.212622763483619,
      "logits/rejected": 4.329810739793858,
      "logps/chosen": -321.5377799987793,
      "logps/rejected": -311.98607025146487,
      "loss": 0.46921277046203613,
      "mean_token_accuracy": 0.7168540626764297,
      "num_tokens": 21299826.0,
      "rewards/accuracies": 0.8,
      "rewards/chosen": 0.3227839909493923,
      "rewards/margins": 0.9929535686969757,
      "rewards/rejected": -0.6701695755124092,
      "step": 1550
    },
    {
      "entropy": 1.0051219373941422,
      "epoch": 0.40169949787562764,
      "grad_norm": 64.10111999511719,
      "learning_rate": 3.3998439705577587e-07,
      "logits/chosen": 4.336165566431968,
      "logits/rejected": 4.434400236337686,
      "logps/chosen": -340.68482971191406,
      "logps/rejected": -334.61559906005857,
      "loss": 0.5105271339416504,
      "mean_token_accuracy": 0.7102333426475524,
      "num_tokens": 21437856.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.20026715993881225,
      "rewards/margins": 0.8899520173668861,
      "rewards/rejected": -0.689684884250164,
      "step": 1560
    },
    {
      "entropy": 0.9653350681066513,
      "epoch": 0.4042744946568817,
      "grad_norm": 57.03731918334961,
      "learning_rate": 3.3803608741773137e-07,
      "logits/chosen": 4.112749837286741,
      "logits/rejected": 4.341371862717738,
      "logps/chosen": -324.7610061645508,
      "logps/rejected": -320.7888603210449,
      "loss": 0.5658888339996337,
      "mean_token_accuracy": 0.7201475411653518,
      "num_tokens": 21578723.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.16358607187867164,
      "rewards/margins": 0.8124723732471466,
      "rewards/rejected": -0.6488862888887524,
      "step": 1570
    },
    {
      "entropy": 1.0187890470027923,
      "epoch": 0.4068494914381357,
      "grad_norm": 48.15790939331055,
      "learning_rate": 3.3608165474498463e-07,
      "logits/chosen": 4.511571607388313,
      "logits/rejected": 4.608273734629913,
      "logps/chosen": -290.37970581054685,
      "logps/rejected": -274.29333267211916,
      "loss": 0.5271549224853516,
      "mean_token_accuracy": 0.7093697398900985,
      "num_tokens": 21698766.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.45765426084399224,
      "rewards/margins": 0.9066750418394804,
      "rewards/rejected": -0.44902077317237854,
      "step": 1580
    },
    {
      "entropy": 1.0284575790166854,
      "epoch": 0.4094244882193897,
      "grad_norm": 45.4733772277832,
      "learning_rate": 3.341212349710991e-07,
      "logits/chosen": 4.1811476539506955,
      "logits/rejected": 4.1478347758355625,
      "logps/chosen": -348.23877105712893,
      "logps/rejected": -321.8086814880371,
      "loss": 0.5493577480316162,
      "mean_token_accuracy": 0.7045906782150269,
      "num_tokens": 21826816.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.2270603898912668,
      "rewards/margins": 0.7640220403671265,
      "rewards/rejected": -0.5369616780430079,
      "step": 1590
    },
    {
      "entropy": 1.0212238907814026,
      "epoch": 0.41199948500064376,
      "grad_norm": 89.39241790771484,
      "learning_rate": 3.3215496444604964e-07,
      "logits/chosen": 4.532756426511286,
      "logits/rejected": 4.629493190244512,
      "logps/chosen": -339.993953704834,
      "logps/rejected": -311.24218521118166,
      "loss": 0.5322290420532226,
      "mean_token_accuracy": 0.7075557291507721,
      "num_tokens": 21968861.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.2599800042808056,
      "rewards/margins": 0.9229485109448433,
      "rewards/rejected": -0.6629685159772635,
      "step": 1600
    },
    {
      "entropy": 1.0632631063461304,
      "epoch": 0.41457448178189776,
      "grad_norm": 41.36448287963867,
      "learning_rate": 3.3018297992673924e-07,
      "logits/chosen": 4.507129678744716,
      "logits/rejected": 4.7003194517972515,
      "logps/chosen": -340.9761138916016,
      "logps/rejected": -311.91967544555666,
      "loss": 0.4883114814758301,
      "mean_token_accuracy": 0.6983400255441665,
      "num_tokens": 22106242.0,
      "rewards/accuracies": 0.775,
      "rewards/chosen": 0.480826123803854,
      "rewards/margins": 1.0077935695648192,
      "rewards/rejected": -0.526967428997159,
      "step": 1610
    },
    {
      "entropy": 1.0190553188323974,
      "epoch": 0.4171494785631518,
      "grad_norm": 93.61823272705078,
      "learning_rate": 3.282054185674871e-07,
      "logits/chosen": 4.566390577731012,
      "logits/rejected": 4.165170132755452,
      "logps/chosen": -347.6525405883789,
      "logps/rejected": -335.3510269165039,
      "loss": 0.5531125545501709,
      "mean_token_accuracy": 0.6961113244295121,
      "num_tokens": 22247306.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.30568349808454515,
      "rewards/margins": 0.7751612067222595,
      "rewards/rejected": -0.46947770081460477,
      "step": 1620
    },
    {
      "entropy": 1.0008302301168441,
      "epoch": 0.4197244753444058,
      "grad_norm": 71.16085815429688,
      "learning_rate": 3.2622241791048964e-07,
      "logits/chosen": 4.582875345835534,
      "logits/rejected": 4.507340951874982,
      "logps/chosen": -308.279963684082,
      "logps/rejected": -283.0868232727051,
      "loss": 0.669698429107666,
      "mean_token_accuracy": 0.7173137634992599,
      "num_tokens": 22381348.0,
      "rewards/accuracies": 0.60625,
      "rewards/chosen": 0.3391913536936045,
      "rewards/margins": 0.5125614479184151,
      "rewards/rejected": -0.17337006516754627,
      "step": 1630
    },
    {
      "entropy": 0.9887471556663513,
      "epoch": 0.4222994721256598,
      "grad_norm": 51.84771728515625,
      "learning_rate": 3.242341158762543e-07,
      "logits/chosen": 4.416430634733047,
      "logits/rejected": 4.549421839075295,
      "logps/chosen": -342.06642150878906,
      "logps/rejected": -331.0380546569824,
      "loss": 0.4823248863220215,
      "mean_token_accuracy": 0.7201027363538742,
      "num_tokens": 22528483.0,
      "rewards/accuracies": 0.75625,
      "rewards/chosen": 0.3955455271527171,
      "rewards/margins": 1.0124763235449792,
      "rewards/rejected": -0.6169307995587587,
      "step": 1640
    },
    {
      "entropy": 1.0046769827604294,
      "epoch": 0.4248744689069139,
      "grad_norm": 45.738059997558594,
      "learning_rate": 3.222406507540066e-07,
      "logits/chosen": 4.169730618839731,
      "logits/rejected": 4.504782422516071,
      "logps/chosen": -318.0526466369629,
      "logps/rejected": -323.7085357666016,
      "loss": 0.4680798530578613,
      "mean_token_accuracy": 0.7223367244005203,
      "num_tokens": 22667434.0,
      "rewards/accuracies": 0.76875,
      "rewards/chosen": 0.32473539747297764,
      "rewards/margins": 1.0296514950692655,
      "rewards/rejected": -0.7049161233007908,
      "step": 1650
    },
    {
      "entropy": 0.9911033987998963,
      "epoch": 0.4274494656881679,
      "grad_norm": 62.42966079711914,
      "learning_rate": 3.2024216119207215e-07,
      "logits/chosen": 4.388264832498901,
      "logits/rejected": 4.388930285050414,
      "logps/chosen": -344.140438079834,
      "logps/rejected": -330.0677917480469,
      "loss": 0.5633188724517822,
      "mean_token_accuracy": 0.7085156321525574,
      "num_tokens": 22799973.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.24126881808042527,
      "rewards/margins": 0.7521100163459777,
      "rewards/rejected": -0.510841203480959,
      "step": 1660
    },
    {
      "entropy": 1.0932520985603333,
      "epoch": 0.4300244624694219,
      "grad_norm": 56.770660400390625,
      "learning_rate": 3.182387861882337e-07,
      "logits/chosen": 4.264878523978969,
      "logits/rejected": 4.41654907158112,
      "logps/chosen": -350.5899627685547,
      "logps/rejected": -378.4750503540039,
      "loss": 0.5012292385101318,
      "mean_token_accuracy": 0.7019307494163514,
      "num_tokens": 22942405.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.3956910580396652,
      "rewards/margins": 1.0485083594918252,
      "rewards/rejected": -0.6528172954916954,
      "step": 1670
    },
    {
      "entropy": 1.0346484899520874,
      "epoch": 0.43259945925067594,
      "grad_norm": 60.96067810058594,
      "learning_rate": 3.162306650800632e-07,
      "logits/chosen": 4.381951759660245,
      "logits/rejected": 4.3761065400867185,
      "logps/chosen": -319.1274444580078,
      "logps/rejected": -284.38352127075194,
      "loss": 0.5894469738006591,
      "mean_token_accuracy": 0.6966201603412628,
      "num_tokens": 23073290.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.222568079829216,
      "rewards/margins": 0.7801389444619418,
      "rewards/rejected": -0.5575708828866481,
      "step": 1680
    },
    {
      "entropy": 1.0130227506160736,
      "epoch": 0.43517445603192995,
      "grad_norm": 88.51482391357422,
      "learning_rate": 3.1421793753523085e-07,
      "logits/chosen": 4.153179485330449,
      "logits/rejected": 4.468525957066152,
      "logps/chosen": -347.164697265625,
      "logps/rejected": -333.5994499206543,
      "loss": 0.5185692787170411,
      "mean_token_accuracy": 0.7093759685754776,
      "num_tokens": 23208834.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": 0.19649834968149663,
      "rewards/margins": 0.9458184786140919,
      "rewards/rejected": -0.7493201218545437,
      "step": 1690
    },
    {
      "entropy": 0.9987653195858002,
      "epoch": 0.437749452813184,
      "grad_norm": 59.26670837402344,
      "learning_rate": 3.1220074354179134e-07,
      "logits/chosen": 4.447430238036,
      "logits/rejected": 4.345364868742253,
      "logps/chosen": -335.15634689331057,
      "logps/rejected": -322.3852783203125,
      "loss": 0.5455127239227295,
      "mean_token_accuracy": 0.707678034901619,
      "num_tokens": 23339859.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": 0.10780363157391548,
      "rewards/margins": 0.8008625339716673,
      "rewards/rejected": -0.6930589221417904,
      "step": 1700
    },
    {
      "entropy": 0.9553779780864715,
      "epoch": 0.440324449594438,
      "grad_norm": 67.1646728515625,
      "learning_rate": 3.101792233984468e-07,
      "logits/chosen": 4.317778320205003,
      "logits/rejected": 4.481137207239907,
      "logps/chosen": -333.0953964233398,
      "logps/rejected": -353.4780700683594,
      "loss": 0.5301013469696045,
      "mean_token_accuracy": 0.72568961083889,
      "num_tokens": 23483179.0,
      "rewards/accuracies": 0.7,
      "rewards/chosen": 0.2193666160106659,
      "rewards/margins": 0.8763925023376942,
      "rewards/rejected": -0.6570258844643831,
      "step": 1710
    },
    {
      "entropy": 1.0156360119581223,
      "epoch": 0.442899446375692,
      "grad_norm": 63.09981155395508,
      "learning_rate": 3.081535177047896e-07,
      "logits/chosen": 4.205427199102777,
      "logits/rejected": 4.318676160866055,
      "logps/chosen": -335.0963165283203,
      "logps/rejected": -340.5821472167969,
      "loss": 0.5540195941925049,
      "mean_token_accuracy": 0.7033586442470551,
      "num_tokens": 23624445.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.2030243966728449,
      "rewards/margins": 0.8909270189702511,
      "rewards/rejected": -0.6879026241600513,
      "step": 1720
    },
    {
      "entropy": 0.9994647204875946,
      "epoch": 0.44547444315694606,
      "grad_norm": 59.10143280029297,
      "learning_rate": 3.061237673515228e-07,
      "logits/chosen": 4.475974708431387,
      "logits/rejected": 4.4807601896793,
      "logps/chosen": -341.10973205566404,
      "logps/rejected": -337.8475936889648,
      "loss": 0.5034106254577637,
      "mean_token_accuracy": 0.7194854825735092,
      "num_tokens": 23772998.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": 0.2501206941902637,
      "rewards/margins": 1.0072236776351928,
      "rewards/rejected": -0.7571029774844646,
      "step": 1730
    },
    {
      "entropy": 0.9580833286046981,
      "epoch": 0.44804943993820007,
      "grad_norm": 65.97066497802734,
      "learning_rate": 3.040901135106614e-07,
      "logits/chosen": 4.6918887133458735,
      "logits/rejected": 4.679432626279566,
      "logps/chosen": -314.99814758300784,
      "logps/rejected": -301.6247161865234,
      "loss": 0.5239108085632325,
      "mean_token_accuracy": 0.7052569478750229,
      "num_tokens": 23904394.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.13549816319718958,
      "rewards/margins": 0.9139077693223954,
      "rewards/rejected": -0.7784095827490092,
      "step": 1740
    },
    {
      "entropy": 0.9939440310001373,
      "epoch": 0.4506244367194541,
      "grad_norm": 47.2442741394043,
      "learning_rate": 3.020526976257133e-07,
      "logits/chosen": 4.3844166827864575,
      "logits/rejected": 4.618266336702991,
      "logps/chosen": -324.671704864502,
      "logps/rejected": -311.7133255004883,
      "loss": 0.5350105285644531,
      "mean_token_accuracy": 0.7167317628860473,
      "num_tokens": 24029143.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.28501694574952124,
      "rewards/margins": 0.875099265575409,
      "rewards/rejected": -0.59008234962821,
      "step": 1750
    },
    {
      "entropy": 0.9728594481945038,
      "epoch": 0.4531994335007081,
      "grad_norm": 52.896759033203125,
      "learning_rate": 3.000116614018423e-07,
      "logits/chosen": 4.347222045065487,
      "logits/rejected": 4.491772509839542,
      "logps/chosen": -345.4105422973633,
      "logps/rejected": -311.8397155761719,
      "loss": 0.5155512809753418,
      "mean_token_accuracy": 0.7080474406480789,
      "num_tokens": 24168843.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.22332061640918255,
      "rewards/margins": 0.9133861869573593,
      "rewards/rejected": -0.6900655575096607,
      "step": 1760
    },
    {
      "entropy": 0.9716287791728974,
      "epoch": 0.45577443028196213,
      "grad_norm": 69.11155700683594,
      "learning_rate": 2.9796714679601153e-07,
      "logits/chosen": 4.07666419766162,
      "logits/rejected": 4.541536803168063,
      "logps/chosen": -333.53333587646483,
      "logps/rejected": -364.5257148742676,
      "loss": 0.5594016551971436,
      "mean_token_accuracy": 0.7196703106164932,
      "num_tokens": 24313406.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": 0.09260394554585219,
      "rewards/margins": 0.7315813470631838,
      "rewards/rejected": -0.6389774307608604,
      "step": 1770
    },
    {
      "entropy": 0.968604001402855,
      "epoch": 0.4583494270632162,
      "grad_norm": 75.13587951660156,
      "learning_rate": 2.9591929600711064e-07,
      "logits/chosen": 4.4954023446079905,
      "logits/rejected": 4.318362773308535,
      "logps/chosen": -354.32204360961913,
      "logps/rejected": -324.70655517578126,
      "loss": 0.6192696571350098,
      "mean_token_accuracy": 0.7113049805164338,
      "num_tokens": 24458893.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.11326459385454654,
      "rewards/margins": 0.6298577159643173,
      "rewards/rejected": -0.5165931291878223,
      "step": 1780
    },
    {
      "entropy": 1.0130292028188705,
      "epoch": 0.4609244238444702,
      "grad_norm": 42.6855354309082,
      "learning_rate": 2.9386825146606556e-07,
      "logits/chosen": 4.242645254840911,
      "logits/rejected": 4.208730296274518,
      "logps/chosen": -302.8190490722656,
      "logps/rejected": -287.4302803039551,
      "loss": 0.5245326519012451,
      "mean_token_accuracy": 0.7104728162288666,
      "num_tokens": 24592529.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.13887676037847996,
      "rewards/margins": 0.9169051334261894,
      "rewards/rejected": -0.7780283778905869,
      "step": 1790
    },
    {
      "entropy": 0.939438796043396,
      "epoch": 0.46349942062572425,
      "grad_norm": 53.44596481323242,
      "learning_rate": 2.918141558259323e-07,
      "logits/chosen": 4.425804985311169,
      "logits/rejected": 4.579291347565833,
      "logps/chosen": -314.8403045654297,
      "logps/rejected": -319.444881439209,
      "loss": 0.5266315937042236,
      "mean_token_accuracy": 0.7189203172922134,
      "num_tokens": 24727873.0,
      "rewards/accuracies": 0.76875,
      "rewards/chosen": 0.22815715186297894,
      "rewards/margins": 0.9557700768113137,
      "rewards/rejected": -0.7276129089295864,
      "step": 1800
    },
    {
      "entropy": 1.017608442902565,
      "epoch": 0.46607441740697825,
      "grad_norm": 75.32769775390625,
      "learning_rate": 2.897571519519749e-07,
      "logits/chosen": 4.325712846533475,
      "logits/rejected": 4.327608559472024,
      "logps/chosen": -321.98156585693357,
      "logps/rejected": -333.84306411743165,
      "loss": 0.5792953968048096,
      "mean_token_accuracy": 0.7041509211063385,
      "num_tokens": 24867266.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.039030957221984866,
      "rewards/margins": 0.7474001370370388,
      "rewards/rejected": -0.7083691634237766,
      "step": 1810
    },
    {
      "entropy": 0.8716294199228287,
      "epoch": 0.46864941418823225,
      "grad_norm": 69.67833709716797,
      "learning_rate": 2.8769738291172937e-07,
      "logits/chosen": 4.729007949358574,
      "logits/rejected": 4.748424687543743,
      "logps/chosen": -286.38878173828124,
      "logps/rejected": -315.22359466552734,
      "loss": 0.4433790683746338,
      "mean_token_accuracy": 0.7467392474412918,
      "num_tokens": 25008627.0,
      "rewards/accuracies": 0.775,
      "rewards/chosen": 0.3098715987056494,
      "rewards/margins": 1.0342625588178636,
      "rewards/rejected": -0.7243909515440464,
      "step": 1820
    },
    {
      "entropy": 1.0027720093727113,
      "epoch": 0.4712244109694863,
      "grad_norm": 58.2140007019043,
      "learning_rate": 2.856349919650529e-07,
      "logits/chosen": 4.342801686238408,
      "logits/rejected": 4.463185245965893,
      "logps/chosen": -312.2874137878418,
      "logps/rejected": -302.03475723266604,
      "loss": 0.49503297805786134,
      "mean_token_accuracy": 0.7056602656841278,
      "num_tokens": 25144657.0,
      "rewards/accuracies": 0.75625,
      "rewards/chosen": 0.2989152632653713,
      "rewards/margins": 1.038015954941511,
      "rewards/rejected": -0.739100681990385,
      "step": 1830
    },
    {
      "entropy": 0.9499877840280533,
      "epoch": 0.4737994077507403,
      "grad_norm": 66.1916275024414,
      "learning_rate": 2.835701225541601e-07,
      "logits/chosen": 4.750521566268771,
      "logits/rejected": 4.538883212279146,
      "logps/chosen": -328.4700454711914,
      "logps/rejected": -303.7730224609375,
      "loss": 0.5446010112762452,
      "mean_token_accuracy": 0.7132113128900528,
      "num_tokens": 25277668.0,
      "rewards/accuracies": 0.7,
      "rewards/chosen": 0.16393240839242934,
      "rewards/margins": 0.8867032006382942,
      "rewards/rejected": -0.7227707915008068,
      "step": 1840
    },
    {
      "entropy": 1.031875056028366,
      "epoch": 0.4763744045319943,
      "grad_norm": 66.70272064208984,
      "learning_rate": 2.815029182936461e-07,
      "logits/chosen": 4.415904153295584,
      "logits/rejected": 4.60825803093547,
      "logps/chosen": -355.8702056884766,
      "logps/rejected": -331.3533233642578,
      "loss": 0.4908586025238037,
      "mean_token_accuracy": 0.6996257096529007,
      "num_tokens": 25417326.0,
      "rewards/accuracies": 0.75625,
      "rewards/chosen": 0.27313601449131963,
      "rewards/margins": 1.024405437707901,
      "rewards/rejected": -0.751269418001175,
      "step": 1850
    },
    {
      "entropy": 0.9652377277612686,
      "epoch": 0.47894940131324837,
      "grad_norm": 51.2459716796875,
      "learning_rate": 2.7943352296049817e-07,
      "logits/chosen": 4.381936753944478,
      "logits/rejected": 4.503009937215559,
      "logps/chosen": -319.6825668334961,
      "logps/rejected": -317.8706504821777,
      "loss": 0.49027070999145506,
      "mean_token_accuracy": 0.7136070847511291,
      "num_tokens": 25549942.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.12705591693520546,
      "rewards/margins": 1.0133403688669205,
      "rewards/rejected": -0.8862844713032245,
      "step": 1860
    },
    {
      "entropy": 0.9866015404462815,
      "epoch": 0.48152439809450237,
      "grad_norm": 48.989219665527344,
      "learning_rate": 2.773620804840959e-07,
      "logits/chosen": 4.288209644175543,
      "logits/rejected": 4.304385423177918,
      "logps/chosen": -311.4419464111328,
      "logps/rejected": -301.3541030883789,
      "loss": 0.5566805839538574,
      "mean_token_accuracy": 0.7075719892978668,
      "num_tokens": 25680942.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1245550274848938,
      "rewards/margins": 0.8268403589725495,
      "rewards/rejected": -0.7022853277623653,
      "step": 1870
    },
    {
      "entropy": 0.9705629885196686,
      "epoch": 0.48409939487575643,
      "grad_norm": 52.36819076538086,
      "learning_rate": 2.7528873493620057e-07,
      "logits/chosen": 4.472752466035425,
      "logits/rejected": 4.677669941701231,
      "logps/chosen": -368.9326988220215,
      "logps/rejected": -323.53197250366213,
      "loss": 0.509603214263916,
      "mean_token_accuracy": 0.7059235632419586,
      "num_tokens": 25819603.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1403234925121069,
      "rewards/margins": 0.8788333332166076,
      "rewards/rejected": -0.7385098546743393,
      "step": 1880
    },
    {
      "entropy": 0.995087307691574,
      "epoch": 0.48667439165701043,
      "grad_norm": 62.67254638671875,
      "learning_rate": 2.732136305209347e-07,
      "logits/chosen": 4.383197580484635,
      "logits/rejected": 4.69916580317545,
      "logps/chosen": -340.8980545043945,
      "logps/rejected": -320.3641952514648,
      "loss": 0.5173936367034913,
      "mean_token_accuracy": 0.7051976472139359,
      "num_tokens": 25951235.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": 0.16847603507339953,
      "rewards/margins": 1.022395919263363,
      "rewards/rejected": -0.8539198614656925,
      "step": 1890
    },
    {
      "entropy": 0.9622721552848816,
      "epoch": 0.48924938843826443,
      "grad_norm": 45.17765426635742,
      "learning_rate": 2.711369115647525e-07,
      "logits/chosen": 4.274382834525982,
      "logits/rejected": 4.567524067046074,
      "logps/chosen": -298.41376571655275,
      "logps/rejected": -279.6089782714844,
      "loss": 0.5523276329040527,
      "mean_token_accuracy": 0.7140445560216904,
      "num_tokens": 26086978.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1847171064466238,
      "rewards/margins": 0.7757710345089436,
      "rewards/rejected": -0.591053931415081,
      "step": 1900
    },
    {
      "entropy": 0.9933630079030991,
      "epoch": 0.4918243852195185,
      "grad_norm": 48.34173583984375,
      "learning_rate": 2.690587225064019e-07,
      "logits/chosen": 4.248255535456623,
      "logits/rejected": 4.705522751035231,
      "logps/chosen": -320.63128356933595,
      "logps/rejected": -322.36269226074216,
      "loss": 0.5072047233581543,
      "mean_token_accuracy": 0.7172810703516006,
      "num_tokens": 26215024.0,
      "rewards/accuracies": 0.775,
      "rewards/chosen": 0.07452852092683315,
      "rewards/margins": 1.0095476165413857,
      "rewards/rejected": -0.9350190743803978,
      "step": 1910
    },
    {
      "entropy": 1.03436821103096,
      "epoch": 0.4943993820007725,
      "grad_norm": 43.470863342285156,
      "learning_rate": 2.669792078868785e-07,
      "logits/chosen": 4.445727344457447,
      "logits/rejected": 4.469930971599314,
      "logps/chosen": -319.6712905883789,
      "logps/rejected": -319.20749282836914,
      "loss": 0.4812634944915771,
      "mean_token_accuracy": 0.6964850157499314,
      "num_tokens": 26353198.0,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.27224585637450216,
      "rewards/margins": 1.200338999927044,
      "rewards/rejected": -0.9280931562185287,
      "step": 1920
    },
    {
      "entropy": 0.9880954831838608,
      "epoch": 0.49697437878202655,
      "grad_norm": 60.059425354003906,
      "learning_rate": 2.6489851233937254e-07,
      "logits/chosen": 4.360795670967695,
      "logits/rejected": 4.647031218142274,
      "logps/chosen": -337.48805923461913,
      "logps/rejected": -315.1701011657715,
      "loss": 0.5536554336547852,
      "mean_token_accuracy": 0.7152973026037216,
      "num_tokens": 26490614.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": -0.12552479170262815,
      "rewards/margins": 0.8430479489266872,
      "rewards/rejected": -0.9685727544128895,
      "step": 1930
    },
    {
      "entropy": 0.930216184258461,
      "epoch": 0.49954937556328055,
      "grad_norm": 57.515750885009766,
      "learning_rate": 2.628167805792094e-07,
      "logits/chosen": 4.484900366278743,
      "logits/rejected": 4.580445152126293,
      "logps/chosen": -337.2224060058594,
      "logps/rejected": -303.9986000061035,
      "loss": 0.47016096115112305,
      "mean_token_accuracy": 0.7162662833929062,
      "num_tokens": 26633836.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": 0.27869669497013094,
      "rewards/margins": 1.0826191306114197,
      "rewards/rejected": -0.803922425955534,
      "step": 1940
    },
    {
      "entropy": 0.9742702841758728,
      "epoch": 0.5021243723445346,
      "grad_norm": 39.161773681640625,
      "learning_rate": 2.607341573937847e-07,
      "logits/chosen": 4.283901718371682,
      "logits/rejected": 4.398404167911892,
      "logps/chosen": -346.7345329284668,
      "logps/rejected": -307.449609375,
      "loss": 0.6053523063659668,
      "mean_token_accuracy": 0.7093511551618576,
      "num_tokens": 26772785.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.0554534487426281,
      "rewards/margins": 0.7310394756495953,
      "rewards/rejected": -0.6755860306322574,
      "step": 1950
    },
    {
      "entropy": 1.0009024798870088,
      "epoch": 0.5046993691257886,
      "grad_norm": 65.297119140625,
      "learning_rate": 2.5865078763249374e-07,
      "logits/chosen": 4.496857392609219,
      "logits/rejected": 4.232990657321963,
      "logps/chosen": -321.77712478637693,
      "logps/rejected": -332.6124717712402,
      "loss": 0.5037482261657715,
      "mean_token_accuracy": 0.7017572969198227,
      "num_tokens": 26910902.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.061662624776363376,
      "rewards/margins": 1.0938476547598839,
      "rewards/rejected": -1.0321850061416626,
      "step": 1960
    },
    {
      "entropy": 0.9611604005098343,
      "epoch": 0.5072743659070427,
      "grad_norm": 57.55832290649414,
      "learning_rate": 2.565668161966575e-07,
      "logits/chosen": 4.354855323553224,
      "logits/rejected": 4.303488830095201,
      "logps/chosen": -384.1266235351562,
      "logps/rejected": -349.87569961547854,
      "loss": 0.610211992263794,
      "mean_token_accuracy": 0.7133847624063492,
      "num_tokens": 27067130.0,
      "rewards/accuracies": 0.7,
      "rewards/chosen": 0.10608787275850773,
      "rewards/margins": 0.7891268633306027,
      "rewards/rejected": -0.6830389894545078,
      "step": 1970
    },
    {
      "entropy": 0.9564415842294693,
      "epoch": 0.5098493626882966,
      "grad_norm": 45.074100494384766,
      "learning_rate": 2.5448238802944384e-07,
      "logits/chosen": 4.411397196718537,
      "logits/rejected": 4.597745429208741,
      "logps/chosen": -306.9679183959961,
      "logps/rejected": -336.1702850341797,
      "loss": 0.5392940998077392,
      "mean_token_accuracy": 0.7255575984716416,
      "num_tokens": 27204645.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.14039542600512506,
      "rewards/margins": 0.9036742106080056,
      "rewards/rejected": -0.7632787860929966,
      "step": 1980
    },
    {
      "entropy": 0.9326091140508652,
      "epoch": 0.5124243594695507,
      "grad_norm": 46.505699157714844,
      "learning_rate": 2.523976481057874e-07,
      "logits/chosen": 4.324125159695785,
      "logits/rejected": 4.345816921216889,
      "logps/chosen": -316.32717361450193,
      "logps/rejected": -306.45115127563474,
      "loss": 0.5290246486663819,
      "mean_token_accuracy": 0.708903294801712,
      "num_tokens": 27345750.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.1300957791507244,
      "rewards/margins": 0.841148529201746,
      "rewards/rejected": -0.7110527500510215,
      "step": 1990
    },
    {
      "entropy": 1.0109939306974411,
      "epoch": 0.5149993562508047,
      "grad_norm": 44.72649002075195,
      "learning_rate": 2.5031274142230576e-07,
      "logits/chosen": 4.510163501684517,
      "logits/rejected": 4.325580930649053,
      "logps/chosen": -332.6693832397461,
      "logps/rejected": -321.6415710449219,
      "loss": 0.5182650566101075,
      "mean_token_accuracy": 0.7005341708660126,
      "num_tokens": 27480837.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": 0.2247800800949335,
      "rewards/margins": 0.8958122998476028,
      "rewards/rejected": -0.6710322130471468,
      "step": 2000
    },
    {
      "entropy": 0.9889083057641983,
      "epoch": 0.5175743530320587,
      "grad_norm": 58.55170822143555,
      "learning_rate": 2.4822781298721473e-07,
      "logits/chosen": 4.579308480886338,
      "logits/rejected": 4.72775180226153,
      "logps/chosen": -332.1877388000488,
      "logps/rejected": -307.91585540771484,
      "loss": 0.5243938446044922,
      "mean_token_accuracy": 0.704783484339714,
      "num_tokens": 27619535.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.07122649792581796,
      "rewards/margins": 0.8846869133412838,
      "rewards/rejected": -0.8134604260325432,
      "step": 2010
    },
    {
      "entropy": 0.9544982284307479,
      "epoch": 0.5201493498133127,
      "grad_norm": 91.9818344116211,
      "learning_rate": 2.4614300781024314e-07,
      "logits/chosen": 4.510144881846659,
      "logits/rejected": 4.53069605625107,
      "logps/chosen": -339.508846282959,
      "logps/rejected": -335.85532531738284,
      "loss": 0.5688719749450684,
      "mean_token_accuracy": 0.7240165054798127,
      "num_tokens": 27762485.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": -0.06838485449552537,
      "rewards/margins": 1.0121475353837013,
      "rewards/rejected": -1.0805324025452137,
      "step": 2020
    },
    {
      "entropy": 0.983922865986824,
      "epoch": 0.5227243465945668,
      "grad_norm": 57.940250396728516,
      "learning_rate": 2.440584708925469e-07,
      "logits/chosen": 4.572384071478625,
      "logits/rejected": 4.769022806281224,
      "logps/chosen": -337.8588836669922,
      "logps/rejected": -308.79794082641604,
      "loss": 0.47387022972106935,
      "mean_token_accuracy": 0.7007263749837875,
      "num_tokens": 27904937.0,
      "rewards/accuracies": 0.75625,
      "rewards/chosen": 0.19583048969507216,
      "rewards/margins": 0.9539054423570633,
      "rewards/rejected": -0.7580749623477459,
      "step": 2030
    },
    {
      "entropy": 0.9752614676952363,
      "epoch": 0.5252993433758207,
      "grad_norm": 51.395687103271484,
      "learning_rate": 2.4197434721662453e-07,
      "logits/chosen": 4.555002191529634,
      "logits/rejected": 4.620000321200531,
      "logps/chosen": -284.87156143188474,
      "logps/rejected": -286.2675308227539,
      "loss": 0.50951247215271,
      "mean_token_accuracy": 0.7125829070806503,
      "num_tokens": 28030489.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.20675006583333017,
      "rewards/margins": 0.9260029315948486,
      "rewards/rejected": -0.719252859801054,
      "step": 2040
    },
    {
      "entropy": 1.0117036342620849,
      "epoch": 0.5278743401570748,
      "grad_norm": 60.130470275878906,
      "learning_rate": 2.3989078173623224e-07,
      "logits/chosen": 4.333817315595856,
      "logits/rejected": 4.47111231732413,
      "logps/chosen": -344.92832641601564,
      "logps/rejected": -313.57475128173826,
      "loss": 0.5121257305145264,
      "mean_token_accuracy": 0.7067907392978668,
      "num_tokens": 28166707.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.12466732561588287,
      "rewards/margins": 0.9099546924233437,
      "rewards/rejected": -0.7852873660624027,
      "step": 2050
    },
    {
      "entropy": 0.982884556055069,
      "epoch": 0.5304493369383289,
      "grad_norm": 51.04621124267578,
      "learning_rate": 2.3780791936630366e-07,
      "logits/chosen": 4.4138088975037535,
      "logits/rejected": 4.569439114318762,
      "logps/chosen": -301.08812103271487,
      "logps/rejected": -289.3797378540039,
      "loss": 0.5101655960083008,
      "mean_token_accuracy": 0.718460476398468,
      "num_tokens": 28301909.0,
      "rewards/accuracies": 0.75625,
      "rewards/chosen": 0.07634416520595551,
      "rewards/margins": 1.081505411863327,
      "rewards/rejected": -1.0051612228155136,
      "step": 2060
    },
    {
      "entropy": 0.9662475287914276,
      "epoch": 0.5330243337195828,
      "grad_norm": 58.388267517089844,
      "learning_rate": 2.3572590497286988e-07,
      "logits/chosen": 4.402496806036643,
      "logits/rejected": 4.572015991746902,
      "logps/chosen": -306.4839553833008,
      "logps/rejected": -327.9119476318359,
      "loss": 0.5104366302490234,
      "mean_token_accuracy": 0.7234220892190933,
      "num_tokens": 28441535.0,
      "rewards/accuracies": 0.76875,
      "rewards/chosen": -0.0008957050740718842,
      "rewards/margins": 0.96529905423522,
      "rewards/rejected": -0.9661947466433048,
      "step": 2070
    },
    {
      "entropy": 0.9819912523031235,
      "epoch": 0.5355993305008369,
      "grad_norm": 52.43979263305664,
      "learning_rate": 2.3364488336298363e-07,
      "logits/chosen": 4.221476865602507,
      "logits/rejected": 4.190381915320627,
      "logps/chosen": -368.37193832397463,
      "logps/rejected": -325.319921875,
      "loss": 0.5923614025115966,
      "mean_token_accuracy": 0.6977939248085022,
      "num_tokens": 28586273.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": 0.05655666179955006,
      "rewards/margins": 0.7294862203299999,
      "rewards/rejected": -0.6729295663535595,
      "step": 2080
    },
    {
      "entropy": 0.9639265269041062,
      "epoch": 0.5381743272820909,
      "grad_norm": 45.972591400146484,
      "learning_rate": 2.315649992746488e-07,
      "logits/chosen": 4.343052913177901,
      "logits/rejected": 4.35323733514684,
      "logps/chosen": -316.8616611480713,
      "logps/rejected": -304.16934967041016,
      "loss": 0.5641426563262939,
      "mean_token_accuracy": 0.7136311322450638,
      "num_tokens": 28724776.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": 0.0941460222005844,
      "rewards/margins": 0.7165497049689293,
      "rewards/rejected": -0.6224036753177643,
      "step": 2090
    },
    {
      "entropy": 0.9250091910362244,
      "epoch": 0.540749324063345,
      "grad_norm": 84.40534210205078,
      "learning_rate": 2.294863973667525e-07,
      "logits/chosen": 4.494238621336267,
      "logits/rejected": 4.506737881799265,
      "logps/chosen": -323.89574279785154,
      "logps/rejected": -318.222599029541,
      "loss": 0.5222518920898438,
      "mean_token_accuracy": 0.7188939899206161,
      "num_tokens": 28870500.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.10355070717632771,
      "rewards/margins": 0.9211230382323266,
      "rewards/rejected": -0.8175723269581795,
      "step": 2100
    },
    {
      "entropy": 0.9546181976795196,
      "epoch": 0.5433243208445989,
      "grad_norm": 75.78016662597656,
      "learning_rate": 2.2740922220900462e-07,
      "logits/chosen": 4.565091308177751,
      "logits/rejected": 4.517123364012567,
      "logps/chosen": -345.64021530151365,
      "logps/rejected": -328.6051383972168,
      "loss": 0.5602751731872558,
      "mean_token_accuracy": 0.7249366581439972,
      "num_tokens": 29007529.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.18785378597676755,
      "rewards/margins": 0.9705551497638225,
      "rewards/rejected": -0.782701363414526,
      "step": 2110
    },
    {
      "entropy": 0.9966236472129821,
      "epoch": 0.545899317625853,
      "grad_norm": 54.55280303955078,
      "learning_rate": 2.2533361827188264e-07,
      "logits/chosen": 4.558325017683928,
      "logits/rejected": 4.712489848699003,
      "logps/chosen": -333.2004821777344,
      "logps/rejected": -316.39307327270507,
      "loss": 0.49320497512817385,
      "mean_token_accuracy": 0.7029337018728257,
      "num_tokens": 29149979.0,
      "rewards/accuracies": 0.79375,
      "rewards/chosen": 0.05198279172182083,
      "rewards/margins": 1.0450399309396743,
      "rewards/rejected": -0.9930571526288986,
      "step": 2120
    },
    {
      "entropy": 0.9686165571212768,
      "epoch": 0.548474314407107,
      "grad_norm": 78.59687042236328,
      "learning_rate": 2.2325972991658306e-07,
      "logits/chosen": 4.431529517037036,
      "logits/rejected": 4.4224376064404805,
      "logps/chosen": -363.5590950012207,
      "logps/rejected": -340.79425888061525,
      "loss": 0.4562540054321289,
      "mean_token_accuracy": 0.7150961607694626,
      "num_tokens": 29302390.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": -0.03903376907110214,
      "rewards/margins": 1.1957050636410713,
      "rewards/rejected": -1.2347388774156571,
      "step": 2130
    },
    {
      "entropy": 0.9467013031244278,
      "epoch": 0.551049311188361,
      "grad_norm": 37.64112091064453,
      "learning_rate": 2.2118770138498167e-07,
      "logits/chosen": 4.643776640567135,
      "logits/rejected": 4.611367951965497,
      "logps/chosen": -309.24220275878906,
      "logps/rejected": -328.61732330322263,
      "loss": 0.5229010105133056,
      "mean_token_accuracy": 0.7172821938991547,
      "num_tokens": 29436778.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": -0.11375233680009841,
      "rewards/margins": 1.0564178586006165,
      "rewards/rejected": -1.1701701939105988,
      "step": 2140
    },
    {
      "entropy": 0.940066397190094,
      "epoch": 0.553624307969615,
      "grad_norm": 73.62627410888672,
      "learning_rate": 2.1911767678960063e-07,
      "logits/chosen": 4.652108957909488,
      "logits/rejected": 4.715885326711968,
      "logps/chosen": -331.7106544494629,
      "logps/rejected": -311.7936897277832,
      "loss": 0.48636856079101565,
      "mean_token_accuracy": 0.7166824817657471,
      "num_tokens": 29579500.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": 0.12549700997769833,
      "rewards/margins": 1.1011155344545842,
      "rewards/rejected": -0.9756185300648212,
      "step": 2150
    },
    {
      "entropy": 0.9694902867078781,
      "epoch": 0.5561993047508691,
      "grad_norm": 61.72739028930664,
      "learning_rate": 2.1704980010358542e-07,
      "logits/chosen": 4.602444838255726,
      "logits/rejected": 4.799831230044429,
      "logps/chosen": -331.89060668945314,
      "logps/rejected": -331.9415641784668,
      "loss": 0.5689295768737793,
      "mean_token_accuracy": 0.7171097755432129,
      "num_tokens": 29708042.0,
      "rewards/accuracies": 0.775,
      "rewards/chosen": -0.16421702206134797,
      "rewards/margins": 0.9721270004287362,
      "rewards/rejected": -1.1363440088927745,
      "step": 2160
    },
    {
      "entropy": 0.9587494820356369,
      "epoch": 0.558774301532123,
      "grad_norm": 72.0654525756836,
      "learning_rate": 2.1498421515069171e-07,
      "logits/chosen": 4.567917515442895,
      "logits/rejected": 4.510748764102793,
      "logps/chosen": -348.9438766479492,
      "logps/rejected": -326.5355796813965,
      "loss": 0.5357170104980469,
      "mean_token_accuracy": 0.7194243431091308,
      "num_tokens": 29853333.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.10092468392103911,
      "rewards/margins": 0.9197949707508087,
      "rewards/rejected": -0.8188702866435051,
      "step": 2170
    },
    {
      "entropy": 1.0412338376045227,
      "epoch": 0.5613492983133771,
      "grad_norm": 74.85493469238281,
      "learning_rate": 2.1292106559528147e-07,
      "logits/chosen": 4.408832687099317,
      "logits/rejected": 4.534272302491532,
      "logps/chosen": -300.82046737670896,
      "logps/rejected": -297.51931228637693,
      "loss": 0.4682475566864014,
      "mean_token_accuracy": 0.6946048319339753,
      "num_tokens": 29988640.0,
      "rewards/accuracies": 0.76875,
      "rewards/chosen": 0.2746724203228951,
      "rewards/margins": 1.281447449326515,
      "rewards/rejected": -1.0067750446498394,
      "step": 2180
    },
    {
      "entropy": 0.9617623418569565,
      "epoch": 0.5639242950946312,
      "grad_norm": 59.6571159362793,
      "learning_rate": 2.108604949323316e-07,
      "logits/chosen": 4.478970340786703,
      "logits/rejected": 4.599134912622948,
      "logps/chosen": -311.82995834350584,
      "logps/rejected": -325.49252166748045,
      "loss": 0.47739534378051757,
      "mean_token_accuracy": 0.7158630758523941,
      "num_tokens": 30124183.0,
      "rewards/accuracies": 0.76875,
      "rewards/chosen": 0.1551291227340698,
      "rewards/margins": 1.0965686649084092,
      "rewards/rejected": -0.9414395779371262,
      "step": 2190
    },
    {
      "entropy": 0.9513105571269989,
      "epoch": 0.5664992918758851,
      "grad_norm": 60.03646469116211,
      "learning_rate": 2.088026464774534e-07,
      "logits/chosen": 4.214004904189293,
      "logits/rejected": 4.465290087756984,
      "logps/chosen": -327.4006637573242,
      "logps/rejected": -319.3096549987793,
      "loss": 0.5218130111694336,
      "mean_token_accuracy": 0.7127509474754333,
      "num_tokens": 30271400.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.17084717899560928,
      "rewards/margins": 1.027149811387062,
      "rewards/rejected": -1.1979969710111618,
      "step": 2200
    },
    {
      "entropy": 0.9537039786577225,
      "epoch": 0.5690742886571392,
      "grad_norm": 44.66596603393555,
      "learning_rate": 2.0674766335692445e-07,
      "logits/chosen": 4.303240877933076,
      "logits/rejected": 4.435057485614915,
      "logps/chosen": -321.6051383972168,
      "logps/rejected": -317.53667602539065,
      "loss": 0.5237411499023438,
      "mean_token_accuracy": 0.7187791794538498,
      "num_tokens": 30408848.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.05160765647888184,
      "rewards/margins": 0.9479971334338189,
      "rewards/rejected": -0.9996047995984554,
      "step": 2210
    },
    {
      "entropy": 0.9782517611980438,
      "epoch": 0.5716492854383932,
      "grad_norm": 39.292198181152344,
      "learning_rate": 2.0469568849773453e-07,
      "logits/chosen": 4.094074952160854,
      "logits/rejected": 4.335160989134377,
      "logps/chosen": -288.1838668823242,
      "logps/rejected": -290.0323829650879,
      "loss": 0.4903146743774414,
      "mean_token_accuracy": 0.7082525342702866,
      "num_tokens": 30540736.0,
      "rewards/accuracies": 0.775,
      "rewards/chosen": 0.17911316305398942,
      "rewards/margins": 1.014538937807083,
      "rewards/rejected": -0.835425791144371,
      "step": 2220
    },
    {
      "entropy": 0.9824255883693696,
      "epoch": 0.5742242822196473,
      "grad_norm": 39.250972747802734,
      "learning_rate": 2.026468646176446e-07,
      "logits/chosen": 4.0822044548447,
      "logits/rejected": 4.332966515519744,
      "logps/chosen": -305.91218147277834,
      "logps/rejected": -340.64088401794436,
      "loss": 0.45061264038085935,
      "mean_token_accuracy": 0.7125394314527511,
      "num_tokens": 30682984.0,
      "rewards/accuracies": 0.75625,
      "rewards/chosen": 0.1678448911756277,
      "rewards/margins": 1.1703709587454796,
      "rewards/rejected": -1.0025260582566262,
      "step": 2230
    },
    {
      "entropy": 0.9766992688179016,
      "epoch": 0.5767992790009012,
      "grad_norm": 52.70896530151367,
      "learning_rate": 2.0060133421526027e-07,
      "logits/chosen": 4.443482124148224,
      "logits/rejected": 4.4387098977954285,
      "logps/chosen": -329.52354125976564,
      "logps/rejected": -299.1279350280762,
      "loss": 0.5766659259796143,
      "mean_token_accuracy": 0.7177125632762908,
      "num_tokens": 30819840.0,
      "rewards/accuracies": 0.6625,
      "rewards/chosen": -0.01829192042350769,
      "rewards/margins": 0.9055951349437237,
      "rewards/rejected": -0.9238870710134506,
      "step": 2240
    },
    {
      "entropy": 0.9193742334842682,
      "epoch": 0.5793742757821553,
      "grad_norm": 63.07462692260742,
      "learning_rate": 1.985592395601215e-07,
      "logits/chosen": 4.402515272289742,
      "logits/rejected": 4.686042422271998,
      "logps/chosen": -321.8515800476074,
      "logps/rejected": -332.01871795654296,
      "loss": 0.5270612239837646,
      "mean_token_accuracy": 0.7229076892137527,
      "num_tokens": 30962735.0,
      "rewards/accuracies": 0.75625,
      "rewards/chosen": 0.09906055107712745,
      "rewards/margins": 0.9753070950508118,
      "rewards/rejected": -0.8762465573847293,
      "step": 2250
    },
    {
      "entropy": 1.0063863813877105,
      "epoch": 0.5819492725634093,
      "grad_norm": 55.75368118286133,
      "learning_rate": 1.9652072268280675e-07,
      "logits/chosen": 4.4105326671434835,
      "logits/rejected": 4.474947993035541,
      "logps/chosen": -361.05382080078124,
      "logps/rejected": -325.942626953125,
      "loss": 0.548794174194336,
      "mean_token_accuracy": 0.69999920129776,
      "num_tokens": 31104260.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08069399669766426,
      "rewards/margins": 0.8124750137329102,
      "rewards/rejected": -0.8931690063327551,
      "step": 2260
    },
    {
      "entropy": 0.9078447461128235,
      "epoch": 0.5845242693446633,
      "grad_norm": 71.37114715576172,
      "learning_rate": 1.944859253650552e-07,
      "logits/chosen": 4.3516229551206225,
      "logits/rejected": 4.715358662549993,
      "logps/chosen": -310.53897018432616,
      "logps/rejected": -311.94091567993166,
      "loss": 0.47094364166259767,
      "mean_token_accuracy": 0.7269211143255234,
      "num_tokens": 31250584.0,
      "rewards/accuracies": 0.7875,
      "rewards/chosen": -0.03624087125062943,
      "rewards/margins": 1.1930074781179427,
      "rewards/rejected": -1.2292483642697334,
      "step": 2270
    },
    {
      "entropy": 0.952119767665863,
      "epoch": 0.5870992661259173,
      "grad_norm": 65.18846130371094,
      "learning_rate": 1.9245498912990552e-07,
      "logits/chosen": 4.3513078256927535,
      "logits/rejected": 4.516444060921208,
      "logps/chosen": -320.1900581359863,
      "logps/rejected": -319.03382568359376,
      "loss": 0.4853237152099609,
      "mean_token_accuracy": 0.7136391550302505,
      "num_tokens": 31391197.0,
      "rewards/accuracies": 0.75625,
      "rewards/chosen": -0.00556834451854229,
      "rewards/margins": 1.0650015987455845,
      "rewards/rejected": -1.0705699451267718,
      "step": 2280
    },
    {
      "entropy": 1.0051916390657425,
      "epoch": 0.5896742629071714,
      "grad_norm": 60.125362396240234,
      "learning_rate": 1.9042805523185224e-07,
      "logits/chosen": 4.136806267133233,
      "logits/rejected": 4.373445331875144,
      "logps/chosen": -347.1316879272461,
      "logps/rejected": -330.32265548706056,
      "loss": 0.5287572383880615,
      "mean_token_accuracy": 0.699402442574501,
      "num_tokens": 31525878.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.14854914993047713,
      "rewards/margins": 0.9958145916461945,
      "rewards/rejected": -0.8472654297947884,
      "step": 2290
    },
    {
      "entropy": 0.9312989771366119,
      "epoch": 0.5922492596884253,
      "grad_norm": 47.99911880493164,
      "learning_rate": 1.8840526464702187e-07,
      "logits/chosen": 4.427626339654708,
      "logits/rejected": 4.65078433694862,
      "logps/chosen": -323.1201881408691,
      "logps/rejected": -335.28249282836913,
      "loss": 0.48755817413330077,
      "mean_token_accuracy": 0.7307901710271836,
      "num_tokens": 31669884.0,
      "rewards/accuracies": 0.775,
      "rewards/chosen": 0.07594985216856003,
      "rewards/margins": 1.0695114340633154,
      "rewards/rejected": -0.9935616031289101,
      "step": 2300
    },
    {
      "entropy": 1.0104268401861192,
      "epoch": 0.5948242564696794,
      "grad_norm": 56.197410583496094,
      "learning_rate": 1.8638675806336776e-07,
      "logits/chosen": 4.1933119218725965,
      "logits/rejected": 4.310008726056373,
      "logps/chosen": -382.15055084228516,
      "logps/rejected": -380.93878707885744,
      "loss": 0.4801497936248779,
      "mean_token_accuracy": 0.7114536106586457,
      "num_tokens": 31812649.0,
      "rewards/accuracies": 0.75625,
      "rewards/chosen": 0.029319702088832854,
      "rewards/margins": 1.0714796632528305,
      "rewards/rejected": -1.042159953713417,
      "step": 2310
    },
    {
      "entropy": 1.0495285987854004,
      "epoch": 0.5973992532509335,
      "grad_norm": 55.15418243408203,
      "learning_rate": 1.8437267587088457e-07,
      "logits/chosen": 4.433101498582317,
      "logits/rejected": 4.382495202363422,
      "logps/chosen": -335.66616287231443,
      "logps/rejected": -305.47173233032225,
      "loss": 0.5913334369659424,
      "mean_token_accuracy": 0.6869961947202683,
      "num_tokens": 31955299.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": -0.1284914530813694,
      "rewards/margins": 0.8338379189372063,
      "rewards/rejected": -0.9623293563723564,
      "step": 2320
    },
    {
      "entropy": 0.9694363415241242,
      "epoch": 0.5999742500321874,
      "grad_norm": 74.43938446044922,
      "learning_rate": 1.823631581518446e-07,
      "logits/chosen": 4.342257492087476,
      "logits/rejected": 4.333678647145307,
      "logps/chosen": -306.73903198242186,
      "logps/rejected": -328.825910949707,
      "loss": 0.5228142738342285,
      "mean_token_accuracy": 0.703788360953331,
      "num_tokens": 32088877.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.07906525041908026,
      "rewards/margins": 1.1507192701101303,
      "rewards/rejected": -1.0716539956629276,
      "step": 2330
    },
    {
      "entropy": 0.9773776948451995,
      "epoch": 0.6025492468134415,
      "grad_norm": 62.20151138305664,
      "learning_rate": 1.8035834467105442e-07,
      "logits/chosen": 4.194597717165923,
      "logits/rejected": 4.441323402074882,
      "logps/chosen": -319.6354751586914,
      "logps/rejected": -337.572087097168,
      "loss": 0.5659647941589355,
      "mean_token_accuracy": 0.7077054262161255,
      "num_tokens": 32228732.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": -0.014954057708382606,
      "rewards/margins": 0.9510046035051346,
      "rewards/rejected": -0.9659586466848851,
      "step": 2340
    },
    {
      "entropy": 1.0027844697237014,
      "epoch": 0.6051242435946955,
      "grad_norm": 41.43119430541992,
      "learning_rate": 1.78358374866134e-07,
      "logits/chosen": 4.454848762213653,
      "logits/rejected": 4.433416337835228,
      "logps/chosen": -327.75243911743166,
      "logps/rejected": -332.934854888916,
      "loss": 0.5612287998199463,
      "mean_token_accuracy": 0.7106603384017944,
      "num_tokens": 32374141.0,
      "rewards/accuracies": 0.6625,
      "rewards/chosen": 0.16538103260099887,
      "rewards/margins": 0.9340221837162972,
      "rewards/rejected": -0.768641147762537,
      "step": 2350
    },
    {
      "entropy": 0.9379937499761581,
      "epoch": 0.6076992403759496,
      "grad_norm": 94.86339569091797,
      "learning_rate": 1.763633878378191e-07,
      "logits/chosen": 4.312113700941093,
      "logits/rejected": 4.511808621681159,
      "logps/chosen": -318.45185470581055,
      "logps/rejected": -320.56103057861327,
      "loss": 0.5496462821960449,
      "mean_token_accuracy": 0.7269904494285584,
      "num_tokens": 32504564.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": 0.05075220391154289,
      "rewards/margins": 0.9266988523304462,
      "rewards/rejected": -0.8759466748684644,
      "step": 2360
    },
    {
      "entropy": 0.9562380164861679,
      "epoch": 0.6102742371572035,
      "grad_norm": 39.19052505493164,
      "learning_rate": 1.743735223402861e-07,
      "logits/chosen": 4.421962083003923,
      "logits/rejected": 4.475477171427551,
      "logps/chosen": -309.61843185424806,
      "logps/rejected": -303.66740493774415,
      "loss": 0.5537517547607422,
      "mean_token_accuracy": 0.7129708468914032,
      "num_tokens": 32647109.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": 0.13659467920660973,
      "rewards/margins": 0.8689602486789226,
      "rewards/rejected": -0.7323655590415001,
      "step": 2370
    },
    {
      "entropy": 0.9923003166913986,
      "epoch": 0.6128492339384576,
      "grad_norm": 60.206581115722656,
      "learning_rate": 1.7238891677150157e-07,
      "logits/chosen": 4.403583009165136,
      "logits/rejected": 4.752100390667911,
      "logps/chosen": -316.19840774536135,
      "logps/rejected": -323.16968765258787,
      "loss": 0.48160157203674314,
      "mean_token_accuracy": 0.7170157432556152,
      "num_tokens": 32772750.0,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.09984740689396858,
      "rewards/margins": 1.0032228454947472,
      "rewards/rejected": -0.9033754229545593,
      "step": 2380
    },
    {
      "entropy": 0.9690987765789032,
      "epoch": 0.6154242307197116,
      "grad_norm": 54.384742736816406,
      "learning_rate": 1.7040970916359693e-07,
      "logits/chosen": 4.389778037955121,
      "logits/rejected": 4.460626551878917,
      "logps/chosen": -298.3137832641602,
      "logps/rejected": -292.2226303100586,
      "loss": 0.48703317642211913,
      "mean_token_accuracy": 0.703352552652359,
      "num_tokens": 32896953.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.1597156472504139,
      "rewards/margins": 1.1024471048265696,
      "rewards/rejected": -0.9427314698696136,
      "step": 2390
    },
    {
      "entropy": 0.9835656076669693,
      "epoch": 0.6179992275009656,
      "grad_norm": 85.71637725830078,
      "learning_rate": 1.6843603717326733e-07,
      "logits/chosen": 4.281219507832217,
      "logits/rejected": 4.6274769425241455,
      "logps/chosen": -329.02880783081054,
      "logps/rejected": -329.16113433837893,
      "loss": 0.5334079265594482,
      "mean_token_accuracy": 0.7152191966772079,
      "num_tokens": 33024356.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.21461917832493782,
      "rewards/margins": 1.0890141651034355,
      "rewards/rejected": -0.8743950046598912,
      "step": 2400
    },
    {
      "entropy": 0.9442358911037445,
      "epoch": 0.6205742242822196,
      "grad_norm": 63.576297760009766,
      "learning_rate": 1.664680380721981e-07,
      "logits/chosen": 4.415484036032346,
      "logits/rejected": 4.553295847904644,
      "logps/chosen": -344.47364807128906,
      "logps/rejected": -302.069393157959,
      "loss": 0.5017064094543457,
      "mean_token_accuracy": 0.7172648161649704,
      "num_tokens": 33171454.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.01607281230390072,
      "rewards/margins": 1.0055124536156654,
      "rewards/rejected": -0.9894396156072617,
      "step": 2410
    },
    {
      "entropy": 1.0014096409082414,
      "epoch": 0.6231492210634737,
      "grad_norm": 53.30178451538086,
      "learning_rate": 1.6450584873751723e-07,
      "logits/chosen": 4.5171440294255785,
      "logits/rejected": 4.685752037350361,
      "logps/chosen": -329.97341079711913,
      "logps/rejected": -309.59109725952146,
      "loss": 0.6140494346618652,
      "mean_token_accuracy": 0.701422393321991,
      "num_tokens": 33309587.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.17000230513513087,
      "rewards/margins": 0.6981745153665543,
      "rewards/rejected": -0.5281722396612167,
      "step": 2420
    },
    {
      "entropy": 0.9768936216831208,
      "epoch": 0.6257242178447276,
      "grad_norm": 69.7081069946289,
      "learning_rate": 1.6254960564227484e-07,
      "logits/chosen": 4.329045615259164,
      "logits/rejected": 4.660475225543115,
      "logps/chosen": -324.4511905670166,
      "logps/rejected": -323.71324462890624,
      "loss": 0.49992985725402833,
      "mean_token_accuracy": 0.7213058680295944,
      "num_tokens": 33445123.0,
      "rewards/accuracies": 0.75625,
      "rewards/chosen": 0.13972454741597176,
      "rewards/margins": 1.0301117934286594,
      "rewards/rejected": -0.8903872383758425,
      "step": 2430
    },
    {
      "entropy": 1.019405362010002,
      "epoch": 0.6282992146259817,
      "grad_norm": 54.75837707519531,
      "learning_rate": 1.6059944484595207e-07,
      "logits/chosen": 4.291267243289228,
      "logits/rejected": 4.509141716767159,
      "logps/chosen": -328.6258911132812,
      "logps/rejected": -336.042707824707,
      "loss": 0.5735377788543701,
      "mean_token_accuracy": 0.701374489068985,
      "num_tokens": 33579270.0,
      "rewards/accuracies": 0.7,
      "rewards/chosen": -0.011339339613914489,
      "rewards/margins": 0.7998064674437046,
      "rewards/rejected": -0.8111458018422126,
      "step": 2440
    },
    {
      "entropy": 0.9283707648515701,
      "epoch": 0.6308742114072358,
      "grad_norm": 69.55551147460938,
      "learning_rate": 1.5865550198499715e-07,
      "logits/chosen": 4.602162313008973,
      "logits/rejected": 4.594571587360844,
      "logps/chosen": -310.7761627197266,
      "logps/rejected": -310.3740631103516,
      "loss": 0.5215212821960449,
      "mean_token_accuracy": 0.7308780759572983,
      "num_tokens": 33717844.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.05332138538360596,
      "rewards/margins": 0.9332276426255703,
      "rewards/rejected": -0.8799062594771385,
      "step": 2450
    },
    {
      "entropy": 0.9819339632987976,
      "epoch": 0.6334492081884897,
      "grad_norm": 80.49530029296875,
      "learning_rate": 1.5671791226339232e-07,
      "logits/chosen": 4.500601053193192,
      "logits/rejected": 4.426130239662253,
      "logps/chosen": -326.73095932006834,
      "logps/rejected": -338.1872108459473,
      "loss": 0.5379696846008301,
      "mean_token_accuracy": 0.7148545265197754,
      "num_tokens": 33852296.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.029502155259251595,
      "rewards/margins": 0.88140639513731,
      "rewards/rejected": -0.9109085343778134,
      "step": 2460
    },
    {
      "entropy": 1.000627225637436,
      "epoch": 0.6360242049697438,
      "grad_norm": 51.98258972167969,
      "learning_rate": 1.547868104432501e-07,
      "logits/chosen": 4.514064345831087,
      "logits/rejected": 4.54695116008311,
      "logps/chosen": -359.0768356323242,
      "logps/rejected": -355.0493965148926,
      "loss": 0.5708075523376465,
      "mean_token_accuracy": 0.7088312357664108,
      "num_tokens": 33995837.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": -0.022996275499463082,
      "rewards/margins": 0.8100681878626347,
      "rewards/rejected": -0.8330644458532334,
      "step": 2470
    },
    {
      "entropy": 0.9869578510522843,
      "epoch": 0.6385992017509978,
      "grad_norm": 58.026180267333984,
      "learning_rate": 1.5286233083543997e-07,
      "logits/chosen": 4.404094705198541,
      "logits/rejected": 4.741168195342172,
      "logps/chosen": -318.848078918457,
      "logps/rejected": -329.31809997558594,
      "loss": 0.5148727893829346,
      "mean_token_accuracy": 0.7177086800336838,
      "num_tokens": 34129212.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": 0.0676919974386692,
      "rewards/margins": 1.0266894429922104,
      "rewards/rejected": -0.9589974403381347,
      "step": 2480
    },
    {
      "entropy": 0.9847066342830658,
      "epoch": 0.6411741985322519,
      "grad_norm": 73.6479263305664,
      "learning_rate": 1.5094460729024726e-07,
      "logits/chosen": 4.557787003296725,
      "logits/rejected": 4.52986587523226,
      "logps/chosen": -316.85884857177734,
      "logps/rejected": -306.20886306762696,
      "loss": 0.5156795501708984,
      "mean_token_accuracy": 0.7074246108531952,
      "num_tokens": 34264275.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": -0.027447796985507013,
      "rewards/margins": 1.0159383706748486,
      "rewards/rejected": -1.0433861635625363,
      "step": 2490
    },
    {
      "entropy": 1.0174211353063582,
      "epoch": 0.6437491953135058,
      "grad_norm": 53.14724349975586,
      "learning_rate": 1.4903377318806371e-07,
      "logits/chosen": 4.508990891910876,
      "logits/rejected": 4.518605516985489,
      "logps/chosen": -333.36309204101565,
      "logps/rejected": -317.0201698303223,
      "loss": 0.5784246444702148,
      "mean_token_accuracy": 0.7028531193733215,
      "num_tokens": 34402752.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.05209047831594944,
      "rewards/margins": 0.8696140378713608,
      "rewards/rejected": -0.9217044733464718,
      "step": 2500
    },
    {
      "entropy": 0.9878538995981216,
      "epoch": 0.6463241920947599,
      "grad_norm": 56.932682037353516,
      "learning_rate": 1.4712996143011037e-07,
      "logits/chosen": 4.578728757916737,
      "logits/rejected": 4.4917693545516215,
      "logps/chosen": -340.391495513916,
      "logps/rejected": -313.9547912597656,
      "loss": 0.5982888221740723,
      "mean_token_accuracy": 0.718751209974289,
      "num_tokens": 34547977.0,
      "rewards/accuracies": 0.64375,
      "rewards/chosen": -0.02102447971701622,
      "rewards/margins": 0.8910175286233425,
      "rewards/rejected": -0.9120420113205909,
      "step": 2510
    },
    {
      "entropy": 1.0095184803009034,
      "epoch": 0.6488991888760139,
      "grad_norm": 94.94068908691406,
      "learning_rate": 1.4523330442919411e-07,
      "logits/chosen": 4.567168908759323,
      "logits/rejected": 4.278807339492092,
      "logps/chosen": -350.88896255493165,
      "logps/rejected": -324.29125823974607,
      "loss": 0.584876298904419,
      "mean_token_accuracy": 0.6987872898578644,
      "num_tokens": 34688434.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": 0.01219734326004982,
      "rewards/margins": 0.7425737954676151,
      "rewards/rejected": -0.7303764313459397,
      "step": 2520
    },
    {
      "entropy": 0.9569864004850388,
      "epoch": 0.6514741856572679,
      "grad_norm": 52.566650390625,
      "learning_rate": 1.43343934100499e-07,
      "logits/chosen": 4.308113043827216,
      "logits/rejected": 4.494314852834433,
      "logps/chosen": -341.7305908203125,
      "logps/rejected": -324.85329284667966,
      "loss": 0.45043315887451174,
      "mean_token_accuracy": 0.7243666499853134,
      "num_tokens": 34838666.0,
      "rewards/accuracies": 0.80625,
      "rewards/chosen": 0.11741002500057221,
      "rewards/margins": 1.2841462165117263,
      "rewards/rejected": -1.1667361974716186,
      "step": 2530
    },
    {
      "entropy": 1.00680111348629,
      "epoch": 0.654049182438522,
      "grad_norm": 50.91255569458008,
      "learning_rate": 1.4146198185240974e-07,
      "logits/chosen": 4.45827913044039,
      "logits/rejected": 4.550586306424201,
      "logps/chosen": -360.7131713867187,
      "logps/rejected": -294.3333106994629,
      "loss": 0.5361333370208741,
      "mean_token_accuracy": 0.7095499604940414,
      "num_tokens": 34968051.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.20790192447602748,
      "rewards/margins": 0.9727953381836414,
      "rewards/rejected": -0.7648934207856655,
      "step": 2540
    },
    {
      "entropy": 0.9529173254966736,
      "epoch": 0.656624179219776,
      "grad_norm": 67.677490234375,
      "learning_rate": 1.3958757857737386e-07,
      "logits/chosen": 4.4871638388789705,
      "logits/rejected": 4.6531658406392005,
      "logps/chosen": -298.5674705505371,
      "logps/rejected": -290.7885871887207,
      "loss": 0.5661542415618896,
      "mean_token_accuracy": 0.7199174910783768,
      "num_tokens": 35088183.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": -0.008777842298150063,
      "rewards/margins": 0.8344557493925094,
      "rewards/rejected": -0.8432335741817951,
      "step": 2550
    },
    {
      "entropy": 0.9984112858772278,
      "epoch": 0.65919917600103,
      "grad_norm": 29.420278549194336,
      "learning_rate": 1.3772085464279675e-07,
      "logits/chosen": 4.400277528325536,
      "logits/rejected": 4.468486933165357,
      "logps/chosen": -317.15233001708987,
      "logps/rejected": -320.6111999511719,
      "loss": 0.5059976577758789,
      "mean_token_accuracy": 0.7072679579257966,
      "num_tokens": 35218826.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.10976639427244664,
      "rewards/margins": 1.1474495999515057,
      "rewards/rejected": -1.0376832112669945,
      "step": 2560
    },
    {
      "entropy": 0.955791312456131,
      "epoch": 0.661774172782284,
      "grad_norm": 107.9035873413086,
      "learning_rate": 1.3586193988197453e-07,
      "logits/chosen": 4.595885989595397,
      "logits/rejected": 4.541103714693393,
      "logps/chosen": -340.237361907959,
      "logps/rejected": -327.3079864501953,
      "loss": 0.582268476486206,
      "mean_token_accuracy": 0.7249088078737259,
      "num_tokens": 35358921.0,
      "rewards/accuracies": 0.675,
      "rewards/chosen": -0.051393065601587296,
      "rewards/margins": 0.7945620276033878,
      "rewards/rejected": -0.8459550872445106,
      "step": 2570
    },
    {
      "entropy": 1.0618447273969651,
      "epoch": 0.6643491695635381,
      "grad_norm": 54.83210754394531,
      "learning_rate": 1.3401096358506476e-07,
      "logits/chosen": 4.331693795635436,
      "logits/rejected": 4.644192663326451,
      "logps/chosen": -360.5722297668457,
      "logps/rejected": -333.0255714416504,
      "loss": 0.5570868968963623,
      "mean_token_accuracy": 0.6983685493469238,
      "num_tokens": 35502346.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": -0.15737950913608073,
      "rewards/margins": 0.8818966701626778,
      "rewards/rejected": -1.0392761876806618,
      "step": 2580
    },
    {
      "entropy": 0.9186001747846604,
      "epoch": 0.6669241663447921,
      "grad_norm": 50.079200744628906,
      "learning_rate": 1.3216805449009332e-07,
      "logits/chosen": 4.687191874734405,
      "logits/rejected": 4.603947518986728,
      "logps/chosen": -303.88953018188477,
      "logps/rejected": -309.2432388305664,
      "loss": 0.42603626251220705,
      "mean_token_accuracy": 0.7322681874036789,
      "num_tokens": 35632285.0,
      "rewards/accuracies": 0.76875,
      "rewards/chosen": 0.16260901503264905,
      "rewards/margins": 1.1081829860806465,
      "rewards/rejected": -0.945573964342475,
      "step": 2590
    },
    {
      "entropy": 0.9982148051261902,
      "epoch": 0.6694991631260461,
      "grad_norm": 40.70330810546875,
      "learning_rate": 1.3033334077400044e-07,
      "logits/chosen": 4.402947773957061,
      "logits/rejected": 4.453650887548149,
      "logps/chosen": -337.1904579162598,
      "logps/rejected": -315.55087051391604,
      "loss": 0.511660099029541,
      "mean_token_accuracy": 0.6992412239313126,
      "num_tokens": 35762029.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.027360453456640243,
      "rewards/margins": 1.019322784245014,
      "rewards/rejected": -0.9919623393565417,
      "step": 2600
    },
    {
      "entropy": 1.0451726645231247,
      "epoch": 0.6720741599073001,
      "grad_norm": 63.92889404296875,
      "learning_rate": 1.2850695004372686e-07,
      "logits/chosen": 4.40729617097136,
      "logits/rejected": 4.577327136628008,
      "logps/chosen": -321.3320426940918,
      "logps/rejected": -323.710400390625,
      "loss": 0.4458984375,
      "mean_token_accuracy": 0.7113217651844025,
      "num_tokens": 35895643.0,
      "rewards/accuracies": 0.76875,
      "rewards/chosen": 0.1277332842350006,
      "rewards/margins": 1.0387729123234748,
      "rewards/rejected": -0.9110396340489387,
      "step": 2610
    },
    {
      "entropy": 0.9464722245931625,
      "epoch": 0.6746491566885542,
      "grad_norm": 73.1920394897461,
      "learning_rate": 1.2668900932733695e-07,
      "logits/chosen": 4.404658291124084,
      "logits/rejected": 4.62908634142962,
      "logps/chosen": -319.60567169189454,
      "logps/rejected": -343.53348007202146,
      "loss": 0.5372816562652588,
      "mean_token_accuracy": 0.7348382592201232,
      "num_tokens": 36045193.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": 0.048183007165789604,
      "rewards/margins": 0.9457500413060188,
      "rewards/rejected": -0.8975670389831066,
      "step": 2620
    },
    {
      "entropy": 1.0007650017738343,
      "epoch": 0.6772241534698081,
      "grad_norm": 51.42596435546875,
      "learning_rate": 1.248796450651855e-07,
      "logits/chosen": 4.343561634989639,
      "logits/rejected": 4.435974349810258,
      "logps/chosen": -322.3106376647949,
      "logps/rejected": -290.7184883117676,
      "loss": 0.5228497982025146,
      "mean_token_accuracy": 0.7019443780183792,
      "num_tokens": 36171248.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.00893600434064865,
      "rewards/margins": 0.8154723040759564,
      "rewards/rejected": -0.806536290794611,
      "step": 2630
    },
    {
      "entropy": 0.9472282439470291,
      "epoch": 0.6797991502510622,
      "grad_norm": 62.84988021850586,
      "learning_rate": 1.230789831011223e-07,
      "logits/chosen": 4.436925138350209,
      "logits/rejected": 4.608209699765224,
      "logps/chosen": -323.9646141052246,
      "logps/rejected": -316.40479888916013,
      "loss": 0.49807300567626955,
      "mean_token_accuracy": 0.7247744888067246,
      "num_tokens": 36303477.0,
      "rewards/accuracies": 0.79375,
      "rewards/chosen": 0.08416349403560161,
      "rewards/margins": 1.0327687010169029,
      "rewards/rejected": -0.9486052393913269,
      "step": 2640
    },
    {
      "entropy": 0.9816150844097138,
      "epoch": 0.6823741470323162,
      "grad_norm": 64.08195495605469,
      "learning_rate": 1.2128714867373977e-07,
      "logits/chosen": 4.419836130212663,
      "logits/rejected": 4.61189906946818,
      "logps/chosen": -337.7493865966797,
      "logps/rejected": -317.8284484863281,
      "loss": 0.4829238414764404,
      "mean_token_accuracy": 0.7087979733943939,
      "num_tokens": 36446619.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.10568625628948211,
      "rewards/margins": 1.0961877018213273,
      "rewards/rejected": -0.9905014172196388,
      "step": 2650
    },
    {
      "entropy": 1.009509739279747,
      "epoch": 0.6849491438135702,
      "grad_norm": 50.11562728881836,
      "learning_rate": 1.1950426640766314e-07,
      "logits/chosen": 4.408743922486776,
      "logits/rejected": 4.44753181965117,
      "logps/chosen": -339.89669647216795,
      "logps/rejected": -310.1831184387207,
      "loss": 0.4841722011566162,
      "mean_token_accuracy": 0.7028984099626541,
      "num_tokens": 36578459.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.11314288526773453,
      "rewards/margins": 1.1558299280703068,
      "rewards/rejected": -1.042687040567398,
      "step": 2660
    },
    {
      "entropy": 0.9401783347129822,
      "epoch": 0.6875241405948243,
      "grad_norm": 35.34513854980469,
      "learning_rate": 1.177304603048819e-07,
      "logits/chosen": 4.57718549025906,
      "logits/rejected": 4.888864889147892,
      "logps/chosen": -303.99970321655275,
      "logps/rejected": -301.6977195739746,
      "loss": 0.5017678260803222,
      "mean_token_accuracy": 0.7172846794128418,
      "num_tokens": 36719334.0,
      "rewards/accuracies": 0.7,
      "rewards/chosen": -0.029189274087548255,
      "rewards/margins": 0.9483950551599264,
      "rewards/rejected": -0.977584321796894,
      "step": 2670
    },
    {
      "entropy": 1.0551141589879989,
      "epoch": 0.6900991373760783,
      "grad_norm": 61.1798095703125,
      "learning_rate": 1.1596585373612525e-07,
      "logits/chosen": 4.541497685054158,
      "logits/rejected": 4.65840884700258,
      "logps/chosen": -334.9914184570313,
      "logps/rejected": -346.8951156616211,
      "loss": 0.4910571098327637,
      "mean_token_accuracy": 0.692557942867279,
      "num_tokens": 36849707.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": 0.06038952544331551,
      "rewards/margins": 1.144432232528925,
      "rewards/rejected": -1.0840427078306676,
      "step": 2680
    },
    {
      "entropy": 0.9442234963178635,
      "epoch": 0.6926741341573323,
      "grad_norm": 47.575958251953125,
      "learning_rate": 1.1421056943228233e-07,
      "logits/chosen": 4.404051901571395,
      "logits/rejected": 4.524723351667393,
      "logps/chosen": -332.95044174194334,
      "logps/rejected": -323.7625015258789,
      "loss": 0.5233668327331543,
      "mean_token_accuracy": 0.7174067795276642,
      "num_tokens": 36992329.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.06839351318776607,
      "rewards/margins": 1.0658936589956283,
      "rewards/rejected": -0.9975001350045204,
      "step": 2690
    },
    {
      "entropy": 0.9637559443712235,
      "epoch": 0.6952491309385863,
      "grad_norm": 55.56652069091797,
      "learning_rate": 1.1246472947586494e-07,
      "logits/chosen": 4.281454236228097,
      "logits/rejected": 4.303310600480479,
      "logps/chosen": -345.25632553100587,
      "logps/rejected": -318.4501441955566,
      "loss": 0.5608476638793946,
      "mean_token_accuracy": 0.7196205258369446,
      "num_tokens": 37136230.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": -0.12094516679644585,
      "rewards/margins": 1.0600071348249913,
      "rewards/rejected": -1.1809523344039916,
      "step": 2700
    },
    {
      "entropy": 1.0444719851017,
      "epoch": 0.6978241277198404,
      "grad_norm": 48.556495666503906,
      "learning_rate": 1.1072845529251742e-07,
      "logits/chosen": 4.281288189609699,
      "logits/rejected": 4.4158216345852,
      "logps/chosen": -349.4377815246582,
      "logps/rejected": -323.73373184204104,
      "loss": 0.48833293914794923,
      "mean_token_accuracy": 0.6984021216630936,
      "num_tokens": 37267297.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.1243469189852476,
      "rewards/margins": 1.0905327267944813,
      "rewards/rejected": -0.966185811161995,
      "step": 2710
    },
    {
      "entropy": 1.0229749262332917,
      "epoch": 0.7003991245010944,
      "grad_norm": 75.61956787109375,
      "learning_rate": 1.0900186764257127e-07,
      "logits/chosen": 4.459821607232414,
      "logits/rejected": 4.460600032873114,
      "logps/chosen": -347.7802589416504,
      "logps/rejected": -340.0708801269531,
      "loss": 0.5790718078613282,
      "mean_token_accuracy": 0.7006754904985428,
      "num_tokens": 37406239.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.026463273912668228,
      "rewards/margins": 0.9200003072619438,
      "rewards/rejected": -0.9464636027812958,
      "step": 2720
    },
    {
      "entropy": 0.9510831981897354,
      "epoch": 0.7029741212823484,
      "grad_norm": 63.93190002441406,
      "learning_rate": 1.0728508661264521e-07,
      "logits/chosen": 4.4445433403575905,
      "logits/rejected": 4.43910397449009,
      "logps/chosen": -338.7940216064453,
      "logps/rejected": -314.04123840332034,
      "loss": 0.5228251457214356,
      "mean_token_accuracy": 0.717621025443077,
      "num_tokens": 37531340.0,
      "rewards/accuracies": 0.7,
      "rewards/chosen": 0.23341020233929158,
      "rewards/margins": 0.9731593623757362,
      "rewards/rejected": -0.739749164134264,
      "step": 2730
    },
    {
      "entropy": 0.9506629645824433,
      "epoch": 0.7055491180636024,
      "grad_norm": 55.87628936767578,
      "learning_rate": 1.0557823160729431e-07,
      "logits/chosen": 4.33434736424992,
      "logits/rejected": 4.626212192499031,
      "logps/chosen": -331.46177368164064,
      "logps/rejected": -308.30202560424806,
      "loss": 0.45470657348632815,
      "mean_token_accuracy": 0.716735503077507,
      "num_tokens": 37678597.0,
      "rewards/accuracies": 0.775,
      "rewards/chosen": 0.21226003915071487,
      "rewards/margins": 1.2418378800153733,
      "rewards/rejected": -1.02957783639431,
      "step": 2740
    },
    {
      "entropy": 1.0071748048067093,
      "epoch": 0.7081241148448565,
      "grad_norm": 48.8303337097168,
      "learning_rate": 1.0388142134070424e-07,
      "logits/chosen": 4.286711720057296,
      "logits/rejected": 4.364682946553529,
      "logps/chosen": -330.69200325012207,
      "logps/rejected": -327.5856483459473,
      "loss": 0.5594644546508789,
      "mean_token_accuracy": 0.7034598410129547,
      "num_tokens": 37812907.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": 0.13423703461885453,
      "rewards/margins": 0.7663955822587013,
      "rewards/rejected": -0.6321585506200791,
      "step": 2750
    },
    {
      "entropy": 0.9429391235113144,
      "epoch": 0.7106991116261104,
      "grad_norm": 48.388492584228516,
      "learning_rate": 1.0219477382843464e-07,
      "logits/chosen": 4.194248919370122,
      "logits/rejected": 4.5460081216541575,
      "logps/chosen": -294.94847259521487,
      "logps/rejected": -297.35206146240233,
      "loss": 0.5030807018280029,
      "mean_token_accuracy": 0.7219616591930389,
      "num_tokens": 37939278.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.15515281893312932,
      "rewards/margins": 0.9358277872204781,
      "rewards/rejected": -0.7806749522686005,
      "step": 2760
    },
    {
      "entropy": 0.8929346352815628,
      "epoch": 0.7132741084073645,
      "grad_norm": 63.80548858642578,
      "learning_rate": 1.0051840637921174e-07,
      "logits/chosen": 4.409810811158247,
      "logits/rejected": 4.542762150569493,
      "logps/chosen": -291.65123825073243,
      "logps/rejected": -307.8862419128418,
      "loss": 0.5681679725646973,
      "mean_token_accuracy": 0.7367508620023727,
      "num_tokens": 38070789.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.05143854729831219,
      "rewards/margins": 0.8403750531375408,
      "rewards/rejected": -0.7889365002512931,
      "step": 2770
    },
    {
      "entropy": 0.9484794080257416,
      "epoch": 0.7158491051886186,
      "grad_norm": 36.92988204956055,
      "learning_rate": 9.885243558676822e-08,
      "logits/chosen": 4.419774984410074,
      "logits/rejected": 4.647377253694863,
      "logps/chosen": -306.4446388244629,
      "logps/rejected": -284.5430198669434,
      "loss": 0.5263370513916016,
      "mean_token_accuracy": 0.7101660877466202,
      "num_tokens": 38210686.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": 0.20834870748221873,
      "rewards/margins": 0.9040569610893726,
      "rewards/rejected": -0.6957082599401474,
      "step": 2780
    },
    {
      "entropy": 0.9955869674682617,
      "epoch": 0.7184241019698725,
      "grad_norm": 50.16758346557617,
      "learning_rate": 9.719697732173481e-08,
      "logits/chosen": 4.425591581073951,
      "logits/rejected": 4.527296195863723,
      "logps/chosen": -319.05909194946287,
      "logps/rejected": -336.0451545715332,
      "loss": 0.5405101776123047,
      "mean_token_accuracy": 0.7104849278926849,
      "num_tokens": 38353204.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.17830976098775864,
      "rewards/margins": 0.8981378510594368,
      "rewards/rejected": -0.7198280796408654,
      "step": 2790
    },
    {
      "entropy": 1.0241438031196595,
      "epoch": 0.7209990987511266,
      "grad_norm": 47.5545654296875,
      "learning_rate": 9.555214672358128e-08,
      "logits/chosen": 4.299143100957586,
      "logits/rejected": 4.705484058148562,
      "logps/chosen": -307.576830291748,
      "logps/rejected": -335.4364730834961,
      "loss": 0.4629045963287354,
      "mean_token_accuracy": 0.7091218382120132,
      "num_tokens": 38484000.0,
      "rewards/accuracies": 0.76875,
      "rewards/chosen": 0.32216081991791723,
      "rewards/margins": 1.1764334067702293,
      "rewards/rejected": -0.8542725712060928,
      "step": 2800
    },
    {
      "entropy": 1.0045750170946122,
      "epoch": 0.7235740955323806,
      "grad_norm": 50.36190414428711,
      "learning_rate": 9.391805819260753e-08,
      "logits/chosen": 4.461994885639138,
      "logits/rejected": 4.51211255949246,
      "logps/chosen": -332.54803771972655,
      "logps/rejected": -342.58244705200195,
      "loss": 0.5819994449615479,
      "mean_token_accuracy": 0.7181883543729782,
      "num_tokens": 38628258.0,
      "rewards/accuracies": 0.7125,
      "rewards/chosen": 0.03722733799368143,
      "rewards/margins": 0.823909766972065,
      "rewards/rejected": -0.7866824433207512,
      "step": 2810
    },
    {
      "entropy": 0.9943287044763565,
      "epoch": 0.7261490923136346,
      "grad_norm": 50.188690185546875,
      "learning_rate": 9.229482538198804e-08,
      "logits/chosen": 4.086052531524615,
      "logits/rejected": 4.108102212518812,
      "logps/chosen": -336.1692123413086,
      "logps/rejected": -332.55058059692385,
      "loss": 0.5688528537750244,
      "mean_token_accuracy": 0.7213399589061738,
      "num_tokens": 38770992.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.0018285416066646577,
      "rewards/margins": 0.9121986500918865,
      "rewards/rejected": -0.9103701081126928,
      "step": 2820
    },
    {
      "entropy": 0.9544013321399689,
      "epoch": 0.7287240890948886,
      "grad_norm": 53.01747512817383,
      "learning_rate": 9.068256118986631e-08,
      "logits/chosen": 4.1799197337194895,
      "logits/rejected": 4.455103215239529,
      "logps/chosen": -317.2519187927246,
      "logps/rejected": -294.7378273010254,
      "loss": 0.49945316314697263,
      "mean_token_accuracy": 0.7185042858123779,
      "num_tokens": 38906315.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2399644384160638,
      "rewards/margins": 1.0651536084711553,
      "rewards/rejected": -0.8251891598105431,
      "step": 2830
    },
    {
      "entropy": 0.9642573595046997,
      "epoch": 0.7312990858761427,
      "grad_norm": 92.45647430419922,
      "learning_rate": 8.908137775150257e-08,
      "logits/chosen": 4.303308568983775,
      "logits/rejected": 4.412389834513493,
      "logps/chosen": -339.0303047180176,
      "logps/rejected": -317.9662094116211,
      "loss": 0.5016836166381836,
      "mean_token_accuracy": 0.7174497812986373,
      "num_tokens": 39043108.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.1519692622125149,
      "rewards/margins": 1.0284341365098952,
      "rewards/rejected": -0.876464857161045,
      "step": 2840
    },
    {
      "entropy": 0.9783081382513046,
      "epoch": 0.7338740826573967,
      "grad_norm": 41.27275466918945,
      "learning_rate": 8.749138643147558e-08,
      "logits/chosen": 4.697188761436596,
      "logits/rejected": 4.8487493974469675,
      "logps/chosen": -305.1290481567383,
      "logps/rejected": -299.2451332092285,
      "loss": 0.5368721961975098,
      "mean_token_accuracy": 0.7132978916168213,
      "num_tokens": 39177011.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.1731885824352503,
      "rewards/margins": 0.9777018561959266,
      "rewards/rejected": -0.8045132834464311,
      "step": 2850
    },
    {
      "entropy": 0.992822477221489,
      "epoch": 0.7364490794386507,
      "grad_norm": 55.166175842285156,
      "learning_rate": 8.591269781593574e-08,
      "logits/chosen": 4.345998862675562,
      "logits/rejected": 4.54603522894707,
      "logps/chosen": -323.51059646606444,
      "logps/rejected": -318.03050537109374,
      "loss": 0.48357276916503905,
      "mean_token_accuracy": 0.719045877456665,
      "num_tokens": 39313378.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.19722165130078792,
      "rewards/margins": 1.0652718126773835,
      "rewards/rejected": -0.8680501654744148,
      "step": 2860
    },
    {
      "entropy": 0.9903046548366546,
      "epoch": 0.7390240762199047,
      "grad_norm": 28.31355857849121,
      "learning_rate": 8.434542170491466e-08,
      "logits/chosen": 4.410495406036079,
      "logits/rejected": 4.481941922543696,
      "logps/chosen": -304.29503479003904,
      "logps/rejected": -301.4001434326172,
      "loss": 0.5171587944030762,
      "mean_token_accuracy": 0.7006939500570297,
      "num_tokens": 39443882.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.21424660459160805,
      "rewards/margins": 0.8981669448316097,
      "rewards/rejected": -0.6839203275740147,
      "step": 2870
    },
    {
      "entropy": 1.0298338741064073,
      "epoch": 0.7415990730011588,
      "grad_norm": 67.0123519897461,
      "learning_rate": 8.278966710468822e-08,
      "logits/chosen": 4.201646701813575,
      "logits/rejected": 4.241147471196167,
      "logps/chosen": -345.51076583862306,
      "logps/rejected": -325.53188705444336,
      "loss": 0.5303487777709961,
      "mean_token_accuracy": 0.7124982863664627,
      "num_tokens": 39584784.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.08423602096736431,
      "rewards/margins": 1.0374247565865518,
      "rewards/rejected": -0.9531887352466584,
      "step": 2880
    },
    {
      "entropy": 0.9180865705013275,
      "epoch": 0.7441740697824127,
      "grad_norm": 61.85934066772461,
      "learning_rate": 8.124554222019448e-08,
      "logits/chosen": 4.503551553238841,
      "logits/rejected": 4.840787797658568,
      "logps/chosen": -289.37534484863284,
      "logps/rejected": -303.46933517456057,
      "loss": 0.54837646484375,
      "mean_token_accuracy": 0.7242263734340668,
      "num_tokens": 39725942.0,
      "rewards/accuracies": 0.675,
      "rewards/chosen": 0.1193401824682951,
      "rewards/margins": 0.92861771248281,
      "rewards/rejected": -0.8092775233089924,
      "step": 2890
    },
    {
      "entropy": 1.0399951666593552,
      "epoch": 0.7467490665636668,
      "grad_norm": 81.25881958007812,
      "learning_rate": 7.97131544475085e-08,
      "logits/chosen": 4.212558677519399,
      "logits/rejected": 4.5135824060273055,
      "logps/chosen": -371.32031936645507,
      "logps/rejected": -374.5592987060547,
      "loss": 0.5198248863220215,
      "mean_token_accuracy": 0.7017723619937897,
      "num_tokens": 39881804.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.06453087776899338,
      "rewards/margins": 0.9404610253870487,
      "rewards/rejected": -0.8759301364421844,
      "step": 2900
    },
    {
      "entropy": 0.9375022858381271,
      "epoch": 0.7493240633449209,
      "grad_norm": 62.9113883972168,
      "learning_rate": 7.819261036637284e-08,
      "logits/chosen": 4.441437066640102,
      "logits/rejected": 4.26147197062758,
      "logps/chosen": -330.31504974365237,
      "logps/rejected": -302.4228385925293,
      "loss": 0.5019186019897461,
      "mean_token_accuracy": 0.7240248680114746,
      "num_tokens": 40022680.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.27599499225616453,
      "rewards/margins": 1.134216284379363,
      "rewards/rejected": -0.8582212992012501,
      "step": 2910
    },
    {
      "entropy": 1.0441882610321045,
      "epoch": 0.7518990601261748,
      "grad_norm": 54.067020416259766,
      "learning_rate": 7.668401573278387e-08,
      "logits/chosen": 4.317062533761897,
      "logits/rejected": 4.409713854047424,
      "logps/chosen": -324.02139205932616,
      "logps/rejected": -311.38745193481446,
      "loss": 0.47819151878356936,
      "mean_token_accuracy": 0.6976284444332123,
      "num_tokens": 40151504.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": 0.3253614973276854,
      "rewards/margins": 1.0317629069089889,
      "rewards/rejected": -0.7064014218747616,
      "step": 2920
    },
    {
      "entropy": 0.9620579421520233,
      "epoch": 0.7544740569074289,
      "grad_norm": 57.29512023925781,
      "learning_rate": 7.518747547163773e-08,
      "logits/chosen": 4.529682488991996,
      "logits/rejected": 4.549528768541835,
      "logps/chosen": -341.7662750244141,
      "logps/rejected": -341.3739807128906,
      "loss": 0.4937107086181641,
      "mean_token_accuracy": 0.726236167550087,
      "num_tokens": 40294340.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.03524754866957665,
      "rewards/margins": 1.0567084670066833,
      "rewards/rejected": -1.0214609190821649,
      "step": 2930
    },
    {
      "entropy": 1.0326186656951903,
      "epoch": 0.7570490536886829,
      "grad_norm": 70.87602233886719,
      "learning_rate": 7.370309366943114e-08,
      "logits/chosen": 4.416342597151165,
      "logits/rejected": 4.474957801985486,
      "logps/chosen": -374.77448120117185,
      "logps/rejected": -348.7484405517578,
      "loss": 0.45115070343017577,
      "mean_token_accuracy": 0.7073379307985306,
      "num_tokens": 40442011.0,
      "rewards/accuracies": 0.775,
      "rewards/chosen": 0.2300522394478321,
      "rewards/margins": 1.0959902562201023,
      "rewards/rejected": -0.8659380182623864,
      "step": 2940
    },
    {
      "entropy": 0.9630140721797943,
      "epoch": 0.7596240504699369,
      "grad_norm": 32.93770980834961,
      "learning_rate": 7.22309735670231e-08,
      "logits/chosen": 4.174306564433566,
      "logits/rejected": 4.293997769817925,
      "logps/chosen": -317.42327880859375,
      "logps/rejected": -315.3322067260742,
      "loss": 0.4436497211456299,
      "mean_token_accuracy": 0.7245869845151901,
      "num_tokens": 40576718.0,
      "rewards/accuracies": 0.79375,
      "rewards/chosen": 0.2645474158227444,
      "rewards/margins": 1.1843299344182014,
      "rewards/rejected": -0.9197825267910957,
      "step": 2950
    },
    {
      "entropy": 0.9748166382312775,
      "epoch": 0.7621990472511909,
      "grad_norm": 72.24275207519531,
      "learning_rate": 7.077121755245428e-08,
      "logits/chosen": 4.217123138479009,
      "logits/rejected": 4.486150912960931,
      "logps/chosen": -308.8130683898926,
      "logps/rejected": -313.8214279174805,
      "loss": 0.5102333545684814,
      "mean_token_accuracy": 0.7062664836645126,
      "num_tokens": 40711819.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": -0.007556497305631638,
      "rewards/margins": 1.0142739318311214,
      "rewards/rejected": -1.0218304254114627,
      "step": 2960
    },
    {
      "entropy": 1.053650790452957,
      "epoch": 0.764774044032445,
      "grad_norm": 42.517642974853516,
      "learning_rate": 6.932392715382487e-08,
      "logits/chosen": 4.506357549619956,
      "logits/rejected": 4.419823614093227,
      "logps/chosen": -355.1724250793457,
      "logps/rejected": -311.9959014892578,
      "loss": 0.4585103511810303,
      "mean_token_accuracy": 0.6848949640989304,
      "num_tokens": 40846130.0,
      "rewards/accuracies": 0.75625,
      "rewards/chosen": 0.08447664454579354,
      "rewards/margins": 1.195004227757454,
      "rewards/rejected": -1.1105275854468346,
      "step": 2970
    },
    {
      "entropy": 0.9970977663993835,
      "epoch": 0.767349040813699,
      "grad_norm": 48.33706283569336,
      "learning_rate": 6.788920303223425e-08,
      "logits/chosen": 4.363256771998095,
      "logits/rejected": 4.447415007108594,
      "logps/chosen": -345.1931518554687,
      "logps/rejected": -314.6180160522461,
      "loss": 0.46572027206420896,
      "mean_token_accuracy": 0.7173113703727723,
      "num_tokens": 40984083.0,
      "rewards/accuracies": 0.79375,
      "rewards/chosen": 0.11490394063293934,
      "rewards/margins": 1.0985809415578842,
      "rewards/rejected": -0.9836770124733448,
      "step": 2980
    },
    {
      "entropy": 0.9672432780265808,
      "epoch": 0.769924037594953,
      "grad_norm": 66.26432800292969,
      "learning_rate": 6.646714497477948e-08,
      "logits/chosen": 4.366952443799669,
      "logits/rejected": 4.514437732158646,
      "logps/chosen": -353.2722297668457,
      "logps/rejected": -332.82758178710935,
      "loss": 0.5087109565734863,
      "mean_token_accuracy": 0.7158259689807892,
      "num_tokens": 41129956.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.08387112971395254,
      "rewards/margins": 0.9910045996308326,
      "rewards/rejected": -0.9071334734559059,
      "step": 2990
    },
    {
      "entropy": 0.9817124366760254,
      "epoch": 0.772499034376207,
      "grad_norm": 91.40129089355469,
      "learning_rate": 6.505785188761451e-08,
      "logits/chosen": 4.227975146993117,
      "logits/rejected": 4.198954829654662,
      "logps/chosen": -368.0373992919922,
      "logps/rejected": -342.33693771362306,
      "loss": 0.5445797920227051,
      "mean_token_accuracy": 0.7081363558769226,
      "num_tokens": 41269766.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.050302238762378694,
      "rewards/margins": 0.8330787874758243,
      "rewards/rejected": -0.8833810284733772,
      "step": 3000
    },
    {
      "entropy": 1.0685946702957154,
      "epoch": 0.7750740311574611,
      "grad_norm": 56.52578353881836,
      "learning_rate": 6.36614217890718e-08,
      "logits/chosen": 4.127420726294206,
      "logits/rejected": 4.333874667104852,
      "logps/chosen": -348.59493865966795,
      "logps/rejected": -331.4124214172363,
      "loss": 0.5051251888275147,
      "mean_token_accuracy": 0.7025615155696869,
      "num_tokens": 41399201.0,
      "rewards/accuracies": 0.76875,
      "rewards/chosen": 0.06800611373037099,
      "rewards/margins": 1.1953423842787743,
      "rewards/rejected": -1.1273362621665002,
      "step": 3010
    },
    {
      "entropy": 1.0232778668403626,
      "epoch": 0.777649027938715,
      "grad_norm": 39.64238357543945,
      "learning_rate": 6.227795180284465e-08,
      "logits/chosen": 4.3056687184544575,
      "logits/rejected": 4.124631304956351,
      "logps/chosen": -343.99916610717776,
      "logps/rejected": -295.5423843383789,
      "loss": 0.5074767589569091,
      "mean_token_accuracy": 0.6896465122699738,
      "num_tokens": 41531613.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": 0.15039641335606574,
      "rewards/margins": 0.9849472507834435,
      "rewards/rejected": -0.8345508441329003,
      "step": 3020
    },
    {
      "entropy": 0.9815282195806503,
      "epoch": 0.7802240247199691,
      "grad_norm": 52.85367202758789,
      "learning_rate": 6.090753815123209e-08,
      "logits/chosen": 4.558400550496125,
      "logits/rejected": 4.555732151159805,
      "logps/chosen": -327.8596466064453,
      "logps/rejected": -314.9741668701172,
      "loss": 0.46218223571777345,
      "mean_token_accuracy": 0.716248345375061,
      "num_tokens": 41666673.0,
      "rewards/accuracies": 0.8,
      "rewards/chosen": 0.35434846989810465,
      "rewards/margins": 1.1373900040984153,
      "rewards/rejected": -0.7830415509641171,
      "step": 3030
    },
    {
      "entropy": 0.9949847847223282,
      "epoch": 0.7827990215012232,
      "grad_norm": 53.299171447753906,
      "learning_rate": 5.955027614844682e-08,
      "logits/chosen": 4.394556627696557,
      "logits/rejected": 4.386567171595721,
      "logps/chosen": -299.4707862854004,
      "logps/rejected": -303.38683853149416,
      "loss": 0.518274974822998,
      "mean_token_accuracy": 0.7210689902305603,
      "num_tokens": 41803295.0,
      "rewards/accuracies": 0.76875,
      "rewards/chosen": 0.1871395196765661,
      "rewards/margins": 1.0939487650990487,
      "rewards/rejected": -0.9068092465400696,
      "step": 3040
    },
    {
      "entropy": 0.9936753064393997,
      "epoch": 0.7853740182824771,
      "grad_norm": 72.15467071533203,
      "learning_rate": 5.820626019398534e-08,
      "logits/chosen": 4.268774096198235,
      "logits/rejected": 4.6194692725014495,
      "logps/chosen": -323.48271560668945,
      "logps/rejected": -352.22514190673826,
      "loss": 0.4884005546569824,
      "mean_token_accuracy": 0.7117483109235764,
      "num_tokens": 41946617.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": 0.21016546208411455,
      "rewards/margins": 1.0751794874668121,
      "rewards/rejected": -0.8650140359997749,
      "step": 3050
    },
    {
      "entropy": 1.0037529230117799,
      "epoch": 0.7879490150637312,
      "grad_norm": 70.5625,
      "learning_rate": 5.687558376606291e-08,
      "logits/chosen": 4.464402455004146,
      "logits/rejected": 4.337021500618102,
      "logps/chosen": -375.0096603393555,
      "logps/rejected": -367.5603225708008,
      "loss": 0.4842118263244629,
      "mean_token_accuracy": 0.7070386677980423,
      "num_tokens": 42094420.0,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.17200252264738083,
      "rewards/margins": 1.0302217558026314,
      "rewards/rejected": -0.8582192309200763,
      "step": 3060
    },
    {
      "entropy": 0.9708365321159362,
      "epoch": 0.7905240118449852,
      "grad_norm": 101.7674789428711,
      "learning_rate": 5.555833941511198e-08,
      "logits/chosen": 4.605958277092105,
      "logits/rejected": 4.559032011572001,
      "logps/chosen": -317.2640075683594,
      "logps/rejected": -302.55060958862305,
      "loss": 0.560026741027832,
      "mean_token_accuracy": 0.7196670472621918,
      "num_tokens": 42232440.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.15156788676977156,
      "rewards/margins": 0.9224099405109882,
      "rewards/rejected": -0.7708420552313328,
      "step": 3070
    },
    {
      "entropy": 0.9678906172513961,
      "epoch": 0.7930990086262392,
      "grad_norm": 58.81368637084961,
      "learning_rate": 5.425461875734466e-08,
      "logits/chosen": 4.442958832299348,
      "logits/rejected": 4.563597377460239,
      "logps/chosen": -315.42679290771486,
      "logps/rejected": -327.9312545776367,
      "loss": 0.4371212959289551,
      "mean_token_accuracy": 0.7215264499187469,
      "num_tokens": 42370464.0,
      "rewards/accuracies": 0.7875,
      "rewards/chosen": 0.28482050970196726,
      "rewards/margins": 1.1923205159604549,
      "rewards/rejected": -0.9075000010430813,
      "step": 3080
    },
    {
      "entropy": 1.0114638358354568,
      "epoch": 0.7956740054074932,
      "grad_norm": 58.771854400634766,
      "learning_rate": 5.2964512468381157e-08,
      "logits/chosen": 4.449208595583224,
      "logits/rejected": 4.567686457399292,
      "logps/chosen": -347.89423141479494,
      "logps/rejected": -334.30847320556643,
      "loss": 0.5054470062255859,
      "mean_token_accuracy": 0.7074404180049896,
      "num_tokens": 42518097.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.09104574248194694,
      "rewards/margins": 1.058924862742424,
      "rewards/rejected": -0.9678791239857674,
      "step": 3090
    },
    {
      "entropy": 0.9954713702201843,
      "epoch": 0.7982490021887473,
      "grad_norm": 55.89361572265625,
      "learning_rate": 5.168811027694323e-08,
      "logits/chosen": 4.395756512540025,
      "logits/rejected": 4.416072690094935,
      "logps/chosen": -280.47585067749026,
      "logps/rejected": -279.00662994384766,
      "loss": 0.4241953372955322,
      "mean_token_accuracy": 0.7219798177480697,
      "num_tokens": 42642299.0,
      "rewards/accuracies": 0.7875,
      "rewards/chosen": 0.0713214971125126,
      "rewards/margins": 1.2660470068454743,
      "rewards/rejected": -1.1947255536913872,
      "step": 3100
    },
    {
      "entropy": 0.9094302177429199,
      "epoch": 0.8008239989700013,
      "grad_norm": 50.474430084228516,
      "learning_rate": 5.042550095861281e-08,
      "logits/chosen": 4.322333511113185,
      "logits/rejected": 4.444034928428492,
      "logps/chosen": -305.94030380249023,
      "logps/rejected": -325.72607574462893,
      "loss": 0.5195977210998535,
      "mean_token_accuracy": 0.7327635198831558,
      "num_tokens": 42787119.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0955929946154356,
      "rewards/margins": 0.9432641051709652,
      "rewards/rejected": -1.0388571172952652,
      "step": 3110
    },
    {
      "entropy": 0.9371308833360672,
      "epoch": 0.8033989957512553,
      "grad_norm": 53.76936721801758,
      "learning_rate": 4.9176772329658445e-08,
      "logits/chosen": 4.157149298615895,
      "logits/rejected": 4.363022352815242,
      "logps/chosen": -287.5912223815918,
      "logps/rejected": -296.0829788208008,
      "loss": 0.5081684112548828,
      "mean_token_accuracy": 0.7216660618782044,
      "num_tokens": 42928684.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.134003047272563,
      "rewards/margins": 0.9698007084429264,
      "rewards/rejected": -0.8357976607978344,
      "step": 3120
    },
    {
      "entropy": 0.9710374444723129,
      "epoch": 0.8059739925325093,
      "grad_norm": 52.81174087524414,
      "learning_rate": 4.794201124092659e-08,
      "logits/chosen": 4.387222672277018,
      "logits/rejected": 4.36880727899687,
      "logps/chosen": -339.0060348510742,
      "logps/rejected": -332.5059280395508,
      "loss": 0.4661799430847168,
      "mean_token_accuracy": 0.7248915672302246,
      "num_tokens": 43070782.0,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.14939685836434363,
      "rewards/margins": 1.1781152054667472,
      "rewards/rejected": -1.02871835231781,
      "step": 3130
    },
    {
      "entropy": 1.0249121218919754,
      "epoch": 0.8085489893137634,
      "grad_norm": 72.28135681152344,
      "learning_rate": 4.672130357180168e-08,
      "logits/chosen": 4.145989070533242,
      "logits/rejected": 4.332538270159017,
      "logps/chosen": -330.81416244506835,
      "logps/rejected": -327.6256950378418,
      "loss": 0.6214367389678955,
      "mean_token_accuracy": 0.7014501124620438,
      "num_tokens": 43203719.0,
      "rewards/accuracies": 0.675,
      "rewards/chosen": 0.002490393817424774,
      "rewards/margins": 0.7461536779999733,
      "rewards/rejected": -0.7436633050441742,
      "step": 3140
    },
    {
      "entropy": 0.9359357237815857,
      "epoch": 0.8111239860950173,
      "grad_norm": 59.6107063293457,
      "learning_rate": 4.5514734224233076e-08,
      "logits/chosen": 4.399457422597005,
      "logits/rejected": 4.390425372401519,
      "logps/chosen": -318.7063865661621,
      "logps/rejected": -295.77928161621094,
      "loss": 0.5429286956787109,
      "mean_token_accuracy": 0.7206829756498336,
      "num_tokens": 43342940.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": -0.011263814568519593,
      "rewards/margins": 0.9415080212056637,
      "rewards/rejected": -0.9527718558907509,
      "step": 3150
    },
    {
      "entropy": 0.9799446612596512,
      "epoch": 0.8136989828762714,
      "grad_norm": 54.60009002685547,
      "learning_rate": 4.4322387116829454e-08,
      "logits/chosen": 4.244228730951929,
      "logits/rejected": 4.38494076500641,
      "logps/chosen": -321.30857009887694,
      "logps/rejected": -335.9203987121582,
      "loss": 0.5400273323059082,
      "mean_token_accuracy": 0.7213184833526611,
      "num_tokens": 43477920.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.13054923564195633,
      "rewards/margins": 1.0041513681411742,
      "rewards/rejected": -0.8736021481454372,
      "step": 3160
    },
    {
      "entropy": 1.0019416391849518,
      "epoch": 0.8162739796575255,
      "grad_norm": 60.097293853759766,
      "learning_rate": 4.314434517902271e-08,
      "logits/chosen": 4.187407208641154,
      "logits/rejected": 4.537707368797923,
      "logps/chosen": -319.647412109375,
      "logps/rejected": -298.61428756713866,
      "loss": 0.5090551376342773,
      "mean_token_accuracy": 0.7005850493907928,
      "num_tokens": 43616356.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.032133683934807776,
      "rewards/margins": 0.9810623221099377,
      "rewards/rejected": -0.9489286404103041,
      "step": 3170
    },
    {
      "entropy": 0.9200732558965683,
      "epoch": 0.8188489764387794,
      "grad_norm": 58.14112091064453,
      "learning_rate": 4.198069034530008e-08,
      "logits/chosen": 4.217178553337115,
      "logits/rejected": 4.304576324351252,
      "logps/chosen": -338.6818000793457,
      "logps/rejected": -300.8328262329102,
      "loss": 0.540928840637207,
      "mean_token_accuracy": 0.7179835706949234,
      "num_tokens": 43764948.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.0418201744556427,
      "rewards/margins": 0.8899469636380672,
      "rewards/rejected": -0.8481268070638179,
      "step": 3180
    },
    {
      "entropy": 0.9348589479923248,
      "epoch": 0.8214239732200335,
      "grad_norm": 64.86296844482422,
      "learning_rate": 4.0831503549505016e-08,
      "logits/chosen": 4.202605550969819,
      "logits/rejected": 4.265748399265876,
      "logps/chosen": -319.32573852539065,
      "logps/rejected": -280.73924789428713,
      "loss": 0.5497491359710693,
      "mean_token_accuracy": 0.7174855619668961,
      "num_tokens": 43893288.0,
      "rewards/accuracies": 0.7,
      "rewards/chosen": 0.1283643525093794,
      "rewards/margins": 0.948226423561573,
      "rewards/rejected": -0.8198620736598968,
      "step": 3190
    },
    {
      "entropy": 0.9861269295215607,
      "epoch": 0.8239989700012875,
      "grad_norm": 51.75646209716797,
      "learning_rate": 3.969686471920852e-08,
      "logits/chosen": 4.299380680822329,
      "logits/rejected": 4.30685423706581,
      "logps/chosen": -350.1937675476074,
      "logps/rejected": -338.32622756958006,
      "loss": 0.4989126682281494,
      "mean_token_accuracy": 0.7143979758024216,
      "num_tokens": 44043167.0,
      "rewards/accuracies": 0.76875,
      "rewards/chosen": 0.06968585439026356,
      "rewards/margins": 1.104652463644743,
      "rewards/rejected": -1.0349665828049184,
      "step": 3200
    },
    {
      "entropy": 0.9652820855379105,
      "epoch": 0.8265739667825415,
      "grad_norm": 85.18624114990234,
      "learning_rate": 3.857685277015005e-08,
      "logits/chosen": 4.324221564423723,
      "logits/rejected": 4.543678736975258,
      "logps/chosen": -343.1850891113281,
      "logps/rejected": -327.8295722961426,
      "loss": 0.5779080867767334,
      "mean_token_accuracy": 0.7182841390371323,
      "num_tokens": 44185012.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": -0.023977766185998915,
      "rewards/margins": 0.8196044005453587,
      "rewards/rejected": -0.8435821406543255,
      "step": 3210
    },
    {
      "entropy": 0.9091089338064193,
      "epoch": 0.8291489635637955,
      "grad_norm": 63.109886169433594,
      "learning_rate": 3.747154560074861e-08,
      "logits/chosen": 4.429191732784413,
      "logits/rejected": 4.195509661221952,
      "logps/chosen": -299.11571502685547,
      "logps/rejected": -277.0503555297852,
      "loss": 0.4867547512054443,
      "mean_token_accuracy": 0.7231446325778961,
      "num_tokens": 44326839.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": 0.18592803850769996,
      "rewards/margins": 1.041901146247983,
      "rewards/rejected": -0.855973094701767,
      "step": 3220
    },
    {
      "entropy": 0.9785544514656067,
      "epoch": 0.8317239603450496,
      "grad_norm": 50.95145034790039,
      "learning_rate": 3.638102008668517e-08,
      "logits/chosen": 4.432440797187778,
      "logits/rejected": 4.528259753053593,
      "logps/chosen": -327.1904724121094,
      "logps/rejected": -310.68684844970704,
      "loss": 0.43732595443725586,
      "mean_token_accuracy": 0.710511577129364,
      "num_tokens": 44458796.0,
      "rewards/accuracies": 0.79375,
      "rewards/chosen": 0.2300154183059931,
      "rewards/margins": 1.1821377247571945,
      "rewards/rejected": -0.9521223038434983,
      "step": 3230
    },
    {
      "entropy": 0.9591111004352569,
      "epoch": 0.8342989571263036,
      "grad_norm": 63.192962646484375,
      "learning_rate": 3.530535207555527e-08,
      "logits/chosen": 4.2694179077771075,
      "logits/rejected": 4.588908212903034,
      "logps/chosen": -311.78385467529296,
      "logps/rejected": -322.0230285644531,
      "loss": 0.44493680000305175,
      "mean_token_accuracy": 0.7172895401716233,
      "num_tokens": 44596694.0,
      "rewards/accuracies": 0.76875,
      "rewards/chosen": 0.17520832270383835,
      "rewards/margins": 1.1929794155061244,
      "rewards/rejected": -1.017771103978157,
      "step": 3240
    },
    {
      "entropy": 0.9898055881261826,
      "epoch": 0.8368739539075576,
      "grad_norm": 80.0720443725586,
      "learning_rate": 3.42446163815942e-08,
      "logits/chosen": 4.414742148305107,
      "logits/rejected": 4.522412258383963,
      "logps/chosen": -355.3474433898926,
      "logps/rejected": -336.7364868164062,
      "loss": 0.4767888069152832,
      "mean_token_accuracy": 0.7169474422931671,
      "num_tokens": 44729978.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": -0.005838570743799209,
      "rewards/margins": 1.0999968815594912,
      "rewards/rejected": -1.1058354690670966,
      "step": 3250
    },
    {
      "entropy": 1.0163926750421524,
      "epoch": 0.8394489506888116,
      "grad_norm": 39.93063735961914,
      "learning_rate": 3.319888678047364e-08,
      "logits/chosen": 4.1840992037675155,
      "logits/rejected": 4.442582675435789,
      "logps/chosen": -325.13705291748045,
      "logps/rejected": -338.8698677062988,
      "loss": 0.4550458908081055,
      "mean_token_accuracy": 0.7010392010211944,
      "num_tokens": 44864417.0,
      "rewards/accuracies": 0.79375,
      "rewards/chosen": 0.14707549009472132,
      "rewards/margins": 1.2349995456635952,
      "rewards/rejected": -1.0879240721464156,
      "step": 3260
    },
    {
      "entropy": 1.0307422816753387,
      "epoch": 0.8420239474700657,
      "grad_norm": 54.57400131225586,
      "learning_rate": 3.2168236004169873e-08,
      "logits/chosen": 4.306451708066383,
      "logits/rejected": 4.507960301818548,
      "logps/chosen": -371.58898849487304,
      "logps/rejected": -344.5499099731445,
      "loss": 0.5361446380615235,
      "mean_token_accuracy": 0.7136694967746735,
      "num_tokens": 44997890.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": -0.03218022361397743,
      "rewards/margins": 0.9815607778728008,
      "rewards/rejected": -1.0137410029768943,
      "step": 3270
    },
    {
      "entropy": 1.021285566687584,
      "epoch": 0.8445989442513196,
      "grad_norm": 53.717735290527344,
      "learning_rate": 3.115273573590574e-08,
      "logits/chosen": 4.250017985967512,
      "logits/rejected": 4.205670235926026,
      "logps/chosen": -358.8350463867188,
      "logps/rejected": -320.0092658996582,
      "loss": 0.511791181564331,
      "mean_token_accuracy": 0.6992344230413436,
      "num_tokens": 45139221.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.07899544127285481,
      "rewards/margins": 1.021796802431345,
      "rewards/rejected": -0.9428013868629932,
      "step": 3280
    },
    {
      "entropy": 1.0297605782747268,
      "epoch": 0.8471739410325737,
      "grad_norm": 54.42903137207031,
      "learning_rate": 3.015245660516472e-08,
      "logits/chosen": 4.475107655950782,
      "logits/rejected": 4.368668733903215,
      "logps/chosen": -341.9279327392578,
      "logps/rejected": -352.9533218383789,
      "loss": 0.4986253261566162,
      "mean_token_accuracy": 0.6980555474758148,
      "num_tokens": 45283979.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": 0.17783180978149177,
      "rewards/margins": 0.9337885938584805,
      "rewards/rejected": -0.7559567965567112,
      "step": 3290
    },
    {
      "entropy": 1.0129867225885392,
      "epoch": 0.8497489378138278,
      "grad_norm": 58.75669860839844,
      "learning_rate": 2.91674681827786e-08,
      "logits/chosen": 3.8199001294357067,
      "logits/rejected": 4.099848671849001,
      "logps/chosen": -340.3991340637207,
      "logps/rejected": -338.3979187011719,
      "loss": 0.49118432998657224,
      "mean_token_accuracy": 0.7043730199337006,
      "num_tokens": 45420691.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15180539079010485,
      "rewards/margins": 1.1536284662783145,
      "rewards/rejected": -1.0018231131136417,
      "step": 3300
    },
    {
      "entropy": 1.0250534802675246,
      "epoch": 0.8523239345950817,
      "grad_norm": 58.28948211669922,
      "learning_rate": 2.8197838976088823e-08,
      "logits/chosen": 4.361926243219319,
      "logits/rejected": 4.179079459291225,
      "logps/chosen": -410.66667861938475,
      "logps/rejected": -378.39041061401366,
      "loss": 0.5394617080688476,
      "mean_token_accuracy": 0.6966198533773422,
      "num_tokens": 45580397.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.12192578278481961,
      "rewards/margins": 1.0251854434609413,
      "rewards/rejected": -0.9032596364617348,
      "step": 3310
    },
    {
      "entropy": 0.9336719363927841,
      "epoch": 0.8548989313763358,
      "grad_norm": 53.06043243408203,
      "learning_rate": 2.7243636424181426e-08,
      "logits/chosen": 4.435972725142554,
      "logits/rejected": 4.655780388727168,
      "logps/chosen": -311.94235076904295,
      "logps/rejected": -322.9733139038086,
      "loss": 0.4260276794433594,
      "mean_token_accuracy": 0.7199991792440414,
      "num_tokens": 45718068.0,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.15638988018035888,
      "rewards/margins": 1.2638778954744339,
      "rewards/rejected": -1.1074880246073007,
      "step": 3320
    },
    {
      "entropy": 1.085283598303795,
      "epoch": 0.8574739281575898,
      "grad_norm": 70.15311431884766,
      "learning_rate": 2.6304926893196745e-08,
      "logits/chosen": 4.27457033464077,
      "logits/rejected": 4.373139219715083,
      "logps/chosen": -358.29911346435546,
      "logps/rejected": -335.08835525512694,
      "loss": 0.5034091949462891,
      "mean_token_accuracy": 0.6905516624450684,
      "num_tokens": 45850667.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.16071342900395394,
      "rewards/margins": 1.0567363321781158,
      "rewards/rejected": -0.8960228696465492,
      "step": 3330
    },
    {
      "entropy": 1.0259439915418624,
      "epoch": 0.8600489249388438,
      "grad_norm": 50.04902267456055,
      "learning_rate": 2.5381775671713816e-08,
      "logits/chosen": 4.323888256404123,
      "logits/rejected": 4.3945349140457,
      "logps/chosen": -333.95324935913084,
      "logps/rejected": -317.8748207092285,
      "loss": 0.5272077560424805,
      "mean_token_accuracy": 0.7033091694116592,
      "num_tokens": 45984107.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": -0.007355622760951519,
      "rewards/margins": 0.9154392205178737,
      "rewards/rejected": -0.9227948397397995,
      "step": 3340
    },
    {
      "entropy": 0.9725571423768997,
      "epoch": 0.8626239217200978,
      "grad_norm": 57.20749282836914,
      "learning_rate": 2.4474246966208812e-08,
      "logits/chosen": 4.438619917633764,
      "logits/rejected": 4.5021580303400475,
      "logps/chosen": -344.06627960205077,
      "logps/rejected": -323.3831077575684,
      "loss": 0.4868340492248535,
      "mean_token_accuracy": 0.7153410166501999,
      "num_tokens": 46126452.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.028528289496898653,
      "rewards/margins": 1.1265264317393302,
      "rewards/rejected": -1.0979981660842895,
      "step": 3350
    },
    {
      "entropy": 0.9808278143405914,
      "epoch": 0.8651989185013519,
      "grad_norm": 58.57217788696289,
      "learning_rate": 2.3582403896589968e-08,
      "logits/chosen": 4.571170439503101,
      "logits/rejected": 4.5146572611487485,
      "logps/chosen": -348.4826530456543,
      "logps/rejected": -341.46127395629884,
      "loss": 0.513688039779663,
      "mean_token_accuracy": 0.7054894149303437,
      "num_tokens": 46271576.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2376721866428852,
      "rewards/margins": 1.028487491607666,
      "rewards/rejected": -0.7908153183758259,
      "step": 3360
    },
    {
      "entropy": 0.9694880276918412,
      "epoch": 0.867773915282606,
      "grad_norm": 80.97119140625,
      "learning_rate": 2.270630849180735e-08,
      "logits/chosen": 4.648432265594115,
      "logits/rejected": 4.546168698768557,
      "logps/chosen": -330.9306983947754,
      "logps/rejected": -308.4755485534668,
      "loss": 0.49057564735412595,
      "mean_token_accuracy": 0.7102002441883087,
      "num_tokens": 46417059.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": 0.2763052903115749,
      "rewards/margins": 1.0597978465259075,
      "rewards/rejected": -0.7834925360977649,
      "step": 3370
    },
    {
      "entropy": 1.0251123160123825,
      "epoch": 0.8703489120638599,
      "grad_norm": 60.645111083984375,
      "learning_rate": 2.184602168553848e-08,
      "logits/chosen": 4.418524502990852,
      "logits/rejected": 4.563836301864508,
      "logps/chosen": -349.13284454345705,
      "logps/rejected": -311.8704147338867,
      "loss": 0.564413595199585,
      "mean_token_accuracy": 0.7005513310432434,
      "num_tokens": 46548689.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.10500337593257428,
      "rewards/margins": 0.8836007565259933,
      "rewards/rejected": -0.9886041209101677,
      "step": 3380
    },
    {
      "entropy": 0.9626199662685394,
      "epoch": 0.872923908845114,
      "grad_norm": 52.24079513549805,
      "learning_rate": 2.10016033119505e-08,
      "logits/chosen": 4.275510975573473,
      "logits/rejected": 4.560188272155659,
      "logps/chosen": -325.59469375610354,
      "logps/rejected": -311.76687850952146,
      "loss": 0.4976241111755371,
      "mean_token_accuracy": 0.7235046684741974,
      "num_tokens": 46688467.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": -0.009001258015632629,
      "rewards/margins": 1.0198740661144257,
      "rewards/rejected": -1.0288753151893615,
      "step": 3390
    },
    {
      "entropy": 0.9624796003103256,
      "epoch": 0.875498905626368,
      "grad_norm": 60.422061920166016,
      "learning_rate": 2.0173112101538575e-08,
      "logits/chosen": 4.165599870005346,
      "logits/rejected": 4.334379304055487,
      "logps/chosen": -355.1679443359375,
      "logps/rejected": -329.28182525634764,
      "loss": 0.524689769744873,
      "mean_token_accuracy": 0.7226759999990463,
      "num_tokens": 46824018.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.01133049912750721,
      "rewards/margins": 0.9631065726280212,
      "rewards/rejected": -0.9517760708928108,
      "step": 3400
    },
    {
      "entropy": 0.9720148354768753,
      "epoch": 0.878073902407622,
      "grad_norm": 51.23833084106445,
      "learning_rate": 1.936060567704101e-08,
      "logits/chosen": 4.417957401809131,
      "logits/rejected": 4.420553470939984,
      "logps/chosen": -347.849609375,
      "logps/rejected": -325.4596824645996,
      "loss": 0.5499835968017578,
      "mean_token_accuracy": 0.7164855569601059,
      "num_tokens": 46962450.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": 0.06175786927342415,
      "rewards/margins": 1.0169452384114266,
      "rewards/rejected": -0.9551873594522476,
      "step": 3410
    },
    {
      "entropy": 0.9205372929573059,
      "epoch": 0.880648899188876,
      "grad_norm": 71.48031616210938,
      "learning_rate": 1.8564140549431784e-08,
      "logits/chosen": 4.543263375398614,
      "logits/rejected": 4.34733416576674,
      "logps/chosen": -323.281729888916,
      "logps/rejected": -320.5871925354004,
      "loss": 0.5773813247680664,
      "mean_token_accuracy": 0.7166509211063385,
      "num_tokens": 47108489.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.012704336643218994,
      "rewards/margins": 0.8792961195111275,
      "rewards/rejected": -0.8920004740357399,
      "step": 3420
    },
    {
      "entropy": 1.0096068501472473,
      "epoch": 0.8832238959701301,
      "grad_norm": 39.13214874267578,
      "learning_rate": 1.778377211398968e-08,
      "logits/chosen": 4.290272215409019,
      "logits/rejected": 4.223242211919704,
      "logps/chosen": -360.93024291992185,
      "logps/rejected": -334.8608901977539,
      "loss": 0.43523283004760743,
      "mean_token_accuracy": 0.7145869106054306,
      "num_tokens": 47245415.0,
      "rewards/accuracies": 0.7875,
      "rewards/chosen": 0.17010074853897095,
      "rewards/margins": 1.2077522069215774,
      "rewards/rejected": -1.0376514531672,
      "step": 3430
    },
    {
      "entropy": 0.9463884651660919,
      "epoch": 0.885798892751384,
      "grad_norm": 58.91367721557617,
      "learning_rate": 1.7019554646445834e-08,
      "logits/chosen": 4.213345909251523,
      "logits/rejected": 4.4840705751237895,
      "logps/chosen": -325.1591438293457,
      "logps/rejected": -360.0714385986328,
      "loss": 0.4720927238464355,
      "mean_token_accuracy": 0.7402132153511047,
      "num_tokens": 47386884.0,
      "rewards/accuracies": 0.8,
      "rewards/chosen": 0.17252297177910805,
      "rewards/margins": 1.1690418511629104,
      "rewards/rejected": -0.9965188939124345,
      "step": 3440
    },
    {
      "entropy": 0.9820081353187561,
      "epoch": 0.8883738895326381,
      "grad_norm": 70.7621078491211,
      "learning_rate": 1.6271541299208685e-08,
      "logits/chosen": 4.105262020171894,
      "logits/rejected": 4.634721768465438,
      "logps/chosen": -344.67479553222654,
      "logps/rejected": -343.77735366821287,
      "loss": 0.45592031478881834,
      "mean_token_accuracy": 0.7207370430231095,
      "num_tokens": 47530905.0,
      "rewards/accuracies": 0.75625,
      "rewards/chosen": 0.15923903696238995,
      "rewards/margins": 1.182755758613348,
      "rewards/rejected": -1.0235167093575002,
      "step": 3450
    },
    {
      "entropy": 1.001122486591339,
      "epoch": 0.8909488863138921,
      "grad_norm": 60.54716873168945,
      "learning_rate": 1.5539784097667124e-08,
      "logits/chosen": 4.40741366964353,
      "logits/rejected": 4.270748510304506,
      "logps/chosen": -306.87539291381836,
      "logps/rejected": -299.2044052124023,
      "loss": 0.5292521476745605,
      "mean_token_accuracy": 0.7106813669204712,
      "num_tokens": 47659337.0,
      "rewards/accuracies": 0.70625,
      "rewards/chosen": 0.09808586463332176,
      "rewards/margins": 0.9288386881351471,
      "rewards/rejected": -0.8307528175413609,
      "step": 3460
    },
    {
      "entropy": 0.9732944667339325,
      "epoch": 0.8935238830951461,
      "grad_norm": 48.38511657714844,
      "learning_rate": 1.4824333936572037e-08,
      "logits/chosen": 4.468958133684264,
      "logits/rejected": 4.732916791050794,
      "logps/chosen": -304.51147232055666,
      "logps/rejected": -310.21005630493164,
      "loss": 0.5139917373657227,
      "mean_token_accuracy": 0.6996112436056137,
      "num_tokens": 47782251.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.17072734758257865,
      "rewards/margins": 1.0207956433296204,
      "rewards/rejected": -0.850068312138319,
      "step": 3470
    },
    {
      "entropy": 0.9817721247673035,
      "epoch": 0.8960988798764001,
      "grad_norm": 69.75738525390625,
      "learning_rate": 1.41252405764965e-08,
      "logits/chosen": 4.303265819710651,
      "logits/rejected": 4.364314668182876,
      "logps/chosen": -354.1113571166992,
      "logps/rejected": -362.1471939086914,
      "loss": 0.494260311126709,
      "mean_token_accuracy": 0.7172562003135681,
      "num_tokens": 47931522.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.0932394809089601,
      "rewards/margins": 1.0632715754210949,
      "rewards/rejected": -0.9700321093201637,
      "step": 3480
    },
    {
      "entropy": 0.9759364396333694,
      "epoch": 0.8986738766576542,
      "grad_norm": 54.06985855102539,
      "learning_rate": 1.3442552640374943e-08,
      "logits/chosen": 4.454683544464034,
      "logits/rejected": 4.838835310257229,
      "logps/chosen": -325.03923416137695,
      "logps/rejected": -336.8667266845703,
      "loss": 0.46573600769042967,
      "mean_token_accuracy": 0.7211115062236786,
      "num_tokens": 48064681.0,
      "rewards/accuracies": 0.75625,
      "rewards/chosen": 0.0038886144757270813,
      "rewards/margins": 1.223842778801918,
      "rewards/rejected": -1.219954188168049,
      "step": 3490
    },
    {
      "entropy": 0.9556342899799347,
      "epoch": 0.9012488734389082,
      "grad_norm": 60.828941345214844,
      "learning_rate": 1.2776317610121296e-08,
      "logits/chosen": 4.4151058022555665,
      "logits/rejected": 4.379737999278982,
      "logps/chosen": -344.0136268615723,
      "logps/rejected": -304.6720855712891,
      "loss": 0.5218825817108155,
      "mean_token_accuracy": 0.7084827989339828,
      "num_tokens": 48196983.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.16754340901970863,
      "rewards/margins": 1.1194391429424286,
      "rewards/rejected": -0.9518957525491715,
      "step": 3500
    },
    {
      "entropy": 1.016376093029976,
      "epoch": 0.9038238702201622,
      "grad_norm": 38.34528350830078,
      "learning_rate": 1.2126581823326487e-08,
      "logits/chosen": 4.517608908474623,
      "logits/rejected": 4.287675331926841,
      "logps/chosen": -353.27395095825193,
      "logps/rejected": -344.9620536804199,
      "loss": 0.47306151390075685,
      "mean_token_accuracy": 0.7100744724273682,
      "num_tokens": 48334195.0,
      "rewards/accuracies": 0.76875,
      "rewards/chosen": 0.15619846247136593,
      "rewards/margins": 1.1464665360748767,
      "rewards/rejected": -0.9902680486440658,
      "step": 3510
    },
    {
      "entropy": 0.9969549298286438,
      "epoch": 0.9063988670014163,
      "grad_norm": 76.98277282714844,
      "learning_rate": 1.1493390470035752e-08,
      "logits/chosen": 4.47879155288048,
      "logits/rejected": 4.561442948216003,
      "logps/chosen": -340.20881576538085,
      "logps/rejected": -316.81405181884764,
      "loss": 0.5304606437683106,
      "mean_token_accuracy": 0.7083680987358093,
      "num_tokens": 48464269.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.08661839216947556,
      "rewards/margins": 0.9748988553881646,
      "rewards/rejected": -0.8882804535329342,
      "step": 3520
    },
    {
      "entropy": 0.9985833793878556,
      "epoch": 0.9089738637826703,
      "grad_norm": 57.09160614013672,
      "learning_rate": 1.0876787589605502e-08,
      "logits/chosen": 4.438816131902436,
      "logits/rejected": 4.630601574751163,
      "logps/chosen": -326.38246154785156,
      "logps/rejected": -330.35162963867185,
      "loss": 0.5265772819519043,
      "mean_token_accuracy": 0.7096292585134506,
      "num_tokens": 48597852.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.12409729398787021,
      "rewards/margins": 0.9895732261240482,
      "rewards/rejected": -0.865475931763649,
      "step": 3530
    },
    {
      "entropy": 0.9311624348163605,
      "epoch": 0.9115488605639243,
      "grad_norm": 31.453582763671875,
      "learning_rate": 1.0276816067640277e-08,
      "logits/chosen": 4.354476711818544,
      "logits/rejected": 4.563276009580368,
      "logps/chosen": -297.76134796142577,
      "logps/rejected": -302.59604721069337,
      "loss": 0.4661102294921875,
      "mean_token_accuracy": 0.7178372591733932,
      "num_tokens": 48733326.0,
      "rewards/accuracies": 0.75625,
      "rewards/chosen": 0.053084198385477066,
      "rewards/margins": 1.0801889322698117,
      "rewards/rejected": -1.0271047309041024,
      "step": 3540
    },
    {
      "entropy": 1.0048164397478103,
      "epoch": 0.9141238573451783,
      "grad_norm": 54.727439880371094,
      "learning_rate": 9.693517633010157e-09,
      "logits/chosen": 4.331816697040276,
      "logits/rejected": 4.619338770355471,
      "logps/chosen": -323.12854995727537,
      "logps/rejected": -349.59267578125,
      "loss": 0.45178980827331544,
      "mean_token_accuracy": 0.7198001354932785,
      "num_tokens": 48879817.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.1579263649880886,
      "rewards/margins": 1.2303193509578705,
      "rewards/rejected": -1.0723929926753044,
      "step": 3550
    },
    {
      "entropy": 1.0108634173870086,
      "epoch": 0.9166988541264324,
      "grad_norm": 55.493743896484375,
      "learning_rate": 9.126932854948255e-09,
      "logits/chosen": 4.160852975325641,
      "logits/rejected": 4.183551900899551,
      "logps/chosen": -344.4482650756836,
      "logps/rejected": -323.087296295166,
      "loss": 0.5426519393920899,
      "mean_token_accuracy": 0.6971057683229447,
      "num_tokens": 49017367.0,
      "rewards/accuracies": 0.75625,
      "rewards/chosen": 0.11907950565218925,
      "rewards/margins": 0.898277985304594,
      "rewards/rejected": -0.7791984729468823,
      "step": 3560
    },
    {
      "entropy": 0.9394759550690651,
      "epoch": 0.9192738509076863,
      "grad_norm": 66.28225708007812,
      "learning_rate": 8.57710114022922e-09,
      "logits/chosen": 4.451378819490435,
      "logits/rejected": 4.237397623162968,
      "logps/chosen": -331.27843322753904,
      "logps/rejected": -334.40370483398436,
      "loss": 0.4559895992279053,
      "mean_token_accuracy": 0.7293369919061661,
      "num_tokens": 49174005.0,
      "rewards/accuracies": 0.775,
      "rewards/chosen": 0.27012583091855047,
      "rewards/margins": 1.190965287387371,
      "rewards/rejected": -0.92083944901824,
      "step": 3570
    },
    {
      "entropy": 0.976487186551094,
      "epoch": 0.9218488476889404,
      "grad_norm": 84.45362091064453,
      "learning_rate": 8.044060730428431e-09,
      "logits/chosen": 4.495778418241548,
      "logits/rejected": 4.430111787511347,
      "logps/chosen": -330.96579513549807,
      "logps/rejected": -310.9513671875,
      "loss": 0.5034812927246094,
      "mean_token_accuracy": 0.7043195247650147,
      "num_tokens": 49309238.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.06998340710997582,
      "rewards/margins": 0.9363695189356804,
      "rewards/rejected": -0.8663861125707626,
      "step": 3580
    },
    {
      "entropy": 0.9788063317537308,
      "epoch": 0.9244238444701944,
      "grad_norm": 42.2854118347168,
      "learning_rate": 7.527848699262151e-09,
      "logits/chosen": 4.376404988570172,
      "logits/rejected": 4.37388951762519,
      "logps/chosen": -334.96368713378905,
      "logps/rejected": -303.0596351623535,
      "loss": 0.5014265060424805,
      "mean_token_accuracy": 0.7028478592634201,
      "num_tokens": 49444791.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.06081862226128578,
      "rewards/margins": 1.1488137885928154,
      "rewards/rejected": -1.0879951477050782,
      "step": 3590
    },
    {
      "entropy": 0.9703697711229324,
      "epoch": 0.9269988412514485,
      "grad_norm": 63.97004318237305,
      "learning_rate": 7.028500950009036e-09,
      "logits/chosen": 4.33694045197637,
      "logits/rejected": 4.370983334664355,
      "logps/chosen": -337.7773941040039,
      "logps/rejected": -343.25316162109374,
      "loss": 0.47454023361206055,
      "mean_token_accuracy": 0.7206103503704071,
      "num_tokens": 49588105.0,
      "rewards/accuracies": 0.7625,
      "rewards/chosen": 0.12851075902581216,
      "rewards/margins": 1.0958582788705826,
      "rewards/rejected": -0.9673475250601768,
      "step": 3600
    },
    {
      "entropy": 0.9552771568298339,
      "epoch": 0.9295738380327024,
      "grad_norm": 74.13584899902344,
      "learning_rate": 6.546052213013076e-09,
      "logits/chosen": 4.549494020329994,
      "logits/rejected": 4.766378175997125,
      "logps/chosen": -322.0286994934082,
      "logps/rejected": -304.0044807434082,
      "loss": 0.493131160736084,
      "mean_token_accuracy": 0.7168387413024903,
      "num_tokens": 49725681.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.12400502972304821,
      "rewards/margins": 0.9972663760185242,
      "rewards/rejected": -0.873261358588934,
      "step": 3610
    },
    {
      "entropy": 0.9899664372205734,
      "epoch": 0.9321488348139565,
      "grad_norm": 73.20947265625,
      "learning_rate": 6.08053604326797e-09,
      "logits/chosen": 4.3790757163409,
      "logits/rejected": 4.53567903067453,
      "logps/chosen": -368.3445587158203,
      "logps/rejected": -335.86344680786135,
      "loss": 0.5364912986755371,
      "mean_token_accuracy": 0.7091458439826965,
      "num_tokens": 49867742.0,
      "rewards/accuracies": 0.75625,
      "rewards/chosen": 0.019294209219515322,
      "rewards/margins": 0.9568890571594239,
      "rewards/rejected": -0.9375948514789343,
      "step": 3620
    },
    {
      "entropy": 0.9530099391937256,
      "epoch": 0.9347238315952106,
      "grad_norm": 48.23020935058594,
      "learning_rate": 5.631984818083329e-09,
      "logits/chosen": 4.296055677285347,
      "logits/rejected": 4.332563173146445,
      "logps/chosen": -330.5657104492187,
      "logps/rejected": -337.90441665649416,
      "loss": 0.502886962890625,
      "mean_token_accuracy": 0.7261953383684159,
      "num_tokens": 50020610.0,
      "rewards/accuracies": 0.73125,
      "rewards/chosen": 0.011554064974188805,
      "rewards/margins": 0.980589747056365,
      "rewards/rejected": -0.9690356459468603,
      "step": 3630
    },
    {
      "entropy": 1.016004478931427,
      "epoch": 0.9372988283764645,
      "grad_norm": 26.548263549804688,
      "learning_rate": 5.2004297348328675e-09,
      "logits/chosen": 4.448092568149213,
      "logits/rejected": 4.512511965425527,
      "logps/chosen": -342.29409942626955,
      "logps/rejected": -327.07233200073244,
      "loss": 0.5192561626434327,
      "mean_token_accuracy": 0.6933602273464203,
      "num_tokens": 50167159.0,
      "rewards/accuracies": 0.75625,
      "rewards/chosen": -0.028628817386925222,
      "rewards/margins": 0.975167042016983,
      "rewards/rejected": -1.0037958763539792,
      "step": 3640
    },
    {
      "entropy": 1.037110298871994,
      "epoch": 0.9398738251577186,
      "grad_norm": 49.208045959472656,
      "learning_rate": 4.785900808784521e-09,
      "logits/chosen": 4.18404633139661,
      "logits/rejected": 4.366859724127451,
      "logps/chosen": -368.90018463134766,
      "logps/rejected": -326.96028289794924,
      "loss": 0.5454651832580566,
      "mean_token_accuracy": 0.6996820390224456,
      "num_tokens": 50305415.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.1265437588095665,
      "rewards/margins": 0.8717910453677178,
      "rewards/rejected": -0.7452472664415837,
      "step": 3650
    },
    {
      "entropy": 0.9307460784912109,
      "epoch": 0.9424488219389726,
      "grad_norm": 57.491458892822266,
      "learning_rate": 4.38842687101279e-09,
      "logits/chosen": 4.498533801183553,
      "logits/rejected": 4.700642188877437,
      "logps/chosen": -334.06893768310545,
      "logps/rejected": -319.5280303955078,
      "loss": 0.5305843830108643,
      "mean_token_accuracy": 0.7245637029409409,
      "num_tokens": 50440896.0,
      "rewards/accuracies": 0.75625,
      "rewards/chosen": 0.10404398180544376,
      "rewards/margins": 1.0015343960374594,
      "rewards/rejected": -0.8974904328584671,
      "step": 3660
    },
    {
      "entropy": 0.9172339409589767,
      "epoch": 0.9450238187202266,
      "grad_norm": 49.77804183959961,
      "learning_rate": 4.008035566393675e-09,
      "logits/chosen": 4.25483115385353,
      "logits/rejected": 4.390901937743865,
      "logps/chosen": -317.6829383850098,
      "logps/rejected": -300.2950981140137,
      "loss": 0.5132992744445801,
      "mean_token_accuracy": 0.721070921421051,
      "num_tokens": 50582442.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.06545577980577946,
      "rewards/margins": 0.9651219345629215,
      "rewards/rejected": -0.8996661596000195,
      "step": 3670
    },
    {
      "entropy": 0.9238002628087998,
      "epoch": 0.9475988155014806,
      "grad_norm": 56.991695404052734,
      "learning_rate": 3.6447533516816843e-09,
      "logits/chosen": 4.512138319031409,
      "logits/rejected": 4.6444846471796835,
      "logps/chosen": -300.3995414733887,
      "logps/rejected": -277.4478889465332,
      "loss": 0.5094088077545166,
      "mean_token_accuracy": 0.7250993430614472,
      "num_tokens": 50718730.0,
      "rewards/accuracies": 0.75625,
      "rewards/chosen": 0.17520229630172252,
      "rewards/margins": 0.9895223274827003,
      "rewards/rejected": -0.8143200561404228,
      "step": 3680
    },
    {
      "entropy": 0.9735976099967957,
      "epoch": 0.9501738122827347,
      "grad_norm": 52.92755889892578,
      "learning_rate": 3.298605493669976e-09,
      "logits/chosen": 4.401222899972965,
      "logits/rejected": 4.386323304332022,
      "logps/chosen": -317.9237892150879,
      "logps/rejected": -314.15701446533205,
      "loss": 0.5146839141845703,
      "mean_token_accuracy": 0.7224712401628495,
      "num_tokens": 50853694.0,
      "rewards/accuracies": 0.7375,
      "rewards/chosen": 0.12512148544192314,
      "rewards/margins": 1.0661357514560224,
      "rewards/rejected": -0.941014252603054,
      "step": 3690
    },
    {
      "entropy": 1.0029818892478943,
      "epoch": 0.9527488090639886,
      "grad_norm": 47.47979736328125,
      "learning_rate": 2.969616067432873e-09,
      "logits/chosen": 4.289798602571775,
      "logits/rejected": 4.290125592710071,
      "logps/chosen": -332.7917869567871,
      "logps/rejected": -309.6506996154785,
      "loss": 0.5282728195190429,
      "mean_token_accuracy": 0.7061104476451874,
      "num_tokens": 50973757.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.04582103714346886,
      "rewards/margins": 1.0365157380700112,
      "rewards/rejected": -0.990694697573781,
      "step": 3700
    },
    {
      "entropy": 0.948711308836937,
      "epoch": 0.9553238058452427,
      "grad_norm": 59.510738372802734,
      "learning_rate": 2.65780795465137e-09,
      "logits/chosen": 4.399845970240033,
      "logits/rejected": 4.513711278582631,
      "logps/chosen": -344.8980438232422,
      "logps/rejected": -307.91698532104493,
      "loss": 0.5410758018493652,
      "mean_token_accuracy": 0.7102200150489807,
      "num_tokens": 51118168.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": -0.2040230341255665,
      "rewards/margins": 0.9633007615804672,
      "rewards/rejected": -1.1673237793147564,
      "step": 3710
    },
    {
      "entropy": 0.9567266225814819,
      "epoch": 0.9578988026264967,
      "grad_norm": 65.30461120605469,
      "learning_rate": 2.3632028420218496e-09,
      "logits/chosen": 4.403461993684977,
      "logits/rejected": 4.558946817995839,
      "logps/chosen": -352.7233604431152,
      "logps/rejected": -305.87541580200195,
      "loss": 0.5773183345794678,
      "mean_token_accuracy": 0.7121285676956177,
      "num_tokens": 51256978.0,
      "rewards/accuracies": 0.7,
      "rewards/chosen": 0.035476422682404515,
      "rewards/margins": 0.8842684037983417,
      "rewards/rejected": -0.8487919729202986,
      "step": 3720
    },
    {
      "entropy": 1.0510099142789842,
      "epoch": 0.9604737994077508,
      "grad_norm": 26.84271240234375,
      "learning_rate": 2.0858212197476245e-09,
      "logits/chosen": 4.372463580494778,
      "logits/rejected": 4.265729635275705,
      "logps/chosen": -324.15354919433594,
      "logps/rejected": -302.15199890136716,
      "loss": 0.49859132766723635,
      "mean_token_accuracy": 0.7022114306688308,
      "num_tokens": 51385761.0,
      "rewards/accuracies": 0.76875,
      "rewards/chosen": -0.035042554885149,
      "rewards/margins": 1.1020322252064942,
      "rewards/rejected": -1.1370747864246369,
      "step": 3730
    },
    {
      "entropy": 0.994208711385727,
      "epoch": 0.9630487961890047,
      "grad_norm": 46.86901092529297,
      "learning_rate": 1.8256823801137689e-09,
      "logits/chosen": 4.404450478702302,
      "logits/rejected": 4.57243674071521,
      "logps/chosen": -300.2155632019043,
      "logps/rejected": -299.6177520751953,
      "loss": 0.46028990745544435,
      "mean_token_accuracy": 0.7044767260551452,
      "num_tokens": 51516979.0,
      "rewards/accuracies": 0.7875,
      "rewards/chosen": 0.23469365537166595,
      "rewards/margins": 1.0552187602967025,
      "rewards/rejected": -0.8205250978469849,
      "step": 3740
    },
    {
      "entropy": 0.9790441781282425,
      "epoch": 0.9656237929702588,
      "grad_norm": 51.8381233215332,
      "learning_rate": 1.582804416145417e-09,
      "logits/chosen": 4.391877838478205,
      "logits/rejected": 4.334116422860771,
      "logps/chosen": -325.6937026977539,
      "logps/rejected": -335.8347755432129,
      "loss": 0.5244534492492676,
      "mean_token_accuracy": 0.7070640385150909,
      "num_tokens": 51647982.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": 0.10406715907156468,
      "rewards/margins": 1.0737947836518287,
      "rewards/rejected": -0.9697276167571545,
      "step": 3750
    },
    {
      "entropy": 1.053725364804268,
      "epoch": 0.9681987897515129,
      "grad_norm": 39.69020462036133,
      "learning_rate": 1.3572042203493784e-09,
      "logits/chosen": 4.288184999106501,
      "logits/rejected": 4.723794248653045,
      "logps/chosen": -301.32601737976074,
      "logps/rejected": -288.46766204833983,
      "loss": 0.5159613609313964,
      "mean_token_accuracy": 0.7011086165904998,
      "num_tokens": 51773767.0,
      "rewards/accuracies": 0.775,
      "rewards/chosen": -0.0047888275235891346,
      "rewards/margins": 0.9299458310008049,
      "rewards/rejected": -0.9347346603870392,
      "step": 3760
    },
    {
      "entropy": 0.9579876095056534,
      "epoch": 0.9707737865327668,
      "grad_norm": 59.244327545166016,
      "learning_rate": 1.1488974835390796e-09,
      "logits/chosen": 4.175309925483745,
      "logits/rejected": 4.1302942537954745,
      "logps/chosen": -332.2213508605957,
      "logps/rejected": -316.0274543762207,
      "loss": 0.5204493045806885,
      "mean_token_accuracy": 0.6993423879146576,
      "num_tokens": 51915427.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.01833348087966442,
      "rewards/margins": 1.0724851697683335,
      "rewards/rejected": -1.0541516847908496,
      "step": 3770
    },
    {
      "entropy": 0.9610406279563903,
      "epoch": 0.9733487833140209,
      "grad_norm": 46.04349136352539,
      "learning_rate": 9.57898693743464e-10,
      "logits/chosen": 4.1796459032978195,
      "logits/rejected": 4.298540422448867,
      "logps/chosen": -337.83121490478516,
      "logps/rejected": -350.1724571228027,
      "loss": 0.4208075523376465,
      "mean_token_accuracy": 0.7275057882070541,
      "num_tokens": 52055506.0,
      "rewards/accuracies": 0.76875,
      "rewards/chosen": 0.03156646024435759,
      "rewards/margins": 1.250686925649643,
      "rewards/rejected": -1.2191204607486725,
      "step": 3780
    },
    {
      "entropy": 0.9695363134145737,
      "epoch": 0.9759237800952749,
      "grad_norm": 62.886592864990234,
      "learning_rate": 7.842211351991312e-10,
      "logits/chosen": 4.404529722887416,
      "logits/rejected": 4.35932479335686,
      "logps/chosen": -335.45074768066405,
      "logps/rejected": -331.40769653320314,
      "loss": 0.48799610137939453,
      "mean_token_accuracy": 0.7236330389976502,
      "num_tokens": 52199784.0,
      "rewards/accuracies": 0.775,
      "rewards/chosen": -0.024450248479843138,
      "rewards/margins": 1.0116318702697753,
      "rewards/rejected": -1.0360821306705474,
      "step": 3790
    },
    {
      "entropy": 1.002945750951767,
      "epoch": 0.9784987768765289,
      "grad_norm": 55.1344108581543,
      "learning_rate": 6.278768874265206e-10,
      "logits/chosen": 4.345139753849429,
      "logits/rejected": 4.478437201822584,
      "logps/chosen": -299.3524765014648,
      "logps/rejected": -308.5686767578125,
      "loss": 0.5122725009918213,
      "mean_token_accuracy": 0.7084631592035293,
      "num_tokens": 52332543.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.05897114500403404,
      "rewards/margins": 1.132245208323002,
      "rewards/rejected": -1.0732740998268127,
      "step": 3800
    },
    {
      "entropy": 0.9791542381048203,
      "epoch": 0.9810737736577829,
      "grad_norm": 74.19060516357422,
      "learning_rate": 4.888768243898056e-10,
      "logits/chosen": 4.233951982196901,
      "logits/rejected": 4.480447268588881,
      "logps/chosen": -290.825382232666,
      "logps/rejected": -293.841047668457,
      "loss": 0.5422379493713378,
      "mean_token_accuracy": 0.7104681402444839,
      "num_tokens": 52467596.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": 0.043087200820446016,
      "rewards/margins": 1.0256297551095486,
      "rewards/rejected": -0.9825425572693348,
      "step": 3810
    },
    {
      "entropy": 0.9377125799655914,
      "epoch": 0.983648770439037,
      "grad_norm": 62.60303497314453,
      "learning_rate": 3.6723061374036025e-10,
      "logits/chosen": 4.597417417132064,
      "logits/rejected": 4.760054064041833,
      "logps/chosen": -293.8340728759766,
      "logps/rejected": -323.3628387451172,
      "loss": 0.6000535011291503,
      "mean_token_accuracy": 0.7310346335172653,
      "num_tokens": 52603104.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": -0.01746976636350155,
      "rewards/margins": 0.8278767816722393,
      "rewards/rejected": -0.8453465715050698,
      "step": 3820
    },
    {
      "entropy": 0.9405255407094956,
      "epoch": 0.9862237672202909,
      "grad_norm": 69.44036102294922,
      "learning_rate": 2.629467161446852e-10,
      "logits/chosen": 4.497785698829776,
      "logits/rejected": 4.675584302140631,
      "logps/chosen": -315.19013671875,
      "logps/rejected": -322.6870307922363,
      "loss": 0.5892306327819824,
      "mean_token_accuracy": 0.7260134309530258,
      "num_tokens": 52745037.0,
      "rewards/accuracies": 0.69375,
      "rewards/chosen": 0.04895561598241329,
      "rewards/margins": 0.8657831616699696,
      "rewards/rejected": -0.8168275523930788,
      "step": 3830
    },
    {
      "entropy": 0.9618756115436554,
      "epoch": 0.988798764001545,
      "grad_norm": 76.69972229003906,
      "learning_rate": 1.7603238469571234e-10,
      "logits/chosen": 4.24194299584001,
      "logits/rejected": 4.270564125145848,
      "logps/chosen": -352.28253479003905,
      "logps/rejected": -338.86647338867186,
      "loss": 0.5200666427612305,
      "mean_token_accuracy": 0.7175153642892838,
      "num_tokens": 52888306.0,
      "rewards/accuracies": 0.725,
      "rewards/chosen": 0.06362154483795165,
      "rewards/margins": 1.0577692627906798,
      "rewards/rejected": -0.994147739931941,
      "step": 3840
    },
    {
      "entropy": 0.9540875136852265,
      "epoch": 0.991373760782799,
      "grad_norm": 72.23999786376953,
      "learning_rate": 1.0649366440837471e-10,
      "logits/chosen": 4.449182385583951,
      "logits/rejected": 4.436637852350745,
      "logps/chosen": -391.4891944885254,
      "logps/rejected": -323.86412811279297,
      "loss": 0.5273003578186035,
      "mean_token_accuracy": 0.7144930273294449,
      "num_tokens": 53031726.0,
      "rewards/accuracies": 0.74375,
      "rewards/chosen": 0.18170544244349002,
      "rewards/margins": 1.0253052242100238,
      "rewards/rejected": -0.8435997664928436,
      "step": 3850
    },
    {
      "entropy": 0.9514907240867615,
      "epoch": 0.9939487575640531,
      "grad_norm": 53.7625617980957,
      "learning_rate": 5.433539179930391e-11,
      "logits/chosen": 4.684322845812066,
      "logits/rejected": 4.2598464409633285,
      "logps/chosen": -322.4320526123047,
      "logps/rejected": -303.11545791625974,
      "loss": 0.5720946311950683,
      "mean_token_accuracy": 0.7100042164325714,
      "num_tokens": 53161230.0,
      "rewards/accuracies": 0.7,
      "rewards/chosen": -0.11039275005459785,
      "rewards/margins": 0.9645901948213578,
      "rewards/rejected": -1.0749829187989235,
      "step": 3860
    },
    {
      "entropy": 0.9525843232870101,
      "epoch": 0.996523754345307,
      "grad_norm": 60.94110870361328,
      "learning_rate": 1.9561194550293815e-11,
      "logits/chosen": 4.319067716941498,
      "logits/rejected": 4.458056856677681,
      "logps/chosen": -330.46614227294924,
      "logps/rejected": -335.6721809387207,
      "loss": 0.4807633399963379,
      "mean_token_accuracy": 0.7251676887273788,
      "num_tokens": 53304079.0,
      "rewards/accuracies": 0.79375,
      "rewards/chosen": 0.16729226494207977,
      "rewards/margins": 1.125135461986065,
      "rewards/rejected": -0.9578431934118271,
      "step": 3870
    },
    {
      "entropy": 0.9905207127332687,
      "epoch": 0.9990987511265611,
      "grad_norm": 51.55181121826172,
      "learning_rate": 2.1734912560855688e-12,
      "logits/chosen": 4.26814043785083,
      "logits/rejected": 4.257808624133493,
      "logps/chosen": -362.62685546875,
      "logps/rejected": -311.6571701049805,
      "loss": 0.563578462600708,
      "mean_token_accuracy": 0.7050950795412063,
      "num_tokens": 53435910.0,
      "rewards/accuracies": 0.675,
      "rewards/chosen": -0.00768631212413311,
      "rewards/margins": 0.872562188655138,
      "rewards/rejected": -0.8802485037595034,
      "step": 3880
    },
    {
      "entropy": 0.9510993446622577,
      "epoch": 1.0,
      "logits/chosen": 4.121521583799508,
      "logits/rejected": 3.973326885699311,
      "logps/chosen": -317.1263449532645,
      "logps/rejected": -297.800528390067,
      "mean_token_accuracy": 0.7126511675970895,
      "num_tokens": 53480717.0,
      "rewards/accuracies": 0.8392857142857143,
      "rewards/chosen": 0.34361560216971804,
      "rewards/margins": 1.3156322070530482,
      "rewards/rejected": -0.9720166155270168,
      "step": 3884,
      "total_flos": 1.6967093886910464e+17,
      "train_loss": 0.538786339802796,
      "train_runtime": 81210.4656,
      "train_samples_per_second": 0.765,
      "train_steps_per_second": 0.048
    }
  ],
  "logging_steps": 10,
  "max_steps": 3884,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 1.6967093886910464e+17,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}