{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.5089058524173028,
  "eval_steps": 500,
  "global_step": 100,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "entropy": 0.167724609375,
      "epoch": 0.005089058524173028,
      "grad_norm": 63.83088684082031,
      "learning_rate": 5e-06,
      "logits/chosen": -1.1478307765991018,
      "logits/rejected": -1.1531979437304007,
      "logps/chosen": -230.5,
      "logps/rejected": -250.5,
      "loss": 0.726806640625,
      "mean_token_accuracy": 0.9508575797080994,
      "num_tokens": 387774.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0360107421875,
      "rewards/margins": 0.007598876953125,
      "rewards/rejected": -0.04305267333984375,
      "step": 1
    },
    {
      "entropy": 0.1708984375,
      "epoch": 0.010178117048346057,
      "grad_norm": 58.35269546508789,
      "learning_rate": 4.974619289340102e-06,
      "logits/chosen": -1.1465309061137527,
      "logits/rejected": -1.1435811743754352,
      "logps/chosen": -202.75,
      "logps/rejected": -220.75,
      "loss": 0.73284912109375,
      "mean_token_accuracy": 0.9518170654773712,
      "num_tokens": 796045.0,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.01251220703125,
      "rewards/margins": -0.01416015625,
      "rewards/rejected": 0.00146484375,
      "step": 2
    },
    {
      "entropy": 0.1708984375,
      "epoch": 0.015267175572519083,
      "grad_norm": 52.742557525634766,
      "learning_rate": 4.949238578680203e-06,
      "logits/chosen": -1.22388826039798,
      "logits/rejected": -1.2534275052969308,
      "logps/chosen": -217.0,
      "logps/rejected": -210.75,
      "loss": 0.695587158203125,
      "mean_token_accuracy": 0.9499112516641617,
      "num_tokens": 1158499.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.03204345703125,
      "rewards/margins": 0.07501220703125,
      "rewards/rejected": -0.1068115234375,
      "step": 3
    },
    {
      "entropy": 0.175537109375,
      "epoch": 0.020356234096692113,
      "grad_norm": 58.555728912353516,
      "learning_rate": 4.923857868020305e-06,
      "logits/chosen": -1.0887703188820197,
      "logits/rejected": -1.110711324029513,
      "logps/chosen": -247.5,
      "logps/rejected": -225.25,
      "loss": 0.7191162109375,
      "mean_token_accuracy": 0.9449715912342072,
      "num_tokens": 1526178.0,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0124664306640625,
      "rewards/margins": 0.05133056640625,
      "rewards/rejected": -0.0640869140625,
      "step": 4
    },
    {
      "entropy": 0.181884765625,
      "epoch": 0.02544529262086514,
      "grad_norm": 57.25611114501953,
      "learning_rate": 4.898477157360406e-06,
      "logits/chosen": -1.1241738300809683,
      "logits/rejected": -1.1185368980659924,
      "logps/chosen": -197.25,
      "logps/rejected": -204.5,
      "loss": 0.746917724609375,
      "mean_token_accuracy": 0.9447457045316696,
      "num_tokens": 1898739.0,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.036865234375,
      "rewards/margins": 0.0240478515625,
      "rewards/rejected": -0.06103515625,
      "step": 5
    },
    {
      "entropy": 0.172119140625,
      "epoch": 0.030534351145038167,
      "grad_norm": 55.632781982421875,
      "learning_rate": 4.873096446700508e-06,
      "logits/chosen": -1.118042467736593,
      "logits/rejected": -1.1188858755620001,
      "logps/chosen": -195.5,
      "logps/rejected": -213.5,
      "loss": 0.706695556640625,
      "mean_token_accuracy": 0.9515382200479507,
      "num_tokens": 2280846.0,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0267333984375,
      "rewards/margins": 0.051971435546875,
      "rewards/rejected": -0.07904052734375,
      "step": 6
    },
    {
      "entropy": 0.165283203125,
      "epoch": 0.035623409669211195,
      "grad_norm": 58.1217155456543,
      "learning_rate": 4.84771573604061e-06,
      "logits/chosen": -1.2056221529623334,
      "logits/rejected": -1.1698524733043854,
      "logps/chosen": -203.75,
      "logps/rejected": -237.5,
      "loss": 0.778106689453125,
      "mean_token_accuracy": 0.9536973237991333,
      "num_tokens": 2661565.0,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": -0.047119140625,
      "rewards/margins": -0.0155029296875,
      "rewards/rejected": -0.031982421875,
      "step": 7
    },
    {
      "entropy": 0.16015625,
      "epoch": 0.04071246819338423,
      "grad_norm": 57.468605041503906,
      "learning_rate": 4.822335025380711e-06,
      "logits/chosen": -1.285398958255684,
      "logits/rejected": -1.2651885553215019,
      "logps/chosen": -185.25,
      "logps/rejected": -195.0,
      "loss": 0.76959228515625,
      "mean_token_accuracy": 0.9562991708517075,
      "num_tokens": 3011121.0,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.07479095458984375,
      "rewards/margins": -0.06903076171875,
      "rewards/rejected": -0.0052642822265625,
      "step": 8
    },
    {
      "entropy": 0.177978515625,
      "epoch": 0.04580152671755725,
      "grad_norm": 61.59230041503906,
      "learning_rate": 4.796954314720812e-06,
      "logits/chosen": -1.1463901211868202,
      "logits/rejected": -1.151384163231262,
      "logps/chosen": -262.0,
      "logps/rejected": -263.0,
      "loss": 0.77764892578125,
      "mean_token_accuracy": 0.9490418434143066,
      "num_tokens": 3364659.0,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0624542236328125,
      "rewards/margins": -0.07977294921875,
      "rewards/rejected": 0.017181396484375,
      "step": 9
    },
    {
      "entropy": 0.17041015625,
      "epoch": 0.05089058524173028,
      "grad_norm": 66.9396743774414,
      "learning_rate": 4.771573604060914e-06,
      "logits/chosen": -1.1312320566718426,
      "logits/rejected": -1.114105189953177,
      "logps/chosen": -246.25,
      "logps/rejected": -253.5,
      "loss": 0.784912109375,
      "mean_token_accuracy": 0.9495462626218796,
      "num_tokens": 3756722.0,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.11962890625,
      "rewards/margins": -0.0777587890625,
      "rewards/rejected": -0.0416259765625,
      "step": 10
    },
    {
      "entropy": 0.17041015625,
      "epoch": 0.05597964376590331,
      "grad_norm": 60.69718933105469,
      "learning_rate": 4.746192893401016e-06,
      "logits/chosen": -1.2082123033021486,
      "logits/rejected": -1.240414757271401,
      "logps/chosen": -206.75,
      "logps/rejected": -253.0,
      "loss": 0.77130126953125,
      "mean_token_accuracy": 0.9514396339654922,
      "num_tokens": 4150502.0,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.122711181640625,
      "rewards/margins": -0.0660400390625,
      "rewards/rejected": -0.056396484375,
      "step": 11
    },
    {
      "entropy": 0.166015625,
      "epoch": 0.061068702290076333,
      "grad_norm": 68.75598907470703,
      "learning_rate": 4.7208121827411175e-06,
      "logits/chosen": -1.1497054731535286,
      "logits/rejected": -1.1323881415685357,
      "logps/chosen": -233.0,
      "logps/rejected": -214.0,
      "loss": 0.81524658203125,
      "mean_token_accuracy": 0.9499669224023819,
      "num_tokens": 4548017.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.14825439453125,
      "rewards/margins": -0.148681640625,
      "rewards/rejected": 0.0008544921875,
      "step": 12
    },
    {
      "entropy": 0.171142578125,
      "epoch": 0.06615776081424936,
      "grad_norm": 54.791534423828125,
      "learning_rate": 4.695431472081219e-06,
      "logits/chosen": -1.1963053139896285,
      "logits/rejected": -1.164336637055035,
      "logps/chosen": -204.0,
      "logps/rejected": -226.5,
      "loss": 0.71929931640625,
      "mean_token_accuracy": 0.952528104186058,
      "num_tokens": 4902525.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0484619140625,
      "rewards/margins": 0.0706787109375,
      "rewards/rejected": -0.1187744140625,
      "step": 13
    },
    {
      "entropy": 0.165771484375,
      "epoch": 0.07124681933842239,
      "grad_norm": 55.923667907714844,
      "learning_rate": 4.67005076142132e-06,
      "logits/chosen": -1.227218462140038,
      "logits/rejected": -1.2252739974581663,
      "logps/chosen": -204.0,
      "logps/rejected": -245.75,
      "loss": 0.681884765625,
      "mean_token_accuracy": 0.9524703174829483,
      "num_tokens": 5247776.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.017791748046875,
      "rewards/margins": 0.07401275634765625,
      "rewards/rejected": -0.056304931640625,
      "step": 14
    },
    {
      "entropy": 0.16796875,
      "epoch": 0.07633587786259542,
      "grad_norm": 54.7180290222168,
      "learning_rate": 4.644670050761422e-06,
      "logits/chosen": -1.1794452967378677,
      "logits/rejected": -1.213616516672937,
      "logps/chosen": -208.75,
      "logps/rejected": -199.5,
      "loss": 0.7034912109375,
      "mean_token_accuracy": 0.9460672587156296,
      "num_tokens": 5614879.0,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.097320556640625,
      "rewards/margins": 0.0595703125,
      "rewards/rejected": -0.15673828125,
      "step": 15
    },
    {
      "entropy": 0.176513671875,
      "epoch": 0.08142493638676845,
      "grad_norm": 62.994319915771484,
      "learning_rate": 4.6192893401015235e-06,
      "logits/chosen": -1.2212638279189771,
      "logits/rejected": -1.2088961146919388,
      "logps/chosen": -227.25,
      "logps/rejected": -213.0,
      "loss": 0.83746337890625,
      "mean_token_accuracy": 0.9471840560436249,
      "num_tokens": 5971613.0,
      "rewards/accuracies": 0.28125,
      "rewards/chosen": -0.173828125,
      "rewards/margins": -0.18463134765625,
      "rewards/rejected": 0.01068115234375,
      "step": 16
    },
    {
      "entropy": 0.169677734375,
      "epoch": 0.08651399491094147,
      "grad_norm": 60.76310348510742,
      "learning_rate": 4.593908629441624e-06,
      "logits/chosen": -1.2065562340385436,
      "logits/rejected": -1.2022840016395056,
      "logps/chosen": -218.5,
      "logps/rejected": -190.0,
      "loss": 0.77734375,
      "mean_token_accuracy": 0.9474756270647049,
      "num_tokens": 6368369.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.067138671875,
      "rewards/margins": -0.05255126953125,
      "rewards/rejected": -0.0137939453125,
      "step": 17
    },
    {
      "entropy": 0.1748046875,
      "epoch": 0.0916030534351145,
      "grad_norm": 61.58626174926758,
      "learning_rate": 4.568527918781726e-06,
      "logits/chosen": -1.1644959212587278,
      "logits/rejected": -1.1554610862237855,
      "logps/chosen": -213.5,
      "logps/rejected": -256.5,
      "loss": 0.744384765625,
      "mean_token_accuracy": 0.9518721997737885,
      "num_tokens": 6749942.0,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.1009521484375,
      "rewards/margins": 0.003753662109375,
      "rewards/rejected": -0.104736328125,
      "step": 18
    },
    {
      "entropy": 0.170654296875,
      "epoch": 0.09669211195928754,
      "grad_norm": 69.10458374023438,
      "learning_rate": 4.543147208121828e-06,
      "logits/chosen": -1.2018930915056671,
      "logits/rejected": -1.1882594101155102,
      "logps/chosen": -208.75,
      "logps/rejected": -223.0,
      "loss": 0.7578125,
      "mean_token_accuracy": 0.9503921419382095,
      "num_tokens": 7150669.0,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.16357421875,
      "rewards/margins": -0.0054931640625,
      "rewards/rejected": -0.157958984375,
      "step": 19
    },
    {
      "entropy": 0.1708984375,
      "epoch": 0.10178117048346055,
      "grad_norm": 59.20901870727539,
      "learning_rate": 4.5177664974619295e-06,
      "logits/chosen": -1.153684045887259,
      "logits/rejected": -1.1406723101806675,
      "logps/chosen": -214.25,
      "logps/rejected": -235.75,
      "loss": 0.742584228515625,
      "mean_token_accuracy": 0.9510545581579208,
      "num_tokens": 7577633.0,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.1868896484375,
      "rewards/margins": -0.008544921875,
      "rewards/rejected": -0.17822265625,
      "step": 20
    },
    {
      "entropy": 0.160888671875,
      "epoch": 0.10687022900763359,
      "grad_norm": 53.25274658203125,
      "learning_rate": 4.492385786802031e-06,
      "logits/chosen": -1.2115618534718147,
      "logits/rejected": -1.1697568144140837,
      "logps/chosen": -220.5,
      "logps/rejected": -224.5,
      "loss": 0.6365966796875,
      "mean_token_accuracy": 0.954043835401535,
      "num_tokens": 7935697.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0498046875,
      "rewards/margins": 0.22021484375,
      "rewards/rejected": -0.2708282470703125,
      "step": 21
    },
    {
      "entropy": 0.1708984375,
      "epoch": 0.11195928753180662,
      "grad_norm": 64.42499542236328,
      "learning_rate": 4.467005076142132e-06,
      "logits/chosen": -1.163452744541083,
      "logits/rejected": -1.1574276551228084,
      "logps/chosen": -206.75,
      "logps/rejected": -249.0,
      "loss": 0.7249755859375,
      "mean_token_accuracy": 0.9557177871465683,
      "num_tokens": 8344719.0,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.064208984375,
      "rewards/margins": 0.043914794921875,
      "rewards/rejected": -0.108154296875,
      "step": 22
    },
    {
      "entropy": 0.1650390625,
      "epoch": 0.11704834605597965,
      "grad_norm": 57.46247863769531,
      "learning_rate": 4.441624365482234e-06,
      "logits/chosen": -1.184751779280805,
      "logits/rejected": -1.1601956141572203,
      "logps/chosen": -221.5,
      "logps/rejected": -223.5,
      "loss": 0.725341796875,
      "mean_token_accuracy": 0.9495507925748825,
      "num_tokens": 8715576.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.12018585205078125,
      "rewards/margins": 0.1531982421875,
      "rewards/rejected": -0.27362060546875,
      "step": 23
    },
    {
      "entropy": 0.166259765625,
      "epoch": 0.12213740458015267,
      "grad_norm": 88.59230041503906,
      "learning_rate": 4.4162436548223355e-06,
      "logits/chosen": -1.2269864585121328,
      "logits/rejected": -1.2167687340565112,
      "logps/chosen": -190.5,
      "logps/rejected": -223.25,
      "loss": 0.616790771484375,
      "mean_token_accuracy": 0.9541012793779373,
      "num_tokens": 9131729.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.0047607421875,
      "rewards/margins": 0.2396240234375,
      "rewards/rejected": -0.244873046875,
      "step": 24
    },
    {
      "entropy": 0.174560546875,
      "epoch": 0.1272264631043257,
      "grad_norm": 59.16816711425781,
      "learning_rate": 4.390862944162436e-06,
      "logits/chosen": -1.2085149622005857,
      "logits/rejected": -1.1962000528044974,
      "logps/chosen": -209.5,
      "logps/rejected": -248.25,
      "loss": 0.68780517578125,
      "mean_token_accuracy": 0.9498510509729385,
      "num_tokens": 9510884.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.14385986328125,
      "rewards/margins": 0.06866455078125,
      "rewards/rejected": -0.212890625,
      "step": 25
    },
    {
      "entropy": 0.166259765625,
      "epoch": 0.13231552162849872,
      "grad_norm": 58.58485412597656,
      "learning_rate": 4.365482233502538e-06,
      "logits/chosen": -1.2357831115334332,
      "logits/rejected": -1.2414339305565623,
      "logps/chosen": -205.25,
      "logps/rejected": -210.0,
      "loss": 0.735107421875,
      "mean_token_accuracy": 0.9495077282190323,
      "num_tokens": 9892399.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.144287109375,
      "rewards/margins": 0.02099609375,
      "rewards/rejected": -0.16522216796875,
      "step": 26
    },
    {
      "entropy": 0.17333984375,
      "epoch": 0.13740458015267176,
      "grad_norm": 51.31973648071289,
      "learning_rate": 4.34010152284264e-06,
      "logits/chosen": -1.1937940321742593,
      "logits/rejected": -1.20553321824887,
      "logps/chosen": -185.875,
      "logps/rejected": -204.75,
      "loss": 0.66217041015625,
      "mean_token_accuracy": 0.9475287944078445,
      "num_tokens": 10284784.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.072509765625,
      "rewards/margins": 0.17462158203125,
      "rewards/rejected": -0.247314453125,
      "step": 27
    },
    {
      "entropy": 0.16552734375,
      "epoch": 0.14249363867684478,
      "grad_norm": 61.93020248413086,
      "learning_rate": 4.3147208121827415e-06,
      "logits/chosen": -1.2497175985959397,
      "logits/rejected": -1.2349540219835924,
      "logps/chosen": -185.5,
      "logps/rejected": -200.5,
      "loss": 0.785888671875,
      "mean_token_accuracy": 0.9526190459728241,
      "num_tokens": 10640064.0,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.24853515625,
      "rewards/margins": -0.0711669921875,
      "rewards/rejected": -0.1773681640625,
      "step": 28
    },
    {
      "entropy": 0.168701171875,
      "epoch": 0.1475826972010178,
      "grad_norm": 47.82231903076172,
      "learning_rate": 4.289340101522843e-06,
      "logits/chosen": -1.2240958044550552,
      "logits/rejected": -1.2331657411234425,
      "logps/chosen": -164.75,
      "logps/rejected": -183.5,
      "loss": 0.64813232421875,
      "mean_token_accuracy": 0.9512129127979279,
      "num_tokens": 11043531.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0230712890625,
      "rewards/margins": 0.1962890625,
      "rewards/rejected": -0.21942138671875,
      "step": 29
    },
    {
      "entropy": 0.166015625,
      "epoch": 0.15267175572519084,
      "grad_norm": 64.59899139404297,
      "learning_rate": 4.263959390862945e-06,
      "logits/chosen": -1.1938851832774966,
      "logits/rejected": -1.2063193481273322,
      "logps/chosen": -203.375,
      "logps/rejected": -209.25,
      "loss": 0.75030517578125,
      "mean_token_accuracy": 0.9482824355363846,
      "num_tokens": 11433645.0,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.160888671875,
      "rewards/margins": 0.0379638671875,
      "rewards/rejected": -0.1988525390625,
      "step": 30
    },
    {
      "entropy": 0.174560546875,
      "epoch": 0.15776081424936386,
      "grad_norm": 50.42904281616211,
      "learning_rate": 4.238578680203046e-06,
      "logits/chosen": -1.1847193683809207,
      "logits/rejected": -1.2039758508125116,
      "logps/chosen": -187.0,
      "logps/rejected": -196.0,
      "loss": 0.6805324554443359,
      "mean_token_accuracy": 0.9458457380533218,
      "num_tokens": 11802241.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01849365234375,
      "rewards/margins": 0.2666015625,
      "rewards/rejected": -0.285400390625,
      "step": 31
    },
    {
      "entropy": 0.164794921875,
      "epoch": 0.1628498727735369,
      "grad_norm": 65.630126953125,
      "learning_rate": 4.2131979695431475e-06,
      "logits/chosen": -1.2242753640413924,
      "logits/rejected": -1.2582800165640973,
      "logps/chosen": -237.25,
      "logps/rejected": -239.75,
      "loss": 0.762481689453125,
      "mean_token_accuracy": 0.9494172036647797,
      "num_tokens": 12168742.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.256591796875,
      "rewards/margins": 0.0673828125,
      "rewards/rejected": -0.323486328125,
      "step": 32
    },
    {
      "entropy": 0.1669921875,
      "epoch": 0.16793893129770993,
      "grad_norm": 64.15123748779297,
      "learning_rate": 4.187817258883249e-06,
      "logits/chosen": -1.2054564969855512,
      "logits/rejected": -1.2242075139011555,
      "logps/chosen": -225.75,
      "logps/rejected": -242.25,
      "loss": 0.776153564453125,
      "mean_token_accuracy": 0.9507817625999451,
      "num_tokens": 12544019.0,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.259033203125,
      "rewards/margins": -0.0609130859375,
      "rewards/rejected": -0.1982421875,
      "step": 33
    },
    {
      "entropy": 0.16162109375,
      "epoch": 0.17302798982188294,
      "grad_norm": 59.748802185058594,
      "learning_rate": 4.162436548223351e-06,
      "logits/chosen": -1.1906290703191211,
      "logits/rejected": -1.2255973378124254,
      "logps/chosen": -185.5,
      "logps/rejected": -189.5,
      "loss": 0.81256103515625,
      "mean_token_accuracy": 0.951210081577301,
      "num_tokens": 12934809.0,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.261016845703125,
      "rewards/margins": -0.1204071044921875,
      "rewards/rejected": -0.140380859375,
      "step": 34
    },
    {
      "entropy": 0.160400390625,
      "epoch": 0.178117048346056,
      "grad_norm": 62.484031677246094,
      "learning_rate": 4.137055837563453e-06,
      "logits/chosen": -1.1568720249350068,
      "logits/rejected": -1.1932806025584015,
      "logps/chosen": -214.75,
      "logps/rejected": -264.25,
      "loss": 0.743255615234375,
      "mean_token_accuracy": 0.952626571059227,
      "num_tokens": 13333240.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.19916534423828125,
      "rewards/margins": 0.0906982421875,
      "rewards/rejected": -0.2904052734375,
      "step": 35
    },
    {
      "entropy": 0.168701171875,
      "epoch": 0.183206106870229,
      "grad_norm": 57.09201431274414,
      "learning_rate": 4.1116751269035535e-06,
      "logits/chosen": -1.244021900396991,
      "logits/rejected": -1.2053838438463438,
      "logps/chosen": -227.75,
      "logps/rejected": -235.75,
      "loss": 0.598724365234375,
      "mean_token_accuracy": 0.9517507553100586,
      "num_tokens": 13672135.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.099365234375,
      "rewards/margins": 0.307861328125,
      "rewards/rejected": -0.4072265625,
      "step": 36
    },
    {
      "entropy": 0.168212890625,
      "epoch": 0.18829516539440203,
      "grad_norm": 70.22441101074219,
      "learning_rate": 4.086294416243655e-06,
      "logits/chosen": -1.2293163289174895,
      "logits/rejected": -1.2229109217299612,
      "logps/chosen": -224.25,
      "logps/rejected": -227.75,
      "loss": 0.82135009765625,
      "mean_token_accuracy": 0.9494140148162842,
      "num_tokens": 14069079.0,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": -0.0679931640625,
      "rewards/margins": -0.12286376953125,
      "rewards/rejected": 0.0546875,
      "step": 37
    },
    {
      "entropy": 0.16357421875,
      "epoch": 0.19338422391857507,
      "grad_norm": 76.26187896728516,
      "learning_rate": 4.060913705583757e-06,
      "logits/chosen": -1.210362411249237,
      "logits/rejected": -1.2507633848434636,
      "logps/chosen": -263.5,
      "logps/rejected": -245.75,
      "loss": 0.789825439453125,
      "mean_token_accuracy": 0.9464226067066193,
      "num_tokens": 14439718.0,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.00927734375,
      "rewards/margins": -0.00775146484375,
      "rewards/rejected": -0.001953125,
      "step": 38
    },
    {
      "entropy": 0.1767578125,
      "epoch": 0.1984732824427481,
      "grad_norm": 60.25480270385742,
      "learning_rate": 4.035532994923858e-06,
      "logits/chosen": -1.2676762543068074,
      "logits/rejected": -1.2517246495133931,
      "logps/chosen": -240.25,
      "logps/rejected": -250.0,
      "loss": 0.706390380859375,
      "mean_token_accuracy": 0.9461905360221863,
      "num_tokens": 14842983.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.234130859375,
      "rewards/margins": 0.1005859375,
      "rewards/rejected": -0.334716796875,
      "step": 39
    },
    {
      "entropy": 0.158447265625,
      "epoch": 0.2035623409669211,
      "grad_norm": 56.411930084228516,
      "learning_rate": 4.0101522842639595e-06,
      "logits/chosen": -1.253394155349274,
      "logits/rejected": -1.2763997522602422,
      "logps/chosen": -219.5,
      "logps/rejected": -248.0,
      "loss": 0.6590461730957031,
      "mean_token_accuracy": 0.9516128301620483,
      "num_tokens": 15207602.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.115966796875,
      "rewards/margins": 0.2529296875,
      "rewards/rejected": -0.3690185546875,
      "step": 40
    },
    {
      "entropy": 0.162109375,
      "epoch": 0.20865139949109415,
      "grad_norm": 50.47223663330078,
      "learning_rate": 3.984771573604061e-06,
      "logits/chosen": -1.221418931634342,
      "logits/rejected": -1.240179635098703,
      "logps/chosen": -198.75,
      "logps/rejected": -241.0,
      "loss": 0.61651611328125,
      "mean_token_accuracy": 0.9533202648162842,
      "num_tokens": 15615485.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.01708984375,
      "rewards/margins": 0.32470703125,
      "rewards/rejected": -0.3079833984375,
      "step": 41
    },
    {
      "entropy": 0.164794921875,
      "epoch": 0.21374045801526717,
      "grad_norm": 63.18607711791992,
      "learning_rate": 3.959390862944163e-06,
      "logits/chosen": -1.2487682814246732,
      "logits/rejected": -1.2684586865119198,
      "logps/chosen": -230.5,
      "logps/rejected": -251.25,
      "loss": 0.73516845703125,
      "mean_token_accuracy": 0.9500314593315125,
      "num_tokens": 16006587.0,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.0408935546875,
      "rewards/margins": 0.07275390625,
      "rewards/rejected": -0.11383056640625,
      "step": 42
    },
    {
      "entropy": 0.16796875,
      "epoch": 0.21882951653944022,
      "grad_norm": 57.65352249145508,
      "learning_rate": 3.934010152284265e-06,
      "logits/chosen": -1.3110932592597404,
      "logits/rejected": -1.3089875033252198,
      "logps/chosen": -223.0,
      "logps/rejected": -271.25,
      "loss": 0.7037353515625,
      "mean_token_accuracy": 0.952717587351799,
      "num_tokens": 16376034.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.1783447265625,
      "rewards/margins": 0.10040283203125,
      "rewards/rejected": -0.2783203125,
      "step": 43
    },
    {
      "entropy": 0.162353515625,
      "epoch": 0.22391857506361323,
      "grad_norm": 51.37344741821289,
      "learning_rate": 3.9086294416243655e-06,
      "logits/chosen": -1.293287887477467,
      "logits/rejected": -1.2826172419774333,
      "logps/chosen": -169.0,
      "logps/rejected": -188.25,
      "loss": 0.68048095703125,
      "mean_token_accuracy": 0.9513245075941086,
      "num_tokens": 16729880.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.14501953125,
      "rewards/margins": 0.17425537109375,
      "rewards/rejected": -0.31884765625,
      "step": 44
    },
    {
      "entropy": 0.162109375,
      "epoch": 0.22900763358778625,
      "grad_norm": 60.11509704589844,
      "learning_rate": 3.883248730964467e-06,
      "logits/chosen": -1.2668976340142997,
      "logits/rejected": -1.2750247764479736,
      "logps/chosen": -227.0,
      "logps/rejected": -250.0,
      "loss": 0.6617431640625,
      "mean_token_accuracy": 0.9519101679325104,
      "num_tokens": 17112685.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0929718017578125,
      "rewards/margins": 0.20703125,
      "rewards/rejected": -0.2998046875,
      "step": 45
    },
    {
      "entropy": 0.16552734375,
      "epoch": 0.2340966921119593,
      "grad_norm": 70.05858612060547,
      "learning_rate": 3.857868020304569e-06,
      "logits/chosen": -1.256115313351624,
      "logits/rejected": -1.292612336747009,
      "logps/chosen": -203.75,
      "logps/rejected": -204.25,
      "loss": 0.78472900390625,
      "mean_token_accuracy": 0.9469475299119949,
      "num_tokens": 17464688.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.1900482177734375,
      "rewards/margins": -0.072418212890625,
      "rewards/rejected": -0.118408203125,
      "step": 46
    },
    {
      "entropy": 0.16552734375,
      "epoch": 0.23918575063613232,
      "grad_norm": 63.72059631347656,
      "learning_rate": 3.832487309644671e-06,
      "logits/chosen": -1.2366188823977582,
      "logits/rejected": -1.2068790190894856,
      "logps/chosen": -215.75,
      "logps/rejected": -216.25,
      "loss": 0.70928955078125,
      "mean_token_accuracy": 0.947726234793663,
      "num_tokens": 17827406.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.126556396484375,
      "rewards/margins": 0.05908203125,
      "rewards/rejected": -0.18597412109375,
      "step": 47
    },
    {
      "entropy": 0.173095703125,
      "epoch": 0.24427480916030533,
      "grad_norm": 51.02957534790039,
      "learning_rate": 3.8071065989847715e-06,
      "logits/chosen": -1.2951891338286738,
      "logits/rejected": -1.2901734123706725,
      "logps/chosen": -186.375,
      "logps/rejected": -190.375,
      "loss": 0.686798095703125,
      "mean_token_accuracy": 0.9495810121297836,
      "num_tokens": 18168837.0,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.149169921875,
      "rewards/margins": 0.10107421875,
      "rewards/rejected": -0.25054931640625,
      "step": 48
    },
    {
      "entropy": 0.171142578125,
      "epoch": 0.24936386768447838,
      "grad_norm": 65.16993713378906,
      "learning_rate": 3.7817258883248736e-06,
      "logits/chosen": -1.2489111246586129,
      "logits/rejected": -1.2595230297930642,
      "logps/chosen": -226.25,
      "logps/rejected": -253.5,
      "loss": 0.6953582763671875,
      "mean_token_accuracy": 0.9487173557281494,
      "num_tokens": 18520757.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.05126953125,
      "rewards/margins": 0.1453857421875,
      "rewards/rejected": -0.197357177734375,
      "step": 49
    },
    {
      "entropy": 0.161865234375,
      "epoch": 0.2544529262086514,
      "grad_norm": 85.86858367919922,
      "learning_rate": 3.756345177664975e-06,
      "logits/chosen": -1.2788417924140663,
      "logits/rejected": -1.2818688140867107,
      "logps/chosen": -222.25,
      "logps/rejected": -228.0,
      "loss": 0.906097412109375,
      "mean_token_accuracy": 0.9481759369373322,
      "num_tokens": 18907465.0,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.32647705078125,
      "rewards/margins": -0.261962890625,
      "rewards/rejected": -0.06494140625,
      "step": 50
    },
    {
      "entropy": 0.1689453125,
      "epoch": 0.2595419847328244,
      "grad_norm": 63.7056999206543,
      "learning_rate": 3.7309644670050766e-06,
      "logits/chosen": -1.24593670349563,
      "logits/rejected": -1.2525011882667132,
      "logps/chosen": -204.5,
      "logps/rejected": -235.0,
      "loss": 0.809600830078125,
      "mean_token_accuracy": 0.9492675065994263,
      "num_tokens": 19271716.0,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.22967529296875,
      "rewards/margins": -0.108154296875,
      "rewards/rejected": -0.12200927734375,
      "step": 51
    },
    {
      "entropy": 0.16943359375,
      "epoch": 0.26463104325699743,
      "grad_norm": 66.75394439697266,
      "learning_rate": 3.705583756345178e-06,
      "logits/chosen": -1.2951451816324733,
      "logits/rejected": -1.2748576702987078,
      "logps/chosen": -220.0,
      "logps/rejected": -201.5,
      "loss": 0.7838897705078125,
      "mean_token_accuracy": 0.9473998248577118,
      "num_tokens": 19651670.0,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.283111572265625,
      "rewards/margins": -0.046630859375,
      "rewards/rejected": -0.235687255859375,
      "step": 52
    },
    {
      "entropy": 0.167724609375,
      "epoch": 0.2697201017811705,
      "grad_norm": 68.03031921386719,
      "learning_rate": 3.6802030456852796e-06,
      "logits/chosen": -1.2656714275142638,
      "logits/rejected": -1.2242606863621242,
      "logps/chosen": -184.375,
      "logps/rejected": -200.75,
      "loss": 0.81591796875,
      "mean_token_accuracy": 0.9486018866300583,
      "num_tokens": 20048489.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.284912109375,
      "rewards/margins": -0.12548828125,
      "rewards/rejected": -0.158782958984375,
      "step": 53
    },
    {
      "entropy": 0.164306640625,
      "epoch": 0.2748091603053435,
      "grad_norm": 52.74068832397461,
      "learning_rate": 3.654822335025381e-06,
      "logits/chosen": -1.3013878893228477,
      "logits/rejected": -1.3064163255128232,
      "logps/chosen": -192.5,
      "logps/rejected": -185.5,
      "loss": 0.67987060546875,
      "mean_token_accuracy": 0.9475405365228653,
      "num_tokens": 20384513.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.19287109375,
      "rewards/margins": 0.16943359375,
      "rewards/rejected": -0.3623046875,
      "step": 54
    },
    {
      "entropy": 0.162109375,
      "epoch": 0.27989821882951654,
      "grad_norm": 71.49076843261719,
      "learning_rate": 3.629441624365482e-06,
      "logits/chosen": -1.2508704209061283,
      "logits/rejected": -1.2312254570289394,
      "logps/chosen": -225.0,
      "logps/rejected": -227.5,
      "loss": 0.86474609375,
      "mean_token_accuracy": 0.9473535567522049,
      "num_tokens": 20754117.0,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.34716796875,
      "rewards/margins": -0.2258148193359375,
      "rewards/rejected": -0.1211395263671875,
      "step": 55
    },
    {
      "entropy": 0.1640625,
      "epoch": 0.28498727735368956,
      "grad_norm": 70.81625366210938,
      "learning_rate": 3.6040609137055843e-06,
      "logits/chosen": -1.3251861075583002,
      "logits/rejected": -1.32016448634709,
      "logps/chosen": -237.75,
      "logps/rejected": -248.0,
      "loss": 0.78326416015625,
      "mean_token_accuracy": 0.948715329170227,
      "num_tokens": 21157270.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.26702880859375,
      "rewards/margins": -0.0032958984375,
      "rewards/rejected": -0.26409912109375,
      "step": 56
    },
    {
      "entropy": 0.16162109375,
      "epoch": 0.2900763358778626,
      "grad_norm": 57.185699462890625,
      "learning_rate": 3.5786802030456856e-06,
      "logits/chosen": -1.3391477581159092,
      "logits/rejected": -1.2969421235061,
      "logps/chosen": -167.5,
      "logps/rejected": -206.75,
      "loss": 0.7360153198242188,
      "mean_token_accuracy": 0.9557931572198868,
      "num_tokens": 21557901.0,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": -0.13916015625,
      "rewards/margins": 0.08935546875,
      "rewards/rejected": -0.22784423828125,
      "step": 57
    },
    {
      "entropy": 0.167236328125,
      "epoch": 0.2951653944020356,
      "grad_norm": 67.23629760742188,
      "learning_rate": 3.5532994923857873e-06,
      "logits/chosen": -1.2344125058729924,
      "logits/rejected": -1.2176261940150734,
      "logps/chosen": -221.625,
      "logps/rejected": -227.5,
      "loss": 0.833160400390625,
      "mean_token_accuracy": 0.9486464709043503,
      "num_tokens": 21930023.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.220458984375,
      "rewards/margins": -0.0969696044921875,
      "rewards/rejected": -0.1231689453125,
      "step": 58
    },
    {
      "entropy": 0.158447265625,
      "epoch": 0.30025445292620867,
      "grad_norm": 61.011383056640625,
      "learning_rate": 3.5279187817258886e-06,
      "logits/chosen": -1.2608206675107227,
      "logits/rejected": -1.2702774965163959,
      "logps/chosen": -230.5,
      "logps/rejected": -221.0,
      "loss": 0.693756103515625,
      "mean_token_accuracy": 0.9499951750040054,
      "num_tokens": 22300544.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.32568359375,
      "rewards/margins": 0.140350341796875,
      "rewards/rejected": -0.466796875,
      "step": 59
    },
    {
      "entropy": 0.155517578125,
      "epoch": 0.3053435114503817,
      "grad_norm": 60.74739456176758,
      "learning_rate": 3.5025380710659903e-06,
      "logits/chosen": -1.2432721402471825,
      "logits/rejected": -1.1966404831484798,
      "logps/chosen": -188.25,
      "logps/rejected": -196.0,
      "loss": 0.8065185546875,
      "mean_token_accuracy": 0.9513755738735199,
      "num_tokens": 22699715.0,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": -0.320556640625,
      "rewards/margins": -0.060028076171875,
      "rewards/rejected": -0.260986328125,
      "step": 60
    },
    {
      "entropy": 0.17333984375,
      "epoch": 0.3104325699745547,
      "grad_norm": 64.06476593017578,
      "learning_rate": 3.4771573604060916e-06,
      "logits/chosen": -1.23914246177565,
      "logits/rejected": -1.2589912793052047,
      "logps/chosen": -208.25,
      "logps/rejected": -225.25,
      "loss": 0.77069091796875,
      "mean_token_accuracy": 0.9473724365234375,
      "num_tokens": 23067391.0,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.082275390625,
      "rewards/margins": 0.08203125,
      "rewards/rejected": -0.166015625,
      "step": 61
    },
    {
      "entropy": 0.1689453125,
      "epoch": 0.3155216284987277,
      "grad_norm": 48.07145309448242,
      "learning_rate": 3.451776649746193e-06,
      "logits/chosen": -1.2898154412235743,
      "logits/rejected": -1.2884235645949569,
      "logps/chosen": -205.25,
      "logps/rejected": -217.125,
      "loss": 0.5599517822265625,
      "mean_token_accuracy": 0.947222501039505,
      "num_tokens": 23396708.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.03094482421875,
      "rewards/margins": 0.41748046875,
      "rewards/rejected": -0.38671875,
      "step": 62
    },
    {
      "entropy": 0.1640625,
      "epoch": 0.32061068702290074,
      "grad_norm": 65.72078704833984,
      "learning_rate": 3.4263959390862946e-06,
      "logits/chosen": -1.2525791988685302,
      "logits/rejected": -1.2465728013183064,
      "logps/chosen": -226.5,
      "logps/rejected": -198.75,
      "loss": 0.852203369140625,
      "mean_token_accuracy": 0.9457215964794159,
      "num_tokens": 23771644.0,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.45166015625,
      "rewards/margins": -0.1153564453125,
      "rewards/rejected": -0.335205078125,
      "step": 63
    },
    {
      "entropy": 0.16943359375,
      "epoch": 0.3256997455470738,
      "grad_norm": 70.23268127441406,
      "learning_rate": 3.4010152284263963e-06,
      "logits/chosen": -1.2732464634523972,
      "logits/rejected": -1.2514687027641094,
      "logps/chosen": -235.75,
      "logps/rejected": -247.25,
      "loss": 0.781402587890625,
      "mean_token_accuracy": 0.9476925879716873,
      "num_tokens": 24154169.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.297607421875,
      "rewards/margins": -0.0504150390625,
      "rewards/rejected": -0.246826171875,
      "step": 64
    },
    {
      "entropy": 0.1630859375,
      "epoch": 0.33078880407124683,
      "grad_norm": 75.50202941894531,
      "learning_rate": 3.375634517766498e-06,
      "logits/chosen": -1.2872747032412404,
      "logits/rejected": -1.2523587882327665,
      "logps/chosen": -243.0,
      "logps/rejected": -231.0,
      "loss": 0.81689453125,
      "mean_token_accuracy": 0.9463559240102768,
      "num_tokens": 24486129.0,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.22308349609375,
      "rewards/margins": -0.103271484375,
      "rewards/rejected": -0.11993408203125,
      "step": 65
    },
    {
      "entropy": 0.162109375,
      "epoch": 0.33587786259541985,
      "grad_norm": 65.13658142089844,
      "learning_rate": 3.3502538071065993e-06,
      "logits/chosen": -1.2898683758825709,
      "logits/rejected": -1.2849647133801598,
      "logps/chosen": -220.25,
      "logps/rejected": -229.5,
      "loss": 0.7967147827148438,
      "mean_token_accuracy": 0.9507217556238174,
      "num_tokens": 24857818.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2047119140625,
      "rewards/margins": 0.05828857421875,
      "rewards/rejected": -0.263427734375,
      "step": 66
    },
    {
      "entropy": 0.165283203125,
      "epoch": 0.34096692111959287,
      "grad_norm": 61.869327545166016,
      "learning_rate": 3.3248730964467006e-06,
      "logits/chosen": -1.2517319440883818,
      "logits/rejected": -1.2434228301098385,
      "logps/chosen": -205.625,
      "logps/rejected": -222.25,
      "loss": 0.697723388671875,
      "mean_token_accuracy": 0.9499448090791702,
      "num_tokens": 25270495.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.1441650390625,
      "rewards/margins": 0.16796875,
      "rewards/rejected": -0.31201171875,
      "step": 67
    },
    {
      "entropy": 0.156005859375,
      "epoch": 0.3460559796437659,
      "grad_norm": 69.01359558105469,
      "learning_rate": 3.2994923857868023e-06,
      "logits/chosen": -1.2596454913995498,
      "logits/rejected": -1.2822667709063957,
      "logps/chosen": -238.75,
      "logps/rejected": -219.25,
      "loss": 0.788330078125,
      "mean_token_accuracy": 0.9476147145032883,
      "num_tokens": 25639922.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2823486328125,
      "rewards/margins": -0.060791015625,
      "rewards/rejected": -0.2213134765625,
      "step": 68
    },
    {
      "entropy": 0.170166015625,
      "epoch": 0.3511450381679389,
      "grad_norm": 65.33216094970703,
      "learning_rate": 3.2741116751269036e-06,
      "logits/chosen": -1.2824587990174645,
      "logits/rejected": -1.2777971502557461,
      "logps/chosen": -199.25,
      "logps/rejected": -190.5,
      "loss": 0.833953857421875,
      "mean_token_accuracy": 0.945390522480011,
      "num_tokens": 26010115.0,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.238037109375,
      "rewards/margins": -0.1005859375,
      "rewards/rejected": -0.1373443603515625,
      "step": 69
    },
    {
      "entropy": 0.162841796875,
      "epoch": 0.356234096692112,
      "grad_norm": 64.25736999511719,
      "learning_rate": 3.2487309644670053e-06,
      "logits/chosen": -1.332777326624499,
      "logits/rejected": -1.3164871807406162,
      "logps/chosen": -175.625,
      "logps/rejected": -220.75,
      "loss": 0.727630615234375,
      "mean_token_accuracy": 0.950916975736618,
      "num_tokens": 26385610.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.1827850341796875,
      "rewards/margins": 0.056396484375,
      "rewards/rejected": -0.2392578125,
      "step": 70
    },
    {
      "entropy": 0.1611328125,
      "epoch": 0.361323155216285,
      "grad_norm": 57.03504943847656,
      "learning_rate": 3.223350253807107e-06,
      "logits/chosen": -1.2973189023839775,
      "logits/rejected": -1.2664155194693132,
      "logps/chosen": -195.75,
      "logps/rejected": -225.25,
      "loss": 0.6987152099609375,
      "mean_token_accuracy": 0.9523173123598099,
      "num_tokens": 26772574.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.239013671875,
      "rewards/margins": 0.1385498046875,
      "rewards/rejected": -0.37744140625,
      "step": 71
    },
    {
      "entropy": 0.151123046875,
      "epoch": 0.366412213740458,
      "grad_norm": 106.73540496826172,
      "learning_rate": 3.1979695431472087e-06,
      "logits/chosen": -1.3368664755840716,
      "logits/rejected": -1.3149258298106234,
      "logps/chosen": -207.75,
      "logps/rejected": -193.75,
      "loss": 0.76409912109375,
      "mean_token_accuracy": 0.9506115764379501,
      "num_tokens": 27153798.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.32080078125,
      "rewards/margins": -0.001861572265625,
      "rewards/rejected": -0.318359375,
      "step": 72
    },
    {
      "entropy": 0.162841796875,
      "epoch": 0.37150127226463103,
      "grad_norm": 43.361202239990234,
      "learning_rate": 3.17258883248731e-06,
      "logits/chosen": -1.292466410490634,
      "logits/rejected": -1.2795658979837796,
      "logps/chosen": -174.25,
      "logps/rejected": -169.0,
      "loss": 0.6114501953125,
      "mean_token_accuracy": 0.9446200430393219,
      "num_tokens": 27531852.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.11895751953125,
      "rewards/margins": 0.2860107421875,
      "rewards/rejected": -0.40478515625,
      "step": 73
    },
    {
      "entropy": 0.165283203125,
      "epoch": 0.37659033078880405,
      "grad_norm": 258.8431701660156,
      "learning_rate": 3.1472081218274113e-06,
      "logits/chosen": -1.255416233032602,
      "logits/rejected": -1.2341775737485738,
      "logps/chosen": -251.25,
      "logps/rejected": -240.5,
      "loss": 0.8010711669921875,
      "mean_token_accuracy": 0.9430906176567078,
      "num_tokens": 27903266.0,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.41162109375,
      "rewards/margins": 0.0003662109375,
      "rewards/rejected": -0.4114990234375,
      "step": 74
    },
    {
      "entropy": 0.1513671875,
      "epoch": 0.3816793893129771,
      "grad_norm": 59.12688064575195,
      "learning_rate": 3.121827411167513e-06,
      "logits/chosen": -1.3708437654075616,
      "logits/rejected": -1.3749631542345564,
      "logps/chosen": -184.25,
      "logps/rejected": -207.75,
      "loss": 0.7006683349609375,
      "mean_token_accuracy": 0.9533696621656418,
      "num_tokens": 28291715.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.008056640625,
      "rewards/margins": 0.16162109375,
      "rewards/rejected": -0.169677734375,
      "step": 75
    },
    {
      "entropy": 0.168212890625,
      "epoch": 0.38676844783715014,
      "grad_norm": 70.00326538085938,
      "learning_rate": 3.0964467005076143e-06,
      "logits/chosen": -1.2278840581970798,
      "logits/rejected": -1.252243618402488,
      "logps/chosen": -241.25,
      "logps/rejected": -222.25,
      "loss": 0.7878265380859375,
      "mean_token_accuracy": 0.9443240016698837,
      "num_tokens": 28640895.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.22607421875,
      "rewards/margins": -0.0015869140625,
      "rewards/rejected": -0.2257080078125,
      "step": 76
    },
    {
      "entropy": 0.147216796875,
      "epoch": 0.39185750636132316,
      "grad_norm": 52.42808151245117,
      "learning_rate": 3.071065989847716e-06,
      "logits/chosen": -1.3088794461049085,
      "logits/rejected": -1.2848204770716634,
      "logps/chosen": -174.375,
      "logps/rejected": -186.75,
      "loss": 0.730010986328125,
      "mean_token_accuracy": 0.9527680724859238,
      "num_tokens": 29031104.0,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.24163818359375,
      "rewards/margins": 0.07861328125,
      "rewards/rejected": -0.31982421875,
      "step": 77
    },
    {
      "entropy": 0.162109375,
      "epoch": 0.3969465648854962,
      "grad_norm": 66.45071411132812,
      "learning_rate": 3.0456852791878177e-06,
      "logits/chosen": -1.2249955483759087,
      "logits/rejected": -1.2137169350336674,
      "logps/chosen": -216.0,
      "logps/rejected": -232.0,
      "loss": 0.7823486328125,
      "mean_token_accuracy": 0.9497682005167007,
      "num_tokens": 29411929.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.2440185546875,
      "rewards/margins": -0.053955078125,
      "rewards/rejected": -0.1904296875,
      "step": 78
    },
    {
      "entropy": 0.169189453125,
      "epoch": 0.4020356234096692,
      "grad_norm": 71.31132507324219,
      "learning_rate": 3.0203045685279194e-06,
      "logits/chosen": -1.311240796195369,
      "logits/rejected": -1.3000907150950454,
      "logps/chosen": -202.0,
      "logps/rejected": -218.75,
      "loss": 0.727569580078125,
      "mean_token_accuracy": 0.9468916058540344,
      "num_tokens": 29765589.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.289306640625,
      "rewards/margins": 0.17138671875,
      "rewards/rejected": -0.460205078125,
      "step": 79
    },
    {
      "entropy": 0.162353515625,
      "epoch": 0.4071246819338422,
      "grad_norm": 64.0061264038086,
      "learning_rate": 2.9949238578680207e-06,
      "logits/chosen": -1.3101197475281547,
      "logits/rejected": -1.2970568708741963,
      "logps/chosen": -220.25,
      "logps/rejected": -211.5,
      "loss": 0.729522705078125,
      "mean_token_accuracy": 0.9487328827381134,
      "num_tokens": 30163265.0,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.21240234375,
      "rewards/margins": 0.0606689453125,
      "rewards/rejected": -0.273193359375,
      "step": 80
    },
    {
      "entropy": 0.16357421875,
      "epoch": 0.4122137404580153,
      "grad_norm": 65.17308807373047,
      "learning_rate": 2.969543147208122e-06,
      "logits/chosen": -1.3028268301410426,
      "logits/rejected": -1.3046930569719115,
      "logps/chosen": -221.0,
      "logps/rejected": -223.25,
      "loss": 0.70806884765625,
      "mean_token_accuracy": 0.9474814534187317,
      "num_tokens": 30509121.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.2777099609375,
      "rewards/margins": 0.07781982421875,
      "rewards/rejected": -0.35546875,
      "step": 81
    },
    {
      "entropy": 0.165771484375,
      "epoch": 0.4173027989821883,
      "grad_norm": 50.35979080200195,
      "learning_rate": 2.9441624365482237e-06,
      "logits/chosen": -1.2836960568317857,
      "logits/rejected": -1.306755689640039,
      "logps/chosen": -194.25,
      "logps/rejected": -214.75,
      "loss": 0.6282501220703125,
      "mean_token_accuracy": 0.9473407566547394,
      "num_tokens": 30917508.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.16790771484375,
      "rewards/margins": 0.30712890625,
      "rewards/rejected": -0.47509765625,
      "step": 82
    },
    {
      "entropy": 0.156494140625,
      "epoch": 0.4223918575063613,
      "grad_norm": 55.4305419921875,
      "learning_rate": 2.918781725888325e-06,
      "logits/chosen": -1.3347602124239275,
      "logits/rejected": -1.297045195641747,
      "logps/chosen": -168.0,
      "logps/rejected": -182.5,
      "loss": 0.702239990234375,
      "mean_token_accuracy": 0.9523750394582748,
      "num_tokens": 31290108.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.17498779296875,
      "rewards/margins": 0.1236572265625,
      "rewards/rejected": -0.298828125,
      "step": 83
    },
    {
      "entropy": 0.1591796875,
      "epoch": 0.42748091603053434,
      "grad_norm": 72.86205291748047,
      "learning_rate": 2.8934010152284262e-06,
      "logits/chosen": -1.2560234762562408,
      "logits/rejected": -1.2293338403613354,
      "logps/chosen": -207.75,
      "logps/rejected": -219.0,
      "loss": 0.7640380859375,
      "mean_token_accuracy": 0.9502075463533401,
      "num_tokens": 31677755.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6025390625,
      "rewards/margins": 0.078369140625,
      "rewards/rejected": -0.6806640625,
      "step": 84
    },
    {
      "entropy": 0.16015625,
      "epoch": 0.43256997455470736,
      "grad_norm": 60.04010009765625,
      "learning_rate": 2.8680203045685284e-06,
      "logits/chosen": -1.3011174292040961,
      "logits/rejected": -1.2539586523088446,
      "logps/chosen": -210.5,
      "logps/rejected": -211.25,
      "loss": 0.669281005859375,
      "mean_token_accuracy": 0.9515846818685532,
      "num_tokens": 32041342.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.1500244140625,
      "rewards/margins": 0.138671875,
      "rewards/rejected": -0.28857421875,
      "step": 85
    },
    {
      "entropy": 0.1572265625,
      "epoch": 0.43765903307888043,
      "grad_norm": 65.90755462646484,
      "learning_rate": 2.8426395939086297e-06,
      "logits/chosen": -1.2890849091113363,
      "logits/rejected": -1.2671961630083879,
      "logps/chosen": -212.0,
      "logps/rejected": -212.0,
      "loss": 0.8037109375,
      "mean_token_accuracy": 0.9498841613531113,
      "num_tokens": 32426894.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.328125,
      "rewards/margins": -0.07586669921875,
      "rewards/rejected": -0.2518310546875,
      "step": 86
    },
    {
      "entropy": 0.160888671875,
      "epoch": 0.44274809160305345,
      "grad_norm": 64.37542724609375,
      "learning_rate": 2.8172588832487314e-06,
      "logits/chosen": -1.2516031645716161,
      "logits/rejected": -1.2446139578943387,
      "logps/chosen": -238.5,
      "logps/rejected": -244.5,
      "loss": 0.734710693359375,
      "mean_token_accuracy": 0.9513941407203674,
      "num_tokens": 32747265.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.337158203125,
      "rewards/margins": 0.085693359375,
      "rewards/rejected": -0.42431640625,
      "step": 87
    },
    {
      "entropy": 0.164306640625,
      "epoch": 0.44783715012722647,
      "grad_norm": 64.61540222167969,
      "learning_rate": 2.7918781725888327e-06,
      "logits/chosen": -1.3129108468652448,
      "logits/rejected": -1.3204735199009792,
      "logps/chosen": -208.5,
      "logps/rejected": -216.5,
      "loss": 0.674041748046875,
      "mean_token_accuracy": 0.9465741366147995,
      "num_tokens": 33124296.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.34832763671875,
      "rewards/margins": 0.1859130859375,
      "rewards/rejected": -0.533203125,
      "step": 88
    },
    {
      "entropy": 0.166748046875,
      "epoch": 0.4529262086513995,
      "grad_norm": 62.742977142333984,
      "learning_rate": 2.7664974619289344e-06,
      "logits/chosen": -1.2541273019597161,
      "logits/rejected": -1.2661804942223065,
      "logps/chosen": -202.25,
      "logps/rejected": -204.0,
      "loss": 0.775146484375,
      "mean_token_accuracy": 0.9480140656232834,
      "num_tokens": 33498040.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.30126953125,
      "rewards/margins": -0.014892578125,
      "rewards/rejected": -0.28759765625,
      "step": 89
    },
    {
      "entropy": 0.1611328125,
      "epoch": 0.4580152671755725,
      "grad_norm": 70.147216796875,
      "learning_rate": 2.7411167512690357e-06,
      "logits/chosen": -1.2708255735449616,
      "logits/rejected": -1.2573322500480986,
      "logps/chosen": -226.0,
      "logps/rejected": -218.5,
      "loss": 0.7926025390625,
      "mean_token_accuracy": 0.9451356083154678,
      "num_tokens": 33845280.0,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.323974609375,
      "rewards/margins": -0.10205078125,
      "rewards/rejected": -0.2225341796875,
      "step": 90
    },
    {
      "entropy": 0.15625,
      "epoch": 0.4631043256997455,
      "grad_norm": 63.841976165771484,
      "learning_rate": 2.715736040609137e-06,
      "logits/chosen": -1.326140821141854,
      "logits/rejected": -1.3591119533293379,
      "logps/chosen": -200.25,
      "logps/rejected": -205.0,
      "loss": 0.790740966796875,
      "mean_token_accuracy": 0.9486561864614487,
      "num_tokens": 34242803.0,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4453125,
      "rewards/margins": -0.0655517578125,
      "rewards/rejected": -0.380126953125,
      "step": 91
    },
    {
      "entropy": 0.16015625,
      "epoch": 0.4681933842239186,
      "grad_norm": 73.43412780761719,
      "learning_rate": 2.6903553299492387e-06,
      "logits/chosen": -1.252702561138765,
      "logits/rejected": -1.2816603315955593,
      "logps/chosen": -250.25,
      "logps/rejected": -238.75,
      "loss": 0.90521240234375,
      "mean_token_accuracy": 0.9443905353546143,
      "num_tokens": 34628251.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.48974609375,
      "rewards/margins": -0.2421875,
      "rewards/rejected": -0.2470703125,
      "step": 92
    },
    {
      "entropy": 0.171142578125,
      "epoch": 0.4732824427480916,
      "grad_norm": 63.70939254760742,
      "learning_rate": 2.6649746192893404e-06,
      "logits/chosen": -1.254758430715978,
      "logits/rejected": -1.2633837670267376,
      "logps/chosen": -201.5,
      "logps/rejected": -184.25,
      "loss": 0.7750244140625,
      "mean_token_accuracy": 0.9433987885713577,
      "num_tokens": 35007915.0,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.306884765625,
      "rewards/margins": -0.08544921875,
      "rewards/rejected": -0.220947265625,
      "step": 93
    },
    {
      "entropy": 0.162109375,
      "epoch": 0.47837150127226463,
      "grad_norm": 73.37699890136719,
      "learning_rate": 2.639593908629442e-06,
      "logits/chosen": -1.2285106824311274,
      "logits/rejected": -1.2533075679141386,
      "logps/chosen": -218.75,
      "logps/rejected": -242.25,
      "loss": 0.84307861328125,
      "mean_token_accuracy": 0.947221428155899,
      "num_tokens": 35387530.0,
      "rewards/accuracies": 0.28125,
      "rewards/chosen": -0.4287109375,
      "rewards/margins": -0.1712646484375,
      "rewards/rejected": -0.25738525390625,
      "step": 94
    },
    {
      "entropy": 0.16357421875,
      "epoch": 0.48346055979643765,
      "grad_norm": 51.54319381713867,
      "learning_rate": 2.6142131979695434e-06,
      "logits/chosen": -1.276040442817514,
      "logits/rejected": -1.2760191663203924,
      "logps/chosen": -208.5,
      "logps/rejected": -240.0,
      "loss": 0.6256561279296875,
      "mean_token_accuracy": 0.9505224972963333,
      "num_tokens": 35732544.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03521728515625,
      "rewards/margins": 0.306884765625,
      "rewards/rejected": -0.3427734375,
      "step": 95
    },
    {
      "entropy": 0.165283203125,
      "epoch": 0.48854961832061067,
      "grad_norm": 64.21118927001953,
      "learning_rate": 2.588832487309645e-06,
      "logits/chosen": -1.2820478858157205,
      "logits/rejected": -1.2736327297402361,
      "logps/chosen": -211.5,
      "logps/rejected": -197.0,
      "loss": 0.8376922607421875,
      "mean_token_accuracy": 0.9446868598461151,
      "num_tokens": 36116587.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.236328125,
      "rewards/margins": -0.08966064453125,
      "rewards/rejected": -0.146087646484375,
      "step": 96
    },
    {
      "entropy": 0.15869140625,
      "epoch": 0.49363867684478374,
      "grad_norm": 60.89461135864258,
      "learning_rate": 2.5634517766497464e-06,
      "logits/chosen": -1.2462152995908475,
      "logits/rejected": -1.2528073331416407,
      "logps/chosen": -232.0,
      "logps/rejected": -247.25,
      "loss": 0.69921875,
      "mean_token_accuracy": 0.9487164616584778,
      "num_tokens": 36525364.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.1297607421875,
      "rewards/margins": 0.209716796875,
      "rewards/rejected": -0.3388671875,
      "step": 97
    },
    {
      "entropy": 0.162353515625,
      "epoch": 0.49872773536895676,
      "grad_norm": 58.411373138427734,
      "learning_rate": 2.5380710659898476e-06,
      "logits/chosen": -1.2857249020723578,
      "logits/rejected": -1.273766229120395,
      "logps/chosen": -194.0,
      "logps/rejected": -214.5,
      "loss": 0.718597412109375,
      "mean_token_accuracy": 0.949738010764122,
      "num_tokens": 36924408.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.32177734375,
      "rewards/margins": 0.137939453125,
      "rewards/rejected": -0.459716796875,
      "step": 98
    },
    {
      "entropy": 0.155029296875,
      "epoch": 0.5038167938931297,
      "grad_norm": 65.45690155029297,
      "learning_rate": 2.5126903553299493e-06,
      "logits/chosen": -1.3141924885583736,
      "logits/rejected": -1.3078939917330479,
      "logps/chosen": -222.25,
      "logps/rejected": -223.75,
      "loss": 0.7358856201171875,
      "mean_token_accuracy": 0.951283186674118,
      "num_tokens": 37320690.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2308349609375,
      "rewards/margins": 0.076904296875,
      "rewards/rejected": -0.308990478515625,
      "step": 99
    },
    {
      "entropy": 0.175048828125,
      "epoch": 0.5089058524173028,
      "grad_norm": 64.2022705078125,
      "learning_rate": 2.487309644670051e-06,
      "logits/chosen": -1.2262440668616592,
      "logits/rejected": -1.23488919059254,
      "logps/chosen": -232.75,
      "logps/rejected": -212.375,
      "loss": 0.8157958984375,
      "mean_token_accuracy": 0.9397356808185577,
      "num_tokens": 37699769.0,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.3427734375,
      "rewards/margins": -0.0829620361328125,
      "rewards/rejected": -0.2600555419921875,
      "step": 100
    }
  ],
  "logging_steps": 1,
  "max_steps": 197,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 7.965984588168167e+18,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}