{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.1254115065057219,
  "eval_steps": 500,
  "global_step": 200,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "entropy": 1.171857863664627,
      "epoch": 0.0006270575325286095,
      "grad_norm": 36.12955856323242,
      "learning_rate": 0.0,
      "logits/chosen": -16.618545689498976,
      "logits/rejected": -19.895854701940685,
      "logps/chosen": -618.4926280975342,
      "logps/rejected": -544.1511726379395,
      "loss": 1.0,
      "mean_token_accuracy": 0.7207779660820961,
      "num_tokens": 44742.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "entropy": 1.3859521597623825,
      "epoch": 0.001254115065057219,
      "grad_norm": 35.26237487792969,
      "learning_rate": 1e-08,
      "logits/chosen": -16.148485680429634,
      "logits/rejected": -15.444903538915032,
      "logps/chosen": -789.9899635314941,
      "logps/rejected": -629.6869964599609,
      "loss": 1.0,
      "mean_token_accuracy": 0.6869945079088211,
      "num_tokens": 96877.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2
    },
    {
      "entropy": 1.1979895681142807,
      "epoch": 0.0018811725975858284,
      "grad_norm": 41.65127944946289,
      "learning_rate": 2e-08,
      "logits/chosen": -18.29156918013904,
      "logits/rejected": -19.65692519211939,
      "logps/chosen": -1245.2266998291016,
      "logps/rejected": -570.8414497375488,
      "loss": 1.0063536167144775,
      "mean_token_accuracy": 0.6914148852229118,
      "num_tokens": 168920.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.017706198908854276,
      "rewards/margins": -0.025545110227540135,
      "rewards/rejected": 0.007838911202270538,
      "step": 3
    },
    {
      "entropy": 1.1795232072472572,
      "epoch": 0.002508230130114438,
      "grad_norm": 44.819602966308594,
      "learning_rate": 3e-08,
      "logits/chosen": -20.724100083229605,
      "logits/rejected": -19.704480662994772,
      "logps/chosen": -1142.84916305542,
      "logps/rejected": -649.1872062683105,
      "loss": 1.0045382976531982,
      "mean_token_accuracy": 0.7210699692368507,
      "num_tokens": 243342.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.00683369068428874,
      "rewards/margins": -0.01820890378439799,
      "rewards/rejected": 0.025042592780664563,
      "step": 4
    },
    {
      "entropy": 1.2270928248763084,
      "epoch": 0.0031352876626430477,
      "grad_norm": 32.66579055786133,
      "learning_rate": 4e-08,
      "logits/chosen": -16.499294980475682,
      "logits/rejected": -17.05921506779658,
      "logps/chosen": -709.8800754547119,
      "logps/rejected": -586.3846015930176,
      "loss": 1.0000782012939453,
      "mean_token_accuracy": 0.7035035043954849,
      "num_tokens": 295956.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.004917382728308439,
      "rewards/margins": -0.00028197653591632843,
      "rewards/rejected": -0.004635405610315502,
      "step": 5
    },
    {
      "entropy": 0.8437139950692654,
      "epoch": 0.003762345195171657,
      "grad_norm": 35.4331169128418,
      "learning_rate": 5e-08,
      "logits/chosen": -21.37335490682356,
      "logits/rejected": -21.437480680261856,
      "logps/chosen": -284.04754638671875,
      "logps/rejected": -445.4689540863037,
      "loss": 0.9983432292938232,
      "mean_token_accuracy": 0.8082250654697418,
      "num_tokens": 342091.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.01554340252187103,
      "rewards/margins": 0.0067013868829235435,
      "rewards/rejected": 0.008842015580739826,
      "step": 6
    },
    {
      "entropy": 1.0479743406176567,
      "epoch": 0.004389402727700266,
      "grad_norm": 37.445518493652344,
      "learning_rate": 6e-08,
      "logits/chosen": -18.649493652023672,
      "logits/rejected": -18.4066487416478,
      "logps/chosen": -805.4484777450562,
      "logps/rejected": -680.746976852417,
      "loss": 0.9999387264251709,
      "mean_token_accuracy": 0.7536595985293388,
      "num_tokens": 402856.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.002710643340833485,
      "rewards/margins": 0.00014570658095180988,
      "rewards/rejected": 0.0025649368471931666,
      "step": 7
    },
    {
      "entropy": 0.9191301837563515,
      "epoch": 0.005016460260228876,
      "grad_norm": 38.56687927246094,
      "learning_rate": 7e-08,
      "logits/chosen": -23.125927299845493,
      "logits/rejected": -18.183520901420447,
      "logps/chosen": -924.5588388442993,
      "logps/rejected": -373.86534118652344,
      "loss": 0.9900327324867249,
      "mean_token_accuracy": 0.7644752189517021,
      "num_tokens": 466828.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.01720759950694628,
      "rewards/margins": 0.04020594718167558,
      "rewards/rejected": -0.02299834630684927,
      "step": 8
    },
    {
      "entropy": 0.9575163498520851,
      "epoch": 0.005643517792757485,
      "grad_norm": 40.65217590332031,
      "learning_rate": 8e-08,
      "logits/chosen": -21.412901522784708,
      "logits/rejected": -21.05138834665733,
      "logps/chosen": -816.7314529418945,
      "logps/rejected": -663.6353340148926,
      "loss": 1.0026217699050903,
      "mean_token_accuracy": 0.771336242556572,
      "num_tokens": 527777.0,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.005182235909160227,
      "rewards/margins": -0.010497396113350987,
      "rewards/rejected": 0.005315160844475031,
      "step": 9
    },
    {
      "entropy": 1.193418636918068,
      "epoch": 0.006270575325286095,
      "grad_norm": 42.18134689331055,
      "learning_rate": 9e-08,
      "logits/chosen": -20.675864147312936,
      "logits/rejected": -20.81446933923923,
      "logps/chosen": -905.0363216400146,
      "logps/rejected": -529.3571662902832,
      "loss": 1.0143247842788696,
      "mean_token_accuracy": 0.7211106270551682,
      "num_tokens": 582379.0,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": -0.04024720628513023,
      "rewards/margins": -0.05793563392944634,
      "rewards/rejected": 0.01768842909950763,
      "step": 10
    },
    {
      "entropy": 1.0013729706406593,
      "epoch": 0.006897632857814705,
      "grad_norm": 26.49334144592285,
      "learning_rate": 1e-07,
      "logits/chosen": -18.441954576827186,
      "logits/rejected": -20.784630543895993,
      "logps/chosen": -532.7153358459473,
      "logps/rejected": -424.26265144348145,
      "loss": 0.9981816411018372,
      "mean_token_accuracy": 0.7596741318702698,
      "num_tokens": 629715.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.00034768966725096107,
      "rewards/margins": 0.007268631597980857,
      "rewards/rejected": -0.006920939544215798,
      "step": 11
    },
    {
      "entropy": 1.0685330666601658,
      "epoch": 0.007524690390343314,
      "grad_norm": 34.10297393798828,
      "learning_rate": 1.0999999999999999e-07,
      "logits/chosen": -20.029515190554566,
      "logits/rejected": -21.2224080516646,
      "logps/chosen": -728.1428518295288,
      "logps/rejected": -420.13951873779297,
      "loss": 0.9948853850364685,
      "mean_token_accuracy": 0.7521882280707359,
      "num_tokens": 675406.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.010362100991187617,
      "rewards/margins": 0.020601681084372103,
      "rewards/rejected": -0.030963782221078873,
      "step": 12
    },
    {
      "entropy": 1.0431296452879906,
      "epoch": 0.008151747922871924,
      "grad_norm": 26.776187896728516,
      "learning_rate": 1.2e-07,
      "logits/chosen": -19.538335123973816,
      "logits/rejected": -20.38955248873754,
      "logps/chosen": -445.5426654815674,
      "logps/rejected": -416.24525260925293,
      "loss": 0.997660756111145,
      "mean_token_accuracy": 0.7694718763232231,
      "num_tokens": 722623.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010003616625908762,
      "rewards/margins": 0.009433707047719508,
      "rewards/rejected": 0.0005699098983313888,
      "step": 13
    },
    {
      "entropy": 1.328450158238411,
      "epoch": 0.008778805455400532,
      "grad_norm": 40.44959259033203,
      "learning_rate": 1.3e-07,
      "logits/chosen": -16.50173976340228,
      "logits/rejected": -18.966777530094504,
      "logps/chosen": -1086.8657112121582,
      "logps/rejected": -677.222972869873,
      "loss": 0.9905858039855957,
      "mean_token_accuracy": 0.6911511719226837,
      "num_tokens": 786297.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.018621409428305924,
      "rewards/margins": 0.03770919982343912,
      "rewards/rejected": -0.01908779090445023,
      "step": 14
    },
    {
      "entropy": 1.0233074873685837,
      "epoch": 0.009405862987929142,
      "grad_norm": 28.252763748168945,
      "learning_rate": 1.4e-07,
      "logits/chosen": -17.86138401016042,
      "logits/rejected": -19.831209346381634,
      "logps/chosen": -449.9010829925537,
      "logps/rejected": -492.7903289794922,
      "loss": 1.0015244483947754,
      "mean_token_accuracy": 0.7536996155977249,
      "num_tokens": 837006.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.0008059862302616239,
      "rewards/margins": -0.006120585021562874,
      "rewards/rejected": 0.006926572299562395,
      "step": 15
    },
    {
      "entropy": 1.1438564285635948,
      "epoch": 0.010032920520457752,
      "grad_norm": 39.57615280151367,
      "learning_rate": 1.5e-07,
      "logits/chosen": -17.861874499690074,
      "logits/rejected": -19.590980097470435,
      "logps/chosen": -791.6395130157471,
      "logps/rejected": -567.6202964782715,
      "loss": 0.9942368268966675,
      "mean_token_accuracy": 0.7436006516218185,
      "num_tokens": 896266.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.008153757720720023,
      "rewards/margins": 0.023181513650342822,
      "rewards/rejected": -0.015027755754999816,
      "step": 16
    },
    {
      "entropy": 1.1797396391630173,
      "epoch": 0.010659978052986362,
      "grad_norm": 31.603225708007812,
      "learning_rate": 1.6e-07,
      "logits/chosen": -16.27227861739119,
      "logits/rejected": -20.263959422316383,
      "logps/chosen": -687.8001708984375,
      "logps/rejected": -614.8188095092773,
      "loss": 1.00355064868927,
      "mean_token_accuracy": 0.7303128838539124,
      "num_tokens": 962374.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.0015672160079702735,
      "rewards/margins": -0.014266991434851661,
      "rewards/rejected": 0.012699775630608201,
      "step": 17
    },
    {
      "entropy": 0.9821090400218964,
      "epoch": 0.01128703558551497,
      "grad_norm": 30.358339309692383,
      "learning_rate": 1.7000000000000001e-07,
      "logits/chosen": -19.800772516773982,
      "logits/rejected": -17.959288041851217,
      "logps/chosen": -607.3202495574951,
      "logps/rejected": -534.6925010681152,
      "loss": 1.002077579498291,
      "mean_token_accuracy": 0.7688523679971695,
      "num_tokens": 1016007.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0275348040740937,
      "rewards/margins": -0.008413461735472083,
      "rewards/rejected": -0.019121342920698225,
      "step": 18
    },
    {
      "entropy": 1.2196400687098503,
      "epoch": 0.01191409311804358,
      "grad_norm": 37.30662536621094,
      "learning_rate": 1.8e-07,
      "logits/chosen": -17.72504789763709,
      "logits/rejected": -16.12664834863506,
      "logps/chosen": -813.8953742980957,
      "logps/rejected": -505.71807956695557,
      "loss": 1.004115343093872,
      "mean_token_accuracy": 0.7162602841854095,
      "num_tokens": 1080373.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.006188086466863751,
      "rewards/margins": -0.016581419855356216,
      "rewards/rejected": 0.010393334203399718,
      "step": 19
    },
    {
      "entropy": 1.1595325469970703,
      "epoch": 0.01254115065057219,
      "grad_norm": 44.13706970214844,
      "learning_rate": 1.8999999999999998e-07,
      "logits/chosen": -19.441370010137216,
      "logits/rejected": -19.42652252828183,
      "logps/chosen": -1272.5718240737915,
      "logps/rejected": -622.7399673461914,
      "loss": 1.0018830299377441,
      "mean_token_accuracy": 0.7498048022389412,
      "num_tokens": 1154517.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.02443491853773594,
      "rewards/margins": -0.007660747622139752,
      "rewards/rejected": -0.016774169402197003,
      "step": 20
    },
    {
      "entropy": 1.1010279133915901,
      "epoch": 0.013168208183100799,
      "grad_norm": 41.25276184082031,
      "learning_rate": 2e-07,
      "logits/chosen": -21.419456943034934,
      "logits/rejected": -18.181673739978688,
      "logps/chosen": -728.3854866027832,
      "logps/rejected": -525.0471820831299,
      "loss": 0.9977896213531494,
      "mean_token_accuracy": 0.7623501494526863,
      "num_tokens": 1212935.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.010096669750055298,
      "rewards/margins": 0.008904347196221352,
      "rewards/rejected": 0.0011923184501938522,
      "step": 21
    },
    {
      "entropy": 1.028150089085102,
      "epoch": 0.01379526571562941,
      "grad_norm": 36.785099029541016,
      "learning_rate": 2.0999999999999997e-07,
      "logits/chosen": -21.92485367743462,
      "logits/rejected": -18.499413813157435,
      "logps/chosen": -951.3307018280029,
      "logps/rejected": -426.83284759521484,
      "loss": 0.9958165884017944,
      "mean_token_accuracy": 0.7534473612904549,
      "num_tokens": 1281702.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.011519648949615657,
      "rewards/margins": 0.01635956938844174,
      "rewards/rejected": -0.027879221714101732,
      "step": 22
    },
    {
      "entropy": 0.9120204672217369,
      "epoch": 0.01442232324815802,
      "grad_norm": 36.060447692871094,
      "learning_rate": 2.1999999999999998e-07,
      "logits/chosen": -20.064261738727723,
      "logits/rejected": -21.198644490481797,
      "logps/chosen": -380.3253688812256,
      "logps/rejected": -429.5263252258301,
      "loss": 0.9947471022605896,
      "mean_token_accuracy": 0.7963218614459038,
      "num_tokens": 1327486.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.007161529851146042,
      "rewards/margins": 0.02124298777198419,
      "rewards/rejected": -0.014081457338761538,
      "step": 23
    },
    {
      "entropy": 1.282885067164898,
      "epoch": 0.015049380780686628,
      "grad_norm": 29.555295944213867,
      "learning_rate": 2.3e-07,
      "logits/chosen": -14.928810725754646,
      "logits/rejected": -15.783012175933775,
      "logps/chosen": -594.7181587219238,
      "logps/rejected": -425.2836685180664,
      "loss": 1.0085357427597046,
      "mean_token_accuracy": 0.7083301916718483,
      "num_tokens": 1377436.0,
      "rewards/accuracies": 0.28125,
      "rewards/chosen": -0.018809219647664577,
      "rewards/margins": -0.03420144016854465,
      "rewards/rejected": 0.015392220113426447,
      "step": 24
    },
    {
      "entropy": 1.170431300997734,
      "epoch": 0.015676438313215236,
      "grad_norm": 30.94621467590332,
      "learning_rate": 2.4e-07,
      "logits/chosen": -15.975754306334057,
      "logits/rejected": -18.080566240707103,
      "logps/chosen": -616.5587253570557,
      "logps/rejected": -574.7409896850586,
      "loss": 0.995916485786438,
      "mean_token_accuracy": 0.7270784974098206,
      "num_tokens": 1424961.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010408571077277884,
      "rewards/margins": 0.01637996477074921,
      "rewards/rejected": -0.00597139319870621,
      "step": 25
    },
    {
      "entropy": 0.8303776457905769,
      "epoch": 0.016303495845743848,
      "grad_norm": 33.599998474121094,
      "learning_rate": 2.5e-07,
      "logits/chosen": -25.26339476179235,
      "logits/rejected": -21.119193724011815,
      "logps/chosen": -830.6931796073914,
      "logps/rejected": -321.0673656463623,
      "loss": 1.0016303062438965,
      "mean_token_accuracy": 0.7916704788804054,
      "num_tokens": 1479087.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.01703651063144207,
      "rewards/margins": -0.007413207786157727,
      "rewards/rejected": -0.009623300982639194,
      "step": 26
    },
    {
      "entropy": 0.991417720913887,
      "epoch": 0.016930553378272456,
      "grad_norm": 36.997718811035156,
      "learning_rate": 2.6e-07,
      "logits/chosen": -21.212152458499872,
      "logits/rejected": -18.952785907097258,
      "logps/chosen": -704.5350914001465,
      "logps/rejected": -370.6160888671875,
      "loss": 0.9924548864364624,
      "mean_token_accuracy": 0.7522578835487366,
      "num_tokens": 1545047.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.017328572226688266,
      "rewards/margins": 0.03090921661350876,
      "rewards/rejected": -0.013580642873421311,
      "step": 27
    },
    {
      "entropy": 0.9894383996725082,
      "epoch": 0.017557610910801064,
      "grad_norm": 32.38890838623047,
      "learning_rate": 2.7e-07,
      "logits/chosen": -15.813927428167577,
      "logits/rejected": -17.403900338000618,
      "logps/chosen": -491.92512130737305,
      "logps/rejected": -495.05503511428833,
      "loss": 0.9995163083076477,
      "mean_token_accuracy": 0.7497244253754616,
      "num_tokens": 1594476.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008761245291680098,
      "rewards/margins": 0.001895940862596035,
      "rewards/rejected": -0.010657186910975724,
      "step": 28
    },
    {
      "entropy": 0.9598497748374939,
      "epoch": 0.018184668443329676,
      "grad_norm": 37.41246795654297,
      "learning_rate": 2.8e-07,
      "logits/chosen": -19.56551149166531,
      "logits/rejected": -19.280089414645445,
      "logps/chosen": -857.0342273712158,
      "logps/rejected": -602.1430015563965,
      "loss": 0.9875982403755188,
      "mean_token_accuracy": 0.7813946753740311,
      "num_tokens": 1660065.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.013675417518243194,
      "rewards/margins": 0.05004646920133382,
      "rewards/rejected": -0.03637105316738598,
      "step": 29
    },
    {
      "entropy": 1.324970230460167,
      "epoch": 0.018811725975858284,
      "grad_norm": 36.4524040222168,
      "learning_rate": 2.9e-07,
      "logits/chosen": -21.331314482840543,
      "logits/rejected": -18.526820149021674,
      "logps/chosen": -1117.674301147461,
      "logps/rejected": -583.0690956115723,
      "loss": 1.0027996301651,
      "mean_token_accuracy": 0.687925897538662,
      "num_tokens": 1720616.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.020995987113565207,
      "rewards/margins": -0.01269948878325522,
      "rewards/rejected": -0.008296501007862389,
      "step": 30
    },
    {
      "entropy": 1.0734140500426292,
      "epoch": 0.019438783508386896,
      "grad_norm": 46.66982650756836,
      "learning_rate": 3e-07,
      "logits/chosen": -16.946001394868276,
      "logits/rejected": -16.085758178871664,
      "logps/chosen": -1352.3280754089355,
      "logps/rejected": -446.2639093399048,
      "loss": 0.9967788457870483,
      "mean_token_accuracy": 0.7191508114337921,
      "num_tokens": 1811101.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0069078231463208795,
      "rewards/margins": 0.012905016890726984,
      "rewards/rejected": -0.005997193278744817,
      "step": 31
    },
    {
      "entropy": 1.323067456483841,
      "epoch": 0.020065841040915505,
      "grad_norm": 43.25249099731445,
      "learning_rate": 3.1e-07,
      "logits/chosen": -17.15731645945775,
      "logits/rejected": -16.6517620046484,
      "logps/chosen": -1334.5013446807861,
      "logps/rejected": -645.0475730895996,
      "loss": 0.9926663637161255,
      "mean_token_accuracy": 0.7025224380195141,
      "num_tokens": 1874585.0,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0028306127060204744,
      "rewards/margins": 0.0292252313811332,
      "rewards/rejected": -0.032055844203568995,
      "step": 32
    },
    {
      "entropy": 1.0061679631471634,
      "epoch": 0.020692898573444113,
      "grad_norm": 31.95186996459961,
      "learning_rate": 3.2e-07,
      "logits/chosen": -19.57589939389031,
      "logits/rejected": -19.90894640965601,
      "logps/chosen": -616.9296951293945,
      "logps/rejected": -426.7467842102051,
      "loss": 1.0076781511306763,
      "mean_token_accuracy": 0.7674555331468582,
      "num_tokens": 1923915.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.03830282250419259,
      "rewards/margins": -0.031945616006851196,
      "rewards/rejected": -0.006357205333188176,
      "step": 33
    },
    {
      "entropy": 0.9171701222658157,
      "epoch": 0.021319956105972725,
      "grad_norm": 40.395572662353516,
      "learning_rate": 3.3e-07,
      "logits/chosen": -23.769768694096186,
      "logits/rejected": -21.32001457302996,
      "logps/chosen": -1274.3026218414307,
      "logps/rejected": -430.8449001312256,
      "loss": 0.991915225982666,
      "mean_token_accuracy": 0.7600029706954956,
      "num_tokens": 1992060.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0150325192080345,
      "rewards/margins": 0.03390131541527808,
      "rewards/rejected": -0.01886879827361554,
      "step": 34
    },
    {
      "entropy": 1.1749595329165459,
      "epoch": 0.021947013638501333,
      "grad_norm": 44.95557403564453,
      "learning_rate": 3.4000000000000003e-07,
      "logits/chosen": -20.226573322321272,
      "logits/rejected": -19.058830108404514,
      "logps/chosen": -1253.8970890045166,
      "logps/rejected": -554.8555345535278,
      "loss": 1.0085124969482422,
      "mean_token_accuracy": 0.7034121379256248,
      "num_tokens": 2055149.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.031581409042701125,
      "rewards/margins": -0.03442110400646925,
      "rewards/rejected": 0.002839697408489883,
      "step": 35
    },
    {
      "entropy": 1.1094688102602959,
      "epoch": 0.02257407117102994,
      "grad_norm": 40.33211135864258,
      "learning_rate": 3.5e-07,
      "logits/chosen": -20.24939867823679,
      "logits/rejected": -20.34034861023941,
      "logps/chosen": -832.8481521606445,
      "logps/rejected": -694.2218322753906,
      "loss": 0.9992198944091797,
      "mean_token_accuracy": 0.7405928075313568,
      "num_tokens": 2115443.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.009844484797213227,
      "rewards/margins": 0.0029849251732230186,
      "rewards/rejected": -0.01282941095996648,
      "step": 36
    },
    {
      "entropy": 0.9377781078219414,
      "epoch": 0.023201128703558553,
      "grad_norm": 34.241668701171875,
      "learning_rate": 3.6e-07,
      "logits/chosen": -23.224412647259843,
      "logits/rejected": -21.117019989134523,
      "logps/chosen": -721.9840965270996,
      "logps/rejected": -388.344877243042,
      "loss": 0.99440598487854,
      "mean_token_accuracy": 0.7658621892333031,
      "num_tokens": 2172629.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.016015969216823578,
      "rewards/margins": 0.022238188714254647,
      "rewards/rejected": -0.0062222188571467996,
      "step": 37
    },
    {
      "entropy": 1.0614213570952415,
      "epoch": 0.02382818623608716,
      "grad_norm": 34.88137435913086,
      "learning_rate": 3.7e-07,
      "logits/chosen": -21.41548795553992,
      "logits/rejected": -18.69351990419123,
      "logps/chosen": -1103.8941230773926,
      "logps/rejected": -619.8993282318115,
      "loss": 1.011991024017334,
      "mean_token_accuracy": 0.7375566810369492,
      "num_tokens": 2243025.0,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0650398419238627,
      "rewards/margins": -0.04807211959268898,
      "rewards/rejected": -0.016967719770036638,
      "step": 38
    },
    {
      "entropy": 1.2264064773917198,
      "epoch": 0.02445524376861577,
      "grad_norm": 32.48948669433594,
      "learning_rate": 3.7999999999999996e-07,
      "logits/chosen": -18.42769778338538,
      "logits/rejected": -16.890675790644988,
      "logps/chosen": -775.1542854309082,
      "logps/rejected": -521.0977792739868,
      "loss": 1.0028043985366821,
      "mean_token_accuracy": 0.7075130566954613,
      "num_tokens": 2294828.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.009019852615892887,
      "rewards/margins": -0.011407412588596344,
      "rewards/rejected": 0.002387559274211526,
      "step": 39
    },
    {
      "entropy": 1.2194309085607529,
      "epoch": 0.02508230130114438,
      "grad_norm": 35.85207748413086,
      "learning_rate": 3.8999999999999997e-07,
      "logits/chosen": -18.869462390156798,
      "logits/rejected": -17.76926630663928,
      "logps/chosen": -866.6771221160889,
      "logps/rejected": -515.5025405883789,
      "loss": 1.0078489780426025,
      "mean_token_accuracy": 0.7282191216945648,
      "num_tokens": 2359584.0,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.019920013728551567,
      "rewards/margins": -0.03206599899567664,
      "rewards/rejected": 0.012145984219387174,
      "step": 40
    },
    {
      "entropy": 0.9393897280097008,
      "epoch": 0.02570935883367299,
      "grad_norm": 29.508663177490234,
      "learning_rate": 4e-07,
      "logits/chosen": -19.542580737616923,
      "logits/rejected": -17.785255602711995,
      "logps/chosen": -440.89880752563477,
      "logps/rejected": -531.4820680618286,
      "loss": 0.9985625147819519,
      "mean_token_accuracy": 0.7809053212404251,
      "num_tokens": 2407189.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0036070493515580893,
      "rewards/margins": 0.005761435255408287,
      "rewards/rejected": -0.002154385729227215,
      "step": 41
    },
    {
      "entropy": 1.1858881711959839,
      "epoch": 0.026336416366201598,
      "grad_norm": 34.17562484741211,
      "learning_rate": 4.0999999999999994e-07,
      "logits/chosen": -20.121897634772164,
      "logits/rejected": -19.622514667746742,
      "logps/chosen": -804.0400094985962,
      "logps/rejected": -586.3046684265137,
      "loss": 0.9875714778900146,
      "mean_token_accuracy": 0.7288667857646942,
      "num_tokens": 2459432.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.0406644003232941,
      "rewards/margins": 0.05225609429180622,
      "rewards/rejected": -0.011591696529649198,
      "step": 42
    },
    {
      "entropy": 0.8192418366670609,
      "epoch": 0.02696347389873021,
      "grad_norm": 43.9438591003418,
      "learning_rate": 4.1999999999999995e-07,
      "logits/chosen": -24.092886788439017,
      "logits/rejected": -23.664276651461066,
      "logps/chosen": -584.1518545150757,
      "logps/rejected": -503.76978302001953,
      "loss": 0.9815411567687988,
      "mean_token_accuracy": 0.8075317144393921,
      "num_tokens": 2512702.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04232085426338017,
      "rewards/margins": 0.07526338030584157,
      "rewards/rejected": -0.032942528603598475,
      "step": 43
    },
    {
      "entropy": 0.9453827887773514,
      "epoch": 0.02759053143125882,
      "grad_norm": 35.92549514770508,
      "learning_rate": 4.2999999999999996e-07,
      "logits/chosen": -18.338003960845416,
      "logits/rejected": -19.673824728737838,
      "logps/chosen": -925.3223133087158,
      "logps/rejected": -475.0994644165039,
      "loss": 0.9924615025520325,
      "mean_token_accuracy": 0.769691713154316,
      "num_tokens": 2575249.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02094321296317503,
      "rewards/margins": 0.03026525373570621,
      "rewards/rejected": -0.009322041762061417,
      "step": 44
    },
    {
      "entropy": 0.8736219257116318,
      "epoch": 0.028217588963787427,
      "grad_norm": 33.97110366821289,
      "learning_rate": 4.3999999999999997e-07,
      "logits/chosen": -22.332301111125155,
      "logits/rejected": -20.832310241019115,
      "logps/chosen": -462.5166606903076,
      "logps/rejected": -383.00551986694336,
      "loss": 1.0048575401306152,
      "mean_token_accuracy": 0.7859590947628021,
      "num_tokens": 2621631.0,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.009766561212018132,
      "rewards/margins": -0.019437916460447013,
      "rewards/rejected": 0.009671354899182916,
      "step": 45
    },
    {
      "entropy": 1.02370435744524,
      "epoch": 0.02884464649631604,
      "grad_norm": 31.75177764892578,
      "learning_rate": 4.5e-07,
      "logits/chosen": -18.283618627987572,
      "logits/rejected": -18.815319615630727,
      "logps/chosen": -466.81629753112793,
      "logps/rejected": -454.2089099884033,
      "loss": 0.9951180219650269,
      "mean_token_accuracy": 0.7507254704833031,
      "num_tokens": 2674748.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.001771716313669458,
      "rewards/margins": 0.019629769725725055,
      "rewards/rejected": -0.01785805242252536,
      "step": 46
    },
    {
      "entropy": 0.9373743236064911,
      "epoch": 0.029471704028844647,
      "grad_norm": 42.6140251159668,
      "learning_rate": 4.6e-07,
      "logits/chosen": -25.528625869209804,
      "logits/rejected": -21.64687665278925,
      "logps/chosen": -1166.7054986953735,
      "logps/rejected": -438.6943473815918,
      "loss": 0.9925483465194702,
      "mean_token_accuracy": 0.759131945669651,
      "num_tokens": 2744660.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.005512146046385169,
      "rewards/margins": 0.03011180693283677,
      "rewards/rejected": -0.024599659722298384,
      "step": 47
    },
    {
      "entropy": 0.9783306047320366,
      "epoch": 0.030098761561373255,
      "grad_norm": 35.963809967041016,
      "learning_rate": 4.6999999999999995e-07,
      "logits/chosen": -19.222043644762906,
      "logits/rejected": -18.896907946229895,
      "logps/chosen": -515.0348672866821,
      "logps/rejected": -419.7729835510254,
      "loss": 0.9998578429222107,
      "mean_token_accuracy": 0.792612262070179,
      "num_tokens": 2790068.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.02343193959677592,
      "rewards/margins": 0.0005922880955040455,
      "rewards/rejected": -0.024024227866902947,
      "step": 48
    },
    {
      "entropy": 1.10813407599926,
      "epoch": 0.030725819093901867,
      "grad_norm": 35.080543518066406,
      "learning_rate": 4.8e-07,
      "logits/chosen": -22.976597678281987,
      "logits/rejected": -20.831361335827353,
      "logps/chosen": -826.8215522766113,
      "logps/rejected": -406.26672554016113,
      "loss": 1.008866310119629,
      "mean_token_accuracy": 0.7337941229343414,
      "num_tokens": 2845743.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.02741124981548637,
      "rewards/margins": -0.03900438791606575,
      "rewards/rejected": 0.011593140079639852,
      "step": 49
    },
    {
      "entropy": 1.3011442199349403,
      "epoch": 0.03135287662643047,
      "grad_norm": 32.856292724609375,
      "learning_rate": 4.9e-07,
      "logits/chosen": -14.496447176077387,
      "logits/rejected": -17.84699543352937,
      "logps/chosen": -565.9537868499756,
      "logps/rejected": -459.75712490081787,
      "loss": 0.9952901005744934,
      "mean_token_accuracy": 0.7119247242808342,
      "num_tokens": 2899233.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.004313211014959961,
      "rewards/margins": 0.018873692606575787,
      "rewards/rejected": -0.014560480834916234,
      "step": 50
    },
    {
      "entropy": 1.2501383051276207,
      "epoch": 0.031979934158959084,
      "grad_norm": 36.4530143737793,
      "learning_rate": 5e-07,
      "logits/chosen": -21.17299309242602,
      "logits/rejected": -22.865218106484676,
      "logps/chosen": -866.4957427978516,
      "logps/rejected": -508.9821949005127,
      "loss": 1.0008279085159302,
      "mean_token_accuracy": 0.7066351845860481,
      "num_tokens": 2952150.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0076497383415699005,
      "rewards/margins": -0.003307956736534834,
      "rewards/rejected": -0.004341780979302712,
      "step": 51
    },
    {
      "entropy": 1.1913169473409653,
      "epoch": 0.032606991691487695,
      "grad_norm": 40.79720687866211,
      "learning_rate": 5.1e-07,
      "logits/chosen": -17.331403938671592,
      "logits/rejected": -16.946636435404315,
      "logps/chosen": -760.1813850402832,
      "logps/rejected": -703.0359497070312,
      "loss": 1.0072510242462158,
      "mean_token_accuracy": 0.7222515121102333,
      "num_tokens": 3005503.0,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.013277458609081805,
      "rewards/margins": -0.029311579186469316,
      "rewards/rejected": 0.01603412051917985,
      "step": 52
    },
    {
      "entropy": 1.3226243034005165,
      "epoch": 0.0332340492240163,
      "grad_norm": 37.33218002319336,
      "learning_rate": 5.2e-07,
      "logits/chosen": -19.339108194716523,
      "logits/rejected": -17.709144071414954,
      "logps/chosen": -1036.9900550842285,
      "logps/rejected": -671.3515205383301,
      "loss": 1.0005102157592773,
      "mean_token_accuracy": 0.699407085776329,
      "num_tokens": 3062231.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.01915261917747557,
      "rewards/margins": -0.002298903651535511,
      "rewards/rejected": -0.016853714711032808,
      "step": 53
    },
    {
      "entropy": 1.2621377930045128,
      "epoch": 0.03386110675654491,
      "grad_norm": 36.86682891845703,
      "learning_rate": 5.3e-07,
      "logits/chosen": -23.098386869669465,
      "logits/rejected": -18.559492736951306,
      "logps/chosen": -879.0279788970947,
      "logps/rejected": -472.2311248779297,
      "loss": 0.9976851940155029,
      "mean_token_accuracy": 0.687630370259285,
      "num_tokens": 3118527.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.01943317288532853,
      "rewards/margins": 0.009305761312134564,
      "rewards/rejected": -0.028738934081047773,
      "step": 54
    },
    {
      "entropy": 1.0990462750196457,
      "epoch": 0.034488164289073524,
      "grad_norm": 29.28276824951172,
      "learning_rate": 5.4e-07,
      "logits/chosen": -17.590637043087355,
      "logits/rejected": -16.153945977490892,
      "logps/chosen": -548.3314371109009,
      "logps/rejected": -324.43446254730225,
      "loss": 0.9927998781204224,
      "mean_token_accuracy": 0.7380417436361313,
      "num_tokens": 3164828.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.007261359598487616,
      "rewards/margins": 0.02887870534323156,
      "rewards/rejected": -0.0216173455119133,
      "step": 55
    },
    {
      "entropy": 1.2540519461035728,
      "epoch": 0.03511522182160213,
      "grad_norm": 36.696895599365234,
      "learning_rate": 5.5e-07,
      "logits/chosen": -15.279084980657903,
      "logits/rejected": -15.824559065606763,
      "logps/chosen": -903.7292518615723,
      "logps/rejected": -541.010986328125,
      "loss": 0.9985660314559937,
      "mean_token_accuracy": 0.6998207569122314,
      "num_tokens": 3233270.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.016322331794071943,
      "rewards/margins": 0.0056688676122576,
      "rewards/rejected": -0.021991199755575508,
      "step": 56
    },
    {
      "entropy": 1.0729844830930233,
      "epoch": 0.03574227935413074,
      "grad_norm": 26.47355842590332,
      "learning_rate": 5.6e-07,
      "logits/chosen": -19.623318897458958,
      "logits/rejected": -20.258368385656922,
      "logps/chosen": -469.5618600845337,
      "logps/rejected": -361.1621437072754,
      "loss": 1.0013068914413452,
      "mean_token_accuracy": 0.7484212070703506,
      "num_tokens": 3268958.0,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004558162530884147,
      "rewards/margins": -0.005245101172477007,
      "rewards/rejected": 0.009803264052607119,
      "step": 57
    },
    {
      "entropy": 0.9888226762413979,
      "epoch": 0.03636933688665935,
      "grad_norm": 28.04119873046875,
      "learning_rate": 5.699999999999999e-07,
      "logits/chosen": -18.97722177970246,
      "logits/rejected": -17.818980544574217,
      "logps/chosen": -490.36932277679443,
      "logps/rejected": -353.6288299560547,
      "loss": 0.9960744976997375,
      "mean_token_accuracy": 0.763416476547718,
      "num_tokens": 3318736.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0007263210136443377,
      "rewards/margins": 0.015722022857517004,
      "rewards/rejected": -0.016448343638330698,
      "step": 58
    },
    {
      "entropy": 1.2886198982596397,
      "epoch": 0.03699639441918796,
      "grad_norm": 43.716182708740234,
      "learning_rate": 5.8e-07,
      "logits/chosen": -20.97464568747773,
      "logits/rejected": -18.98481146897743,
      "logps/chosen": -1585.0814723968506,
      "logps/rejected": -528.0391731262207,
      "loss": 0.9945598840713501,
      "mean_token_accuracy": 0.7001667320728302,
      "num_tokens": 3386147.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.019920118735171854,
      "rewards/margins": 0.021774652879685163,
      "rewards/rejected": -0.0018545325146988034,
      "step": 59
    },
    {
      "entropy": 1.039273351430893,
      "epoch": 0.03762345195171657,
      "grad_norm": 27.81415367126465,
      "learning_rate": 5.9e-07,
      "logits/chosen": -18.662482620406607,
      "logits/rejected": -20.1330546037938,
      "logps/chosen": -365.54387283325195,
      "logps/rejected": -373.88531494140625,
      "loss": 0.996242105960846,
      "mean_token_accuracy": 0.7572688236832619,
      "num_tokens": 3424538.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.006609153002500534,
      "rewards/margins": 0.015012014424428344,
      "rewards/rejected": -0.02162116818362847,
      "step": 60
    },
    {
      "entropy": 1.2042050436139107,
      "epoch": 0.03825050948424518,
      "grad_norm": 36.16862869262695,
      "learning_rate": 6e-07,
      "logits/chosen": -20.75980830973855,
      "logits/rejected": -20.60451823594014,
      "logps/chosen": -1099.3400793075562,
      "logps/rejected": -421.7832374572754,
      "loss": 1.0015130043029785,
      "mean_token_accuracy": 0.7048044949769974,
      "num_tokens": 3491026.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.00036664726212620735,
      "rewards/margins": -0.005983836483210325,
      "rewards/rejected": 0.00635048293042928,
      "step": 61
    },
    {
      "entropy": 1.0666373148560524,
      "epoch": 0.03887756701677379,
      "grad_norm": 43.32762908935547,
      "learning_rate": 6.1e-07,
      "logits/chosen": -23.173134701752,
      "logits/rejected": -20.672318471013675,
      "logps/chosen": -1242.8998069763184,
      "logps/rejected": -464.8951187133789,
      "loss": 0.9945209622383118,
      "mean_token_accuracy": 0.7305290177464485,
      "num_tokens": 3555810.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.005398005829192698,
      "rewards/margins": 0.022236518329009414,
      "rewards/rejected": -0.016838514362461865,
      "step": 62
    },
    {
      "entropy": 0.7979357466101646,
      "epoch": 0.0395046245493024,
      "grad_norm": 30.137588500976562,
      "learning_rate": 6.2e-07,
      "logits/chosen": -22.89783292156173,
      "logits/rejected": -22.199726863036197,
      "logps/chosen": -536.3147115707397,
      "logps/rejected": -364.88674545288086,
      "loss": 0.9864821434020996,
      "mean_token_accuracy": 0.8112821727991104,
      "num_tokens": 3604824.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.03518715803511441,
      "rewards/margins": 0.056509705260396004,
      "rewards/rejected": -0.02132254181196913,
      "step": 63
    },
    {
      "entropy": 1.1044994071125984,
      "epoch": 0.04013168208183101,
      "grad_norm": 29.391481399536133,
      "learning_rate": 6.3e-07,
      "logits/chosen": -24.507481146010182,
      "logits/rejected": -22.21625134174552,
      "logps/chosen": -702.180121421814,
      "logps/rejected": -308.05684661865234,
      "loss": 0.9998461604118347,
      "mean_token_accuracy": 0.7285795137286186,
      "num_tokens": 3651250.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.004402894992381334,
      "rewards/margins": -0.0003644675016403198,
      "rewards/rejected": -0.004038428800413385,
      "step": 64
    },
    {
      "entropy": 0.8255317062139511,
      "epoch": 0.04075873961435962,
      "grad_norm": 36.75194549560547,
      "learning_rate": 6.4e-07,
      "logits/chosen": -23.442134502527015,
      "logits/rejected": -21.24577292120065,
      "logps/chosen": -481.1418218612671,
      "logps/rejected": -390.04198455810547,
      "loss": 0.9808884859085083,
      "mean_token_accuracy": 0.8040206730365753,
      "num_tokens": 3700811.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03725379565730691,
      "rewards/margins": 0.0775255209300667,
      "rewards/rejected": -0.040271724574267864,
      "step": 65
    },
    {
      "entropy": 0.9590631648898125,
      "epoch": 0.041385797146888226,
      "grad_norm": 31.754987716674805,
      "learning_rate": 6.5e-07,
      "logits/chosen": -23.604474652863118,
      "logits/rejected": -21.18440918134874,
      "logps/chosen": -892.4836721420288,
      "logps/rejected": -423.81532859802246,
      "loss": 0.9878406524658203,
      "mean_token_accuracy": 0.7679838165640831,
      "num_tokens": 3752809.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.035222190992499236,
      "rewards/margins": 0.05575526849133894,
      "rewards/rejected": -0.020533079630695283,
      "step": 66
    },
    {
      "entropy": 0.9526357278227806,
      "epoch": 0.04201285467941684,
      "grad_norm": 38.09070587158203,
      "learning_rate": 6.6e-07,
      "logits/chosen": -19.300392282829993,
      "logits/rejected": -20.892184024152968,
      "logps/chosen": -767.138952255249,
      "logps/rejected": -475.6669750213623,
      "loss": 0.9936155080795288,
      "mean_token_accuracy": 0.7684061825275421,
      "num_tokens": 3817303.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02726819575764239,
      "rewards/margins": 0.025456703966483474,
      "rewards/rejected": 0.0018114912672899663,
      "step": 67
    },
    {
      "entropy": 0.7557753957808018,
      "epoch": 0.04263991221194545,
      "grad_norm": 31.207439422607422,
      "learning_rate": 6.7e-07,
      "logits/chosen": -22.31144983616734,
      "logits/rejected": -22.303479200481075,
      "logps/chosen": -516.0994625091553,
      "logps/rejected": -432.821720123291,
      "loss": 0.993211030960083,
      "mean_token_accuracy": 0.8136529326438904,
      "num_tokens": 3869975.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008595484774559736,
      "rewards/margins": 0.027202091412618756,
      "rewards/rejected": -0.018606607103720307,
      "step": 68
    },
    {
      "entropy": 1.078584998846054,
      "epoch": 0.043266969744474054,
      "grad_norm": 35.849456787109375,
      "learning_rate": 6.800000000000001e-07,
      "logits/chosen": -20.188257880122976,
      "logits/rejected": -19.395488505188574,
      "logps/chosen": -778.6550903320312,
      "logps/rejected": -460.7582092285156,
      "loss": 0.9967914819717407,
      "mean_token_accuracy": 0.739520289003849,
      "num_tokens": 3917591.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0018483520834706724,
      "rewards/margins": 0.013088171719573438,
      "rewards/rejected": -0.014936523628421128,
      "step": 69
    },
    {
      "entropy": 1.0627146326005459,
      "epoch": 0.043894027277002666,
      "grad_norm": 33.957401275634766,
      "learning_rate": 6.9e-07,
      "logits/chosen": -19.09365423111348,
      "logits/rejected": -19.238957994462496,
      "logps/chosen": -625.0276880264282,
      "logps/rejected": -460.5406036376953,
      "loss": 0.9846621155738831,
      "mean_token_accuracy": 0.7464545965194702,
      "num_tokens": 3963982.0,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.005297277413774282,
      "rewards/margins": 0.06182992341928184,
      "rewards/rejected": -0.056532644899562,
      "step": 70
    },
    {
      "entropy": 1.0039971619844437,
      "epoch": 0.04452108480953128,
      "grad_norm": 36.00166702270508,
      "learning_rate": 7e-07,
      "logits/chosen": -21.96269636469623,
      "logits/rejected": -21.29580632185736,
      "logps/chosen": -772.8854808807373,
      "logps/rejected": -615.0897903442383,
      "loss": 0.9930751323699951,
      "mean_token_accuracy": 0.7512786686420441,
      "num_tokens": 4028293.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0168970461236313,
      "rewards/margins": 0.02881601534318179,
      "rewards/rejected": -0.011918970150873065,
      "step": 71
    },
    {
      "entropy": 1.302395537495613,
      "epoch": 0.04514814234205988,
      "grad_norm": 39.504661560058594,
      "learning_rate": 7.1e-07,
      "logits/chosen": -19.878782312586424,
      "logits/rejected": -18.000410834982972,
      "logps/chosen": -1210.3033666610718,
      "logps/rejected": -607.6138458251953,
      "loss": 0.9891315698623657,
      "mean_token_accuracy": 0.703778937458992,
      "num_tokens": 4089535.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.0096849100664258,
      "rewards/margins": 0.043962169205769897,
      "rewards/rejected": -0.034277260303497314,
      "step": 72
    },
    {
      "entropy": 0.9964236989617348,
      "epoch": 0.045775199874588494,
      "grad_norm": 33.938472747802734,
      "learning_rate": 7.2e-07,
      "logits/chosen": -20.415109094760414,
      "logits/rejected": -17.26682495783373,
      "logps/chosen": -721.8188791275024,
      "logps/rejected": -408.42410469055176,
      "loss": 0.995058536529541,
      "mean_token_accuracy": 0.7626092284917831,
      "num_tokens": 4140075.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.015723853604868054,
      "rewards/margins": 0.019197183661162853,
      "rewards/rejected": -0.03492103505413979,
      "step": 73
    },
    {
      "entropy": 1.017537921667099,
      "epoch": 0.046402257407117106,
      "grad_norm": 26.6578311920166,
      "learning_rate": 7.3e-07,
      "logits/chosen": -15.80474927414226,
      "logits/rejected": -16.128822298593438,
      "logps/chosen": -515.7086997032166,
      "logps/rejected": -239.00895309448242,
      "loss": 0.9995874762535095,
      "mean_token_accuracy": 0.7658038064837456,
      "num_tokens": 4194003.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.008220234914915636,
      "rewards/margins": 0.0016142040840350091,
      "rewards/rejected": -0.009834438504185528,
      "step": 74
    },
    {
      "entropy": 0.9735286235809326,
      "epoch": 0.04702931493964571,
      "grad_norm": 33.58881759643555,
      "learning_rate": 7.4e-07,
      "logits/chosen": -20.946241053605,
      "logits/rejected": -19.067083934619777,
      "logps/chosen": -674.0198707580566,
      "logps/rejected": -419.6971263885498,
      "loss": 0.991060733795166,
      "mean_token_accuracy": 0.7505866810679436,
      "num_tokens": 4244222.0,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.015050875255838037,
      "rewards/margins": 0.03578460752032697,
      "rewards/rejected": -0.020733732322696596,
      "step": 75
    },
    {
      "entropy": 1.137626238167286,
      "epoch": 0.04765637247217432,
      "grad_norm": 38.98091125488281,
      "learning_rate": 7.5e-07,
      "logits/chosen": -22.829858763000864,
      "logits/rejected": -19.879969265793594,
      "logps/chosen": -1177.3832550048828,
      "logps/rejected": -511.0323143005371,
      "loss": 0.9940009713172913,
      "mean_token_accuracy": 0.7326341941952705,
      "num_tokens": 4313717.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.021500761155039072,
      "rewards/margins": 0.023731452063657343,
      "rewards/rejected": -0.0452322136843577,
      "step": 76
    },
    {
      "entropy": 1.2288251742720604,
      "epoch": 0.048283430004702935,
      "grad_norm": 28.220809936523438,
      "learning_rate": 7.599999999999999e-07,
      "logits/chosen": -17.088404479801554,
      "logits/rejected": -18.424581121455233,
      "logps/chosen": -600.109245300293,
      "logps/rejected": -335.4466133117676,
      "loss": 1.0012062788009644,
      "mean_token_accuracy": 0.7148683369159698,
      "num_tokens": 4358615.0,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.014499580138362944,
      "rewards/margins": -0.004860305110923946,
      "rewards/rejected": -0.00963927514385432,
      "step": 77
    },
    {
      "entropy": 1.0265920907258987,
      "epoch": 0.04891048753723154,
      "grad_norm": 34.03989791870117,
      "learning_rate": 7.699999999999999e-07,
      "logits/chosen": -22.126280957342644,
      "logits/rejected": -21.625379783333514,
      "logps/chosen": -804.5845255851746,
      "logps/rejected": -453.7879638671875,
      "loss": 0.9970452785491943,
      "mean_token_accuracy": 0.7403309643268585,
      "num_tokens": 4415873.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.008073777018580586,
      "rewards/margins": 0.011700771981850266,
      "rewards/rejected": -0.019774550921283662,
      "step": 78
    },
    {
      "entropy": 1.262473076581955,
      "epoch": 0.04953754506976015,
      "grad_norm": 36.033348083496094,
      "learning_rate": 7.799999999999999e-07,
      "logits/chosen": -18.124917388062837,
      "logits/rejected": -19.211221946008383,
      "logps/chosen": -795.0781173706055,
      "logps/rejected": -577.8308029174805,
      "loss": 0.9934069514274597,
      "mean_token_accuracy": 0.7068465352058411,
      "num_tokens": 4477334.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.004607248993124813,
      "rewards/margins": 0.026270719012245536,
      "rewards/rejected": -0.030877968529239297,
      "step": 79
    },
    {
      "entropy": 0.9185331761837006,
      "epoch": 0.05016460260228876,
      "grad_norm": 36.832279205322266,
      "learning_rate": 7.9e-07,
      "logits/chosen": -20.603378074573,
      "logits/rejected": -17.056396975980135,
      "logps/chosen": -758.8550605773926,
      "logps/rejected": -412.65998458862305,
      "loss": 0.9883898496627808,
      "mean_token_accuracy": 0.7726860344409943,
      "num_tokens": 4533455.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.0016075177991297096,
      "rewards/margins": 0.04657475184649229,
      "rewards/rejected": -0.04818226984934881,
      "step": 80
    },
    {
      "entropy": 1.121558502316475,
      "epoch": 0.05079166013481737,
      "grad_norm": 36.56794738769531,
      "learning_rate": 8e-07,
      "logits/chosen": -20.37862110155874,
      "logits/rejected": -18.992690204889037,
      "logps/chosen": -1184.8006420135498,
      "logps/rejected": -610.0655975341797,
      "loss": 0.9853536486625671,
      "mean_token_accuracy": 0.7357284799218178,
      "num_tokens": 4596652.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.02295642625540495,
      "rewards/margins": 0.059380507678724825,
      "rewards/rejected": -0.03642408235464245,
      "step": 81
    },
    {
      "entropy": 1.0517073720693588,
      "epoch": 0.05141871766734598,
      "grad_norm": 39.78794860839844,
      "learning_rate": 8.1e-07,
      "logits/chosen": -23.220199460872006,
      "logits/rejected": -21.76251201594231,
      "logps/chosen": -901.9024753570557,
      "logps/rejected": -581.7087097167969,
      "loss": 0.9863370656967163,
      "mean_token_accuracy": 0.7483935281634331,
      "num_tokens": 4660574.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.005605014972388744,
      "rewards/margins": 0.05513616371899843,
      "rewards/rejected": -0.04953114781528711,
      "step": 82
    },
    {
      "entropy": 1.031716726720333,
      "epoch": 0.05204577519987459,
      "grad_norm": 34.147884368896484,
      "learning_rate": 8.199999999999999e-07,
      "logits/chosen": -17.43597000771303,
      "logits/rejected": -17.040958005172556,
      "logps/chosen": -611.6086521148682,
      "logps/rejected": -463.6491508483887,
      "loss": 0.9925582408905029,
      "mean_token_accuracy": 0.7591038644313812,
      "num_tokens": 4726456.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.009094578330405056,
      "rewards/margins": 0.029864652664400637,
      "rewards/rejected": -0.03895922936499119,
      "step": 83
    },
    {
      "entropy": 1.1349261552095413,
      "epoch": 0.052672832732403196,
      "grad_norm": 32.638126373291016,
      "learning_rate": 8.299999999999999e-07,
      "logits/chosen": -17.404521363601013,
      "logits/rejected": -19.98172508139376,
      "logps/chosen": -767.6361827850342,
      "logps/rejected": -448.48921966552734,
      "loss": 0.9871995449066162,
      "mean_token_accuracy": 0.7235324308276176,
      "num_tokens": 4792314.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.017612980911508203,
      "rewards/margins": 0.05156451647053473,
      "rewards/rejected": -0.033951534889638424,
      "step": 84
    },
    {
      "entropy": 1.1050493195652962,
      "epoch": 0.05329989026493181,
      "grad_norm": 36.08334732055664,
      "learning_rate": 8.399999999999999e-07,
      "logits/chosen": -16.500999504676678,
      "logits/rejected": -18.25587297431137,
      "logps/chosen": -675.2915000915527,
      "logps/rejected": -561.0027389526367,
      "loss": 0.9919638633728027,
      "mean_token_accuracy": 0.741675615310669,
      "num_tokens": 4849066.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.0004984733532182872,
      "rewards/margins": 0.03216184466145933,
      "rewards/rejected": -0.03166337008588016,
      "step": 85
    },
    {
      "entropy": 1.220378190279007,
      "epoch": 0.05392694779746042,
      "grad_norm": 40.76183319091797,
      "learning_rate": 8.499999999999999e-07,
      "logits/chosen": -23.90722501767462,
      "logits/rejected": -22.38695916119651,
      "logps/chosen": -1048.8524017333984,
      "logps/rejected": -473.95777893066406,
      "loss": 0.9880315661430359,
      "mean_token_accuracy": 0.7109057381749153,
      "num_tokens": 4923190.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0222030496224761,
      "rewards/margins": 0.04874912742525339,
      "rewards/rejected": -0.07095217774622142,
      "step": 86
    },
    {
      "entropy": 1.2091117054224014,
      "epoch": 0.054554005329989025,
      "grad_norm": 35.72580337524414,
      "learning_rate": 8.599999999999999e-07,
      "logits/chosen": -20.30170921476879,
      "logits/rejected": -21.20954152859497,
      "logps/chosen": -951.2183666229248,
      "logps/rejected": -513.4668788909912,
      "loss": 0.9863914251327515,
      "mean_token_accuracy": 0.7123682126402855,
      "num_tokens": 4974868.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.026849399670027196,
      "rewards/margins": 0.05532143288291991,
      "rewards/rejected": -0.028472037287428975,
      "step": 87
    },
    {
      "entropy": 1.073004886507988,
      "epoch": 0.05518106286251764,
      "grad_norm": 36.27262496948242,
      "learning_rate": 8.699999999999999e-07,
      "logits/chosen": -19.685747925382,
      "logits/rejected": -17.267068163974074,
      "logps/chosen": -578.3146362304688,
      "logps/rejected": -475.0130271911621,
      "loss": 0.9875953197479248,
      "mean_token_accuracy": 0.7410635352134705,
      "num_tokens": 5037188.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.014228225103579462,
      "rewards/margins": 0.04966498585417867,
      "rewards/rejected": -0.03543675900436938,
      "step": 88
    },
    {
      "entropy": 1.056903399527073,
      "epoch": 0.05580812039504625,
      "grad_norm": 35.787532806396484,
      "learning_rate": 8.799999999999999e-07,
      "logits/chosen": -20.64660484271999,
      "logits/rejected": -19.71323913593022,
      "logps/chosen": -527.9435653686523,
      "logps/rejected": -577.5134410858154,
      "loss": 0.9897867441177368,
      "mean_token_accuracy": 0.7574738562107086,
      "num_tokens": 5086966.0,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.00012873177183791995,
      "rewards/margins": 0.04125080001540482,
      "rewards/rejected": -0.04112206675927155,
      "step": 89
    },
    {
      "entropy": 0.9986204952001572,
      "epoch": 0.05643517792757485,
      "grad_norm": 33.426368713378906,
      "learning_rate": 8.9e-07,
      "logits/chosen": -18.57544006926955,
      "logits/rejected": -19.12848221709334,
      "logps/chosen": -766.3484830856323,
      "logps/rejected": -476.17732429504395,
      "loss": 0.9895883798599243,
      "mean_token_accuracy": 0.7547919675707817,
      "num_tokens": 5144107.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.013297693920321763,
      "rewards/margins": 0.04271816968685016,
      "rewards/rejected": -0.029420473758364096,
      "step": 90
    },
    {
      "entropy": 1.1071998253464699,
      "epoch": 0.057062235460103465,
      "grad_norm": 46.52241516113281,
      "learning_rate": 9e-07,
      "logits/chosen": -20.750110720341095,
      "logits/rejected": -20.73160492144149,
      "logps/chosen": -1105.2754192352295,
      "logps/rejected": -582.0479850769043,
      "loss": 0.9801906943321228,
      "mean_token_accuracy": 0.7243269085884094,
      "num_tokens": 5211621.0,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.019272988894954324,
      "rewards/margins": 0.0801434232853353,
      "rewards/rejected": -0.06087043380830437,
      "step": 91
    },
    {
      "entropy": 1.1382925510406494,
      "epoch": 0.05768929299263208,
      "grad_norm": 37.77758026123047,
      "learning_rate": 9.1e-07,
      "logits/chosen": -20.904564962987713,
      "logits/rejected": -21.018963758425333,
      "logps/chosen": -640.5741844177246,
      "logps/rejected": -708.7987327575684,
      "loss": 0.9877656102180481,
      "mean_token_accuracy": 0.7462773993611336,
      "num_tokens": 5263236.0,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.032095869741169736,
      "rewards/margins": 0.04855327168479562,
      "rewards/rejected": -0.08064913935959339,
      "step": 92
    },
    {
      "entropy": 1.1960504204034805,
      "epoch": 0.05831635052516068,
      "grad_norm": 27.9012508392334,
      "learning_rate": 9.2e-07,
      "logits/chosen": -17.198728228506177,
      "logits/rejected": -19.989566953539725,
      "logps/chosen": -581.2822170257568,
      "logps/rejected": -365.0048427581787,
      "loss": 0.9875739812850952,
      "mean_token_accuracy": 0.7414722666144371,
      "num_tokens": 5301490.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0007973910542204976,
      "rewards/margins": 0.04991468833759427,
      "rewards/rejected": -0.04911729716695845,
      "step": 93
    },
    {
      "entropy": 0.9865989461541176,
      "epoch": 0.058943408057689294,
      "grad_norm": 31.25222396850586,
      "learning_rate": 9.3e-07,
      "logits/chosen": -18.790141123207366,
      "logits/rejected": -16.14887539333265,
      "logps/chosen": -577.1131019592285,
      "logps/rejected": -340.10778999328613,
      "loss": 0.9858304262161255,
      "mean_token_accuracy": 0.7623919099569321,
      "num_tokens": 5361142.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0011998027912341058,
      "rewards/margins": 0.057147986139170825,
      "rewards/rejected": -0.05594818387180567,
      "step": 94
    },
    {
      "entropy": 1.044435366988182,
      "epoch": 0.059570465590217905,
      "grad_norm": 26.22032928466797,
      "learning_rate": 9.399999999999999e-07,
      "logits/chosen": -17.09738355288199,
      "logits/rejected": -17.458014048093904,
      "logps/chosen": -388.39873027801514,
      "logps/rejected": -326.0696334838867,
      "loss": 0.9898619055747986,
      "mean_token_accuracy": 0.7581661641597748,
      "num_tokens": 5408748.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0021333397598937154,
      "rewards/margins": 0.04059030464850366,
      "rewards/rejected": -0.04272364475764334,
      "step": 95
    },
    {
      "entropy": 0.8697420880198479,
      "epoch": 0.06019752312274651,
      "grad_norm": 42.353424072265625,
      "learning_rate": 9.499999999999999e-07,
      "logits/chosen": -23.41937699390698,
      "logits/rejected": -21.70754765415036,
      "logps/chosen": -1428.7145009040833,
      "logps/rejected": -443.71881103515625,
      "loss": 0.9914920926094055,
      "mean_token_accuracy": 0.7912928014993668,
      "num_tokens": 5483101.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.011147335055284202,
      "rewards/margins": 0.045076546957716346,
      "rewards/rejected": -0.03392921155318618,
      "step": 96
    },
    {
      "entropy": 1.1013405546545982,
      "epoch": 0.06082458065527512,
      "grad_norm": 36.10636520385742,
      "learning_rate": 9.6e-07,
      "logits/chosen": -19.821280825841903,
      "logits/rejected": -19.578973804909,
      "logps/chosen": -985.1940336227417,
      "logps/rejected": -473.689453125,
      "loss": 0.983747124671936,
      "mean_token_accuracy": 0.7495378330349922,
      "num_tokens": 5534649.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0021885630558244884,
      "rewards/margins": 0.06537919119000435,
      "rewards/rejected": -0.06756775546818972,
      "step": 97
    },
    {
      "entropy": 1.2237722724676132,
      "epoch": 0.061451638187803734,
      "grad_norm": 41.582496643066406,
      "learning_rate": 9.7e-07,
      "logits/chosen": -20.153029153130383,
      "logits/rejected": -17.664550339329576,
      "logps/chosen": -1082.3712921142578,
      "logps/rejected": -649.8507308959961,
      "loss": 0.9809252023696899,
      "mean_token_accuracy": 0.718035951256752,
      "num_tokens": 5612387.0,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.02365162328351289,
      "rewards/margins": 0.07657872815616429,
      "rewards/rejected": -0.052927102777175605,
      "step": 98
    },
    {
      "entropy": 1.0099660605192184,
      "epoch": 0.06207869572033234,
      "grad_norm": 30.545495986938477,
      "learning_rate": 9.8e-07,
      "logits/chosen": -19.286327351085962,
      "logits/rejected": -19.547563799363445,
      "logps/chosen": -635.9861097335815,
      "logps/rejected": -379.7309799194336,
      "loss": 0.9843310117721558,
      "mean_token_accuracy": 0.7559159845113754,
      "num_tokens": 5664396.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.005910164851229638,
      "rewards/margins": 0.06448549684137106,
      "rewards/rejected": -0.07039566081948578,
      "step": 99
    },
    {
      "entropy": 1.1891107335686684,
      "epoch": 0.06270575325286094,
      "grad_norm": 30.441099166870117,
      "learning_rate": 9.9e-07,
      "logits/chosen": -16.462167828377467,
      "logits/rejected": -17.668335453566645,
      "logps/chosen": -774.8042068481445,
      "logps/rejected": -352.1152057647705,
      "loss": 0.9829254150390625,
      "mean_token_accuracy": 0.723647378385067,
      "num_tokens": 5712927.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0013654606882482767,
      "rewards/margins": 0.0686984455678612,
      "rewards/rejected": -0.07006390765309334,
      "step": 100
    },
    {
      "entropy": 1.0059744790196419,
      "epoch": 0.06333281078538956,
      "grad_norm": 34.044551849365234,
      "learning_rate": 1e-06,
      "logits/chosen": -22.569647432957222,
      "logits/rejected": -20.571769885244017,
      "logps/chosen": -737.9557514190674,
      "logps/rejected": -456.0193290710449,
      "loss": 0.9836858510971069,
      "mean_token_accuracy": 0.7708024978637695,
      "num_tokens": 5762779.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.02014523115940392,
      "rewards/margins": 0.0654695084085688,
      "rewards/rejected": -0.045324277016334236,
      "step": 101
    },
    {
      "entropy": 1.1221561804413795,
      "epoch": 0.06395986831791817,
      "grad_norm": 35.60519790649414,
      "learning_rate": 9.999988960301596e-07,
      "logits/chosen": -17.63612561400455,
      "logits/rejected": -21.225849866079535,
      "logps/chosen": -851.1271896362305,
      "logps/rejected": -731.4087066650391,
      "loss": 0.9757521152496338,
      "mean_token_accuracy": 0.7322330251336098,
      "num_tokens": 5820523.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.011595683405175805,
      "rewards/margins": 0.09945710864849389,
      "rewards/rejected": -0.08786143315955997,
      "step": 102
    },
    {
      "entropy": 1.033433958888054,
      "epoch": 0.06458692585044677,
      "grad_norm": 33.74344253540039,
      "learning_rate": 9.999955841255138e-07,
      "logits/chosen": -21.6586000014676,
      "logits/rejected": -20.241547622800056,
      "logps/chosen": -748.6037483215332,
      "logps/rejected": -495.99595642089844,
      "loss": 0.9870611429214478,
      "mean_token_accuracy": 0.7368155121803284,
      "num_tokens": 5870733.0,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03833825106266886,
      "rewards/margins": 0.050038286950439215,
      "rewards/rejected": -0.08837653254158795,
      "step": 103
    },
    {
      "entropy": 1.118708185851574,
      "epoch": 0.06521398338297539,
      "grad_norm": 43.72235107421875,
      "learning_rate": 9.999900643006873e-07,
      "logits/chosen": -22.952759755082543,
      "logits/rejected": -18.112035021982514,
      "logps/chosen": -1293.3799285888672,
      "logps/rejected": -614.5923004150391,
      "loss": 0.9773090481758118,
      "mean_token_accuracy": 0.7410418093204498,
      "num_tokens": 5941283.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.02265817968873307,
      "rewards/margins": 0.09182942099869251,
      "rewards/rejected": -0.06917124305618927,
      "step": 104
    },
    {
      "entropy": 1.3742387741804123,
      "epoch": 0.065841040915504,
      "grad_norm": 39.11056137084961,
      "learning_rate": 9.99982336580055e-07,
      "logits/chosen": -16.713279247073885,
      "logits/rejected": -16.98860342760087,
      "logps/chosen": -950.8361873626709,
      "logps/rejected": -464.50708770751953,
      "loss": 0.9773339629173279,
      "mean_token_accuracy": 0.6922967359423637,
      "num_tokens": 6000343.0,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.006588876945897937,
      "rewards/margins": 0.09067038784269243,
      "rewards/rejected": -0.08408151054754853,
      "step": 105
    },
    {
      "entropy": 1.1545687764883041,
      "epoch": 0.0664680984480326,
      "grad_norm": 34.153663635253906,
      "learning_rate": 9.99972400997742e-07,
      "logits/chosen": -20.552279910445396,
      "logits/rejected": -18.52949584159018,
      "logps/chosen": -710.0431318283081,
      "logps/rejected": -350.6925792694092,
      "loss": 0.982926070690155,
      "mean_token_accuracy": 0.7238080576062202,
      "num_tokens": 6047719.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.01743771624751389,
      "rewards/margins": 0.06860713846981525,
      "rewards/rejected": -0.05116942140739411,
      "step": 106
    },
    {
      "entropy": 0.8841631710529327,
      "epoch": 0.06709515598056122,
      "grad_norm": 30.25786018371582,
      "learning_rate": 9.999602575976219e-07,
      "logits/chosen": -16.81424879838689,
      "logits/rejected": -16.14855911409956,
      "logps/chosen": -426.7861785888672,
      "logps/rejected": -396.69312858581543,
      "loss": 0.979150652885437,
      "mean_token_accuracy": 0.8006457611918449,
      "num_tokens": 6096830.0,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.007483278808649629,
      "rewards/margins": 0.0836036205291748,
      "rewards/rejected": -0.07612034154590219,
      "step": 107
    },
    {
      "entropy": 1.0404388830065727,
      "epoch": 0.06772221351308982,
      "grad_norm": 33.5556640625,
      "learning_rate": 9.999459064333188e-07,
      "logits/chosen": -15.91254721701439,
      "logits/rejected": -16.29134417675286,
      "logps/chosen": -601.9646797180176,
      "logps/rejected": -457.3084487915039,
      "loss": 0.9777726531028748,
      "mean_token_accuracy": 0.7538063228130341,
      "num_tokens": 6145266.0,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.019957647193223238,
      "rewards/margins": 0.08928463887423277,
      "rewards/rejected": -0.0693269899347797,
      "step": 108
    },
    {
      "entropy": 1.2834724336862564,
      "epoch": 0.06834927104561843,
      "grad_norm": 34.99026870727539,
      "learning_rate": 9.999293475682062e-07,
      "logits/chosen": -17.259286736211884,
      "logits/rejected": -17.444496267757785,
      "logps/chosen": -667.7793045043945,
      "logps/rejected": -577.6125526428223,
      "loss": 0.9799240231513977,
      "mean_token_accuracy": 0.6992196813225746,
      "num_tokens": 6196039.0,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.005005566752515733,
      "rewards/margins": 0.08093663223553449,
      "rewards/rejected": -0.08594219759106636,
      "step": 109
    },
    {
      "entropy": 1.055838204920292,
      "epoch": 0.06897632857814705,
      "grad_norm": 33.32623291015625,
      "learning_rate": 9.999105810754053e-07,
      "logits/chosen": -18.407450521012187,
      "logits/rejected": -18.873402077141666,
      "logps/chosen": -799.7204494476318,
      "logps/rejected": -477.783242225647,
      "loss": 0.9677953720092773,
      "mean_token_accuracy": 0.7510818913578987,
      "num_tokens": 6263445.0,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.052083960617892444,
      "rewards/margins": 0.1307980790734291,
      "rewards/rejected": -0.07871411880478263,
      "step": 110
    },
    {
      "entropy": 0.9602865129709244,
      "epoch": 0.06960338611067565,
      "grad_norm": 28.93109130859375,
      "learning_rate": 9.99889607037787e-07,
      "logits/chosen": -16.415104591720436,
      "logits/rejected": -18.58728532903226,
      "logps/chosen": -381.7690181732178,
      "logps/rejected": -416.77470779418945,
      "loss": 0.9900650382041931,
      "mean_token_accuracy": 0.7899875342845917,
      "num_tokens": 6311526.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.004977155942469835,
      "rewards/margins": 0.039845253340899944,
      "rewards/rejected": -0.044822409050539136,
      "step": 111
    },
    {
      "entropy": 0.8192609176039696,
      "epoch": 0.07023044364320426,
      "grad_norm": 39.31092071533203,
      "learning_rate": 9.998664255479704e-07,
      "logits/chosen": -25.072994460556252,
      "logits/rejected": -22.42456335487019,
      "logps/chosen": -1064.658107995987,
      "logps/rejected": -414.81433868408203,
      "loss": 0.9966785311698914,
      "mean_token_accuracy": 0.8144989609718323,
      "num_tokens": 6373673.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06160880299285054,
      "rewards/margins": 0.012114129436668009,
      "rewards/rejected": -0.0737229329533875,
      "step": 112
    },
    {
      "entropy": 0.9448361806571484,
      "epoch": 0.07085750117573288,
      "grad_norm": 39.90821075439453,
      "learning_rate": 9.99841036708322e-07,
      "logits/chosen": -23.766030290287702,
      "logits/rejected": -25.34154914444923,
      "logps/chosen": -1000.321138381958,
      "logps/rejected": -514.6866340637207,
      "loss": 0.9712129831314087,
      "mean_token_accuracy": 0.7757420614361763,
      "num_tokens": 6428282.0,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.034964483114890754,
      "rewards/margins": 0.11884536314755678,
      "rewards/rejected": -0.08388087863568217,
      "step": 113
    },
    {
      "entropy": 0.8173889517784119,
      "epoch": 0.07148455870826148,
      "grad_norm": 30.850637435913086,
      "learning_rate": 9.998134406309553e-07,
      "logits/chosen": -25.008434498378293,
      "logits/rejected": -22.165182027384386,
      "logps/chosen": -702.1321802139282,
      "logps/rejected": -378.8375816345215,
      "loss": 0.9969829320907593,
      "mean_token_accuracy": 0.8100090399384499,
      "num_tokens": 6478216.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0617737959837541,
      "rewards/margins": 0.0021084430627524853,
      "rewards/rejected": -0.06388223776593804,
      "step": 114
    },
    {
      "entropy": 1.0738236010074615,
      "epoch": 0.07211161624079009,
      "grad_norm": 37.23641586303711,
      "learning_rate": 9.997836374377318e-07,
      "logits/chosen": -21.598642905098632,
      "logits/rejected": -23.624653024193194,
      "logps/chosen": -677.4854145050049,
      "logps/rejected": -513.8848571777344,
      "loss": 0.9803842306137085,
      "mean_token_accuracy": 0.7514287456870079,
      "num_tokens": 6522645.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0021543916955124587,
      "rewards/margins": 0.0788510333513841,
      "rewards/rejected": -0.07669664057902992,
      "step": 115
    },
    {
      "entropy": 1.0687246397137642,
      "epoch": 0.0727386737733187,
      "grad_norm": 28.08934783935547,
      "learning_rate": 9.997516272602588e-07,
      "logits/chosen": -18.0843256562564,
      "logits/rejected": -17.27698922154224,
      "logps/chosen": -591.8647804260254,
      "logps/rejected": -329.94952392578125,
      "loss": 0.9886791706085205,
      "mean_token_accuracy": 0.7329046651721001,
      "num_tokens": 6565380.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.014423841843381524,
      "rewards/margins": 0.045475234859623015,
      "rewards/rejected": -0.05989907821640372,
      "step": 116
    },
    {
      "entropy": 1.2525769509375095,
      "epoch": 0.07336573130584731,
      "grad_norm": 32.46056365966797,
      "learning_rate": 9.997174102398892e-07,
      "logits/chosen": -19.069984483617883,
      "logits/rejected": -18.31701463203426,
      "logps/chosen": -857.891770362854,
      "logps/rejected": -576.6237850189209,
      "loss": 0.9878202676773071,
      "mean_token_accuracy": 0.7090706676244736,
      "num_tokens": 6621385.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.012965936097316444,
      "rewards/margins": 0.04882583348080516,
      "rewards/rejected": -0.06179177016019821,
      "step": 117
    },
    {
      "entropy": 1.0708193145692348,
      "epoch": 0.07399278883837591,
      "grad_norm": 33.78789138793945,
      "learning_rate": 9.996809865277214e-07,
      "logits/chosen": -17.575822411940983,
      "logits/rejected": -15.71341222924891,
      "logps/chosen": -628.3108444213867,
      "logps/rejected": -381.9364585876465,
      "loss": 0.984896183013916,
      "mean_token_accuracy": 0.7500499114394188,
      "num_tokens": 6671308.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.02526119421236217,
      "rewards/margins": 0.060638573253527284,
      "rewards/rejected": -0.0858997656032443,
      "step": 118
    },
    {
      "entropy": 1.1464358419179916,
      "epoch": 0.07461984637090453,
      "grad_norm": 29.262676239013672,
      "learning_rate": 9.996423562845978e-07,
      "logits/chosen": -18.950197207830822,
      "logits/rejected": -17.89534831083926,
      "logps/chosen": -499.1498966217041,
      "logps/rejected": -397.74127197265625,
      "loss": 0.9772639274597168,
      "mean_token_accuracy": 0.7390127554535866,
      "num_tokens": 6707362.0,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.014790326356887817,
      "rewards/margins": 0.09125107247382402,
      "rewards/rejected": -0.07646074634976685,
      "step": 119
    },
    {
      "entropy": 1.1956902518868446,
      "epoch": 0.07524690390343314,
      "grad_norm": 31.247859954833984,
      "learning_rate": 9.996015196811054e-07,
      "logits/chosen": -16.430566598689513,
      "logits/rejected": -16.301428702284802,
      "logps/chosen": -712.8943672180176,
      "logps/rejected": -467.7664604187012,
      "loss": 0.9875868558883667,
      "mean_token_accuracy": 0.7251445576548576,
      "num_tokens": 6761518.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00015118438750505447,
      "rewards/margins": 0.05001709656789899,
      "rewards/rejected": -0.04986591334454715,
      "step": 120
    },
    {
      "entropy": 1.04892847687006,
      "epoch": 0.07587396143596174,
      "grad_norm": 36.938690185546875,
      "learning_rate": 9.995584768975734e-07,
      "logits/chosen": -20.353520038495564,
      "logits/rejected": -18.411114375084413,
      "logps/chosen": -787.6422328948975,
      "logps/rejected": -501.9888114929199,
      "loss": 0.9638845920562744,
      "mean_token_accuracy": 0.7510411590337753,
      "num_tokens": 6814204.0,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.037587814789731055,
      "rewards/margins": 0.1457526022568345,
      "rewards/rejected": -0.10816478868946433,
      "step": 121
    },
    {
      "entropy": 0.911387711763382,
      "epoch": 0.07650101896849036,
      "grad_norm": 30.251392364501953,
      "learning_rate": 9.995132281240734e-07,
      "logits/chosen": -19.869032639332318,
      "logits/rejected": -18.91703122960338,
      "logps/chosen": -582.7686996459961,
      "logps/rejected": -366.2984085083008,
      "loss": 0.9788750410079956,
      "mean_token_accuracy": 0.7888387069106102,
      "num_tokens": 6861531.0,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.003442022774834186,
      "rewards/margins": 0.0853227588813752,
      "rewards/rejected": -0.08188073709607124,
      "step": 122
    },
    {
      "entropy": 1.0427976697683334,
      "epoch": 0.07712807650101897,
      "grad_norm": 36.89279556274414,
      "learning_rate": 9.994657735604188e-07,
      "logits/chosen": -24.044872176023844,
      "logits/rejected": -22.597699453222447,
      "logps/chosen": -966.8881568908691,
      "logps/rejected": -505.8887948989868,
      "loss": 0.9708135724067688,
      "mean_token_accuracy": 0.7332872226834297,
      "num_tokens": 6921458.0,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.012047640164382756,
      "rewards/margins": 0.11771579552441835,
      "rewards/rejected": -0.10566815291531384,
      "step": 123
    },
    {
      "entropy": 1.0259113758802414,
      "epoch": 0.07775513403354758,
      "grad_norm": 37.096275329589844,
      "learning_rate": 9.994161134161632e-07,
      "logits/chosen": -19.591871441281974,
      "logits/rejected": -20.936406703396575,
      "logps/chosen": -818.1269016265869,
      "logps/rejected": -639.8456077575684,
      "loss": 0.9665595293045044,
      "mean_token_accuracy": 0.7770890146493912,
      "num_tokens": 6982937.0,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.020557800424285233,
      "rewards/margins": 0.13559715449810028,
      "rewards/rejected": -0.11503935337532312,
      "step": 124
    },
    {
      "entropy": 1.1710731238126755,
      "epoch": 0.07838219156607619,
      "grad_norm": 42.78230667114258,
      "learning_rate": 9.993642479105997e-07,
      "logits/chosen": -24.393052790442358,
      "logits/rejected": -18.192713037735032,
      "logps/chosen": -1177.7320861816406,
      "logps/rejected": -522.7479839324951,
      "loss": 0.9867607355117798,
      "mean_token_accuracy": 0.712364636361599,
      "num_tokens": 7046947.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02724925708025694,
      "rewards/margins": 0.052655004314146936,
      "rewards/rejected": -0.07990426244214177,
      "step": 125
    },
    {
      "entropy": 1.1666254326701164,
      "epoch": 0.0790092490986048,
      "grad_norm": 32.3548469543457,
      "learning_rate": 9.993101772727601e-07,
      "logits/chosen": -17.63798263385176,
      "logits/rejected": -17.47313740458377,
      "logps/chosen": -606.1291179656982,
      "logps/rejected": -430.56521797180176,
      "loss": 0.9682327508926392,
      "mean_token_accuracy": 0.7458649724721909,
      "num_tokens": 7102060.0,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.0330769574502483,
      "rewards/margins": 0.13182413554750383,
      "rewards/rejected": -0.09874717495404184,
      "step": 126
    },
    {
      "entropy": 1.1291334107518196,
      "epoch": 0.07963630663113341,
      "grad_norm": 31.756563186645508,
      "learning_rate": 9.99253901741414e-07,
      "logits/chosen": -21.307392332824012,
      "logits/rejected": -20.84085959531398,
      "logps/chosen": -732.371265411377,
      "logps/rejected": -503.3115940093994,
      "loss": 0.9792959690093994,
      "mean_token_accuracy": 0.7251338735222816,
      "num_tokens": 7157801.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.018919202295364812,
      "rewards/margins": 0.08486578240990639,
      "rewards/rejected": -0.06594657967798412,
      "step": 127
    },
    {
      "entropy": 1.1786841079592705,
      "epoch": 0.08026336416366202,
      "grad_norm": 40.29298782348633,
      "learning_rate": 9.99195421565067e-07,
      "logits/chosen": -17.529350354706732,
      "logits/rejected": -20.833092058953017,
      "logps/chosen": -1033.621503829956,
      "logps/rejected": -691.7955369949341,
      "loss": 0.9725984334945679,
      "mean_token_accuracy": 0.7169393450021744,
      "num_tokens": 7224323.0,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.006326087401248515,
      "rewards/margins": 0.11025936878286302,
      "rewards/rejected": -0.11658545769751072,
      "step": 128
    },
    {
      "entropy": 1.1065169423818588,
      "epoch": 0.08089042169619062,
      "grad_norm": 30.228065490722656,
      "learning_rate": 9.991347370019609e-07,
      "logits/chosen": -18.675428265484687,
      "logits/rejected": -18.02665901699452,
      "logps/chosen": -738.0616884231567,
      "logps/rejected": -413.77689933776855,
      "loss": 0.9663163423538208,
      "mean_token_accuracy": 0.7391728013753891,
      "num_tokens": 7273940.0,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04217634559608996,
      "rewards/margins": 0.1416997853666544,
      "rewards/rejected": -0.09952343860641122,
      "step": 129
    },
    {
      "entropy": 1.0223164483904839,
      "epoch": 0.08151747922871924,
      "grad_norm": 36.71884536743164,
      "learning_rate": 9.990718483200711e-07,
      "logits/chosen": -24.017658505382084,
      "logits/rejected": -19.894383210249206,
      "logps/chosen": -853.0483455657959,
      "logps/rejected": -410.2704782485962,
      "loss": 0.9696524739265442,
      "mean_token_accuracy": 0.7612569332122803,
      "num_tokens": 7330670.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.022751914570108056,
      "rewards/margins": 0.12564924580510706,
      "rewards/rejected": -0.10289733344689012,
      "step": 130
    },
    {
      "entropy": 1.188691645860672,
      "epoch": 0.08214453676124785,
      "grad_norm": 45.50562286376953,
      "learning_rate": 9.990067557971066e-07,
      "logits/chosen": -20.96738395725814,
      "logits/rejected": -18.577314710392834,
      "logps/chosen": -1276.1438827514648,
      "logps/rejected": -671.3540458679199,
      "loss": 0.9635441303253174,
      "mean_token_accuracy": 0.7195305675268173,
      "num_tokens": 7400528.0,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.027424347819760442,
      "rewards/margins": 0.14744434040039778,
      "rewards/rejected": -0.1200199886225164,
      "step": 131
    },
    {
      "entropy": 0.9636916220188141,
      "epoch": 0.08277159429377645,
      "grad_norm": 28.21497344970703,
      "learning_rate": 9.989394597205082e-07,
      "logits/chosen": -17.83059240747361,
      "logits/rejected": -17.95696408960033,
      "logps/chosen": -485.29137802124023,
      "logps/rejected": -400.46766471862793,
      "loss": 0.976447343826294,
      "mean_token_accuracy": 0.7697746828198433,
      "num_tokens": 7462922.0,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.010323267662897706,
      "rewards/margins": 0.09466969268396497,
      "rewards/rejected": -0.08434642176143825,
      "step": 132
    },
    {
      "entropy": 0.9702051728963852,
      "epoch": 0.08339865182630507,
      "grad_norm": 35.21070861816406,
      "learning_rate": 9.98869960387447e-07,
      "logits/chosen": -24.15228929902058,
      "logits/rejected": -17.62097141810482,
      "logps/chosen": -707.7485370635986,
      "logps/rejected": -328.29246520996094,
      "loss": 0.9750052690505981,
      "mean_token_accuracy": 0.7517875507473946,
      "num_tokens": 7516053.0,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.020313383429311216,
      "rewards/margins": 0.10041370801627636,
      "rewards/rejected": -0.08010032417951152,
      "step": 133
    },
    {
      "entropy": 1.080765277147293,
      "epoch": 0.08402570935883368,
      "grad_norm": 33.306419372558594,
      "learning_rate": 9.98798258104824e-07,
      "logits/chosen": -16.50647893746725,
      "logits/rejected": -18.638188273587172,
      "logps/chosen": -394.3768768310547,
      "logps/rejected": -530.8435726165771,
      "loss": 0.9684998393058777,
      "mean_token_accuracy": 0.7539431154727936,
      "num_tokens": 7560146.0,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.008416369266342372,
      "rewards/margins": 0.12770695355720818,
      "rewards/rejected": -0.13612332614138722,
      "step": 134
    },
    {
      "entropy": 1.0699984654784203,
      "epoch": 0.08465276689136228,
      "grad_norm": 29.02651596069336,
      "learning_rate": 9.987243531892676e-07,
      "logits/chosen": -17.323968607265364,
      "logits/rejected": -17.327844017915886,
      "logps/chosen": -423.5040798187256,
      "logps/rejected": -401.37097549438477,
      "loss": 0.9706511497497559,
      "mean_token_accuracy": 0.7508202418684959,
      "num_tokens": 7604430.0,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.010786252591060475,
      "rewards/margins": 0.11822709790430963,
      "rewards/rejected": -0.10744084196630865,
      "step": 135
    },
    {
      "entropy": 1.1741472780704498,
      "epoch": 0.0852798244238909,
      "grad_norm": 29.518667221069336,
      "learning_rate": 9.98648245967133e-07,
      "logits/chosen": -17.057511505199,
      "logits/rejected": -15.816799192033697,
      "logps/chosen": -578.4385671615601,
      "logps/rejected": -470.2986297607422,
      "loss": 0.9768874645233154,
      "mean_token_accuracy": 0.7322021424770355,
      "num_tokens": 7661280.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.018278606235980988,
      "rewards/margins": 0.09282787260599434,
      "rewards/rejected": -0.07454926625359803,
      "step": 136
    },
    {
      "entropy": 1.1715349406003952,
      "epoch": 0.0859068819564195,
      "grad_norm": 35.8231086730957,
      "learning_rate": 9.985699367745007e-07,
      "logits/chosen": -20.207632968359622,
      "logits/rejected": -18.918033251926897,
      "logps/chosen": -1063.0584182739258,
      "logps/rejected": -546.3697662353516,
      "loss": 0.9809889793395996,
      "mean_token_accuracy": 0.7140867561101913,
      "num_tokens": 7723139.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.016168745700269938,
      "rewards/margins": 0.07651804899796844,
      "rewards/rejected": -0.09268679865635931,
      "step": 137
    },
    {
      "entropy": 1.0441111102700233,
      "epoch": 0.08653393948894811,
      "grad_norm": 32.78045654296875,
      "learning_rate": 9.984894259571743e-07,
      "logits/chosen": -19.221359907309978,
      "logits/rejected": -18.10666979652929,
      "logps/chosen": -489.88498306274414,
      "logps/rejected": -398.86695098876953,
      "loss": 0.9633641242980957,
      "mean_token_accuracy": 0.7486372217535973,
      "num_tokens": 7766818.0,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.011141044669784606,
      "rewards/margins": 0.14769388129934669,
      "rewards/rejected": -0.1365528372116387,
      "step": 138
    },
    {
      "entropy": 1.1274549514055252,
      "epoch": 0.08716099702147673,
      "grad_norm": 28.61403465270996,
      "learning_rate": 9.984067138706801e-07,
      "logits/chosen": -20.09536989108541,
      "logits/rejected": -19.089721130331522,
      "logps/chosen": -668.453311920166,
      "logps/rejected": -400.32433700561523,
      "loss": 0.9768990278244019,
      "mean_token_accuracy": 0.7357787117362022,
      "num_tokens": 7805514.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.008357231738045812,
      "rewards/margins": 0.09283104329369962,
      "rewards/rejected": -0.08447381528094411,
      "step": 139
    },
    {
      "entropy": 1.072766751050949,
      "epoch": 0.08778805455400533,
      "grad_norm": 36.328330993652344,
      "learning_rate": 9.983218008802647e-07,
      "logits/chosen": -16.597205351543526,
      "logits/rejected": -17.38877890508226,
      "logps/chosen": -845.426420211792,
      "logps/rejected": -446.5038776397705,
      "loss": 0.969933271408081,
      "mean_token_accuracy": 0.7438737154006958,
      "num_tokens": 7867552.0,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.004711989313364029,
      "rewards/margins": 0.12162661645561457,
      "rewards/rejected": -0.12633860250934958,
      "step": 140
    },
    {
      "entropy": 1.1710950955748558,
      "epoch": 0.08841511208653394,
      "grad_norm": 35.29788589477539,
      "learning_rate": 9.982346873608936e-07,
      "logits/chosen": -17.457327570012882,
      "logits/rejected": -17.907635026265908,
      "logps/chosen": -594.5470962524414,
      "logps/rejected": -608.6735458374023,
      "loss": 0.9617944359779358,
      "mean_token_accuracy": 0.7367723286151886,
      "num_tokens": 7913056.0,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.0014770900015719235,
      "rewards/margins": 0.15395231172442436,
      "rewards/rejected": -0.15247522108256817,
      "step": 141
    },
    {
      "entropy": 1.0754098296165466,
      "epoch": 0.08904216961906256,
      "grad_norm": 35.45695877075195,
      "learning_rate": 9.981453736972495e-07,
      "logits/chosen": -19.118562752807563,
      "logits/rejected": -16.91145482295967,
      "logps/chosen": -698.3301868438721,
      "logps/rejected": -497.7921562194824,
      "loss": 0.9720945954322815,
      "mean_token_accuracy": 0.7595574334263802,
      "num_tokens": 7980591.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.01288341250619851,
      "rewards/margins": 0.11340800416655838,
      "rewards/rejected": -0.10052459384314716,
      "step": 142
    },
    {
      "entropy": 0.9044511765241623,
      "epoch": 0.08966922715159116,
      "grad_norm": 30.17085075378418,
      "learning_rate": 9.98053860283731e-07,
      "logits/chosen": -21.207617728808614,
      "logits/rejected": -20.905731823651053,
      "logps/chosen": -628.0943202972412,
      "logps/rejected": -420.4750385284424,
      "loss": 0.9711873531341553,
      "mean_token_accuracy": 0.7811061069369316,
      "num_tokens": 8034884.0,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.023002621717751026,
      "rewards/margins": 0.1186411960516125,
      "rewards/rejected": -0.09563857619650662,
      "step": 143
    },
    {
      "entropy": 1.1677963212132454,
      "epoch": 0.09029628468411977,
      "grad_norm": 31.993118286132812,
      "learning_rate": 9.9796014752445e-07,
      "logits/chosen": -18.57265198556839,
      "logits/rejected": -17.995440262698004,
      "logps/chosen": -658.4226865768433,
      "logps/rejected": -438.4370231628418,
      "loss": 0.9786888360977173,
      "mean_token_accuracy": 0.7285968512296677,
      "num_tokens": 8082039.0,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02490088331978768,
      "rewards/margins": 0.08490909356623888,
      "rewards/rejected": -0.10980997630394995,
      "step": 144
    },
    {
      "entropy": 1.0427600964903831,
      "epoch": 0.09092334221664838,
      "grad_norm": 31.311111450195312,
      "learning_rate": 9.978642358332307e-07,
      "logits/chosen": -19.139753277032113,
      "logits/rejected": -17.59120642360813,
      "logps/chosen": -567.9225654602051,
      "logps/rejected": -465.25221252441406,
      "loss": 0.9686312675476074,
      "mean_token_accuracy": 0.7571733519434929,
      "num_tokens": 8123718.0,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.008705203887075186,
      "rewards/margins": 0.12602692190557718,
      "rewards/rejected": -0.117321718018502,
      "step": 145
    },
    {
      "entropy": 0.8850021287798882,
      "epoch": 0.09155039974917699,
      "grad_norm": 37.63220977783203,
      "learning_rate": 9.97766125633608e-07,
      "logits/chosen": -19.910044793415654,
      "logits/rejected": -22.54540787489115,
      "logps/chosen": -699.3141222000122,
      "logps/rejected": -466.68331146240234,
      "loss": 0.9604589939117432,
      "mean_token_accuracy": 0.7936916127800941,
      "num_tokens": 8187632.0,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.003929961123503745,
      "rewards/margins": 0.168057446135208,
      "rewards/rejected": -0.17198741296306252,
      "step": 146
    },
    {
      "entropy": 1.0326995179057121,
      "epoch": 0.0921774572817056,
      "grad_norm": 37.081241607666016,
      "learning_rate": 9.976658173588243e-07,
      "logits/chosen": -21.190682950448576,
      "logits/rejected": -17.69587061048208,
      "logps/chosen": -751.942907333374,
      "logps/rejected": -442.78441619873047,
      "loss": 0.9758281707763672,
      "mean_token_accuracy": 0.7523355558514595,
      "num_tokens": 8243290.0,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.02370329963741824,
      "rewards/margins": 0.09733096603304148,
      "rewards/rejected": -0.12103426642715931,
      "step": 147
    },
    {
      "entropy": 0.9734203144907951,
      "epoch": 0.09280451481423421,
      "grad_norm": 36.707550048828125,
      "learning_rate": 9.97563311451829e-07,
      "logits/chosen": -23.493140481868807,
      "logits/rejected": -19.432473725597365,
      "logps/chosen": -1202.7809944152832,
      "logps/rejected": -374.36139488220215,
      "loss": 0.9666430950164795,
      "mean_token_accuracy": 0.7533522471785545,
      "num_tokens": 8312496.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.040649566042702645,
      "rewards/margins": 0.13564491388387978,
      "rewards/rejected": -0.0949953489471227,
      "step": 148
    },
    {
      "entropy": 1.3850471526384354,
      "epoch": 0.09343157234676282,
      "grad_norm": 31.51274871826172,
      "learning_rate": 9.974586083652757e-07,
      "logits/chosen": -18.153076228160014,
      "logits/rejected": -16.684204876050245,
      "logps/chosen": -845.3816833496094,
      "logps/rejected": -408.8254728317261,
      "loss": 0.9770760536193848,
      "mean_token_accuracy": 0.6601794064044952,
      "num_tokens": 8362689.0,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.018910931539721787,
      "rewards/margins": 0.09186943899840117,
      "rewards/rejected": -0.11078036949038506,
      "step": 149
    },
    {
      "entropy": 1.0344312191009521,
      "epoch": 0.09405862987929142,
      "grad_norm": 37.89040756225586,
      "learning_rate": 9.97351708561521e-07,
      "logits/chosen": -23.15418936171772,
      "logits/rejected": -19.21865452626319,
      "logps/chosen": -769.2827854156494,
      "logps/rejected": -666.4356002807617,
      "loss": 0.9749352335929871,
      "mean_token_accuracy": 0.7564720064401627,
      "num_tokens": 8420985.0,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.009593813680112362,
      "rewards/margins": 0.10142657672986388,
      "rewards/rejected": -0.1110203885473311,
      "step": 150
    },
    {
      "entropy": 1.175799421966076,
      "epoch": 0.09468568741182004,
      "grad_norm": 40.132625579833984,
      "learning_rate": 9.972426125126207e-07,
      "logits/chosen": -21.38852635915963,
      "logits/rejected": -18.672145327724344,
      "logps/chosen": -1471.715404510498,
      "logps/rejected": -486.47255516052246,
      "loss": 0.980172336101532,
      "mean_token_accuracy": 0.721579059958458,
      "num_tokens": 8489644.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03523680008947849,
      "rewards/margins": 0.0802477840334177,
      "rewards/rejected": -0.11548458551988006,
      "step": 151
    },
    {
      "entropy": 1.4164262264966965,
      "epoch": 0.09531274494434865,
      "grad_norm": 27.696420669555664,
      "learning_rate": 9.971313207003307e-07,
      "logits/chosen": -13.93982345493672,
      "logits/rejected": -14.11517608202279,
      "logps/chosen": -740.9725232124329,
      "logps/rejected": -564.2864780426025,
      "loss": 0.9797288179397583,
      "mean_token_accuracy": 0.6809670105576515,
      "num_tokens": 8537512.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0021716501796618104,
      "rewards/margins": 0.08156625635456294,
      "rewards/rejected": -0.08373790187761188,
      "step": 152
    },
    {
      "entropy": 1.263432890176773,
      "epoch": 0.09593980247687725,
      "grad_norm": 31.5176944732666,
      "learning_rate": 9.970178336161016e-07,
      "logits/chosen": -16.973323202222108,
      "logits/rejected": -19.614638086614725,
      "logps/chosen": -711.4418649673462,
      "logps/rejected": -372.44454193115234,
      "loss": 0.9762611389160156,
      "mean_token_accuracy": 0.7092464938759804,
      "num_tokens": 8591662.0,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0145274052338209,
      "rewards/margins": 0.09459308302029967,
      "rewards/rejected": -0.10912049002945423,
      "step": 153
    },
    {
      "entropy": 1.1620648950338364,
      "epoch": 0.09656686000940587,
      "grad_norm": 30.509321212768555,
      "learning_rate": 9.969021517610792e-07,
      "logits/chosen": -13.79756665559363,
      "logits/rejected": -15.715619490328542,
      "logps/chosen": -604.1418495178223,
      "logps/rejected": -382.564510345459,
      "loss": 0.9641399383544922,
      "mean_token_accuracy": 0.726889930665493,
      "num_tokens": 8636893.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0023322205524891615,
      "rewards/margins": 0.14497205498628318,
      "rewards/rejected": -0.1473042806610465,
      "step": 154
    },
    {
      "entropy": 1.1641441136598587,
      "epoch": 0.09719391754193447,
      "grad_norm": 41.579978942871094,
      "learning_rate": 9.967842756461002e-07,
      "logits/chosen": -20.334094398749123,
      "logits/rejected": -22.011842279254914,
      "logps/chosen": -1403.928207397461,
      "logps/rejected": -554.649658203125,
      "loss": 0.9708878397941589,
      "mean_token_accuracy": 0.72523083537817,
      "num_tokens": 8717868.0,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.00970022683031857,
      "rewards/margins": 0.11776156397536397,
      "rewards/rejected": -0.10806133644655347,
      "step": 155
    },
    {
      "entropy": 1.2410652860999107,
      "epoch": 0.09782097507446308,
      "grad_norm": 125.10015106201172,
      "learning_rate": 9.966642057916914e-07,
      "logits/chosen": -18.70830338783564,
      "logits/rejected": -19.97097714965005,
      "logps/chosen": -1191.791633605957,
      "logps/rejected": -752.3146705627441,
      "loss": 0.9524157643318176,
      "mean_token_accuracy": 0.7021138742566109,
      "num_tokens": 8807483.0,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0013844413042534143,
      "rewards/margins": 0.19747850112617016,
      "rewards/rejected": -0.19886294193565845,
      "step": 156
    },
    {
      "entropy": 1.029944323003292,
      "epoch": 0.0984480326069917,
      "grad_norm": 35.78042221069336,
      "learning_rate": 9.965419427280668e-07,
      "logits/chosen": -23.38625175610134,
      "logits/rejected": -20.01934285422626,
      "logps/chosen": -1151.3675441741943,
      "logps/rejected": -498.4595642089844,
      "loss": 0.9675576090812683,
      "mean_token_accuracy": 0.7536375895142555,
      "num_tokens": 8887962.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03968535171588883,
      "rewards/margins": 0.13185083586722612,
      "rewards/rejected": -0.09216548292897642,
      "step": 157
    },
    {
      "entropy": 1.1238584071397781,
      "epoch": 0.0990750901395203,
      "grad_norm": 35.993568420410156,
      "learning_rate": 9.964174869951254e-07,
      "logits/chosen": -13.67164112165123,
      "logits/rejected": -19.3182056629662,
      "logps/chosen": -457.62388134002686,
      "logps/rejected": -487.38970375061035,
      "loss": 0.9535990953445435,
      "mean_token_accuracy": 0.7514778524637222,
      "num_tokens": 8930150.0,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.004519644775427878,
      "rewards/margins": 0.18916182266548276,
      "rewards/rejected": -0.19368146173655987,
      "step": 158
    },
    {
      "entropy": 0.9919542148709297,
      "epoch": 0.09970214767204891,
      "grad_norm": 35.438880920410156,
      "learning_rate": 9.962908391424487e-07,
      "logits/chosen": -24.406944580639333,
      "logits/rejected": -21.38588868915605,
      "logps/chosen": -899.7554340362549,
      "logps/rejected": -343.76156425476074,
      "loss": 0.9654215574264526,
      "mean_token_accuracy": 0.7644909620285034,
      "num_tokens": 8991665.0,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.006943491520360112,
      "rewards/margins": 0.14082542480900884,
      "rewards/rejected": -0.1338819395750761,
      "step": 159
    },
    {
      "entropy": 1.0232055224478245,
      "epoch": 0.10032920520457753,
      "grad_norm": 31.05710220336914,
      "learning_rate": 9.961619997292983e-07,
      "logits/chosen": -17.982335887836616,
      "logits/rejected": -20.611772617446324,
      "logps/chosen": -540.6755952835083,
      "logps/rejected": -586.5883369445801,
      "loss": 0.9546467065811157,
      "mean_token_accuracy": 0.7635128647089005,
      "num_tokens": 9042126.0,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.0020738002785947174,
      "rewards/margins": 0.19118777592666447,
      "rewards/rejected": -0.19326158007606864,
      "step": 160
    },
    {
      "entropy": 1.0774603635072708,
      "epoch": 0.10095626273710613,
      "grad_norm": 30.445066452026367,
      "learning_rate": 9.960309693246134e-07,
      "logits/chosen": -16.290460512960674,
      "logits/rejected": -18.012763543697705,
      "logps/chosen": -576.4469928741455,
      "logps/rejected": -511.25352478027344,
      "loss": 0.9666774868965149,
      "mean_token_accuracy": 0.7399578765034676,
      "num_tokens": 9088125.0,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.01442447875160724,
      "rewards/margins": 0.1348671750165522,
      "rewards/rejected": -0.14929165970534086,
      "step": 161
    },
    {
      "entropy": 1.0612441897392273,
      "epoch": 0.10158332026963474,
      "grad_norm": 37.424896240234375,
      "learning_rate": 9.958977485070087e-07,
      "logits/chosen": -19.54973934978431,
      "logits/rejected": -21.830906008805396,
      "logps/chosen": -839.8476104736328,
      "logps/rejected": -619.4777336120605,
      "loss": 0.9631219506263733,
      "mean_token_accuracy": 0.7454415857791901,
      "num_tokens": 9147591.0,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.027470878849271685,
      "rewards/margins": 0.14951888378709555,
      "rewards/rejected": -0.17698976676911116,
      "step": 162
    },
    {
      "entropy": 0.8433318734169006,
      "epoch": 0.10221037780216335,
      "grad_norm": 36.499000549316406,
      "learning_rate": 9.957623378647708e-07,
      "logits/chosen": -22.97117871622693,
      "logits/rejected": -23.077180808340206,
      "logps/chosen": -762.8622913360596,
      "logps/rejected": -496.52293968200684,
      "loss": 0.9598798155784607,
      "mean_token_accuracy": 0.7907682359218597,
      "num_tokens": 9204397.0,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.008882903959602118,
      "rewards/margins": 0.16420582123100758,
      "rewards/rejected": -0.15532291657291353,
      "step": 163
    },
    {
      "entropy": 1.1806890442967415,
      "epoch": 0.10283743533469196,
      "grad_norm": 45.862754821777344,
      "learning_rate": 9.956247379958574e-07,
      "logits/chosen": -19.059974763772217,
      "logits/rejected": -18.85946041293872,
      "logps/chosen": -1369.5862731933594,
      "logps/rejected": -567.7572498321533,
      "loss": 0.9552696943283081,
      "mean_token_accuracy": 0.7177683860063553,
      "num_tokens": 9271429.0,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.03492974303662777,
      "rewards/margins": 0.18170535750687122,
      "rewards/rejected": -0.14677561353892088,
      "step": 164
    },
    {
      "entropy": 1.0775589682161808,
      "epoch": 0.10346449286722056,
      "grad_norm": 29.601449966430664,
      "learning_rate": 9.954849495078926e-07,
      "logits/chosen": -19.275060518402277,
      "logits/rejected": -19.166819179421008,
      "logps/chosen": -660.1427116394043,
      "logps/rejected": -383.6678161621094,
      "loss": 0.958690881729126,
      "mean_token_accuracy": 0.7361882030963898,
      "num_tokens": 9312257.0,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03519108361797407,
      "rewards/margins": 0.1664116713218391,
      "rewards/rejected": -0.13122059125453234,
      "step": 165
    },
    {
      "entropy": 1.1816764548420906,
      "epoch": 0.10409155039974918,
      "grad_norm": 30.252086639404297,
      "learning_rate": 9.953429730181652e-07,
      "logits/chosen": -16.37164589704853,
      "logits/rejected": -15.937881459823364,
      "logps/chosen": -442.10915756225586,
      "logps/rejected": -464.68394470214844,
      "loss": 0.9728706479072571,
      "mean_token_accuracy": 0.7251901105046272,
      "num_tokens": 9350579.0,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.025675334269180894,
      "rewards/margins": 0.10957675683312118,
      "rewards/rejected": -0.13525208923965693,
      "step": 166
    },
    {
      "entropy": 1.0316157564520836,
      "epoch": 0.10471860793227779,
      "grad_norm": 39.15115737915039,
      "learning_rate": 9.95198809153627e-07,
      "logits/chosen": -17.97388225518009,
      "logits/rejected": -17.878072330124738,
      "logps/chosen": -963.8222351074219,
      "logps/rejected": -576.2222747802734,
      "loss": 0.9726594686508179,
      "mean_token_accuracy": 0.7661974504590034,
      "num_tokens": 9420912.0,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.039964041672647,
      "rewards/margins": 0.10996103240177035,
      "rewards/rejected": -0.14992507733404636,
      "step": 167
    },
    {
      "entropy": 0.9888377711176872,
      "epoch": 0.10534566546480639,
      "grad_norm": 33.86361312866211,
      "learning_rate": 9.950524585508875e-07,
      "logits/chosen": -21.71417541647197,
      "logits/rejected": -21.1007052103952,
      "logps/chosen": -570.1340103149414,
      "logps/rejected": -569.658317565918,
      "loss": 0.965457558631897,
      "mean_token_accuracy": 0.7609636634588242,
      "num_tokens": 9472501.0,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.004693154362030327,
      "rewards/margins": 0.13876159954816103,
      "rewards/rejected": -0.13406844343990088,
      "step": 168
    },
    {
      "entropy": 1.1498691216111183,
      "epoch": 0.10597272299733501,
      "grad_norm": 35.466957092285156,
      "learning_rate": 9.949039218562137e-07,
      "logits/chosen": -23.389534862650184,
      "logits/rejected": -20.17964972359958,
      "logps/chosen": -982.7829933166504,
      "logps/rejected": -726.9613132476807,
      "loss": 0.9566149711608887,
      "mean_token_accuracy": 0.7251748219132423,
      "num_tokens": 9543720.0,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.003413940197788179,
      "rewards/margins": 0.1751425163820386,
      "rewards/rejected": -0.17855645716190338,
      "step": 169
    },
    {
      "entropy": 0.8980679214000702,
      "epoch": 0.10659978052986362,
      "grad_norm": 40.1265754699707,
      "learning_rate": 9.947531997255256e-07,
      "logits/chosen": -24.576026786001893,
      "logits/rejected": -21.24743446770542,
      "logps/chosen": -646.5030174255371,
      "logps/rejected": -609.3380126953125,
      "loss": 0.946999728679657,
      "mean_token_accuracy": 0.8074663206934929,
      "num_tokens": 9599025.0,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.024876392912119627,
      "rewards/margins": 0.21698989067226648,
      "rewards/rejected": -0.19211349356919527,
      "step": 170
    },
    {
      "entropy": 1.041010521352291,
      "epoch": 0.10722683806239222,
      "grad_norm": 27.908248901367188,
      "learning_rate": 9.946002928243938e-07,
      "logits/chosen": -17.12985163970272,
      "logits/rejected": -18.323306945407023,
      "logps/chosen": -343.0621109008789,
      "logps/rejected": -436.75281524658203,
      "loss": 0.9608134031295776,
      "mean_token_accuracy": 0.7628256380558014,
      "num_tokens": 9635752.0,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.014670340926386416,
      "rewards/margins": 0.15794761665165424,
      "rewards/rejected": -0.1726179551333189,
      "step": 171
    },
    {
      "entropy": 0.9685352593660355,
      "epoch": 0.10785389559492084,
      "grad_norm": 44.593017578125,
      "learning_rate": 9.94445201828037e-07,
      "logits/chosen": -25.693284187291074,
      "logits/rejected": -20.638393007437365,
      "logps/chosen": -1019.63330078125,
      "logps/rejected": -641.4812908172607,
      "loss": 0.9540131092071533,
      "mean_token_accuracy": 0.7625846937298775,
      "num_tokens": 9707928.0,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.051992423948831856,
      "rewards/margins": 0.18558591045439243,
      "rewards/rejected": -0.23757833428680897,
      "step": 172
    },
    {
      "entropy": 1.035721518099308,
      "epoch": 0.10848095312744944,
      "grad_norm": 36.23087692260742,
      "learning_rate": 9.942879274213183e-07,
      "logits/chosen": -25.004740833477207,
      "logits/rejected": -21.763571870178154,
      "logps/chosen": -1005.9208698272705,
      "logps/rejected": -417.8051815032959,
      "loss": 0.9568801522254944,
      "mean_token_accuracy": 0.7389874830842018,
      "num_tokens": 9762747.0,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.041769477422349155,
      "rewards/margins": 0.17441253177821636,
      "rewards/rejected": -0.13264305610209703,
      "step": 173
    },
    {
      "entropy": 1.2502099946141243,
      "epoch": 0.10910801065997805,
      "grad_norm": 26.709156036376953,
      "learning_rate": 9.941284702987425e-07,
      "logits/chosen": -15.603563936993165,
      "logits/rejected": -17.525335746655358,
      "logps/chosen": -483.58291244506836,
      "logps/rejected": -384.8349323272705,
      "loss": 0.9618538618087769,
      "mean_token_accuracy": 0.706192672252655,
      "num_tokens": 9801642.0,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.01173106487840414,
      "rewards/margins": 0.15353717282414436,
      "rewards/rejected": -0.1418061126023531,
      "step": 174
    },
    {
      "entropy": 1.0089079961180687,
      "epoch": 0.10973506819250667,
      "grad_norm": 34.94546890258789,
      "learning_rate": 9.939668311644527e-07,
      "logits/chosen": -21.48884231436644,
      "logits/rejected": -20.73044133310641,
      "logps/chosen": -809.1402287483215,
      "logps/rejected": -620.0923671722412,
      "loss": 0.9594979286193848,
      "mean_token_accuracy": 0.764327310025692,
      "num_tokens": 9861136.0,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.0033170885872095823,
      "rewards/margins": 0.1640703366138041,
      "rewards/rejected": -0.1607532473281026,
      "step": 175
    },
    {
      "entropy": 1.134946659207344,
      "epoch": 0.11036212572503527,
      "grad_norm": 40.56920623779297,
      "learning_rate": 9.938030107322283e-07,
      "logits/chosen": -20.60884003968547,
      "logits/rejected": -21.315807587757533,
      "logps/chosen": -1011.4535865783691,
      "logps/rejected": -677.3356189727783,
      "loss": 0.9518853425979614,
      "mean_token_accuracy": 0.7143147438764572,
      "num_tokens": 9940296.0,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.020753700169734657,
      "rewards/margins": 0.2017311709932983,
      "rewards/rejected": -0.2224848661571741,
      "step": 176
    },
    {
      "entropy": 1.1413284689188004,
      "epoch": 0.11098918325756388,
      "grad_norm": 34.17391586303711,
      "learning_rate": 9.936370097254803e-07,
      "logits/chosen": -18.22259319683248,
      "logits/rejected": -16.596884329014106,
      "logps/chosen": -748.4427871704102,
      "logps/rejected": -550.8146286010742,
      "loss": 0.9616140127182007,
      "mean_token_accuracy": 0.7435401454567909,
      "num_tokens": 9992025.0,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.020526117412373424,
      "rewards/margins": 0.15512995416065678,
      "rewards/rejected": -0.1346038356423378,
      "step": 177
    },
    {
      "entropy": 1.1366123706102371,
      "epoch": 0.1116162407900925,
      "grad_norm": 42.503440856933594,
      "learning_rate": 9.93468828877249e-07,
      "logits/chosen": -22.75767775859101,
      "logits/rejected": -21.918987652590392,
      "logps/chosen": -1206.1099195480347,
      "logps/rejected": -610.3419342041016,
      "loss": 0.9570979475975037,
      "mean_token_accuracy": 0.7211766019463539,
      "num_tokens": 10059360.0,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0131734365131706,
      "rewards/margins": 0.18104426749050617,
      "rewards/rejected": -0.1942176972515881,
      "step": 178
    },
    {
      "entropy": 1.107018142938614,
      "epoch": 0.1122432983226211,
      "grad_norm": 39.66263198852539,
      "learning_rate": 9.93298468930201e-07,
      "logits/chosen": -23.260314784767324,
      "logits/rejected": -21.11647381786282,
      "logps/chosen": -1436.7120723724365,
      "logps/rejected": -508.69267654418945,
      "loss": 0.9501864910125732,
      "mean_token_accuracy": 0.7304199710488319,
      "num_tokens": 10132130.0,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.028971436579013243,
      "rewards/margins": 0.20222028624266386,
      "rewards/rejected": -0.17324885539710522,
      "step": 179
    },
    {
      "entropy": 1.021133229136467,
      "epoch": 0.1128703558551497,
      "grad_norm": 29.389991760253906,
      "learning_rate": 9.93125930636625e-07,
      "logits/chosen": -18.06104104884531,
      "logits/rejected": -19.473802853816192,
      "logps/chosen": -539.9062232971191,
      "logps/rejected": -461.96774101257324,
      "loss": 0.9612171053886414,
      "mean_token_accuracy": 0.7571789473295212,
      "num_tokens": 10182254.0,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.02405679877847433,
      "rewards/margins": 0.1570815034210682,
      "rewards/rejected": -0.1330247106961906,
      "step": 180
    },
    {
      "entropy": 1.1228575333952904,
      "epoch": 0.11349741338767833,
      "grad_norm": 31.7424373626709,
      "learning_rate": 9.929512147584296e-07,
      "logits/chosen": -19.25161771955138,
      "logits/rejected": -19.300870767626026,
      "logps/chosen": -471.5840435028076,
      "logps/rejected": -377.43126487731934,
      "loss": 0.9580326080322266,
      "mean_token_accuracy": 0.7341411262750626,
      "num_tokens": 10231221.0,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.00470810174010694,
      "rewards/margins": 0.1703320173546672,
      "rewards/rejected": -0.16562391445040703,
      "step": 181
    },
    {
      "entropy": 1.2480486258864403,
      "epoch": 0.11412447092020693,
      "grad_norm": 33.407691955566406,
      "learning_rate": 9.927743220671389e-07,
      "logits/chosen": -15.707493700178858,
      "logits/rejected": -18.14101813743978,
      "logps/chosen": -644.9055366516113,
      "logps/rejected": -403.51587295532227,
      "loss": 0.9577570557594299,
      "mean_token_accuracy": 0.695245198905468,
      "num_tokens": 10278764.0,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.00990894460119307,
      "rewards/margins": 0.1714292955584824,
      "rewards/rejected": -0.18133824318647385,
      "step": 182
    },
    {
      "entropy": 0.9243633225560188,
      "epoch": 0.11475152845273554,
      "grad_norm": 34.56248474121094,
      "learning_rate": 9.925952533438897e-07,
      "logits/chosen": -17.46200834389154,
      "logits/rejected": -16.492948873374484,
      "logps/chosen": -617.6435585021973,
      "logps/rejected": -372.62390518188477,
      "loss": 0.948718786239624,
      "mean_token_accuracy": 0.7732345685362816,
      "num_tokens": 10341069.0,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.010069805663079023,
      "rewards/margins": 0.2102643116377294,
      "rewards/rejected": -0.22033411543816328,
      "step": 183
    },
    {
      "entropy": 0.9938921704888344,
      "epoch": 0.11537858598526415,
      "grad_norm": 30.834369659423828,
      "learning_rate": 9.924140093794277e-07,
      "logits/chosen": -19.204634086255844,
      "logits/rejected": -19.86160843964533,
      "logps/chosen": -502.1897392272949,
      "logps/rejected": -514.1468658447266,
      "loss": 0.9647988080978394,
      "mean_token_accuracy": 0.7703093513846397,
      "num_tokens": 10385519.0,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.006156077841296792,
      "rewards/margins": 0.14176303520798683,
      "rewards/rejected": -0.13560695759952068,
      "step": 184
    },
    {
      "entropy": 0.9982673227787018,
      "epoch": 0.11600564351779276,
      "grad_norm": 33.376678466796875,
      "learning_rate": 9.922305909741046e-07,
      "logits/chosen": -20.081957609627672,
      "logits/rejected": -20.08288046634073,
      "logps/chosen": -563.1763916015625,
      "logps/rejected": -455.9494934082031,
      "loss": 0.940411388874054,
      "mean_token_accuracy": 0.7691720277070999,
      "num_tokens": 10451619.0,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.03211024316260591,
      "rewards/margins": 0.24388167914003134,
      "rewards/rejected": -0.21177144069224596,
      "step": 185
    },
    {
      "entropy": 1.2043246403336525,
      "epoch": 0.11663270105032136,
      "grad_norm": 35.30168151855469,
      "learning_rate": 9.92044998937874e-07,
      "logits/chosen": -15.935400744656912,
      "logits/rejected": -17.80423133241425,
      "logps/chosen": -507.5350112915039,
      "logps/rejected": -506.1169013977051,
      "loss": 0.9446260929107666,
      "mean_token_accuracy": 0.7261253371834755,
      "num_tokens": 10504967.0,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.011272144154645503,
      "rewards/margins": 0.2257434045895934,
      "rewards/rejected": -0.21447127033025026,
      "step": 186
    },
    {
      "entropy": 0.8656372576951981,
      "epoch": 0.11725975858284998,
      "grad_norm": 36.30891799926758,
      "learning_rate": 9.918572340902878e-07,
      "logits/chosen": -20.357606409612657,
      "logits/rejected": -20.43430765009451,
      "logps/chosen": -806.364107131958,
      "logps/rejected": -409.4738140106201,
      "loss": 0.9610611796379089,
      "mean_token_accuracy": 0.7912320047616959,
      "num_tokens": 10559810.0,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.01855588040780276,
      "rewards/margins": 0.16032809671014547,
      "rewards/rejected": -0.14177220640704036,
      "step": 187
    },
    {
      "entropy": 1.2360781207680702,
      "epoch": 0.11788681611537859,
      "grad_norm": 36.44474411010742,
      "learning_rate": 9.916672972604927e-07,
      "logits/chosen": -21.232208354402246,
      "logits/rejected": -20.78721834595899,
      "logps/chosen": -872.3558025360107,
      "logps/rejected": -562.6851234436035,
      "loss": 0.956852912902832,
      "mean_token_accuracy": 0.720037579536438,
      "num_tokens": 10620148.0,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.012437668745405972,
      "rewards/margins": 0.17596057945047505,
      "rewards/rejected": -0.16352291277144104,
      "step": 188
    },
    {
      "entropy": 1.016458660364151,
      "epoch": 0.11851387364790719,
      "grad_norm": 29.050859451293945,
      "learning_rate": 9.914751892872274e-07,
      "logits/chosen": -19.08313772226747,
      "logits/rejected": -18.82169734271658,
      "logps/chosen": -434.9751796722412,
      "logps/rejected": -346.7390670776367,
      "loss": 0.9528908133506775,
      "mean_token_accuracy": 0.7646205425262451,
      "num_tokens": 10670772.0,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.008685210370458663,
      "rewards/margins": 0.19308780319988728,
      "rewards/rejected": -0.20177301485091448,
      "step": 189
    },
    {
      "entropy": 1.1021007224917412,
      "epoch": 0.11914093118043581,
      "grad_norm": 44.599769592285156,
      "learning_rate": 9.91280911018817e-07,
      "logits/chosen": -21.5532820720735,
      "logits/rejected": -20.40933043838709,
      "logps/chosen": -1575.5354919433594,
      "logps/rejected": -422.48606395721436,
      "loss": 0.941632866859436,
      "mean_token_accuracy": 0.7219045013189316,
      "num_tokens": 10758488.0,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.06574870133772492,
      "rewards/margins": 0.2363713993690908,
      "rewards/rejected": -0.1706227008253336,
      "step": 190
    },
    {
      "entropy": 1.1249773353338242,
      "epoch": 0.11976798871296442,
      "grad_norm": 37.6546745300293,
      "learning_rate": 9.910844633131712e-07,
      "logits/chosen": -17.602501168387953,
      "logits/rejected": -18.726467814742886,
      "logps/chosen": -641.0849552154541,
      "logps/rejected": -560.3315982818604,
      "loss": 0.935768723487854,
      "mean_token_accuracy": 0.7447556182742119,
      "num_tokens": 10819612.0,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.0099241798161529,
      "rewards/margins": 0.26406861934810877,
      "rewards/rejected": -0.27399280294775963,
      "step": 191
    },
    {
      "entropy": 1.0111653581261635,
      "epoch": 0.12039504624549302,
      "grad_norm": 33.29735565185547,
      "learning_rate": 9.908858470377793e-07,
      "logits/chosen": -17.99586764005896,
      "logits/rejected": -16.641471216812302,
      "logps/chosen": -754.2721481323242,
      "logps/rejected": -414.00395679473877,
      "loss": 0.9521582126617432,
      "mean_token_accuracy": 0.7499677836894989,
      "num_tokens": 10882013.0,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.00994637084659189,
      "rewards/margins": 0.19447646429762244,
      "rewards/rejected": -0.1845300872810185,
      "step": 192
    },
    {
      "entropy": 1.0526621490716934,
      "epoch": 0.12102210377802164,
      "grad_norm": 35.13934326171875,
      "learning_rate": 9.906850630697066e-07,
      "logits/chosen": -17.88888213857138,
      "logits/rejected": -17.117701311736873,
      "logps/chosen": -626.3125686645508,
      "logps/rejected": -390.99532318115234,
      "loss": 0.9473018646240234,
      "mean_token_accuracy": 0.7478787750005722,
      "num_tokens": 10932194.0,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.013837772188708186,
      "rewards/margins": 0.21603740751743317,
      "rewards/rejected": -0.22987518273293972,
      "step": 193
    },
    {
      "entropy": 1.2843475490808487,
      "epoch": 0.12164916131055024,
      "grad_norm": 33.156471252441406,
      "learning_rate": 9.904821122955914e-07,
      "logits/chosen": -16.072002517290578,
      "logits/rejected": -17.32703283993392,
      "logps/chosen": -858.3284645080566,
      "logps/rejected": -468.63574600219727,
      "loss": 0.957085132598877,
      "mean_token_accuracy": 0.6953159794211388,
      "num_tokens": 10989560.0,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0002469784813001752,
      "rewards/margins": 0.174753800034523,
      "rewards/rejected": -0.1750007774680853,
      "step": 194
    },
    {
      "entropy": 0.9444542303681374,
      "epoch": 0.12227621884307885,
      "grad_norm": 33.4073371887207,
      "learning_rate": 9.90276995611639e-07,
      "logits/chosen": -19.142496042660422,
      "logits/rejected": -21.297342713346943,
      "logps/chosen": -639.1740913391113,
      "logps/rejected": -354.38869857788086,
      "loss": 0.9653919339179993,
      "mean_token_accuracy": 0.7545767053961754,
      "num_tokens": 11034532.0,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0012790057808160782,
      "rewards/margins": 0.1389640721026808,
      "rewards/rejected": -0.13768507237546146,
      "step": 195
    },
    {
      "entropy": 1.106497973203659,
      "epoch": 0.12290327637560747,
      "grad_norm": 30.54036521911621,
      "learning_rate": 9.900697139236208e-07,
      "logits/chosen": -17.157277160405563,
      "logits/rejected": -18.095409943495174,
      "logps/chosen": -551.8531856536865,
      "logps/rejected": -366.1446056365967,
      "loss": 0.9476866722106934,
      "mean_token_accuracy": 0.7291347607970238,
      "num_tokens": 11079071.0,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.026466174283996224,
      "rewards/margins": 0.21335551515221596,
      "rewards/rejected": -0.18688933923840523,
      "step": 196
    },
    {
      "entropy": 1.0959510654211044,
      "epoch": 0.12353033390813607,
      "grad_norm": 36.96934127807617,
      "learning_rate": 9.898602681468674e-07,
      "logits/chosen": -19.027734293968553,
      "logits/rejected": -19.864234419673636,
      "logps/chosen": -954.4189186096191,
      "logps/rejected": -566.7688503265381,
      "loss": 0.9575784206390381,
      "mean_token_accuracy": 0.7271228209137917,
      "num_tokens": 11136634.0,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.016784888110123575,
      "rewards/margins": 0.17304366454482079,
      "rewards/rejected": -0.18982854578644037,
      "step": 197
    },
    {
      "entropy": 0.8774717822670937,
      "epoch": 0.12415739144066468,
      "grad_norm": 36.82571792602539,
      "learning_rate": 9.896486592062659e-07,
      "logits/chosen": -28.647222960438814,
      "logits/rejected": -22.060415168768344,
      "logps/chosen": -836.5278701782227,
      "logps/rejected": -480.254337310791,
      "loss": 0.9511983394622803,
      "mean_token_accuracy": 0.7737741991877556,
      "num_tokens": 11197525.0,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.008797007380053401,
      "rewards/margins": 0.20592515252064914,
      "rewards/rejected": -0.21472215885296464,
      "step": 198
    },
    {
      "entropy": 1.1285011321306229,
      "epoch": 0.1247844489731933,
      "grad_norm": 42.2977409362793,
      "learning_rate": 9.89434888036256e-07,
      "logits/chosen": -18.49698845016767,
      "logits/rejected": -17.254445681297014,
      "logps/chosen": -1127.2061748504639,
      "logps/rejected": -956.0397682189941,
      "loss": 0.9597939848899841,
      "mean_token_accuracy": 0.731256939470768,
      "num_tokens": 11292505.0,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.003246246255002916,
      "rewards/margins": 0.16256380267441273,
      "rewards/rejected": -0.1658100476488471,
      "step": 199
    },
    {
      "entropy": 0.9618688002228737,
      "epoch": 0.1254115065057219,
      "grad_norm": 35.041629791259766,
      "learning_rate": 9.892189555808251e-07,
      "logits/chosen": -23.16837855908106,
      "logits/rejected": -22.01573163400953,
      "logps/chosen": -595.0505418777466,
      "logps/rejected": -505.8145923614502,
      "loss": 0.9585968852043152,
      "mean_token_accuracy": 0.7569021657109261,
      "num_tokens": 11345306.0,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.027557444758713245,
      "rewards/margins": 0.16784314159303904,
      "rewards/rejected": -0.19540058448910713,
      "step": 200
    }
  ],
  "logging_steps": 1,
  "max_steps": 1595,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 5.084673057646182e+16,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}